New: Fetch all albums from metadata and filter client-side (#803)

* Retrieve all albums from metadata and filter client-side

Should significantly increase the proportion of artist requests hitting
cloudflare cache

* Add tests for album filtering

src/NzbDrone.Core.Test/MetadataSource/SkyHook/SkyHookProxyFixture.cs

@@ -10,6 +10,7 @@ using NzbDrone.Core.Music;
 using NzbDrone.Test.Common.Categories;
 using Moq;
 using NzbDrone.Core.Profiles.Metadata;
+using NzbDrone.Core.MetadataSource.SkyHook.Resource;
 
 namespace NzbDrone.Core.Test.MetadataSource.SkyHook
 {
@@ -17,12 +18,14 @@ namespace NzbDrone.Core.Test.MetadataSource.SkyHook
     [IntegrationTest]
     public class SkyHookProxyFixture : CoreTest<SkyHookProxy>
     {
+        private MetadataProfile _metadataProfile;
+
         [SetUp]
         public void Setup()
         {
             UseRealHttp();
 
-            var _metadataProfile = new MetadataProfile
+            _metadataProfile = new MetadataProfile
             {
                 PrimaryAlbumTypes = new List<ProfilePrimaryAlbumTypeItem>
                 {
@@ -30,7 +33,6 @@ namespace NzbDrone.Core.Test.MetadataSource.SkyHook
                     {
                         PrimaryAlbumType = PrimaryAlbumType.Album,
                         Allowed = true
-                        
                     }
                 },
                 SecondaryAlbumTypes = new List<ProfileSecondaryAlbumTypeItem>
@@ -59,6 +61,30 @@ namespace NzbDrone.Core.Test.MetadataSource.SkyHook
                 .Setup(s => s.Exists(It.IsAny<int>()))
                 .Returns(true);
         }
+        
+        public List<AlbumResource> GivenExampleAlbums()
+        {
+            var result = new List<AlbumResource>();
+            
+            foreach (var primaryType in PrimaryAlbumType.All)
+            {
+                foreach (var secondaryType in SecondaryAlbumType.All)
+                {
+                    var secondaryTypes = secondaryType.Name == "Studio" ? new List<string>() : new List<string> { secondaryType.Name };
+                    foreach (var releaseStatus in ReleaseStatus.All)
+                    {
+                        var releaseStatuses = new List<string> { releaseStatus.Name };
+                        result.Add(new AlbumResource {
+                                Type = primaryType.Name,
+                                SecondaryTypes = secondaryTypes,
+                                ReleaseStatuses = releaseStatuses
+                            });
+                    }
+                }
+            }
+            
+            return result;
+        }
 
         [TestCase("f59c5520-5f46-4d2c-b2c4-822eabf53419", "Linkin Park")]
         [TestCase("66c662b6-6e2f-4930-8610-912e24c63ed1", "AC/DC")]
@@ -67,10 +93,64 @@ namespace NzbDrone.Core.Test.MetadataSource.SkyHook
             var details = Subject.GetArtistInfo(mbId, 1);
 
             ValidateArtist(details);
-            ValidateAlbums(details.Albums.Value);
+            ValidateAlbums(details.Albums.Value, true);
 
             details.Name.Should().Be(name);
         }
+
+        [TestCaseSource(typeof(PrimaryAlbumType), "All")]
+        public void should_filter_albums_by_primary_release_type(PrimaryAlbumType type)
+        {
+            _metadataProfile.PrimaryAlbumTypes = new List<ProfilePrimaryAlbumTypeItem> {
+                    new ProfilePrimaryAlbumTypeItem
+                    {
+                        PrimaryAlbumType = type,
+                        Allowed = true
+                    }
+            };
+
+
+            var albums = GivenExampleAlbums();
+            Subject.FilterAlbums(albums, 1).Select(x => x.Type).Distinct()
+                   .Should().BeEquivalentTo(new List<string> { type.Name });
+        }
+        
+        [TestCaseSource(typeof(SecondaryAlbumType), "All")]
+        public void should_filter_albums_by_secondary_release_type(SecondaryAlbumType type)
+        {
+            _metadataProfile.SecondaryAlbumTypes = new List<ProfileSecondaryAlbumTypeItem> {
+                    new ProfileSecondaryAlbumTypeItem
+                    {
+                        SecondaryAlbumType = type,
+                        Allowed = true
+                    }
+            };
+
+            var albums = GivenExampleAlbums();
+            var filtered = Subject.FilterAlbums(albums, 1);
+            TestLogger.Debug(filtered.Count());
+            
+            filtered.SelectMany(x => x.SecondaryTypes.Select(SkyHookProxy.MapSecondaryTypes))
+                    .Select(x => x.Name)
+                    .Distinct()
+                    .Should().BeEquivalentTo(type.Name == "Studio" ? new List<string>() : new List<string> { type.Name });
+        }
+
+        [TestCaseSource(typeof(ReleaseStatus), "All")]
+        public void should_filter_albums_by_release_status(ReleaseStatus type)
+        {
+            _metadataProfile.ReleaseStatuses = new List<ProfileReleaseStatusItem> {
+                    new ProfileReleaseStatusItem
+                    {
+                        ReleaseStatus = type,
+                        Allowed = true
+                    }
+            };
+
+            var albums = GivenExampleAlbums();
+            Subject.FilterAlbums(albums, 1).SelectMany(x => x.ReleaseStatuses).Distinct()
+                   .Should().BeEquivalentTo(new List<string> { type.Name });
+        }
         
         [TestCase("12fa3845-7c62-36e5-a8da-8be137155a72", "Hysteria")]
         public void should_be_able_to_get_album_detail(string mbId, string name)
@@ -130,16 +210,24 @@ namespace NzbDrone.Core.Test.MetadataSource.SkyHook
             artist.ForeignArtistId.Should().NotBeNullOrWhiteSpace();
         }
 
-        private void ValidateAlbums(List<Album> albums)
+        private void ValidateAlbums(List<Album> albums, bool idOnly = false)
         {
             albums.Should().NotBeEmpty();
 
             foreach (var album in albums)
             {
-                ValidateAlbum(album);
+                album.ForeignAlbumId.Should().NotBeNullOrWhiteSpace();
+                if (!idOnly)
+                {
+                    ValidateAlbum(album);
+                }
 
-                //if atleast one album has title it means parse it working.
+            }
-                albums.Should().Contain(c => !string.IsNullOrWhiteSpace(c.Title));
+            
+            //if atleast one album has title it means parse it working.
+            if (!idOnly)
+            {
+                albums.Should().Contain(c => !string.IsNullOrWhiteSpace(c.Title));                
             }
         }
 

src/NzbDrone.Core/MetadataSource/SkyHook/Resource/AlbumResource.cs

@@ -1,7 +1,5 @@
 using System;
 using System.Collections.Generic;
-using System.Linq;
-using System.Text;
 
 namespace NzbDrone.Core.MetadataSource.SkyHook.Resource
 {
@@ -20,5 +18,6 @@ namespace NzbDrone.Core.MetadataSource.SkyHook.Resource
         public List<string> SecondaryTypes { get; set; }
         public string Title { get; set; }
         public string Type { get; set; }
+        public List<string> ReleaseStatuses { get; set; }
     }
 }

src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs

@@ -50,18 +50,9 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
 
             _logger.Debug("Getting Artist with LidarrAPI.MetadataID of {0}", foreignArtistId);
 
-            var metadataProfile = _metadataProfileService.Exists(metadataProfileId) ? _metadataProfileService.Get(metadataProfileId) : _metadataProfileService.All().First();
-
-            var primaryTypes = metadataProfile.PrimaryAlbumTypes.Where(s => s.Allowed).Select(s => s.PrimaryAlbumType.Name);
-            var secondaryTypes = metadataProfile.SecondaryAlbumTypes.Where(s => s.Allowed).Select(s => s.SecondaryAlbumType.Name);
-            var releaseStatuses = metadataProfile.ReleaseStatuses.Where(s => s.Allowed).Select(s => s.ReleaseStatus.Name);
-
             var httpRequest = _requestBuilder.GetRequestBuilder().Create()
-                                            .SetSegment("route", "artist/" + foreignArtistId)
+                                             .SetSegment("route", "artist/" + foreignArtistId)
-                                            .AddQueryParam("primTypes", string.Join("|", primaryTypes))
+                                             .Build();
-                                            .AddQueryParam("secTypes", string.Join("|", secondaryTypes))
-                                            .AddQueryParam("releaseStatuses", string.Join("|", releaseStatuses))
-                                            .Build();
 
             httpRequest.AllowAutoRedirect = true;
             httpRequest.SuppressHttpError = true;
@@ -89,11 +80,30 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
             artist.Metadata = MapArtistMetadata(httpResponse.Resource);
             artist.CleanName = Parser.Parser.CleanArtistName(artist.Metadata.Value.Name);
             artist.SortName = Parser.Parser.NormalizeTitle(artist.Metadata.Value.Name);
-            artist.Albums = httpResponse.Resource.Albums.Select(x => MapAlbum(x, null)).ToList();
+
+            artist.Albums = FilterAlbums(httpResponse.Resource.Albums, metadataProfileId)
+                .Select(x => new Album {
+                        ForeignAlbumId = x.Id
+                    })
+                .ToList();
 
             return artist;
         }
 
+        public IEnumerable<AlbumResource> FilterAlbums(IEnumerable<AlbumResource> albums, int metadataProfileId)
+        {
+            var metadataProfile = _metadataProfileService.Exists(metadataProfileId) ? _metadataProfileService.Get(metadataProfileId) : _metadataProfileService.All().First();
+            var primaryTypes = new HashSet<string>(metadataProfile.PrimaryAlbumTypes.Where(s => s.Allowed).Select(s => s.PrimaryAlbumType.Name));
+            var secondaryTypes = new HashSet<string>(metadataProfile.SecondaryAlbumTypes.Where(s => s.Allowed).Select(s => s.SecondaryAlbumType.Name));
+            var releaseStatuses = new HashSet<string>(metadataProfile.ReleaseStatuses.Where(s => s.Allowed).Select(s => s.ReleaseStatus.Name));
+
+
+            return albums.Where(album => primaryTypes.Contains(album.Type) &&
+                                (!album.SecondaryTypes.Any() && secondaryTypes.Contains("Studio") ||
+                                 album.SecondaryTypes.Any(x => secondaryTypes.Contains(x))) &&
+                                album.ReleaseStatuses.Any(x => releaseStatuses.Contains(x)));
+        }
+
         public Tuple<string, Album, List<ArtistMetadata>> GetAlbumInfo(string foreignAlbumId)
         {
             _logger.Debug("Getting Album with LidarrAPI.MetadataID of {0}", foreignAlbumId);
@@ -470,7 +480,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
             }
         }
 
-        private static SecondaryAlbumType MapSecondaryTypes(string albumType)
+        public static SecondaryAlbumType MapSecondaryTypes(string albumType)
         {
             switch (albumType.ToLowerInvariant())
             {