New: Import List Exclusions (#608)

* New: Import List Exclusions

* Fixed: ImportExclusion ForeignId Checks, Unique. RefreshArtist Duplicate

* Fixed: Copy/Paste typos
This commit is contained in:
Qstick 2019-03-01 17:26:36 -05:00 committed by GitHub
commit 42c16c227e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
35 changed files with 1160 additions and 20 deletions

View file

@ -171,8 +171,9 @@ namespace Lidarr.Api.V1.Artist
private void DeleteArtist(int id)
{
var deleteFiles = Request.GetBooleanQueryParameter("deleteFiles");
var addImportListExclusion = Request.GetBooleanQueryParameter("addImportListExclusion");
_artistService.DeleteArtist(id, deleteFiles);
_artistService.DeleteArtist(id, deleteFiles, addImportListExclusion);
}
private void MapCoversToLocal(params ArtistResource[] artists)

View file

@ -0,0 +1,56 @@
using System.Collections.Generic;
using NzbDrone.Core.ImportLists.Exclusions;
using Lidarr.Http;
using FluentValidation;
using NzbDrone.Core.Validation;
namespace Lidarr.Api.V1.ImportLists
{
public class ImportListExclusionModule : LidarrRestModule<ImportListExclusionResource>
{
private readonly IImportListExclusionService _importListExclusionService;
public ImportListExclusionModule(IImportListExclusionService importListExclusionService,
ImportListExclusionExistsValidator importListExclusionExistsValidator,
GuidValidator guidValidator)
{
_importListExclusionService = importListExclusionService;
GetResourceById = GetImportListExclusion;
GetResourceAll = GetImportListExclusions;
CreateResource = AddImportListExclusion;
UpdateResource = UpdateImportListExclusion;
DeleteResource = DeleteImportListExclusionResource;
SharedValidator.RuleFor(c => c.ForeignId).NotEmpty().SetValidator(guidValidator).SetValidator(importListExclusionExistsValidator);
SharedValidator.RuleFor(c => c.ArtistName).NotEmpty();
}
private ImportListExclusionResource GetImportListExclusion(int id)
{
return _importListExclusionService.Get(id).ToResource();
}
private List<ImportListExclusionResource> GetImportListExclusions()
{
return _importListExclusionService.All().ToResource();
}
private int AddImportListExclusion(ImportListExclusionResource resource)
{
var customFilter = _importListExclusionService.Add(resource.ToModel());
return customFilter.Id;
}
private void UpdateImportListExclusion(ImportListExclusionResource resource)
{
_importListExclusionService.Update(resource.ToModel());
}
private void DeleteImportListExclusionResource(int id)
{
_importListExclusionService.Delete(id);
}
}
}

View file

@ -0,0 +1,45 @@
using System.Collections.Generic;
using System.Linq;
using NzbDrone.Core.ImportLists.Exclusions;
using Lidarr.Http.REST;
namespace Lidarr.Api.V1.ImportLists
{
public class ImportListExclusionResource : RestResource
{
public string ForeignId { get; set; }
public string ArtistName { get; set; }
}
public static class ImportListExclusionResourceMapper
{
public static ImportListExclusionResource ToResource(this ImportListExclusion model)
{
if (model == null) return null;
return new ImportListExclusionResource
{
Id = model.Id,
ForeignId = model.ForeignId,
ArtistName = model.Name,
};
}
public static ImportListExclusion ToModel(this ImportListExclusionResource resource)
{
if (resource == null) return null;
return new ImportListExclusion
{
Id = resource.Id,
ForeignId = resource.ForeignId,
Name = resource.ArtistName
};
}
public static List<ImportListExclusionResource> ToResource(this IEnumerable<ImportListExclusion> filters)
{
return filters.Select(ToResource).ToList();
}
}
}

View file

@ -99,6 +99,8 @@
<Compile Include="Config\MetadataProviderConfigResource.cs" />
<Compile Include="CustomFilters\CustomFilterModule.cs" />
<Compile Include="CustomFilters\CustomFilterResource.cs" />
<Compile Include="ImportLists\ImportListExclusionModule.cs" />
<Compile Include="ImportLists\ImportListExclusionResource.cs" />
<Compile Include="ImportLists\ImportListModule.cs" />
<Compile Include="ImportLists\ImportListResource.cs" />
<Compile Include="Profiles\Metadata\MetadataProfileModule.cs" />

View file

@ -7,6 +7,7 @@ using NzbDrone.Core.MetadataSource;
using NzbDrone.Core.Music;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Test.Framework;
using NzbDrone.Core.ImportLists.Exclusions;
namespace NzbDrone.Core.Test.ImportListTests
{
@ -43,6 +44,10 @@ namespace NzbDrone.Core.Test.ImportListTests
Mocker.GetMock<IFetchAndParseImportList>()
.Setup(v => v.Fetch())
.Returns(_importListReports);
Mocker.GetMock<IImportListExclusionService>()
.Setup(v => v.All())
.Returns(new List<ImportListExclusion>());
}
private void WithAlbum()
@ -67,6 +72,17 @@ namespace NzbDrone.Core.Test.ImportListTests
.Returns(new Artist{ForeignArtistId = _importListReports.First().ArtistMusicBrainzId });
}
private void WithExcludedArtist()
{
Mocker.GetMock<IImportListExclusionService>()
.Setup(v => v.All())
.Returns(new List<ImportListExclusion> {
new ImportListExclusion {
ForeignId = "f59c5520-5f46-4d2c-b2c4-822eabf53419"
}
});
}
[Test]
public void should_search_if_artist_title_and_no_artist_id()
{
@ -123,7 +139,7 @@ namespace NzbDrone.Core.Test.ImportListTests
}
[Test]
public void should_not_try_add_if_existing_artist()
public void should_not_add_if_existing_artist()
{
WithArtistId();
WithAlbum();
@ -149,6 +165,20 @@ namespace NzbDrone.Core.Test.ImportListTests
.Verify(v => v.AddArtists(It.Is<List<Artist>>(t => t.Count == 1)));
}
[Test]
public void should_not_add_if_excluded_artist()
{
WithArtistId();
WithAlbum();
WithAlbumId();
WithExcludedArtist();
Subject.Execute(new ImportListSyncCommand());
Mocker.GetMock<IAddArtistService>()
.Verify(v => v.AddArtists(It.Is<List<Artist>>(t => t.Count == 0)));
}
[Test]
public void should_mark_album_for_monitor_if_album_id()
{

View file

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -387,6 +387,7 @@
<Compile Include="ThingiProviderTests\ProviderStatusServiceFixture.cs" />
<Compile Include="UpdateTests\UpdatePackageProviderFixture.cs" />
<Compile Include="UpdateTests\UpdateServiceFixture.cs" />
<Compile Include="ValidationTests\GuidValidationFixture.cs" />
<Compile Include="ValidationTests\SystemFolderValidatorFixture.cs" />
<Compile Include="XbmcVersionTests.cs" />
<None Include="Files\Nzbs\NoFiles.nzb">
@ -610,9 +611,6 @@
<ItemGroup>
<IdentificationTestCases Include="Files\Identification\*.json" />
</ItemGroup>
<Copy
SourceFiles="@(IdentificationTestCases)"
DestinationFolder="$(OutputPath)\Files\Identification\"
SkipUnchangedFiles="true"/>
<Copy SourceFiles="@(IdentificationTestCases)" DestinationFolder="$(OutputPath)\Files\Identification\" SkipUnchangedFiles="true" />
</Target>
</Project>
</Project>

View file

@ -0,0 +1,44 @@
using FizzWare.NBuilder;
using FluentAssertions;
using NUnit.Framework;
using NzbDrone.Core.Test.Framework;
using NzbDrone.Core.Validation;
using NzbDrone.Test.Common;
using NzbDrone.Core.ImportLists.Exclusions;
namespace NzbDrone.Core.Test.ValidationTests
{
public class GuidValidationFixture : CoreTest<GuidValidator>
{
private TestValidator<ImportListExclusion> _validator;
[SetUp]
public void Setup()
{
_validator = new TestValidator<ImportListExclusion>
{
v => v.RuleFor(s => s.ForeignId).SetValidator(Subject)
};
}
[Test]
public void should_not_be_valid_if_invalid_guid()
{
var listExclusion = Builder<ImportListExclusion>.CreateNew()
.With(s => s.ForeignId = "e1f1e33e-2e4c-4d43-b91b-7064068d328")
.Build();
_validator.Validate(listExclusion).IsValid.Should().BeFalse();
}
[Test]
public void should_be_valid_if_valid_guid()
{
var listExclusion = Builder<ImportListExclusion>.CreateNew()
.With(s => s.ForeignId = "e1f1e33e-2e4c-4d43-b91b-7064068d3283")
.Build();
_validator.Validate(listExclusion).IsValid.Should().BeTrue();
}
}
}

View file

@ -0,0 +1,16 @@
using FluentMigrator;
using NzbDrone.Core.Datastore.Migration.Framework;
namespace NzbDrone.Core.Datastore.Migration
{
[Migration(027)]
public class add_import_exclusions : NzbDroneMigrationBase
{
protected override void MainDbUpgrade()
{
Create.TableForModel("ImportListExclusions")
.WithColumn("ForeignId").AsString().NotNullable().Unique()
.WithColumn("Name").AsString().NotNullable();
}
}
}

View file

@ -12,6 +12,7 @@ using NzbDrone.Core.Download;
using NzbDrone.Core.Download.Pending;
using NzbDrone.Core.Indexers;
using NzbDrone.Core.ImportLists;
using NzbDrone.Core.ImportLists.Exclusions;
using NzbDrone.Core.Instrumentation;
using NzbDrone.Core.Jobs;
using NzbDrone.Core.MediaFiles;
@ -191,6 +192,7 @@ namespace NzbDrone.Core.Datastore
Mapper.Entity<ImportListStatus>().RegisterModel("ImportListStatus");
Mapper.Entity<CustomFilter>().RegisterModel("CustomFilters");
Mapper.Entity<ImportListExclusion>().RegisterModel("ImportListExclusions");
}
private static void RegisterMappers()

View file

@ -0,0 +1,10 @@
using NzbDrone.Core.Datastore;
namespace NzbDrone.Core.ImportLists.Exclusions
{
public class ImportListExclusion : ModelBase
{
public string ForeignId { get; set; }
public string Name { get; set; }
}
}

View file

@ -0,0 +1,22 @@
using FluentValidation.Validators;
namespace NzbDrone.Core.ImportLists.Exclusions
{
public class ImportListExclusionExistsValidator : PropertyValidator
{
private readonly IImportListExclusionService _importListExclusionService;
public ImportListExclusionExistsValidator(IImportListExclusionService importListExclusionService)
: base("This exclusion has already been added.")
{
_importListExclusionService = importListExclusionService;
}
protected override bool IsValid(PropertyValidatorContext context)
{
if (context.PropertyValue == null) return true;
return (!_importListExclusionService.All().Exists(s => s.ForeignId == context.PropertyValue.ToString()));
}
}
}

View file

@ -0,0 +1,24 @@
using NzbDrone.Core.Datastore;
using NzbDrone.Core.Messaging.Events;
using System.Linq;
namespace NzbDrone.Core.ImportLists.Exclusions
{
public interface IImportListExclusionRepository : IBasicRepository<ImportListExclusion>
{
ImportListExclusion FindByForeignId(string foreignId);
}
public class ImportListExclusionRepository : BasicRepository<ImportListExclusion>, IImportListExclusionRepository
{
public ImportListExclusionRepository(IMainDatabase database, IEventAggregator eventAggregator)
: base(database, eventAggregator)
{
}
public ImportListExclusion FindByForeignId(string foreignId)
{
return Query.Where<ImportListExclusion>(m => m.ForeignId == foreignId).SingleOrDefault();
}
}
}

View file

@ -0,0 +1,80 @@
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.Music.Events;
using System.Collections.Generic;
using System.Linq;
namespace NzbDrone.Core.ImportLists.Exclusions
{
public interface IImportListExclusionService
{
ImportListExclusion Add(ImportListExclusion importListExclusion);
List<ImportListExclusion> All();
void Delete(int id);
ImportListExclusion Get(int id);
ImportListExclusion FindByForeignId(string foreignId);
ImportListExclusion Update(ImportListExclusion importListExclusion);
}
public class ImportListExclusionService : IImportListExclusionService, IHandleAsync<ArtistDeletedEvent>
{
private readonly IImportListExclusionRepository _repo;
public ImportListExclusionService(IImportListExclusionRepository repo)
{
_repo = repo;
}
public ImportListExclusion Add(ImportListExclusion importListExclusion)
{
return _repo.Insert(importListExclusion);
}
public ImportListExclusion Update(ImportListExclusion importListExclusion)
{
return _repo.Update(importListExclusion);
}
public void Delete(int id)
{
_repo.Delete(id);
}
public ImportListExclusion Get(int id)
{
return _repo.Get(id);
}
public ImportListExclusion FindByForeignId(string foreignId)
{
return _repo.FindByForeignId(foreignId);
}
public List<ImportListExclusion> All()
{
return _repo.All().ToList();
}
public void HandleAsync(ArtistDeletedEvent message)
{
if (!message.AddImportListExclusion)
{
return;
}
var existingExclusion = _repo.FindByForeignId(message.Artist.ForeignArtistId);
if (existingExclusion != null)
{
return;
}
var importExclusion = new ImportListExclusion
{
ForeignId = message.Artist.ForeignArtistId,
Name = message.Artist.Name
};
_repo.Insert(importExclusion);
}
}
}

View file

@ -3,6 +3,7 @@ using System.Linq;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Instrumentation.Extensions;
using NzbDrone.Core.ImportLists.Exclusions;
using NzbDrone.Core.Messaging.Commands;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.MetadataSource;
@ -15,6 +16,7 @@ namespace NzbDrone.Core.ImportLists
{
private readonly IImportListStatusService _importListStatusService;
private readonly IImportListFactory _importListFactory;
private readonly IImportListExclusionService _importListExclusionService;
private readonly IFetchAndParseImportList _listFetcherAndParser;
private readonly ISearchForNewAlbum _albumSearchService;
private readonly ISearchForNewArtist _artistSearchService;
@ -25,6 +27,7 @@ namespace NzbDrone.Core.ImportLists
public ImportListSyncService(IImportListStatusService importListStatusService,
IImportListFactory importListFactory,
IImportListExclusionService importListExclusionService,
IFetchAndParseImportList listFetcherAndParser,
ISearchForNewAlbum albumSearchService,
ISearchForNewArtist artistSearchService,
@ -35,6 +38,7 @@ namespace NzbDrone.Core.ImportLists
{
_importListStatusService = importListStatusService;
_importListFactory = importListFactory;
_importListExclusionService = importListExclusionService;
_listFetcherAndParser = listFetcherAndParser;
_albumSearchService = albumSearchService;
_artistSearchService = artistSearchService;
@ -78,6 +82,8 @@ namespace NzbDrone.Core.ImportLists
var reportNumber = 1;
var listExclusions = _importListExclusionService.All();
foreach (var report in reports)
{
_logger.ProgressTrace("Processing list item {0}/{1}", reportNumber, reports.Count);
@ -112,9 +118,17 @@ namespace NzbDrone.Core.ImportLists
// Check to see if artist in DB
var existingArtist = _artistService.FindById(report.ArtistMusicBrainzId);
// Check to see if artist excluded
var excludedArtist = listExclusions.Where(s => s.ForeignId == report.ArtistMusicBrainzId).SingleOrDefault();
if (excludedArtist != null)
{
_logger.Debug("{0} [{1}] Rejected due to list exlcusion", report.ArtistMusicBrainzId, report.Artist);
}
// Append Artist if not already in DB or already on add list
if (existingArtist == null && artistsToAdd.All(s => s.Metadata.Value.ForeignArtistId != report.ArtistMusicBrainzId))
if (existingArtist == null && excludedArtist == null && artistsToAdd.All(s => s.Metadata.Value.ForeignArtistId != report.ArtistMusicBrainzId))
{
artistsToAdd.Add(new Artist
{

View file

@ -4,9 +4,10 @@ using NzbDrone.Core.Music.Events;
using System;
using System.Collections.Generic;
using System.Linq;
using NzbDrone.Core.Parser;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Cache;
using NzbDrone.Core.ImportLists.Exclusions;
using NzbDrone.Core.Parser;
namespace NzbDrone.Core.Music
{
@ -21,7 +22,7 @@ namespace NzbDrone.Core.Music
Artist FindByName(string title);
Artist FindByNameInexact(string title);
List<Artist> GetCandidates(string title);
void DeleteArtist(int artistId, bool deleteFiles);
void DeleteArtist(int artistId, bool deleteFiles, bool addImportListExclusion = false);
List<Artist> GetAllArtists();
List<Artist> AllForTag(int tagId);
Artist UpdateArtist(Artist artist);
@ -36,6 +37,7 @@ namespace NzbDrone.Core.Music
private readonly IArtistMetadataRepository _artistMetadataRepository;
private readonly IEventAggregator _eventAggregator;
private readonly ITrackService _trackService;
private readonly IImportListExclusionService _importListExclusionService;
private readonly IBuildArtistPaths _artistPathBuilder;
private readonly Logger _logger;
private readonly ICached<List<Artist>> _cache;
@ -44,6 +46,7 @@ namespace NzbDrone.Core.Music
IArtistMetadataRepository artistMetadataRepository,
IEventAggregator eventAggregator,
ITrackService trackService,
IImportListExclusionService importListExclusionService,
IBuildArtistPaths artistPathBuilder,
ICacheManager cacheManager,
Logger logger)
@ -52,6 +55,7 @@ namespace NzbDrone.Core.Music
_artistMetadataRepository = artistMetadataRepository;
_eventAggregator = eventAggregator;
_trackService = trackService;
_importListExclusionService = importListExclusionService;
_artistPathBuilder = artistPathBuilder;
_cache = cacheManager.GetCache<List<Artist>>(GetType());
_logger = logger;
@ -82,12 +86,12 @@ namespace NzbDrone.Core.Music
return _artistRepository.ArtistPathExists(folder);
}
public void DeleteArtist(int artistId, bool deleteFiles)
public void DeleteArtist(int artistId, bool deleteFiles, bool addImportListExclusion = false)
{
_cache.Clear();
var artist = _artistRepository.Get(artistId);
_artistRepository.Delete(artistId);
_eventAggregator.PublishEvent(new ArtistDeletedEvent(artist, deleteFiles));
_eventAggregator.PublishEvent(new ArtistDeletedEvent(artist, deleteFiles, addImportListExclusion));
}
public Artist FindById(string spotifyId)

View file

@ -1,4 +1,4 @@
using NzbDrone.Common.Messaging;
using NzbDrone.Common.Messaging;
using System;
using System.Collections.Generic;
using System.Linq;
@ -10,11 +10,13 @@ namespace NzbDrone.Core.Music.Events
{
public Artist Artist { get; private set; }
public bool DeleteFiles { get; private set; }
public bool AddImportListExclusion { get; private set; }
public ArtistDeletedEvent(Artist artist, bool deleteFiles)
public ArtistDeletedEvent(Artist artist, bool deleteFiles, bool addImportListExclusion)
{
Artist = artist;
DeleteFiles = deleteFiles;
AddImportListExclusion = addImportListExclusion;
}
}
}

View file

@ -14,6 +14,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using NzbDrone.Core.ImportLists.Exclusions;
namespace NzbDrone.Core.Music
{
@ -29,6 +30,7 @@ namespace NzbDrone.Core.Music
private readonly IDiskScanService _diskScanService;
private readonly ICheckIfArtistShouldBeRefreshed _checkIfArtistShouldBeRefreshed;
private readonly IConfigService _configService;
private readonly IImportListExclusionService _importListExclusionService;
private readonly Logger _logger;
public RefreshArtistService(IProvideArtistInfo artistInfo,
@ -41,6 +43,7 @@ namespace NzbDrone.Core.Music
IDiskScanService diskScanService,
ICheckIfArtistShouldBeRefreshed checkIfArtistShouldBeRefreshed,
IConfigService configService,
IImportListExclusionService importListExclusionService,
Logger logger)
{
_artistInfo = artistInfo;
@ -53,6 +56,7 @@ namespace NzbDrone.Core.Music
_diskScanService = diskScanService;
_checkIfArtistShouldBeRefreshed = checkIfArtistShouldBeRefreshed;
_configService = configService;
_importListExclusionService = importListExclusionService;
_logger = logger;
}
@ -75,6 +79,16 @@ namespace NzbDrone.Core.Music
if (artist.Metadata.Value.ForeignArtistId != artistInfo.Metadata.Value.ForeignArtistId)
{
_logger.Warn("Artist '{0}' (Artist {1}) was replaced with '{2}' (LidarrAPI {3}), because the original was a duplicate.", artist.Name, artist.Metadata.Value.ForeignArtistId, artistInfo.Name, artistInfo.Metadata.Value.ForeignArtistId);
// Update list exclusion if one exists
var importExclusion = _importListExclusionService.FindByForeignId(artist.Metadata.Value.ForeignArtistId);
if (importExclusion != null)
{
importExclusion.ForeignId = artistInfo.Metadata.Value.ForeignArtistId;
_importListExclusionService.Update(importExclusion);
}
artist.Metadata.Value.ForeignArtistId = artistInfo.Metadata.Value.ForeignArtistId;
}

View file

@ -194,6 +194,7 @@
<Compile Include="Datastore\Migration\017_remove_nma.cs" />
<Compile Include="Datastore\Migration\018_album_disambiguation.cs" />
<Compile Include="Datastore\Migration\019_add_ape_quality_in_profiles.cs" />
<Compile Include="Datastore\Migration\027_add_import_exclusions.cs" />
<Compile Include="Datastore\Migration\021_add_custom_filters.cs" />
<Compile Include="Datastore\Migration\022_import_list_tags.cs" />
<Compile Include="Datastore\Migration\023_add_release_groups_etc.cs" />
@ -545,6 +546,10 @@
<Compile Include="Http\HttpProxySettingsProvider.cs" />
<Compile Include="Http\TorcacheHttpInterceptor.cs" />
<Compile Include="ImportLists\Exceptions\ImportListException.cs" />
<Compile Include="ImportLists\Exclusions\ImportListExclusionExistsValidator.cs" />
<Compile Include="ImportLists\Exclusions\ImportListExclusionService.cs" />
<Compile Include="ImportLists\Exclusions\ImportListExclusionRepository.cs" />
<Compile Include="ImportLists\Exclusions\ImportListExclusion.cs" />
<Compile Include="ImportLists\FetchAndParseImportListService.cs" />
<Compile Include="ImportLists\HeadphonesImport\HeadphonesImport.cs" />
<Compile Include="ImportLists\HeadphonesImport\HeadphonesImportApi.cs" />
@ -1200,6 +1205,7 @@
<Compile Include="Update\UpdateVerification.cs" />
<Compile Include="Update\UpdateVerificationFailedException.cs" />
<Compile Include="Validation\FolderValidator.cs" />
<Compile Include="Validation\GuidValidator.cs" />
<Compile Include="Validation\IpValidation.cs" />
<Compile Include="Validation\LanguageProfileExistsValidator.cs" />
<Compile Include="Validation\MetadataProfileExistsValidator.cs" />

View file

@ -0,0 +1,20 @@
using System;
using FluentValidation.Validators;
namespace NzbDrone.Core.Validation
{
public class GuidValidator : PropertyValidator
{
public GuidValidator()
: base("String is not a valid Guid")
{
}
protected override bool IsValid(PropertyValidatorContext context)
{
if (context.PropertyValue == null) return false;
return Guid.TryParse(context.PropertyValue.ToString(), out Guid guidOutput);
}
}
}