New: Import Lists Base (#196)

* New: Import Lists Base
This commit is contained in:
Qstick 2018-02-06 18:08:36 -05:00 committed by GitHub
parent c712d932a0
commit c105c9a65e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
74 changed files with 3538 additions and 4 deletions

View file

@ -0,0 +1,23 @@
using NzbDrone.Core.ImportLists;
namespace Lidarr.Api.V1.ImportLists
{
public class ImportListModule : ProviderModuleBase<ImportListResource, IImportList, ImportListDefinition>
{
public static readonly ImportListResourceMapper ResourceMapper = new ImportListResourceMapper();
public ImportListModule(ImportListFactory importListFactory)
: base(importListFactory, "importlist", ResourceMapper)
{
}
protected override void Validate(ImportListDefinition definition, bool includeWarnings)
{
if (!definition.Enable)
{
return;
}
base.Validate(definition, includeWarnings);
}
}
}

View file

@ -0,0 +1,55 @@
using NzbDrone.Core.ImportLists;
namespace Lidarr.Api.V1.ImportLists
{
public class ImportListResource : ProviderResource
{
public bool EnableAutomaticAdd { get; set; }
public bool ShouldMonitor { get; set; }
public string RootFolderPath { get; set; }
public int ProfileId { get; set; }
public int LanguageProfileId { get; set; }
public int MetadataProfileId { get; set; }
}
public class ImportListResourceMapper : ProviderResourceMapper<ImportListResource, ImportListDefinition>
{
public override ImportListResource ToResource(ImportListDefinition definition)
{
if (definition == null)
{
return null;
}
var resource = base.ToResource(definition);
resource.EnableAutomaticAdd = definition.EnableAutomaticAdd;
resource.ShouldMonitor = definition.ShouldMonitor;
resource.RootFolderPath = definition.RootFolderPath;
resource.ProfileId = definition.ProfileId;
resource.LanguageProfileId = definition.LanguageProfileId;
resource.MetadataProfileId = definition.MetadataProfileId;
return resource;
}
public override ImportListDefinition ToModel(ImportListResource resource)
{
if (resource == null)
{
return null;
}
var definition = base.ToModel(resource);
definition.EnableAutomaticAdd = resource.EnableAutomaticAdd;
definition.ShouldMonitor = resource.ShouldMonitor;
definition.RootFolderPath = resource.RootFolderPath;
definition.ProfileId = resource.ProfileId;
definition.LanguageProfileId = resource.LanguageProfileId;
definition.MetadataProfileId = resource.MetadataProfileId;
return definition;
}
}
}

View file

@ -96,6 +96,8 @@
<Compile Include="Commands\CommandResource.cs" />
<Compile Include="Config\MetadataProviderConfigModule.cs" />
<Compile Include="Config\MetadataProviderConfigResource.cs" />
<Compile Include="ImportLists\ImportListModule.cs" />
<Compile Include="ImportLists\ImportListResource.cs" />
<Compile Include="Profiles\Metadata\MetadataProfileModule.cs" />
<Compile Include="Profiles\Metadata\MetadataProfileResource.cs" />
<Compile Include="Profiles\Metadata\MetadataProfileSchemaModule.cs" />

View file

@ -0,0 +1,87 @@
using System;
using System.Collections.Generic;
using Moq;
using NUnit.Framework;
using NzbDrone.Core.HealthCheck.Checks;
using NzbDrone.Core.ImportLists;
using NzbDrone.Core.Test.Framework;
namespace NzbDrone.Core.Test.HealthCheck.Checks
{
[TestFixture]
public class ImportListStatusCheckFixture : CoreTest<ImportListStatusCheck>
{
private List<IImportList> _importLists = new List<IImportList>();
private List<ImportListStatus> _blockedImportLists = new List<ImportListStatus>();
[SetUp]
public void SetUp()
{
Mocker.GetMock<IImportListFactory>()
.Setup(v => v.GetAvailableProviders())
.Returns(_importLists);
Mocker.GetMock<IImportListStatusService>()
.Setup(v => v.GetBlockedProviders())
.Returns(_blockedImportLists);
}
private Mock<IImportList> GivenImportList(int i, double backoffHours, double failureHours)
{
var id = i;
var mockImportList = new Mock<IImportList>();
mockImportList.SetupGet(s => s.Definition).Returns(new ImportListDefinition { Id = id });
_importLists.Add(mockImportList.Object);
if (backoffHours != 0.0)
{
_blockedImportLists.Add(new ImportListStatus
{
ProviderId = id,
InitialFailure = DateTime.UtcNow.AddHours(-failureHours),
MostRecentFailure = DateTime.UtcNow.AddHours(-0.1),
EscalationLevel = 5,
DisabledTill = DateTime.UtcNow.AddHours(backoffHours)
});
}
return mockImportList;
}
[Test]
public void should_not_return_error_when_no_import_lists()
{
Subject.Check().ShouldBeOk();
}
[Test]
public void should_return_warning_if_import_list_unavailable()
{
GivenImportList(1, 10.0, 24.0);
GivenImportList(2, 0.0, 0.0);
Subject.Check().ShouldBeWarning();
}
[Test]
public void should_return_error_if_all_import_lists_unavailable()
{
GivenImportList(1, 10.0, 24.0);
Subject.Check().ShouldBeError();
}
[Test]
public void should_return_warning_if_few_import_lists_unavailable()
{
GivenImportList(1, 10.0, 24.0);
GivenImportList(2, 10.0, 24.0);
GivenImportList(3, 0.0, 0.0);
Subject.Check().ShouldBeWarning();
}
}
}

View file

@ -0,0 +1,54 @@
using FizzWare.NBuilder;
using FluentAssertions;
using NUnit.Framework;
using NzbDrone.Core.Housekeeping.Housekeepers;
using NzbDrone.Core.ImportLists;
using NzbDrone.Core.Test.Framework;
namespace NzbDrone.Core.Test.Housekeeping.Housekeepers
{
[TestFixture]
public class CleanupOrphanedImportListStatusFixture : DbTest<CleanupOrphanedImportListStatus, ImportListStatus>
{
private ImportListDefinition _importList;
[SetUp]
public void Setup()
{
_importList = Builder<ImportListDefinition>.CreateNew()
.BuildNew();
}
private void GivenImportList()
{
Db.Insert(_importList);
}
[Test]
public void should_delete_orphaned_importliststatus()
{
var status = Builder<ImportListStatus>.CreateNew()
.With(h => h.ProviderId = _importList.Id)
.BuildNew();
Db.Insert(status);
Subject.Clean();
AllStoredModels.Should().BeEmpty();
}
[Test]
public void should_not_delete_unorphaned_importliststatus()
{
GivenImportList();
var status = Builder<ImportListStatus>.CreateNew()
.With(h => h.ProviderId = _importList.Id)
.BuildNew();
Db.Insert(status);
Subject.Clean();
AllStoredModels.Should().HaveCount(1);
AllStoredModels.Should().Contain(h => h.ProviderId == _importList.Id);
}
}
}

View file

@ -0,0 +1,119 @@
using System;
using System.Collections.Generic;
using System.Linq;
using FizzWare.NBuilder;
using Moq;
using NUnit.Framework;
using NzbDrone.Core.Housekeeping.Housekeepers;
using NzbDrone.Core.ImportLists;
using NzbDrone.Core.Test.Framework;
using NzbDrone.Core.ThingiProvider.Status;
namespace NzbDrone.Core.Test.Housekeeping.Housekeepers
{
[TestFixture]
public class FixFutureImportListStatusTimesFixture : CoreTest<FixFutureImportListStatusTimes>
{
[Test]
public void should_set_disabled_till_when_its_too_far_in_the_future()
{
var disabledTillTime = EscalationBackOff.Periods[1];
var importListStatuses = Builder<ImportListStatus>.CreateListOfSize(5)
.All()
.With(t => t.DisabledTill = DateTime.UtcNow.AddDays(5))
.With(t => t.InitialFailure = DateTime.UtcNow.AddDays(-5))
.With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(-5))
.With(t => t.EscalationLevel = 1)
.BuildListOfNew();
Mocker.GetMock<IImportListStatusRepository>()
.Setup(s => s.All())
.Returns(importListStatuses);
Subject.Clean();
Mocker.GetMock<IImportListStatusRepository>()
.Verify(v => v.UpdateMany(
It.Is<List<ImportListStatus>>(i => i.All(
s => s.DisabledTill.Value <= DateTime.UtcNow.AddMinutes(disabledTillTime)))
)
);
}
[Test]
public void should_set_initial_failure_when_its_in_the_future()
{
var importListStatuses = Builder<ImportListStatus>.CreateListOfSize(5)
.All()
.With(t => t.DisabledTill = DateTime.UtcNow.AddDays(-5))
.With(t => t.InitialFailure = DateTime.UtcNow.AddDays(5))
.With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(-5))
.With(t => t.EscalationLevel = 1)
.BuildListOfNew();
Mocker.GetMock<IImportListStatusRepository>()
.Setup(s => s.All())
.Returns(importListStatuses);
Subject.Clean();
Mocker.GetMock<IImportListStatusRepository>()
.Verify(v => v.UpdateMany(
It.Is<List<ImportListStatus>>(i => i.All(
s => s.InitialFailure.Value <= DateTime.UtcNow))
)
);
}
[Test]
public void should_set_most_recent_failure_when_its_in_the_future()
{
var importListStatuses = Builder<ImportListStatus>.CreateListOfSize(5)
.All()
.With(t => t.DisabledTill = DateTime.UtcNow.AddDays(-5))
.With(t => t.InitialFailure = DateTime.UtcNow.AddDays(-5))
.With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(5))
.With(t => t.EscalationLevel = 1)
.BuildListOfNew();
Mocker.GetMock<IImportListStatusRepository>()
.Setup(s => s.All())
.Returns(importListStatuses);
Subject.Clean();
Mocker.GetMock<IImportListStatusRepository>()
.Verify(v => v.UpdateMany(
It.Is<List<ImportListStatus>>(i => i.All(
s => s.MostRecentFailure.Value <= DateTime.UtcNow))
)
);
}
[Test]
public void should_not_change_statuses_when_times_are_in_the_past()
{
var importListStatuses = Builder<ImportListStatus>.CreateListOfSize(5)
.All()
.With(t => t.DisabledTill = DateTime.UtcNow.AddDays(-5))
.With(t => t.InitialFailure = DateTime.UtcNow.AddDays(-5))
.With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(-5))
.With(t => t.EscalationLevel = 0)
.BuildListOfNew();
Mocker.GetMock<IImportListStatusRepository>()
.Setup(s => s.All())
.Returns(importListStatuses);
Subject.Clean();
Mocker.GetMock<IImportListStatusRepository>()
.Verify(v => v.UpdateMany(
It.Is<List<ImportListStatus>>(i => i.Count == 0)
)
);
}
}
}

View file

@ -0,0 +1,43 @@
using System.Collections.Generic;
using FizzWare.NBuilder;
using FluentAssertions;
using NUnit.Framework;
using NzbDrone.Core.ImportLists;
using NzbDrone.Core.ImportLists.LidarrLists;
using NzbDrone.Core.Lifecycle;
using NzbDrone.Core.Test.Framework;
namespace NzbDrone.Core.Test.ImportListTests
{
public class ImportListServiceFixture : DbTest<ImportListFactory, ImportListDefinition>
{
private List<IImportList> _importLists;
[SetUp]
public void Setup()
{
_importLists = new List<IImportList>();
_importLists.Add(Mocker.Resolve<LidarrLists>());
Mocker.SetConstant<IEnumerable<IImportList>>(_importLists);
}
[Test]
public void should_remove_missing_import_lists_on_startup()
{
var repo = Mocker.Resolve<ImportListRepository>();
Mocker.SetConstant<IImportListRepository>(repo);
var existingImportLists = Builder<ImportListDefinition>.CreateNew().BuildNew();
existingImportLists.ConfigContract = typeof (LidarrListsSettings).Name;
repo.Insert(existingImportLists);
Subject.Handle(new ApplicationStartedEvent());
AllStoredModels.Should().NotContain(c => c.Id == existingImportLists.Id);
}
}
}

View file

@ -0,0 +1,67 @@
using System;
using System.Linq;
using FluentAssertions;
using Moq;
using NUnit.Framework;
using NzbDrone.Core.ImportLists;
using NzbDrone.Core.Test.Framework;
namespace NzbDrone.Core.Test.ImportListTests
{
public class ImportListStatusServiceFixture : CoreTest<ImportListStatusService>
{
private DateTime _epoch;
[SetUp]
public void SetUp()
{
_epoch = DateTime.UtcNow;
}
private void WithStatus(ImportListStatus status)
{
Mocker.GetMock<IImportListStatusRepository>()
.Setup(v => v.FindByProviderId(1))
.Returns(status);
Mocker.GetMock<IImportListStatusRepository>()
.Setup(v => v.All())
.Returns(new[] { status });
}
private void VerifyUpdate()
{
Mocker.GetMock<IImportListStatusRepository>()
.Verify(v => v.Upsert(It.IsAny<ImportListStatus>()), Times.Once());
}
private void VerifyNoUpdate()
{
Mocker.GetMock<IImportListStatusRepository>()
.Verify(v => v.Upsert(It.IsAny<ImportListStatus>()), Times.Never());
}
[Test]
public void should_cancel_backoff_on_success()
{
WithStatus(new ImportListStatus { EscalationLevel = 2 });
Subject.RecordSuccess(1);
VerifyUpdate();
var status = Subject.GetBlockedProviders().FirstOrDefault();
status.Should().BeNull();
}
[Test]
public void should_not_store_update_if_already_okay()
{
WithStatus(new ImportListStatus { EscalationLevel = 0 });
Subject.RecordSuccess(1);
VerifyNoUpdate();
}
}
}

View file

@ -0,0 +1,177 @@
using System.Linq;
using System.Collections.Generic;
using Moq;
using NUnit.Framework;
using NzbDrone.Core.ImportLists;
using NzbDrone.Core.MetadataSource;
using NzbDrone.Core.Music;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Test.Framework;
namespace NzbDrone.Core.Test.ImportListTests
{
public class ImportListSyncServiceFixture : CoreTest<ImportListSyncService>
{
private List<ImportListItemInfo> _importListReports;
[SetUp]
public void SetUp()
{
var importListItem1 = new ImportListItemInfo
{
Artist = "Linkin Park"
};
_importListReports = new List<ImportListItemInfo>{importListItem1};
Mocker.GetMock<IFetchAndParseImportList>()
.Setup(v => v.Fetch())
.Returns(_importListReports);
Mocker.GetMock<ISearchForNewArtist>()
.Setup(v => v.SearchForNewArtist(It.IsAny<string>()))
.Returns(new List<Artist>());
Mocker.GetMock<ISearchForNewAlbum>()
.Setup(v => v.SearchForNewAlbum(It.IsAny<string>(), It.IsAny<string>()))
.Returns(new List<Album>());
Mocker.GetMock<IImportListFactory>()
.Setup(v => v.Get(It.IsAny<int>()))
.Returns(new ImportListDefinition());
Mocker.GetMock<IFetchAndParseImportList>()
.Setup(v => v.Fetch())
.Returns(_importListReports);
}
private void WithAlbum()
{
_importListReports.First().Album = "Meteora";
}
private void WithArtistId()
{
_importListReports.First().ArtistMusicBrainzId = "f59c5520-5f46-4d2c-b2c4-822eabf53419";
}
private void WithAlbumId()
{
_importListReports.First().AlbumMusicBrainzId = "09474d62-17dd-3a4f-98fb-04c65f38a479";
}
private void WithExistingArtist()
{
Mocker.GetMock<IArtistService>()
.Setup(v => v.FindById(_importListReports.First().ArtistMusicBrainzId))
.Returns(new Artist{ForeignArtistId = _importListReports.First().ArtistMusicBrainzId });
}
[Test]
public void should_search_if_artist_title_and_no_artist_id()
{
Subject.Execute(new ImportListSyncCommand());
Mocker.GetMock<ISearchForNewArtist>()
.Verify(v => v.SearchForNewArtist(It.IsAny<string>()), Times.Once());
}
[Test]
public void should_not_search_if_artist_title_and_artist_id()
{
WithArtistId();
Subject.Execute(new ImportListSyncCommand());
Mocker.GetMock<ISearchForNewArtist>()
.Verify(v => v.SearchForNewArtist(It.IsAny<string>()), Times.Never());
}
[Test]
public void should_search_if_album_title_and_no_album_id()
{
WithAlbum();
Subject.Execute(new ImportListSyncCommand());
Mocker.GetMock<ISearchForNewAlbum>()
.Verify(v => v.SearchForNewAlbum(It.IsAny<string>(), It.IsAny<string>()), Times.Once());
}
[Test]
public void should_not_search_if_album_title_and_album_id()
{
WithAlbum();
WithAlbumId();
Subject.Execute(new ImportListSyncCommand());
Mocker.GetMock<ISearchForNewAlbum>()
.Verify(v => v.SearchForNewAlbum(It.IsAny<string>(), It.IsAny<string>()), Times.Never());
}
[Test]
public void should_not_search_if_all_info()
{
WithArtistId();
WithAlbum();
WithAlbumId();
Subject.Execute(new ImportListSyncCommand());
Mocker.GetMock<ISearchForNewArtist>()
.Verify(v => v.SearchForNewArtist(It.IsAny<string>()), Times.Never());
Mocker.GetMock<ISearchForNewAlbum>()
.Verify(v => v.SearchForNewAlbum(It.IsAny<string>(), It.IsAny<string>()), Times.Never());
}
[Test]
public void should_not_try_add_if_existing_artist()
{
WithArtistId();
WithAlbum();
WithAlbumId();
WithExistingArtist();
Subject.Execute(new ImportListSyncCommand());
Mocker.GetMock<IAddArtistService>()
.Verify(v => v.AddArtists(It.Is<List<Artist>>(t=>t.Count == 0)));
}
[Test]
public void should_add_if_not_existing_artist()
{
WithArtistId();
WithAlbum();
WithAlbumId();
Subject.Execute(new ImportListSyncCommand());
Mocker.GetMock<IAddArtistService>()
.Verify(v => v.AddArtists(It.Is<List<Artist>>(t => t.Count == 1)));
}
[Test]
public void should_mark_album_for_monitor_if_album_id()
{
WithArtistId();
WithAlbum();
WithAlbumId();
Subject.Execute(new ImportListSyncCommand());
Mocker.GetMock<IAddArtistService>()
.Verify(v => v.AddArtists(It.Is<List<Artist>>(t => t.Count == 1 && t.First().AddOptions.AlbumsToMonitor.Contains("09474d62-17dd-3a4f-98fb-04c65f38a479"))));
}
[Test]
public void should_not_mark_album_for_monitor_if_no_album_id()
{
WithArtistId();
Subject.Execute(new ImportListSyncCommand());
Mocker.GetMock<IAddArtistService>()
.Verify(v => v.AddArtists(It.Is<List<Artist>>(t => t.Count == 1 && t.First().AddOptions.AlbumsToMonitor.Count == 0)));
}
}
}

View file

@ -205,6 +205,7 @@
<Compile Include="HealthCheck\Checks\ImportMechanismCheckFixture.cs" />
<Compile Include="HealthCheck\Checks\IndexerSearchCheckFixture.cs" />
<Compile Include="HealthCheck\Checks\IndexerRssCheckFixture.cs" />
<Compile Include="HealthCheck\Checks\ImportListStatusCheckFixture.cs" />
<Compile Include="HealthCheck\Checks\MonoVersionCheckFixture.cs" />
<Compile Include="HealthCheck\Checks\IndexerStatusCheckFixture.cs" />
<Compile Include="HealthCheck\Checks\RootFolderCheckFixture.cs" />
@ -219,6 +220,7 @@
<Compile Include="Housekeeping\Housekeepers\CleanupDuplicateMetadataFilesFixture.cs" />
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedAlbumsFixture.cs" />
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedBlacklistFixture.cs" />
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedImportListStatusFixture.cs" />
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedTrackFilesFixture.cs" />
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedTracksFixture.cs" />
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedIndexerStatusFixture.cs" />
@ -227,9 +229,13 @@
<Compile Include="Housekeeping\Housekeepers\CleanupUnusedTagsFixture.cs" />
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedPendingReleasesFixture.cs" />
<Compile Include="Housekeeping\Housekeepers\FixFutureDownloadClientStatusTimesFixture.cs" />
<Compile Include="Housekeeping\Housekeepers\FixFutureImportListStatusTimesFixture.cs" />
<Compile Include="Housekeeping\Housekeepers\FixFutureIndexerStatusTimesFixture.cs" />
<Compile Include="Http\HttpProxySettingsProviderFixture.cs" />
<Compile Include="Http\TorCacheHttpRequestInterceptorFixture.cs" />
<Compile Include="ImportListTests\ImportListServiceFixture.cs" />
<Compile Include="ImportListTests\ImportListStatusServiceFixture.cs" />
<Compile Include="ImportListTests\ImportListSyncServiceFixture.cs" />
<Compile Include="IndexerSearchTests\ArtistSearchServiceFixture.cs" />
<Compile Include="IndexerSearchTests\SearchDefinitionFixture.cs" />
<Compile Include="IndexerTests\BasicRssParserFixture.cs" />

View file

@ -0,0 +1,32 @@
using FluentMigrator;
using NzbDrone.Core.Datastore.Migration.Framework;
namespace NzbDrone.Core.Datastore.Migration
{
[Migration(11)]
public class import_lists : NzbDroneMigrationBase
{
protected override void MainDbUpgrade()
{
Create.TableForModel("ImportLists")
.WithColumn("Name").AsString().Unique()
.WithColumn("Implementation").AsString()
.WithColumn("Settings").AsString().Nullable()
.WithColumn("ConfigContract").AsString().Nullable()
.WithColumn("EnableAutomaticAdd").AsBoolean().Nullable()
.WithColumn("RootFolderPath").AsString()
.WithColumn("ShouldMonitor").AsInt32()
.WithColumn("ProfileId").AsInt32()
.WithColumn("LanguageProfileId").AsInt32()
.WithColumn("MetadataProfileId").AsInt32();
Create.TableForModel("ImportListStatus")
.WithColumn("ProviderId").AsInt32().NotNullable().Unique()
.WithColumn("InitialFailure").AsDateTime().Nullable()
.WithColumn("MostRecentFailure").AsDateTime().Nullable()
.WithColumn("EscalationLevel").AsInt32().NotNullable()
.WithColumn("DisabledTill").AsDateTime().Nullable()
.WithColumn("LastSyncListInfo").AsString().Nullable();
}
}
}

View file

@ -10,6 +10,7 @@ using NzbDrone.Core.Datastore.Extensions;
using NzbDrone.Core.Download;
using NzbDrone.Core.Download.Pending;
using NzbDrone.Core.Indexers;
using NzbDrone.Core.ImportLists;
using NzbDrone.Core.Instrumentation;
using NzbDrone.Core.Jobs;
using NzbDrone.Core.MediaFiles;
@ -59,6 +60,10 @@ namespace NzbDrone.Core.Datastore
.Ignore(i => i.SupportsSearch)
.Ignore(d => d.Tags);
Mapper.Entity<ImportListDefinition>().RegisterDefinition("ImportLists")
.Ignore(i => i.Enable)
.Ignore(d => d.Tags);
Mapper.Entity<NotificationDefinition>().RegisterDefinition("Notifications")
.Ignore(i => i.SupportsOnGrab)
.Ignore(i => i.SupportsOnDownload)
@ -130,6 +135,7 @@ namespace NzbDrone.Core.Datastore
Mapper.Entity<IndexerStatus>().RegisterModel("IndexerStatus");
Mapper.Entity<DownloadClientStatus>().RegisterModel("DownloadClientStatus");
Mapper.Entity<ImportListStatus>().RegisterModel("ImportListStatus");
}
private static void RegisterMappers()

View file

@ -0,0 +1,45 @@
using System;
using System.Linq;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.ImportLists;
using NzbDrone.Core.ThingiProvider.Events;
namespace NzbDrone.Core.HealthCheck.Checks
{
[CheckOn(typeof(ProviderUpdatedEvent<IImportList>))]
[CheckOn(typeof(ProviderDeletedEvent<IImportList>))]
[CheckOn(typeof(ProviderStatusChangedEvent<IImportList>))]
public class ImportListStatusCheck : HealthCheckBase
{
private readonly IImportListFactory _providerFactory;
private readonly IImportListStatusService _providerStatusService;
public ImportListStatusCheck(IImportListFactory providerFactory, IImportListStatusService providerStatusService)
{
_providerFactory = providerFactory;
_providerStatusService = providerStatusService;
}
public override HealthCheck Check()
{
var enabledProviders = _providerFactory.GetAvailableProviders();
var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(),
i => i.Definition.Id,
s => s.ProviderId,
(i, s) => new { ImportList = i, Status = s })
.ToList();
if (backOffProviders.Empty())
{
return new HealthCheck(GetType());
}
if (backOffProviders.Count == enabledProviders.Count)
{
return new HealthCheck(GetType(), HealthCheckResult.Error, "All import lists are unavailable due to failures", "#import-lists-are-unavailable-due-to-failures");
}
return new HealthCheck(GetType(), HealthCheckResult.Warning, string.Format("Import lists unavailable due to failures: {0}", string.Join(", ", backOffProviders.Select(v => v.ImportList.Definition.Name))), "#import-lsits-are-unavailable-due-to-failures");
}
}
}

View file

@ -0,0 +1,26 @@
using NzbDrone.Core.Datastore;
namespace NzbDrone.Core.Housekeeping.Housekeepers
{
public class CleanupOrphanedImportListStatus : IHousekeepingTask
{
private readonly IMainDatabase _database;
public CleanupOrphanedImportListStatus(IMainDatabase database)
{
_database = database;
}
public void Clean()
{
var mapper = _database.GetDataMapper();
mapper.ExecuteNonQuery(@"DELETE FROM ImportListStatus
WHERE Id IN (
SELECT ImportListStatus.Id FROM ImportListStatus
LEFT OUTER JOIN ImportLists
ON ImportListStatus.ProviderId = ImportLists.Id
WHERE ImportLists.Id IS NULL)");
}
}
}

View file

@ -0,0 +1,12 @@
using NzbDrone.Core.ImportLists;
namespace NzbDrone.Core.Housekeeping.Housekeepers
{
public class FixFutureImportListStatusTimes : FixFutureProviderStatusTimes<ImportListStatus>, IHousekeepingTask
{
public FixFutureImportListStatusTimes(IImportListStatusRepository importListStatusRepository)
: base(importListStatusRepository)
{
}
}
}

View file

@ -0,0 +1,23 @@
using NzbDrone.Common.Exceptions;
namespace NzbDrone.Core.ImportLists.Exceptions
{
public class ImportListException : NzbDroneException
{
private readonly ImportListResponse _importListResponse;
public ImportListException(ImportListResponse response, string message, params object[] args)
: base(message, args)
{
_importListResponse = response;
}
public ImportListException(ImportListResponse response, string message)
: base(message)
{
_importListResponse = response;
}
public ImportListResponse Response => _importListResponse;
}
}

View file

@ -0,0 +1,80 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using NLog;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Common.TPL;
using System;
using NzbDrone.Common.Extensions;
namespace NzbDrone.Core.ImportLists
{
public interface IFetchAndParseImportList
{
List<ImportListItemInfo> Fetch();
}
public class FetchAndParseImportListService : IFetchAndParseImportList
{
private readonly IImportListFactory _importListFactory;
private readonly Logger _logger;
public FetchAndParseImportListService(IImportListFactory importListFactory, Logger logger)
{
_importListFactory = importListFactory;
_logger = logger;
}
public List<ImportListItemInfo> Fetch()
{
var result = new List<ImportListItemInfo>();
var importLists = _importListFactory.AutomaticAddEnabled();
if (!importLists.Any())
{
_logger.Warn("No available import lists. check your configuration.");
return result;
}
_logger.Debug("Available import lists {0}", importLists.Count);
var taskList = new List<Task>();
var taskFactory = new TaskFactory(TaskCreationOptions.LongRunning, TaskContinuationOptions.None);
foreach (var importList in importLists)
{
var importListLocal = importList;
var task = taskFactory.StartNew(() =>
{
try
{
var importListReports = importListLocal.Fetch();
lock (result)
{
_logger.Debug("Found {0} from {1}", importListReports.Count, importList.Name);
result.AddRange(importListReports);
}
}
catch (Exception e)
{
_logger.Error(e, "Error during Import List Sync");
}
}).LogExceptions();
taskList.Add(task);
}
Task.WaitAll(taskList.ToArray());
result = result.DistinctBy(r => new {r.Artist, r.Album}).ToList();
_logger.Debug("Found {0} reports", result.Count);
return result;
}
}
}

View file

@ -0,0 +1,30 @@
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Parser;
namespace NzbDrone.Core.ImportLists.HeadphonesImport
{
public class HeadphonesImport : HttpImportListBase<HeadphonesImportSettings>
{
public override string Name => "Headphones";
public override int PageSize => 1000;
public HeadphonesImport(IHttpClient httpClient, IImportListStatusService importListStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
: base(httpClient, importListStatusService, configService, parsingService, logger)
{
}
public override IImportListRequestGenerator GetRequestGenerator()
{
return new HeadphonesImportRequestGenerator { Settings = Settings};
}
public override IParseImportListResponse GetParser()
{
return new HeadphonesImportParser();
}
}
}

View file

@ -0,0 +1,9 @@
namespace NzbDrone.Core.ImportLists.HeadphonesImport
{
public class HeadphonesImportArtist
{
public string ArtistName { get; set; }
public string ArtistId { get; set; }
}
}

View file

@ -0,0 +1,64 @@
using Newtonsoft.Json;
using NzbDrone.Core.ImportLists.Exceptions;
using NzbDrone.Core.Parser.Model;
using System.Collections.Generic;
using System.Net;
using NLog;
using NzbDrone.Common.Extensions;
namespace NzbDrone.Core.ImportLists.HeadphonesImport
{
public class HeadphonesImportParser : IParseImportListResponse
{
private ImportListResponse _importListResponse;
private readonly Logger _logger;
public IList<ImportListItemInfo> ParseResponse(ImportListResponse importListResponse)
{
_importListResponse = importListResponse;
var items = new List<ImportListItemInfo>();
if (!PreProcess(_importListResponse))
{
return items;
}
var jsonResponse = JsonConvert.DeserializeObject<List<HeadphonesImportArtist>>(_importListResponse.Content);
// no albums were return
if (jsonResponse == null)
{
return items;
}
foreach (var item in jsonResponse)
{
items.AddIfNotNull(new ImportListItemInfo
{
Artist = item.ArtistName,
ArtistMusicBrainzId = item.ArtistId
});
}
return items;
}
protected virtual bool PreProcess(ImportListResponse importListResponse)
{
if (importListResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
{
throw new ImportListException(importListResponse, "Import List API call resulted in an unexpected StatusCode [{0}]", importListResponse.HttpResponse.StatusCode);
}
if (importListResponse.HttpResponse.Headers.ContentType != null && importListResponse.HttpResponse.Headers.ContentType.Contains("text/json") &&
importListResponse.HttpRequest.Headers.Accept != null && !importListResponse.HttpRequest.Headers.Accept.Contains("text/json"))
{
throw new ImportListException(importListResponse, "Import List responded with html content. Site is likely blocked or unavailable.");
}
return true;
}
}
}

View file

@ -0,0 +1,34 @@
using System.Collections.Generic;
using NzbDrone.Common.Http;
namespace NzbDrone.Core.ImportLists.HeadphonesImport
{
public class HeadphonesImportRequestGenerator : IImportListRequestGenerator
{
public HeadphonesImportSettings Settings { get; set; }
public int MaxPages { get; set; }
public int PageSize { get; set; }
public HeadphonesImportRequestGenerator()
{
MaxPages = 1;
PageSize = 1000;
}
public virtual ImportListPageableRequestChain GetListItems()
{
var pageableRequests = new ImportListPageableRequestChain();
pageableRequests.Add(GetPagedRequests());
return pageableRequests;
}
private IEnumerable<ImportListRequest> GetPagedRequests()
{
yield return new ImportListRequest(string.Format("{0}/api?cmd=getIndex&apikey={1}", Settings.BaseUrl.TrimEnd('/'), Settings.ApiKey), HttpAccept.Json);
}
}
}

View file

@ -0,0 +1,35 @@
using FluentValidation;
using NzbDrone.Core.Annotations;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.ImportLists.HeadphonesImport
{
public class HeadphonesImportSettingsValidator : AbstractValidator<HeadphonesImportSettings>
{
public HeadphonesImportSettingsValidator()
{
RuleFor(c => c.BaseUrl).ValidRootUrl();
}
}
public class HeadphonesImportSettings : IImportListSettings
{
private static readonly HeadphonesImportSettingsValidator Validator = new HeadphonesImportSettingsValidator();
public HeadphonesImportSettings()
{
BaseUrl = "http://localhost:8181/";
}
[FieldDefinition(0, Label = "Headphones URL")]
public string BaseUrl { get; set; }
[FieldDefinition(1, Label = "API Key")]
public string ApiKey { get; set; }
public NzbDroneValidationResult Validate()
{
return new NzbDroneValidationResult(Validator.Validate(this));
}
}
}

View file

@ -0,0 +1,246 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Http.CloudFlare;
using NzbDrone.Core.ImportLists.Exceptions;
using NzbDrone.Core.Indexers.Exceptions;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.ThingiProvider;
namespace NzbDrone.Core.ImportLists
{
public abstract class HttpImportListBase<TSettings> : ImportListBase<TSettings>
where TSettings : IImportListSettings, new()
{
protected const int MaxNumResultsPerQuery = 1000;
protected readonly IHttpClient _httpClient;
public bool SupportsPaging => PageSize > 0;
public virtual int PageSize => 0;
public virtual TimeSpan RateLimit => TimeSpan.FromSeconds(2);
public abstract IImportListRequestGenerator GetRequestGenerator();
public abstract IParseImportListResponse GetParser();
public HttpImportListBase(IHttpClient httpClient, IImportListStatusService importListStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
: base(importListStatusService, configService, parsingService, logger)
{
_httpClient = httpClient;
}
public override IList<ImportListItemInfo> Fetch()
{
return FetchReleases(g => g.GetListItems(), true);
}
protected virtual IList<ImportListItemInfo> FetchReleases(Func<IImportListRequestGenerator, ImportListPageableRequestChain> pageableRequestChainSelector, bool isRecent = false)
{
var releases = new List<ImportListItemInfo>();
var url = string.Empty;
try
{
var generator = GetRequestGenerator();
var parser = GetParser();
var pageableRequestChain = pageableRequestChainSelector(generator);
for (int i = 0; i < pageableRequestChain.Tiers; i++)
{
var pageableRequests = pageableRequestChain.GetTier(i);
foreach (var pageableRequest in pageableRequests)
{
var pagedReleases = new List<ImportListItemInfo>();
foreach (var request in pageableRequest)
{
url = request.Url.FullUri;
var page = FetchPage(request, parser);
pagedReleases.AddRange(page);
if (pagedReleases.Count >= MaxNumResultsPerQuery)
{
break;
}
if (!IsFullPage(page))
{
break;
}
}
releases.AddRange(pagedReleases.Where(IsValidRelease));
}
if (releases.Any())
{
break;
}
}
_importListStatusService.RecordSuccess(Definition.Id);
}
catch (WebException webException)
{
if (webException.Status == WebExceptionStatus.NameResolutionFailure ||
webException.Status == WebExceptionStatus.ConnectFailure)
{
_importListStatusService.RecordConnectionFailure(Definition.Id);
}
else
{
_importListStatusService.RecordFailure(Definition.Id);
}
if (webException.Message.Contains("502") || webException.Message.Contains("503") ||
webException.Message.Contains("timed out"))
{
_logger.Warn("{0} server is currently unavailable. {1} {2}", this, url, webException.Message);
}
else
{
_logger.Warn("{0} {1} {2}", this, url, webException.Message);
}
}
catch (TooManyRequestsException ex)
{
if (ex.RetryAfter != TimeSpan.Zero)
{
_importListStatusService.RecordFailure(Definition.Id, ex.RetryAfter);
}
else
{
_importListStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
}
_logger.Warn("API Request Limit reached for {0}", this);
}
catch (HttpException ex)
{
_importListStatusService.RecordFailure(Definition.Id);
_logger.Warn("{0} {1}", this, ex.Message);
}
catch (RequestLimitReachedException)
{
_importListStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
_logger.Warn("API Request Limit reached for {0}", this);
}
catch (CloudFlareCaptchaException ex)
{
_importListStatusService.RecordFailure(Definition.Id);
ex.WithData("FeedUrl", url);
if (ex.IsExpired)
{
_logger.Error(ex, "Expired CAPTCHA token for {0}, please refresh in import list settings.", this);
}
else
{
_logger.Error(ex, "CAPTCHA token required for {0}, check import list settings.", this);
}
}
catch (ImportListException ex)
{
_importListStatusService.RecordFailure(Definition.Id);
_logger.Warn(ex, "{0}", url);
}
catch (Exception ex)
{
_importListStatusService.RecordFailure(Definition.Id);
ex.WithData("FeedUrl", url);
_logger.Error(ex, "An error occurred while processing feed. {0}", url);
}
return CleanupListItems(releases);
}
protected virtual bool IsValidRelease(ImportListItemInfo release)
{
if (release.Album.IsNullOrWhiteSpace() && release.Artist.IsNullOrWhiteSpace())
{
return false;
}
return true;
}
protected virtual bool IsFullPage(IList<ImportListItemInfo> page)
{
return PageSize != 0 && page.Count >= PageSize;
}
protected virtual IList<ImportListItemInfo> FetchPage(ImportListRequest request, IParseImportListResponse parser)
{
var response = FetchImportListResponse(request);
return parser.ParseResponse(response).ToList();
}
protected virtual ImportListResponse FetchImportListResponse(ImportListRequest request)
{
_logger.Debug("Downloading Feed " + request.HttpRequest.ToString(false));
if (request.HttpRequest.RateLimit < RateLimit)
{
request.HttpRequest.RateLimit = RateLimit;
}
return new ImportListResponse(request, _httpClient.Execute(request.HttpRequest));
}
protected override void Test(List<ValidationFailure> failures)
{
failures.AddIfNotNull(TestConnection());
}
protected virtual ValidationFailure TestConnection()
{
try
{
var parser = GetParser();
var generator = GetRequestGenerator();
var releases = FetchPage(generator.GetListItems().GetAllTiers().First().First(), parser);
if (releases.Empty())
{
return new ValidationFailure(string.Empty, "No results were returned from your import list, please check your settings.");
}
}
catch (RequestLimitReachedException)
{
_logger.Warn("Request limit reached");
}
catch (UnsupportedFeedException ex)
{
_logger.Warn(ex, "Import list feed is not supported");
return new ValidationFailure(string.Empty, "Import list feed is not supported: " + ex.Message);
}
catch (ImportListException ex)
{
_logger.Warn(ex, "Unable to connect to import list");
return new ValidationFailure(string.Empty, "Unable to connect to import list. " + ex.Message);
}
catch (Exception ex)
{
_logger.Warn(ex, "Unable to connect to import list");
return new ValidationFailure(string.Empty, "Unable to connect to import list, check the log for more details");
}
return null;
}
}
}

View file

@ -0,0 +1,11 @@
using System.Collections.Generic;
using NzbDrone.Core.ThingiProvider;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.ImportLists
{
public interface IImportList : IProvider
{
IList<ImportListItemInfo> Fetch();
}
}

View file

@ -0,0 +1,7 @@
namespace NzbDrone.Core.ImportLists
{
public interface IImportListRequestGenerator
{
ImportListPageableRequestChain GetListItems();
}
}

View file

@ -0,0 +1,9 @@
using NzbDrone.Core.ThingiProvider;
namespace NzbDrone.Core.ImportLists
{
public interface IImportListSettings : IProviderConfig
{
string BaseUrl { get; set; }
}
}

View file

@ -0,0 +1,10 @@
using System.Collections.Generic;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.ImportLists
{
public interface IParseImportListResponse
{
IList<ImportListItemInfo> ParseResponse(ImportListResponse importListResponse);
}
}

View file

@ -0,0 +1,97 @@
using System;
using System.Collections.Generic;
using System.Linq;
using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.ThingiProvider;
namespace NzbDrone.Core.ImportLists
{
public abstract class ImportListBase<TSettings> : IImportList
where TSettings : IImportListSettings, new()
{
protected readonly IImportListStatusService _importListStatusService;
protected readonly IConfigService _configService;
protected readonly IParsingService _parsingService;
protected readonly Logger _logger;
public abstract string Name { get; }
public ImportListBase(IImportListStatusService importListStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
{
_importListStatusService = importListStatusService;
_configService = configService;
_parsingService = parsingService;
_logger = logger;
}
public Type ConfigContract => typeof(TSettings);
public virtual ProviderMessage Message => null;
public virtual IEnumerable<ProviderDefinition> DefaultDefinitions
{
get
{
var config = (IProviderConfig)new TSettings();
yield return new ImportListDefinition
{
Name = GetType().Name,
EnableAutomaticAdd = config.Validate().IsValid,
Implementation = GetType().Name,
Settings = config
};
}
}
public virtual ProviderDefinition Definition { get; set; }
public virtual object RequestAction(string action, IDictionary<string, string> query) { return null; }
protected TSettings Settings => (TSettings)Definition.Settings;
public abstract IList<ImportListItemInfo> Fetch();
protected virtual IList<ImportListItemInfo> CleanupListItems(IEnumerable<ImportListItemInfo> releases)
{
var result = releases.DistinctBy(r => new {r.Artist, r.Album}).ToList();
result.ForEach(c =>
{
c.ImportListId = Definition.Id;
c.ImportList = Definition.Name;
});
return result;
}
public ValidationResult Test()
{
var failures = new List<ValidationFailure>();
try
{
Test(failures);
}
catch (Exception ex)
{
_logger.Error(ex, "Test aborted due to exception");
failures.Add(new ValidationFailure(string.Empty, "Test was aborted due to an error: " + ex.Message));
}
return new ValidationResult(failures);
}
protected abstract void Test(List<ValidationFailure> failures);
public override string ToString()
{
return Definition.Name;
}
}
}

View file

@ -0,0 +1,19 @@
using NzbDrone.Core.ThingiProvider;
namespace NzbDrone.Core.ImportLists
{
public class ImportListDefinition : ProviderDefinition
{
public bool EnableAutomaticAdd { get; set; }
public bool ShouldMonitor { get; set; }
public int ProfileId { get; set; }
public int LanguageProfileId { get; set; }
public int MetadataProfileId { get; set; }
public string RootFolderPath { get; set; }
public override bool Enable => EnableAutomaticAdd;
public ImportListStatus Status { get; set; }
}
}

View file

@ -0,0 +1,79 @@
using System.Collections.Generic;
using System.Linq;
using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Composition;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.ThingiProvider;
namespace NzbDrone.Core.ImportLists
{
public interface IImportListFactory : IProviderFactory<IImportList, ImportListDefinition>
{
List<IImportList> AutomaticAddEnabled(bool filterBlockedImportLists = true);
}
public class ImportListFactory : ProviderFactory<IImportList, ImportListDefinition>, IImportListFactory
{
private readonly IImportListStatusService _importListStatusService;
private readonly Logger _logger;
public ImportListFactory(IImportListStatusService importListStatusService,
IImportListRepository providerRepository,
IEnumerable<IImportList> providers,
IContainer container,
IEventAggregator eventAggregator,
Logger logger)
: base(providerRepository, providers, container, eventAggregator, logger)
{
_importListStatusService = importListStatusService;
_logger = logger;
}
protected override List<ImportListDefinition> Active()
{
return base.Active().Where(c => c.Enable).ToList();
}
public List<IImportList> AutomaticAddEnabled(bool filterBlockedImportLists = true)
{
var enabledImportLists = GetAvailableProviders().Where(n => ((ImportListDefinition)n.Definition).EnableAutomaticAdd);
if (filterBlockedImportLists)
{
return FilterBlockedImportLists(enabledImportLists).ToList();
}
return enabledImportLists.ToList();
}
private IEnumerable<IImportList> FilterBlockedImportLists(IEnumerable<IImportList> importLists)
{
var blockedImportLists = _importListStatusService.GetBlockedProviders().ToDictionary(v => v.ProviderId, v => v);
foreach (var importList in importLists)
{
ImportListStatus blockedImportListStatus;
if (blockedImportLists.TryGetValue(importList.Definition.Id, out blockedImportListStatus))
{
_logger.Debug("Temporarily ignoring import list {0} till {1} due to recent failures.", importList.Definition.Name, blockedImportListStatus.DisabledTill.Value.ToLocalTime());
continue;
}
yield return importList;
}
}
public override ValidationResult Test(ImportListDefinition definition)
{
var result = base.Test(definition);
if ((result == null || result.IsValid) && definition.Id != 0)
{
_importListStatusService.RecordSuccess(definition.Id);
}
return result;
}
}
}

View file

@ -0,0 +1,25 @@
using System.Collections;
using System.Collections.Generic;
namespace NzbDrone.Core.ImportLists
{
public class ImportListPageableRequest : IEnumerable<ImportListRequest>
{
private readonly IEnumerable<ImportListRequest> _enumerable;
public ImportListPageableRequest(IEnumerable<ImportListRequest> enumerable)
{
_enumerable = enumerable;
}
public IEnumerator<ImportListRequest> GetEnumerator()
{
return _enumerable.GetEnumerator();
}
IEnumerator IEnumerable.GetEnumerator()
{
return _enumerable.GetEnumerator();
}
}
}

View file

@ -0,0 +1,54 @@
using System.Collections.Generic;
using System.Linq;
namespace NzbDrone.Core.ImportLists
{
public class ImportListPageableRequestChain
{
private List<List<ImportListPageableRequest>> _chains;
public ImportListPageableRequestChain()
{
_chains = new List<List<ImportListPageableRequest>>();
_chains.Add(new List<ImportListPageableRequest>());
}
public int Tiers => _chains.Count;
public IEnumerable<ImportListPageableRequest> GetAllTiers()
{
return _chains.SelectMany(v => v);
}
public IEnumerable<ImportListPageableRequest> GetTier(int index)
{
return _chains[index];
}
public void Add(IEnumerable<ImportListRequest> request)
{
if (request == null)
{
return;
}
_chains.Last().Add(new ImportListPageableRequest(request));
}
public void AddTier(IEnumerable<ImportListRequest> request)
{
AddTier();
Add(request);
}
public void AddTier()
{
if (_chains.Last().Count == 0)
{
return;
}
_chains.Add(new List<ImportListPageableRequest>());
}
}
}

View file

@ -0,0 +1,18 @@
using NzbDrone.Core.Datastore;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.ThingiProvider;
namespace NzbDrone.Core.ImportLists
{
public interface IImportListRepository : IProviderRepository<ImportListDefinition>
{
}
public class ImportListRepository : ProviderRepository<ImportListDefinition>, IImportListRepository
{
public ImportListRepository(IMainDatabase database, IEventAggregator eventAggregator)
: base(database, eventAggregator)
{
}
}
}

View file

@ -0,0 +1,21 @@
using NzbDrone.Common.Http;
namespace NzbDrone.Core.ImportLists
{
public class ImportListRequest
{
public HttpRequest HttpRequest { get; private set; }
public ImportListRequest(string url, HttpAccept httpAccept)
{
HttpRequest = new HttpRequest(url, httpAccept);
}
public ImportListRequest(HttpRequest httpRequest)
{
HttpRequest = httpRequest;
}
public HttpUri Url => HttpRequest.Url;
}
}

View file

@ -0,0 +1,24 @@
using NzbDrone.Common.Http;
namespace NzbDrone.Core.ImportLists
{
public class ImportListResponse
{
private readonly ImportListRequest _importListRequest;
private readonly HttpResponse _httpResponse;
public ImportListResponse(ImportListRequest importListRequest, HttpResponse httpResponse)
{
_importListRequest = importListRequest;
_httpResponse = httpResponse;
}
public ImportListRequest Request => _importListRequest;
public HttpRequest HttpRequest => _httpResponse.Request;
public HttpResponse HttpResponse => _httpResponse;
public string Content => _httpResponse.Content;
}
}

View file

@ -0,0 +1,10 @@
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.ThingiProvider.Status;
namespace NzbDrone.Core.ImportLists
{
public class ImportListStatus : ProviderStatusBase
{
public ImportListItemInfo LastSyncListInfo { get; set; }
}
}

View file

@ -0,0 +1,19 @@
using NzbDrone.Core.Datastore;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.ThingiProvider.Status;
namespace NzbDrone.Core.ImportLists
{
public interface IImportListStatusRepository : IProviderStatusRepository<ImportListStatus>
{
}
public class ImportListStatusRepository : ProviderStatusRepository<ImportListStatus>, IImportListStatusRepository
{
public ImportListStatusRepository(IMainDatabase database, IEventAggregator eventAggregator)
: base(database, eventAggregator)
{
}
}
}

View file

@ -0,0 +1,40 @@
using NLog;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.ThingiProvider.Status;
namespace NzbDrone.Core.ImportLists
{
public interface IImportListStatusService : IProviderStatusServiceBase<ImportListStatus>
{
ImportListItemInfo GetLastSyncListInfo(int importListId);
void UpdateListSyncStatus(int importListId, ImportListItemInfo listItemInfo);
}
public class ImportListStatusService : ProviderStatusServiceBase<IImportList, ImportListStatus>, IImportListStatusService
{
public ImportListStatusService(IImportListStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger)
: base(providerStatusRepository, eventAggregator, logger)
{
}
public ImportListItemInfo GetLastSyncListInfo(int importListId)
{
return GetProviderStatus(importListId).LastSyncListInfo;
}
public void UpdateListSyncStatus(int importListId, ImportListItemInfo listItemInfo)
{
lock (_syncRoot)
{
var status = GetProviderStatus(importListId);
status.LastSyncListInfo = listItemInfo;
_providerStatusRepository.Upsert(status);
}
}
}
}

View file

@ -0,0 +1,9 @@
using NzbDrone.Core.Messaging.Commands;
namespace NzbDrone.Core.ImportLists
{
public class ImportListSyncCommand : Command
{
public override bool SendUpdatesToClient => true;
}
}

View file

@ -0,0 +1,16 @@
using System.Collections.Generic;
using NzbDrone.Common.Messaging;
using NzbDrone.Core.Music;
namespace NzbDrone.Core.ImportLists
{
public class ImportListSyncCompleteEvent : IEvent
{
public List<Album> ProcessedDecisions { get; private set; }
public ImportListSyncCompleteEvent(List<Album> processedDecisions)
{
ProcessedDecisions = processedDecisions;
}
}
}

View file

@ -0,0 +1,136 @@
using System.Collections.Generic;
using System.Linq;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Instrumentation.Extensions;
using NzbDrone.Core.Messaging.Commands;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.MetadataSource;
using NzbDrone.Core.Music;
namespace NzbDrone.Core.ImportLists
{
public class ImportListSyncService : IExecute<ImportListSyncCommand>
{
private readonly IImportListStatusService _importListStatusService;
private readonly IImportListFactory _importListFactory;
private readonly IFetchAndParseImportList _listFetcherAndParser;
private readonly ISearchForNewAlbum _albumSearchService;
private readonly ISearchForNewArtist _artistSearchService;
private readonly IArtistService _artistService;
private readonly IAddArtistService _addArtistService;
private readonly IEventAggregator _eventAggregator;
private readonly Logger _logger;
public ImportListSyncService(IImportListStatusService importListStatusService,
IImportListFactory importListFactory,
IFetchAndParseImportList listFetcherAndParser,
ISearchForNewAlbum albumSearchService,
ISearchForNewArtist artistSearchService,
IArtistService artistService,
IAddArtistService addArtistService,
IEventAggregator eventAggregator,
Logger logger)
{
_importListStatusService = importListStatusService;
_importListFactory = importListFactory;
_listFetcherAndParser = listFetcherAndParser;
_albumSearchService = albumSearchService;
_artistSearchService = artistSearchService;
_artistService = artistService;
_addArtistService = addArtistService;
_eventAggregator = eventAggregator;
_logger = logger;
}
private List<Album> Sync()
{
_logger.ProgressInfo("Starting Import List Sync");
var rssReleases = _listFetcherAndParser.Fetch();
var reports = rssReleases.ToList();
var processed = new List<Album>();
var artistsToAdd = new List<Artist>();
_logger.ProgressInfo("Processing {0} list items", reports.Count);
var reportNumber = 1;
foreach (var report in reports)
{
_logger.ProgressTrace("Processing list item {0}/{1}", reportNumber, reports.Count);
reportNumber++;
var importList = _importListFactory.Get(report.ImportListId);
// Map MBid if we only have an album title
if (report.AlbumMusicBrainzId.IsNullOrWhiteSpace() && report.Album.IsNotNullOrWhiteSpace())
{
var mappedAlbum = _albumSearchService.SearchForNewAlbum(report.Album, report.Artist)
.FirstOrDefault();
if (mappedAlbum == null) continue; // Break if we are looking for an album and cant find it. This will avoid us from adding the artist and possibly getting it wrong.
report.AlbumMusicBrainzId = mappedAlbum.ForeignAlbumId;
report.Album = mappedAlbum.Title;
report.Artist = mappedAlbum.Artist?.Name;
report.ArtistMusicBrainzId = mappedAlbum?.Artist?.ForeignArtistId;
}
// Map MBid if we only have a artist name
if (report.ArtistMusicBrainzId.IsNullOrWhiteSpace() && report.Artist.IsNotNullOrWhiteSpace())
{
var mappedArtist = _artistSearchService.SearchForNewArtist(report.Artist)
.FirstOrDefault();
report.ArtistMusicBrainzId = mappedArtist?.ForeignArtistId;
report.Artist = mappedArtist?.Name;
}
// Check to see if artist in DB
var existingArtist = _artistService.FindById(report.ArtistMusicBrainzId);
// Append Artist if not already in DB or already on add list
if (existingArtist == null && artistsToAdd.All(s => s.ForeignArtistId != report.ArtistMusicBrainzId))
{
artistsToAdd.Add(new Artist
{
ForeignArtistId = report.ArtistMusicBrainzId,
Name = report.Artist,
Monitored = importList.ShouldMonitor,
RootFolderPath = importList.RootFolderPath,
ProfileId = importList.ProfileId,
LanguageProfileId = importList.LanguageProfileId,
MetadataProfileId = importList.MetadataProfileId,
AlbumFolder = true,
AddOptions = new AddArtistOptions{SearchForMissingAlbums = true, Monitored = importList.ShouldMonitor }
});
}
// Add Album so we know what to monitor
if (report.AlbumMusicBrainzId.IsNotNullOrWhiteSpace() && artistsToAdd.Any(s=>s.ForeignArtistId == report.ArtistMusicBrainzId) && importList.ShouldMonitor)
{
artistsToAdd.Find(s => s.ForeignArtistId == report.ArtistMusicBrainzId).AddOptions.AlbumsToMonitor.Add(report.AlbumMusicBrainzId);
}
}
_addArtistService.AddArtists(artistsToAdd);
var message = string.Format("Import List Sync Completed. Reports found: {0}, Reports grabbed: {1}", reports.Count, processed.Count);
_logger.ProgressInfo(message);
return processed;
}
public void Execute(ImportListSyncCommand message)
{
var processed = Sync();
_eventAggregator.PublishEvent(new ImportListSyncCompleteEvent(processed));
}
}
}

View file

@ -0,0 +1,63 @@
using System.Collections.Generic;
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Parser;
using NzbDrone.Core.ThingiProvider;
namespace NzbDrone.Core.ImportLists.LidarrLists
{
public class LidarrLists : HttpImportListBase<LidarrListsSettings>
{
public override string Name => "Lidarr Lists";
public override int PageSize => 10;
public LidarrLists(IHttpClient httpClient, IImportListStatusService importListStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
: base(httpClient, importListStatusService, configService, parsingService, logger)
{
}
public override IEnumerable<ProviderDefinition> DefaultDefinitions
{
get
{
yield return GetDefinition("iTunes Top Albums", GetSettings("itunes/album/top"));
yield return GetDefinition("Billboard Top Albums", GetSettings("billboard/album/top"));
yield return GetDefinition("Billboard Top Artists", GetSettings("billboard/artist/top"));
yield return GetDefinition("Last.fm Top Albums", GetSettings("lastfm/album/top"));
yield return GetDefinition("Last.fm Top Artists", GetSettings("lastfm/artist/top"));
}
}
private ImportListDefinition GetDefinition(string name, LidarrListsSettings settings)
{
return new ImportListDefinition
{
EnableAutomaticAdd = false,
Name = name,
Implementation = GetType().Name,
Settings = settings
};
}
private LidarrListsSettings GetSettings(string url)
{
var settings = new LidarrListsSettings { ListId = url };
return settings;
}
public override IImportListRequestGenerator GetRequestGenerator()
{
return new LidarrListsRequestGenerator { Settings = Settings, PageSize = PageSize };
}
public override IParseImportListResponse GetParser()
{
return new LidarrListsParser(Settings);
}
}
}

View file

@ -0,0 +1,13 @@
using System;
namespace NzbDrone.Core.ImportLists.LidarrLists
{
public class LidarrListsAlbum
{
public string ArtistName { get; set; }
public string AlbumTitle { get; set; }
public string ArtistId { get; set; }
public string AlbumId { get; set; }
public DateTime? ReleaseDate { get; set; }
}
}

View file

@ -0,0 +1,73 @@
using Newtonsoft.Json;
using NzbDrone.Core.ImportLists.Exceptions;
using NzbDrone.Core.Parser.Model;
using System.Collections.Generic;
using System.Net;
using NLog;
using NzbDrone.Common.Extensions;
namespace NzbDrone.Core.ImportLists.LidarrLists
{
public class LidarrListsParser : IParseImportListResponse
{
private readonly LidarrListsSettings _settings;
private ImportListResponse _importListResponse;
private readonly Logger _logger;
public LidarrListsParser(LidarrListsSettings settings)
{
_settings = settings;
}
public IList<ImportListItemInfo> ParseResponse(ImportListResponse importListResponse)
{
_importListResponse = importListResponse;
var items = new List<ImportListItemInfo>();
if (!PreProcess(_importListResponse))
{
return items;
}
var jsonResponse = JsonConvert.DeserializeObject<List<LidarrListsAlbum>>(_importListResponse.Content);
// no albums were return
if (jsonResponse == null)
{
return items;
}
foreach (var item in jsonResponse)
{
items.AddIfNotNull(new ImportListItemInfo
{
Artist = item.ArtistName,
Album = item.AlbumTitle,
ArtistMusicBrainzId = item.ArtistId,
AlbumMusicBrainzId = item.AlbumId,
ReleaseDate = item.ReleaseDate.GetValueOrDefault()
});
}
return items;
}
protected virtual bool PreProcess(ImportListResponse importListResponse)
{
if (importListResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
{
throw new ImportListException(importListResponse, "Import List API call resulted in an unexpected StatusCode [{0}]", importListResponse.HttpResponse.StatusCode);
}
if (importListResponse.HttpResponse.Headers.ContentType != null && importListResponse.HttpResponse.Headers.ContentType.Contains("text/json") &&
importListResponse.HttpRequest.Headers.Accept != null && !importListResponse.HttpRequest.Headers.Accept.Contains("text/json"))
{
throw new ImportListException(importListResponse, "Import List responded with html content. Site is likely blocked or unavailable.");
}
return true;
}
}
}

View file

@ -0,0 +1,34 @@
using System.Collections.Generic;
using NzbDrone.Common.Http;
namespace NzbDrone.Core.ImportLists.LidarrLists
{
public class LidarrListsRequestGenerator : IImportListRequestGenerator
{
public LidarrListsSettings Settings { get; set; }
public int MaxPages { get; set; }
public int PageSize { get; set; }
public LidarrListsRequestGenerator()
{
MaxPages = 1;
PageSize = 10;
}
public virtual ImportListPageableRequestChain GetListItems()
{
var pageableRequests = new ImportListPageableRequestChain();
pageableRequests.Add(GetPagedRequests());
return pageableRequests;
}
private IEnumerable<ImportListRequest> GetPagedRequests()
{
yield return new ImportListRequest(string.Format("{0}{1}", Settings.BaseUrl, Settings.ListId), HttpAccept.Json);
}
}
}

View file

@ -0,0 +1,34 @@
using FluentValidation;
using NzbDrone.Core.Annotations;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.ImportLists.LidarrLists
{
public class LidarrListsSettingsValidator : AbstractValidator<LidarrListsSettings>
{
public LidarrListsSettingsValidator()
{
RuleFor(c => c.BaseUrl).ValidRootUrl();
}
}
public class LidarrListsSettings : IImportListSettings
{
private static readonly LidarrListsSettingsValidator Validator = new LidarrListsSettingsValidator();
public LidarrListsSettings()
{
BaseUrl = "https://api.lidarr.audio/api/v0.3/chart/";
}
public string BaseUrl { get; set; }
[FieldDefinition(0, Label = "List Id")]
public string ListId { get; set; }
public NzbDroneValidationResult Validate()
{
return new NzbDroneValidationResult(Validator.Validate(this));
}
}
}

View file

@ -9,6 +9,7 @@ using NzbDrone.Core.Download;
using NzbDrone.Core.HealthCheck;
using NzbDrone.Core.Housekeeping;
using NzbDrone.Core.Indexers;
using NzbDrone.Core.ImportLists;
using NzbDrone.Core.Lifecycle;
using NzbDrone.Core.Messaging.Commands;
using NzbDrone.Core.Messaging.Events;
@ -72,6 +73,12 @@ namespace NzbDrone.Core.Jobs
TypeName = typeof(BackupCommand).FullName
},
new ScheduledTask
{
Interval = 24 * 60, // TODO: Add a setting?
TypeName = typeof(ImportListSyncCommand).FullName
},
new ScheduledTask
{
Interval = GetRssSyncInterval(),

View file

@ -34,14 +34,14 @@ namespace NzbDrone.Core.Music
var albums = _albumService.GetAlbumsByArtist(artist.Id);
var monitoredAlbums = artist.Albums;
var monitoredAlbums = monitoringOptions.AlbumsToMonitor;
if (monitoredAlbums != null)
if (monitoredAlbums.Any())
{
ToggleAlbumsMonitoredState(
albums.Where(s => monitoredAlbums.Any(t => t.ForeignAlbumId == s.ForeignAlbumId)), true);
albums.Where(s => monitoredAlbums.Any(t => t == s.ForeignAlbumId)), true);
ToggleAlbumsMonitoredState(
albums.Where(s => monitoredAlbums.Any(t => t.ForeignAlbumId != s.ForeignAlbumId)), false);
albums.Where(s => monitoredAlbums.Any(t => t != s.ForeignAlbumId)), false);
}
else
{

View file

@ -1,11 +1,18 @@
using NzbDrone.Core.Datastore;
using System.Collections.Generic;
namespace NzbDrone.Core.Music
{
public class MonitoringOptions : IEmbeddedDocument
{
public MonitoringOptions()
{
AlbumsToMonitor = new List<string>();
}
public bool IgnoreAlbumsWithFiles { get; set; }
public bool IgnoreAlbumsWithoutFiles { get; set; }
public List<string> AlbumsToMonitor { get; set; }
public bool Monitored { get; set; }
}
}

View file

@ -183,6 +183,7 @@
<Compile Include="Datastore\Migration\007_change_album_path_to_relative.cs" />
<Compile Include="Datastore\Migration\009_album_releases.cs" />
<Compile Include="Datastore\Migration\010_album_releases_fix.cs" />
<Compile Include="Datastore\Migration\011_import_lists.cs" />
<Compile Include="Datastore\Migration\Framework\MigrationContext.cs" />
<Compile Include="Datastore\Migration\Framework\MigrationController.cs" />
<Compile Include="Datastore\Migration\Framework\MigrationDbFactory.cs" />
@ -464,6 +465,7 @@
<Compile Include="HealthCheck\Checks\AppDataLocationCheck.cs" />
<Compile Include="HealthCheck\Checks\DownloadClientCheck.cs" />
<Compile Include="HealthCheck\Checks\DownloadClientStatusCheck.cs" />
<Compile Include="HealthCheck\Checks\ImportListStatusCheck.cs" />
<Compile Include="HealthCheck\Checks\MonoTlsCheck.cs" />
<Compile Include="HealthCheck\Checks\MountCheck.cs" />
<Compile Include="HealthCheck\Checks\ImportMechanismCheck.cs" />
@ -493,6 +495,7 @@
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedAlbums.cs" />
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedBlacklist.cs" />
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedDownloadClientStatus.cs" />
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedImportListStatus.cs" />
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedTrackFiles.cs" />
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedTracks.cs" />
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedIndexerStatus.cs" />
@ -502,6 +505,7 @@
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedPendingReleases.cs" />
<Compile Include="Housekeeping\Housekeepers\DeleteBadMediaCovers.cs" />
<Compile Include="Housekeeping\Housekeepers\FixFutureDownloadClientStatusTimes.cs" />
<Compile Include="Housekeeping\Housekeepers\FixFutureImportListStatusTimes.cs" />
<Compile Include="Housekeeping\Housekeepers\FixFutureIndexerStatusTimes.cs" />
<Compile Include="Housekeeping\Housekeepers\FixFutureProviderStatusTimes.cs" />
<Compile Include="Housekeeping\Housekeepers\FixFutureRunScheduledTasks.cs" />
@ -515,6 +519,37 @@
<Compile Include="Http\CloudFlare\CloudFlareHttpInterceptor.cs" />
<Compile Include="Http\HttpProxySettingsProvider.cs" />
<Compile Include="Http\TorcacheHttpInterceptor.cs" />
<Compile Include="ImportLists\Exceptions\ImportListException.cs" />
<Compile Include="ImportLists\FetchAndParseImportListService.cs" />
<Compile Include="ImportLists\HeadphonesImport\HeadphonesImport.cs" />
<Compile Include="ImportLists\HeadphonesImport\HeadphonesImportApi.cs" />
<Compile Include="ImportLists\HeadphonesImport\HeadphonesImportParser.cs" />
<Compile Include="ImportLists\HeadphonesImport\HeadphonesImportRequestGenerator.cs" />
<Compile Include="ImportLists\HeadphonesImport\HeadphonesImportSettings.cs" />
<Compile Include="ImportLists\HttpImportListBase.cs" />
<Compile Include="ImportLists\IImportList.cs" />
<Compile Include="ImportLists\IImportListSettings.cs" />
<Compile Include="ImportLists\IImportListRequestGenerator.cs" />
<Compile Include="ImportLists\ImportListDefinition.cs" />
<Compile Include="ImportLists\ImportListFactory.cs" />
<Compile Include="ImportLists\ImportListRepository.cs" />
<Compile Include="ImportLists\ImportListStatus.cs" />
<Compile Include="ImportLists\ImportListStatusRepository.cs" />
<Compile Include="ImportLists\ImportListStatusService.cs" />
<Compile Include="ImportLists\ImportListRequest.cs" />
<Compile Include="ImportLists\ImportListResponse.cs" />
<Compile Include="ImportLists\ImportListSyncCommand.cs" />
<Compile Include="ImportLists\ImportListBase.cs" />
<Compile Include="ImportLists\ImportListPageableRequestChain.cs" />
<Compile Include="ImportLists\ImportListPageableRequest.cs" />
<Compile Include="ImportLists\IProcessImportListResponse.cs" />
<Compile Include="ImportLists\ImportListSyncService.cs" />
<Compile Include="ImportLists\ImportListSyncCompleteEvent.cs" />
<Compile Include="ImportLists\LidarrLists\LidarrLists.cs" />
<Compile Include="ImportLists\LidarrLists\LidarrListsApi.cs" />
<Compile Include="ImportLists\LidarrLists\LidarrListsParser.cs" />
<Compile Include="ImportLists\LidarrLists\LidarrListsRequestGenerator.cs" />
<Compile Include="ImportLists\LidarrLists\LidarrListsSettings.cs" />
<Compile Include="IndexerSearch\AlbumSearchCommand.cs" />
<Compile Include="IndexerSearch\AlbumSearchService.cs" />
<Compile Include="IndexerSearch\ArtistSearchCommand.cs" />
@ -883,6 +918,7 @@
<Compile Include="Parser\IsoLanguages.cs" />
<Compile Include="Parser\LanguageParser.cs" />
<Compile Include="Parser\Model\ArtistTitleInfo.cs" />
<Compile Include="Parser\Model\ImportListItemInfo.cs" />
<Compile Include="Parser\Model\LocalTrack.cs" />
<Compile Include="Parser\Model\ParsedAlbumInfo.cs" />
<Compile Include="Parser\Model\ParsedTrackInfo.cs" />

View file

@ -0,0 +1,21 @@
using System;
using System.Text;
namespace NzbDrone.Core.Parser.Model
{
public class ImportListItemInfo
{
public int ImportListId { get; set; }
public string ImportList { get; set; }
public string Artist { get; set; }
public string ArtistMusicBrainzId { get; set; }
public string Album { get; set; }
public string AlbumMusicBrainzId { get; set; }
public DateTime ReleaseDate { get; set; }
public override string ToString()
{
return string.Format("[{0}] {1} [{2}]", ReleaseDate, Artist, Album);
}
}
}