Upstream Changes to DownloadClients and Indexers

src/NzbDrone.Common/Reflection/ReflectionExtensions.cs

@@ -60,6 +60,11 @@ namespace NzbDrone.Common.Reflection
             return (T)attribute;
         }
 
+        public static T[] GetAttributes<T>(this MemberInfo member) where T : Attribute
+        {
+            return member.GetCustomAttributes(typeof(T), false).OfType<T>().ToArray();
+        }
+
         public static Type FindTypeByName(this Assembly assembly, string name)
         {
             return assembly.GetTypes().SingleOrDefault(c => c.Name.Equals(name, StringComparison.InvariantCultureIgnoreCase));

src/NzbDrone.Core.Test/DecisionEngineTests/BlockedIndexerSpecificationFixture.cs

@@ -0,0 +1,54 @@
+using System;
+using System.Collections.Generic;
+using FluentAssertions;
+using NUnit.Framework;
+using NzbDrone.Core.DecisionEngine;
+using NzbDrone.Core.DecisionEngine.Specifications;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.DecisionEngineTests
+{
+    [TestFixture]
+
+    public class BlockedIndexerSpecificationFixture : CoreTest<BlockedIndexerSpecification>
+    {
+        private RemoteAlbum _remoteAlbum;
+
+        [SetUp]
+        public void Setup()
+        {
+            _remoteAlbum = new RemoteAlbum
+            {
+                Release = new ReleaseInfo { IndexerId = 1 }
+            };
+
+            Mocker.GetMock<IIndexerStatusService>()
+                  .Setup(v => v.GetBlockedProviders())
+                  .Returns(new List<IndexerStatus>());
+        }
+
+        private void WithBlockedIndexer()
+        {
+            Mocker.GetMock<IIndexerStatusService>()
+                  .Setup(v => v.GetBlockedProviders())
+                  .Returns(new List<IndexerStatus> { new IndexerStatus { ProviderId = 1, DisabledTill = DateTime.UtcNow } });
+        }
+
+        [Test]
+        public void should_return_true_if_no_blocked_indexer()
+        {
+            Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeTrue();
+        }
+
+        [Test]
+        public void should_return_false_if_blocked_indexer()
+        {
+            WithBlockedIndexer();
+
+            Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeFalse();
+            Subject.Type.Should().Be(RejectionType.Temporary);
+        }
+    }
+}

src/NzbDrone.Core.Test/DecisionEngineTests/Search/TorrentSeedingSpecificationFixture.cs

@@ -0,0 +1,111 @@
+using FizzWare.NBuilder;
+using FluentAssertions;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Core.Datastore;
+using NzbDrone.Core.DecisionEngine.Specifications.Search;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Indexers.TorrentRss;
+using NzbDrone.Core.IndexerSearch.Definitions;
+using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.Music;
+using NzbDrone.Test.Common;
+
+namespace NzbDrone.Core.Test.DecisionEngineTests.Search
+{
+    [TestFixture]
+    public class TorrentSeedingSpecificationFixture : TestBase<TorrentSeedingSpecification>
+    {
+        private Artist _artist;
+        private RemoteAlbum _remoteAlbum;
+        private IndexerDefinition _indexerDefinition;
+
+        [SetUp]
+        public void Setup()
+        {
+            _artist = Builder<Artist>.CreateNew().With(s => s.Id = 1).Build();
+
+            _remoteAlbum = new RemoteAlbum
+            {
+                Artist = _artist,
+                Release = new TorrentInfo
+                {
+                    IndexerId = 1,
+                    Title = "Artist - Album [FLAC-RlsGrp]",
+                    Seeders = 0
+                }
+            };
+
+            _indexerDefinition = new IndexerDefinition
+            {
+                Settings = new TorrentRssIndexerSettings { MinimumSeeders = 5 }
+            };
+
+            Mocker.GetMock<IIndexerFactory>()
+                  .Setup(v => v.Get(1))
+                  .Returns(_indexerDefinition);
+
+        }
+
+        private void GivenReleaseSeeders(int? seeders)
+        {
+            (_remoteAlbum.Release as TorrentInfo).Seeders = seeders;
+        }
+
+        [Test]
+        public void should_return_true_if_not_torrent()
+        {
+            _remoteAlbum.Release = new ReleaseInfo
+            {
+                IndexerId = 1,
+                Title = "Artist - Album [FLAC-RlsGrp]"
+            };
+
+            Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeTrue();
+        }
+
+        [Test]
+        public void should_return_true_if_indexer_not_specified()
+        {
+            _remoteAlbum.Release.IndexerId = 0;
+
+            Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeTrue();
+        }
+
+        [Test]
+        public void should_return_true_if_indexer_no_longer_exists()
+        {
+            Mocker.GetMock<IIndexerFactory>()
+                  .Setup(v => v.Get(It.IsAny<int>()))
+                  .Callback<int>(i => { throw new ModelNotFoundException(typeof(IndexerDefinition), i); });
+
+            Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeTrue();
+        }
+
+        [Test]
+        public void should_return_true_if_seeds_unknown()
+        {
+            GivenReleaseSeeders(null);
+
+            Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeTrue();
+        }
+
+        [TestCase(5)]
+        [TestCase(6)]
+        public void should_return_true_if_seeds_above_or_equal_to_limit(int seeders)
+        {
+            GivenReleaseSeeders(seeders);
+
+            Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeTrue();
+        }
+
+        [TestCase(0)]
+        [TestCase(4)]
+        public void should_return_false_if_seeds_belove_limit(int seeders)
+        {
+            GivenReleaseSeeders(seeders);
+
+            Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeFalse();
+        }
+    }
+}

src/NzbDrone.Core.Test/Download/DownloadApprovedReportsTests/DownloadApprovedFixture.cs

@@ -6,7 +6,9 @@ using Moq;
 using NUnit.Framework;
 using NzbDrone.Core.DecisionEngine;
 using NzbDrone.Core.Download;
+using NzbDrone.Core.Download.Clients;
 using NzbDrone.Core.Download.Pending;
+using NzbDrone.Core.Indexers;
 using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.Profiles.Qualities;
 using NzbDrone.Core.Qualities;
@@ -34,7 +36,7 @@ namespace NzbDrone.Core.Test.Download.DownloadApprovedReportsTests
                             .Build();
         }
 
-        private RemoteAlbum GetRemoteAlbum(List<Album> albums, QualityModel quality)
+        private RemoteAlbum GetRemoteAlbum(List<Album> albums, QualityModel quality, DownloadProtocol downloadProtocol = DownloadProtocol.Usenet)
         {
             var remoteAlbum = new RemoteAlbum();
             remoteAlbum.ParsedAlbumInfo = new ParsedAlbumInfo();
@@ -44,6 +46,7 @@ namespace NzbDrone.Core.Test.Download.DownloadApprovedReportsTests
             remoteAlbum.Albums.AddRange(albums);
 
             remoteAlbum.Release = new ReleaseInfo();
+            remoteAlbum.Release.DownloadProtocol = downloadProtocol;
             remoteAlbum.Release.PublishDate = DateTime.UtcNow;
 
             remoteAlbum.Artist = Builder<Artist>.CreateNew()
@@ -191,7 +194,6 @@ namespace NzbDrone.Core.Test.Download.DownloadApprovedReportsTests
 
             var decisions = new List<DownloadDecision>();
             decisions.Add(new DownloadDecision(remoteAlbum, new Rejection("Failure!", RejectionType.Temporary)));
-            decisions.Add(new DownloadDecision(remoteAlbum));
 
             Subject.ProcessDecisions(decisions);
             Mocker.GetMock<IDownloadService>().Verify(v => v.DownloadReport(It.IsAny<RemoteAlbum>()), Times.Never());
@@ -208,7 +210,7 @@ namespace NzbDrone.Core.Test.Download.DownloadApprovedReportsTests
             decisions.Add(new DownloadDecision(remoteAlbum, new Rejection("Failure!", RejectionType.Temporary)));
 
             Subject.ProcessDecisions(decisions);
-            Mocker.GetMock<IPendingReleaseService>().Verify(v => v.Add(It.IsAny<DownloadDecision>()), Times.Never());
+            Mocker.GetMock<IPendingReleaseService>().Verify(v => v.Add(It.IsAny<DownloadDecision>(), It.IsAny<PendingReleaseReason>()), Times.Never());
         }
 
         [Test]
@@ -222,7 +224,43 @@ namespace NzbDrone.Core.Test.Download.DownloadApprovedReportsTests
             decisions.Add(new DownloadDecision(remoteAlbum, new Rejection("Failure!", RejectionType.Temporary)));
 
             Subject.ProcessDecisions(decisions);
-            Mocker.GetMock<IPendingReleaseService>().Verify(v => v.Add(It.IsAny<DownloadDecision>()), Times.Exactly(2));
+            Mocker.GetMock<IPendingReleaseService>().Verify(v => v.Add(It.IsAny<DownloadDecision>(), It.IsAny<PendingReleaseReason>()), Times.Exactly(2));
+        }
+
+        [Test]
+        public void should_add_to_failed_if_already_failed_for_that_protocol()
+        {
+            var albums = new List<Album> { GetAlbum(1) };
+            var remoteAlbum = GetRemoteAlbum(albums, new QualityModel(Quality.MP3_320));
+
+            var decisions = new List<DownloadDecision>();
+            decisions.Add(new DownloadDecision(remoteAlbum));
+            decisions.Add(new DownloadDecision(remoteAlbum));
+
+            Mocker.GetMock<IDownloadService>().Setup(s => s.DownloadReport(It.IsAny<RemoteAlbum>()))
+                  .Throws(new DownloadClientUnavailableException("Download client failed"));
+
+            Subject.ProcessDecisions(decisions);
+            Mocker.GetMock<IDownloadService>().Verify(v => v.DownloadReport(It.IsAny<RemoteAlbum>()), Times.Once());
+        }
+
+        [Test]
+        public void should_not_add_to_failed_if_failed_for_a_different_protocol()
+        {
+            var episodes = new List<Album> { GetAlbum(1) };
+            var remoteEpisode = GetRemoteAlbum(episodes, new QualityModel(Quality.MP3_320), DownloadProtocol.Usenet);
+            var remoteEpisode2 = GetRemoteAlbum(episodes, new QualityModel(Quality.MP3_320), DownloadProtocol.Torrent);
+
+            var decisions = new List<DownloadDecision>();
+            decisions.Add(new DownloadDecision(remoteEpisode));
+            decisions.Add(new DownloadDecision(remoteEpisode2));
+
+            Mocker.GetMock<IDownloadService>().Setup(s => s.DownloadReport(It.Is<RemoteAlbum>(r => r.Release.DownloadProtocol == DownloadProtocol.Usenet)))
+                  .Throws(new DownloadClientUnavailableException("Download client failed"));
+
+            Subject.ProcessDecisions(decisions);
+            Mocker.GetMock<IDownloadService>().Verify(v => v.DownloadReport(It.Is<RemoteAlbum>(r => r.Release.DownloadProtocol == DownloadProtocol.Usenet)), Times.Once());
+            Mocker.GetMock<IDownloadService>().Verify(v => v.DownloadReport(It.Is<RemoteAlbum>(r => r.Release.DownloadProtocol == DownloadProtocol.Torrent)), Times.Once());
         }
     }
 }

src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs

@@ -0,0 +1,156 @@
+using System;
+using System.Linq;
+using FluentAssertions;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Core.Download;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.Download
+{
+    public class DownloadClientStatusServiceFixture : CoreTest<DownloadClientStatusService>
+    {
+        private DateTime _epoch;
+
+        [SetUp]
+        public void SetUp()
+        {
+            _epoch = DateTime.UtcNow;
+        }
+
+        private DownloadClientStatus WithStatus(DownloadClientStatus status)
+        {
+            Mocker.GetMock<IDownloadClientStatusRepository>()
+                .Setup(v => v.FindByProviderId(1))
+                .Returns(status);
+
+            Mocker.GetMock<IDownloadClientStatusRepository>()
+                .Setup(v => v.All())
+                .Returns(new[] { status });
+
+            return status;
+        }
+
+        private void VerifyUpdate()
+        {
+            Mocker.GetMock<IDownloadClientStatusRepository>()
+                .Verify(v => v.Upsert(It.IsAny<DownloadClientStatus>()), Times.Once());
+        }
+
+        private void VerifyNoUpdate()
+        {
+            Mocker.GetMock<IDownloadClientStatusRepository>()
+                .Verify(v => v.Upsert(It.IsAny<DownloadClientStatus>()), Times.Never());
+        }
+
+        [Test]
+        public void should_not_consider_blocked_within_5_minutes_since_initial_failure()
+        {
+            WithStatus(new DownloadClientStatus
+            {
+                InitialFailure = _epoch - TimeSpan.FromMinutes(4),
+                MostRecentFailure = _epoch - TimeSpan.FromSeconds(4),
+                EscalationLevel = 3
+            });
+
+            Subject.RecordFailure(1);
+
+            VerifyUpdate();
+
+            var status = Subject.GetBlockedProviders().FirstOrDefault();
+            status.Should().BeNull();
+        }
+
+        [Test]
+        public void should_consider_blocked_after_5_minutes_since_initial_failure()
+        {
+            WithStatus(new DownloadClientStatus
+            {
+                InitialFailure = _epoch - TimeSpan.FromMinutes(6),
+                MostRecentFailure = _epoch - TimeSpan.FromSeconds(120),
+                EscalationLevel = 3
+            });
+
+            Subject.RecordFailure(1);
+
+            VerifyUpdate();
+
+            var status = Subject.GetBlockedProviders().FirstOrDefault();
+            status.Should().NotBeNull();
+        }
+
+        [Test]
+        public void should_not_escalate_further_till_after_5_minutes_since_initial_failure()
+        {
+            var origStatus = WithStatus(new DownloadClientStatus
+            {
+                InitialFailure = _epoch - TimeSpan.FromMinutes(4),
+                MostRecentFailure = _epoch - TimeSpan.FromSeconds(4),
+                EscalationLevel = 3
+            });
+
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+
+            var status = Subject.GetBlockedProviders().FirstOrDefault();
+            status.Should().BeNull();
+
+            origStatus.EscalationLevel.Should().Be(3);
+        }
+
+        [Test]
+        public void should_escalate_further_after_5_minutes_since_initial_failure()
+        {
+            WithStatus(new DownloadClientStatus
+            {
+                InitialFailure = _epoch - TimeSpan.FromMinutes(6),
+                MostRecentFailure = _epoch - TimeSpan.FromSeconds(120),
+                EscalationLevel = 3
+            });
+
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+
+            var status = Subject.GetBlockedProviders().FirstOrDefault();
+            status.Should().NotBeNull();
+
+            status.EscalationLevel.Should().BeGreaterThan(3);
+        }
+
+        [Test]
+        public void should_not_escalate_beyond_3_hours()
+        {
+            WithStatus(new DownloadClientStatus
+            {
+                InitialFailure = _epoch - TimeSpan.FromMinutes(6),
+                MostRecentFailure = _epoch - TimeSpan.FromSeconds(120),
+                EscalationLevel = 3
+            });
+
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+
+            var status = Subject.GetBlockedProviders().FirstOrDefault();
+            status.Should().NotBeNull();
+            status.DisabledTill.Should().HaveValue();
+            status.DisabledTill.Should().NotBeAfter(_epoch + TimeSpan.FromHours(3.1));
+        }
+    }
+}

src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/AddFixture.cs

@@ -102,7 +102,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
         [Test]
         public void should_add()
         {
-            Subject.Add(_temporarilyRejected);
+            Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay);
 
             VerifyInsert();
         }
@@ -112,7 +112,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
         {
             GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate);
 
-            Subject.Add(_temporarilyRejected);
+            Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay);
 
             VerifyNoInsert();
         }
@@ -122,7 +122,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
         {
             GivenHeldRelease(_release.Title + "-RP", _release.Indexer, _release.PublishDate);
 
-            Subject.Add(_temporarilyRejected);
+            Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay);
 
             VerifyInsert();
         }
@@ -132,7 +132,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
         {
             GivenHeldRelease(_release.Title, "AnotherIndexer", _release.PublishDate);
 
-            Subject.Add(_temporarilyRejected);
+            Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay);
 
             VerifyInsert();
         }
@@ -142,7 +142,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
         {
             GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate.AddHours(1));
 
-            Subject.Add(_temporarilyRejected);
+            Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay);
 
             VerifyInsert();
         }

src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/PendingReleaseServiceFixture.cs

@@ -27,7 +27,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
         public void should_not_ignore_pending_items_from_available_indexer()
         {
             Mocker.GetMock<IIndexerStatusService>()
-                .Setup(v => v.GetBlockedIndexers())
+                .Setup(v => v.GetBlockedProviders())
                 .Returns(new List<IndexerStatus>());
 
             GivenPendingRelease();
@@ -43,7 +43,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
         public void should_ignore_pending_items_from_unavailable_indexer()
         {
             Mocker.GetMock<IIndexerStatusService>()
-                .Setup(v => v.GetBlockedIndexers())
+                .Setup(v => v.GetBlockedProviders())
                 .Returns(new List<IndexerStatus> { new IndexerStatus { ProviderId = 1, DisabledTill = DateTime.UtcNow.AddHours(2) } });
 
             GivenPendingRelease();

src/NzbDrone.Core.Test/HealthCheck/Checks/DownloadClientCheckFixture.cs

@@ -1,9 +1,8 @@
-using System;
+using System;
 using System.Collections.Generic;
 using NUnit.Framework;
 using NzbDrone.Core.Download;
 using NzbDrone.Core.HealthCheck.Checks;
-using NzbDrone.Core.Indexers;
 using NzbDrone.Core.Test.Framework;
 using NzbDrone.Test.Common;
 
@@ -26,7 +25,7 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks
         public void should_return_error_when_download_client_throws()
         {
             var downloadClient = Mocker.GetMock<IDownloadClient>();
-            downloadClient.Setup(s => s.Definition).Returns(new IndexerDefinition{Name = "Test"});
+            downloadClient.Setup(s => s.Definition).Returns(new DownloadClientDefinition{Name = "Test"});
 
             downloadClient.Setup(s => s.GetItems())
                           .Throws<Exception>();
@@ -36,8 +35,6 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks
                   .Returns(new IDownloadClient[] { downloadClient.Object });
 
             Subject.Check().ShouldBeError();
-            
-            ExceptionVerification.ExpectedErrors(1);
         }
 
         [Test]

src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs

@@ -22,7 +22,7 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks
                   .Returns(_indexers);
 
             Mocker.GetMock<IIndexerStatusService>()
-                   .Setup(v => v.GetBlockedIndexers())
+                   .Setup(v => v.GetBlockedProviders())
                    .Returns(_blockedIndexers);
         }
 
@@ -57,13 +57,6 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks
         {
             Subject.Check().ShouldBeOk();
         }
-        [Test]
-        public void should_not_return_error_when_indexer_failed_less_than_an_hour()
-        {
-            GivenIndexer(1, 0.1, 0.5);
-
-            Subject.Check().ShouldBeOk();
-        }
 
         [Test]
         public void should_return_warning_if_indexer_unavailable()

src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupDownloadClientUnavailablePendingReleasesFixture.cs

@@ -0,0 +1,60 @@
+using System;
+using FizzWare.NBuilder;
+using FluentAssertions;
+using NUnit.Framework;
+using NzbDrone.Core.Download.Pending;
+using NzbDrone.Core.Housekeeping.Housekeepers;
+using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.Housekeeping.Housekeepers
+{
+    [TestFixture]
+    public class CleanupDownloadClientUnavailablePendingReleasesFixture : DbTest<CleanupDownloadClientUnavailablePendingReleases, PendingRelease>
+    {
+        [Test]
+        public void should_delete_old_DownloadClientUnavailable_pending_items()
+        {
+            var pendingRelease = Builder<PendingRelease>.CreateNew()
+                .With(h => h.Reason = PendingReleaseReason.DownloadClientUnavailable)
+                .With(h => h.Added = DateTime.UtcNow.AddDays(-21))
+                .With(h => h.ParsedAlbumInfo = new ParsedAlbumInfo())
+                .With(h => h.Release = new ReleaseInfo())
+                .BuildNew();
+
+            Db.Insert(pendingRelease);
+            Subject.Clean();
+            AllStoredModels.Should().BeEmpty();
+        }
+
+        [Test]
+        public void should_delete_old_Fallback_pending_items()
+        {
+            var pendingRelease = Builder<PendingRelease>.CreateNew()
+                .With(h => h.Reason = PendingReleaseReason.Fallback)
+                .With(h => h.Added = DateTime.UtcNow.AddDays(-21))
+                .With(h => h.ParsedAlbumInfo = new ParsedAlbumInfo())
+                .With(h => h.Release = new ReleaseInfo())
+                .BuildNew();
+
+            Db.Insert(pendingRelease);
+            Subject.Clean();
+            AllStoredModels.Should().BeEmpty();
+        }
+
+        [Test]
+        public void should_not_delete_old_Delay_pending_items()
+        {
+            var pendingRelease = Builder<PendingRelease>.CreateNew()
+                .With(h => h.Reason = PendingReleaseReason.Delay)
+                .With(h => h.Added = DateTime.UtcNow.AddDays(-21))
+                .With(h => h.ParsedAlbumInfo = new ParsedAlbumInfo())
+                .With(h => h.Release = new ReleaseInfo())
+                .BuildNew();
+
+            Db.Insert(pendingRelease);
+            Subject.Clean();
+            AllStoredModels.Should().HaveCount(1);
+        }
+    }
+}

src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Linq;
 using FluentAssertions;
 using Moq;
@@ -21,7 +21,7 @@ namespace NzbDrone.Core.Test.IndexerTests
         private void WithStatus(IndexerStatus status)
         {
             Mocker.GetMock<IIndexerStatusRepository>()
-                .Setup(v => v.FindByIndexerId(1))
+                .Setup(v => v.FindByProviderId(1))
                 .Returns(status);
 
             Mocker.GetMock<IIndexerStatusRepository>()
@@ -29,25 +29,16 @@ namespace NzbDrone.Core.Test.IndexerTests
                 .Returns(new[] { status });
         }
 
-        private void VerifyUpdate(bool updated = true)
+        private void VerifyUpdate()
         {
             Mocker.GetMock<IIndexerStatusRepository>()
-                .Verify(v => v.Upsert(It.IsAny<IndexerStatus>()), Times.Exactly(updated ? 1 : 0));
+                .Verify(v => v.Upsert(It.IsAny<IndexerStatus>()), Times.Once());
         }
 
-        [Test]
+        private void VerifyNoUpdate()
-        public void should_start_backoff_on_first_failure()
         {
-            WithStatus(new IndexerStatus());
+            Mocker.GetMock<IIndexerStatusRepository>()
-
+                  .Verify(v => v.Upsert(It.IsAny<IndexerStatus>()), Times.Never());
-            Subject.RecordFailure(1);
-
-            VerifyUpdate();
-
-            var status = Subject.GetBlockedIndexers().FirstOrDefault();
-            status.Should().NotBeNull();
-            status.DisabledTill.Should().HaveValue();
-            status.DisabledTill.Value.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(5), 500);
         }
 
         [Test]
@@ -59,7 +50,7 @@ namespace NzbDrone.Core.Test.IndexerTests
 
             VerifyUpdate();
 
-            var status = Subject.GetBlockedIndexers().FirstOrDefault();
+            var status = Subject.GetBlockedProviders().FirstOrDefault();
             status.Should().BeNull();
         }
 
@@ -70,22 +61,7 @@ namespace NzbDrone.Core.Test.IndexerTests
 
             Subject.RecordSuccess(1);
 
-            VerifyUpdate(false);
+            VerifyNoUpdate();
-        }
-
-        [Test]
-        public void should_preserve_escalation_on_intermittent_success()
-        {
-            WithStatus(new IndexerStatus { MostRecentFailure = _epoch - TimeSpan.FromSeconds(4), EscalationLevel = 3 });
-
-            Subject.RecordSuccess(1);
-            Subject.RecordSuccess(1);
-            Subject.RecordFailure(1);
-
-            var status = Subject.GetBlockedIndexers().FirstOrDefault();
-            status.Should().NotBeNull();
-            status.DisabledTill.Should().HaveValue();
-            status.DisabledTill.Value.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(15), 500);
         }
     }
 }

src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj

@@ -117,6 +117,7 @@
     <Compile Include="Datastore\ReflectionStrategyFixture\Benchmarks.cs" />
     <Compile Include="Datastore\SqliteSchemaDumperTests\SqliteSchemaDumperFixture.cs" />
     <Compile Include="DecisionEngineTests\AcceptableSizeSpecificationFixture.cs" />
+    <Compile Include="DecisionEngineTests\BlockedIndexerSpecificationFixture.cs" />
     <Compile Include="DecisionEngineTests\ProtocolSpecificationFixture.cs" />
     <Compile Include="DecisionEngineTests\CutoffSpecificationFixture.cs" />
     <Compile Include="DecisionEngineTests\DownloadDecisionMakerFixture.cs" />
@@ -135,10 +136,12 @@
     <Compile Include="DecisionEngineTests\RssSync\ProperSpecificationFixture.cs" />
     <Compile Include="DecisionEngineTests\Search\ArtistSpecificationFixture.cs" />
     <Compile Include="DecisionEngineTests\RawDiskSpecificationFixture.cs" />
+    <Compile Include="DecisionEngineTests\Search\TorrentSeedingSpecificationFixture.cs" />
     <Compile Include="DecisionEngineTests\UpgradeDiskSpecificationFixture.cs" />
     <Compile Include="DiskSpace\DiskSpaceServiceFixture.cs" />
     <Compile Include="Download\CompletedDownloadServiceFixture.cs" />
     <Compile Include="Download\DownloadApprovedReportsTests\DownloadApprovedFixture.cs" />
+    <Compile Include="Download\DownloadClientStatusServiceFixture.cs" />
     <Compile Include="Download\DownloadClientTests\Blackhole\ScanWatchFolderFixture.cs" />
     <Compile Include="Download\DownloadClientTests\Blackhole\TorrentBlackholeFixture.cs" />
     <Compile Include="Download\DownloadClientTests\Blackhole\UsenetBlackholeFixture.cs" />
@@ -198,6 +201,7 @@
     <Compile Include="Housekeeping\Housekeepers\CleanupAdditionalUsersFixture.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupAdditionalNamingSpecsFixture.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupAbsolutePathMetadataFilesFixture.cs" />
+    <Compile Include="Housekeeping\Housekeepers\CleanupDownloadClientUnavailablePendingReleasesFixture.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupDuplicateMetadataFilesFixture.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupOrphanedAlbumsFixture.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupOrphanedBlacklistFixture.cs" />
@@ -324,11 +328,12 @@
     <Compile Include="Qualities\QualityFixture.cs" />
     <Compile Include="Qualities\QualityModelComparerFixture.cs" />
     <Compile Include="RootFolderTests\RootFolderServiceFixture.cs" />
-    <Compile Include="ThingiProvider\ProviderBaseFixture.cs" />
+    <Compile Include="ThingiProviderTests\ProviderBaseFixture.cs" />
     <Compile Include="ThingiProviderTests\NullConfigFixture.cs" />
     <Compile Include="MusicTests\MoveArtistServiceFixture.cs" />
     <Compile Include="MusicTests\RefreshArtistServiceFixture.cs" />
     <Compile Include="MusicTests\ShouldRefreshArtistFixture.cs" />
+    <Compile Include="ThingiProviderTests\ProviderStatusServiceFixture.cs" />
     <Compile Include="UpdateTests\UpdatePackageProviderFixture.cs" />
     <Compile Include="UpdateTests\UpdateServiceFixture.cs" />
     <Compile Include="XbmcVersionTests.cs" />
@@ -522,6 +527,7 @@
     <Folder Include="InstrumentationTests\" />
     <Folder Include="Providers\" />
     <Folder Include="ProviderTests\UpdateProviderTests\" />
+    <Folder Include="ThingiProvider\" />
   </ItemGroup>
   <ItemGroup>
     <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />

src/NzbDrone.Core.Test/ThingiProviderTests/ProviderBaseFixture.cs

@@ -1,13 +1,12 @@
-using FizzWare.NBuilder;
+using FizzWare.NBuilder;
 using FluentAssertions;
 using NUnit.Framework;
 using NzbDrone.Core.Indexers;
 using NzbDrone.Core.Indexers.Newznab;
 using NzbDrone.Core.Test.Framework;
 
-namespace NzbDrone.Core.Test.ThingiProvider
+namespace NzbDrone.Core.Test.ThingiProviderTests
 {
-
     public class ProviderRepositoryFixture : DbTest<IndexerRepository, IndexerDefinition>
     {
         [Test]

src/NzbDrone.Core.Test/ThingiProviderTests/ProviderStatusServiceFixture.cs

@@ -0,0 +1,126 @@
+using System;
+using System.Linq;
+using FluentAssertions;
+using Moq;
+using NLog;
+using NUnit.Framework;
+using NzbDrone.Core.Messaging.Events;
+using NzbDrone.Core.Test.Framework;
+using NzbDrone.Core.ThingiProvider;
+using NzbDrone.Core.ThingiProvider.Status;
+
+namespace NzbDrone.Core.Test.ThingiProviderTests
+{
+    public class MockProviderStatus : ProviderStatusBase
+    {
+    }
+
+    public interface IMockProvider : IProvider
+    {
+    }
+
+    public interface IMockProviderStatusRepository : IProviderStatusRepository<MockProviderStatus>
+    {
+    }
+
+    public class MockProviderStatusService : ProviderStatusServiceBase<IMockProvider, MockProviderStatus>
+    {
+        public MockProviderStatusService(IMockProviderStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger)
+            : base(providerStatusRepository, eventAggregator, logger)
+        {
+
+        }
+    }
+
+    public class ProviderStatusServiceFixture : CoreTest<MockProviderStatusService>
+    {
+        private DateTime _epoch;
+
+        [SetUp]
+        public void SetUp()
+        {
+            _epoch = DateTime.UtcNow;
+        }
+
+        private void WithStatus(MockProviderStatus status)
+        {
+            Mocker.GetMock<IMockProviderStatusRepository>()
+                .Setup(v => v.FindByProviderId(1))
+                .Returns(status);
+
+            Mocker.GetMock<IMockProviderStatusRepository>()
+                .Setup(v => v.All())
+                .Returns(new[] { status });
+        }
+
+        private void VerifyUpdate()
+        {
+            Mocker.GetMock<IMockProviderStatusRepository>()
+                .Verify(v => v.Upsert(It.IsAny<MockProviderStatus>()), Times.Once());
+        }
+
+        private void VerifyNoUpdate()
+        {
+            Mocker.GetMock<IMockProviderStatusRepository>()
+                .Verify(v => v.Upsert(It.IsAny<MockProviderStatus>()), Times.Never());
+        }
+
+        [Test]
+        public void should_start_backoff_on_first_failure()
+        {
+            WithStatus(new MockProviderStatus());
+
+            Subject.RecordFailure(1);
+
+            VerifyUpdate();
+
+            var status = Subject.GetBlockedProviders().FirstOrDefault();
+            status.Should().NotBeNull();
+            status.DisabledTill.Should().HaveValue();
+            status.DisabledTill.Value.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(5), 500);
+        }
+
+        [Test]
+        public void should_cancel_backoff_on_success()
+        {
+            WithStatus(new MockProviderStatus { EscalationLevel = 2 });
+
+            Subject.RecordSuccess(1);
+
+            VerifyUpdate();
+
+            var status = Subject.GetBlockedProviders().FirstOrDefault();
+            status.Should().BeNull();
+        }
+
+        [Test]
+        public void should_not_store_update_if_already_okay()
+        {
+            WithStatus(new MockProviderStatus { EscalationLevel = 0 });
+
+            Subject.RecordSuccess(1);
+
+            VerifyNoUpdate();
+        }
+
+        [Test]
+        public void should_preserve_escalation_on_intermittent_success()
+        {
+            WithStatus(new MockProviderStatus
+            {
+                InitialFailure = _epoch - TimeSpan.FromSeconds(20),
+                MostRecentFailure = _epoch - TimeSpan.FromSeconds(4),
+                EscalationLevel = 3
+            });
+
+            Subject.RecordSuccess(1);
+            Subject.RecordSuccess(1);
+            Subject.RecordFailure(1);
+
+            var status = Subject.GetBlockedProviders().FirstOrDefault();
+            status.Should().NotBeNull();
+            status.DisabledTill.Should().HaveValue();
+            status.DisabledTill.Value.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(15), 500);
+        }
+    }
+}

src/NzbDrone.Core/Datastore/Migration/002_add_release_to_pending_releases.cs

@@ -0,0 +1,14 @@
+using FluentMigrator;
+using NzbDrone.Core.Datastore.Migration.Framework;
+
+namespace NzbDrone.Core.Datastore.Migration
+{
+    [Migration(2)]
+    public class add_reason_to_pending_releases : NzbDroneMigrationBase
+    {
+        protected override void MainDbUpgrade()
+        {
+            Alter.Table("PendingReleases").AddColumn("Reason").AsInt32().WithDefaultValue(0);
+        }
+    }
+}

src/NzbDrone.Core/Datastore/TableMapping.cs

@@ -123,6 +123,7 @@ namespace NzbDrone.Core.Datastore
                 .Ignore(c => c.Message);
 
             Mapper.Entity<IndexerStatus>().RegisterModel("IndexerStatus");
+            Mapper.Entity<DownloadClientStatus>().RegisterModel("DownloadClientStatus");
         }
 
         private static void RegisterMappers()

src/NzbDrone.Core/DecisionEngine/Specifications/BlockedIndexerSpecification.cs

@@ -0,0 +1,46 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using NLog;
+using NzbDrone.Common.Cache;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.IndexerSearch.Definitions;
+using NzbDrone.Core.Parser.Model;
+
+namespace NzbDrone.Core.DecisionEngine.Specifications
+{
+    public class BlockedIndexerSpecification : IDecisionEngineSpecification
+    {
+        private readonly IIndexerStatusService _indexerStatusService;
+        private readonly Logger _logger;
+
+        private readonly ICachedDictionary<IndexerStatus> _blockedIndexerCache;
+
+        public BlockedIndexerSpecification(IIndexerStatusService indexerStatusService, ICacheManager cacheManager, Logger logger)
+        {
+            _indexerStatusService = indexerStatusService;
+            _logger = logger;
+
+            _blockedIndexerCache = cacheManager.GetCacheDictionary(GetType(), "blocked", FetchBlockedIndexer, TimeSpan.FromSeconds(15));
+        }
+
+        public SpecificationPriority Priority => SpecificationPriority.Database;
+        public RejectionType Type => RejectionType.Temporary;
+
+        public virtual Decision IsSatisfiedBy(RemoteAlbum subject, SearchCriteriaBase searchCriteria)
+        {
+            var status = _blockedIndexerCache.Find(subject.Release.IndexerId.ToString());
+            if (status != null)
+            {
+                return Decision.Reject($"Indexer {subject.Release.Indexer} is blocked till {status.DisabledTill} due to failures, cannot grab release.");
+            }
+
+            return Decision.Accept();
+        }
+
+        private IDictionary<string, IndexerStatus> FetchBlockedIndexer()
+        {
+            return _indexerStatusService.GetBlockedProviders().ToDictionary(v => v.ProviderId.ToString());
+        }
+    }
+}

src/NzbDrone.Core/DecisionEngine/Specifications/TorrentSeedingSpecification.cs

@@ -1,6 +1,5 @@
-using System.Linq;
 using NLog;
-using NzbDrone.Common.Reflection;
+using NzbDrone.Core.Datastore;
 using NzbDrone.Core.Indexers;
 using NzbDrone.Core.IndexerSearch.Definitions;
 using NzbDrone.Core.Parser.Model;
@@ -9,10 +8,10 @@ namespace NzbDrone.Core.DecisionEngine.Specifications.Search
 {
     public class TorrentSeedingSpecification : IDecisionEngineSpecification
     {
-        private readonly IndexerFactory _indexerFactory;
+        private readonly IIndexerFactory _indexerFactory;
         private readonly Logger _logger;
 
-        public TorrentSeedingSpecification(IndexerFactory indexerFactory, Logger logger)
+        public TorrentSeedingSpecification(IIndexerFactory indexerFactory, Logger logger)
         {
             _indexerFactory = indexerFactory;
             _logger = logger;
@@ -26,12 +25,22 @@ namespace NzbDrone.Core.DecisionEngine.Specifications.Search
         {
             var torrentInfo = remoteAlbum.Release as TorrentInfo;
 
-            if (torrentInfo == null)
+            if (torrentInfo == null || torrentInfo.IndexerId == 0)
             {
                 return Decision.Accept();
             }
 
-            var indexer = _indexerFactory.Get(torrentInfo.IndexerId);
+            IndexerDefinition indexer;
+            try
+            {
+                indexer = _indexerFactory.Get(torrentInfo.IndexerId);
+            }
+            catch (ModelNotFoundException)
+            {
+                _logger.Debug("Indexer with id {0} does not exist, skipping seeders check", torrentInfo.IndexerId);
+                return Decision.Accept();
+            }
+
             var torrentIndexerSettings = indexer.Settings as ITorrentIndexerSettings;
 
             if (torrentIndexerSettings != null)

src/NzbDrone.Core/Download/Clients/Deluge/Deluge.cs

@@ -81,8 +81,6 @@ namespace NzbDrone.Core.Download.Clients.Deluge
         {
             IEnumerable<DelugeTorrent> torrents;
 
-            try
-            {
             if (!Settings.TvCategory.IsNullOrWhiteSpace())
             {
                 torrents = _proxy.GetTorrentsByLabel(Settings.TvCategory, Settings);
@@ -91,12 +89,6 @@ namespace NzbDrone.Core.Download.Clients.Deluge
             {
                 torrents = _proxy.GetTorrents(Settings);
             }
-            }
-            catch (DownloadClientException ex)
-            {
-                _logger.Error(ex, "Couldn't get list of torrents");
-                return Enumerable.Empty<DownloadClientItem>();
-            }
 
             var items = new List<DownloadClientItem>();
 

src/NzbDrone.Core/Download/Clients/Deluge/DelugeProxy.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Net;
@@ -231,7 +231,7 @@ namespace NzbDrone.Core.Download.Clients.Deluge
             }
             catch (WebException ex)
             {
-                throw new DownloadClientException("Unable to connect to Deluge, please check your settings", ex);
+                throw new DownloadClientUnavailableException("Unable to connect to Deluge, please check your settings", ex);
             }
         }
 

src/NzbDrone.Core/Download/Clients/DownloadClientException.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using NzbDrone.Common.Exceptions;
 
 namespace NzbDrone.Core.Download.Clients
@@ -8,19 +8,16 @@ namespace NzbDrone.Core.Download.Clients
         public DownloadClientException(string message, params object[] args)
             : base(string.Format(message, args))
         {
-
         }
 
         public DownloadClientException(string message)
             : base(message)
         {
-
         }
 
         public DownloadClientException(string message, Exception innerException, params object[] args)
             : base(string.Format(message, args), innerException)
         {
-
         }
 
         public DownloadClientException(string message, Exception innerException)

src/NzbDrone.Core/Download/Clients/DownloadClientUnavailableException.cs

@@ -0,0 +1,27 @@
+using System;
+
+namespace NzbDrone.Core.Download.Clients
+{
+    public class DownloadClientUnavailableException : DownloadClientException
+    {
+        public DownloadClientUnavailableException(string message, params object[] args)
+            : base(string.Format(message, args))
+        {
+        }
+
+        public DownloadClientUnavailableException(string message)
+            : base(message)
+        {
+        }
+
+        public DownloadClientUnavailableException(string message, Exception innerException, params object[] args)
+            : base(string.Format(message, args), innerException)
+        {
+        }
+
+        public DownloadClientUnavailableException(string message, Exception innerException)
+            : base(message, innerException)
+        {
+        }
+    }
+}

src/NzbDrone.Core/Download/Clients/DownloadStation/Proxies/DiskStationProxyBase.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Net;
@@ -72,7 +72,20 @@ namespace NzbDrone.Core.Download.Clients.DownloadStation.Proxies
                                                          DownloadStationSettings settings) where T : new()
         {
             var request = requestBuilder.Build();
-            var response = _httpClient.Execute(request);
+            HttpResponse response;
+
+            try
+            {
+                response = _httpClient.Execute(request);
+            }
+            catch (HttpException ex)
+            {
+                throw new DownloadClientException("Unable to connect to Diskstation, please check your settings", ex);
+            }
+            catch (WebException ex)
+            {
+                throw new DownloadClientUnavailableException("Unable to connect to Diskstation, please check your settings", ex);
+            }
 
             _logger.Debug("Trying to {0}", operation);
 

src/NzbDrone.Core/Download/Clients/Hadouken/Hadouken.cs

@@ -35,17 +35,7 @@ namespace NzbDrone.Core.Download.Clients.Hadouken
 
         public override IEnumerable<DownloadClientItem> GetItems()
         {
-            HadoukenTorrent[] torrents;
+            var torrents = _proxy.GetTorrents(Settings);
-
-            try
-            {
-                torrents = _proxy.GetTorrents(Settings);
-            }
-            catch (DownloadClientException ex)
-            {
-                _logger.ErrorException(ex.Message, ex);
-                return Enumerable.Empty<DownloadClientItem>();
-            }
 
             var items = new List<DownloadClientItem>();
 

src/NzbDrone.Core/Download/Clients/Hadouken/HadoukenProxy.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Net;
 using NLog;
@@ -77,7 +77,21 @@ namespace NzbDrone.Core.Download.Clients.Hadouken
             requestBuilder.Headers.Add("Accept-Encoding", "gzip,deflate");
 
             var httpRequest = requestBuilder.Build();
-            var response = _httpClient.Execute(httpRequest);
+            HttpResponse response;
+
+            try
+            {
+                response = _httpClient.Execute(httpRequest);
+            }
+            catch (HttpException ex)
+            {
+                throw new DownloadClientException("Unable to connect to Hadouken, please check your settings", ex);
+            }
+            catch (WebException ex)
+            {
+                throw new DownloadClientUnavailableException("Unable to connect to Hadouken, please check your settings", ex);
+            }
+
             var result = Json.Deserialize<JsonRpcResponse<T>>(response.Content);
 
             if (result.Error != null)

src/NzbDrone.Core/Download/Clients/NzbVortex/NzbVortex.cs

@@ -47,17 +47,7 @@ namespace NzbDrone.Core.Download.Clients.NzbVortex
 
         public override IEnumerable<DownloadClientItem> GetItems()
         {
-            List<NzbVortexQueueItem> vortexQueue;
+            var vortexQueue = _proxy.GetQueue(30, Settings);
-
-            try
-            {
-                vortexQueue = _proxy.GetQueue(30, Settings);
-            }
-            catch (DownloadClientException ex)
-            {
-                _logger.Warn("Couldn't get download queue. {0}", ex.Message);
-                return Enumerable.Empty<DownloadClientItem>();
-            }
 
             var queueItems = new List<DownloadClientItem>();
 

src/NzbDrone.Core/Download/Clients/NzbVortex/NzbVortexProxy.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Net;
 using Newtonsoft.Json;
@@ -164,7 +164,7 @@ namespace NzbDrone.Core.Download.Clients.NzbVortex
             }
             catch (WebException ex)
             {
-                throw new DownloadClientException("Unable to connect to NZBVortex, please check your settings", ex);
+                throw new DownloadClientUnavailableException("Unable to connect to NZBVortex, please check your settings", ex);
             }
         }
 

src/NzbDrone.Core/Download/Clients/Nzbget/Nzbget.cs

@@ -51,19 +51,8 @@ namespace NzbDrone.Core.Download.Clients.Nzbget
 
         private IEnumerable<DownloadClientItem> GetQueue()
         {
-            NzbgetGlobalStatus globalStatus;
+            var globalStatus = _proxy.GetGlobalStatus(Settings);
-            List<NzbgetQueueItem> queue;
+            var queue = _proxy.GetQueue(Settings);
-
-            try
-            {
-                globalStatus = _proxy.GetGlobalStatus(Settings);
-                queue = _proxy.GetQueue(Settings);
-            }
-            catch (DownloadClientException ex)
-            {
-                _logger.Error(ex, ex.Message);
-                return Enumerable.Empty<DownloadClientItem>();
-            }
 
             var queueItems = new List<DownloadClientItem>();
 
@@ -119,17 +108,7 @@ namespace NzbDrone.Core.Download.Clients.Nzbget
 
         private IEnumerable<DownloadClientItem> GetHistory()
         {
-            List<NzbgetHistoryItem> history;
+            var history = _proxy.GetHistory(Settings).Take(_configService.DownloadClientHistoryLimit).ToList();
-
-            try
-            {
-                history = _proxy.GetHistory(Settings).Take(_configService.DownloadClientHistoryLimit).ToList();
-            }
-            catch (DownloadClientException ex)
-            {
-                _logger.Error(ex, ex.Message);
-                return Enumerable.Empty<DownloadClientItem>();
-            }
 
             var historyItems = new List<DownloadClientItem>();
 

src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetProxy.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using NLog;
@@ -235,14 +235,14 @@ namespace NzbDrone.Core.Download.Clients.Nzbget
             {
                 if (ex.Response.StatusCode == HttpStatusCode.Unauthorized)
                 {
-                    throw new DownloadClientException("Authentication failed for NzbGet, please check your settings", ex);
+                    throw new DownloadClientAuthenticationException("Authentication failed for NzbGet, please check your settings", ex);
                 }
 
                 throw new DownloadClientException("Unable to connect to NzbGet. " + ex.Message, ex);
             }
             catch (WebException ex)
             {
-                throw new DownloadClientException("Unable to connect to NzbGet. " + ex.Message, ex);
+                throw new DownloadClientUnavailableException("Unable to connect to NzbGet. " + ex.Message, ex);
             }
 
             var result = Json.Deserialize<JsonRpcResponse<T>>(response.Content);

src/NzbDrone.Core/Download/Clients/QBittorrent/QBittorrent.cs

@@ -89,19 +89,8 @@ namespace NzbDrone.Core.Download.Clients.QBittorrent
 
         public override IEnumerable<DownloadClientItem> GetItems()
         {
-            QBittorrentPreferences config;
+            var config = _proxy.GetConfig(Settings);
-            List<QBittorrentTorrent> torrents;
+            var torrents = _proxy.GetTorrents(Settings);
-
-            try
-            {
-                config = _proxy.GetConfig(Settings);
-                torrents = _proxy.GetTorrents(Settings);
-            }
-            catch (DownloadClientException ex)
-            {
-                _logger.Error(ex, ex.Message);
-                return Enumerable.Empty<DownloadClientItem>();
-            }
 
             var queueItems = new List<DownloadClientItem>();
 

src/NzbDrone.Core/Download/Clients/QBittorrent/QBittorrentProxy.cs

@@ -226,7 +226,7 @@ namespace NzbDrone.Core.Download.Clients.QBittorrent
                 }
                 catch (WebException ex)
                 {
-                    throw new DownloadClientException("Failed to connect to qBitTorrent, please check your settings.", ex);
+                    throw new DownloadClientUnavailableException("Failed to connect to qBitTorrent, please check your settings.", ex);
                 }
 
                 if (response.Content != "Ok.") // returns "Fails." on bad login

src/NzbDrone.Core/Download/Clients/Sabnzbd/Sabnzbd.cs

@@ -112,17 +112,7 @@ namespace NzbDrone.Core.Download.Clients.Sabnzbd
 
         private IEnumerable<DownloadClientItem> GetHistory()
         {
-            SabnzbdHistory sabHistory;
+            var sabHistory = _proxy.GetHistory(0, _configService.DownloadClientHistoryLimit, Settings.TvCategory, Settings);
-
-            try
-            {
-                sabHistory = _proxy.GetHistory(0, _configService.DownloadClientHistoryLimit, Settings.TvCategory, Settings);
-            }
-            catch (DownloadClientException ex)
-            {
-                _logger.Warn(ex, "Couldn't get download queue. {0}", ex.Message);
-                return Enumerable.Empty<DownloadClientItem>();
-            }
 
             var historyItems = new List<DownloadClientItem>();
 

src/NzbDrone.Core/Download/Clients/Sabnzbd/SabnzbdProxy.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using Newtonsoft.Json.Linq;
 using NLog;
 using NzbDrone.Common.Extensions;
@@ -183,7 +183,7 @@ namespace NzbDrone.Core.Download.Clients.Sabnzbd
             }
             catch (WebException ex)
             {
-                throw new DownloadClientException("Unable to connect to SABnzbd, please check your settings", ex);
+                throw new DownloadClientUnavailableException("Unable to connect to SABnzbd, please check your settings", ex);
             }
 
             CheckForError(response);

src/NzbDrone.Core/Download/Clients/Transmission/TransmissionBase.cs

@@ -33,17 +33,7 @@ namespace NzbDrone.Core.Download.Clients.Transmission
 
         public override IEnumerable<DownloadClientItem> GetItems()
         {
-            List<TransmissionTorrent> torrents;
+            var torrents = _proxy.GetTorrents(Settings);
-
-            try
-            {
-                torrents = _proxy.GetTorrents(Settings);
-            }
-            catch (DownloadClientException ex)
-            {
-                _logger.Error(ex, ex.Message);
-                return Enumerable.Empty<DownloadClientItem>();
-            }
 
             var items = new List<DownloadClientItem>();
 
@@ -211,18 +201,14 @@ namespace NzbDrone.Core.Download.Clients.Transmission
                     DetailedDescription = string.Format("Please verify your username and password. Also verify if the host running Lidarr isn't blocked from accessing {0} by WhiteList limitations in the {0} configuration.", Name)
                 };
             }
-            catch (WebException ex)
+            catch (DownloadClientUnavailableException ex)
             {
                 _logger.Error(ex, ex.Message);
-                if (ex.Status == WebExceptionStatus.ConnectFailure)
-                {
                 return new NzbDroneValidationFailure("Host", "Unable to connect")
                 {
                     DetailedDescription = "Please verify the hostname and port."
                 };
             }
-                return new NzbDroneValidationFailure(string.Empty, "Unknown exception: " + ex.Message);
-            }
             catch (Exception ex)
             {
                 _logger.Error(ex, "Failed to test");

src/NzbDrone.Core/Download/Clients/Transmission/TransmissionProxy.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Net;
 using System.Collections.Generic;
 using NzbDrone.Common.Extensions;
@@ -237,6 +237,8 @@ namespace NzbDrone.Core.Download.Clients.Transmission
         }
         
         public TransmissionResponse ProcessRequest(string action, object arguments, TransmissionSettings settings)
+        {
+            try
             {
                 var requestBuilder = BuildRequest(settings);
                 requestBuilder.Headers.ContentType = "application/json";
@@ -258,6 +260,7 @@ namespace NzbDrone.Core.Download.Clients.Transmission
                 request.ContentSummary = string.Format("{0}(...)", action);
 
                 var response = _httpClient.Execute(request);
+
                 if (response.StatusCode == HttpStatusCode.Conflict)
                 {
                     AuthenticateClient(requestBuilder, settings, true);
@@ -284,8 +287,16 @@ namespace NzbDrone.Core.Download.Clients.Transmission
                 {
                     throw new TransmissionException(transmissionResponse.Result);
                 }
-
                 return transmissionResponse;
             }
+            catch (HttpException ex)
+            {
+                throw new DownloadClientException("Unable to connect to Transmission, please check your settings", ex);
+            }
+            catch (WebException ex)
+            {
+                throw new DownloadClientUnavailableException("Unable to connect to Transmission, please check your settings", ex);
+            }
+        }
     }
 }

src/NzbDrone.Core/Download/Clients/rTorrent/RTorrent.cs

@@ -80,8 +80,6 @@ namespace NzbDrone.Core.Download.Clients.RTorrent
         public override ProviderMessage Message => new ProviderMessage("Lidarr is unable to remove torrents that have finished seeding when using rTorrent", ProviderMessageType.Warning);
 
         public override IEnumerable<DownloadClientItem> GetItems()
-        {
-            try
         {
             var torrents = _proxy.GetTorrents(Settings);
 
@@ -138,13 +136,6 @@ namespace NzbDrone.Core.Download.Clients.RTorrent
 
             return items;
         }
-            catch (DownloadClientException ex)
-            {
-                _logger.Error(ex, ex.Message);
-                return Enumerable.Empty<DownloadClientItem>();
-            }
-
-        }
 
         public override void RemoveItem(string downloadId, bool deleteData)
         {

src/NzbDrone.Core/Download/Clients/rTorrent/RTorrentProxy.cs

@@ -1,7 +1,9 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Net;
+using System.Net.Sockets;
+using System.Runtime.InteropServices.ComTypes;
 using NLog;
 using NzbDrone.Common.Extensions;
 using CookComputing.XmlRpc;
@@ -54,8 +56,7 @@ namespace NzbDrone.Core.Download.Clients.RTorrent
             _logger.Debug("Executing remote method: system.client_version");
 
             var client = BuildClient(settings);
-
+            var version = ExecuteRequest(() => client.GetVersion());
-            var version = client.GetVersion();
 
             return version;
         }
@@ -65,7 +66,7 @@ namespace NzbDrone.Core.Download.Clients.RTorrent
             _logger.Debug("Executing remote method: d.multicall2");
 
             var client = BuildClient(settings);
-            var ret = client.TorrentMulticall("", "",
+            var ret = ExecuteRequest(() => client.TorrentMulticall("", "",
                     "d.name=", // string
                     "d.hash=", // string
                     "d.base_path=", // string
@@ -76,9 +77,11 @@ namespace NzbDrone.Core.Download.Clients.RTorrent
                     "d.ratio=", // long
                     "d.is_open=", // long
                     "d.is_active=", // long
-                "d.complete="); //long
+                    "d.complete=") //long
+            );
 
             var items = new List<RTorrentTorrent>();
+
             foreach (object[] torrent in ret)
             {
                 var labelDecoded = System.Web.HttpUtility.UrlDecode((string) torrent[3]);
@@ -107,8 +110,8 @@ namespace NzbDrone.Core.Download.Clients.RTorrent
             _logger.Debug("Executing remote method: load.normal");
 
             var client = BuildClient(settings);
+            var response = ExecuteRequest(() => client.LoadStart("", torrentUrl, GetCommands(label, priority, directory)));
 
-            var response = client.LoadStart("", torrentUrl, GetCommands(label, priority, directory));
             if (response != 0)
             {
                 throw new DownloadClientException("Could not add torrent: {0}.", torrentUrl);
@@ -120,8 +123,8 @@ namespace NzbDrone.Core.Download.Clients.RTorrent
             _logger.Debug("Executing remote method: load.raw");
 
             var client = BuildClient(settings);
+            var response = ExecuteRequest(() => client.LoadRawStart("", fileContent, GetCommands(label, priority, directory)));
 
-            var response = client.LoadRawStart("", fileContent, GetCommands(label, priority, directory));
             if (response != 0)
             {
                 throw new DownloadClientException("Could not add torrent: {0}.", fileName);
@@ -133,14 +136,39 @@ namespace NzbDrone.Core.Download.Clients.RTorrent
             _logger.Debug("Executing remote method: d.erase");
 
             var client = BuildClient(settings);
+            var response = ExecuteRequest(() => client.Remove(hash));
 
-            var response = client.Remove(hash);
             if (response != 0)
             {
                 throw new DownloadClientException("Could not remove torrent: {0}.", hash);
             }
         }
 
+        public bool HasHashTorrent(string hash, RTorrentSettings settings)
+        {
+            _logger.Debug("Executing remote method: d.name");
+
+            var client = BuildClient(settings);
+
+            try
+            {
+                var name = ExecuteRequest(() => client.GetName(hash));
+
+                if (name.IsNullOrWhiteSpace())
+                {
+                    return false;
+                }
+
+                var metaTorrent = name == (hash + ".meta");
+
+                return !metaTorrent;
+            }
+            catch (Exception)
+            {
+                return false;
+            }
+        }
+
         private string[] GetCommands(string label, RTorrentPriority priority, string directory)
         {
             var result = new List<string>();
@@ -163,25 +191,6 @@ namespace NzbDrone.Core.Download.Clients.RTorrent
             return result.ToArray();
         }
 
-        public bool HasHashTorrent(string hash, RTorrentSettings settings)
-        {
-            _logger.Debug("Executing remote method: d.name");
-
-            var client = BuildClient(settings);
-
-            try
-            {
-                var name = client.GetName(hash);
-                if (name.IsNullOrWhiteSpace()) return false;
-                bool metaTorrent = name == (hash + ".meta");
-                return !metaTorrent;
-            }
-            catch (Exception)
-            {
-                return false;
-            }
-        }
-
         private IRTorrent BuildClient(RTorrentSettings settings)
         {
             var client = XmlRpcProxyGen.Create<IRTorrent>();
@@ -201,5 +210,21 @@ namespace NzbDrone.Core.Download.Clients.RTorrent
 
             return client;
         }
+
+        private T ExecuteRequest<T>(Func<T> task)
+        {
+            try
+            {
+                return task();
+            }
+            catch (XmlRpcServerException ex)
+            {
+                throw new DownloadClientException("Unable to connect to rTorrent, please check your settings", ex);
+            }
+            catch (WebException ex)
+            {
+                throw new DownloadClientUnavailableException("Unable to connect to rTorrent, please check your settings", ex);
+            }
+        }
     }
 }

src/NzbDrone.Core/Download/Clients/uTorrent/UTorrent.cs

@@ -72,42 +72,7 @@ namespace NzbDrone.Core.Download.Clients.UTorrent
 
         public override IEnumerable<DownloadClientItem> GetItems()
         {
-            List<UTorrentTorrent> torrents;
+            var torrents = GetTorrents();
-
-            try
-            {
-                var cacheKey = string.Format("{0}:{1}:{2}", Settings.Host, Settings.Port, Settings.TvCategory);
-                var cache = _torrentCache.Find(cacheKey);
-
-                var response = _proxy.GetTorrents(cache == null ? null : cache.CacheID, Settings);
-
-                if (cache != null && response.Torrents == null)
-                {
-                    var removedAndUpdated = new HashSet<string>(response.TorrentsChanged.Select(v => v.Hash).Concat(response.TorrentsRemoved));
-
-                    torrents = cache.Torrents
-                        .Where(v => !removedAndUpdated.Contains(v.Hash))
-                        .Concat(response.TorrentsChanged)
-                        .ToList();
-                }
-                else
-                {
-                    torrents = response.Torrents;
-                }
-
-                cache = new UTorrentTorrentCache
-                {
-                    CacheID = response.CacheNumber,
-                    Torrents = torrents
-                };
-
-                _torrentCache.Set(cacheKey, cache, TimeSpan.FromMinutes(15));
-            }
-            catch (DownloadClientException ex)
-            {
-                _logger.Error(ex, ex.Message);
-                return Enumerable.Empty<DownloadClientItem>();
-            }
 
             var queueItems = new List<DownloadClientItem>();
 
@@ -173,6 +138,40 @@ namespace NzbDrone.Core.Download.Clients.UTorrent
             return queueItems;
         }
 
+        private List<UTorrentTorrent> GetTorrents()
+        {
+            List<UTorrentTorrent> torrents;
+
+            var cacheKey = string.Format("{0}:{1}:{2}", Settings.Host, Settings.Port, Settings.TvCategory);
+            var cache = _torrentCache.Find(cacheKey);
+
+            var response = _proxy.GetTorrents(cache == null ? null : cache.CacheID, Settings);
+
+            if (cache != null && response.Torrents == null)
+            {
+                var removedAndUpdated = new HashSet<string>(response.TorrentsChanged.Select(v => v.Hash).Concat(response.TorrentsRemoved));
+
+                torrents = cache.Torrents
+                    .Where(v => !removedAndUpdated.Contains(v.Hash))
+                    .Concat(response.TorrentsChanged)
+                    .ToList();
+            }
+            else
+            {
+                torrents = response.Torrents;
+            }
+
+            cache = new UTorrentTorrentCache
+            {
+                CacheID = response.CacheNumber,
+                Torrents = torrents
+            };
+
+            _torrentCache.Set(cacheKey, cache, TimeSpan.FromMinutes(15));
+
+            return torrents;
+        }
+
         public override void RemoveItem(string downloadId, bool deleteData)
         {
             _proxy.RemoveTorrent(downloadId, deleteData, Settings);

src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentProxy.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Net;
 using NLog;
@@ -244,7 +244,7 @@ namespace NzbDrone.Core.Download.Clients.UTorrent
                 }
                 catch (WebException ex)
                 {
-                    throw new DownloadClientException("Unable to connect to uTorrent, please check your settings", ex);
+                    throw new DownloadClientUnavailableException("Unable to connect to uTorrent, please check your settings", ex);
                 }
 
                 cookies = response.GetCookies();

src/NzbDrone.Core/Download/DownloadClientFactory.cs

@@ -1,5 +1,7 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Linq;
+using FluentValidation.Results;
 using NLog;
 using NzbDrone.Common.Composition;
 using NzbDrone.Core.Messaging.Events;
@@ -9,17 +11,24 @@ namespace NzbDrone.Core.Download
 {
     public interface IDownloadClientFactory : IProviderFactory<IDownloadClient, DownloadClientDefinition>
     {
-
+        List<IDownloadClient> DownloadHandlingEnabled(bool filterBlockedClients = true);
     }
 
     public class DownloadClientFactory : ProviderFactory<IDownloadClient, DownloadClientDefinition>, IDownloadClientFactory
     {
-        private readonly IDownloadClientRepository _providerRepository;
+        private readonly IDownloadClientStatusService _downloadClientStatusService;
+        private readonly Logger _logger;
 
-        public DownloadClientFactory(IDownloadClientRepository providerRepository, IEnumerable<IDownloadClient> providers, IContainer container, IEventAggregator eventAggregator, Logger logger)
+        public DownloadClientFactory(IDownloadClientStatusService downloadClientStatusService,
+                                             IDownloadClientRepository providerRepository,
+                                             IEnumerable<IDownloadClient> providers,
+                                             IContainer container,
+                                             IEventAggregator eventAggregator,
+                                             Logger logger)
             : base(providerRepository, providers, container, eventAggregator, logger)
         {
-            _providerRepository = providerRepository;
+            _downloadClientStatusService = downloadClientStatusService;
+            _logger = logger;
         }
 
         protected override List<DownloadClientDefinition> Active()
@@ -33,5 +42,46 @@ namespace NzbDrone.Core.Download
 
             definition.Protocol = provider.Protocol;
         }
+
+        public List<IDownloadClient> DownloadHandlingEnabled(bool filterBlockedClients = true)
+        {
+            var enabledClients = GetAvailableProviders();
+
+            if (filterBlockedClients)
+            {
+                return FilterBlockedClients(enabledClients).ToList();
+            }
+
+            return enabledClients.ToList();
+        }
+
+        private IEnumerable<IDownloadClient> FilterBlockedClients(IEnumerable<IDownloadClient> clients)
+        {
+            var blockedIndexers = _downloadClientStatusService.GetBlockedProviders().ToDictionary(v => v.ProviderId, v => v);
+
+            foreach (var client in clients)
+            {
+                DownloadClientStatus downloadClientStatus;
+                if (blockedIndexers.TryGetValue(client.Definition.Id, out downloadClientStatus))
+                {
+                    _logger.Debug("Temporarily ignoring download client {0} till {1} due to recent failures.", client.Definition.Name, downloadClientStatus.DisabledTill.Value.ToLocalTime());
+                    continue;
+                }
+
+                yield return client;
+            }
+        }
+
+        public override ValidationResult Test(DownloadClientDefinition definition)
+        {
+            var result = base.Test(definition);
+
+            if ((result == null || result.IsValid) && definition.Id != 0)
+            {
+                _downloadClientStatusService.RecordSuccess(definition.Id);
+            }
+
+            return result;
+        }
     }
 }

src/NzbDrone.Core/Download/DownloadClientProvider.cs

@@ -1,4 +1,4 @@
-using System.Linq;
+using System.Linq;
 using System.Collections.Generic;
 using NzbDrone.Core.Indexers;
 
@@ -27,19 +27,12 @@ namespace NzbDrone.Core.Download
 
         public IEnumerable<IDownloadClient> GetDownloadClients()
         {
-            return _downloadClientFactory.GetAvailableProviders();//.Select(MapDownloadClient);
+            return _downloadClientFactory.GetAvailableProviders();
         }
 
         public IDownloadClient Get(int id)
         {
             return _downloadClientFactory.GetAvailableProviders().Single(d => d.Definition.Id == id);
         }
-
-        public IDownloadClient MapDownloadClient(IDownloadClient downloadClient)
-        {
-            _downloadClientFactory.SetProviderCharacteristics(downloadClient, (DownloadClientDefinition)downloadClient.Definition);
-
-            return downloadClient;
-        }
     }
 }

src/NzbDrone.Core/Download/DownloadClientStatus.cs

@@ -0,0 +1,9 @@
+using NzbDrone.Core.ThingiProvider.Status;
+
+namespace NzbDrone.Core.Download
+{
+    public class DownloadClientStatus : ProviderStatusBase
+    {
+
+    }
+}

src/NzbDrone.Core/Download/DownloadClientStatusRepository.cs

@@ -0,0 +1,19 @@
+using NzbDrone.Core.Datastore;
+using NzbDrone.Core.Messaging.Events;
+using NzbDrone.Core.ThingiProvider.Status;
+
+namespace NzbDrone.Core.Download
+{
+    public interface IDownloadClientStatusRepository : IProviderStatusRepository<DownloadClientStatus>
+    {
+
+    }
+
+    public class DownloadClientStatusRepository : ProviderStatusRepository<DownloadClientStatus>, IDownloadClientStatusRepository
+    {
+        public DownloadClientStatusRepository(IMainDatabase database, IEventAggregator eventAggregator)
+            : base(database, eventAggregator)
+        {
+        }
+    }
+}

src/NzbDrone.Core/Download/DownloadClientStatusService.cs

@@ -0,0 +1,22 @@
+using System;
+using NLog;
+using NzbDrone.Core.Messaging.Events;
+using NzbDrone.Core.ThingiProvider.Status;
+
+namespace NzbDrone.Core.Download
+{
+    public interface IDownloadClientStatusService : IProviderStatusServiceBase<DownloadClientStatus>
+    {
+
+    }
+
+    public class DownloadClientStatusService : ProviderStatusServiceBase<IDownloadClient, DownloadClientStatus>, IDownloadClientStatusService
+    {
+        public DownloadClientStatusService(IDownloadClientStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger)
+            : base(providerStatusRepository, eventAggregator, logger)
+        {
+            MinimumTimeSinceInitialFailure = TimeSpan.FromMinutes(5);
+            MaximumEscalationLevel = 5;
+        }
+    }
+}

src/NzbDrone.Core/Download/DownloadService.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using NLog;
 using NzbDrone.Common.EnsureThat;
 using NzbDrone.Common.Extensions;
@@ -20,18 +20,21 @@ namespace NzbDrone.Core.Download
     public class DownloadService : IDownloadService
     {
         private readonly IProvideDownloadClient _downloadClientProvider;
+        private readonly IDownloadClientStatusService _downloadClientStatusService;
         private readonly IIndexerStatusService _indexerStatusService;
         private readonly IRateLimitService _rateLimitService;
         private readonly IEventAggregator _eventAggregator;
         private readonly Logger _logger;
 
         public DownloadService(IProvideDownloadClient downloadClientProvider,
+                               IDownloadClientStatusService downloadClientStatusService,
                                IIndexerStatusService indexerStatusService,
                                IRateLimitService rateLimitService,
                                IEventAggregator eventAggregator,
                                Logger logger)
         {
             _downloadClientProvider = downloadClientProvider;
+            _downloadClientStatusService = downloadClientStatusService;
             _indexerStatusService = indexerStatusService;
             _rateLimitService = rateLimitService;
             _eventAggregator = eventAggregator;
@@ -63,6 +66,7 @@ namespace NzbDrone.Core.Download
             try
             {
                 downloadClientId = downloadClient.Download(remoteAlbum);
+                _downloadClientStatusService.RecordSuccess(downloadClient.Definition.Id);
                 _indexerStatusService.RecordSuccess(remoteAlbum.Release.IndexerId);
             }
             catch (ReleaseDownloadException ex)

src/NzbDrone.Core/Download/Pending/PendingRelease.cs

@@ -11,6 +11,7 @@ namespace NzbDrone.Core.Download.Pending
         public DateTime Added { get; set; }
         public ParsedAlbumInfo ParsedAlbumInfo { get; set; }
         public ReleaseInfo Release { get; set; }
+        public PendingReleaseReason Reason { get; set; }
 
         //Not persisted
         public RemoteAlbum RemoteAlbum { get; set; }

src/NzbDrone.Core/Download/Pending/PendingReleaseReason.cs

@@ -0,0 +1,9 @@
+namespace NzbDrone.Core.Download.Pending
+{
+    public enum PendingReleaseReason
+    {
+        Delay = 0,
+        DownloadClientUnavailable = 1,
+        Fallback = 2
+    }
+}

src/NzbDrone.Core/Download/Pending/PendingReleaseService.cs

@@ -20,8 +20,7 @@ namespace NzbDrone.Core.Download.Pending
 {
     public interface IPendingReleaseService
     {
-        void Add(DownloadDecision decision);
+        void Add(DownloadDecision decision, PendingReleaseReason reason);
-
         List<ReleaseInfo> GetPending();
         List<RemoteAlbum> GetPendingRemoteAlbums(int artistId);
         List<Queue.Queue> GetPendingQueue();
@@ -67,7 +66,7 @@ namespace NzbDrone.Core.Download.Pending
         }
 
 
-        public void Add(DownloadDecision decision)
+        public void Add(DownloadDecision decision, PendingReleaseReason reason)
         {
             var alreadyPending = GetPendingReleases();
 
@@ -77,14 +76,32 @@ namespace NzbDrone.Core.Download.Pending
                                                              .Intersect(albumIds)
                                                              .Any());
 
-            if (existingReports.Any(MatchingReleasePredicate(decision.RemoteAlbum.Release)))
+            var matchingReports = existingReports.Where(MatchingReleasePredicate(decision.RemoteAlbum.Release)).ToList();
+
+            if (matchingReports.Any())
             {
-                _logger.Debug("This release is already pending, not adding again");
+                var sameReason = true;
-                return;
+
+                foreach (var matchingReport in matchingReports)
+                {
+                    if (matchingReport.Reason != reason)
+                    {
+                        _logger.Debug("This release is already pending with reason {0}, changing to {1}", matchingReport.Reason, reason);
+                        matchingReport.Reason = reason;
+                        _repository.Update(matchingReport);
+                        sameReason = false;
+                    }
                 }
 
-            _logger.Debug("Adding release to pending releases");
+                if (sameReason)
-            Insert(decision);
+                {
+                    _logger.Debug("This release is already pending with reason {0}, not adding again", reason);
+                    return;
+                }
+            }
+
+            _logger.Debug("Adding release to pending releases with reason {0}", reason);
+            Insert(decision, reason);
         }
 
         public List<ReleaseInfo> GetPending()
@@ -101,7 +118,7 @@ namespace NzbDrone.Core.Download.Pending
 
         private List<ReleaseInfo> FilterBlockedIndexers(List<ReleaseInfo> releases)
         {
-            var blockedIndexers = new HashSet<int>(_indexerStatusService.GetBlockedIndexers().Select(v => v.ProviderId));
+            var blockedIndexers = new HashSet<int>(_indexerStatusService.GetBlockedProviders().Select(v => v.ProviderId));
 
             return releases.Where(release => !blockedIndexers.Contains(release.IndexerId)).ToList();
         }
@@ -117,7 +134,7 @@ namespace NzbDrone.Core.Download.Pending
 
             var nextRssSync = new Lazy<DateTime>(() => _taskManager.GetNextExecution(typeof(RssSyncCommand)));
 
-            foreach (var pendingRelease in GetPendingReleases())
+            foreach (var pendingRelease in GetPendingReleases().Where(p => p.Reason != PendingReleaseReason.Fallback))
             {
                 foreach (var album in pendingRelease.RemoteAlbum.Albums)
                 {
@@ -132,6 +149,13 @@ namespace NzbDrone.Core.Download.Pending
                         ect = ect.AddMinutes(_configService.RssSyncInterval);
                     }
 
+                    var timeleft = ect.Subtract(DateTime.UtcNow);
+
+                    if (timeleft.TotalSeconds < 0)
+                    {
+                        timeleft = TimeSpan.Zero;
+                    }
+
                     var queue = new Queue.Queue
                                 {
                                     Id = GetQueueId(pendingRelease, album),
@@ -142,12 +166,13 @@ namespace NzbDrone.Core.Download.Pending
                                     Size = pendingRelease.RemoteAlbum.Release.Size,
                                     Sizeleft = pendingRelease.RemoteAlbum.Release.Size,
                                     RemoteAlbum = pendingRelease.RemoteAlbum,
-                                    Timeleft = ect.Subtract(DateTime.UtcNow),
+                                    Timeleft = timeleft,
                                     EstimatedCompletionTime = ect,
-                                    Status = "Pending",
+                                    Status = pendingRelease.Reason.ToString(),
                                     Protocol = pendingRelease.RemoteAlbum.Release.DownloadProtocol,
                                     Indexer = pendingRelease.RemoteAlbum.Release.Indexer
                     };
+
                     queued.Add(queue);
                 }
             }
@@ -230,7 +255,7 @@ namespace NzbDrone.Core.Download.Pending
             };
         }
 
-        private void Insert(DownloadDecision decision)
+        private void Insert(DownloadDecision decision, PendingReleaseReason reason)
         {
             _repository.Insert(new PendingRelease
             {
@@ -238,7 +263,8 @@ namespace NzbDrone.Core.Download.Pending
                 ParsedAlbumInfo = decision.RemoteAlbum.ParsedAlbumInfo,
                 Release = decision.RemoteAlbum.Release,
                 Title = decision.RemoteAlbum.Release.Title,
-                Added = DateTime.UtcNow
+                Added = DateTime.UtcNow,
+                Reason = reason
             });
 
             _eventAggregator.PublishEvent(new PendingReleasesUpdatedEvent());

src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs

@@ -1,9 +1,12 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Net;
 using NLog;
 using NzbDrone.Core.DecisionEngine;
+using NzbDrone.Core.Download.Clients;
 using NzbDrone.Core.Download.Pending;
+using NzbDrone.Core.Indexers;
 
 namespace NzbDrone.Core.Download
 {
@@ -36,36 +39,33 @@ namespace NzbDrone.Core.Download
             var prioritizedDecisions = _prioritizeDownloadDecision.PrioritizeDecisions(qualifiedReports);
             var grabbed = new List<DownloadDecision>();
             var pending = new List<DownloadDecision>();
+            var failed = new List<DownloadDecision>();
+
+            var usenetFailed = false;
+            var torrentFailed = false;
 
             foreach (var report in prioritizedDecisions)
             {
                 var remoteAlbum = report.RemoteAlbum;
-
+                var downloadProtocol = report.RemoteAlbum.Release.DownloadProtocol;
-                var albumIds = remoteAlbum.Albums.Select(e => e.Id).ToList();
 
                 //Skip if already grabbed
-                if (grabbed.SelectMany(r => r.RemoteAlbum.Albums)
+                if (IsAlbumProcessed(grabbed, report))
-                                .Select(e => e.Id)
-                                .ToList()
-                                .Intersect(albumIds)
-                                .Any())
                 {
                     continue;
                 }
 
                 if (report.TemporarilyRejected)
                 {
-                    _pendingReleaseService.Add(report);
+                    _pendingReleaseService.Add(report, PendingReleaseReason.Delay);
                     pending.Add(report);
                     continue;
                 }
 
-                if (pending.SelectMany(r => r.RemoteAlbum.Albums)
+                if (downloadProtocol == DownloadProtocol.Usenet && usenetFailed ||
-                        .Select(e => e.Id)
+                    downloadProtocol == DownloadProtocol.Torrent && torrentFailed)
-                        .ToList()
-                        .Intersect(albumIds)
-                        .Any())
                 {
+                    failed.Add(report);
                     continue;
                 }
 
@@ -74,13 +74,30 @@ namespace NzbDrone.Core.Download
                     _downloadService.DownloadReport(remoteAlbum);
                     grabbed.Add(report);
                 }
-                catch (Exception e)
+                catch (Exception ex)
                 {
-                    //TODO: support for store & forward
+                    if (ex is DownloadClientUnavailableException || ex is DownloadClientAuthenticationException)
-                    //We'll need to differentiate between a download client error and an indexer error
+                    {
-                    _logger.Warn(e, "Couldn't add report to download queue. " + remoteAlbum);
+                        _logger.Debug("Failed to send release to download client, storing until later");
+                        failed.Add(report);
+
+                        if (downloadProtocol == DownloadProtocol.Usenet)
+                        {
+                            usenetFailed = true;
+                        }
+                        else if (downloadProtocol == DownloadProtocol.Torrent)
+                        {
+                            torrentFailed = true;
                         }
                     }
+                    else
+                    {
+                        _logger.Warn(ex, "Couldn't add report to download queue. " + remoteAlbum);
+                    }
+                }
+            }
+
+            pending.AddRange(ProcessFailedGrabs(grabbed, failed));
 
             return new ProcessedDecisions(grabbed, pending, decisions.Where(d => d.Rejected).ToList());
         }
@@ -90,5 +107,50 @@ namespace NzbDrone.Core.Download
             //Process both approved and temporarily rejected
             return decisions.Where(c => (c.Approved || c.TemporarilyRejected) && c.RemoteAlbum.Albums.Any()).ToList();
         }
+
+        private bool IsAlbumProcessed(List<DownloadDecision> decisions, DownloadDecision report)
+        {
+            var albumIds = report.RemoteAlbum.Albums.Select(e => e.Id).ToList();
+
+            return decisions.SelectMany(r => r.RemoteAlbum.Albums)
+                            .Select(e => e.Id)
+                            .ToList()
+                            .Intersect(albumIds)
+                            .Any();
+        }
+
+        private List<DownloadDecision> ProcessFailedGrabs(List<DownloadDecision> grabbed, List<DownloadDecision> failed)
+        {
+            var pending = new List<DownloadDecision>();
+            var stored = new List<DownloadDecision>();
+
+            foreach (var report in failed)
+            {
+                // If a release was already grabbed with matching albums we should store it as a fallback
+                // and filter it out the next time it is processed incase a higher quality release failed to
+                // add to the download client, but a lower quality release was sent to another client
+                // If the release wasn't grabbed already, but was already stored, store it as a fallback,
+                // otherwise store it as DownloadClientUnavailable.
+
+                if (IsAlbumProcessed(grabbed, report))
+                {
+                    _pendingReleaseService.Add(report, PendingReleaseReason.Fallback);
+                    pending.Add(report);
+                }
+                else if (IsAlbumProcessed(stored, report))
+                {
+                    _pendingReleaseService.Add(report, PendingReleaseReason.Fallback);
+                    pending.Add(report);
+                }
+                else
+                {
+                    _pendingReleaseService.Add(report, PendingReleaseReason.DownloadClientUnavailable);
+                    pending.Add(report);
+                    stored.Add(report);
+                }
+            }
+
+            return pending;
+        }
     }
 }

src/NzbDrone.Core/Download/TrackedDownloads/DownloadMonitoringService.cs

@@ -17,7 +17,8 @@ namespace NzbDrone.Core.Download.TrackedDownloads
                                              IHandle<TrackedDownloadsRemovedEvent>
 
     {
-        private readonly IProvideDownloadClient _downloadClientProvider;
+        private readonly IDownloadClientStatusService _downloadClientStatusService;
+        private readonly IDownloadClientFactory _downloadClientFactory;
         private readonly IEventAggregator _eventAggregator;
         private readonly IManageCommandQueue _manageCommandQueue;
         private readonly IConfigService _configService;
@@ -27,7 +28,8 @@ namespace NzbDrone.Core.Download.TrackedDownloads
         private readonly Logger _logger;
         private readonly Debouncer _refreshDebounce;
 
-        public DownloadMonitoringService(IProvideDownloadClient downloadClientProvider,
+        public DownloadMonitoringService(IDownloadClientStatusService downloadClientStatusService,
+                                         IDownloadClientFactory downloadClientFactory,
                                          IEventAggregator eventAggregator,
                                          IManageCommandQueue manageCommandQueue,
                                          IConfigService configService,
@@ -36,7 +38,8 @@ namespace NzbDrone.Core.Download.TrackedDownloads
                                          ITrackedDownloadService trackedDownloadService,
                                          Logger logger)
         {
-            _downloadClientProvider = downloadClientProvider;
+            _downloadClientStatusService = downloadClientStatusService;
+            _downloadClientFactory = downloadClientFactory;
             _eventAggregator = eventAggregator;
             _manageCommandQueue = manageCommandQueue;
             _configService = configService;
@@ -58,7 +61,7 @@ namespace NzbDrone.Core.Download.TrackedDownloads
             _refreshDebounce.Pause();
             try
             {
-                var downloadClients = _downloadClientProvider.GetDownloadClients();
+                var downloadClients = _downloadClientFactory.DownloadHandlingEnabled();
 
                 var trackedDownloads = new List<TrackedDownload>();
 
@@ -86,9 +89,12 @@ namespace NzbDrone.Core.Download.TrackedDownloads
             try
             {
                 downloadClientHistory = downloadClient.GetItems().ToList();
+
+                _downloadClientStatusService.RecordSuccess(downloadClient.Definition.Id);
             }
             catch (Exception ex)
             {
+                _downloadClientStatusService.RecordFailure(downloadClient.Definition.Id);
                 _logger.Warn(ex, "Unable to retrieve queue and history items from " + downloadClient.Definition.Name);
             }
 

src/NzbDrone.Core/HealthCheck/CheckOnAttribute.cs

@@ -0,0 +1,16 @@
+using System;
+using NzbDrone.Common.Messaging;
+
+namespace NzbDrone.Core.HealthCheck
+{
+    [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)]
+    public class CheckOnAttribute : Attribute
+    {
+        public Type EventType { get; set; }
+
+        public CheckOnAttribute(Type eventType)
+        {
+            EventType = eventType;
+        }
+    }
+}

src/NzbDrone.Core/HealthCheck/Checks/AppDataLocationCheck.cs

@@ -1,5 +1,6 @@
-using NzbDrone.Common.EnvironmentInfo;
+using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Common.Extensions;
+using NzbDrone.Core.Configuration.Events;
 
 namespace NzbDrone.Core.HealthCheck.Checks
 {
@@ -23,6 +24,5 @@ namespace NzbDrone.Core.HealthCheck.Checks
             return new HealthCheck(GetType());
         }
         
-        public override bool CheckOnConfigChange => false;
     }
 }

src/NzbDrone.Core/HealthCheck/Checks/DownloadClientCheck.cs

@@ -1,10 +1,13 @@
-using System;
+using System;
 using System.Linq;
 using NLog;
 using NzbDrone.Core.Download;
+using NzbDrone.Core.ThingiProvider.Events;
 
 namespace NzbDrone.Core.HealthCheck.Checks
 {
+    [CheckOn(typeof(ProviderUpdatedEvent<IDownloadClient>))]
+    [CheckOn(typeof(ProviderDeletedEvent<IDownloadClient>))]
     public class DownloadClientCheck : HealthCheckBase
     {
         private readonly IProvideDownloadClient _downloadClientProvider;
@@ -33,11 +36,10 @@ namespace NzbDrone.Core.HealthCheck.Checks
                 }
                 catch (Exception ex)
                 {
-
+                    _logger.Debug(ex, "Unable to communicate with {0}", downloadClient.Definition.Name);
-                    _logger.Error(ex, "Unable to communicate with {0}", downloadClient.Definition.Name);
 
                     var message = $"Unable to communicate with {downloadClient.Definition.Name}.";
-                    return new HealthCheck(GetType(), HealthCheckResult.Error, $"{message} {ex.Message}");
+                    return new HealthCheck(GetType(), HealthCheckResult.Error, $"{message} {ex.Message}", "#unable-to-communicate-with-download-client");
                 }
             }
 

src/NzbDrone.Core/HealthCheck/Checks/DownloadClientStatusCheck.cs

@@ -0,0 +1,45 @@
+using System;
+using System.Linq;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Core.Download;
+using NzbDrone.Core.ThingiProvider.Events;
+
+namespace NzbDrone.Core.HealthCheck.Checks
+{
+    [CheckOn(typeof(ProviderUpdatedEvent<IDownloadClient>))]
+    [CheckOn(typeof(ProviderDeletedEvent<IDownloadClient>))]
+    [CheckOn(typeof(ProviderStatusChangedEvent<IDownloadClient>))]
+    public class DownloadClientStatusCheck : HealthCheckBase
+    {
+        private readonly IDownloadClientFactory _providerFactory;
+        private readonly IDownloadClientStatusService _providerStatusService;
+
+        public DownloadClientStatusCheck(IDownloadClientFactory providerFactory, IDownloadClientStatusService providerStatusService)
+        {
+            _providerFactory = providerFactory;
+            _providerStatusService = providerStatusService;
+        }
+
+        public override HealthCheck Check()
+        {
+            var enabledProviders = _providerFactory.GetAvailableProviders();
+            var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(),
+                    i => i.Definition.Id,
+                    s => s.ProviderId,
+                    (i, s) => new { Provider = i, Status = s })
+                .ToList();
+
+            if (backOffProviders.Empty())
+            {
+                return new HealthCheck(GetType());
+            }
+
+            if (backOffProviders.Count == enabledProviders.Count)
+            {
+                return new HealthCheck(GetType(), HealthCheckResult.Error, "All download clients are unavailable due to failures", "#download-clients-are-unavailable-due-to-failures");
+            }
+
+            return new HealthCheck(GetType(), HealthCheckResult.Warning, string.Format("Download clients unavailable due to failures: {0}", string.Join(", ", backOffProviders.Select(v => v.Provider.Definition.Name))), "#download-clients-are-unavailable-due-to-failures");
+        }
+    }
+}

src/NzbDrone.Core/HealthCheck/Checks/ImportMechanismCheck.cs

@@ -2,13 +2,18 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using NzbDrone.Core.Configuration;
+using NzbDrone.Core.Configuration.Events;
 using NzbDrone.Core.Download;
 using NzbDrone.Core.Download.Clients;
 using NzbDrone.Core.Download.Clients.Nzbget;
 using NzbDrone.Core.Download.Clients.Sabnzbd;
+using NzbDrone.Core.ThingiProvider.Events;
 
 namespace NzbDrone.Core.HealthCheck.Checks
 {
+    [CheckOn(typeof(ProviderUpdatedEvent<IDownloadClient>))]
+    [CheckOn(typeof(ProviderDeletedEvent<IDownloadClient>))]
+    [CheckOn(typeof(ConfigSavedEvent))]
     public class ImportMechanismCheck : HealthCheckBase
     {
         private readonly IConfigService _configService;
@@ -33,7 +38,7 @@ namespace NzbDrone.Core.HealthCheck.Checks
                     Status = v.GetStatus()
                 }).ToList();
             }
-            catch (DownloadClientException)
+            catch (Exception)
             {
                 // One or more download clients failed, assume the health is okay and verify later
                 return new HealthCheck(GetType());

src/NzbDrone.Core/HealthCheck/Checks/IndexerRssCheck.cs

@@ -1,9 +1,13 @@
-using System.Linq;
+using System.Linq;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Indexers;
+using NzbDrone.Core.ThingiProvider.Events;
 
 namespace NzbDrone.Core.HealthCheck.Checks
 {
+    [CheckOn(typeof(ProviderUpdatedEvent<IIndexer>))]
+    [CheckOn(typeof(ProviderDeletedEvent<IIndexer>))]
+    [CheckOn(typeof(ProviderStatusChangedEvent<IIndexer>))]
     public class IndexerRssCheck : HealthCheckBase
     {
         private readonly IIndexerFactory _indexerFactory;

src/NzbDrone.Core/HealthCheck/Checks/IndexerSearchCheck.cs

@@ -1,9 +1,13 @@
-using System.Linq;
+using System.Linq;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Indexers;
+using NzbDrone.Core.ThingiProvider.Events;
 
 namespace NzbDrone.Core.HealthCheck.Checks
 {
+    [CheckOn(typeof(ProviderUpdatedEvent<IIndexer>))]
+    [CheckOn(typeof(ProviderDeletedEvent<IIndexer>))]
+    [CheckOn(typeof(ProviderStatusChangedEvent<IIndexer>))]
     public class IndexerSearchCheck : HealthCheckBase
     {
         private readonly IIndexerFactory _indexerFactory;

src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs

@@ -1,42 +1,45 @@
-using System;
+using System;
 using System.Linq;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Indexers;
+using NzbDrone.Core.ThingiProvider.Events;
 
 namespace NzbDrone.Core.HealthCheck.Checks
 {
+    [CheckOn(typeof(ProviderUpdatedEvent<IIndexer>))]
+    [CheckOn(typeof(ProviderDeletedEvent<IIndexer>))]
+    [CheckOn(typeof(ProviderStatusChangedEvent<IIndexer>))]
     public class IndexerStatusCheck : HealthCheckBase
     {
-        private readonly IIndexerFactory _indexerFactory;
+        private readonly IIndexerFactory _providerFactory;
-        private readonly IIndexerStatusService _indexerStatusService;
+        private readonly IIndexerStatusService _providerStatusService;
 
-        public IndexerStatusCheck(IIndexerFactory indexerFactory, IIndexerStatusService indexerStatusService)
+        public IndexerStatusCheck(IIndexerFactory providerFactory, IIndexerStatusService providerStatusService)
         {
-            _indexerFactory = indexerFactory;
+            _providerFactory = providerFactory;
-            _indexerStatusService = indexerStatusService;
+            _providerStatusService = providerStatusService;
         }
 
         public override HealthCheck Check()
         {
-            var enabledIndexers = _indexerFactory.GetAvailableProviders();
+            var enabledProviders = _providerFactory.GetAvailableProviders();
-            var backOffIndexers = enabledIndexers.Join(_indexerStatusService.GetBlockedIndexers(),
+            var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(),
                     i => i.Definition.Id,
                     s => s.ProviderId,
                     (i, s) => new { Indexer = i, Status = s })
-                .Where(v => (v.Status.MostRecentFailure - v.Status.InitialFailure) > TimeSpan.FromHours(1))
                 .ToList();
 
-            if (backOffIndexers.Empty())
+            if (backOffProviders.Empty())
             {
                 return new HealthCheck(GetType());
             }
 
-            if (backOffIndexers.Count == enabledIndexers.Count)
+            if (backOffProviders.Count == enabledProviders.Count)
             {
                 return new HealthCheck(GetType(), HealthCheckResult.Error, "All indexers are unavailable due to failures", "#indexers-are-unavailable-due-to-failures");
             }
 
-            return new HealthCheck(GetType(), HealthCheckResult.Warning, string.Format("Indexers unavailable due to failures: {0}", string.Join(", ", backOffIndexers.Select(v => v.Indexer.Definition.Name))), "#indexers-are-unavailable-due-to-failures");
+            return new HealthCheck(GetType(), HealthCheckResult.Warning, string.Format("Indexers unavailable due to failures: {0}", string.Join(", ", backOffProviders.Select(v => v.Indexer.Definition.Name))), "#indexers-are-unavailable-due-to-failures");
         }
     }
 }

src/NzbDrone.Core/HealthCheck/Checks/MediaInfoDllCheck.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Runtime.CompilerServices;
 using NzbDrone.Core.MediaFiles.MediaInfo;
 
@@ -20,7 +20,5 @@ namespace NzbDrone.Core.HealthCheck.Checks
 
             return new HealthCheck(GetType());
         }
-
-        public override bool CheckOnConfigChange => false;
     }
 }

src/NzbDrone.Core/HealthCheck/Checks/MonoVersionCheck.cs

@@ -41,8 +41,6 @@ namespace NzbDrone.Core.HealthCheck.Checks
             return new HealthCheck(GetType(), HealthCheckResult.Warning, "You are running an old and unsupported version of Mono. Please upgrade Mono for improved stability.");
         }
 
-        public override bool CheckOnConfigChange => false;
-
         public override bool CheckOnSchedule => false;
 
     }

src/NzbDrone.Core/HealthCheck/Checks/ProxyCheck.cs

@@ -1,13 +1,15 @@
-using NLog;
+using NLog;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
 using System;
 using System.Linq;
 using System.Net;
 using NzbDrone.Common.Cloud;
+using NzbDrone.Core.Configuration.Events;
 
 namespace NzbDrone.Core.HealthCheck.Checks
 {
+    [CheckOn(typeof(ConfigSavedEvent))]
     public class ProxyCheck : HealthCheckBase
     {
         private readonly Logger _logger;

src/NzbDrone.Core/HealthCheck/Checks/RootFolderCheck.cs

@@ -1,9 +1,12 @@
 using System.Linq;
 using NzbDrone.Common.Disk;
 using NzbDrone.Core.Music;
+using NzbDrone.Core.Music.Events;
 
 namespace NzbDrone.Core.HealthCheck.Checks
 {
+    [CheckOn(typeof(ArtistDeletedEvent))]
+    [CheckOn(typeof(ArtistMovedEvent))]
     public class RootFolderCheck : HealthCheckBase
     {
         private readonly IArtistService _artistService;
@@ -36,7 +39,5 @@ namespace NzbDrone.Core.HealthCheck.Checks
 
             return new HealthCheck(GetType());
         }
-
-        public override bool CheckOnConfigChange => false;
     }
 }

src/NzbDrone.Core/HealthCheck/Checks/UpdateCheck.cs

@@ -1,13 +1,15 @@
-using System;
+using System;
 using System.IO;
 using NzbDrone.Common.Disk;
 using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Configuration;
+using NzbDrone.Core.Configuration.Events;
 using NzbDrone.Core.Update;
 
 namespace NzbDrone.Core.HealthCheck.Checks
 {
+    [CheckOn(typeof(ConfigFileSavedEvent))]
     public class UpdateCheck : HealthCheckBase
     {
         private readonly IDiskProvider _diskProvider;
@@ -66,7 +68,5 @@ namespace NzbDrone.Core.HealthCheck.Checks
 
             return new HealthCheck(GetType());
         }
-
-        public override bool CheckOnConfigChange => false;
     }
 }

src/NzbDrone.Core/HealthCheck/HealthCheckBase.cs

@@ -1,4 +1,4 @@
-namespace NzbDrone.Core.HealthCheck
+namespace NzbDrone.Core.HealthCheck
 {
     public abstract class HealthCheckBase : IProvideHealthCheck
     {
@@ -6,8 +6,6 @@
 
         public virtual bool CheckOnStartup => true;
 
-        public virtual bool CheckOnConfigChange => true;
-
         public virtual bool CheckOnSchedule => true;
     }
 }

src/NzbDrone.Core/HealthCheck/HealthCheckService.cs

@@ -1,8 +1,11 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using NLog;
 using NzbDrone.Common.Cache;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Messaging;
+using NzbDrone.Common.Reflection;
 using NzbDrone.Core.Configuration.Events;
 using NzbDrone.Core.Download;
 using NzbDrone.Core.Indexers;
@@ -21,13 +24,12 @@ namespace NzbDrone.Core.HealthCheck
     public class HealthCheckService : IHealthCheckService,
                                       IExecute<CheckHealthCommand>,
                                       IHandleAsync<ApplicationStartedEvent>,
-                                      IHandleAsync<ConfigSavedEvent>,
+                                      IHandleAsync<IEvent>
-                                      IHandleAsync<ProviderUpdatedEvent<IIndexer>>,
-                                      IHandleAsync<ProviderDeletedEvent<IIndexer>>,
-                                      IHandleAsync<ProviderUpdatedEvent<IDownloadClient>>,
-                                      IHandleAsync<ProviderDeletedEvent<IDownloadClient>>
     {
-        private readonly IEnumerable<IProvideHealthCheck> _healthChecks;
+        private readonly IProvideHealthCheck[] _healthChecks;
+        private readonly IProvideHealthCheck[] _startupHealthChecks;
+        private readonly IProvideHealthCheck[] _scheduledHealthChecks;
+        private readonly Dictionary<Type, IProvideHealthCheck[]> _eventDrivenHealthChecks;
         private readonly IEventAggregator _eventAggregator;
         private readonly ICacheManager _cacheManager;
         private readonly Logger _logger;
@@ -39,12 +41,16 @@ namespace NzbDrone.Core.HealthCheck
                                   ICacheManager cacheManager,
                                   Logger logger)
         {
-            _healthChecks = healthChecks;
+            _healthChecks = healthChecks.ToArray();
             _eventAggregator = eventAggregator;
             _cacheManager = cacheManager;
             _logger = logger;
 
             _healthCheckResults = _cacheManager.GetCache<HealthCheck>(GetType());
+
+            _startupHealthChecks = _healthChecks.Where(v => v.CheckOnStartup).ToArray();
+            _scheduledHealthChecks = _healthChecks.Where(v => v.CheckOnSchedule).ToArray();
+            _eventDrivenHealthChecks = GetEventDrivenHealthChecks();
         }
 
         public List<HealthCheck> Results()
@@ -52,10 +58,17 @@ namespace NzbDrone.Core.HealthCheck
             return _healthCheckResults.Values.ToList();
         }
 
-        private void PerformHealthCheck(Func<IProvideHealthCheck, bool> predicate)
+        private Dictionary<Type, IProvideHealthCheck[]> GetEventDrivenHealthChecks()
         {
-            var results = _healthChecks.Where(predicate)
+            return _healthChecks
-                                       .Select(c => c.Check())
+                .SelectMany(h => h.GetType().GetAttributes<CheckOnAttribute>().Select(a => Tuple.Create(a.EventType, h)))
+                .GroupBy(t => t.Item1, t => t.Item2)
+                .ToDictionary(g => g.Key, g => g.ToArray());
+        }
+
+        private void PerformHealthCheck(IProvideHealthCheck[] healthChecks)
+        {
+            var results = healthChecks.Select(c => c.Check())
                                       .ToList();
 
             foreach (var result in results)
@@ -76,37 +89,37 @@ namespace NzbDrone.Core.HealthCheck
 
         public void Execute(CheckHealthCommand message)
         {
-            PerformHealthCheck(c => message.Trigger == CommandTrigger.Manual || c.CheckOnSchedule);
+            if (message.Trigger == CommandTrigger.Manual)
+            {
+                PerformHealthCheck(_healthChecks);
+            }
+            else
+            {
+                PerformHealthCheck(_scheduledHealthChecks);
+            }
         }
 
         public void HandleAsync(ApplicationStartedEvent message)
         {
-            PerformHealthCheck(c => c.CheckOnStartup);
+            PerformHealthCheck(_startupHealthChecks);
         }
 
-        public void HandleAsync(ConfigSavedEvent message)
+        public void HandleAsync(IEvent message)
         {
-            PerformHealthCheck(c => c.CheckOnConfigChange);
+            if (message is HealthCheckCompleteEvent)
+            {
+                return;
             }
 
-        public void HandleAsync(ProviderUpdatedEvent<IIndexer> message)
+            IProvideHealthCheck[] checks;
+            if (!_eventDrivenHealthChecks.TryGetValue(message.GetType(), out checks))
             {
-            PerformHealthCheck(c => c.CheckOnConfigChange);
+                return;
             }
 
-        public void HandleAsync(ProviderDeletedEvent<IIndexer> message)
+            // TODO: Add debounce
-        {
-            PerformHealthCheck(c => c.CheckOnConfigChange);
-        }
 
-        public void HandleAsync(ProviderUpdatedEvent<IDownloadClient> message)
+            PerformHealthCheck(checks);
-        {
-            PerformHealthCheck(c => c.CheckOnConfigChange);
-        }
-
-        public void HandleAsync(ProviderDeletedEvent<IDownloadClient> message)
-        {
-            PerformHealthCheck(c => c.CheckOnConfigChange);
         }
     }
 }

src/NzbDrone.Core/HealthCheck/IProvideHealthCheck.cs

@@ -1,10 +1,9 @@
-namespace NzbDrone.Core.HealthCheck
+namespace NzbDrone.Core.HealthCheck
 {
     public interface IProvideHealthCheck
     {
         HealthCheck Check();
         bool CheckOnStartup { get; }
-        bool CheckOnConfigChange { get; }
         bool CheckOnSchedule { get; }
     }
 }

src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupDownloadClientUnavailablePendingReleases.cs

@@ -0,0 +1,32 @@
+using System;
+using NzbDrone.Core.Datastore;
+using NzbDrone.Core.Download.Pending;
+
+namespace NzbDrone.Core.Housekeeping.Housekeepers
+{
+    public class CleanupDownloadClientUnavailablePendingReleases : IHousekeepingTask
+    {
+        private readonly IMainDatabase _database;
+
+        public CleanupDownloadClientUnavailablePendingReleases(IMainDatabase database)
+        {
+            _database = database;
+        }
+
+        public void Clean()
+        {
+            var mapper = _database.GetDataMapper();
+            var twoWeeksAgo = DateTime.UtcNow.AddDays(-14);
+
+            mapper.Delete<PendingRelease>(p => p.Added < twoWeeksAgo &&
+                                               (p.Reason == PendingReleaseReason.DownloadClientUnavailable ||
+                                                p.Reason == PendingReleaseReason.Fallback));
+
+            //            mapper.AddParameter("twoWeeksAgo", $"{DateTime.UtcNow.AddDays(-14).ToString("s")}Z");
+
+            //            mapper.ExecuteNonQuery(@"DELETE FROM PendingReleases
+            //                                     WHERE Added < @twoWeeksAgo
+            //                                     AND (Reason = 'DownloadClientUnavailable' OR Reason = 'Fallback')");
+        }
+    }
+}

src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedDownloadClientStatus.cs

@@ -0,0 +1,26 @@
+using NzbDrone.Core.Datastore;
+
+namespace NzbDrone.Core.Housekeeping.Housekeepers
+{
+    public class CleanupOrphanedDownloadClientStatus : IHousekeepingTask
+    {
+        private readonly IMainDatabase _database;
+
+        public CleanupOrphanedDownloadClientStatus(IMainDatabase database)
+        {
+            _database = database;
+        }
+
+        public void Clean()
+        {
+            var mapper = _database.GetDataMapper();
+
+            mapper.ExecuteNonQuery(@"DELETE FROM DownloadClientStatus
+                                     WHERE Id IN (
+                                     SELECT DownloadClientStatus.Id FROM DownloadClientStatus
+                                     LEFT OUTER JOIN DownloadClients
+                                     ON DownloadClientStatus.ProviderId = DownloadClients.Id
+                                     WHERE DownloadClients.Id IS NULL)");
+        }
+    }
+}

src/NzbDrone.Core/Indexers/IndexerBase.cs

@@ -95,11 +95,6 @@ namespace NzbDrone.Core.Indexers
                 failures.Add(new ValidationFailure(string.Empty, "Test was aborted due to an error: " + ex.Message));
             }
 
-            if (Definition.Id != 0)
-            {
-                _indexerStatusService.RecordSuccess(Definition.Id);
-            }
-
             return new ValidationResult(failures);
         }
 

src/NzbDrone.Core/Indexers/IndexerFactory.cs

@@ -1,5 +1,6 @@
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.Linq;
+using FluentValidation.Results;
 using NLog;
 using NzbDrone.Common.Composition;
 using NzbDrone.Core.Messaging.Events;
@@ -70,7 +71,7 @@ namespace NzbDrone.Core.Indexers
 
         private IEnumerable<IIndexer> FilterBlockedIndexers(IEnumerable<IIndexer> indexers)
         {
-            var blockedIndexers = _indexerStatusService.GetBlockedIndexers().ToDictionary(v => v.ProviderId, v => v);
+            var blockedIndexers = _indexerStatusService.GetBlockedProviders().ToDictionary(v => v.ProviderId, v => v);
 
             foreach (var indexer in indexers)
             {
@@ -84,5 +85,17 @@ namespace NzbDrone.Core.Indexers
                 yield return indexer;
             }
         }
+
+        public override ValidationResult Test(IndexerDefinition definition)
+        {
+            var result = base.Test(definition);
+
+            if ((result == null || result.IsValid) && definition.Id != 0)
+            {
+                _indexerStatusService.RecordSuccess(definition.Id);
+            }
+
+            return result;
+        }
     }
 }

src/NzbDrone.Core/Indexers/IndexerStatus.cs

@@ -1,23 +1,10 @@
-using System;
-using NzbDrone.Core.Datastore;
 using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.ThingiProvider.Status;
 
 namespace NzbDrone.Core.Indexers
 {
-    public class IndexerStatus : ModelBase
+    public class IndexerStatus : ProviderStatusBase
     {
-        public int ProviderId { get; set; }
-
-        public DateTime? InitialFailure { get; set; }
-        public DateTime? MostRecentFailure { get; set; }
-        public int EscalationLevel { get; set; }
-        public DateTime? DisabledTill { get; set; }
-
         public ReleaseInfo LastRssSyncReleaseInfo { get; set; }
-
-        public bool IsDisabled()
-        {
-            return DisabledTill.HasValue && DisabledTill.Value > DateTime.UtcNow;
-        }
     }
 }

src/NzbDrone.Core/Indexers/IndexerStatusRepository.cs

@@ -1,26 +1,20 @@
-using System.Linq;
 using NzbDrone.Core.Datastore;
 using NzbDrone.Core.Messaging.Events;
-using NzbDrone.Core.ThingiProvider;
+using NzbDrone.Core.ThingiProvider.Status;
 
 
 namespace NzbDrone.Core.Indexers
 {
-    public interface IIndexerStatusRepository : IProviderRepository<IndexerStatus>
+    public interface IIndexerStatusRepository : IProviderStatusRepository<IndexerStatus>
     {
-        IndexerStatus FindByIndexerId(int indexerId);
     }
 
-    public class IndexerStatusRepository : ProviderRepository<IndexerStatus>, IIndexerStatusRepository
+    public class IndexerStatusRepository : ProviderStatusRepository<IndexerStatus>, IIndexerStatusRepository
+
     {
         public IndexerStatusRepository(IMainDatabase database, IEventAggregator eventAggregator)
             : base(database, eventAggregator)
         {
         }
-
-        public IndexerStatus FindByIndexerId(int indexerId)
-        {
-            return Query.Where(c => c.ProviderId == indexerId).SingleOrDefault();
-        }
     }
 }

src/NzbDrone.Core/Indexers/IndexerStatusService.cs

@@ -1,131 +1,31 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using NLog;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.ThingiProvider.Events;
+using NzbDrone.Core.ThingiProvider.Status;
 
 namespace NzbDrone.Core.Indexers
 {
-    public interface IIndexerStatusService
+    public interface IIndexerStatusService : IProviderStatusServiceBase<IndexerStatus>
     {
-        List<IndexerStatus> GetBlockedIndexers();
         ReleaseInfo GetLastRssSyncReleaseInfo(int indexerId);
-        void RecordSuccess(int indexerId);
-        void RecordFailure(int indexerId, TimeSpan minimumBackOff = default(TimeSpan));
-        void RecordConnectionFailure(int indexerId);
 
         void UpdateRssSyncStatus(int indexerId, ReleaseInfo releaseInfo);
     }
 
-    public class IndexerStatusService : IIndexerStatusService, IHandleAsync<ProviderDeletedEvent<IIndexer>>
+    public class IndexerStatusService : ProviderStatusServiceBase<IIndexer, IndexerStatus>, IIndexerStatusService
     {
-        private static readonly int[] EscalationBackOffPeriods = {
+        public IndexerStatusService(IIndexerStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger)
-                                                                     0,
+            : base(providerStatusRepository, eventAggregator, logger)
-                                                                     5 * 60,
-                                                                     15 * 60,
-                                                                     30 * 60,
-                                                                     60 * 60,
-                                                                     3 * 60 * 60,
-                                                                     6 * 60 * 60,
-                                                                     12 * 60 * 60,
-                                                                     24 * 60 * 60
-                                                                 };
-        private static readonly int MaximumEscalationLevel = EscalationBackOffPeriods.Length - 1;
-
-        private static readonly object _syncRoot = new object();
-
-        private readonly IIndexerStatusRepository _indexerStatusRepository;
-        private readonly Logger _logger;
-
-        public IndexerStatusService(IIndexerStatusRepository indexerStatusRepository, Logger logger)
         {
-            _indexerStatusRepository = indexerStatusRepository;
-            _logger = logger;
-        }
-
-        public List<IndexerStatus> GetBlockedIndexers()
-        {
-            return _indexerStatusRepository.All().Where(v => v.IsDisabled()).ToList();
         }
 
         public ReleaseInfo GetLastRssSyncReleaseInfo(int indexerId)
         {
-            return GetIndexerStatus(indexerId).LastRssSyncReleaseInfo;
+            return GetProviderStatus(indexerId).LastRssSyncReleaseInfo;
-        }
-
-        private IndexerStatus GetIndexerStatus(int indexerId)
-        {
-            return _indexerStatusRepository.FindByIndexerId(indexerId) ?? new IndexerStatus { ProviderId = indexerId };
-        }
-
-        private TimeSpan CalculateBackOffPeriod(IndexerStatus status)
-        {
-            var level = Math.Min(MaximumEscalationLevel, status.EscalationLevel);
-
-            return TimeSpan.FromSeconds(EscalationBackOffPeriods[level]);
-        }
-
-        public void RecordSuccess(int indexerId)
-        {
-            lock (_syncRoot)
-            {
-                var status = GetIndexerStatus(indexerId);
-
-                if (status.EscalationLevel == 0)
-                {
-                    return;
-                }
-
-                status.EscalationLevel--;
-                status.DisabledTill = null;
-
-                _indexerStatusRepository.Upsert(status);
-            }
-        }
-
-        protected void RecordFailure(int indexerId, TimeSpan minimumBackOff, bool escalate)
-        {
-            lock (_syncRoot)
-            {
-                var status = GetIndexerStatus(indexerId);
-
-                var now = DateTime.UtcNow;
-
-                if (status.EscalationLevel == 0)
-                {
-                    status.InitialFailure = now;
-                }
-
-                status.MostRecentFailure = now;
-                if (escalate)
-                {
-                    status.EscalationLevel = Math.Min(MaximumEscalationLevel, status.EscalationLevel + 1);
-                }
-
-                if (minimumBackOff != TimeSpan.Zero)
-                {
-                    while (status.EscalationLevel < MaximumEscalationLevel && CalculateBackOffPeriod(status) < minimumBackOff)
-                    {
-                        status.EscalationLevel++;
-                    }
-                }
-
-                status.DisabledTill = now + CalculateBackOffPeriod(status);
-
-                _indexerStatusRepository.Upsert(status);
-            }
-        }
-
-        public void RecordFailure(int indexerId, TimeSpan minimumBackOff = default(TimeSpan))
-        {
-            RecordFailure(indexerId, minimumBackOff, true);
-        }
-
-        public void RecordConnectionFailure(int indexerId)
-        {
-            RecordFailure(indexerId, default(TimeSpan), false);
         }
 
 
@@ -133,21 +33,11 @@ namespace NzbDrone.Core.Indexers
         {
             lock (_syncRoot)
             {
-                var status = GetIndexerStatus(indexerId);
+                var status = GetProviderStatus(indexerId);
 
                 status.LastRssSyncReleaseInfo = releaseInfo;
 
-                _indexerStatusRepository.Upsert(status);
+                _providerStatusRepository.Upsert(status);
-            }
-        }
-
-        public void HandleAsync(ProviderDeletedEvent<IIndexer> message)
-        {
-            var indexerStatus = _indexerStatusRepository.FindByIndexerId(message.ProviderId);
-
-            if (indexerStatus != null)
-            {
-                _indexerStatusRepository.Delete(indexerStatus);
             }
         }
     }

src/NzbDrone.Core/Messaging/Events/EventAggregator.cs

@@ -62,6 +62,17 @@ namespace NzbDrone.Core.Messaging.Events
                 }
             }
 
+            foreach (var handler in _serviceFactory.BuildAll<IHandleAsync<IEvent>>())
+            {
+                var handlerLocal = handler;
+
+                _taskFactory.StartNew(() =>
+                {
+                    handlerLocal.HandleAsync(@event);
+                }, TaskCreationOptions.PreferFairness)
+                .LogExceptions();
+            }
+
             foreach (var handler in _serviceFactory.BuildAll<IHandleAsync<TEvent>>())
             {
                 var handlerLocal = handler;

src/NzbDrone.Core/NzbDrone.Core.csproj

@@ -170,6 +170,7 @@
     <Compile Include="Datastore\MainDatabase.cs" />
     <Compile Include="Datastore\LogDatabase.cs" />
     <Compile Include="Datastore\Migration\001_initial_setup.cs" />
+    <Compile Include="Datastore\Migration\002_add_release_to_pending_releases.cs" />
     <Compile Include="Datastore\Migration\Framework\MigrationContext.cs" />
     <Compile Include="Datastore\Migration\Framework\MigrationController.cs" />
     <Compile Include="Datastore\Migration\Framework\MigrationDbFactory.cs" />
@@ -200,6 +201,7 @@
     <Compile Include="DecisionEngine\SpecificationPriority.cs" />
     <Compile Include="DecisionEngine\Specifications\AcceptableSizeSpecification.cs" />
     <Compile Include="DecisionEngine\Specifications\BlacklistSpecification.cs" />
+    <Compile Include="DecisionEngine\Specifications\BlockedIndexerSpecification.cs" />
     <Compile Include="DecisionEngine\Specifications\DiscographySpecification.cs" />
     <Compile Include="DecisionEngine\Specifications\CutoffSpecification.cs" />
     <Compile Include="DecisionEngine\Specifications\ProtocolSpecification.cs" />
@@ -242,6 +244,7 @@
     <Compile Include="Download\Clients\Deluge\DelugeUpdateUIResult.cs" />
     <Compile Include="Download\Clients\DownloadClientAuthenticationException.cs" />
     <Compile Include="Download\Clients\DownloadClientException.cs" />
+    <Compile Include="Download\Clients\DownloadClientUnavailableException.cs" />
     <Compile Include="Download\Clients\DownloadStation\Proxies\DownloadStationInfoProxy.cs" />
     <Compile Include="Download\Clients\DownloadStation\TorrentDownloadStation.cs" />
     <Compile Include="Download\Clients\DownloadStation\Proxies\DownloadStationTaskProxy.cs" />
@@ -373,7 +376,11 @@
     <Compile Include="Download\Clients\uTorrent\UTorrentTorrentStatus.cs" />
     <Compile Include="Download\Clients\Vuze\Vuze.cs" />
     <Compile Include="Download\CompletedDownloadService.cs" />
+    <Compile Include="Download\DownloadClientStatus.cs" />
+    <Compile Include="Download\DownloadClientStatusRepository.cs" />
+    <Compile Include="Download\DownloadClientStatusService.cs" />
     <Compile Include="Download\DownloadEventHub.cs" />
+    <Compile Include="Download\Pending\PendingReleaseReason.cs" />
     <Compile Include="Download\TrackedDownloads\DownloadMonitoringService.cs" />
     <Compile Include="Download\TrackedDownloads\TrackedDownload.cs" />
     <Compile Include="Download\TrackedDownloads\TrackedDownloadService.cs" />
@@ -435,8 +442,10 @@
     <Compile Include="Extras\Lyrics\LyricService.cs" />
     <Compile Include="Fluent.cs" />
     <Compile Include="HealthCheck\CheckHealthCommand.cs" />
+    <Compile Include="HealthCheck\CheckOnAttribute.cs" />
     <Compile Include="HealthCheck\Checks\AppDataLocationCheck.cs" />
     <Compile Include="HealthCheck\Checks\DownloadClientCheck.cs" />
+    <Compile Include="HealthCheck\Checks\DownloadClientStatusCheck.cs" />
     <Compile Include="HealthCheck\Checks\MountCheck.cs" />
     <Compile Include="HealthCheck\Checks\ImportMechanismCheck.cs" />
     <Compile Include="HealthCheck\Checks\IndexerRssCheck.cs" />
@@ -459,9 +468,11 @@
     <Compile Include="Housekeeping\Housekeepers\CleanupAdditionalNamingSpecs.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupCommandQueue.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupAbsolutePathMetadataFiles.cs" />
+    <Compile Include="Housekeeping\Housekeepers\CleanupDownloadClientUnavailablePendingReleases.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupDuplicateMetadataFiles.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupOrphanedAlbums.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupOrphanedBlacklist.cs" />
+    <Compile Include="Housekeeping\Housekeepers\CleanupOrphanedDownloadClientStatus.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupOrphanedTrackFiles.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupOrphanedTracks.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupOrphanedIndexerStatus.cs" />
@@ -994,6 +1005,7 @@
     <Compile Include="Tags\TagsUpdatedEvent.cs" />
     <Compile Include="ThingiProvider\ConfigContractNotFoundException.cs" />
     <Compile Include="ThingiProvider\Events\ProviderDeletedEvent.cs" />
+    <Compile Include="ThingiProvider\Events\ProviderStatusChangedEvent.cs" />
     <Compile Include="ThingiProvider\Events\ProviderUpdatedEvent.cs" />
     <Compile Include="ThingiProvider\IProvider.cs" />
     <Compile Include="ThingiProvider\IProviderConfig.cs" />
@@ -1004,6 +1016,9 @@
     <Compile Include="ThingiProvider\ProviderFactory.cs" />
     <Compile Include="ThingiProvider\ProviderMessage.cs" />
     <Compile Include="ThingiProvider\ProviderRepository.cs" />
+    <Compile Include="ThingiProvider\Status\ProviderStatusBase.cs" />
+    <Compile Include="ThingiProvider\Status\ProviderStatusRepository.cs" />
+    <Compile Include="ThingiProvider\Status\ProviderStatusServiceBase.cs" />
     <Compile Include="TinyTwitter.cs" />
     <Compile Include="Update\Commands\ApplicationUpdateCommand.cs" />
     <Compile Include="Update\InstallUpdateService.cs" />

src/NzbDrone.Core/ThingiProvider/Events/ProviderStatusChangedEvent.cs

@@ -0,0 +1,18 @@
+using NzbDrone.Common.Messaging;
+using NzbDrone.Core.ThingiProvider.Status;
+
+namespace NzbDrone.Core.ThingiProvider.Events
+{
+    public class ProviderStatusChangedEvent<TProvider> : IEvent
+    {
+        public int ProviderId { get; private set; }
+
+        public ProviderStatusBase Status { get; private set; }
+
+        public ProviderStatusChangedEvent(int id, ProviderStatusBase status)
+        {
+            ProviderId = id;
+            Status = status;
+        }
+    }
+}

src/NzbDrone.Core/ThingiProvider/ProviderFactory.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using FluentValidation.Results;

src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusBase.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;

src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusRepository.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;

src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs

@@ -1,8 +1,9 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using NLog;
 using NzbDrone.Core.Messaging.Events;
+using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.ThingiProvider.Events;
 
 namespace NzbDrone.Core.ThingiProvider.Status
@@ -20,13 +21,25 @@ namespace NzbDrone.Core.ThingiProvider.Status
         where TProvider : IProvider
         where TModel : ProviderStatusBase, new()
     {
+        private static readonly int[] EscalationBackOffPeriods = {
+                                                                     0,
+                                                                     5 * 60,
+                                                                     15 * 60,
+                                                                     30 * 60,
+                                                                     60 * 60,
+                                                                     3 * 60 * 60,
+                                                                     6 * 60 * 60,
+                                                                     12 * 60 * 60,
+                                                                     24 * 60 * 60
+                                                                 };
+
         protected readonly object _syncRoot = new object();
 
         protected readonly IProviderStatusRepository<TModel> _providerStatusRepository;
         protected readonly IEventAggregator _eventAggregator;
         protected readonly Logger _logger;
 
-        protected int MaximumEscalationLevel { get; set; } = EscalationBackOff.Periods.Length - 1;
+        protected int MaximumEscalationLevel { get; set; } = EscalationBackOffPeriods.Length - 1;
         protected TimeSpan MinimumTimeSinceInitialFailure { get; set; } = TimeSpan.Zero;
 
         public ProviderStatusServiceBase(IProviderStatusRepository<TModel> providerStatusRepository, IEventAggregator eventAggregator, Logger logger)
@@ -50,7 +63,7 @@ namespace NzbDrone.Core.ThingiProvider.Status
         {
             var level = Math.Min(MaximumEscalationLevel, status.EscalationLevel);
 
-            return TimeSpan.FromSeconds(EscalationBackOff.Periods[level]);
+            return TimeSpan.FromSeconds(EscalationBackOffPeriods[level]);
         }
 
         public virtual void RecordSuccess(int providerId)