diff --git a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/AddFixture.cs b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/AddFixture.cs index f98c66dc3..da64265e1 100644 --- a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/AddFixture.cs +++ b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/AddFixture.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using FizzWare.NBuilder; using Marr.Data; using Moq; @@ -26,6 +27,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests private ReleaseInfo _release; private ParsedAlbumInfo _parsedAlbumInfo; private RemoteAlbum _remoteAlbum; + private List _heldReleases; [SetUp] public void Setup() @@ -63,14 +65,24 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests _temporarilyRejected = new DownloadDecision(_remoteAlbum, new Rejection("Temp Rejected", RejectionType.Temporary)); + _heldReleases = new List(); + Mocker.GetMock() .Setup(s => s.All()) - .Returns(new List()); + .Returns(_heldReleases); + + Mocker.GetMock() + .Setup(s => s.AllByArtistId(It.IsAny())) + .Returns(i => _heldReleases.Where(v => v.ArtistId == i).ToList()); Mocker.GetMock() .Setup(s => s.GetArtist(It.IsAny())) .Returns(_artist); + Mocker.GetMock() + .Setup(s => s.GetArtists(It.IsAny>())) + .Returns(new List { _artist }); + Mocker.GetMock() .Setup(s => s.GetAlbums(It.IsAny(), _artist, null)) .Returns(new List {_album}); @@ -80,7 +92,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests .Returns((List d) => d); } - private void GivenHeldRelease(string title, string indexer, DateTime publishDate) + private void GivenHeldRelease(string title, string indexer, DateTime publishDate, PendingReleaseReason reason = PendingReleaseReason.Delay) { var release = _release.JsonClone(); release.Indexer = indexer; @@ -92,11 +104,10 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests .With(h => h.ArtistId = _artist.Id) .With(h => h.Title = title) .With(h => h.Release = release) + .With(h => h.Reason = reason) .Build(); - Mocker.GetMock() - .Setup(s => s.All()) - .Returns(heldReleases); + _heldReleases.AddRange(heldReleases); } [Test] @@ -117,6 +128,29 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests VerifyNoInsert(); } + [Test] + public void should_not_add_if_it_is_the_same_release_from_the_same_indexer_twice() + { + GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate, PendingReleaseReason.DownloadClientUnavailable); + GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate, PendingReleaseReason.Fallback); + + Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay); + + VerifyNoInsert(); + } + + [Test] + public void should_remove_duplicate_if_it_is_the_same_release_from_the_same_indexer_twice() + { + GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate, PendingReleaseReason.DownloadClientUnavailable); + GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate, PendingReleaseReason.Fallback); + + Subject.Add(_temporarilyRejected, PendingReleaseReason.Fallback); + + Mocker.GetMock() + .Verify(v => v.Delete(It.IsAny()), Times.Once()); + } + [Test] public void should_add_if_title_is_different() { diff --git a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveGrabbedFixture.cs b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveGrabbedFixture.cs index cf72a88d4..f213eeac6 100644 --- a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveGrabbedFixture.cs +++ b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveGrabbedFixture.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Linq; using FizzWare.NBuilder; using Marr.Data; using Moq; @@ -26,6 +27,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests private ReleaseInfo _release; private ParsedAlbumInfo _parsedAlbumInfo; private RemoteAlbum _remoteAlbum; + private List _heldReleases; [SetUp] public void Setup() @@ -63,14 +65,24 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests _temporarilyRejected = new DownloadDecision(_remoteAlbum, new Rejection("Temp Rejected", RejectionType.Temporary)); + _heldReleases = new List(); + Mocker.GetMock() .Setup(s => s.All()) - .Returns(new List()); + .Returns(_heldReleases); + + Mocker.GetMock() + .Setup(s => s.AllByArtistId(It.IsAny())) + .Returns(i => _heldReleases.Where(v => v.ArtistId == i).ToList()); Mocker.GetMock() .Setup(s => s.GetArtist(It.IsAny())) .Returns(_artist); + Mocker.GetMock() + .Setup(s => s.GetArtists(It.IsAny>())) + .Returns(new List { _artist }); + Mocker.GetMock() .Setup(s => s.GetAlbums(It.IsAny(), _artist, null)) .Returns(new List {_album}); @@ -92,9 +104,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests .With(h => h.ParsedAlbumInfo = parsedEpisodeInfo) .Build(); - Mocker.GetMock() - .Setup(s => s.All()) - .Returns(heldReleases); + _heldReleases.AddRange(heldReleases); } [Test] diff --git a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemovePendingFixture.cs b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemovePendingFixture.cs index adab71ee9..642547e36 100644 --- a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemovePendingFixture.cs +++ b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemovePendingFixture.cs @@ -1,4 +1,4 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; using FizzWare.NBuilder; using Moq; @@ -38,6 +38,10 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests .Setup(s => s.GetArtist(It.IsAny())) .Returns(new Artist()); + Mocker.GetMock() + .Setup(s => s.GetArtists(It.IsAny>())) + .Returns(new List { new Artist() }); + Mocker.GetMock() .Setup(s => s.GetAlbums(It.IsAny(), It.IsAny(), null)) .Returns(new List{ _album }); diff --git a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveRejectedFixture.cs b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveRejectedFixture.cs index 05ae539ad..36d6da6bc 100644 --- a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveRejectedFixture.cs +++ b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveRejectedFixture.cs @@ -62,7 +62,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests _remoteAlbum.Artist = _artist; _remoteAlbum.ParsedAlbumInfo = _parsedAlbumInfo; _remoteAlbum.Release = _release; - + _temporarilyRejected = new DownloadDecision(_remoteAlbum, new Rejection("Temp Rejected", RejectionType.Temporary)); Mocker.GetMock() @@ -73,6 +73,10 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests .Setup(s => s.GetArtist(It.IsAny())) .Returns(_artist); + Mocker.GetMock() + .Setup(s => s.GetArtists(It.IsAny>())) + .Returns(new List { _artist }); + Mocker.GetMock() .Setup(s => s.GetAlbums(It.IsAny(), _artist, null)) .Returns(new List {_album}); diff --git a/src/NzbDrone.Core/DecisionEngine/Specifications/RssSync/DelaySpecification.cs b/src/NzbDrone.Core/DecisionEngine/Specifications/RssSync/DelaySpecification.cs index 8c341dc59..b5cfc52b4 100644 --- a/src/NzbDrone.Core/DecisionEngine/Specifications/RssSync/DelaySpecification.cs +++ b/src/NzbDrone.Core/DecisionEngine/Specifications/RssSync/DelaySpecification.cs @@ -94,7 +94,7 @@ namespace NzbDrone.Core.DecisionEngine.Specifications.RssSync var albumIds = subject.Albums.Select(e => e.Id); - var oldest = _pendingReleaseService.OldestPendingRelease(subject.Artist.Id, albumIds); + var oldest = _pendingReleaseService.OldestPendingRelease(subject.Artist.Id, albumIds.ToArray()); if (oldest != null && oldest.Release.AgeMinutes > delay) { diff --git a/src/NzbDrone.Core/Download/Pending/PendingReleaseRepository.cs b/src/NzbDrone.Core/Download/Pending/PendingReleaseRepository.cs index 50e670721..ded73e2a6 100644 --- a/src/NzbDrone.Core/Download/Pending/PendingReleaseRepository.cs +++ b/src/NzbDrone.Core/Download/Pending/PendingReleaseRepository.cs @@ -1,4 +1,4 @@ -using System.Collections.Generic; +using System.Collections.Generic; using NzbDrone.Core.Datastore; using NzbDrone.Core.Messaging.Events; @@ -8,6 +8,7 @@ namespace NzbDrone.Core.Download.Pending { void DeleteByArtistId(int artistId); List AllByArtistId(int artistId); + List WithoutFallback(); } public class PendingReleaseRepository : BasicRepository, IPendingReleaseRepository @@ -26,5 +27,10 @@ namespace NzbDrone.Core.Download.Pending { return Query.Where(p => p.ArtistId == artistId); } + + public List WithoutFallback() + { + return Query.Where(p => p.Reason != PendingReleaseReason.Fallback); + } } -} \ No newline at end of file +} diff --git a/src/NzbDrone.Core/Download/Pending/PendingReleaseService.cs b/src/NzbDrone.Core/Download/Pending/PendingReleaseService.cs index 65cefcc13..ea67c88b5 100644 --- a/src/NzbDrone.Core/Download/Pending/PendingReleaseService.cs +++ b/src/NzbDrone.Core/Download/Pending/PendingReleaseService.cs @@ -21,12 +21,13 @@ namespace NzbDrone.Core.Download.Pending public interface IPendingReleaseService { void Add(DownloadDecision decision, PendingReleaseReason reason); + void AddMany(List> decisions); List GetPending(); List GetPendingRemoteAlbums(int artistId); List GetPendingQueue(); Queue.Queue FindPendingQueueItem(int queueId); void RemovePendingQueueItems(int queueId); - RemoteAlbum OldestPendingRelease(int artistId, IEnumerable albumIds); + RemoteAlbum OldestPendingRelease(int artistId, int[] albumIds); } public class PendingReleaseService : IPendingReleaseService, @@ -68,8 +69,27 @@ namespace NzbDrone.Core.Download.Pending public void Add(DownloadDecision decision, PendingReleaseReason reason) { - var alreadyPending = GetPendingReleases(); + var alreadyPending = _repository.AllByArtistId(decision.RemoteAlbum.Artist.Id); + alreadyPending = IncludeRemoteAlbums(alreadyPending); + + Add(alreadyPending, decision, reason); + } + + public void AddMany(List> decisions) + { + var alreadyPending = decisions.Select(v => v.Item1.RemoteAlbum.Artist.Id).Distinct().SelectMany(_repository.AllByArtistId).ToList(); + + alreadyPending = IncludeRemoteAlbums(alreadyPending); + + foreach (var pair in decisions) + { + Add(alreadyPending, pair.Item1, pair.Item2); + } + } + + private void Add(List alreadyPending, DownloadDecision decision, PendingReleaseReason reason) + { var albumIds = decision.RemoteAlbum.Albums.Select(e => e.Id); var existingReports = alreadyPending.Where(r => r.RemoteAlbum.Albums.Select(e => e.Id) @@ -80,22 +100,32 @@ namespace NzbDrone.Core.Download.Pending if (matchingReports.Any()) { - var sameReason = true; + var matchingReport = matchingReports.First(); - foreach (var matchingReport in matchingReports) + if (matchingReport.Reason != reason) { - if (matchingReport.Reason != reason) - { - _logger.Debug("The release {0} is already pending with reason {1}, changing to {2}", decision.RemoteAlbum, matchingReport.Reason, reason); matchingReport.Reason = reason; - _repository.Update(matchingReport); - sameReason = false; - } + _logger.Debug("The release {0} is already pending with reason {1}, changing to {2}", decision.RemoteAlbum, matchingReport.Reason, reason); + matchingReport.Reason = reason; + _repository.Update(matchingReport); } - if (sameReason) + else { _logger.Debug("The release {0} is already pending with reason {1}, not adding again", decision.RemoteAlbum, reason); return; } + + if (matchingReports.Count() > 1) + { + _logger.Debug("The release {0} had {1} duplicate pending, removing duplicates.", decision.RemoteAlbum, matchingReports.Count() - 1); + + foreach (var duplicate in matchingReports.Skip(1)) + { + _repository.Delete(duplicate.Id); + alreadyPending.Remove(duplicate); + } + } + + return; } _logger.Debug("Adding release {0} to pending releases with reason {1}", decision.RemoteAlbum, reason); Insert(decision, reason); @@ -122,7 +152,7 @@ namespace NzbDrone.Core.Download.Pending public List GetPendingRemoteAlbums(int artistId) { - return _repository.AllByArtistId(artistId).Select(GetRemoteAlbum).ToList(); + return IncludeRemoteAlbums(_repository.AllByArtistId(artistId)).Select(v => v.RemoteAlbum).ToList(); } public List GetPendingQueue() @@ -131,7 +161,8 @@ namespace NzbDrone.Core.Download.Pending var nextRssSync = new Lazy(() => _taskManager.GetNextExecution(typeof(RssSyncCommand))); - foreach (var pendingRelease in GetPendingReleases().Where(p => p.Reason != PendingReleaseReason.Fallback)) + var pendingReleases = IncludeRemoteAlbums(_repository.WithoutFallback()); + foreach (var pendingRelease in pendingReleases) { foreach (var album in pendingRelease.RemoteAlbum.Albums) { @@ -203,27 +234,48 @@ namespace NzbDrone.Core.Download.Pending _repository.DeleteMany(releasesToRemove.Select(c => c.Id)); } - public RemoteAlbum OldestPendingRelease(int artistId, IEnumerable albumIds) + public RemoteAlbum OldestPendingRelease(int artistId, int[] albumIds) { - return GetPendingRemoteAlbums(artistId).Where(r => r.Albums.Select(e => e.Id).Intersect(albumIds).Any()) - .OrderByDescending(p => p.Release.AgeHours) - .FirstOrDefault(); + var artistReleases = GetPendingReleases(artistId); + + return artistReleases.Select(r => r.RemoteAlbum) + .Where(r => r.Albums.Select(e => e.Id).Intersect(albumIds).Any()) + .OrderByDescending(p => p.Release.AgeHours) + .FirstOrDefault(); } private List GetPendingReleases() + { + return IncludeRemoteAlbums(_repository.All().ToList()); + } + + private List GetPendingReleases(int artistId) + { + return IncludeRemoteAlbums(_repository.AllByArtistId(artistId).ToList()); + } + + private List IncludeRemoteAlbums(List releases) { var result = new List(); + var artistMap = _artistService.GetArtists(releases.Select(v => v.ArtistId).Distinct()) + .ToDictionary(v => v.Id); - foreach (var release in _repository.All()) + foreach (var release in releases) { - var remoteAlbum = GetRemoteAlbum(release); + var artist = artistMap.GetValueOrDefault(release.ArtistId); - if (remoteAlbum == null) + // Just in case the artist was removed, but wasn't cleaned up yet (housekeeper will clean it up) + if (artist == null) return null; + + var albums = _parsingService.GetAlbums(release.ParsedAlbumInfo, artist); + + release.RemoteAlbum = new RemoteAlbum { - continue; - } - - release.RemoteAlbum = remoteAlbum; + Artist = artist, + Albums = albums, + ParsedAlbumInfo = release.ParsedAlbumInfo, + Release = release.Release + }; result.Add(release); } @@ -231,27 +283,6 @@ namespace NzbDrone.Core.Download.Pending return result; } - private RemoteAlbum GetRemoteAlbum(PendingRelease release) - { - var artist = _artistService.GetArtist(release.ArtistId); - - //Just in case the series was removed, but wasn't cleaned up yet (housekeeper will clean it up) - if (artist == null) - { - return null; - } - - var album = _parsingService.GetAlbums(release.ParsedAlbumInfo, artist); - - return new RemoteAlbum - { - Artist = artist, - Albums = album, - ParsedAlbumInfo = release.ParsedAlbumInfo, - Release = release.Release - }; - } - private void Insert(DownloadDecision decision, PendingReleaseReason reason) { _repository.Insert(new PendingRelease @@ -291,7 +322,7 @@ namespace NzbDrone.Core.Download.Pending private void RemoveGrabbed(RemoteAlbum remoteAlbum) { - var pendingReleases = GetPendingReleases(); + var pendingReleases = GetPendingReleases(remoteAlbum.Artist.Id); var albumIds = remoteAlbum.Albums.Select(e => e.Id); var existingReports = pendingReleases.Where(r => r.RemoteAlbum.Albums.Select(e => e.Id) diff --git a/src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs b/src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs index 42e713b91..baaf02d30 100644 --- a/src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs +++ b/src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs @@ -131,6 +131,8 @@ namespace NzbDrone.Core.Download var pending = new List(); var stored = new List(); + var addQueue = new List>(); + foreach (var report in failed) { // If a release was already grabbed with matching albums we should store it as a fallback @@ -141,22 +143,27 @@ namespace NzbDrone.Core.Download if (IsAlbumProcessed(grabbed, report)) { - _pendingReleaseService.Add(report, PendingReleaseReason.Fallback); + addQueue.Add(Tuple.Create(report, PendingReleaseReason.Fallback)); pending.Add(report); } else if (IsAlbumProcessed(stored, report)) { - _pendingReleaseService.Add(report, PendingReleaseReason.Fallback); + addQueue.Add(Tuple.Create(report, PendingReleaseReason.Fallback)); pending.Add(report); } else { - _pendingReleaseService.Add(report, PendingReleaseReason.DownloadClientUnavailable); + addQueue.Add(Tuple.Create(report, PendingReleaseReason.DownloadClientUnavailable)); pending.Add(report); stored.Add(report); } } + if (addQueue.Any()) + { + _pendingReleaseService.AddMany(addQueue); + } + return pending; } }