Updated HttpNetImporterBase. Still needs work to correctly handle failures.

src/NzbDrone.Core.Test/NetImport/RSSImportFixture.cs

@@ -1,32 +1,42 @@
-using System.Linq;
-using System.Text;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using FizzWare.NBuilder;
 using FluentAssertions;
+using Moq;
 using NUnit.Framework;
 using NzbDrone.Common.Http;
-using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Datastore;
 using NzbDrone.Core.NetImport;
 using NzbDrone.Core.NetImport.RSSImport;
 using NzbDrone.Core.Test.Framework;
 
 namespace NzbDrone.Core.Test.NetImport
 {
-    public class RSSImportTest : CoreTest<RSSImportParser>
+    [TestFixture]
+    public class RSSImportFixture : CoreTest<RSSImport>
     {
-        private NetImportResponse CreateResponse(string url, string content)
-        {
-            var httpRequest = new HttpRequest(url);
-            var httpResponse = new HttpResponse(httpRequest, new HttpHeader(), Encoding.UTF8.GetBytes(content));
 
-            return new NetImportResponse(new NetImportRequest(httpRequest), httpResponse);
+        [SetUp]
+        public void Setup()
+        {
+            Subject.Definition = Subject.DefaultDefinitions.First();
+        }
+        private void GivenRecentFeedResponse(string rssXmlFile)
+        {
+            var recentFeed = ReadAllText(@"Files/" + rssXmlFile);
+
+            Mocker.GetMock<IHttpClient>()
+                .Setup(o => o.Execute(It.IsAny<HttpRequest>()))
+                .Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), recentFeed));
         }
 
-
         [Test]
-        public void should_handle_relative_url()
+        public void should_fetch_imdb_list()
         {
-            var xml = ReadAllText("Files/imdb_watchlist.xml");
+            GivenRecentFeedResponse("imdb_watchlist.xml");
 
-            var result = Subject.ParseResponse(CreateResponse("http://my.indexer.com/api?q=My+Favourite+Show", xml));
+            var result = Subject.Fetch();
 
             result.First().Title.Should().Be("Think Like a Man Too");
             result.First().ImdbId.Should().Be("tt2239832");

src/NzbDrone.Core.Test/NetImport/RSSImportParserFixture.cs

@@ -0,0 +1,36 @@
+using System.Linq;
+using System.Text;
+using FluentAssertions;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.NetImport;
+using NzbDrone.Core.NetImport.RSSImport;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.NetImport
+{
+    public class RSSImportTest : CoreTest<RSSImportParser>
+    {
+        private NetImportResponse CreateResponse(string url, string content)
+        {
+            var httpRequest = new HttpRequest(url);
+            var httpResponse = new HttpResponse(httpRequest, new HttpHeader(), Encoding.UTF8.GetBytes(content));
+
+            return new NetImportResponse(new NetImportRequest(httpRequest), httpResponse);
+        }
+
+
+        [Test]
+        public void should_parse_xml_of_imdb()
+        {
+            var xml = ReadAllText("Files/imdb_watchlist.xml");
+
+            var result = Subject.ParseResponse(CreateResponse("http://my.indexer.com/api?q=My+Favourite+Show", xml));
+
+            result.First().Title.Should().Be("Think Like a Man Too");
+            result.First().ImdbId.Should().Be("tt2239832");
+        }
+    }
+}

src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj

@@ -285,6 +285,7 @@
     <Compile Include="MetadataSource\SearchSeriesComparerFixture.cs" />
     <Compile Include="MetadataSource\SkyHook\SkyHookProxyFixture.cs" />
     <Compile Include="NetImport\RSSImportFixture.cs" />
+    <Compile Include="NetImport\RSSImportParserFixture.cs" />
     <Compile Include="NotificationTests\SynologyIndexerFixture.cs" />
     <Compile Include="OrganizerTests\FileNameBuilderTests\CleanTitleFixture.cs" />
     <Compile Include="OrganizerTests\FileNameBuilderTests\EpisodeTitleCollapseFixture.cs" />

src/NzbDrone.Core/NetImport/HttpNetImportBase.cs

@@ -26,6 +26,10 @@ namespace NzbDrone.Core.NetImport
 
         public override bool Enabled => true;
 
+        public bool SupportsPaging => PageSize > 0;
+
+        public virtual int PageSize => 0;
+
         public virtual TimeSpan RateLimit => TimeSpan.FromSeconds(2);
 
         public abstract INetImportRequestGenerator GetRequestGenerator();
@@ -39,7 +43,189 @@ namespace NzbDrone.Core.NetImport
 
         public override IList<Movie> Fetch()
         {
-            return new List<Movie>();
+            var generator = GetRequestGenerator();
+            
+            return FetchMovies(generator.GetMovies());
+        }
+
+        protected virtual IList<Movie> FetchMovies(NetImportPageableRequestChain pageableRequestChain, bool isRecent = false)
+        {
+            var movies = new List<Movie>();
+            var url = string.Empty;
+
+            var parser = GetParser();
+
+            try
+            {
+                var fullyUpdated = false;
+                Movie lastMovie = null;
+                if (isRecent)
+                {
+                    //lastReleaseInfo = _indexerStatusService.GetLastRssSyncReleaseInfo(Definition.Id);
+                }
+
+                for (int i = 0; i < pageableRequestChain.Tiers; i++)
+                {
+                    var pageableRequests = pageableRequestChain.GetTier(i);
+
+                    foreach (var pageableRequest in pageableRequests)
+                    {
+                        var pagedReleases = new List<Movie>();
+
+                        foreach (var request in pageableRequest)
+                        {
+                            url = request.Url.FullUri;
+
+                            var page = FetchPage(request, parser);
+
+                            pagedReleases.AddRange(page);
+
+                            if (isRecent && page.Any())
+                            {
+                                if (lastMovie == null)
+                                {
+                                    fullyUpdated = true;
+                                    break;
+                                }/*
+                                var oldestReleaseDate = page.Select(v => v.PublishDate).Min();
+                                if (oldestReleaseDate < lastReleaseInfo.PublishDate || page.Any(v => v.DownloadUrl == lastReleaseInfo.DownloadUrl))
+                                {
+                                    fullyUpdated = true;
+                                    break;
+                                }
+
+                                if (pagedReleases.Count >= MaxNumResultsPerQuery &&
+                                    oldestReleaseDate < DateTime.UtcNow - TimeSpan.FromHours(24))
+                                {
+                                    fullyUpdated = false;
+                                    break;
+                                }*///update later
+                            }
+                            else if (pagedReleases.Count >= MaxNumResultsPerQuery)
+                            {
+                                break;
+                            }
+
+                            if (!IsFullPage(page))
+                            {
+                                break;
+                            }
+                        }
+
+                        movies.AddRange(pagedReleases);
+                    }
+
+                    if (movies.Any())
+                    {
+                        break;
+                    }
+                }
+
+                if (isRecent && !movies.Empty())
+                {
+                    var ordered = movies.OrderByDescending(v => v.Title).ToList();
+
+                    lastMovie = ordered.First();
+                    //_indexerStatusService.UpdateRssSyncStatus(Definition.Id, lastReleaseInfo);
+                }
+
+                //_indexerStatusService.RecordSuccess(Definition.Id);
+            }
+            catch (WebException webException)
+            {
+                if (webException.Status == WebExceptionStatus.NameResolutionFailure ||
+                    webException.Status == WebExceptionStatus.ConnectFailure)
+                {
+                    //_indexerStatusService.RecordConnectionFailure(Definition.Id);
+                }
+                else
+                {
+                    //_indexerStatusService.RecordFailure(Definition.Id);
+                }
+
+                if (webException.Message.Contains("502") || webException.Message.Contains("503") ||
+                    webException.Message.Contains("timed out"))
+                {
+                    _logger.Warn("{0} server is currently unavailable. {1} {2}", this, url, webException.Message);
+                }
+                else
+                {
+                    _logger.Warn("{0} {1} {2}", this, url, webException.Message);
+                }
+            }
+            catch (HttpException httpException)
+            {
+                if ((int)httpException.Response.StatusCode == 429)
+                {
+                    //_indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
+                    _logger.Warn("API Request Limit reached for {0}", this);
+                }
+                else
+                {
+                    //_indexerStatusService.RecordFailure(Definition.Id);
+                    _logger.Warn("{0} {1}", this, httpException.Message);
+                }
+            }
+            catch (RequestLimitReachedException)
+            {
+                //_indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
+                _logger.Warn("API Request Limit reached for {0}", this);
+            }
+            catch (ApiKeyException)
+            {
+                //_indexerStatusService.RecordFailure(Definition.Id);
+                _logger.Warn("Invalid API Key for {0} {1}", this, url);
+            }
+            catch (CloudFlareCaptchaException ex)
+            {
+                //_indexerStatusService.RecordFailure(Definition.Id);
+                if (ex.IsExpired)
+                {
+                    _logger.Error(ex, "Expired CAPTCHA token for {0}, please refresh in indexer settings.", this);
+                }
+                else
+                {
+                    _logger.Error(ex, "CAPTCHA token required for {0}, check indexer settings.", this);
+                }
+            }
+            catch (IndexerException ex)
+            {
+                //_indexerStatusService.RecordFailure(Definition.Id);
+                var message = string.Format("{0} - {1}", ex.Message, url);
+                _logger.Warn(ex, message);
+            }
+            catch (Exception feedEx)
+            {
+                //_indexerStatusService.RecordFailure(Definition.Id);
+                feedEx.Data.Add("FeedUrl", url);
+                _logger.Error(feedEx, "An error occurred while processing feed. " + url);
+            }
+
+            return movies;
+        }
+
+        protected virtual bool IsFullPage(IList<Movie> page)
+        {
+            return PageSize != 0 && page.Count >= PageSize;
+        }
+
+        protected virtual IList<Movie> FetchPage(NetImportRequest request, IParseNetImportResponse parser)
+        {
+            var response = FetchIndexerResponse(request);
+
+            return parser.ParseResponse(response).ToList();
+        }
+
+        protected virtual NetImportResponse FetchIndexerResponse(NetImportRequest request)
+        {
+            _logger.Debug("Downloading List " + request.HttpRequest.ToString(false));
+
+            if (request.HttpRequest.RateLimit < RateLimit)
+            {
+                request.HttpRequest.RateLimit = RateLimit;
+            }
+
+            return new NetImportResponse(request, _httpClient.Execute(request.HttpRequest));
         }
 
         protected override void Test(List<ValidationFailure> failures)