From 83f871da86289de02d4eb29003884bb2062f2dfe Mon Sep 17 00:00:00 2001 From: Jamie Rees Date: Thu, 17 May 2018 12:05:06 +0100 Subject: [PATCH] Improved the way we sync the plex content and then get the metadata. #2243 --- .../Jobs/Ombi/IRefreshMetadata.cs | 4 +- .../Jobs/Ombi/RefreshMetadata.cs | 57 ++++++++++--- .../Jobs/Plex/Interfaces/IPlexEpisodeSync.cs | 4 + .../Jobs/Plex/PlexContentSync.cs | 79 ++++++++++++++++--- .../Jobs/Plex/PlexEpisodeSync.cs | 8 +- .../Settings/Models/JobSettingsHelper.cs | 4 +- src/Ombi/Startup.cs | 14 ++-- 7 files changed, 139 insertions(+), 31 deletions(-) diff --git a/src/Ombi.Schedule/Jobs/Ombi/IRefreshMetadata.cs b/src/Ombi.Schedule/Jobs/Ombi/IRefreshMetadata.cs index a08db74d0..ed13280b0 100644 --- a/src/Ombi.Schedule/Jobs/Ombi/IRefreshMetadata.cs +++ b/src/Ombi.Schedule/Jobs/Ombi/IRefreshMetadata.cs @@ -1,9 +1,11 @@ -using System.Threading.Tasks; +using System.Collections.Generic; +using System.Threading.Tasks; namespace Ombi.Schedule.Jobs.Ombi { public interface IRefreshMetadata : IBaseJob { Task Start(); + Task ProcessPlexServerContent(IEnumerable contentIds); } } \ No newline at end of file diff --git a/src/Ombi.Schedule/Jobs/Ombi/RefreshMetadata.cs b/src/Ombi.Schedule/Jobs/Ombi/RefreshMetadata.cs index 9b7726a15..4eae94666 100644 --- a/src/Ombi.Schedule/Jobs/Ombi/RefreshMetadata.cs +++ b/src/Ombi.Schedule/Jobs/Ombi/RefreshMetadata.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -17,7 +18,7 @@ namespace Ombi.Schedule.Jobs.Ombi { public RefreshMetadata(IPlexContentRepository plexRepo, IEmbyContentRepository embyRepo, ILogger log, ITvMazeApi tvApi, ISettingsService plexSettings, - IMovieDbApi movieApi) + IMovieDbApi movieApi, ISettingsService embySettings) { _plexRepo = plexRepo; _embyRepo = embyRepo; @@ -25,6 +26,7 @@ namespace Ombi.Schedule.Jobs.Ombi _movieApi = movieApi; _tvApi = tvApi; _plexSettings = plexSettings; + _embySettings = embySettings; } private readonly IPlexContentRepository _plexRepo; @@ -33,6 +35,7 @@ namespace Ombi.Schedule.Jobs.Ombi private readonly IMovieDbApi _movieApi; private readonly ITvMazeApi _tvApi; private readonly ISettingsService _plexSettings; + private readonly ISettingsService _embySettings; public async Task Start() { @@ -43,6 +46,11 @@ namespace Ombi.Schedule.Jobs.Ombi if (settings.Enable) { await StartPlex(); + } + + var embySettings = await _embySettings.GetSettingsAsync(); + if (embySettings.Enable) + { await StartEmby(); } } @@ -53,12 +61,45 @@ namespace Ombi.Schedule.Jobs.Ombi } } - private async Task StartPlex() + public async Task ProcessPlexServerContent(IEnumerable contentIds) { - await StartPlexMovies(); + _log.LogInformation("Starting the Metadata refresh from RecentlyAddedSync"); + try + { + var settings = await _plexSettings.GetSettingsAsync(); + if (settings.Enable) + { + await StartPlexWithKnownContent(contentIds); + } + } + catch (Exception e) + { + _log.LogError(e, "Exception when refreshing the Plex Metadata"); + throw; + } + } + + private async Task StartPlexWithKnownContent(IEnumerable contentids) + { + var everything = _plexRepo.GetAll().Where(x => contentids.Contains(x.Id)); + var allMovies = everything.Where(x => x.Type == PlexMediaTypeEntity.Movie); + await StartPlexMovies(allMovies); // Now Tv - await StartPlexTv(); + var allTv = everything.Where(x => x.Type == PlexMediaTypeEntity.Show); + await StartPlexTv(allTv); + } + + private async Task StartPlex() + { + var allMovies = _plexRepo.GetAll().Where(x => + x.Type == PlexMediaTypeEntity.Movie && (!x.TheMovieDbId.HasValue() || !x.ImdbId.HasValue())); + await StartPlexMovies(allMovies); + + // Now Tv + var allTv = _plexRepo.GetAll().Where(x => + x.Type == PlexMediaTypeEntity.Show && (!x.TheMovieDbId.HasValue() || !x.ImdbId.HasValue() || !x.TvDbId.HasValue())); + await StartPlexTv(allTv); } private async Task StartEmby() @@ -67,10 +108,8 @@ namespace Ombi.Schedule.Jobs.Ombi await StartEmbyTv(); } - private async Task StartPlexTv() + private async Task StartPlexTv(IQueryable allTv) { - var allTv = _plexRepo.GetAll().Where(x => - x.Type == PlexMediaTypeEntity.Show && (!x.TheMovieDbId.HasValue() || !x.ImdbId.HasValue() || !x.TvDbId.HasValue())); var tvCount = 0; foreach (var show in allTv) { @@ -147,10 +186,8 @@ namespace Ombi.Schedule.Jobs.Ombi await _embyRepo.SaveChangesAsync(); } - private async Task StartPlexMovies() + private async Task StartPlexMovies(IQueryable allMovies) { - var allMovies = _plexRepo.GetAll().Where(x => - x.Type == PlexMediaTypeEntity.Movie && (!x.TheMovieDbId.HasValue() || !x.ImdbId.HasValue())); int movieCount = 0; foreach (var movie in allMovies) { diff --git a/src/Ombi.Schedule/Jobs/Plex/Interfaces/IPlexEpisodeSync.cs b/src/Ombi.Schedule/Jobs/Plex/Interfaces/IPlexEpisodeSync.cs index 7d97381be..ede393790 100644 --- a/src/Ombi.Schedule/Jobs/Plex/Interfaces/IPlexEpisodeSync.cs +++ b/src/Ombi.Schedule/Jobs/Plex/Interfaces/IPlexEpisodeSync.cs @@ -1,10 +1,14 @@ using System; +using System.Linq; using System.Threading.Tasks; +using Ombi.Api.Plex.Models; +using Ombi.Store.Entities; namespace Ombi.Schedule.Jobs.Plex.Interfaces { public interface IPlexEpisodeSync : IBaseJob { Task Start(); + Task ProcessEpsiodes(Metadata[] episodes, IQueryable currentEpisodes); } } \ No newline at end of file diff --git a/src/Ombi.Schedule/Jobs/Plex/PlexContentSync.cs b/src/Ombi.Schedule/Jobs/Plex/PlexContentSync.cs index 4593b7ea1..95cd3ba38 100644 --- a/src/Ombi.Schedule/Jobs/Plex/PlexContentSync.cs +++ b/src/Ombi.Schedule/Jobs/Plex/PlexContentSync.cs @@ -77,11 +77,22 @@ namespace Ombi.Schedule.Jobs.Plex Logger.LogError("Plex Settings are not valid"); return; } - + var processedContent = new HashSet(); Logger.LogInformation("Starting Plex Content Cacher"); try { - await StartTheCache(plexSettings, recentlyAddedSearch); + if (recentlyAddedSearch) + { + var result = await StartTheCache(plexSettings, true); + foreach (var r in result) + { + processedContent.Add(r); + } + } + else + { + await StartTheCache(plexSettings, false); + } } catch (Exception e) { @@ -94,33 +105,58 @@ namespace Ombi.Schedule.Jobs.Plex BackgroundJob.Enqueue(() => EpisodeSync.Start()); } - BackgroundJob.Enqueue(() => Metadata.Start()); + if (processedContent.Any() && recentlyAddedSearch) + { + // Just check what we send it + BackgroundJob.Enqueue(() => Metadata.ProcessPlexServerContent(processedContent)); + } } - private async Task StartTheCache(PlexSettings plexSettings, bool recentlyAddedSearch) + private async Task> StartTheCache(PlexSettings plexSettings, bool recentlyAddedSearch) { + var processedContent = new HashSet(); foreach (var servers in plexSettings.Servers ?? new List()) { try { Logger.LogInformation("Starting to cache the content on server {0}", servers.Name); - await ProcessServer(servers, recentlyAddedSearch); + + if (recentlyAddedSearch) + { + // If it's recently added search then we want the results to pass to the metadata job + // This way the metadata job is smaller in size to process, it only need to look at newly added shit + var result = await ProcessServer(servers, true); + foreach (var plexServerContent in result) + { + processedContent.Add(plexServerContent); + } + } + else + { + await ProcessServer(servers, false); + } } catch (Exception e) { Logger.LogWarning(LoggingEvents.PlexContentCacher, e, "Exception thrown when attempting to cache the Plex Content in server {0}", servers.Name); } } + + return processedContent; } - private async Task ProcessServer(PlexServers servers, bool recentlyAddedSearch) + private async Task> ProcessServer(PlexServers servers, bool recentlyAddedSearch) { + var processedContent = new HashSet(); Logger.LogInformation("Getting all content from server {0}", servers.Name); var allContent = await GetAllContent(servers, recentlyAddedSearch); Logger.LogInformation("We found {0} items", allContent.Count); // Let's now process this. var contentToAdd = new HashSet(); + + var allEps = Repo.GetAllEpisodes(); + foreach (var content in allContent) { if (content.viewGroup.Equals(PlexMediaType.Episode.ToString(), StringComparison.CurrentCultureIgnoreCase)) @@ -137,8 +173,10 @@ namespace Ombi.Schedule.Jobs.Plex continue; } - await ProcessTvShow(servers, show, contentToAdd, recentlyAddedSearch); + await ProcessTvShow(servers, show, contentToAdd, recentlyAddedSearch, processedContent); } + + await EpisodeSync.ProcessEpsiodes(content.Metadata, allEps); } if (content.viewGroup.Equals(PlexMediaType.Show.ToString(), StringComparison.CurrentCultureIgnoreCase)) { @@ -146,7 +184,7 @@ namespace Ombi.Schedule.Jobs.Plex Logger.LogInformation("Processing TV Shows"); foreach (var show in content.Metadata ?? new Metadata[] { }) { - await ProcessTvShow(servers, show, contentToAdd, recentlyAddedSearch); + await ProcessTvShow(servers, show, contentToAdd, recentlyAddedSearch, processedContent); } } if (content.viewGroup.Equals(PlexMediaType.Movie.ToString(), StringComparison.CurrentCultureIgnoreCase)) @@ -216,6 +254,10 @@ namespace Ombi.Schedule.Jobs.Plex if (contentToAdd.Count > 500) { await Repo.AddRange(contentToAdd); + foreach (var c in contentToAdd) + { + processedContent.Add(c.Id); + } contentToAdd.Clear(); } } @@ -223,6 +265,10 @@ namespace Ombi.Schedule.Jobs.Plex if (contentToAdd.Count > 500) { await Repo.AddRange(contentToAdd); + foreach (var c in contentToAdd) + { + processedContent.Add(c.Id); + } contentToAdd.Clear(); } } @@ -230,10 +276,16 @@ namespace Ombi.Schedule.Jobs.Plex if (contentToAdd.Any()) { await Repo.AddRange(contentToAdd); + foreach (var c in contentToAdd) + { + processedContent.Add(c.Id); + } } + + return processedContent; } - private async Task ProcessTvShow(PlexServers servers, Metadata show, HashSet contentToAdd, bool recentlyAdded) + private async Task ProcessTvShow(PlexServers servers, Metadata show, HashSet contentToAdd, bool recentlyAdded, HashSet contentProcessed) { var seasonList = await PlexApi.GetSeasons(servers.PlexAuthToken, servers.FullUri, show.ratingKey); @@ -416,8 +468,17 @@ namespace Ombi.Schedule.Jobs.Plex if (contentToAdd.Count > 500 || recentlyAdded) { await Repo.AddRange(contentToAdd); + foreach (var plexServerContent in contentToAdd) + { + contentProcessed.Add(plexServerContent.Id); + } contentToAdd.Clear(); } + + if (contentToAdd.Any()) + { + await Repo.AddRange(contentToAdd); + } } /// diff --git a/src/Ombi.Schedule/Jobs/Plex/PlexEpisodeSync.cs b/src/Ombi.Schedule/Jobs/Plex/PlexEpisodeSync.cs index 8e8cce411..d98eace4a 100644 --- a/src/Ombi.Schedule/Jobs/Plex/PlexEpisodeSync.cs +++ b/src/Ombi.Schedule/Jobs/Plex/PlexEpisodeSync.cs @@ -111,7 +111,7 @@ namespace Ombi.Schedule.Jobs.Plex // 12.03.2017 - I think we should be able to match them now //await _repo.ExecuteSql("DELETE FROM PlexEpisode"); - await ProcessEpsiodes(episodes, currentEpisodes); + await ProcessEpsiodes(episodes?.MediaContainer?.Metadata ?? new Metadata[] { }, currentEpisodes); currentPosition += resultCount; while (currentPosition < episodes.MediaContainer.totalSize) @@ -119,7 +119,7 @@ namespace Ombi.Schedule.Jobs.Plex var ep = await _api.GetAllEpisodes(settings.PlexAuthToken, settings.FullUri, section.key, currentPosition, resultCount); - await ProcessEpsiodes(ep, currentEpisodes); + await ProcessEpsiodes(ep?.MediaContainer?.Metadata ?? new Metadata[] { }, currentEpisodes); _log.LogInformation(LoggingEvents.PlexEpisodeCacher, $"Processed {resultCount} more episodes. Total Remaining {episodes.MediaContainer.totalSize - currentPosition}"); currentPosition += resultCount; } @@ -129,12 +129,12 @@ namespace Ombi.Schedule.Jobs.Plex await _repo.SaveChangesAsync(); } - private async Task ProcessEpsiodes(PlexContainer episodes, IQueryable currentEpisodes) + public async Task ProcessEpsiodes(Metadata[] episodes, IQueryable currentEpisodes) { var ep = new HashSet(); try { - foreach (var episode in episodes?.MediaContainer?.Metadata ?? new Metadata[] { }) + foreach (var episode in episodes) { // I don't think we need to get the metadata, we only need to get the metadata if we need the provider id (TheTvDbid). Why do we need it for episodes? // We have the parent and grandparent rating keys to link up to the season and series diff --git a/src/Ombi.Settings/Settings/Models/JobSettingsHelper.cs b/src/Ombi.Settings/Settings/Models/JobSettingsHelper.cs index 2e56f17db..a5afbeba7 100644 --- a/src/Ombi.Settings/Settings/Models/JobSettingsHelper.cs +++ b/src/Ombi.Settings/Settings/Models/JobSettingsHelper.cs @@ -25,7 +25,7 @@ namespace Ombi.Settings.Settings.Models } public static string PlexRecentlyAdded(JobSettings s) { - return Get(s.PlexRecentlyAddedSync, Cron.Hourly(0)); + return Get(s.PlexRecentlyAddedSync, Cron.MinuteInterval(30)); } public static string CouchPotato(JobSettings s) { @@ -50,7 +50,7 @@ namespace Ombi.Settings.Settings.Models } public static string RefreshMetadata(JobSettings s) { - return Get(s.RefreshMetadata, Cron.DayInterval(2)); + return Get(s.RefreshMetadata, Cron.DayInterval(3)); } private static string Get(string settings, string defaultCron) diff --git a/src/Ombi/Startup.cs b/src/Ombi/Startup.cs index fc80be838..4b373d457 100644 --- a/src/Ombi/Startup.cs +++ b/src/Ombi/Startup.cs @@ -191,11 +191,15 @@ namespace Ombi } app.UseHangfireServer(new BackgroundJobServerOptions { WorkerCount = 1, ServerTimeout = TimeSpan.FromDays(1), ShutdownTimeout = TimeSpan.FromDays(1)}); - app.UseHangfireDashboard(settings.BaseUrl.HasValue() ? $"{settings.BaseUrl}/hangfire" : "/hangfire", - new DashboardOptions - { - Authorization = new[] { new HangfireAuthorizationFilter() } - }); + if (env.IsDevelopment()) + { + app.UseHangfireDashboard(settings.BaseUrl.HasValue() ? $"{settings.BaseUrl}/hangfire" : "/hangfire", + new DashboardOptions + { + Authorization = new[] {new HangfireAuthorizationFilter()} + }); + } + GlobalJobFilters.Filters.Add(new AutomaticRetryAttribute { Attempts = 3 }); // Setup the scheduler