github/Ombi
1
0
Fork 0
mirror of https://github.com/Ombi-app/Ombi.git synced 2025-08-14 02:26:55 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Made a massive improvement on the Smaller more frequent Plex Job. This should pick up content a lot quicker now and also get their metadata a lot quicker.

Browse source
This commit is contained in:
Jamie Rees 2018-05-01 09:00:51 +01:00
commit 5529a049d6
4 changed files with 331 additions and 298 deletions
Download patch file Download diff file Expand all files Collapse all files

8
src/Ombi.Api.Plex/Models/Metadata.cs
View file

@ -11,15 +11,15 @@ namespace Ombi.Api.Plex.Models
public string summary { get; set; }
public int index { get; set; }
public float rating { get; set; }
public int viewCount { get; set; }
public int lastViewedAt { get; set; }
//public int viewCount { get; set; }
//public int lastViewedAt { get; set; }
public int year { get; set; }
public string thumb { get; set; }
public string art { get; set; }
public string banner { get; set; }
public string theme { get; set; }
public string duration { get; set; }
public string originallyAvailableAt { get; set; }
//public string duration { get; set; }
//public string originallyAvailableAt { get; set; }
public int leafCount { get; set; }
public int viewedLeafCount { get; set; }
public int childCount { get; set; }

380
src/Ombi.Schedule/Jobs/Plex/PlexContentSync.cs
View file

@ -119,187 +119,30 @@ namespace Ombi.Schedule.Jobs.Plex
var contentToAdd = new HashSet<PlexServerContent>();
foreach (var content in allContent)
{
if (content.viewGroup.Equals(PlexMediaType.Episode.ToString(), StringComparison.CurrentCultureIgnoreCase))
{
Logger.LogInformation("Found some episodes, this must be a recently added sync");
foreach (var epInfo in content.Metadata)
{
var grandParentKey = epInfo.grandparentRatingKey;
// Lookup the rating key
var showMetadata = await PlexApi.GetMetadata(servers.PlexAuthToken, servers.FullUri, grandParentKey);
var show = showMetadata.MediaContainer.Metadata.FirstOrDefault();
if(show == null)
{
continue;
}
await ProcessTvShow(servers, show, contentToAdd, recentlyAddedSearch);
}
}
if (content.viewGroup.Equals(PlexMediaType.Show.ToString(), StringComparison.CurrentCultureIgnoreCase))
{
// Process Shows
Logger.LogInformation("Processing TV Shows");
foreach (var show in content.Metadata ?? new Metadata[] { })
{
var seasonList = await PlexApi.GetSeasons(servers.PlexAuthToken, servers.FullUri,
show.ratingKey);
var seasonsContent = new List<PlexSeasonsContent>();
foreach (var season in seasonList.MediaContainer.Metadata)
{
seasonsContent.Add(new PlexSeasonsContent
{
ParentKey = season.parentRatingKey,
SeasonKey = season.ratingKey,
SeasonNumber = season.index,
PlexContentId = show.ratingKey
});
}
// Do we already have this item?
// Let's try and match
var existingContent = await Repo.GetFirstContentByCustom(x => x.Title == show.title
&& x.ReleaseYear == show.year.ToString()
&& x.Type == PlexMediaTypeEntity.Show);
// Just double check the rating key, since this is our unique constraint
var existingKey = await Repo.GetByKey(show.ratingKey);
if (existingKey != null)
{
// Damn son.
// Let's check if they match up
var doesMatch = show.title.Equals(existingKey.Title,
StringComparison.CurrentCulture);
if (!doesMatch)
{
// Something fucked up on Plex at somepoint... Damn, rebuild of lib maybe?
// Lets delete the matching key
await Repo.Delete(existingKey);
existingKey = null;
}
}
if (existingContent != null)
{
// Just check the key
if (existingKey != null)
{
// The rating key is all good!
}
else
{
// This means the rating key has changed somehow.
// Should probably delete this and get the new one
var oldKey = existingContent.Key;
Repo.DeleteWithoutSave(existingContent);
// Because we have changed the rating key, we need to change all children too
var episodeToChange = Repo.GetAllEpisodes().Where(x => x.GrandparentKey == oldKey);
if (episodeToChange.Any())
{
foreach (var e in episodeToChange)
{
Repo.DeleteWithoutSave(e);
}
}
await Repo.SaveChangesAsync();
existingContent = null;
}
}
// The ratingKey keeps changing...
//var existingContent = await Repo.GetByKey(show.ratingKey);
if (existingContent != null)
{
try
{
Logger.LogInformation("We already have show {0} checking for new seasons",
existingContent.Title);
// Ok so we have it, let's check if there are any new seasons
var itemAdded = false;
foreach (var season in seasonsContent)
{
var seasonExists =
existingContent.Seasons.FirstOrDefault(x => x.SeasonKey == season.SeasonKey);
if (seasonExists != null)
{
// We already have this season
continue;
}
existingContent.Seasons.Add(season);
itemAdded = true;
}
if (itemAdded) await Repo.Update(existingContent);
}
catch (Exception e)
{
Logger.LogError(LoggingEvents.PlexContentCacher, e,
"Exception when adding new seasons to title {0}", existingContent.Title);
}
}
else
{
try
{
Logger.LogInformation("New show {0}, so add it", show.title);
// Get the show metadata... This sucks since the `metadata` var contains all information about the show
// But it does not contain the `guid` property that we need to pull out thetvdb id...
var showMetadata = await PlexApi.GetMetadata(servers.PlexAuthToken, servers.FullUri,
show.ratingKey);
var providerIds =
PlexHelper.GetProviderIdFromPlexGuid(showMetadata.MediaContainer.Metadata.FirstOrDefault()
.guid);
var item = new PlexServerContent
{
AddedAt = DateTime.Now,
Key = show.ratingKey,
ReleaseYear = show.year.ToString(),
Type = PlexMediaTypeEntity.Show,
Title = show.title,
Url = PlexHelper.GetPlexMediaUrl(servers.MachineIdentifier, show.ratingKey),
Seasons = new List<PlexSeasonsContent>()
};
if (providerIds.Type == ProviderType.ImdbId)
{
item.ImdbId = providerIds.ImdbId;
}
if (providerIds.Type == ProviderType.TheMovieDbId)
{
item.TheMovieDbId = providerIds.TheMovieDb;
}
if (providerIds.Type == ProviderType.TvDbId)
{
item.TvDbId = providerIds.TheTvDb;
}
// Let's just double check to make sure we do not have it now we have some id's
var existingImdb = false;
var existingMovieDbId = false;
var existingTvDbId = false;
if (item.ImdbId.HasValue())
{
existingImdb = await Repo.GetAll().AnyAsync(x =>
x.ImdbId == item.ImdbId && x.Type == PlexMediaTypeEntity.Show);
}
if (item.TheMovieDbId.HasValue())
{
existingMovieDbId = await Repo.GetAll().AnyAsync(x =>
x.TheMovieDbId == item.TheMovieDbId && x.Type == PlexMediaTypeEntity.Show);
}
if (item.TvDbId.HasValue())
{
existingTvDbId = await Repo.GetAll().AnyAsync(x =>
x.TvDbId == item.TvDbId && x.Type == PlexMediaTypeEntity.Show);
}
if (existingImdb || existingTvDbId || existingMovieDbId)
{
// We already have it!
continue;
}
item.Seasons.ToList().AddRange(seasonsContent);
contentToAdd.Add(item);
}
catch (Exception e)
{
Logger.LogError(LoggingEvents.PlexContentCacher, e, "Exception when adding tv show {0}",
show.title);
}
}
if (contentToAdd.Count > 500)
{
await Repo.AddRange(contentToAdd);
contentToAdd.Clear();
}
await ProcessTvShow(servers, show, contentToAdd, recentlyAddedSearch);
}
}
if (content.viewGroup.Equals(PlexMediaType.Movie.ToString(), StringComparison.CurrentCultureIgnoreCase))
@ -386,6 +229,193 @@ namespace Ombi.Schedule.Jobs.Plex
}
}
private async Task ProcessTvShow(PlexServers servers, Metadata show, HashSet<PlexServerContent> contentToAdd, bool recentlyAdded)
{
var seasonList = await PlexApi.GetSeasons(servers.PlexAuthToken, servers.FullUri,
show.ratingKey);
var seasonsContent = new List<PlexSeasonsContent>();
foreach (var season in seasonList.MediaContainer.Metadata)
{
seasonsContent.Add(new PlexSeasonsContent
{
ParentKey = season.parentRatingKey,
SeasonKey = season.ratingKey,
SeasonNumber = season.index,
PlexContentId = show.ratingKey
});
}
// Do we already have this item?
// Let's try and match
var existingContent = await Repo.GetFirstContentByCustom(x => x.Title == show.title
&& x.ReleaseYear == show.year.ToString()
&& x.Type == PlexMediaTypeEntity.Show);
// Just double check the rating key, since this is our unique constraint
var existingKey = await Repo.GetByKey(show.ratingKey);
if (existingKey != null)
{
// Damn son.
// Let's check if they match up
var doesMatch = show.title.Equals(existingKey.Title,
StringComparison.CurrentCulture);
if (!doesMatch)
{
// Something fucked up on Plex at somepoint... Damn, rebuild of lib maybe?
// Lets delete the matching key
await Repo.Delete(existingKey);
existingKey = null;
}
}
if (existingContent != null)
{
// Just check the key
if (existingKey != null)
{
// The rating key is all good!
}
else
{
// This means the rating key has changed somehow.
// Should probably delete this and get the new one
var oldKey = existingContent.Key;
Repo.DeleteWithoutSave(existingContent);
// Because we have changed the rating key, we need to change all children too
var episodeToChange = Repo.GetAllEpisodes().Where(x => x.GrandparentKey == oldKey);
if (episodeToChange.Any())
{
foreach (var e in episodeToChange)
{
Repo.DeleteWithoutSave(e);
}
}
await Repo.SaveChangesAsync();
existingContent = null;
}
}
// The ratingKey keeps changing...
//var existingContent = await Repo.GetByKey(show.ratingKey);
if (existingContent != null)
{
try
{
Logger.LogInformation("We already have show {0} checking for new seasons",
existingContent.Title);
// Ok so we have it, let's check if there are any new seasons
var itemAdded = false;
foreach (var season in seasonsContent)
{
var seasonExists =
existingContent.Seasons.FirstOrDefault(x => x.SeasonKey == season.SeasonKey);
if (seasonExists != null)
{
// We already have this season
continue;
}
existingContent.Seasons.Add(season);
itemAdded = true;
}
if (itemAdded) await Repo.Update(existingContent);
}
catch (Exception e)
{
Logger.LogError(LoggingEvents.PlexContentCacher, e,
"Exception when adding new seasons to title {0}", existingContent.Title);
}
}
else
{
try
{
Logger.LogInformation("New show {0}, so add it", show.title);
// Get the show metadata... This sucks since the `metadata` var contains all information about the show
// But it does not contain the `guid` property that we need to pull out thetvdb id...
var showMetadata = await PlexApi.GetMetadata(servers.PlexAuthToken, servers.FullUri,
show.ratingKey);
var providerIds =
PlexHelper.GetProviderIdFromPlexGuid(showMetadata.MediaContainer.Metadata.FirstOrDefault()
.guid);
var item = new PlexServerContent
{
AddedAt = DateTime.Now,
Key = show.ratingKey,
ReleaseYear = show.year.ToString(),
Type = PlexMediaTypeEntity.Show,
Title = show.title,
Url = PlexHelper.GetPlexMediaUrl(servers.MachineIdentifier, show.ratingKey),
Seasons = new List<PlexSeasonsContent>()
};
if (providerIds.Type == ProviderType.ImdbId)
{
item.ImdbId = providerIds.ImdbId;
}
if (providerIds.Type == ProviderType.TheMovieDbId)
{
item.TheMovieDbId = providerIds.TheMovieDb;
}
if (providerIds.Type == ProviderType.TvDbId)
{
item.TvDbId = providerIds.TheTvDb;
}
// Let's just double check to make sure we do not have it now we have some id's
var existingImdb = false;
var existingMovieDbId = false;
var existingTvDbId = false;
if (item.ImdbId.HasValue())
{
existingImdb = await Repo.GetAll().AnyAsync(x =>
x.ImdbId == item.ImdbId && x.Type == PlexMediaTypeEntity.Show);
}
if (item.TheMovieDbId.HasValue())
{
existingMovieDbId = await Repo.GetAll().AnyAsync(x =>
x.TheMovieDbId == item.TheMovieDbId && x.Type == PlexMediaTypeEntity.Show);
}
if (item.TvDbId.HasValue())
{
existingTvDbId = await Repo.GetAll().AnyAsync(x =>
x.TvDbId == item.TvDbId && x.Type == PlexMediaTypeEntity.Show);
}
if (existingImdb || existingTvDbId || existingMovieDbId)
{
// We already have it!
return;
}
item.Seasons.ToList().AddRange(seasonsContent);
contentToAdd.Add(item);
}
catch (Exception e)
{
Logger.LogError(LoggingEvents.PlexContentCacher, e, "Exception when adding tv show {0}",
show.title);
}
}
if (contentToAdd.Count > 500 || recentlyAdded)
{
await Repo.AddRange(contentToAdd);
contentToAdd.Clear();
}
}
/// <summary>
/// Gets all the library sections.
/// If the user has specified only certain libraries then we will only look for those

3
src/Ombi.Schedule/Jobs/Plex/PlexMediaType.cs
View file

@ -31,6 +31,7 @@ namespace Ombi.Schedule.Jobs
public enum PlexMediaType
{
Movie = 0,
Show = 1
Show = 1,
Episode = 2,
}
}

238
src/Ombi.Tests/IdentityControllerTests.cs
View file

@ -1,132 +1,134 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using AutoMapper;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Http.Features.Authentication;
using Microsoft.AspNetCore.Identity;
using Microsoft.AspNetCore.Identity.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using Moq;
using NUnit.Framework;
using Ombi.Api.Emby;
using Ombi.Api.Plex;
using Ombi.Config;
using Ombi.Controllers;
using Ombi.Core.Authentication;
using Ombi.Core.Settings;
using Ombi.Core.Settings.Models.External;
using Ombi.Models;
using Ombi.Notifications;
using Ombi.Schedule.Jobs.Ombi;
using Ombi.Settings.Settings.Models;
using Ombi.Settings.Settings.Models.Notifications;
using Ombi.Store.Context;
using Ombi.Store.Entities;
//using System.Collections.Generic;
//using System.Linq;
//using System.Threading;
//using System.Threading.Tasks;
//using AutoMapper;
//using Microsoft.AspNetCore.Http;
//using Microsoft.AspNetCore.Http.Features.Authentication;
//using Microsoft.AspNetCore.Identity;
//using Microsoft.AspNetCore.Identity.EntityFrameworkCore;
//using Microsoft.EntityFrameworkCore;
//using Microsoft.EntityFrameworkCore.Infrastructure;
//using Microsoft.Extensions.DependencyInjection;
//using Microsoft.Extensions.Options;
//using Moq;
//using NUnit.Framework;
//using Ombi.Api.Emby;
//using Ombi.Api.Plex;
//using Ombi.Config;
//using Ombi.Controllers;
//using Ombi.Core.Authentication;
//using Ombi.Core.Settings;
//using Ombi.Core.Settings.Models.External;
//using Ombi.Models;
//using Ombi.Notifications;
//using Ombi.Schedule.Jobs.Ombi;
//using Ombi.Settings.Settings.Models;
//using Ombi.Settings.Settings.Models.Notifications;
//using Ombi.Store.Context;
//using Ombi.Store.Entities;
namespace Ombi.Tests
{
[TestFixture]
[Ignore("Need to sort out the DB, looks like it's using the real one...")]
public class IdentityControllerTests
{
[SetUp]
public void Setup()
{
_plexApi = new Mock<IPlexApi>();
_embyApi = new Mock<IEmbyApi>();
_mapper = new Mock<IMapper>();
_emailProvider = new Mock<IEmailProvider>();
_emailSettings = new Mock<ISettingsService<EmailNotificationSettings>>();
_customizationSettings = new Mock<ISettingsService<CustomizationSettings>>();
_welcomeEmail = new Mock<IWelcomeEmail>();
_embySettings = new Mock<ISettingsService<EmbySettings>>();
_plexSettings = new Mock<ISettingsService<PlexSettings>>();
//namespace Ombi.Tests
//{
// [TestFixture]
// [Ignore("Need to sort out the DB, looks like it's using the real one...")]
// public class IdentityControllerTests
// {
// [SetUp]
// public void Setup()
// {
// _plexApi = new Mock<IPlexApi>();
// _embyApi = new Mock<IEmbyApi>();
// _mapper = new Mock<IMapper>();
// _emailProvider = new Mock<IEmailProvider>();
// _emailSettings = new Mock<ISettingsService<EmailNotificationSettings>>();
// _customizationSettings = new Mock<ISettingsService<CustomizationSettings>>();
// _welcomeEmail = new Mock<IWelcomeEmail>();
// _embySettings = new Mock<ISettingsService<EmbySettings>>();
// _plexSettings = new Mock<ISettingsService<PlexSettings>>();
var services = new ServiceCollection();
services.AddEntityFrameworkInMemoryDatabase()
.AddDbContext<OmbiContext>();
services.AddIdentity<OmbiUser, IdentityRole>()
.AddEntityFrameworkStores<OmbiContext>().AddUserManager<OmbiUserManager>();
// var services = new ServiceCollection();
// services.AddEntityFrameworkInMemoryDatabase()
// .AddDbContext<OmbiContext>();
// services.AddIdentity<OmbiUser, IdentityRole>()
// .AddEntityFrameworkStores<OmbiContext>().AddUserManager<OmbiUserManager>();
services.AddTransient(x => _plexApi.Object);
services.AddTransient(x => _embyApi.Object);
services.AddTransient(x => _customizationSettings.Object);
services.AddTransient(x => _welcomeEmail.Object);
services.AddTransient(x => _emailSettings.Object);
services.AddTransient(x => _emailProvider.Object);
services.AddTransient(x => _mapper.Object);
services.AddTransient(x => _embySettings.Object);
services.AddTransient(x => _plexSettings.Object);
// services.AddTransient(x => _plexApi.Object);
// services.AddTransient(x => _embyApi.Object);
// services.AddTransient(x => _customizationSettings.Object);
// services.AddTransient(x => _welcomeEmail.Object);
// services.AddTransient(x => _emailSettings.Object);
// services.AddTransient(x => _emailProvider.Object);
// services.AddTransient(x => _mapper.Object);
// services.AddTransient(x => _embySettings.Object);
// services.AddTransient(x => _plexSettings.Object);
services.Configure<IdentityOptions>(options =>
{
options.Password.RequireDigit = false;
options.Password.RequiredLength = 1;
options.Password.RequireLowercase = false;
options.Password.RequireNonAlphanumeric = false;
options.Password.RequireUppercase = false;
options.User.AllowedUserNameCharacters = string.Empty;
});
// services.Configure<IdentityOptions>(options =>
// {
// options.Password.RequireDigit = false;
// options.Password.RequiredLength = 1;
// options.Password.RequireLowercase = false;
// options.Password.RequireNonAlphanumeric = false;
// options.Password.RequireUppercase = false;
// options.User.AllowedUserNameCharacters = string.Empty;
// });
// Taken from https://github.com/aspnet/MusicStore/blob/dev/test/MusicStore.Test/ManageControllerTest.cs (and modified)
var context = new DefaultHttpContext();
context.Features.Set<IHttpAuthenticationFeature>(new HttpAuthenticationFeature());
services.AddSingleton<IHttpContextAccessor>(h => new HttpContextAccessor { HttpContext = context });
_serviceProvider = services.BuildServiceProvider();
_userManager = _serviceProvider.GetRequiredService<OmbiUserManager>();
Controller = new IdentityController(_userManager, _mapper.Object, _serviceProvider.GetService<RoleManager<IdentityRole>>(), _emailProvider.Object,
_emailSettings.Object, _customizationSettings.Object,_welcomeEmail.Object, null, null, null, null, null, null, null, null);
}
// // Taken from https://github.com/aspnet/MusicStore/blob/dev/test/MusicStore.Test/ManageControllerTest.cs (and modified)
// var context = new DefaultHttpContext();
// context.Features.Set<IHttpAuthenticationFeature>(new HttpAuthenticationFeature());
// services.AddSingleton<IHttpContextAccessor>(h => new HttpContextAccessor { HttpContext = context });
// _serviceProvider = services.BuildServiceProvider();
// _userManager = _serviceProvider.GetRequiredService<OmbiUserManager>();
private OmbiUserManager _userManager;
private Mock<IEmailProvider> _emailProvider;
private Mock<ISettingsService<EmailNotificationSettings>> _emailSettings;
private Mock<ISettingsService<CustomizationSettings>> _customizationSettings;
private Mock<ISettingsService<EmbySettings>> _embySettings;
private Mock<ISettingsService<PlexSettings>> _plexSettings;
private Mock<IWelcomeEmail> _welcomeEmail;
private Mock<IMapper> _mapper;
private Mock<IPlexApi> _plexApi;
private Mock<IEmbyApi> _embyApi;
private ServiceProvider _serviceProvider;
// Controller = new IdentityController(_userManager, _mapper.Object,
// _serviceProvider.GetService<RoleManager<IdentityRole>>(), _emailProvider.Object,
// _emailSettings.Object, _customizationSettings.Object, _welcomeEmail.Object, null, null, null, null,
// null, null, null, null, null);
// }
private IdentityController Controller { get; set; }
// private OmbiUserManager _userManager;
// private Mock<IEmailProvider> _emailProvider;
// private Mock<ISettingsService<EmailNotificationSettings>> _emailSettings;
// private Mock<ISettingsService<CustomizationSettings>> _customizationSettings;
// private Mock<ISettingsService<EmbySettings>> _embySettings;
// private Mock<ISettingsService<PlexSettings>> _plexSettings;
// private Mock<IWelcomeEmail> _welcomeEmail;
// private Mock<IMapper> _mapper;
// private Mock<IPlexApi> _plexApi;
// private Mock<IEmbyApi> _embyApi;
// private ServiceProvider _serviceProvider;
[Test]
public async Task CreateWizardUser_Should_CreateUser_WhenThereAreNoOtherUsers()
{
var model = new CreateUserWizardModel()
{
Password = "a",
Username = "b"
};
// private IdentityController Controller { get; set; }
var result = await Controller.CreateWizardUser(model);
// [Test]
// public async Task CreateWizardUser_Should_CreateUser_WhenThereAreNoOtherUsers()
// {
// var model = new CreateUserWizardModel()
// {
// Password = "a",
// Username = "b"
// };
Assert.That(result, Is.True);
}
// var result = await Controller.CreateWizardUser(model);
[Test]
public async Task CreateWizardUser_ShouldNot_CreateUser_WhenThereAreOtherUsers()
{
var um = _serviceProvider.GetService<OmbiUserManager>();
var r = await um.CreateAsync(new OmbiUser { UserName = "aaaa",UserType = UserType.LocalUser}, "bbb");
var model = new CreateUserWizardModel
{
Password = "a",
Username = "b"
};
// Assert.That(result, Is.True);
// }
var result = await Controller.CreateWizardUser(model);
// [Test]
// public async Task CreateWizardUser_ShouldNot_CreateUser_WhenThereAreOtherUsers()
// {
// var um = _serviceProvider.GetService<OmbiUserManager>();
// var r = await um.CreateAsync(new OmbiUser { UserName = "aaaa",UserType = UserType.LocalUser}, "bbb");
// var model = new CreateUserWizardModel
// {
// Password = "a",
// Username = "b"
// };
Assert.That(result, Is.False);
}
}
}
// var result = await Controller.CreateWizardUser(model);
// Assert.That(result, Is.False);
// }
// }
//}