From cb7a7ec24f98045f63311513b5ad8dd5e9413590 Mon Sep 17 00:00:00 2001 From: zhangdoa Date: Thu, 12 Oct 2023 00:48:34 +0200 Subject: [PATCH] Refactor the cue sheet file loader to read the track titles, disc ID and other useful information. Add support to import releases with multiple cue sheets. Add the cue sheet support to the disc scan service. Use the track info from cue sheet files to map local tracks. Use the disc ID to group cue sheet files and deduce the disc count. (cherry picked from commit fac76b7cfb746e05f9924047e698ef41203efe5e) --- src/NzbDrone.Core/MediaFiles/CueSheet.cs | 207 +++++++++++---- .../MediaFiles/DiskScanService.cs | 89 +++++-- .../Aggregators/AggregateFilenameInfo.cs | 2 - .../Identification/IdentificationService.cs | 55 ++-- .../TrackImport/ImportDecisionMaker.cs | 109 +++++++- .../TrackImport/Manual/ManualImportService.cs | 243 ++++++++++-------- 6 files changed, 504 insertions(+), 201 deletions(-) diff --git a/src/NzbDrone.Core/MediaFiles/CueSheet.cs b/src/NzbDrone.Core/MediaFiles/CueSheet.cs index a22cb6c0a..84374635e 100644 --- a/src/NzbDrone.Core/MediaFiles/CueSheet.cs +++ b/src/NzbDrone.Core/MediaFiles/CueSheet.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.IO.Abstractions; -using System.Linq; using System.Text; using System.Text.RegularExpressions; using NzbDrone.Core.Datastore; @@ -23,68 +22,180 @@ namespace NzbDrone.Core.MediaFiles { content = encoding.GetString(bytes); var lines = content.Split(new string[] { Environment.NewLine }, StringSplitOptions.None); + ParseCueSheet(lines); - // Single-file cue means it's an unsplit image - FileNames = ReadField(lines, "FILE"); - IsSingleFileRelease = FileNames.Count == 1; - - var performers = ReadField(lines, "PERFORMER"); - if (performers.Count > 0) - { - Performer = performers[0]; - } - - var titles = ReadField(lines, "TITLE"); - if (titles.Count > 0) - { - Title = titles[0]; - } - - var dates = ReadField(lines, "REM DATE"); - if (dates.Count > 0) - { - Date = dates[0]; - } + // Single-file cue means it's an unsplit image, which should be specially treated in the pipeline + IsSingleFileRelease = Files.Count == 1; } } } + public class IndexEntry + { + public int Key { get; set; } + public string Time { get; set; } + } + + public class TrackEntry + { + public int Number { get; set; } + public string Title { get; set; } + public string Performer { get; set; } + public List Indices { get; set; } = new List(); + } + + public class FileEntry + { + public string Name { get; set; } + public IndexEntry Index { get; set; } + public List Tracks { get; set; } = new List(); + } + public string Path { get; set; } public bool IsSingleFileRelease { get; set; } - public List FileNames { get; set; } + public List Files { get; set; } = new List(); + public string Genre { get; set; } + public string Date { get; set; } + public string DiscID { get; set; } public string Title { get; set; } public string Performer { get; set; } - public string Date { get; set; } + private static string _FileKey = "FILE"; + private static string _TrackKey = "TRACK"; + private static string _IndexKey = "INDEX"; + private static string _GenreKey = "REM GENRE"; + private static string _DateKey = "REM DATE"; + private static string _DiscIdKey = "REM DISCID"; + private static string _PerformerKey = "PERFORMER"; + private static string _TitleKey = "TITLE"; - private static List ReadField(string[] lines, string fieldName) + private string ExtractValue(string line, string keyword) { - var inQuotePattern = "\"(.*?)\""; - var flatPattern = fieldName + " (.+)"; + var pattern = keyword + @"\s+(?:(?:\""(.*?)\"")|(.+))"; + var match = Regex.Match(line, pattern); - var results = new List(); - var candidates = lines.Where(l => l.StartsWith(fieldName)).ToList(); - foreach (var candidate in candidates) + if (match.Success) { - var matches = Regex.Matches(candidate, inQuotePattern).ToList(); - if (matches.Count == 0) - { - matches = Regex.Matches(candidate, flatPattern).ToList(); - } - - if (matches.Count == 0) - { - continue; - } - - var groups = matches[0].Groups; - if (groups.Count > 0) - { - var result = groups[1].Value; - results.Add(result); - } + var value = match.Groups[1].Success ? match.Groups[1].Value : match.Groups[2].Value; + return value; } - return results; + return ""; + } + + private void ParseCueSheet(string[] lines) + { + var i = 0; + try + { + while (true) + { + var line = lines[i]; + if (line.StartsWith(_FileKey)) + { + line = line.Trim(); + line = line.Substring(_FileKey.Length).Trim(); + var filename = line.Split('"')[1]; + var fileDetails = new FileEntry { Name = filename }; + + i++; + line = lines[i]; + while (line.StartsWith(" ")) + { + line = line.Trim(); + if (line.StartsWith(_TrackKey)) + { + line = line.Substring(_TrackKey.Length).Trim(); + } + + var trackDetails = new TrackEntry(); + var trackInfo = line.Split(' '); + if (trackInfo.Length > 0) + { + if (int.TryParse(trackInfo[0], out var number)) + { + trackDetails.Number = number; + } + } + + i++; + line = lines[i]; + while (line.StartsWith(" ")) + { + line = line.Trim(); + if (line.StartsWith(_IndexKey)) + { + line = line.Substring(_IndexKey.Length).Trim(); + var parts = line.Split(' '); + if (parts.Length > 1) + { + if (int.TryParse(parts[0], out var key)) + { + var value = parts[1].Trim('"'); + trackDetails.Indices.Add(new IndexEntry { Key = key, Time = value }); + } + } + + i++; + line = lines[i]; + } + else if (line.StartsWith(_TitleKey)) + { + trackDetails.Title = ExtractValue(line, _TitleKey); + i++; + line = lines[i]; + } + else if (line.StartsWith(_PerformerKey)) + { + trackDetails.Performer = ExtractValue(line, _PerformerKey); + i++; + line = lines[i]; + } + else + { + i++; + line = lines[i]; + } + } + + fileDetails.Tracks.Add(trackDetails); + } + + Files.Add(fileDetails); + } + else if (line.StartsWith(_GenreKey)) + { + Genre = ExtractValue(line, _GenreKey); + i++; + } + else if (line.StartsWith(_DateKey)) + { + Date = ExtractValue(line, _DateKey); + i++; + } + else if (line.StartsWith(_DiscIdKey)) + { + DiscID = ExtractValue(line, _DiscIdKey); + i++; + } + else if (line.StartsWith(_PerformerKey)) + { + Performer = ExtractValue(line, _PerformerKey); + i++; + } + else if (line.StartsWith(_TitleKey)) + { + Title = ExtractValue(line, _TitleKey); + i++; + } + else + { + i++; + } + } + } + catch (IndexOutOfRangeException) + { + } } } } diff --git a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs index f5e6bb5cc..145f40cd6 100644 --- a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs +++ b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs @@ -11,12 +11,14 @@ using NzbDrone.Common.Disk; using NzbDrone.Common.Extensions; using NzbDrone.Common.Instrumentation.Extensions; using NzbDrone.Core.Configuration; +using NzbDrone.Core.Datastore; using NzbDrone.Core.MediaFiles.Commands; using NzbDrone.Core.MediaFiles.Events; using NzbDrone.Core.MediaFiles.TrackImport; using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Music; +using NzbDrone.Core.Parser.Model; using NzbDrone.Core.RootFolders; namespace NzbDrone.Core.MediaFiles @@ -83,6 +85,66 @@ namespace NzbDrone.Core.MediaFiles artistIds = new List(); } + var mediaFileList = GetMediaFiles(folders, artistIds); + + var decisionsStopwatch = Stopwatch.StartNew(); + + var itemInfo = new ImportDecisionMakerInfo(); + var config = new ImportDecisionMakerConfig + { + Filter = filter, + IncludeExisting = true, + AddNewArtists = addNewArtists + }; + + var decisions = new List>(); + var cueFiles = mediaFileList.Where(x => x.Extension.Equals(".cue")).ToList(); + mediaFileList.RemoveAll(l => cueFiles.Contains(l)); + var cueSheetInfos = new List(); + foreach (var cueFile in cueFiles) + { + var cueSheetInfo = _importDecisionMaker.GetCueSheetInfo(cueFile, mediaFileList); + cueSheetInfos.Add(cueSheetInfo); + } + + var cueSheetInfosGroupedByDiscId = cueSheetInfos.GroupBy(x => x.CueSheet.DiscID).ToList(); + foreach (var cueSheetInfoGroup in cueSheetInfosGroupedByDiscId) + { + var audioFilesForCues = new List(); + foreach (var cueSheetInfo in cueSheetInfoGroup) + { + audioFilesForCues.AddRange(cueSheetInfo.MusicFiles); + } + + decisions.AddRange(_importDecisionMaker.GetImportDecisions(audioFilesForCues, cueSheetInfos[0].IdOverrides, itemInfo, config, cueSheetInfos)); + + foreach (var cueSheetInfo in cueSheetInfos) + { + if (cueSheetInfo.CueSheet != null) + { + decisions.ForEach(item => + { + if (cueSheetInfo.IsForMediaFile(item.Item.Path)) + { + item.Item.CueSheetPath = cueSheetInfo.CueSheet.Path; + } + }); + } + + mediaFileList.RemoveAll(x => cueSheetInfo.MusicFiles.Contains(x)); + } + } + + decisions.AddRange(_importDecisionMaker.GetImportDecisions(mediaFileList, null, itemInfo, config)); + + decisionsStopwatch.Stop(); + _logger.Debug("Import decisions complete [{0}]", decisionsStopwatch.Elapsed); + + Import(folders, artistIds, decisions); + } + + private List GetMediaFiles(List folders, List artistIds) + { var mediaFileList = new List(); var musicFilesStopwatch = Stopwatch.StartNew(); @@ -96,7 +158,7 @@ namespace NzbDrone.Core.MediaFiles if (rootFolder == null) { _logger.Error("Not scanning {0}, it's not a subdirectory of a defined root folder", folder); - return; + return mediaFileList; } var folderExists = _diskProvider.FolderExists(folder); @@ -108,7 +170,7 @@ namespace NzbDrone.Core.MediaFiles _logger.Warn("Artists' root folder ({0}) doesn't exist.", rootFolder.Path); var skippedArtists = _artistService.GetArtists(artistIds); skippedArtists.ForEach(x => _eventAggregator.PublishEvent(new ArtistScanSkippedEvent(x, ArtistScanSkippedReason.RootFolderDoesNotExist))); - return; + return mediaFileList; } if (_diskProvider.FolderEmpty(rootFolder.Path)) @@ -116,7 +178,7 @@ namespace NzbDrone.Core.MediaFiles _logger.Warn("Artists' root folder ({0}) is empty.", rootFolder.Path); var skippedArtists = _artistService.GetArtists(artistIds); skippedArtists.ForEach(x => _eventAggregator.PublishEvent(new ArtistScanSkippedEvent(x, ArtistScanSkippedReason.RootFolderIsEmpty))); - return; + return mediaFileList; } } @@ -140,26 +202,16 @@ namespace NzbDrone.Core.MediaFiles CleanMediaFiles(folder, files.Select(x => x.FullName).ToList()); mediaFileList.AddRange(files); - mediaFileList.RemoveAll(x => x.Extension == ".cue"); } musicFilesStopwatch.Stop(); _logger.Trace("Finished getting track files for:\n{0} [{1}]", folders.ConcatToString("\n"), musicFilesStopwatch.Elapsed); - var decisionsStopwatch = Stopwatch.StartNew(); - - var config = new ImportDecisionMakerConfig - { - Filter = filter, - IncludeExisting = true, - AddNewArtists = addNewArtists - }; - - var decisions = _importDecisionMaker.GetImportDecisions(mediaFileList, null, null, config); - - decisionsStopwatch.Stop(); - _logger.Debug("Import decisions complete [{0}]", decisionsStopwatch.Elapsed); + return mediaFileList; + } + private void Import(List folders, List artistIds, List> decisions) + { var importStopwatch = Stopwatch.StartNew(); _importApprovedTracks.Import(decisions, false); @@ -178,7 +230,8 @@ namespace NzbDrone.Core.MediaFiles Modified = decision.Item.Modified, DateAdded = DateTime.UtcNow, Quality = decision.Item.Quality, - MediaInfo = decision.Item.FileTrackInfo.MediaInfo + MediaInfo = decision.Item.FileTrackInfo.MediaInfo, + IsSingleFileRelease = decision.Item.IsSingleFileRelease, }) .ToList(); _mediaFileService.AddMany(newFiles); diff --git a/src/NzbDrone.Core/MediaFiles/TrackImport/Aggregation/Aggregators/AggregateFilenameInfo.cs b/src/NzbDrone.Core/MediaFiles/TrackImport/Aggregation/Aggregators/AggregateFilenameInfo.cs index 63d57512f..12be78699 100644 --- a/src/NzbDrone.Core/MediaFiles/TrackImport/Aggregation/Aggregators/AggregateFilenameInfo.cs +++ b/src/NzbDrone.Core/MediaFiles/TrackImport/Aggregation/Aggregators/AggregateFilenameInfo.cs @@ -71,8 +71,6 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Aggregation.Aggregators { tracks[i].FileTrackInfo.ArtistTitle = tracks[i].Artist.Name; tracks[i].FileTrackInfo.AlbumTitle = tracks[i].Album.Title; - tracks[i].FileTrackInfo.DiscNumber = i + 1; - tracks[i].FileTrackInfo.DiscCount = tracks.Count; // TODO this is too bold, the release year is not the one from the .cue file tracks[i].FileTrackInfo.Year = (uint)tracks[i].Album.ReleaseDate.Value.Year; diff --git a/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs b/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs index bf5b76501..537919552 100644 --- a/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs +++ b/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs @@ -17,7 +17,7 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Identification { public interface IIdentificationService { - List Identify(List localTracks, IdentificationOverrides idOverrides, ImportDecisionMakerConfig config); + List Identify(List localTracks, IdentificationOverrides idOverrides, ImportDecisionMakerConfig config, List cueSheetInfos = null); } public class IdentificationService : IIdentificationService @@ -114,7 +114,7 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Identification return releases; } - public List Identify(List localTracks, IdentificationOverrides idOverrides, ImportDecisionMakerConfig config) + public List Identify(List localTracks, IdentificationOverrides idOverrides, ImportDecisionMakerConfig config, List cueSheetInfos = null) { // 1 group localTracks so that we think they represent a single release // 2 get candidates given specified artist, album and release. Candidates can include extra files already on disk. @@ -132,6 +132,41 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Identification i++; _logger.ProgressInfo($"Identifying album {i}/{releases.Count}"); IdentifyRelease(localRelease, idOverrides, config); + + if (cueSheetInfos != null && localRelease.IsSingleFileRelease) + { + var addedMbTracks = new List(); + localRelease.LocalTracks.ForEach(localTrack => + { + var cueSheetFindResult = cueSheetInfos.Find(x => x.IsForMediaFile(localTrack.Path)); + var cueSheet = cueSheetFindResult?.CueSheet; + if (cueSheet == null) + { + return; + } + + localTrack.Tracks.Clear(); + localRelease.AlbumRelease.Tracks.Value.ForEach(mbTrack => + { + cueSheet.Files[0].Tracks.ForEach(cueTrack => + { + if (!string.Equals(cueTrack.Title, mbTrack.Title, StringComparison.OrdinalIgnoreCase)) + { + return; + } + + if (addedMbTracks.Contains(mbTrack)) + { + return; + } + + mbTrack.IsSingleFileRelease = true; + localTrack.Tracks.Add(mbTrack); + addedMbTracks.Add(mbTrack); + }); + }); + }); + } } watch.Stop(); @@ -187,7 +222,8 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Identification FileTrackInfo = _audioTagService.ReadTags(x.Path), ExistingFile = true, AdditionalFile = true, - Quality = x.Quality + Quality = x.Quality, + IsSingleFileRelease = x.IsSingleFileRelease, })) .ToList(); @@ -340,19 +376,6 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Identification localAlbumRelease.AlbumRelease = release; localAlbumRelease.ExistingTracks = extraTracks; localAlbumRelease.TrackMapping = mapping; - if (localAlbumRelease.IsSingleFileRelease) - { - localAlbumRelease.LocalTracks.ForEach(x => x.Tracks.Clear()); - for (var i = 0; i < release.Tracks.Value.Count; i++) - { - var track = release.Tracks.Value[i]; - var localTrackIndex = localAlbumRelease.LocalTracks.FindIndex(x => x.FileTrackInfo.DiscNumber == track.MediumNumber); - if (localTrackIndex != -1) - { - localAlbumRelease.LocalTracks[localTrackIndex].Tracks.Add(track); - } - } - } if (currDistance == 0.0) { diff --git a/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs b/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs index b1bbc4ec6..3236d82c1 100644 --- a/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs +++ b/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.IO.Abstractions; using System.Linq; using DryIoc.ImTools; @@ -11,6 +12,7 @@ using NzbDrone.Core.Download; using NzbDrone.Core.MediaFiles.TrackImport.Aggregation; using NzbDrone.Core.MediaFiles.TrackImport.Identification; using NzbDrone.Core.Music; +using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Profiles.Qualities; using NzbDrone.Core.RootFolders; @@ -19,7 +21,9 @@ namespace NzbDrone.Core.MediaFiles.TrackImport { public interface IMakeImportDecision { - List> GetImportDecisions(List musicFiles, IdentificationOverrides idOverrides, ImportDecisionMakerInfo itemInfo, ImportDecisionMakerConfig config); + List> GetImportDecisions(List musicFiles, IdentificationOverrides idOverrides, ImportDecisionMakerInfo itemInfo, ImportDecisionMakerConfig config, List cueSheetInfos = null); + List> GetImportDecisions(List cueSheetInfos, ImportDecisionMakerInfo itemInfo, ImportDecisionMakerConfig config); + CueSheetInfo GetCueSheetInfo(IFileInfo cueFile, List musicFiles); } public class IdentificationOverrides @@ -29,12 +33,18 @@ namespace NzbDrone.Core.MediaFiles.TrackImport public AlbumRelease AlbumRelease { get; set; } } + public class CueSheetInfo + { + public List MusicFiles { get; set; } + public IdentificationOverrides IdOverrides { get; set; } + public CueSheet CueSheet { get; set; } + public bool IsForMediaFile(string path) => CueSheet != null && CueSheet.Files.Count > 0 && CueSheet.Files.Any(x => Path.GetFileName(path) == x.Name); + } + public class ImportDecisionMakerInfo { public DownloadClientItem DownloadClientItem { get; set; } public ParsedAlbumInfo ParsedAlbumInfo { get; set; } - public bool IsSingleFileRelease { get; set; } - public CueSheet CueSheet { get; set; } } public class ImportDecisionMakerConfig @@ -51,6 +61,7 @@ namespace NzbDrone.Core.MediaFiles.TrackImport private readonly IEnumerable> _trackSpecifications; private readonly IEnumerable> _albumSpecifications; private readonly IMediaFileService _mediaFileService; + private readonly IParsingService _parsingService; private readonly IAudioTagService _audioTagService; private readonly IAugmentingService _augmentingService; private readonly IIdentificationService _identificationService; @@ -61,6 +72,7 @@ namespace NzbDrone.Core.MediaFiles.TrackImport public ImportDecisionMaker(IEnumerable> trackSpecifications, IEnumerable> albumSpecifications, IMediaFileService mediaFileService, + IParsingService parsingService, IAudioTagService audioTagService, IAugmentingService augmentingService, IIdentificationService identificationService, @@ -71,6 +83,7 @@ namespace NzbDrone.Core.MediaFiles.TrackImport _trackSpecifications = trackSpecifications; _albumSpecifications = albumSpecifications; _mediaFileService = mediaFileService; + _parsingService = parsingService; _audioTagService = audioTagService; _augmentingService = augmentingService; _identificationService = identificationService; @@ -79,6 +92,46 @@ namespace NzbDrone.Core.MediaFiles.TrackImport _logger = logger; } + public CueSheetInfo GetCueSheetInfo(IFileInfo cueFile, List musicFiles) + { + var cueSheetInfo = new CueSheetInfo(); + var cueSheet = new CueSheet(cueFile); + if (cueSheet == null) + { + return cueSheetInfo; + } + + cueSheetInfo.CueSheet = cueSheet; + cueSheetInfo.IdOverrides = new IdentificationOverrides(); + + Artist artistFromCue = null; + if (!cueSheet.Performer.Empty()) + { + artistFromCue = _parsingService.GetArtist(cueSheet.Performer); + if (artistFromCue != null) + { + cueSheetInfo.IdOverrides.Artist = artistFromCue; + } + } + + var parsedAlbumInfo = new ParsedAlbumInfo + { + AlbumTitle = cueSheet.Title, + ArtistName = artistFromCue.Name, + ReleaseDate = cueSheet.Date, + }; + + var albumsFromCue = _parsingService.GetAlbums(parsedAlbumInfo, artistFromCue); + if (albumsFromCue != null && albumsFromCue.Count > 0) + { + cueSheetInfo.IdOverrides.Album = albumsFromCue[0]; + } + + cueSheetInfo.MusicFiles = musicFiles.Where(musicFile => cueSheet.Files.Any(musicFileFromCue => musicFileFromCue.Name == musicFile.Name)).ToList(); + + return cueSheetInfo; + } + public Tuple, List>> GetLocalTracks(List musicFiles, DownloadClientItem downloadClientItem, ParsedAlbumInfo folderInfo, FilterFilesType filter) { var watch = new System.Diagnostics.Stopwatch(); @@ -116,7 +169,7 @@ namespace NzbDrone.Core.MediaFiles.TrackImport Size = file.Length, Modified = file.LastWriteTimeUtc, FileTrackInfo = _audioTagService.ReadTags(file.FullName), - AdditionalFile = false + AdditionalFile = false, }; try @@ -142,7 +195,7 @@ namespace NzbDrone.Core.MediaFiles.TrackImport return Tuple.Create(localTracks, decisions); } - public List> GetImportDecisions(List musicFiles, IdentificationOverrides idOverrides, ImportDecisionMakerInfo itemInfo, ImportDecisionMakerConfig config) + public List> GetImportDecisions(List musicFiles, IdentificationOverrides idOverrides, ImportDecisionMakerInfo itemInfo, ImportDecisionMakerConfig config, List cueSheetInfos) { idOverrides ??= new IdentificationOverrides(); itemInfo ??= new ImportDecisionMakerInfo(); @@ -152,14 +205,39 @@ namespace NzbDrone.Core.MediaFiles.TrackImport var decisions = trackData.Item2; localTracks.ForEach(x => x.ExistingFile = !config.NewDownload); - localTracks.ForEach(x => x.IsSingleFileRelease = itemInfo.IsSingleFileRelease); - if (itemInfo.IsSingleFileRelease) + if (cueSheetInfos != null) { - localTracks.ForEach(x => x.Artist = idOverrides.Artist); - localTracks.ForEach(x => x.Album = idOverrides.Album); + localTracks.ForEach(localTrack => + { + var cueSheetFindResult = cueSheetInfos.Find(x => x.IsForMediaFile(localTrack.Path)); + var cueSheet = cueSheetFindResult?.CueSheet; + if (cueSheet != null) + { + localTrack.IsSingleFileRelease = cueSheet.IsSingleFileRelease; + localTrack.Artist = idOverrides.Artist; + localTrack.Album = idOverrides.Album; + } + }); } - var releases = _identificationService.Identify(localTracks, idOverrides, config); + var localTracksByAlbums = localTracks.GroupBy(x => x.Album); + foreach (var localTracksByAlbum in localTracksByAlbums) + { + if (!localTracksByAlbum.All(x => x.IsSingleFileRelease == true)) + { + continue; + } + + localTracks.ForEach(x => + { + if (x.IsSingleFileRelease && localTracksByAlbum.Contains(x)) + { + x.FileTrackInfo.DiscCount = localTracksByAlbum.Count(); + } + }); + } + + var releases = _identificationService.Identify(localTracks, idOverrides, config, cueSheetInfos); var albums = releases.GroupBy(x => x.AlbumRelease?.Album?.Value.ForeignAlbumId); @@ -206,6 +284,17 @@ namespace NzbDrone.Core.MediaFiles.TrackImport return decisions; } + public List> GetImportDecisions(List cueSheetInfos, ImportDecisionMakerInfo itemInfo, ImportDecisionMakerConfig config) + { + var decisions = new List>(); + foreach (var cueSheetInfo in cueSheetInfos) + { + decisions.AddRange(GetImportDecisions(cueSheetInfo.MusicFiles, cueSheetInfo.IdOverrides, itemInfo, config, cueSheetInfos)); + } + + return decisions; + } + private void EnsureData(LocalAlbumRelease release) { if (release.AlbumRelease != null && release.AlbumRelease.Album.Value.Artist.Value.QualityProfileId == 0) diff --git a/src/NzbDrone.Core/MediaFiles/TrackImport/Manual/ManualImportService.cs b/src/NzbDrone.Core/MediaFiles/TrackImport/Manual/ManualImportService.cs index 3adf9cd4f..598ebf4b1 100644 --- a/src/NzbDrone.Core/MediaFiles/TrackImport/Manual/ManualImportService.cs +++ b/src/NzbDrone.Core/MediaFiles/TrackImport/Manual/ManualImportService.cs @@ -10,6 +10,7 @@ using NzbDrone.Common.Disk; using NzbDrone.Common.Extensions; using NzbDrone.Common.Instrumentation.Extensions; using NzbDrone.Core.CustomFormats; +using NzbDrone.Core.Datastore; using NzbDrone.Core.DecisionEngine; using NzbDrone.Core.Download; using NzbDrone.Core.Download.TrackedDownloads; @@ -155,63 +156,67 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Manual // Split cue and non-cue files var cueFiles = audioFiles.Where(x => x.Extension.Equals(".cue")).ToList(); audioFiles.RemoveAll(l => cueFiles.Contains(l)); + var cueSheetInfos = new List(); foreach (var cueFile in cueFiles) { - var cueSheet = new CueSheet(cueFile); - - Artist artistFromCue = null; - if (!cueSheet.Performer.Empty()) - { - artistFromCue = _parsingService.GetArtist(cueSheet.Performer); - } - - if (artistFromCue == null) - { - continue; - } - - // TODO use the audio files from the cue sheet - var validAudioFiles = audioFiles.FindAll(x => cueSheet.FileNames.Contains(x.Name)); - if (validAudioFiles.Count == 0) - { - continue; - } - - var parsedAlbumInfo = new ParsedAlbumInfo - { - AlbumTitle = cueSheet.Title, - ArtistName = artistFromCue.Name, - ReleaseDate = cueSheet.Date, - }; - var albumsFromCue = _parsingService.GetAlbums(parsedAlbumInfo, artistFromCue); - if (albumsFromCue == null || albumsFromCue.Count == 0) - { - continue; - } - - results.AddRange(ProcessFolder(downloadId, artistFromCue, albumsFromCue[0], filter, replaceExistingFiles, downloadClientItem, cueSheet.Title, validAudioFiles, cueSheet)); - audioFiles.RemoveAll(x => validAudioFiles.Contains(x)); + var cueSheetInfo = _importDecisionMaker.GetCueSheetInfo(cueFile, audioFiles); + cueSheetInfos.Add(cueSheetInfo); } - results.AddRange(ProcessFolder(downloadId, artist, null, filter, replaceExistingFiles, downloadClientItem, directoryInfo.Name, audioFiles, null)); + var cueSheetInfosGroupedByDiscId = cueSheetInfos.GroupBy(x => x.CueSheet.DiscID).ToList(); + foreach (var cueSheetInfoGroup in cueSheetInfosGroupedByDiscId) + { + var audioFilesForCues = new List(); + foreach (var cueSheetInfo in cueSheetInfoGroup) + { + audioFilesForCues.AddRange(cueSheetInfo.MusicFiles); + } + + var manualImportItems = ProcessFolder(downloadId, cueSheetInfos[0].IdOverrides, filter, replaceExistingFiles, downloadClientItem, cueSheetInfos[0].IdOverrides.Album.Title, audioFilesForCues, cueSheetInfos); + results.AddRange(manualImportItems); + + RemoveProcessedAudioFiles(audioFiles, cueSheetInfos, manualImportItems); + } + + var idOverrides = new IdentificationOverrides + { + Artist = artist, + Album = null + }; + + results.AddRange(ProcessFolder(downloadId, idOverrides, filter, replaceExistingFiles, downloadClientItem, directoryInfo.Name, audioFiles)); return results; } - private List ProcessFolder(string downloadId, Artist overrideArtist, Album overrideAlbum, FilterFilesType filter, bool replaceExistingFiles, DownloadClientItem downloadClientItem, string albumTitle, List audioFiles, CueSheet cueSheet) + private void RemoveProcessedAudioFiles(List audioFiles, List cueSheetInfos, List manualImportItems) { - var idOverrides = new IdentificationOverrides + foreach (var cueSheetInfo in cueSheetInfos) { - Artist = overrideArtist, - Album = overrideAlbum - }; + if (cueSheetInfo.CueSheet != null) + { + manualImportItems.ForEach(item => + { + if (cueSheetInfo.IsForMediaFile(item.Path)) + { + item.CueSheetPath = cueSheetInfo.CueSheet.Path; + } + }); + } + + audioFiles.RemoveAll(x => cueSheetInfo.MusicFiles.Contains(x)); + } + } + + private List ProcessFolder(string downloadId, IdentificationOverrides idOverrides, FilterFilesType filter, bool replaceExistingFiles, DownloadClientItem downloadClientItem, string albumTitle, List audioFiles, List cueSheetInfos = null) + { + idOverrides ??= new IdentificationOverrides(); var itemInfo = new ImportDecisionMakerInfo { DownloadClientItem = downloadClientItem, ParsedAlbumInfo = Parser.Parser.ParseAlbumTitle(albumTitle), - CueSheet = cueSheet, - IsSingleFileRelease = cueSheet != null ? cueSheet.IsSingleFileRelease : false, }; + var config = new ImportDecisionMakerConfig { Filter = filter, @@ -221,7 +226,7 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Manual AddNewArtists = false }; - var decisions = _importDecisionMaker.GetImportDecisions(audioFiles, idOverrides, itemInfo, config); + var decisions = _importDecisionMaker.GetImportDecisions(audioFiles, idOverrides, itemInfo, config, cueSheetInfos); // paths will be different for new and old files which is why we need to map separately var newFiles = audioFiles.Join(decisions, @@ -230,16 +235,12 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Manual (f, d) => new { File = f, Decision = d }, PathEqualityComparer.Instance); - var newItems = newFiles.Select(x => MapItem(x.Decision, downloadId, replaceExistingFiles, false)); + var newItemsList = newFiles.Select(x => MapItem(x.Decision, downloadId, replaceExistingFiles, false)).ToList(); + var existingDecisions = decisions.Except(newFiles.Select(x => x.Decision)); var existingItems = existingDecisions.Select(x => MapItem(x, null, replaceExistingFiles, false)); - var itemsList = newItems.Concat(existingItems).ToList(); - if (cueSheet != null) - { - itemsList.ForEach(item => { item.CueSheetPath = cueSheet.Path; }); - } - + var itemsList = newItemsList.Concat(existingItems.ToList()).ToList(); return itemsList; } @@ -257,13 +258,6 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Manual var disableReleaseSwitching = group.First().DisableReleaseSwitching; - var files = group.Select(x => _diskProvider.GetFileInfo(x.Path)).ToList(); - var idOverride = new IdentificationOverrides - { - Artist = group.First().Artist, - Album = group.First().Album, - AlbumRelease = group.First().Release - }; var config = new ImportDecisionMakerConfig { Filter = FilterFilesType.None, @@ -273,64 +267,99 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Manual AddNewArtists = false }; - var itemInfo = new ImportDecisionMakerInfo + var audioFiles = new List(); + foreach (var item in group) { - IsSingleFileRelease = group.All(x => x.IsSingleFileRelease == true) - }; - - // TODO support with the cue sheet - var decisions = _importDecisionMaker.GetImportDecisions(files, idOverride, itemInfo, config); - - var existingItems = group.Join(decisions, - i => i.Path, - d => d.Item.Path, - (i, d) => new { Item = i, Decision = d }, - PathEqualityComparer.Instance); - - foreach (var pair in existingItems) - { - var item = pair.Item; - var decision = pair.Decision; - - if (decision.Item.Artist != null) - { - item.Artist = decision.Item.Artist; - } - - if (decision.Item.Album != null) - { - item.Album = decision.Item.Album; - item.Release = decision.Item.Release; - } - - if (decision.Item.Tracks.Any()) - { - item.Tracks = decision.Item.Tracks; - } - - if (item.Quality?.Quality == Quality.Unknown) - { - item.Quality = decision.Item.Quality; - } - - if (item.ReleaseGroup.IsNullOrWhiteSpace()) - { - item.ReleaseGroup = decision.Item.ReleaseGroup; - } - - item.Rejections = decision.Rejections; - item.Size = decision.Item.Size; - - result.Add(item); + var file = _diskProvider.GetFileInfo(item.Path); + audioFiles.Add(file); } - var newDecisions = decisions.Except(existingItems.Select(x => x.Decision)); - result.AddRange(newDecisions.Select(x => MapItem(x, null, replaceExistingFiles, disableReleaseSwitching))); + var cueSheetInfos = new List(); + var audioFilesForCues = new List(); + var itemInfo = new ImportDecisionMakerInfo(); + foreach (var item in group) + { + if (item.IsSingleFileRelease) + { + var cueFile = _diskProvider.GetFileInfo(item.CueSheetPath); + var cueSheetInfo = _importDecisionMaker.GetCueSheetInfo(cueFile, audioFiles); + cueSheetInfos.Add(cueSheetInfo); + audioFilesForCues.AddRange(cueSheetInfo.MusicFiles); + } + } + + var singleFileReleaseDecisions = _importDecisionMaker.GetImportDecisions(audioFilesForCues, cueSheetInfos[0].IdOverrides, itemInfo, config, cueSheetInfos); + var manualImportItems = UpdateItems(group, singleFileReleaseDecisions, replaceExistingFiles, disableReleaseSwitching); + result.AddRange(manualImportItems); + + RemoveProcessedAudioFiles(audioFiles, cueSheetInfos, manualImportItems); + + var idOverride = new IdentificationOverrides + { + Artist = group.First().Artist, + Album = group.First().Album, + AlbumRelease = group.First().Release + }; + var decisions = _importDecisionMaker.GetImportDecisions(audioFiles, idOverride, itemInfo, config); + result.AddRange(UpdateItems(group, decisions, replaceExistingFiles, disableReleaseSwitching)); } return result; } + private List UpdateItems(IGrouping group, List> decisions, bool replaceExistingFiles, bool disableReleaseSwitching) + { + var result = new List(); + + var existingItems = group.Join(decisions, + i => i.Path, + d => d.Item.Path, + (i, d) => new { Item = i, Decision = d }, + PathEqualityComparer.Instance); + + foreach (var pair in existingItems) + { + var item = pair.Item; + var decision = pair.Decision; + + if (decision.Item.Artist != null) + { + item.Artist = decision.Item.Artist; + } + + if (decision.Item.Album != null) + { + item.Album = decision.Item.Album; + item.Release = decision.Item.Release; + } + + if (decision.Item.Tracks.Any()) + { + item.Tracks = decision.Item.Tracks; + } + + if (item.Quality?.Quality == Quality.Unknown) + { + item.Quality = decision.Item.Quality; + } + + if (item.ReleaseGroup.IsNullOrWhiteSpace()) + { + item.ReleaseGroup = decision.Item.ReleaseGroup; + } + + item.Rejections = decision.Rejections; + item.Size = decision.Item.Size; + + result.Add(item); + } + + var newDecisions = decisions.Except(existingItems.Select(x => x.Decision)); + result.AddRange(newDecisions.Select(x => MapItem(x, null, replaceExistingFiles, disableReleaseSwitching))); + + return result; + } + private ManualImportItem MapItem(ImportDecision decision, string downloadId, bool replaceExistingFiles, bool disableReleaseSwitching) { var item = new ManualImportItem();