Refactor the cue sheet file loader to read the track titles, disc ID and other useful information.

Add support to import releases with multiple cue sheets.
Add the cue sheet support to the disc scan service.
Use the track info from cue sheet files to map local tracks.
Use the disc ID to group cue sheet files and deduce the disc count.

(cherry picked from commit fac76b7cfb746e05f9924047e698ef41203efe5e)
This commit is contained in:
zhangdoa 2023-10-12 00:48:34 +02:00
commit cb7a7ec24f
6 changed files with 504 additions and 201 deletions

View file

@ -1,7 +1,6 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO.Abstractions; using System.IO.Abstractions;
using System.Linq;
using System.Text; using System.Text;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using NzbDrone.Core.Datastore; using NzbDrone.Core.Datastore;
@ -23,68 +22,180 @@ namespace NzbDrone.Core.MediaFiles
{ {
content = encoding.GetString(bytes); content = encoding.GetString(bytes);
var lines = content.Split(new string[] { Environment.NewLine }, StringSplitOptions.None); var lines = content.Split(new string[] { Environment.NewLine }, StringSplitOptions.None);
ParseCueSheet(lines);
// Single-file cue means it's an unsplit image // Single-file cue means it's an unsplit image, which should be specially treated in the pipeline
FileNames = ReadField(lines, "FILE"); IsSingleFileRelease = Files.Count == 1;
IsSingleFileRelease = FileNames.Count == 1;
var performers = ReadField(lines, "PERFORMER");
if (performers.Count > 0)
{
Performer = performers[0];
}
var titles = ReadField(lines, "TITLE");
if (titles.Count > 0)
{
Title = titles[0];
}
var dates = ReadField(lines, "REM DATE");
if (dates.Count > 0)
{
Date = dates[0];
}
} }
} }
} }
public class IndexEntry
{
public int Key { get; set; }
public string Time { get; set; }
}
public class TrackEntry
{
public int Number { get; set; }
public string Title { get; set; }
public string Performer { get; set; }
public List<IndexEntry> Indices { get; set; } = new List<IndexEntry>();
}
public class FileEntry
{
public string Name { get; set; }
public IndexEntry Index { get; set; }
public List<TrackEntry> Tracks { get; set; } = new List<TrackEntry>();
}
public string Path { get; set; } public string Path { get; set; }
public bool IsSingleFileRelease { get; set; } public bool IsSingleFileRelease { get; set; }
public List<string> FileNames { get; set; } public List<FileEntry> Files { get; set; } = new List<FileEntry>();
public string Genre { get; set; }
public string Date { get; set; }
public string DiscID { get; set; }
public string Title { get; set; } public string Title { get; set; }
public string Performer { get; set; } public string Performer { get; set; }
public string Date { get; set; } private static string _FileKey = "FILE";
private static string _TrackKey = "TRACK";
private static string _IndexKey = "INDEX";
private static string _GenreKey = "REM GENRE";
private static string _DateKey = "REM DATE";
private static string _DiscIdKey = "REM DISCID";
private static string _PerformerKey = "PERFORMER";
private static string _TitleKey = "TITLE";
private static List<string> ReadField(string[] lines, string fieldName) private string ExtractValue(string line, string keyword)
{ {
var inQuotePattern = "\"(.*?)\""; var pattern = keyword + @"\s+(?:(?:\""(.*?)\"")|(.+))";
var flatPattern = fieldName + " (.+)"; var match = Regex.Match(line, pattern);
var results = new List<string>(); if (match.Success)
var candidates = lines.Where(l => l.StartsWith(fieldName)).ToList();
foreach (var candidate in candidates)
{ {
var matches = Regex.Matches(candidate, inQuotePattern).ToList(); var value = match.Groups[1].Success ? match.Groups[1].Value : match.Groups[2].Value;
if (matches.Count == 0) return value;
{
matches = Regex.Matches(candidate, flatPattern).ToList();
}
if (matches.Count == 0)
{
continue;
}
var groups = matches[0].Groups;
if (groups.Count > 0)
{
var result = groups[1].Value;
results.Add(result);
}
} }
return results; return "";
}
private void ParseCueSheet(string[] lines)
{
var i = 0;
try
{
while (true)
{
var line = lines[i];
if (line.StartsWith(_FileKey))
{
line = line.Trim();
line = line.Substring(_FileKey.Length).Trim();
var filename = line.Split('"')[1];
var fileDetails = new FileEntry { Name = filename };
i++;
line = lines[i];
while (line.StartsWith(" "))
{
line = line.Trim();
if (line.StartsWith(_TrackKey))
{
line = line.Substring(_TrackKey.Length).Trim();
}
var trackDetails = new TrackEntry();
var trackInfo = line.Split(' ');
if (trackInfo.Length > 0)
{
if (int.TryParse(trackInfo[0], out var number))
{
trackDetails.Number = number;
}
}
i++;
line = lines[i];
while (line.StartsWith(" "))
{
line = line.Trim();
if (line.StartsWith(_IndexKey))
{
line = line.Substring(_IndexKey.Length).Trim();
var parts = line.Split(' ');
if (parts.Length > 1)
{
if (int.TryParse(parts[0], out var key))
{
var value = parts[1].Trim('"');
trackDetails.Indices.Add(new IndexEntry { Key = key, Time = value });
}
}
i++;
line = lines[i];
}
else if (line.StartsWith(_TitleKey))
{
trackDetails.Title = ExtractValue(line, _TitleKey);
i++;
line = lines[i];
}
else if (line.StartsWith(_PerformerKey))
{
trackDetails.Performer = ExtractValue(line, _PerformerKey);
i++;
line = lines[i];
}
else
{
i++;
line = lines[i];
}
}
fileDetails.Tracks.Add(trackDetails);
}
Files.Add(fileDetails);
}
else if (line.StartsWith(_GenreKey))
{
Genre = ExtractValue(line, _GenreKey);
i++;
}
else if (line.StartsWith(_DateKey))
{
Date = ExtractValue(line, _DateKey);
i++;
}
else if (line.StartsWith(_DiscIdKey))
{
DiscID = ExtractValue(line, _DiscIdKey);
i++;
}
else if (line.StartsWith(_PerformerKey))
{
Performer = ExtractValue(line, _PerformerKey);
i++;
}
else if (line.StartsWith(_TitleKey))
{
Title = ExtractValue(line, _TitleKey);
i++;
}
else
{
i++;
}
}
}
catch (IndexOutOfRangeException)
{
}
} }
} }
} }

View file

@ -11,12 +11,14 @@ using NzbDrone.Common.Disk;
using NzbDrone.Common.Extensions; using NzbDrone.Common.Extensions;
using NzbDrone.Common.Instrumentation.Extensions; using NzbDrone.Common.Instrumentation.Extensions;
using NzbDrone.Core.Configuration; using NzbDrone.Core.Configuration;
using NzbDrone.Core.Datastore;
using NzbDrone.Core.MediaFiles.Commands; using NzbDrone.Core.MediaFiles.Commands;
using NzbDrone.Core.MediaFiles.Events; using NzbDrone.Core.MediaFiles.Events;
using NzbDrone.Core.MediaFiles.TrackImport; using NzbDrone.Core.MediaFiles.TrackImport;
using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.Messaging.Commands;
using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.Music; using NzbDrone.Core.Music;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.RootFolders; using NzbDrone.Core.RootFolders;
namespace NzbDrone.Core.MediaFiles namespace NzbDrone.Core.MediaFiles
@ -83,6 +85,66 @@ namespace NzbDrone.Core.MediaFiles
artistIds = new List<int>(); artistIds = new List<int>();
} }
var mediaFileList = GetMediaFiles(folders, artistIds);
var decisionsStopwatch = Stopwatch.StartNew();
var itemInfo = new ImportDecisionMakerInfo();
var config = new ImportDecisionMakerConfig
{
Filter = filter,
IncludeExisting = true,
AddNewArtists = addNewArtists
};
var decisions = new List<ImportDecision<LocalTrack>>();
var cueFiles = mediaFileList.Where(x => x.Extension.Equals(".cue")).ToList();
mediaFileList.RemoveAll(l => cueFiles.Contains(l));
var cueSheetInfos = new List<CueSheetInfo>();
foreach (var cueFile in cueFiles)
{
var cueSheetInfo = _importDecisionMaker.GetCueSheetInfo(cueFile, mediaFileList);
cueSheetInfos.Add(cueSheetInfo);
}
var cueSheetInfosGroupedByDiscId = cueSheetInfos.GroupBy(x => x.CueSheet.DiscID).ToList();
foreach (var cueSheetInfoGroup in cueSheetInfosGroupedByDiscId)
{
var audioFilesForCues = new List<IFileInfo>();
foreach (var cueSheetInfo in cueSheetInfoGroup)
{
audioFilesForCues.AddRange(cueSheetInfo.MusicFiles);
}
decisions.AddRange(_importDecisionMaker.GetImportDecisions(audioFilesForCues, cueSheetInfos[0].IdOverrides, itemInfo, config, cueSheetInfos));
foreach (var cueSheetInfo in cueSheetInfos)
{
if (cueSheetInfo.CueSheet != null)
{
decisions.ForEach(item =>
{
if (cueSheetInfo.IsForMediaFile(item.Item.Path))
{
item.Item.CueSheetPath = cueSheetInfo.CueSheet.Path;
}
});
}
mediaFileList.RemoveAll(x => cueSheetInfo.MusicFiles.Contains(x));
}
}
decisions.AddRange(_importDecisionMaker.GetImportDecisions(mediaFileList, null, itemInfo, config));
decisionsStopwatch.Stop();
_logger.Debug("Import decisions complete [{0}]", decisionsStopwatch.Elapsed);
Import(folders, artistIds, decisions);
}
private List<IFileInfo> GetMediaFiles(List<string> folders, List<int> artistIds)
{
var mediaFileList = new List<IFileInfo>(); var mediaFileList = new List<IFileInfo>();
var musicFilesStopwatch = Stopwatch.StartNew(); var musicFilesStopwatch = Stopwatch.StartNew();
@ -96,7 +158,7 @@ namespace NzbDrone.Core.MediaFiles
if (rootFolder == null) if (rootFolder == null)
{ {
_logger.Error("Not scanning {0}, it's not a subdirectory of a defined root folder", folder); _logger.Error("Not scanning {0}, it's not a subdirectory of a defined root folder", folder);
return; return mediaFileList;
} }
var folderExists = _diskProvider.FolderExists(folder); var folderExists = _diskProvider.FolderExists(folder);
@ -108,7 +170,7 @@ namespace NzbDrone.Core.MediaFiles
_logger.Warn("Artists' root folder ({0}) doesn't exist.", rootFolder.Path); _logger.Warn("Artists' root folder ({0}) doesn't exist.", rootFolder.Path);
var skippedArtists = _artistService.GetArtists(artistIds); var skippedArtists = _artistService.GetArtists(artistIds);
skippedArtists.ForEach(x => _eventAggregator.PublishEvent(new ArtistScanSkippedEvent(x, ArtistScanSkippedReason.RootFolderDoesNotExist))); skippedArtists.ForEach(x => _eventAggregator.PublishEvent(new ArtistScanSkippedEvent(x, ArtistScanSkippedReason.RootFolderDoesNotExist)));
return; return mediaFileList;
} }
if (_diskProvider.FolderEmpty(rootFolder.Path)) if (_diskProvider.FolderEmpty(rootFolder.Path))
@ -116,7 +178,7 @@ namespace NzbDrone.Core.MediaFiles
_logger.Warn("Artists' root folder ({0}) is empty.", rootFolder.Path); _logger.Warn("Artists' root folder ({0}) is empty.", rootFolder.Path);
var skippedArtists = _artistService.GetArtists(artistIds); var skippedArtists = _artistService.GetArtists(artistIds);
skippedArtists.ForEach(x => _eventAggregator.PublishEvent(new ArtistScanSkippedEvent(x, ArtistScanSkippedReason.RootFolderIsEmpty))); skippedArtists.ForEach(x => _eventAggregator.PublishEvent(new ArtistScanSkippedEvent(x, ArtistScanSkippedReason.RootFolderIsEmpty)));
return; return mediaFileList;
} }
} }
@ -140,26 +202,16 @@ namespace NzbDrone.Core.MediaFiles
CleanMediaFiles(folder, files.Select(x => x.FullName).ToList()); CleanMediaFiles(folder, files.Select(x => x.FullName).ToList());
mediaFileList.AddRange(files); mediaFileList.AddRange(files);
mediaFileList.RemoveAll(x => x.Extension == ".cue");
} }
musicFilesStopwatch.Stop(); musicFilesStopwatch.Stop();
_logger.Trace("Finished getting track files for:\n{0} [{1}]", folders.ConcatToString("\n"), musicFilesStopwatch.Elapsed); _logger.Trace("Finished getting track files for:\n{0} [{1}]", folders.ConcatToString("\n"), musicFilesStopwatch.Elapsed);
var decisionsStopwatch = Stopwatch.StartNew(); return mediaFileList;
}
var config = new ImportDecisionMakerConfig
{
Filter = filter,
IncludeExisting = true,
AddNewArtists = addNewArtists
};
var decisions = _importDecisionMaker.GetImportDecisions(mediaFileList, null, null, config);
decisionsStopwatch.Stop();
_logger.Debug("Import decisions complete [{0}]", decisionsStopwatch.Elapsed);
private void Import(List<string> folders, List<int> artistIds, List<ImportDecision<LocalTrack>> decisions)
{
var importStopwatch = Stopwatch.StartNew(); var importStopwatch = Stopwatch.StartNew();
_importApprovedTracks.Import(decisions, false); _importApprovedTracks.Import(decisions, false);
@ -178,7 +230,8 @@ namespace NzbDrone.Core.MediaFiles
Modified = decision.Item.Modified, Modified = decision.Item.Modified,
DateAdded = DateTime.UtcNow, DateAdded = DateTime.UtcNow,
Quality = decision.Item.Quality, Quality = decision.Item.Quality,
MediaInfo = decision.Item.FileTrackInfo.MediaInfo MediaInfo = decision.Item.FileTrackInfo.MediaInfo,
IsSingleFileRelease = decision.Item.IsSingleFileRelease,
}) })
.ToList(); .ToList();
_mediaFileService.AddMany(newFiles); _mediaFileService.AddMany(newFiles);

View file

@ -71,8 +71,6 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Aggregation.Aggregators
{ {
tracks[i].FileTrackInfo.ArtistTitle = tracks[i].Artist.Name; tracks[i].FileTrackInfo.ArtistTitle = tracks[i].Artist.Name;
tracks[i].FileTrackInfo.AlbumTitle = tracks[i].Album.Title; tracks[i].FileTrackInfo.AlbumTitle = tracks[i].Album.Title;
tracks[i].FileTrackInfo.DiscNumber = i + 1;
tracks[i].FileTrackInfo.DiscCount = tracks.Count;
// TODO this is too bold, the release year is not the one from the .cue file // TODO this is too bold, the release year is not the one from the .cue file
tracks[i].FileTrackInfo.Year = (uint)tracks[i].Album.ReleaseDate.Value.Year; tracks[i].FileTrackInfo.Year = (uint)tracks[i].Album.ReleaseDate.Value.Year;

View file

@ -17,7 +17,7 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Identification
{ {
public interface IIdentificationService public interface IIdentificationService
{ {
List<LocalAlbumRelease> Identify(List<LocalTrack> localTracks, IdentificationOverrides idOverrides, ImportDecisionMakerConfig config); List<LocalAlbumRelease> Identify(List<LocalTrack> localTracks, IdentificationOverrides idOverrides, ImportDecisionMakerConfig config, List<CueSheetInfo> cueSheetInfos = null);
} }
public class IdentificationService : IIdentificationService public class IdentificationService : IIdentificationService
@ -114,7 +114,7 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Identification
return releases; return releases;
} }
public List<LocalAlbumRelease> Identify(List<LocalTrack> localTracks, IdentificationOverrides idOverrides, ImportDecisionMakerConfig config) public List<LocalAlbumRelease> Identify(List<LocalTrack> localTracks, IdentificationOverrides idOverrides, ImportDecisionMakerConfig config, List<CueSheetInfo> cueSheetInfos = null)
{ {
// 1 group localTracks so that we think they represent a single release // 1 group localTracks so that we think they represent a single release
// 2 get candidates given specified artist, album and release. Candidates can include extra files already on disk. // 2 get candidates given specified artist, album and release. Candidates can include extra files already on disk.
@ -132,6 +132,41 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Identification
i++; i++;
_logger.ProgressInfo($"Identifying album {i}/{releases.Count}"); _logger.ProgressInfo($"Identifying album {i}/{releases.Count}");
IdentifyRelease(localRelease, idOverrides, config); IdentifyRelease(localRelease, idOverrides, config);
if (cueSheetInfos != null && localRelease.IsSingleFileRelease)
{
var addedMbTracks = new List<Track>();
localRelease.LocalTracks.ForEach(localTrack =>
{
var cueSheetFindResult = cueSheetInfos.Find(x => x.IsForMediaFile(localTrack.Path));
var cueSheet = cueSheetFindResult?.CueSheet;
if (cueSheet == null)
{
return;
}
localTrack.Tracks.Clear();
localRelease.AlbumRelease.Tracks.Value.ForEach(mbTrack =>
{
cueSheet.Files[0].Tracks.ForEach(cueTrack =>
{
if (!string.Equals(cueTrack.Title, mbTrack.Title, StringComparison.OrdinalIgnoreCase))
{
return;
}
if (addedMbTracks.Contains(mbTrack))
{
return;
}
mbTrack.IsSingleFileRelease = true;
localTrack.Tracks.Add(mbTrack);
addedMbTracks.Add(mbTrack);
});
});
});
}
} }
watch.Stop(); watch.Stop();
@ -187,7 +222,8 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Identification
FileTrackInfo = _audioTagService.ReadTags(x.Path), FileTrackInfo = _audioTagService.ReadTags(x.Path),
ExistingFile = true, ExistingFile = true,
AdditionalFile = true, AdditionalFile = true,
Quality = x.Quality Quality = x.Quality,
IsSingleFileRelease = x.IsSingleFileRelease,
})) }))
.ToList(); .ToList();
@ -340,19 +376,6 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Identification
localAlbumRelease.AlbumRelease = release; localAlbumRelease.AlbumRelease = release;
localAlbumRelease.ExistingTracks = extraTracks; localAlbumRelease.ExistingTracks = extraTracks;
localAlbumRelease.TrackMapping = mapping; localAlbumRelease.TrackMapping = mapping;
if (localAlbumRelease.IsSingleFileRelease)
{
localAlbumRelease.LocalTracks.ForEach(x => x.Tracks.Clear());
for (var i = 0; i < release.Tracks.Value.Count; i++)
{
var track = release.Tracks.Value[i];
var localTrackIndex = localAlbumRelease.LocalTracks.FindIndex(x => x.FileTrackInfo.DiscNumber == track.MediumNumber);
if (localTrackIndex != -1)
{
localAlbumRelease.LocalTracks[localTrackIndex].Tracks.Add(track);
}
}
}
if (currDistance == 0.0) if (currDistance == 0.0)
{ {

View file

@ -1,5 +1,6 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO;
using System.IO.Abstractions; using System.IO.Abstractions;
using System.Linq; using System.Linq;
using DryIoc.ImTools; using DryIoc.ImTools;
@ -11,6 +12,7 @@ using NzbDrone.Core.Download;
using NzbDrone.Core.MediaFiles.TrackImport.Aggregation; using NzbDrone.Core.MediaFiles.TrackImport.Aggregation;
using NzbDrone.Core.MediaFiles.TrackImport.Identification; using NzbDrone.Core.MediaFiles.TrackImport.Identification;
using NzbDrone.Core.Music; using NzbDrone.Core.Music;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Profiles.Qualities; using NzbDrone.Core.Profiles.Qualities;
using NzbDrone.Core.RootFolders; using NzbDrone.Core.RootFolders;
@ -19,7 +21,9 @@ namespace NzbDrone.Core.MediaFiles.TrackImport
{ {
public interface IMakeImportDecision public interface IMakeImportDecision
{ {
List<ImportDecision<LocalTrack>> GetImportDecisions(List<IFileInfo> musicFiles, IdentificationOverrides idOverrides, ImportDecisionMakerInfo itemInfo, ImportDecisionMakerConfig config); List<ImportDecision<LocalTrack>> GetImportDecisions(List<IFileInfo> musicFiles, IdentificationOverrides idOverrides, ImportDecisionMakerInfo itemInfo, ImportDecisionMakerConfig config, List<CueSheetInfo> cueSheetInfos = null);
List<ImportDecision<LocalTrack>> GetImportDecisions(List<CueSheetInfo> cueSheetInfos, ImportDecisionMakerInfo itemInfo, ImportDecisionMakerConfig config);
CueSheetInfo GetCueSheetInfo(IFileInfo cueFile, List<IFileInfo> musicFiles);
} }
public class IdentificationOverrides public class IdentificationOverrides
@ -29,12 +33,18 @@ namespace NzbDrone.Core.MediaFiles.TrackImport
public AlbumRelease AlbumRelease { get; set; } public AlbumRelease AlbumRelease { get; set; }
} }
public class CueSheetInfo
{
public List<IFileInfo> MusicFiles { get; set; }
public IdentificationOverrides IdOverrides { get; set; }
public CueSheet CueSheet { get; set; }
public bool IsForMediaFile(string path) => CueSheet != null && CueSheet.Files.Count > 0 && CueSheet.Files.Any(x => Path.GetFileName(path) == x.Name);
}
public class ImportDecisionMakerInfo public class ImportDecisionMakerInfo
{ {
public DownloadClientItem DownloadClientItem { get; set; } public DownloadClientItem DownloadClientItem { get; set; }
public ParsedAlbumInfo ParsedAlbumInfo { get; set; } public ParsedAlbumInfo ParsedAlbumInfo { get; set; }
public bool IsSingleFileRelease { get; set; }
public CueSheet CueSheet { get; set; }
} }
public class ImportDecisionMakerConfig public class ImportDecisionMakerConfig
@ -51,6 +61,7 @@ namespace NzbDrone.Core.MediaFiles.TrackImport
private readonly IEnumerable<IImportDecisionEngineSpecification<LocalTrack>> _trackSpecifications; private readonly IEnumerable<IImportDecisionEngineSpecification<LocalTrack>> _trackSpecifications;
private readonly IEnumerable<IImportDecisionEngineSpecification<LocalAlbumRelease>> _albumSpecifications; private readonly IEnumerable<IImportDecisionEngineSpecification<LocalAlbumRelease>> _albumSpecifications;
private readonly IMediaFileService _mediaFileService; private readonly IMediaFileService _mediaFileService;
private readonly IParsingService _parsingService;
private readonly IAudioTagService _audioTagService; private readonly IAudioTagService _audioTagService;
private readonly IAugmentingService _augmentingService; private readonly IAugmentingService _augmentingService;
private readonly IIdentificationService _identificationService; private readonly IIdentificationService _identificationService;
@ -61,6 +72,7 @@ namespace NzbDrone.Core.MediaFiles.TrackImport
public ImportDecisionMaker(IEnumerable<IImportDecisionEngineSpecification<LocalTrack>> trackSpecifications, public ImportDecisionMaker(IEnumerable<IImportDecisionEngineSpecification<LocalTrack>> trackSpecifications,
IEnumerable<IImportDecisionEngineSpecification<LocalAlbumRelease>> albumSpecifications, IEnumerable<IImportDecisionEngineSpecification<LocalAlbumRelease>> albumSpecifications,
IMediaFileService mediaFileService, IMediaFileService mediaFileService,
IParsingService parsingService,
IAudioTagService audioTagService, IAudioTagService audioTagService,
IAugmentingService augmentingService, IAugmentingService augmentingService,
IIdentificationService identificationService, IIdentificationService identificationService,
@ -71,6 +83,7 @@ namespace NzbDrone.Core.MediaFiles.TrackImport
_trackSpecifications = trackSpecifications; _trackSpecifications = trackSpecifications;
_albumSpecifications = albumSpecifications; _albumSpecifications = albumSpecifications;
_mediaFileService = mediaFileService; _mediaFileService = mediaFileService;
_parsingService = parsingService;
_audioTagService = audioTagService; _audioTagService = audioTagService;
_augmentingService = augmentingService; _augmentingService = augmentingService;
_identificationService = identificationService; _identificationService = identificationService;
@ -79,6 +92,46 @@ namespace NzbDrone.Core.MediaFiles.TrackImport
_logger = logger; _logger = logger;
} }
public CueSheetInfo GetCueSheetInfo(IFileInfo cueFile, List<IFileInfo> musicFiles)
{
var cueSheetInfo = new CueSheetInfo();
var cueSheet = new CueSheet(cueFile);
if (cueSheet == null)
{
return cueSheetInfo;
}
cueSheetInfo.CueSheet = cueSheet;
cueSheetInfo.IdOverrides = new IdentificationOverrides();
Artist artistFromCue = null;
if (!cueSheet.Performer.Empty())
{
artistFromCue = _parsingService.GetArtist(cueSheet.Performer);
if (artistFromCue != null)
{
cueSheetInfo.IdOverrides.Artist = artistFromCue;
}
}
var parsedAlbumInfo = new ParsedAlbumInfo
{
AlbumTitle = cueSheet.Title,
ArtistName = artistFromCue.Name,
ReleaseDate = cueSheet.Date,
};
var albumsFromCue = _parsingService.GetAlbums(parsedAlbumInfo, artistFromCue);
if (albumsFromCue != null && albumsFromCue.Count > 0)
{
cueSheetInfo.IdOverrides.Album = albumsFromCue[0];
}
cueSheetInfo.MusicFiles = musicFiles.Where(musicFile => cueSheet.Files.Any(musicFileFromCue => musicFileFromCue.Name == musicFile.Name)).ToList();
return cueSheetInfo;
}
public Tuple<List<LocalTrack>, List<ImportDecision<LocalTrack>>> GetLocalTracks(List<IFileInfo> musicFiles, DownloadClientItem downloadClientItem, ParsedAlbumInfo folderInfo, FilterFilesType filter) public Tuple<List<LocalTrack>, List<ImportDecision<LocalTrack>>> GetLocalTracks(List<IFileInfo> musicFiles, DownloadClientItem downloadClientItem, ParsedAlbumInfo folderInfo, FilterFilesType filter)
{ {
var watch = new System.Diagnostics.Stopwatch(); var watch = new System.Diagnostics.Stopwatch();
@ -116,7 +169,7 @@ namespace NzbDrone.Core.MediaFiles.TrackImport
Size = file.Length, Size = file.Length,
Modified = file.LastWriteTimeUtc, Modified = file.LastWriteTimeUtc,
FileTrackInfo = _audioTagService.ReadTags(file.FullName), FileTrackInfo = _audioTagService.ReadTags(file.FullName),
AdditionalFile = false AdditionalFile = false,
}; };
try try
@ -142,7 +195,7 @@ namespace NzbDrone.Core.MediaFiles.TrackImport
return Tuple.Create(localTracks, decisions); return Tuple.Create(localTracks, decisions);
} }
public List<ImportDecision<LocalTrack>> GetImportDecisions(List<IFileInfo> musicFiles, IdentificationOverrides idOverrides, ImportDecisionMakerInfo itemInfo, ImportDecisionMakerConfig config) public List<ImportDecision<LocalTrack>> GetImportDecisions(List<IFileInfo> musicFiles, IdentificationOverrides idOverrides, ImportDecisionMakerInfo itemInfo, ImportDecisionMakerConfig config, List<CueSheetInfo> cueSheetInfos)
{ {
idOverrides ??= new IdentificationOverrides(); idOverrides ??= new IdentificationOverrides();
itemInfo ??= new ImportDecisionMakerInfo(); itemInfo ??= new ImportDecisionMakerInfo();
@ -152,14 +205,39 @@ namespace NzbDrone.Core.MediaFiles.TrackImport
var decisions = trackData.Item2; var decisions = trackData.Item2;
localTracks.ForEach(x => x.ExistingFile = !config.NewDownload); localTracks.ForEach(x => x.ExistingFile = !config.NewDownload);
localTracks.ForEach(x => x.IsSingleFileRelease = itemInfo.IsSingleFileRelease); if (cueSheetInfos != null)
if (itemInfo.IsSingleFileRelease)
{ {
localTracks.ForEach(x => x.Artist = idOverrides.Artist); localTracks.ForEach(localTrack =>
localTracks.ForEach(x => x.Album = idOverrides.Album); {
var cueSheetFindResult = cueSheetInfos.Find(x => x.IsForMediaFile(localTrack.Path));
var cueSheet = cueSheetFindResult?.CueSheet;
if (cueSheet != null)
{
localTrack.IsSingleFileRelease = cueSheet.IsSingleFileRelease;
localTrack.Artist = idOverrides.Artist;
localTrack.Album = idOverrides.Album;
}
});
} }
var releases = _identificationService.Identify(localTracks, idOverrides, config); var localTracksByAlbums = localTracks.GroupBy(x => x.Album);
foreach (var localTracksByAlbum in localTracksByAlbums)
{
if (!localTracksByAlbum.All(x => x.IsSingleFileRelease == true))
{
continue;
}
localTracks.ForEach(x =>
{
if (x.IsSingleFileRelease && localTracksByAlbum.Contains(x))
{
x.FileTrackInfo.DiscCount = localTracksByAlbum.Count();
}
});
}
var releases = _identificationService.Identify(localTracks, idOverrides, config, cueSheetInfos);
var albums = releases.GroupBy(x => x.AlbumRelease?.Album?.Value.ForeignAlbumId); var albums = releases.GroupBy(x => x.AlbumRelease?.Album?.Value.ForeignAlbumId);
@ -206,6 +284,17 @@ namespace NzbDrone.Core.MediaFiles.TrackImport
return decisions; return decisions;
} }
public List<ImportDecision<LocalTrack>> GetImportDecisions(List<CueSheetInfo> cueSheetInfos, ImportDecisionMakerInfo itemInfo, ImportDecisionMakerConfig config)
{
var decisions = new List<ImportDecision<LocalTrack>>();
foreach (var cueSheetInfo in cueSheetInfos)
{
decisions.AddRange(GetImportDecisions(cueSheetInfo.MusicFiles, cueSheetInfo.IdOverrides, itemInfo, config, cueSheetInfos));
}
return decisions;
}
private void EnsureData(LocalAlbumRelease release) private void EnsureData(LocalAlbumRelease release)
{ {
if (release.AlbumRelease != null && release.AlbumRelease.Album.Value.Artist.Value.QualityProfileId == 0) if (release.AlbumRelease != null && release.AlbumRelease.Album.Value.Artist.Value.QualityProfileId == 0)

View file

@ -10,6 +10,7 @@ using NzbDrone.Common.Disk;
using NzbDrone.Common.Extensions; using NzbDrone.Common.Extensions;
using NzbDrone.Common.Instrumentation.Extensions; using NzbDrone.Common.Instrumentation.Extensions;
using NzbDrone.Core.CustomFormats; using NzbDrone.Core.CustomFormats;
using NzbDrone.Core.Datastore;
using NzbDrone.Core.DecisionEngine; using NzbDrone.Core.DecisionEngine;
using NzbDrone.Core.Download; using NzbDrone.Core.Download;
using NzbDrone.Core.Download.TrackedDownloads; using NzbDrone.Core.Download.TrackedDownloads;
@ -155,63 +156,67 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Manual
// Split cue and non-cue files // Split cue and non-cue files
var cueFiles = audioFiles.Where(x => x.Extension.Equals(".cue")).ToList(); var cueFiles = audioFiles.Where(x => x.Extension.Equals(".cue")).ToList();
audioFiles.RemoveAll(l => cueFiles.Contains(l)); audioFiles.RemoveAll(l => cueFiles.Contains(l));
var cueSheetInfos = new List<CueSheetInfo>();
foreach (var cueFile in cueFiles) foreach (var cueFile in cueFiles)
{ {
var cueSheet = new CueSheet(cueFile); var cueSheetInfo = _importDecisionMaker.GetCueSheetInfo(cueFile, audioFiles);
cueSheetInfos.Add(cueSheetInfo);
Artist artistFromCue = null;
if (!cueSheet.Performer.Empty())
{
artistFromCue = _parsingService.GetArtist(cueSheet.Performer);
}
if (artistFromCue == null)
{
continue;
}
// TODO use the audio files from the cue sheet
var validAudioFiles = audioFiles.FindAll(x => cueSheet.FileNames.Contains(x.Name));
if (validAudioFiles.Count == 0)
{
continue;
}
var parsedAlbumInfo = new ParsedAlbumInfo
{
AlbumTitle = cueSheet.Title,
ArtistName = artistFromCue.Name,
ReleaseDate = cueSheet.Date,
};
var albumsFromCue = _parsingService.GetAlbums(parsedAlbumInfo, artistFromCue);
if (albumsFromCue == null || albumsFromCue.Count == 0)
{
continue;
}
results.AddRange(ProcessFolder(downloadId, artistFromCue, albumsFromCue[0], filter, replaceExistingFiles, downloadClientItem, cueSheet.Title, validAudioFiles, cueSheet));
audioFiles.RemoveAll(x => validAudioFiles.Contains(x));
} }
results.AddRange(ProcessFolder(downloadId, artist, null, filter, replaceExistingFiles, downloadClientItem, directoryInfo.Name, audioFiles, null)); var cueSheetInfosGroupedByDiscId = cueSheetInfos.GroupBy(x => x.CueSheet.DiscID).ToList();
foreach (var cueSheetInfoGroup in cueSheetInfosGroupedByDiscId)
{
var audioFilesForCues = new List<IFileInfo>();
foreach (var cueSheetInfo in cueSheetInfoGroup)
{
audioFilesForCues.AddRange(cueSheetInfo.MusicFiles);
}
var manualImportItems = ProcessFolder(downloadId, cueSheetInfos[0].IdOverrides, filter, replaceExistingFiles, downloadClientItem, cueSheetInfos[0].IdOverrides.Album.Title, audioFilesForCues, cueSheetInfos);
results.AddRange(manualImportItems);
RemoveProcessedAudioFiles(audioFiles, cueSheetInfos, manualImportItems);
}
var idOverrides = new IdentificationOverrides
{
Artist = artist,
Album = null
};
results.AddRange(ProcessFolder(downloadId, idOverrides, filter, replaceExistingFiles, downloadClientItem, directoryInfo.Name, audioFiles));
return results; return results;
} }
private List<ManualImportItem> ProcessFolder(string downloadId, Artist overrideArtist, Album overrideAlbum, FilterFilesType filter, bool replaceExistingFiles, DownloadClientItem downloadClientItem, string albumTitle, List<IFileInfo> audioFiles, CueSheet cueSheet) private void RemoveProcessedAudioFiles(List<IFileInfo> audioFiles, List<CueSheetInfo> cueSheetInfos, List<ManualImportItem> manualImportItems)
{ {
var idOverrides = new IdentificationOverrides foreach (var cueSheetInfo in cueSheetInfos)
{ {
Artist = overrideArtist, if (cueSheetInfo.CueSheet != null)
Album = overrideAlbum {
}; manualImportItems.ForEach(item =>
{
if (cueSheetInfo.IsForMediaFile(item.Path))
{
item.CueSheetPath = cueSheetInfo.CueSheet.Path;
}
});
}
audioFiles.RemoveAll(x => cueSheetInfo.MusicFiles.Contains(x));
}
}
private List<ManualImportItem> ProcessFolder(string downloadId, IdentificationOverrides idOverrides, FilterFilesType filter, bool replaceExistingFiles, DownloadClientItem downloadClientItem, string albumTitle, List<IFileInfo> audioFiles, List<CueSheetInfo> cueSheetInfos = null)
{
idOverrides ??= new IdentificationOverrides();
var itemInfo = new ImportDecisionMakerInfo var itemInfo = new ImportDecisionMakerInfo
{ {
DownloadClientItem = downloadClientItem, DownloadClientItem = downloadClientItem,
ParsedAlbumInfo = Parser.Parser.ParseAlbumTitle(albumTitle), ParsedAlbumInfo = Parser.Parser.ParseAlbumTitle(albumTitle),
CueSheet = cueSheet,
IsSingleFileRelease = cueSheet != null ? cueSheet.IsSingleFileRelease : false,
}; };
var config = new ImportDecisionMakerConfig var config = new ImportDecisionMakerConfig
{ {
Filter = filter, Filter = filter,
@ -221,7 +226,7 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Manual
AddNewArtists = false AddNewArtists = false
}; };
var decisions = _importDecisionMaker.GetImportDecisions(audioFiles, idOverrides, itemInfo, config); var decisions = _importDecisionMaker.GetImportDecisions(audioFiles, idOverrides, itemInfo, config, cueSheetInfos);
// paths will be different for new and old files which is why we need to map separately // paths will be different for new and old files which is why we need to map separately
var newFiles = audioFiles.Join(decisions, var newFiles = audioFiles.Join(decisions,
@ -230,16 +235,12 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Manual
(f, d) => new { File = f, Decision = d }, (f, d) => new { File = f, Decision = d },
PathEqualityComparer.Instance); PathEqualityComparer.Instance);
var newItems = newFiles.Select(x => MapItem(x.Decision, downloadId, replaceExistingFiles, false)); var newItemsList = newFiles.Select(x => MapItem(x.Decision, downloadId, replaceExistingFiles, false)).ToList();
var existingDecisions = decisions.Except(newFiles.Select(x => x.Decision)); var existingDecisions = decisions.Except(newFiles.Select(x => x.Decision));
var existingItems = existingDecisions.Select(x => MapItem(x, null, replaceExistingFiles, false)); var existingItems = existingDecisions.Select(x => MapItem(x, null, replaceExistingFiles, false));
var itemsList = newItems.Concat(existingItems).ToList(); var itemsList = newItemsList.Concat(existingItems.ToList()).ToList();
if (cueSheet != null)
{
itemsList.ForEach(item => { item.CueSheetPath = cueSheet.Path; });
}
return itemsList; return itemsList;
} }
@ -257,13 +258,6 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Manual
var disableReleaseSwitching = group.First().DisableReleaseSwitching; var disableReleaseSwitching = group.First().DisableReleaseSwitching;
var files = group.Select(x => _diskProvider.GetFileInfo(x.Path)).ToList();
var idOverride = new IdentificationOverrides
{
Artist = group.First().Artist,
Album = group.First().Album,
AlbumRelease = group.First().Release
};
var config = new ImportDecisionMakerConfig var config = new ImportDecisionMakerConfig
{ {
Filter = FilterFilesType.None, Filter = FilterFilesType.None,
@ -273,64 +267,99 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Manual
AddNewArtists = false AddNewArtists = false
}; };
var itemInfo = new ImportDecisionMakerInfo var audioFiles = new List<IFileInfo>();
foreach (var item in group)
{ {
IsSingleFileRelease = group.All(x => x.IsSingleFileRelease == true) var file = _diskProvider.GetFileInfo(item.Path);
}; audioFiles.Add(file);
// TODO support with the cue sheet
var decisions = _importDecisionMaker.GetImportDecisions(files, idOverride, itemInfo, config);
var existingItems = group.Join(decisions,
i => i.Path,
d => d.Item.Path,
(i, d) => new { Item = i, Decision = d },
PathEqualityComparer.Instance);
foreach (var pair in existingItems)
{
var item = pair.Item;
var decision = pair.Decision;
if (decision.Item.Artist != null)
{
item.Artist = decision.Item.Artist;
}
if (decision.Item.Album != null)
{
item.Album = decision.Item.Album;
item.Release = decision.Item.Release;
}
if (decision.Item.Tracks.Any())
{
item.Tracks = decision.Item.Tracks;
}
if (item.Quality?.Quality == Quality.Unknown)
{
item.Quality = decision.Item.Quality;
}
if (item.ReleaseGroup.IsNullOrWhiteSpace())
{
item.ReleaseGroup = decision.Item.ReleaseGroup;
}
item.Rejections = decision.Rejections;
item.Size = decision.Item.Size;
result.Add(item);
} }
var newDecisions = decisions.Except(existingItems.Select(x => x.Decision)); var cueSheetInfos = new List<CueSheetInfo>();
result.AddRange(newDecisions.Select(x => MapItem(x, null, replaceExistingFiles, disableReleaseSwitching))); var audioFilesForCues = new List<IFileInfo>();
var itemInfo = new ImportDecisionMakerInfo();
foreach (var item in group)
{
if (item.IsSingleFileRelease)
{
var cueFile = _diskProvider.GetFileInfo(item.CueSheetPath);
var cueSheetInfo = _importDecisionMaker.GetCueSheetInfo(cueFile, audioFiles);
cueSheetInfos.Add(cueSheetInfo);
audioFilesForCues.AddRange(cueSheetInfo.MusicFiles);
}
}
var singleFileReleaseDecisions = _importDecisionMaker.GetImportDecisions(audioFilesForCues, cueSheetInfos[0].IdOverrides, itemInfo, config, cueSheetInfos);
var manualImportItems = UpdateItems(group, singleFileReleaseDecisions, replaceExistingFiles, disableReleaseSwitching);
result.AddRange(manualImportItems);
RemoveProcessedAudioFiles(audioFiles, cueSheetInfos, manualImportItems);
var idOverride = new IdentificationOverrides
{
Artist = group.First().Artist,
Album = group.First().Album,
AlbumRelease = group.First().Release
};
var decisions = _importDecisionMaker.GetImportDecisions(audioFiles, idOverride, itemInfo, config);
result.AddRange(UpdateItems(group, decisions, replaceExistingFiles, disableReleaseSwitching));
} }
return result; return result;
} }
private List<ManualImportItem> UpdateItems(IGrouping<int?, ManualImportItem> group, List<ImportDecision<LocalTrack>> decisions, bool replaceExistingFiles, bool disableReleaseSwitching)
{
var result = new List<ManualImportItem>();
var existingItems = group.Join(decisions,
i => i.Path,
d => d.Item.Path,
(i, d) => new { Item = i, Decision = d },
PathEqualityComparer.Instance);
foreach (var pair in existingItems)
{
var item = pair.Item;
var decision = pair.Decision;
if (decision.Item.Artist != null)
{
item.Artist = decision.Item.Artist;
}
if (decision.Item.Album != null)
{
item.Album = decision.Item.Album;
item.Release = decision.Item.Release;
}
if (decision.Item.Tracks.Any())
{
item.Tracks = decision.Item.Tracks;
}
if (item.Quality?.Quality == Quality.Unknown)
{
item.Quality = decision.Item.Quality;
}
if (item.ReleaseGroup.IsNullOrWhiteSpace())
{
item.ReleaseGroup = decision.Item.ReleaseGroup;
}
item.Rejections = decision.Rejections;
item.Size = decision.Item.Size;
result.Add(item);
}
var newDecisions = decisions.Except(existingItems.Select(x => x.Decision));
result.AddRange(newDecisions.Select(x => MapItem(x, null, replaceExistingFiles, disableReleaseSwitching)));
return result;
}
private ManualImportItem MapItem(ImportDecision<LocalTrack> decision, string downloadId, bool replaceExistingFiles, bool disableReleaseSwitching) private ManualImportItem MapItem(ImportDecision<LocalTrack> decision, string downloadId, bool replaceExistingFiles, bool disableReleaseSwitching)
{ {
var item = new ManualImportItem(); var item = new ManualImportItem();