New: Aria2

Fixes #2334
Fixes #2385

src/NzbDrone.Common/Extensions/PathExtensions.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Text.RegularExpressions;
 using NzbDrone.Common.Disk;
 using NzbDrone.Common.EnsureThat;
@@ -238,6 +239,33 @@ namespace NzbDrone.Common.Extensions
             return null;
         }
 
+        public static string GetLongestCommonPath(this List<string> paths)
+        {
+            var firstPath = paths.First();
+            var length = firstPath.Length;
+
+            for (int i = 1; i < paths.Count; i++)
+            {
+                var path = paths[i];
+
+                length = Math.Min(length, path.Length);
+
+                for (int characterIndex = 0; characterIndex < length; characterIndex++)
+                {
+                    if (path[characterIndex] != firstPath[characterIndex])
+                    {
+                        length = characterIndex;
+                        break;
+                    }
+                }
+            }
+
+            var substring = firstPath.Substring(0, length);
+            var lastSeparatorIndex = substring.LastIndexOfAny(new[] { '/', '\\' });
+
+            return substring.Substring(0, lastSeparatorIndex);
+        }
+
         public static string ProcessNameToExe(this string processName, PlatformType runtime)
         {
             if (OsInfo.IsWindows || runtime != PlatformType.NetCore)

src/NzbDrone.Core/Download/Clients/Aria2/Aria2.cs

@@ -0,0 +1,268 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using CookComputing.XmlRpc;
+using FluentValidation.Results;
+using NLog;
+using NzbDrone.Common.Disk;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Configuration;
+using NzbDrone.Core.MediaFiles.TorrentInfo;
+using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.RemotePathMappings;
+using NzbDrone.Core.Validation;
+
+namespace NzbDrone.Core.Download.Clients.Aria2
+{
+    public class Aria2 : TorrentClientBase<Aria2Settings>
+    {
+        private readonly IAria2Proxy _proxy;
+
+        public override string Name => "Aria2";
+
+        public Aria2(IAria2Proxy proxy,
+                        ITorrentFileInfoReader torrentFileInfoReader,
+                        IHttpClient httpClient,
+                        IConfigService configService,
+                        IDiskProvider diskProvider,
+                        IRemotePathMappingService remotePathMappingService,
+                        Logger logger)
+            : base(torrentFileInfoReader, httpClient, configService, diskProvider, remotePathMappingService, logger)
+        {
+            _proxy = proxy;
+        }
+
+        protected override string AddFromMagnetLink(RemoteAlbum remoteAlbum, string hash, string magnetLink)
+        {
+            var gid = _proxy.AddMagnet(Settings, magnetLink);
+
+            var tries = 10;
+            var retryDelay = 500;
+
+            // Wait a bit for the magnet to be resolved.
+            if (!WaitForTorrent(gid, hash, tries, retryDelay))
+            {
+                _logger.Warn($"Aria2 could not add magnent within {tries * retryDelay / 1000} seconds, download may remain stuck: {magnetLink}.");
+                return hash;
+            }
+
+            _logger.Debug($"Aria2 AddFromMagnetLink '{hash}' -> '{gid}'");
+
+            return hash;
+        }
+
+        protected override string AddFromTorrentFile(RemoteAlbum remoteAlbum, string hash, string filename, byte[] fileContent)
+        {
+            var gid = _proxy.AddTorrent(Settings, fileContent);
+
+            var tries = 10;
+            var retryDelay = 500;
+
+            // Wait a bit for the magnet to be resolved.
+            if (!WaitForTorrent(gid, hash, tries, retryDelay))
+            {
+                _logger.Warn($"Aria2 could not add torrent within {tries * retryDelay / 1000} seconds, download may remain stuck: {filename}.");
+                return hash;
+            }
+
+            return hash;
+        }
+
+        public override IEnumerable<DownloadClientItem> GetItems()
+        {
+            var torrents = _proxy.GetTorrents(Settings);
+
+            foreach (var torrent in torrents)
+            {
+                var firstFile = torrent.Files?.FirstOrDefault();
+
+                //skip metadata download
+                if (firstFile?.Path?.Contains("[METADATA]") == true)
+                {
+                    continue;
+                }
+
+                var completedLength = long.Parse(torrent.CompletedLength);
+                var totalLength = long.Parse(torrent.TotalLength);
+                var uploadedLength = long.Parse(torrent.UploadLength);
+                var downloadSpeed = long.Parse(torrent.DownloadSpeed);
+
+                var status = DownloadItemStatus.Failed;
+                var title = "";
+
+                if (torrent.Bittorrent?.ContainsKey("info") == true && ((XmlRpcStruct)torrent.Bittorrent["info"]).ContainsKey("name"))
+                {
+                    title = ((XmlRpcStruct)torrent.Bittorrent["info"])["name"].ToString();
+                }
+
+                switch (torrent.Status)
+                {
+                    case "active":
+                        if (completedLength == totalLength)
+                        {
+                            status = DownloadItemStatus.Completed;
+                        }
+                        else
+                        {
+                            status = DownloadItemStatus.Downloading;
+                        }
+
+                        break;
+                    case "waiting":
+                        status = DownloadItemStatus.Queued;
+                        break;
+                    case "paused":
+                        status = DownloadItemStatus.Paused;
+                        break;
+                    case "error":
+                        status = DownloadItemStatus.Failed;
+                        break;
+                    case "complete":
+                        status = DownloadItemStatus.Completed;
+                        break;
+                    case "removed":
+                        status = DownloadItemStatus.Failed;
+                        break;
+                }
+
+                _logger.Trace($"- aria2 getstatus hash:'{torrent.InfoHash}' gid:'{torrent.Gid}' status:'{status}' total:{totalLength} completed:'{completedLength}'");
+
+                var outputPath = _remotePathMappingService.RemapRemoteToLocal(Settings.Host, new OsPath(GetOutputPath(torrent)));
+
+                yield return new DownloadClientItem
+                {
+                    CanMoveFiles = false,
+                    CanBeRemoved = torrent.Status == "complete",
+                    Category = null,
+                    DownloadClientInfo = DownloadClientItemClientInfo.FromDownloadClient(this),
+                    DownloadId = torrent.InfoHash?.ToUpper(),
+                    IsEncrypted = false,
+                    Message = torrent.ErrorMessage,
+                    OutputPath = outputPath,
+                    RemainingSize = totalLength - completedLength,
+                    RemainingTime = downloadSpeed == 0 ? (TimeSpan?)null : new TimeSpan(0, 0, (int)((totalLength - completedLength) / downloadSpeed)),
+                    Removed = torrent.Status == "removed",
+                    SeedRatio = totalLength > 0 ? (double)uploadedLength / totalLength : 0,
+                    Status = status,
+                    Title = title,
+                    TotalSize = totalLength,
+                };
+            }
+        }
+
+        public override void RemoveItem(string downloadId, bool deleteData)
+        {
+            // Aria2 doesn't support file deletion: https://github.com/aria2/aria2/issues/728
+            var hash = downloadId.ToLower();
+            var aria2Item = _proxy.GetTorrents(Settings).FirstOrDefault(t => t.InfoHash?.ToLower() == hash);
+
+            if (aria2Item == null)
+            {
+                _logger.Error($"Aria2 could not find infoHash '{hash}' for deletion.");
+                return;
+            }
+
+            _logger.Debug($"Aria2 removing hash:'{hash}' gid:'{aria2Item.Gid}'");
+
+            if (aria2Item.Status == "complete" || aria2Item.Status == "error" || aria2Item.Status == "removed")
+            {
+                if (!_proxy.RemoveCompletedTorrent(Settings, aria2Item.Gid))
+                {
+                    _logger.Error($"Aria2 error while deleting {hash}.");
+
+                    return;
+                }
+            }
+            else
+            {
+                if (!_proxy.RemoveTorrent(Settings, aria2Item.Gid))
+                {
+                    _logger.Error($"Aria2 error while deleting {hash}.");
+
+                    return;
+                }
+            }
+
+            if (deleteData)
+            {
+                DeleteItemData(downloadId);
+            }
+        }
+
+        public override DownloadClientInfo GetStatus()
+        {
+            var destDir = _proxy.GetGlobals(Settings);
+
+            return new DownloadClientInfo
+            {
+                IsLocalhost = Settings.Host.Contains("127.0.0.1") || Settings.Host.Contains("localhost"),
+                OutputRootFolders = new List<OsPath> { _remotePathMappingService.RemapRemoteToLocal(Settings.Host, new OsPath(destDir["dir"])) }
+            };
+        }
+
+        private bool WaitForTorrent(string gid, string hash, int tries, int retryDelay)
+        {
+            for (var i = 0; i < tries; i++)
+            {
+                var found = _proxy.GetFromGID(Settings, gid);
+
+                if (found?.InfoHash?.ToLower() == hash?.ToLower())
+                {
+                    return true;
+                }
+
+                Thread.Sleep(retryDelay);
+            }
+
+            _logger.Debug("Could not find hash {0} in {1} tries at {2} ms intervals.", hash, tries, retryDelay);
+
+            return false;
+        }
+
+        protected override void Test(List<ValidationFailure> failures)
+        {
+            failures.AddIfNotNull(TestConnection());
+
+            if (failures.HasErrors())
+            {
+                return;
+            }
+        }
+
+        private ValidationFailure TestConnection()
+        {
+            try
+            {
+                var version = _proxy.GetVersion(Settings);
+
+                if (new Version(version) < new Version("1.34.0"))
+                {
+                    return new ValidationFailure(string.Empty, "Aria2 version should be at least 1.34.0. Version reported is {0}", version);
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.Error(ex, "Failed to test Aria2");
+
+                return new NzbDroneValidationFailure("Host", "Unable to connect to Aria2")
+                {
+                    DetailedDescription = ex.Message
+                };
+            }
+
+            return null;
+        }
+
+        private string GetOutputPath(Aria2Status torrent)
+        {
+            if (torrent.Files.Length == 1)
+            {
+                return torrent.Files.First().Path;
+            }
+
+            return torrent.Files.Select(f => f.Path).ToList().GetLongestCommonPath();
+        }
+    }
+}

src/NzbDrone.Core/Download/Clients/Aria2/Aria2Containers.cs

@@ -0,0 +1,111 @@
+using CookComputing.XmlRpc;
+
+namespace NzbDrone.Core.Download.Clients.Aria2
+{
+    public class Aria2Version
+    {
+        [XmlRpcMember("version")]
+        public string Version;
+
+        [XmlRpcMember("enabledFeatures")]
+        public string[] EnabledFeatures;
+    }
+
+    public class Aria2Uri
+    {
+        [XmlRpcMember("status")]
+        public string Status;
+
+        [XmlRpcMember("uri")]
+        public string Uri;
+    }
+
+    public class Aria2File
+    {
+        [XmlRpcMember("index")]
+        public string Index;
+
+        [XmlRpcMember("length")]
+        public string Length;
+
+        [XmlRpcMember("completedLength")]
+        public string CompletedLength;
+
+        [XmlRpcMember("path")]
+        public string Path;
+
+        [XmlRpcMember("selected")]
+        [XmlRpcMissingMapping(MappingAction.Ignore)]
+        public string Selected;
+
+        [XmlRpcMember("uris")]
+        [XmlRpcMissingMapping(MappingAction.Ignore)]
+        public Aria2Uri[] Uris;
+    }
+
+    public class Aria2Status
+    {
+        [XmlRpcMember("bittorrent")]
+        [XmlRpcMissingMapping(MappingAction.Ignore)]
+        public XmlRpcStruct Bittorrent;
+
+        [XmlRpcMember("bitfield")]
+        [XmlRpcMissingMapping(MappingAction.Ignore)]
+        public string Bitfield;
+
+        [XmlRpcMember("infoHash")]
+        [XmlRpcMissingMapping(MappingAction.Ignore)]
+        public string InfoHash;
+
+        [XmlRpcMember("completedLength")]
+        [XmlRpcMissingMapping(MappingAction.Ignore)]
+        public string CompletedLength;
+
+        [XmlRpcMember("connections")]
+        [XmlRpcMissingMapping(MappingAction.Ignore)]
+        public string Connections;
+
+        [XmlRpcMember("dir")]
+        [XmlRpcMissingMapping(MappingAction.Ignore)]
+        public string Dir;
+
+        [XmlRpcMember("downloadSpeed")]
+        [XmlRpcMissingMapping(MappingAction.Ignore)]
+        public string DownloadSpeed;
+
+        [XmlRpcMember("files")]
+        [XmlRpcMissingMapping(MappingAction.Ignore)]
+        public Aria2File[] Files;
+
+        [XmlRpcMember("gid")]
+        public string Gid;
+
+        [XmlRpcMember("numPieces")]
+        [XmlRpcMissingMapping(MappingAction.Ignore)]
+        public string NumPieces;
+
+        [XmlRpcMember("pieceLength")]
+        [XmlRpcMissingMapping(MappingAction.Ignore)]
+        public string PieceLength;
+
+        [XmlRpcMember("status")]
+        [XmlRpcMissingMapping(MappingAction.Ignore)]
+        public string Status;
+
+        [XmlRpcMember("totalLength")]
+        [XmlRpcMissingMapping(MappingAction.Ignore)]
+        public string TotalLength;
+
+        [XmlRpcMember("uploadLength")]
+        [XmlRpcMissingMapping(MappingAction.Ignore)]
+        public string UploadLength;
+
+        [XmlRpcMember("uploadSpeed")]
+        [XmlRpcMissingMapping(MappingAction.Ignore)]
+        public string UploadSpeed;
+
+        [XmlRpcMember("errorMessage")]
+        [XmlRpcMissingMapping(MappingAction.Ignore)]
+        public string ErrorMessage;
+    }
+}

src/NzbDrone.Core/Download/Clients/Aria2/Aria2Proxy.cs

@@ -0,0 +1,225 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Net;
+using CookComputing.XmlRpc;
+using NLog;
+
+namespace NzbDrone.Core.Download.Clients.Aria2
+{
+    public interface IAria2Proxy
+    {
+        string GetVersion(Aria2Settings settings);
+        string AddMagnet(Aria2Settings settings, string magnet);
+        string AddTorrent(Aria2Settings settings, byte[] torrent);
+        bool RemoveTorrent(Aria2Settings settings, string gid);
+        bool RemoveCompletedTorrent(Aria2Settings settings, string gid);
+        Dictionary<string, string> GetGlobals(Aria2Settings settings);
+        List<Aria2Status> GetTorrents(Aria2Settings settings);
+        Aria2Status GetFromGID(Aria2Settings settings, string gid);
+    }
+
+    public interface IAria2 : IXmlRpcProxy
+    {
+        [XmlRpcMethod("aria2.getVersion")]
+        Aria2Version GetVersion(string token);
+
+        [XmlRpcMethod("aria2.addUri")]
+        string AddUri(string token, string[] uri);
+
+        [XmlRpcMethod("aria2.addTorrent")]
+        string AddTorrent(string token, byte[] torrent);
+
+        [XmlRpcMethod("aria2.forceRemove")]
+        string Remove(string token, string gid);
+
+        [XmlRpcMethod("aria2.removeDownloadResult")]
+        string RemoveResult(string token, string gid);
+
+        [XmlRpcMethod("aria2.tellStatus")]
+        Aria2Status GetFromGid(string token, string gid);
+
+        [XmlRpcMethod("aria2.getGlobalOption")]
+        XmlRpcStruct GetGlobalOption(string token);
+
+        [XmlRpcMethod("aria2.tellActive")]
+        Aria2Status[] GetActive(string token);
+
+        [XmlRpcMethod("aria2.tellWaiting")]
+        Aria2Status[] GetWaiting(string token, int offset, int num);
+
+        [XmlRpcMethod("aria2.tellStopped")]
+        Aria2Status[] GetStopped(string token, int offset, int num);
+    }
+
+    public class Aria2Proxy : IAria2Proxy
+    {
+        private readonly Logger _logger;
+
+        public Aria2Proxy(Logger logger)
+        {
+            _logger = logger;
+        }
+
+        private string GetToken(Aria2Settings settings)
+        {
+            return $"token:{settings?.SecretToken}";
+        }
+
+        private string GetURL(Aria2Settings settings)
+        {
+            return $"http{(settings.UseSsl ? "s" : "")}://{settings.Host}:{settings.Port}{settings.RpcPath}";
+        }
+
+        public string GetVersion(Aria2Settings settings)
+        {
+            _logger.Trace("> aria2.getVersion");
+
+            var client = BuildClient(settings);
+            var version = ExecuteRequest(() => client.GetVersion(GetToken(settings)));
+
+            _logger.Trace("< aria2.getVersion");
+
+            return version.Version;
+        }
+
+        public Aria2Status GetFromGID(Aria2Settings settings, string gid)
+        {
+            _logger.Trace("> aria2.tellStatus");
+
+            var client = BuildClient(settings);
+            var found = ExecuteRequest(() => client.GetFromGid(GetToken(settings), gid));
+
+            _logger.Trace("< aria2.tellStatus");
+
+            return found;
+        }
+
+        public List<Aria2Status> GetTorrents(Aria2Settings settings)
+        {
+            _logger.Trace("> aria2.tellActive");
+
+            var client = BuildClient(settings);
+
+            var active = ExecuteRequest(() => client.GetActive(GetToken(settings)));
+
+            _logger.Trace("< aria2.tellActive");
+
+            _logger.Trace("> aria2.tellWaiting");
+
+            var waiting = ExecuteRequest(() => client.GetWaiting(GetToken(settings), 0, 10 * 1024));
+
+            _logger.Trace("< aria2.tellWaiting");
+
+            _logger.Trace("> aria2.tellStopped");
+
+            var stopped = ExecuteRequest(() => client.GetStopped(GetToken(settings), 0, 10 * 1024));
+
+            _logger.Trace("< aria2.tellStopped");
+
+            var items = new List<Aria2Status>();
+
+            items.AddRange(active);
+            items.AddRange(waiting);
+            items.AddRange(stopped);
+
+            return items;
+        }
+
+        public Dictionary<string, string> GetGlobals(Aria2Settings settings)
+        {
+            _logger.Trace("> aria2.getGlobalOption");
+
+            var client = BuildClient(settings);
+            var options = ExecuteRequest(() => client.GetGlobalOption(GetToken(settings)));
+
+            _logger.Trace("< aria2.getGlobalOption");
+
+            var ret = new Dictionary<string, string>();
+
+            foreach (DictionaryEntry option in options)
+            {
+                ret.Add(option.Key.ToString(), option.Value?.ToString());
+            }
+
+            return ret;
+        }
+
+        public string AddMagnet(Aria2Settings settings, string magnet)
+        {
+            _logger.Trace("> aria2.addUri");
+
+            var client = BuildClient(settings);
+            var gid = ExecuteRequest(() => client.AddUri(GetToken(settings), new[] { magnet }));
+
+            _logger.Trace("< aria2.addUri");
+
+            return gid;
+        }
+
+        public string AddTorrent(Aria2Settings settings, byte[] torrent)
+        {
+            _logger.Trace("> aria2.addTorrent");
+
+            var client = BuildClient(settings);
+            var gid = ExecuteRequest(() => client.AddTorrent(GetToken(settings), torrent));
+
+            _logger.Trace("< aria2.addTorrent");
+
+            return gid;
+        }
+
+        public bool RemoveTorrent(Aria2Settings settings, string gid)
+        {
+            _logger.Trace("> aria2.forceRemove");
+
+            var client = BuildClient(settings);
+            var gidres = ExecuteRequest(() => client.Remove(GetToken(settings), gid));
+
+            _logger.Trace("< aria2.forceRemove");
+
+            return gid == gidres;
+        }
+
+        public bool RemoveCompletedTorrent(Aria2Settings settings, string gid)
+        {
+            _logger.Trace("> aria2.removeDownloadResult");
+
+            var client = BuildClient(settings);
+            var result = ExecuteRequest(() => client.RemoveResult(GetToken(settings), gid));
+
+            _logger.Trace("< aria2.removeDownloadResult");
+
+            return result == "OK";
+        }
+
+        private IAria2 BuildClient(Aria2Settings settings)
+        {
+            var client = XmlRpcProxyGen.Create<IAria2>();
+            client.Url = GetURL(settings);
+
+            return client;
+        }
+
+        private T ExecuteRequest<T>(Func<T> task)
+        {
+            try
+            {
+                return task();
+            }
+            catch (XmlRpcServerException ex)
+            {
+                throw new DownloadClientException("Unable to connect to aria2, please check your settings", ex);
+            }
+            catch (WebException ex)
+            {
+                if (ex.Status == WebExceptionStatus.TrustFailure)
+                {
+                    throw new DownloadClientUnavailableException("Unable to connect to aria2, certificate validation failed.", ex);
+                }
+
+                throw new DownloadClientUnavailableException("Unable to connect to aria2, please check your settings", ex);
+            }
+        }
+    }
+}

src/NzbDrone.Core/Download/Clients/Aria2/Aria2Settings.cs

@@ -0,0 +1,49 @@
+using FluentValidation;
+using NzbDrone.Core.Annotations;
+using NzbDrone.Core.ThingiProvider;
+using NzbDrone.Core.Validation;
+
+namespace NzbDrone.Core.Download.Clients.Aria2
+{
+    public class Aria2SettingsValidator : AbstractValidator<Aria2Settings>
+    {
+        public Aria2SettingsValidator()
+        {
+            RuleFor(c => c.Host).ValidHost();
+        }
+    }
+
+    public class Aria2Settings : IProviderConfig
+    {
+        private static readonly Aria2SettingsValidator Validator = new Aria2SettingsValidator();
+
+        public Aria2Settings()
+        {
+            Host = "localhost";
+            Port = 6800;
+            RpcPath = "/rpc";
+            UseSsl = false;
+            SecretToken = "MySecretToken";
+        }
+
+        [FieldDefinition(0, Label = "Host", Type = FieldType.Textbox)]
+        public string Host { get; set; }
+
+        [FieldDefinition(1, Label = "Port", Type = FieldType.Number)]
+        public int Port { get; set; }
+
+        [FieldDefinition(2, Label = "RPC Path", Type = FieldType.Textbox)]
+        public string RpcPath { get; set; }
+
+        [FieldDefinition(3, Label = "Use SSL", Type = FieldType.Checkbox)]
+        public bool UseSsl { get; set; }
+
+        [FieldDefinition(4, Label = "Secret token", Type = FieldType.Password, Privacy = PrivacyLevel.Password)]
+        public string SecretToken { get; set; }
+
+        public NzbDroneValidationResult Validate()
+        {
+                return new NzbDroneValidationResult(Validator.Validate(this));
+            }
+    }
+}