Command queue

New: Adding multiple series will queue them instead of running all at once
New: Slower scheduled tasks won't be block others from running

src/NzbDrone.Core/Messaging/Commands/CommandExecutor.cs

@@ -1,117 +1,59 @@
 using System;
-using System.Linq;
-using System.Threading.Tasks;
+using System.Threading;
 using NLog;
 using NzbDrone.Common;
-using NzbDrone.Common.EnsureThat;
-using NzbDrone.Common.Serializer;
-using NzbDrone.Common.TPL;
-using NzbDrone.Core.Messaging.Commands.Tracking;
+using NzbDrone.Core.Lifecycle;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.ProgressMessaging;
 
 namespace NzbDrone.Core.Messaging.Commands
 {
-    public class CommandExecutor : ICommandExecutor
+    public class CommandExecutor : IHandle<ApplicationStartedEvent>,
+                                   IHandle<ApplicationShutdownRequested>
     {
         private readonly Logger _logger;
         private readonly IServiceFactory _serviceFactory;
-        private readonly ITrackCommands _trackCommands;
+        private readonly IManageCommandQueue _commandQueueManager;
         private readonly IEventAggregator _eventAggregator;
-        private readonly TaskFactory _taskFactory;
 
-        public CommandExecutor(Logger logger, IServiceFactory serviceFactory, ITrackCommands trackCommands, IEventAggregator eventAggregator)
+        private static CancellationTokenSource _cancellationTokenSource;
+        private const int THREAD_LIMIT = 3;
+
+        public CommandExecutor(IServiceFactory serviceFactory,
+                               IManageCommandQueue commandQueueManager,
+                               IEventAggregator eventAggregator,
+                               Logger logger)
         {
-            var scheduler = new LimitedConcurrencyLevelTaskScheduler(3);
-
             _logger = logger;
             _serviceFactory = serviceFactory;
-            _trackCommands = trackCommands;
+            _commandQueueManager = commandQueueManager;
             _eventAggregator = eventAggregator;
-            _taskFactory = new TaskFactory(scheduler);
         }
 
-        public void PublishCommand<TCommand>(TCommand command) where TCommand : Command
+        private void ExecuteCommands()
         {
-            Ensure.That(command, () => command).IsNotNull();
-
-            _logger.Trace("Publishing {0}", command.GetType().Name);
-
-            if (_trackCommands.FindExisting(command) != null)
+            try
             {
-                _logger.Trace("Command is already in progress: {0}", command.GetType().Name);
-                return;
+                foreach (var command in _commandQueueManager.Queue(_cancellationTokenSource.Token))
+                {
+                    try
+                    {
+                        ExecuteCommand((dynamic)command.Body, command);
+                    }
+                    catch (Exception ex)
+                    {
+                        _logger.ErrorException("Error occurred while executing task " + command.Name, ex);
+                    }
+                }
             }
-
-            _trackCommands.Store(command);
-
-            ExecuteCommand<TCommand>(command);
-        }
-
-        public void PublishCommand(string commandTypeName)
-        {
-            PublishCommand(commandTypeName, null);
-        }
-
-        public void PublishCommand(string commandTypeName, DateTime? lastExecutionTime)
-        {
-            dynamic command = GetCommand(commandTypeName);
-            command.LastExecutionTime = lastExecutionTime;
-
-            PublishCommand(command);
-        }
-
-        public Command PublishCommandAsync<TCommand>(TCommand command) where TCommand : Command
-        {
-            Ensure.That(command, () => command).IsNotNull();
-
-            _logger.Trace("Publishing {0}", command.GetType().Name);
-
-            var existingCommand = _trackCommands.FindExisting(command);
-
-            if (existingCommand != null)
+            catch (ThreadAbortException ex)
             {
-                _logger.Trace("Command is already in progress: {0}", command.GetType().Name);
-                return existingCommand;
+                _logger.ErrorException(ex.Message, ex);
+                Thread.ResetAbort();
             }
-
-            _trackCommands.Store(command);
-
-            // TODO: We should use async await (once we get 4.5) or normal Task Continuations on Command processing to prevent blocking the TaskScheduler.
-            //       For now we use TaskCreationOptions 0x10, which is actually .net 4.5 HideScheduler.
-            //       This will detach the scheduler from the thread, causing new Task creating in the command to be executed on the ThreadPool, avoiding a deadlock.
-            //       Please note that the issue only shows itself on mono because since Microsoft .net implementation supports Task inlining on WaitAll.
-            if (Enum.IsDefined(typeof(TaskCreationOptions), (TaskCreationOptions)0x10))
-            {
-                _taskFactory.StartNew(() => ExecuteCommand<TCommand>(command)
-                    , TaskCreationOptions.PreferFairness | (TaskCreationOptions)0x10)
-                    .LogExceptions();
-            }
-            else
-            {
-                _taskFactory.StartNew(() => ExecuteCommand<TCommand>(command)
-                    , TaskCreationOptions.PreferFairness)
-                    .LogExceptions();
-            }
-
-            return command;
         }
 
-        public Command PublishCommandAsync(string commandTypeName)
-        {
-            dynamic command = GetCommand(commandTypeName);
-            return PublishCommandAsync(command);
-        }
-
-        private dynamic GetCommand(string commandTypeName)
-        {
-            var commandType = _serviceFactory.GetImplementations(typeof(Command))
-                .Single(c => c.FullName.Equals(commandTypeName, StringComparison.InvariantCultureIgnoreCase));
-
-            return Json.Deserialize("{}", commandType);
-        }
-
-        private void ExecuteCommand<TCommand>(Command command) where TCommand : Command
+        private void ExecuteCommand<TCommand>(TCommand command, CommandModel commandModel) where TCommand : Command
         {
             var handlerContract = typeof(IExecute<>).MakeGenericType(command.GetType());
             var handler = (IExecute<TCommand>)_serviceFactory.Build(handlerContract);
@@ -120,47 +62,67 @@ namespace NzbDrone.Core.Messaging.Commands
 
             try
             {
-                _trackCommands.Start(command);
-                BroadcastCommandUpdate(command);
+                _commandQueueManager.Start(commandModel);
+                BroadcastCommandUpdate(commandModel);
 
-                if (!MappedDiagnosticsContext.Contains("CommandId") && command.SendUpdatesToClient)
+                if (!MappedDiagnosticsContext.Contains("CommandId"))
                 {
-                    MappedDiagnosticsContext.Set("CommandId", command.Id.ToString());
+                    MappedDiagnosticsContext.Set("CommandId", commandModel.Id.ToString());
                 }
 
-                handler.Execute((TCommand)command);
+                handler.Execute(command);
 
-                if (command.State == CommandStatus.Running)
-                {
-                    _trackCommands.Completed(command);
-                }
+                _commandQueueManager.Complete(commandModel, command.CompletionMessage);
             }
-            catch (Exception e)
+            catch (CommandFailedException ex)
             {
-                _trackCommands.Failed(command, e);
+                _commandQueueManager.Fail(commandModel, ex.Message, ex);
+                throw;
+            }
+            catch (Exception ex)
+            {
+                _commandQueueManager.SetMessage(commandModel, "Failed");
+                _commandQueueManager.Fail(commandModel, "Failed", ex);
                 throw;
             }
             finally
             {
-                BroadcastCommandUpdate(command);
-                _eventAggregator.PublishEvent(new CommandExecutedEvent(command));
+                BroadcastCommandUpdate(commandModel);
 
-                if (MappedDiagnosticsContext.Get("CommandId") == command.Id.ToString())
+                _eventAggregator.PublishEvent(new CommandExecutedEvent(commandModel));
+
+                if (MappedDiagnosticsContext.Get("CommandId") == commandModel.Id.ToString())
                 {
                     MappedDiagnosticsContext.Remove("CommandId");
                 }
             }
 
-            _logger.Trace("{0} <- {1} [{2}]", command.GetType().Name, handler.GetType().Name, command.Runtime.ToString(""));
+            _logger.Trace("{0} <- {1} [{2}]", command.GetType().Name, handler.GetType().Name, commandModel.Duration.ToString());
         }
-
-
-        private void BroadcastCommandUpdate(Command command)
+        
+        private void BroadcastCommandUpdate(CommandModel command)
         {
-            if (command.SendUpdatesToClient)
+            if (command.Body.SendUpdatesToClient)
             {
                 _eventAggregator.PublishEvent(new CommandUpdatedEvent(command));
             }
         }
+
+        public void Handle(ApplicationStartedEvent message)
+        {
+            _cancellationTokenSource = new CancellationTokenSource();
+
+            for (int i = 0; i < THREAD_LIMIT; i++)
+            {
+                var thread = new Thread(ExecuteCommands);
+                thread.Start();
+            }
+        }
+
+        public void Handle(ApplicationShutdownRequested message)
+        {
+            _logger.Info("Shutting down task execution");
+            _cancellationTokenSource.Cancel(true);
+        }
     }
 }