New: Failed download handling for Nzbget

src/NzbDrone.Common/Extensions/StreamExtensions.cs

@@ -0,0 +1,21 @@
+using System.IO;
+
+namespace NzbDrone.Common.Extensions
+{
+    public static class StreamExtensions
+    {
+        public static byte[] ToBytes(this Stream input)
+        {
+            var buffer = new byte[16 * 1024];
+            using (var ms = new MemoryStream())
+            {
+                int read;
+                while ((read = input.Read(buffer, 0, buffer.Length)) > 0)
+                {
+                    ms.Write(buffer, 0, read);
+                }
+                return ms.ToArray();
+            }
+        }
+    }
+}

src/NzbDrone.Common/NzbDrone.Common.csproj

@@ -107,6 +107,7 @@
     <Compile Include="Processes\ProcessOutput.cs" />
     <Compile Include="Serializer\IntConverter.cs" />
     <Compile Include="Services.cs" />
+    <Compile Include="Extensions\StreamExtensions.cs" />
     <Compile Include="TPL\LimitedConcurrencyLevelTaskScheduler.cs" />
     <Compile Include="Security\IgnoreCertErrorPolicy.cs" />
     <Compile Include="StringExtensions.cs" />

src/NzbDrone.Core.Test/ParserTests/CrapParserFixture.cs

@@ -27,6 +27,7 @@ namespace NzbDrone.Core.Test.ParserTests
         [TestCase("86420f8ee425340d8894bf3bc636b66404b95f18")]
         [TestCase("ce39afb7da6cf7c04eba3090f0a309f609883862")]
         [TestCase("THIS SHOULD NEVER PARSE")]
+        [TestCase("Vh1FvU3bJXw6zs8EEUX4bMo5vbbMdHghxHirc.mkv")]
         public void should_not_parse_crap(string title)
         {
             Parser.Parser.ParseTitle(title).Should().BeNull();

src/NzbDrone.Core.Test/ParserTests/PathParserFixture.cs

@@ -31,6 +31,7 @@ namespace NzbDrone.Core.Test.ParserTests
         [TestCase(@"C:\Test\Unsorted\The.Big.Bang.Theory.S01E01.720p.HDTV\tbbt101.avi", 1, 1)]
         [TestCase(@"C:\Test\Unsorted\Terminator.The.Sarah.Connor.Chronicles.S02E19.720p.BluRay.x264-SiNNERS-RP\ba27283b17c00d01193eacc02a8ba98eeb523a76.mkv", 2, 19)]
         [TestCase(@"C:\Test\Unsorted\Terminator.The.Sarah.Connor.Chronicles.S02E18.720p.BluRay.x264-SiNNERS-RP\45a55debe3856da318cc35882ad07e43cd32fd15.mkv", 2, 18)]
+        [TestCase(@"C:\Test\The.Blacklist.S01E16.720p.HDTV.X264-DIMENSION\XRmZciqkBopq4851Ddbipe\Vh1FvU3bJXw6zs8EEUX4bMo5vbbMdHghxHirc.mkv", 1, 16)]
         public void should_parse_from_path(string path, int season, int episode)
         {
             var result = Parser.Parser.ParsePath(path);

src/NzbDrone.Core/Download/Clients/Nzbget/NzbGetQueueItem.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 
 namespace NzbDrone.Core.Download.Clients.Nzbget
 {
@@ -6,10 +7,13 @@ namespace NzbDrone.Core.Download.Clients.Nzbget
     {
         private string _nzbName;
         public Int32 NzbId { get; set; }
+        public Int32 FirstId { get; set; }
+        public Int32 LastId { get; set; }
         public string NzbName { get; set; }
         public String Category { get; set; }
         public Int32 FileSizeMb { get; set; }
         public Int32 RemainingSizeMb { get; set; }
         public Int32 PausedSizeMb { get; set; }
+        public List<NzbgetParameter> Parameters { get; set; }
     }
 }

src/NzbDrone.Core/Download/Clients/Nzbget/Nzbget.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using NLog;
+using NzbDrone.Common;
 using NzbDrone.Core.Messaging.Commands;
 using NzbDrone.Core.Parser;
 using NzbDrone.Core.Parser.Model;
@@ -13,14 +14,17 @@ namespace NzbDrone.Core.Download.Clients.Nzbget
     {
         private readonly INzbgetProxy _proxy;
         private readonly IParsingService _parsingService;
+        private readonly IHttpProvider _httpProvider;
         private readonly Logger _logger;
 
         public Nzbget(INzbgetProxy proxy,
                       IParsingService parsingService,
+                      IHttpProvider httpProvider,
                       Logger logger)
         {
             _proxy = proxy;
             _parsingService = parsingService;
+            _httpProvider = httpProvider;
             _logger = logger;
         }
 
@@ -29,16 +33,18 @@ namespace NzbDrone.Core.Download.Clients.Nzbget
             var url = remoteEpisode.Release.DownloadUrl;
             var title = remoteEpisode.Release.Title + ".nzb";
 
-            string cat = Settings.TvCategory;
+            string category = Settings.TvCategory;
             int priority = remoteEpisode.IsRecentEpisode() ? Settings.RecentTvPriority : Settings.OlderTvPriority;
 
             _logger.Info("Adding report [{0}] to the queue.", title);
 
-            var success = _proxy.AddNzb(Settings, title, cat, priority, false, url);
+            using (var nzb = _httpProvider.DownloadStream(url))
+            {
+                _logger.Info("Adding report [{0}] to the queue.", title);
+                var response = _proxy.DownloadNzb(nzb, title, category, priority, Settings);
 
-            _logger.Debug("Queue Response: [{0}]", success);
-
-            return null;
+                return response;
+            }
         }
 
         public override IEnumerable<QueueItem> GetQueue()
@@ -57,14 +63,16 @@ namespace NzbDrone.Core.Download.Clients.Nzbget
 
             var queueItems = new List<QueueItem>();
 
-            foreach (var nzbGetQueueItem in queue)
+            foreach (var item in queue)
             {
+                var droneParameter = item.Parameters.SingleOrDefault(p => p.Name == "drone");
+
                 var queueItem = new QueueItem();
-                queueItem.Id = nzbGetQueueItem.NzbId.ToString();
-                queueItem.Title = nzbGetQueueItem.NzbName;
-                queueItem.Size = nzbGetQueueItem.FileSizeMb;
-                queueItem.Sizeleft = nzbGetQueueItem.RemainingSizeMb;
-                queueItem.Status = nzbGetQueueItem.FileSizeMb == nzbGetQueueItem.PausedSizeMb ? "paused" : "queued";
+                queueItem.Id = droneParameter == null ? item.NzbId.ToString() : droneParameter.Value.ToString();
+                queueItem.Title = item.NzbName;
+                queueItem.Size = item.FileSizeMb;
+                queueItem.Sizeleft = item.RemainingSizeMb;
+                queueItem.Status = item.FileSizeMb == item.PausedSizeMb ? "paused" : "queued";
 
                 var parsedEpisodeInfo = Parser.Parser.ParseTitle(queueItem.Title);
                 if (parsedEpisodeInfo == null) continue;
@@ -81,7 +89,43 @@ namespace NzbDrone.Core.Download.Clients.Nzbget
 
         public override IEnumerable<HistoryItem> GetHistory(int start = 0, int limit = 10)
         {
-            return new HistoryItem[0];
+            List<NzbgetHistoryItem> history;
+
+            try
+            {
+                history = _proxy.GetHistory(Settings);
+            }
+            catch (DownloadClientException ex)
+            {
+                _logger.ErrorException(ex.Message, ex);
+                return Enumerable.Empty<HistoryItem>();
+            }
+
+            var historyItems = new List<HistoryItem>();
+            var successStatues = new[] {"SUCCESS", "NONE"};
+
+            foreach (var item in history)
+            {
+                var droneParameter = item.Parameters.SingleOrDefault(p => p.Name == "drone");
+                var status = successStatues.Contains(item.ParStatus) &&
+                             successStatues.Contains(item.ScriptStatus)
+                    ? HistoryStatus.Completed
+                    : HistoryStatus.Failed;
+
+                var historyItem = new HistoryItem();
+                historyItem.Id = droneParameter == null ? item.Id.ToString() : droneParameter.Value.ToString();
+                historyItem.Title = item.Name;
+                historyItem.Size = item.FileSizeMb.ToString(); //Why is this a string?
+                historyItem.DownloadTime = 0;
+                historyItem.Storage = item.DestDir;
+                historyItem.Category = item.Category;
+                historyItem.Message = String.Format("PAR Status: {0} - Script Status: {1}", item.ParStatus, item.ScriptStatus);
+                historyItem.Status = status;
+
+                historyItems.Add(historyItem);
+            }
+
+            return historyItems;
         }
 
         public override void RemoveFromQueue(string id)
@@ -91,7 +135,7 @@ namespace NzbDrone.Core.Download.Clients.Nzbget
 
         public override void RemoveFromHistory(string id)
         {
-            throw new NotImplementedException();
+            _proxy.RemoveFromHistory(id, Settings);
         }
 
         public override void Test()

src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetBooleanResponse.cs

@@ -2,7 +2,7 @@
 
 namespace NzbDrone.Core.Download.Clients.Nzbget
 {
-    public class EnqueueResponse
+    public class NzbgetBooleanResponse
     {
         public String Version { get; set; }
         public Boolean Result { get; set; }

src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetHistoryItem.cs

@@ -0,0 +1,18 @@
+using System;
+using System.Collections.Generic;
+
+namespace NzbDrone.Core.Download.Clients.Nzbget
+{
+    public class NzbgetHistoryItem
+    {
+        private string _nzbName;
+        public Int32 Id { get; set; }
+        public String Name { get; set; }
+        public String Category { get; set; }
+        public Int32 FileSizeMb { get; set; }
+        public String ParStatus { get; set; }
+        public String ScriptStatus { get; set; }
+        public String DestDir { get; set; }
+        public List<NzbgetParameter> Parameters { get; set; }
+    }
+}

src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetListResponse.cs

@@ -4,11 +4,11 @@ using Newtonsoft.Json;
 
 namespace NzbDrone.Core.Download.Clients.Nzbget
 {
-    public class NzbgetQueue
+    public class NzbgetListResponse<T>
     {
         public String Version { get; set; }
 
         [JsonProperty(PropertyName = "result")]
-        public List<NzbgetQueueItem> QueueItems { get; set; }
+        public List<T> QueueItems { get; set; }
     }
 }

src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetParameter.cs

@@ -0,0 +1,9 @@
+using System;
+namespace NzbDrone.Core.Download.Clients.Nzbget
+{
+    public class NzbgetParameter
+    {
+        public String Name { get; set; }
+        public object Value { get; set; }
+    }
+}

src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetProxy.cs

@@ -1,6 +1,9 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
+using System.Linq;
 using NLog;
+using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Serializer;
 using NzbDrone.Core.Rest;
 using RestSharp;
@@ -9,9 +12,11 @@ namespace NzbDrone.Core.Download.Clients.Nzbget
 {
     public interface INzbgetProxy
     {
-        bool AddNzb(NzbgetSettings settings, params object[] parameters);
+        string DownloadNzb(Stream nzb, string title, string category, int priority, NzbgetSettings settings);
         List<NzbgetQueueItem> GetQueue(NzbgetSettings settings);
+        List<NzbgetHistoryItem> GetHistory(NzbgetSettings settings);
         VersionResponse GetVersion(NzbgetSettings settings);
+        void RemoveFromHistory(string id, NzbgetSettings settings);
     }
 
     public class NzbgetProxy : INzbgetProxy
@@ -23,18 +28,50 @@ namespace NzbDrone.Core.Download.Clients.Nzbget
             _logger = logger;
         }
 
-        public bool AddNzb(NzbgetSettings settings, params object[] parameters)
+        public string DownloadNzb(Stream nzb, string title, string category, int priority, NzbgetSettings settings)
         {
-            var request = BuildRequest(new JsonRequest("appendurl", parameters));
+            var parameters = new object[] { title, category, priority, false, Convert.ToBase64String(nzb.ToBytes()) };
+            var request = BuildRequest(new JsonRequest("append", parameters));
 
-            return Json.Deserialize<EnqueueResponse>(ProcessRequest(request, settings)).Result;
+            var response = Json.Deserialize<NzbgetBooleanResponse>(ProcessRequest(request, settings));
+            _logger.Debug("Queue Response: [{0}]", response.Result);
+
+            if (!response.Result)
+            {
+                return null;
+            }
+
+            var queue = GetQueue(settings);
+            var item = queue.FirstOrDefault(q => q.NzbName == title.Substring(0, title.Length - 4));
+
+            if (item == null)
+            {
+                return null;
+            }
+
+            var droneId = Guid.NewGuid().ToString().Replace("-", "");
+            var editResult = EditQueue("GroupSetParameter", 0, "drone=" + droneId, item.LastId, settings);
+
+            if (editResult)
+            {
+                _logger.Debug("Nzbget download drone parameter set to: {0}", droneId);
+            }
+
+            return droneId;
         }
 
         public List<NzbgetQueueItem> GetQueue(NzbgetSettings settings)
         {
             var request = BuildRequest(new JsonRequest("listgroups"));
 
-            return Json.Deserialize<NzbgetQueue>(ProcessRequest(request, settings)).QueueItems;
+            return Json.Deserialize<NzbgetListResponse<NzbgetQueueItem>>(ProcessRequest(request, settings)).QueueItems;
+        }
+
+        public List<NzbgetHistoryItem> GetHistory(NzbgetSettings settings)
+        {
+            var request = BuildRequest(new JsonRequest("history"));
+
+            return Json.Deserialize<NzbgetListResponse<NzbgetHistoryItem>>(ProcessRequest(request, settings)).QueueItems;
         }
 
         public VersionResponse GetVersion(NzbgetSettings settings)
@@ -44,6 +81,32 @@ namespace NzbDrone.Core.Download.Clients.Nzbget
             return Json.Deserialize<VersionResponse>(ProcessRequest(request, settings));
         }
 
+        public void RemoveFromHistory(string id, NzbgetSettings settings)
+        {
+            var history = GetHistory(settings);
+            var item = history.SingleOrDefault(h => h.Parameters.SingleOrDefault(p => p.Name == "drone") != null);
+
+            if (item == null)
+            {
+                _logger.Warn("Unable to remove item from nzbget's history, Unknown ID: {0}", id);
+                return;
+            }
+
+            if (!EditQueue("HistoryDelete", 0, "", item.Id, settings))
+            {
+                _logger.Warn("Failed to remove item from nzbget history, {0} [{1}]", item.Name, item.Id);
+            }
+        }
+
+        private bool EditQueue(string command, int offset, string editText, int id, NzbgetSettings settings)
+        {
+            var parameters = new object[] { command, offset, editText, id };
+            var request = BuildRequest(new JsonRequest("editqueue", parameters));
+            var response = Json.Deserialize<NzbgetBooleanResponse>(ProcessRequest(request, settings));
+
+            return response.Result;
+        }
+
         private string ProcessRequest(IRestRequest restRequest, NzbgetSettings settings)
         {
             var client = BuildClient(settings);

src/NzbDrone.Core/Download/Clients/Sabnzbd/SabnzbdProxy.cs

@@ -3,6 +3,7 @@ using System.IO;
 using Newtonsoft.Json.Linq;
 using NLog;
 using NzbDrone.Common;
+using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Serializer;
 using NzbDrone.Core.Download.Clients.Sabnzbd.Responses;
 using NzbDrone.Core.Instrumentation.Extensions;
@@ -35,7 +36,7 @@ namespace NzbDrone.Core.Download.Clients.Sabnzbd
             var request = new RestRequest(Method.POST);
             var action = String.Format("mode=addfile&cat={0}&priority={1}", category, priority);
 
-            request.AddFile("name", ReadFully(nzb), title, "application/x-nzb");
+            request.AddFile("name", nzb.ToBytes(), title, "application/x-nzb");
 
             SabnzbdAddResponse response;
 
@@ -161,20 +162,5 @@ namespace NzbDrone.Core.Download.Clients.Sabnzbd
             if (result.Failed)
                 throw new DownloadClientException("Error response received from SABnzbd: {0}", result.Error);
         }
-
-        //TODO: Find a better home for this
-        private byte[] ReadFully(Stream input)
-        {
-            byte[] buffer = new byte[16 * 1024];
-            using (MemoryStream ms = new MemoryStream())
-            {
-                int read;
-                while ((read = input.Read(buffer, 0, buffer.Length)) > 0)
-                {
-                    ms.Write(buffer, 0, read);
-                }
-                return ms.ToArray();
-            }
-        }
     }
 }

src/NzbDrone.Core/Download/DownloadClientBase.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Collections.Generic;
-using NLog;
 using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.ThingiProvider;
 

src/NzbDrone.Core/NzbDrone.Core.csproj

@@ -236,6 +236,8 @@
     <Compile Include="Download\Clients\Blackhole\TestBlackholeCommand.cs" />
     <Compile Include="Download\Clients\DownloadClientException.cs" />
     <Compile Include="Download\Clients\FolderSettings.cs" />
+    <Compile Include="Download\Clients\Nzbget\NzbgetHistoryItem.cs" />
+    <Compile Include="Download\Clients\Nzbget\NzbgetParameter.cs" />
     <Compile Include="Download\Clients\Nzbget\NzbgetSettings.cs" />
     <Compile Include="Download\Clients\Nzbget\TestNzbgetCommand.cs" />
     <Compile Include="Download\Clients\Pneumatic\Pneumatic.cs" />
@@ -500,10 +502,10 @@
     <Compile Include="Instrumentation\LogService.cs" />
     <Compile Include="Instrumentation\DatabaseTarget.cs" />
     <Compile Include="MediaFiles\MediaInfo\MediaInfoModel.cs" />
-    <Compile Include="Download\Clients\Nzbget\EnqueueResponse.cs" />
+    <Compile Include="Download\Clients\Nzbget\NzbgetBooleanResponse.cs" />
     <Compile Include="Download\Clients\Nzbget\ErrorModel.cs" />
     <Compile Include="Download\Clients\Nzbget\JsonError.cs" />
-    <Compile Include="Download\Clients\Nzbget\NzbgetQueue.cs" />
+    <Compile Include="Download\Clients\Nzbget\NzbgetListResponse.cs" />
     <Compile Include="Download\Clients\Nzbget\NzbgetQueueItem.cs" />
     <Compile Include="Download\Clients\Nzbget\NzbgetPriority.cs" />
     <Compile Include="Download\Clients\Nzbget\VersionResponse.cs" />