From 17bf438cad110a656fb9ffdde025c61776a7f354 Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Thu, 17 Mar 2016 22:30:21 +0100 Subject: [PATCH] New: uTorrent differential api support to handle larger lists of torrents without hogging the api. Fixes #1109 --- .gitignore | 3 + .../UTorrentTests/UTorrentFixture.cs | 61 ++++++++++++++++--- .../Download/Clients/uTorrent/UTorrent.cs | 38 ++++++++++-- .../Clients/uTorrent/UTorrentProxy.cs | 11 +++- .../Clients/uTorrent/UTorrentResponse.cs | 4 ++ .../Clients/uTorrent/UTorrentTorrentCache.cs | 14 +++++ src/NzbDrone.Core/NzbDrone.Core.csproj | 1 + 7 files changed, 115 insertions(+), 17 deletions(-) create mode 100644 src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentTorrentCache.cs diff --git a/.gitignore b/.gitignore index 06b734042..03e66a1ef 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,9 @@ src/**/[Oo]bj/ _ReSharper* _dotCover* +# DevExpress CodeRush +src/.cr/ + # NCrunch *.ncrunch* .*crunch*.local.xml diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientTests/UTorrentTests/UTorrentFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadClientTests/UTorrentTests/UTorrentFixture.cs index 21130cfd1..1d9f037d2 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadClientTests/UTorrentTests/UTorrentFixture.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadClientTests/UTorrentTests/UTorrentFixture.cs @@ -84,7 +84,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.UTorrentTests DownloadUrl = _downloadUrl, RootDownloadPath = "somepath" }; - + Mocker.GetMock() .Setup(s => s.GetHashFromTorrentFile(It.IsAny())) .Returns("CBC2F069FE8BB2F544EAE707D75BCD3DE9DCF951"); @@ -130,8 +130,8 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.UTorrentTests PrepareClientToReturnQueuedItem(); }); } - - protected virtual void GivenTorrents(List torrents) + + protected virtual void GivenTorrents(List torrents, string cacheNumber = null) { if (torrents == null) { @@ -139,13 +139,30 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.UTorrentTests } Mocker.GetMock() - .Setup(s => s.GetTorrents(It.IsAny())) - .Returns(torrents); + .Setup(s => s.GetTorrents(It.IsAny(), It.IsAny())) + .Returns(new UTorrentResponse { Torrents = torrents, CacheNumber = cacheNumber }); + } + + protected virtual void GivenDifferentialTorrents(string oldCacheNumber, List changed, List removed, string cacheNumber) + { + if (changed == null) + { + changed = new List(); + } + + if (removed == null) + { + removed = new List(); + } + + Mocker.GetMock() + .Setup(s => s.GetTorrents(oldCacheNumber, It.IsAny())) + .Returns(new UTorrentResponse { TorrentsChanged = changed, TorrentsRemoved = removed, CacheNumber = cacheNumber }); } protected void PrepareClientToReturnQueuedItem() { - GivenTorrents(new List + GivenTorrents(new List { _queued }); @@ -153,7 +170,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.UTorrentTests protected void PrepareClientToReturnDownloadingItem() { - GivenTorrents(new List + GivenTorrents(new List { _downloading }); @@ -161,7 +178,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.UTorrentTests protected void PrepareClientToReturnFailedItem() { - GivenTorrents(new List + GivenTorrents(new List { _failed }); @@ -296,13 +313,13 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.UTorrentTests public void should_return_status_with_outputdirs() { var configItems = new Dictionary(); - + configItems.Add("dir_active_download_flag", "true"); configItems.Add("dir_active_download", @"C:\Downloads\Downloading\utorrent".AsOsAgnostic()); configItems.Add("dir_completed_download", @"C:\Downloads\Finished\utorrent".AsOsAgnostic()); configItems.Add("dir_completed_download_flag", "true"); configItems.Add("dir_add_label", "true"); - + Mocker.GetMock() .Setup(v => v.GetConfig(It.IsAny())) .Returns(configItems); @@ -353,5 +370,29 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.UTorrentTests id.Should().NotBeNullOrEmpty(); } + + [Test] + public void GetItems_should_query_with_cache_id_if_available() + { + _downloading.Status = UTorrentTorrentStatus.Started; + + GivenTorrents(new List { _downloading }, "abc"); + + var item1 = Subject.GetItems().Single(); + + Mocker.GetMock().Verify(v => v.GetTorrents(null, It.IsAny()), Times.Once()); + + GivenTorrents(new List { _downloading, _queued }, "abc2"); + GivenDifferentialTorrents("abc", new List { _queued }, new List(), "abc2"); + GivenDifferentialTorrents("abc2", new List(), new List(), "abc2"); + + var item2 = Subject.GetItems().Single(); + + Mocker.GetMock().Verify(v => v.GetTorrents("abc", It.IsAny()), Times.Once()); + + var item3 = Subject.GetItems().Single(); + + Mocker.GetMock().Verify(v => v.GetTorrents("abc2", It.IsAny()), Times.Once()); + } } } diff --git a/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrent.cs b/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrent.cs index 64888a4bd..3e76ad4eb 100644 --- a/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrent.cs +++ b/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrent.cs @@ -12,14 +12,17 @@ using FluentValidation.Results; using System.Net; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.RemotePathMappings; +using NzbDrone.Common.Cache; namespace NzbDrone.Core.Download.Clients.UTorrent { public class UTorrent : TorrentClientBase { private readonly IUTorrentProxy _proxy; + private readonly ICached _torrentCache; public UTorrent(IUTorrentProxy proxy, + ICacheManager cacheManager, ITorrentFileInfoReader torrentFileInfoReader, IHttpClient httpClient, IConfigService configService, @@ -29,6 +32,8 @@ namespace NzbDrone.Core.Download.Clients.UTorrent : base(torrentFileInfoReader, httpClient, configService, diskProvider, remotePathMappingService, logger) { _proxy = proxy; + + _torrentCache = cacheManager.GetCache(GetType(), "differentialTorrents"); } protected override string AddFromMagnetLink(RemoteEpisode remoteEpisode, string hash, string magnetLink) @@ -72,12 +77,37 @@ namespace NzbDrone.Core.Download.Clients.UTorrent } public override IEnumerable GetItems() - { + { List torrents; try { - torrents = _proxy.GetTorrents(Settings); + var cacheKey = string.Format("{0}:{1}:{2}", Settings.Host, Settings.Port, Settings.TvCategory); + var cache = _torrentCache.Find(cacheKey); + + var response = _proxy.GetTorrents(cache == null ? null : cache.CacheID, Settings); + + if (cache != null && response.Torrents == null) + { + var removedAndUpdated = new HashSet(response.TorrentsChanged.Select(v => v.Hash).Concat(response.TorrentsRemoved)); + + torrents = cache.Torrents + .Where(v => !removedAndUpdated.Contains(v.Hash)) + .Concat(response.TorrentsChanged) + .ToList(); + } + else + { + torrents = response.Torrents; + } + + cache = new UTorrentTorrentCache + { + CacheID = response.CacheNumber, + Torrents = torrents + }; + + _torrentCache.Set(cacheKey, cache, TimeSpan.FromMinutes(15)); } catch (DownloadClientException ex) { @@ -122,7 +152,7 @@ namespace NzbDrone.Core.Download.Clients.UTorrent item.Status = DownloadItemStatus.Warning; item.Message = "uTorrent is reporting an error"; } - else if (torrent.Status.HasFlag(UTorrentTorrentStatus.Loaded) && + else if (torrent.Status.HasFlag(UTorrentTorrentStatus.Loaded) && torrent.Status.HasFlag(UTorrentTorrentStatus.Checked) && torrent.Remaining == 0 && torrent.Progress == 1.0) { item.Status = DownloadItemStatus.Completed; @@ -239,7 +269,7 @@ namespace NzbDrone.Core.Download.Clients.UTorrent { try { - _proxy.GetTorrents(Settings); + _proxy.GetTorrents(null, Settings); } catch (Exception ex) { diff --git a/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentProxy.cs b/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentProxy.cs index 1ca49bfec..be360c32c 100644 --- a/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentProxy.cs +++ b/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentProxy.cs @@ -15,7 +15,7 @@ namespace NzbDrone.Core.Download.Clients.UTorrent { int GetVersion(UTorrentSettings settings); Dictionary GetConfig(UTorrentSettings settings); - List GetTorrents(UTorrentSettings settings); + UTorrentResponse GetTorrents(string cacheID, UTorrentSettings settings); void AddTorrentFromUrl(string torrentUrl, UTorrentSettings settings); void AddTorrentFromFile(string fileName, byte[] fileContent, UTorrentSettings settings); @@ -70,14 +70,19 @@ namespace NzbDrone.Core.Download.Clients.UTorrent return configuration; } - public List GetTorrents(UTorrentSettings settings) + public UTorrentResponse GetTorrents(string cacheID, UTorrentSettings settings) { var requestBuilder = BuildRequest(settings) .AddQueryParam("list", 1); + if (cacheID.IsNotNullOrWhiteSpace()) + { + requestBuilder.AddQueryParam("cid", cacheID); + } + var result = ProcessRequest(requestBuilder, settings); - return result.Torrents; + return result; } public void AddTorrentFromUrl(string torrentUrl, UTorrentSettings settings) diff --git a/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentResponse.cs b/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentResponse.cs index 46f560b84..aa045e056 100644 --- a/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentResponse.cs +++ b/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentResponse.cs @@ -12,6 +12,10 @@ namespace NzbDrone.Core.Download.Clients.UTorrent public List RssFeeds { get; set; } public List RssFilters { get; set; } + [JsonProperty(PropertyName = "torrentp")] + public List TorrentsChanged { get; set; } + [JsonProperty(PropertyName = "torrentm")] + public List TorrentsRemoved { get; set; } [JsonProperty(PropertyName = "torrentc")] public string CacheNumber { get; set; } diff --git a/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentTorrentCache.cs b/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentTorrentCache.cs new file mode 100644 index 000000000..38d007812 --- /dev/null +++ b/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentTorrentCache.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace NzbDrone.Core.Download.Clients.UTorrent +{ + public class UTorrentTorrentCache + { + public string CacheID { get; set; } + + public List Torrents { get; set; } + } +} diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 7a32cde0d..c333a4492 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -451,6 +451,7 @@ +