From f2a70677e4ecd66a65f11ab4ea96ef2c49b871c9 Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Sat, 27 Jun 2015 11:43:17 +0200 Subject: [PATCH] New: Will now temporarily stop using an indexer if the indexer reported an error. --- src/NzbDrone.Api/Indexers/ReleaseResource.cs | 1 + .../Http/HttpClientFixture.cs | 10 ++ src/NzbDrone.Common/Http/HttpClient.cs | 10 +- .../Http/TooManyRequestsException.cs | 21 +++ src/NzbDrone.Common/NzbDrone.Common.csproj | 1 + .../Download/DownloadServiceFixture.cs | 56 ++++++- .../PendingReleaseServiceFixture.cs | 63 ++++++++ .../Checks/IndexerStatusCheckFixture.cs | 99 +++++++++++++ .../CleanupOrphanedIndexerStatusFixture.cs | 56 +++++++ .../NzbSearchServiceFixture.cs | 1 + .../BroadcastheNetFixture.cs | 5 +- .../IndexerStatusServiceFixture.cs | 94 ++++++++++++ .../NewznabTests/NewznabFixture.cs | 2 + .../NewznabRequestGeneratorFixture.cs | 12 -- .../IndexerTests/TestIndexer.cs | 4 +- .../TestTorrentRssIndexer.cs | 4 +- .../TorznabRequestGeneratorFixture.cs | 12 -- .../NzbDrone.Core.Test.csproj | 4 + .../Migration/091_added_indexerstatus.cs | 25 ++++ src/NzbDrone.Core/Datastore/TableMapping.cs | 2 + src/NzbDrone.Core/Download/DownloadService.cs | 27 +++- .../Download/Pending/PendingReleaseService.cs | 21 ++- .../Download/TorrentClientBase.cs | 8 +- .../Download/UsenetClientBase.cs | 7 + .../HealthCheck/Checks/IndexerStatusCheck.cs | 48 ++++++ .../CleanupOrphanedIndexerStatus.cs | 26 ++++ .../IndexerSearch/EpisodeSearchService.cs | 31 +--- .../IndexerSearch/NzbSearchService.cs | 2 +- src/NzbDrone.Core/Indexers/BitMeTv/BitMeTv.cs | 4 +- .../Indexers/BroadcastheNet/BroadcastheNet.cs | 4 +- .../BroadcastheNetRequestGenerator.cs | 2 +- src/NzbDrone.Core/Indexers/Fanzub/Fanzub.cs | 4 +- .../Indexers/FetchAndParseRssService.cs | 5 +- src/NzbDrone.Core/Indexers/HDBits/HDBits.cs | 7 +- src/NzbDrone.Core/Indexers/HttpIndexerBase.cs | 75 ++++++++-- .../Indexers/IPTorrents/IPTorrents.cs | 4 +- src/NzbDrone.Core/Indexers/IndexerBase.cs | 10 +- .../Indexers/IndexerDefinition.cs | 2 + src/NzbDrone.Core/Indexers/IndexerFactory.cs | 43 ++++-- src/NzbDrone.Core/Indexers/IndexerStatus.cs | 25 ++++ .../Indexers/IndexerStatusRepository.cs | 26 ++++ .../Indexers/IndexerStatusService.cs | 140 ++++++++++++++++++ .../KickassTorrents/KickassTorrents.cs | 4 +- .../KickassTorrentsRequestGenerator.cs | 3 +- src/NzbDrone.Core/Indexers/Newznab/Newznab.cs | 4 +- .../Newznab/NewznabRequestGenerator.cs | 3 +- src/NzbDrone.Core/Indexers/Nyaa/Nyaa.cs | 4 +- .../Indexers/Nyaa/NyaaRequestGenerator.cs | 2 +- .../Indexers/Omgwtfnzbs/Omgwtfnzbs.cs | 4 +- src/NzbDrone.Core/Indexers/Rarbg/Rarbg.cs | 4 +- src/NzbDrone.Core/Indexers/RssSyncService.cs | 23 +-- .../Indexers/TorrentRss/TorrentRssIndexer.cs | 4 +- .../Indexers/Torrentleech/Torrentleech.cs | 4 +- src/NzbDrone.Core/Indexers/Torznab/Torznab.cs | 4 +- .../Torznab/TorznabRequestGenerator.cs | 3 +- src/NzbDrone.Core/Indexers/Wombles/Wombles.cs | 4 +- src/NzbDrone.Core/NzbDrone.Core.csproj | 41 ++--- src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs | 28 ++-- .../Events/ProviderDeletedEvent.cs | 14 ++ .../Events/ProviderUpdatedEvent.cs | 8 +- .../ThingiProvider/ProviderFactory.cs | 3 +- 61 files changed, 994 insertions(+), 173 deletions(-) create mode 100644 src/NzbDrone.Common/Http/TooManyRequestsException.cs create mode 100644 src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/PendingReleaseServiceFixture.cs create mode 100644 src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs create mode 100644 src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupOrphanedIndexerStatusFixture.cs create mode 100644 src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs create mode 100644 src/NzbDrone.Core/Datastore/Migration/091_added_indexerstatus.cs create mode 100644 src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs create mode 100644 src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedIndexerStatus.cs create mode 100644 src/NzbDrone.Core/Indexers/IndexerStatus.cs create mode 100644 src/NzbDrone.Core/Indexers/IndexerStatusRepository.cs create mode 100644 src/NzbDrone.Core/Indexers/IndexerStatusService.cs create mode 100644 src/NzbDrone.Core/ThingiProvider/Events/ProviderDeletedEvent.cs diff --git a/src/NzbDrone.Api/Indexers/ReleaseResource.cs b/src/NzbDrone.Api/Indexers/ReleaseResource.cs index 29111d835..57fc39456 100644 --- a/src/NzbDrone.Api/Indexers/ReleaseResource.cs +++ b/src/NzbDrone.Api/Indexers/ReleaseResource.cs @@ -16,6 +16,7 @@ namespace NzbDrone.Api.Indexers public Double AgeHours { get; set; } public Double AgeMinutes { get; set; } public Int64 Size { get; set; } + public Int32 IndexerId { get; set; } public String Indexer { get; set; } public String ReleaseGroup { get; set; } public String SubGroup { get; set; } diff --git a/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs b/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs index 39d72fa82..8357d4926 100644 --- a/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs +++ b/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs @@ -249,6 +249,16 @@ namespace NzbDrone.Common.Test.Http ExceptionVerification.IgnoreErrors(); } + + [Test] + public void should_throw_on_http429_too_many_requests() + { + var request = new HttpRequest("http://eu.httpbin.org/status/429"); + + Assert.Throws(() => Subject.Get(request)); + + ExceptionVerification.IgnoreWarns(); + } } public class HttpBinResource diff --git a/src/NzbDrone.Common/Http/HttpClient.cs b/src/NzbDrone.Common/Http/HttpClient.cs index df6485488..da1d3379e 100644 --- a/src/NzbDrone.Common/Http/HttpClient.cs +++ b/src/NzbDrone.Common/Http/HttpClient.cs @@ -94,7 +94,15 @@ namespace NzbDrone.Common.Http if (!request.SuppressHttpError && response.HasHttpError) { _logger.Warn("HTTP Error - {0}", response); - throw new HttpException(request, response); + + if ((int)response.StatusCode == 429) + { + throw new TooManyRequestsException(request, response); + } + else + { + throw new HttpException(request, response); + } } return response; diff --git a/src/NzbDrone.Common/Http/TooManyRequestsException.cs b/src/NzbDrone.Common/Http/TooManyRequestsException.cs new file mode 100644 index 000000000..1c16f0b71 --- /dev/null +++ b/src/NzbDrone.Common/Http/TooManyRequestsException.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace NzbDrone.Common.Http +{ + public class TooManyRequestsException : HttpException + { + public TimeSpan RetryAfter { get; private set; } + + public TooManyRequestsException(HttpRequest request, HttpResponse response) + : base(request, response) + { + if (response.Headers.ContainsKey("Retry-After")) + { + RetryAfter = TimeSpan.FromSeconds(int.Parse(response.Headers["Retry-After"].ToString())); + } + } + } +} diff --git a/src/NzbDrone.Common/NzbDrone.Common.csproj b/src/NzbDrone.Common/NzbDrone.Common.csproj index f3110d9d3..f2b0ac12b 100644 --- a/src/NzbDrone.Common/NzbDrone.Common.csproj +++ b/src/NzbDrone.Common/NzbDrone.Common.csproj @@ -158,6 +158,7 @@ Component + diff --git a/src/NzbDrone.Core.Test/Download/DownloadServiceFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadServiceFixture.cs index 7eadd8746..5c624b34b 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadServiceFixture.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadServiceFixture.cs @@ -1,15 +1,20 @@ +using System; +using System.Collections.Generic; using System.Linq; using System.Net; using FizzWare.NBuilder; using Moq; using NUnit.Framework; +using NzbDrone.Common.Http; +using NzbDrone.Common.TPL; using NzbDrone.Core.Download; +using NzbDrone.Core.Download.Clients; +using NzbDrone.Core.Exceptions; +using NzbDrone.Core.Indexers; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Test.Framework; using NzbDrone.Core.Tv; using NzbDrone.Test.Common; -using System.Collections.Generic; -using NzbDrone.Core.Indexers; namespace NzbDrone.Core.Test.Download { @@ -107,6 +112,53 @@ namespace NzbDrone.Core.Test.Download VerifyEventNotPublished(); } + [Test] + public void Download_report_should_trigger_indexer_backoff_on_indexer_error() + { + var mock = WithUsenetClient(); + mock.Setup(s => s.Download(It.IsAny())) + .Callback(v => { + throw new ReleaseDownloadException(v.Release, "Error", new WebException()); + }); + + Assert.Throws(() => Subject.DownloadReport(_parseResult)); + + Mocker.GetMock() + .Verify(v => v.RecordFailure(It.IsAny(), It.IsAny()), Times.Once()); + } + + [Test] + public void Download_report_should_trigger_indexer_backoff_on_http429_with_long_time() + { + var request = new HttpRequest("http://my.indexer.com"); + var response = new HttpResponse(request, new HttpHeader(), new byte[0], (HttpStatusCode)429); + response.Headers["Retry-After"] = "300"; + + var mock = WithUsenetClient(); + mock.Setup(s => s.Download(It.IsAny())) + .Callback(v => { + throw new ReleaseDownloadException(v.Release, "Error", new TooManyRequestsException(request, response)); + }); + + Assert.Throws(() => Subject.DownloadReport(_parseResult)); + + Mocker.GetMock() + .Verify(v => v.RecordFailure(It.IsAny(), TimeSpan.FromMinutes(5)), Times.Once()); + } + + [Test] + public void Download_report_should_not_trigger_indexer_backoff_on_downloadclient_error() + { + var mock = WithUsenetClient(); + mock.Setup(s => s.Download(It.IsAny())) + .Throws(new DownloadClientException("Some Error")); + + Assert.Throws(() => Subject.DownloadReport(_parseResult)); + + Mocker.GetMock() + .Verify(v => v.RecordFailure(It.IsAny(), It.IsAny()), Times.Never()); + } + [Test] public void should_not_attempt_download_if_client_isnt_configure() { diff --git a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/PendingReleaseServiceFixture.cs b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/PendingReleaseServiceFixture.cs new file mode 100644 index 000000000..5e2c756f3 --- /dev/null +++ b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/PendingReleaseServiceFixture.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections.Generic; +using FizzWare.NBuilder; +using Marr.Data; +using Moq; +using NUnit.Framework; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.DecisionEngine; +using NzbDrone.Core.Download.Pending; +using NzbDrone.Core.Indexers; +using NzbDrone.Core.Parser; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Profiles; +using NzbDrone.Core.Qualities; +using NzbDrone.Core.Test.Framework; +using NzbDrone.Core.Tv; +using FluentAssertions; + +namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests +{ + [TestFixture] + public class PendingReleaseServiceFixture : CoreTest + { + private void GivenPendingRelease() + { + Mocker.GetMock() + .Setup(v => v.All()) + .Returns(new List { + new PendingRelease { Release = new ReleaseInfo { IndexerId = 1 } } + }); + } + + [Test] + public void should_not_ignore_pending_items_from_available_indexer() + { + Mocker.GetMock() + .Setup(v => v.GetBlockedIndexers()) + .Returns(new List()); + + GivenPendingRelease(); + + var results = Subject.GetPending(); + + results.Should().NotBeEmpty(); + Mocker.GetMock() + .Verify(v => v.GetRssDecision(It.Is>(d => d.Count == 0)), Times.Never()); + } + + [Test] + public void should_ignore_pending_items_from_unavailable_indexer() + { + Mocker.GetMock() + .Setup(v => v.GetBlockedIndexers()) + .Returns(new List { new IndexerStatus { IndexerId = 1, DisabledTill = DateTime.UtcNow.AddHours(2) } }); + + GivenPendingRelease(); + + var results = Subject.GetPending(); + + results.Should().BeEmpty(); + } + } +} diff --git a/src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs b/src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs new file mode 100644 index 000000000..d182f52e2 --- /dev/null +++ b/src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs @@ -0,0 +1,99 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using FizzWare.NBuilder; +using Moq; +using NUnit.Framework; +using NzbDrone.Common.Disk; +using NzbDrone.Core.HealthCheck.Checks; +using NzbDrone.Core.Indexers; +using NzbDrone.Core.Test.Framework; +using NzbDrone.Core.Tv; + +namespace NzbDrone.Core.Test.HealthCheck.Checks +{ + [TestFixture] + public class IndexerStatusCheckFixture : CoreTest + { + private List _indexers = new List(); + private List _blockedIndexers = new List(); + + [SetUp] + public void SetUp() + { + Mocker.GetMock() + .Setup(v => v.GetAvailableProviders()) + .Returns(_indexers); + + Mocker.GetMock() + .Setup(v => v.GetBlockedIndexers()) + .Returns(_blockedIndexers); + } + + private Mock GivenIndexer(int i, double backoffHours, double failureHours) + { + var id = i; + + var mockIndexer = new Mock(); + mockIndexer.SetupGet(s => s.Definition).Returns(new IndexerDefinition { Id = id }); + mockIndexer.SetupGet(s => s.SupportsSearch).Returns(true); + + _indexers.Add(mockIndexer.Object); + + if (backoffHours != 0.0) + { + _blockedIndexers.Add(new IndexerStatus + { + IndexerId = id, + InitialFailure = DateTime.UtcNow.AddHours(-failureHours), + MostRecentFailure = DateTime.UtcNow.AddHours(-0.1), + EscalationLevel = 5, + DisabledTill = DateTime.UtcNow.AddHours(backoffHours) + }); + } + + return mockIndexer; + } + + + [Test] + public void should_not_return_error_when_no_indexers() + { + Subject.Check().ShouldBeOk(); + } + [Test] + public void should_not_return_error_when_indexer_failed_less_than_an_hour() + { + GivenIndexer(1, 0.1, 0.5); + + Subject.Check().ShouldBeOk(); + } + + [Test] + public void should_return_warning_if_indexer_unavailable() + { + GivenIndexer(1, 10.0, 24.0); + GivenIndexer(2, 0.0, 0.0); + + Subject.Check().ShouldBeWarning(); + } + + [Test] + public void should_return_error_if_all_indexers_unavailable() + { + GivenIndexer(1, 10.0, 24.0); + + Subject.Check().ShouldBeError(); + } + + [Test] + public void should_return_warning_if_few_indexers_unavailable() + { + GivenIndexer(1, 10.0, 24.0); + GivenIndexer(2, 10.0, 24.0); + GivenIndexer(3, 0.0, 0.0); + + Subject.Check().ShouldBeWarning(); + } + } +} diff --git a/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupOrphanedIndexerStatusFixture.cs b/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupOrphanedIndexerStatusFixture.cs new file mode 100644 index 000000000..2dc3eb3a3 --- /dev/null +++ b/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupOrphanedIndexerStatusFixture.cs @@ -0,0 +1,56 @@ +using FizzWare.NBuilder; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Core.Housekeeping.Housekeepers; +using NzbDrone.Core.Indexers; +using NzbDrone.Core.Qualities; +using NzbDrone.Core.Test.Framework; +using NzbDrone.Core.Tv; + +namespace NzbDrone.Core.Test.Housekeeping.Housekeepers +{ + [TestFixture] + public class CleanupOrphanedIndexerStatusFixture : DbTest + { + private IndexerDefinition _indexer; + + [SetUp] + public void Setup() + { + _indexer = Builder.CreateNew() + .BuildNew(); + } + + private void GivenIndexer() + { + Db.Insert(_indexer); + } + + [Test] + public void should_delete_orphaned_indexerstatus() + { + var status = Builder.CreateNew() + .With(h => h.IndexerId = _indexer.Id) + .BuildNew(); + Db.Insert(status); + + Subject.Clean(); + AllStoredModels.Should().BeEmpty(); + } + + [Test] + public void should_not_delete_unorphaned_indexerstatus() + { + GivenIndexer(); + + var status = Builder.CreateNew() + .With(h => h.IndexerId = _indexer.Id) + .BuildNew(); + Db.Insert(status); + + Subject.Clean(); + AllStoredModels.Should().HaveCount(1); + AllStoredModels.Should().Contain(h => h.IndexerId == _indexer.Id); + } + } +} \ No newline at end of file diff --git a/src/NzbDrone.Core.Test/IndexerSearchTests/NzbSearchServiceFixture.cs b/src/NzbDrone.Core.Test/IndexerSearchTests/NzbSearchServiceFixture.cs index bdfc6afa7..4de560e28 100644 --- a/src/NzbDrone.Core.Test/IndexerSearchTests/NzbSearchServiceFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerSearchTests/NzbSearchServiceFixture.cs @@ -25,6 +25,7 @@ namespace NzbDrone.Core.Test.IndexerSearchTests public void SetUp() { _mockIndexer = Mocker.GetMock(); + _mockIndexer.SetupGet(s => s.Definition).Returns(new IndexerDefinition { Id = 1 }); _mockIndexer.SetupGet(s => s.SupportsSearch).Returns(true); Mocker.GetMock() diff --git a/src/NzbDrone.Core.Test/IndexerTests/BroadcastheNetTests/BroadcastheNetFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/BroadcastheNetTests/BroadcastheNetFixture.cs index 506cd8155..fcb18c4df 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/BroadcastheNetTests/BroadcastheNetFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/BroadcastheNetTests/BroadcastheNetFixture.cs @@ -59,7 +59,8 @@ namespace NzbDrone.Core.Test.IndexerTests.BroadcastheNetTests private void VerifyBackOff() { - // TODO How to detect (and implement) back-off logic. + Mocker.GetMock() + .Verify(v => v.RecordFailure(It.IsAny(), It.IsAny()), Times.Once()); } [Test] @@ -89,8 +90,6 @@ namespace NzbDrone.Core.Test.IndexerTests.BroadcastheNetTests results.Should().BeEmpty(); - results.Should().BeEmpty(); - VerifyBackOff(); ExceptionVerification.ExpectedWarns(1); diff --git a/src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs new file mode 100644 index 000000000..6d3f28d0e --- /dev/null +++ b/src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using System.Text; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Core.Indexers; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.IndexerTests +{ + public class IndexerStatusServiceFixture : CoreTest + { + private DateTime _epoch; + + [SetUp] + public void SetUp() + { + _epoch = DateTime.UtcNow; + } + + private void WithStatus(IndexerStatus status) + { + Mocker.GetMock() + .Setup(v => v.FindByIndexerId(1)) + .Returns(status); + + Mocker.GetMock() + .Setup(v => v.All()) + .Returns(new[] { status }); + } + + private void VerifyUpdate(bool updated = true) + { + Mocker.GetMock() + .Verify(v => v.Upsert(It.IsAny()), Times.Exactly(updated ? 1 : 0)); + } + + [Test] + public void should_start_backoff_on_first_failure() + { + WithStatus(new IndexerStatus()); + + Subject.RecordFailure(1); + + VerifyUpdate(); + + var status = Subject.GetBlockedIndexers().FirstOrDefault(); + status.Should().NotBeNull(); + status.DisabledTill.Should().HaveValue(); + status.DisabledTill.Value.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(5), 500); + } + + [Test] + public void should_cancel_backoff_on_success() + { + WithStatus(new IndexerStatus { EscalationLevel = 2 }); + + Subject.RecordSuccess(1); + + VerifyUpdate(); + + var status = Subject.GetBlockedIndexers().FirstOrDefault(); + status.Should().BeNull(); + } + + [Test] + public void should_not_store_update_if_already_okay() + { + WithStatus(new IndexerStatus { EscalationLevel = 0 }); + + Subject.RecordSuccess(1); + + VerifyUpdate(false); + } + + [Test] + public void should_preserve_escalation_on_intermittent_success() + { + WithStatus(new IndexerStatus { MostRecentFailure = _epoch - TimeSpan.FromSeconds(4), EscalationLevel = 3 }); + + Subject.RecordSuccess(1); + Subject.RecordSuccess(1); + Subject.RecordFailure(1); + + var status = Subject.GetBlockedIndexers().FirstOrDefault(); + status.Should().NotBeNull(); + status.DisabledTill.Should().HaveValue(); + status.DisabledTill.Value.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(15), 500); + } + } +} diff --git a/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabFixture.cs index a50616bf8..5fe0390cc 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabFixture.cs @@ -18,6 +18,7 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests { Subject.Definition = new IndexerDefinition() { + Id = 5, Name = "Newznab", Settings = new NewznabSettings() { @@ -47,6 +48,7 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests releaseInfo.DownloadUrl.Should().Be("http://nzb.su/getnzb/24967ef4c2e26296c65d3bbfa97aa8fe.nzb&i=37292&r=xxx"); releaseInfo.InfoUrl.Should().Be("http://nzb.su/details/24967ef4c2e26296c65d3bbfa97aa8fe"); releaseInfo.CommentUrl.Should().Be("http://nzb.su/details/24967ef4c2e26296c65d3bbfa97aa8fe#comments"); + releaseInfo.IndexerId.Should().Be(Subject.Definition.Id); releaseInfo.Indexer.Should().Be(Subject.Definition.Name); releaseInfo.PublishDate.Should().Be(DateTime.Parse("2012/02/27 16:09:39")); releaseInfo.Size.Should().Be(1183105773); diff --git a/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabRequestGeneratorFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabRequestGeneratorFixture.cs index 3649ede76..f95e5f626 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabRequestGeneratorFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabRequestGeneratorFixture.cs @@ -30,18 +30,6 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests AbsoluteEpisodeNumber = 100 }; } - - [Test] - public void should_return_one_page_for_feed() - { - var results = Subject.GetRecentRequests(); - - results.Should().HaveCount(1); - - var pages = results.First().Take(10).ToList(); - - pages.Should().HaveCount(1); - } [Test] public void should_use_all_categories_for_feed() diff --git a/src/NzbDrone.Core.Test/IndexerTests/TestIndexer.cs b/src/NzbDrone.Core.Test/IndexerTests/TestIndexer.cs index 0f9e334ea..357dbd48e 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/TestIndexer.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/TestIndexer.cs @@ -22,8 +22,8 @@ namespace NzbDrone.Core.Test.IndexerTests public Int32 _supportedPageSize; public override Int32 PageSize { get { return _supportedPageSize; } } - public TestIndexer(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger) - : base(httpClient, configService, parsingService, logger) + public TestIndexer(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) + : base(httpClient, indexerStatusService, configService, parsingService, logger) { } diff --git a/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TestTorrentRssIndexer.cs b/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TestTorrentRssIndexer.cs index 71bf3b22f..ce7ec77b5 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TestTorrentRssIndexer.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TestTorrentRssIndexer.cs @@ -13,8 +13,8 @@ namespace NzbDrone.Core.Test.IndexerTests.TorrentRssIndexerTests { public class TestTorrentRssIndexer : TorrentRssIndexer { - public TestTorrentRssIndexer(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, ITorrentRssParserFactory torrentRssParserFactory, Logger logger) - : base(httpClient, configService, parsingService, torrentRssParserFactory, logger) + public TestTorrentRssIndexer(ITorrentRssParserFactory torrentRssParserFactory, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) + : base(torrentRssParserFactory, httpClient, indexerStatusService, configService, parsingService, logger) { } diff --git a/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabRequestGeneratorFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabRequestGeneratorFixture.cs index 03943179a..ff33ba6d8 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabRequestGeneratorFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabRequestGeneratorFixture.cs @@ -48,18 +48,6 @@ namespace NzbDrone.Core.Test.IndexerTests.TorznabTests .Setup(v => v.GetCapabilities(It.IsAny())) .Returns(_capabilities); } - - [Test] - public void should_return_one_page_for_feed() - { - var results = Subject.GetRecentRequests(); - - results.Should().HaveCount(1); - - var pages = results.First().Take(10).ToList(); - - pages.Should().HaveCount(1); - } [Test] public void should_use_all_categories_for_feed() diff --git a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj index 129579313..761a3ff94 100644 --- a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj +++ b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj @@ -168,6 +168,7 @@ + @@ -193,6 +194,7 @@ + @@ -203,6 +205,7 @@ + @@ -215,6 +218,7 @@ + diff --git a/src/NzbDrone.Core/Datastore/Migration/091_added_indexerstatus.cs b/src/NzbDrone.Core/Datastore/Migration/091_added_indexerstatus.cs new file mode 100644 index 000000000..1a6027149 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/091_added_indexerstatus.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.Data; +using FluentMigrator; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Serializer; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(91)] + public class added_indexerstatus : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Create.TableForModel("IndexerStatus") + .WithColumn("IndexerId").AsInt32().NotNullable().Unique() + .WithColumn("InitialFailure").AsDateTime().Nullable() + .WithColumn("MostRecentFailure").AsDateTime().Nullable() + .WithColumn("EscalationLevel").AsInt32().NotNullable() + .WithColumn("DisabledTill").AsDateTime().Nullable() + .WithColumn("LastRssSyncReleaseInfo").AsString().Nullable(); + } + } +} diff --git a/src/NzbDrone.Core/Datastore/TableMapping.cs b/src/NzbDrone.Core/Datastore/TableMapping.cs index 5f8eb8c2d..760a07c91 100644 --- a/src/NzbDrone.Core/Datastore/TableMapping.cs +++ b/src/NzbDrone.Core/Datastore/TableMapping.cs @@ -111,6 +111,8 @@ namespace NzbDrone.Core.Datastore Mapper.Entity().RegisterModel("Users"); Mapper.Entity().RegisterModel("Commands") .Ignore(c => c.Message); + + Mapper.Entity().RegisterModel("IndexerStatus"); } private static void RegisterMappers() diff --git a/src/NzbDrone.Core/Download/DownloadService.cs b/src/NzbDrone.Core/Download/DownloadService.cs index 6e19c0963..115ba1f94 100644 --- a/src/NzbDrone.Core/Download/DownloadService.cs +++ b/src/NzbDrone.Core/Download/DownloadService.cs @@ -2,8 +2,11 @@ using NLog; using NzbDrone.Common.EnsureThat; using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; using NzbDrone.Common.Instrumentation.Extensions; using NzbDrone.Common.TPL; +using NzbDrone.Core.Exceptions; +using NzbDrone.Core.Indexers; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Parser.Model; @@ -18,16 +21,19 @@ namespace NzbDrone.Core.Download public class DownloadService : IDownloadService { private readonly IProvideDownloadClient _downloadClientProvider; + private readonly IIndexerStatusService _indexerStatusService; private readonly IRateLimitService _rateLimitService; private readonly IEventAggregator _eventAggregator; private readonly Logger _logger; public DownloadService(IProvideDownloadClient downloadClientProvider, + IIndexerStatusService indexerStatusService, IRateLimitService rateLimitService, IEventAggregator eventAggregator, Logger logger) { _downloadClientProvider = downloadClientProvider; + _indexerStatusService = indexerStatusService; _rateLimitService = rateLimitService; _eventAggregator = eventAggregator; _logger = logger; @@ -54,7 +60,26 @@ namespace NzbDrone.Core.Download _rateLimitService.WaitAndPulse(uri.Host, TimeSpan.FromSeconds(2)); } - var downloadClientId = downloadClient.Download(remoteEpisode); + string downloadClientId; + try + { + downloadClientId = downloadClient.Download(remoteEpisode); + _indexerStatusService.RecordSuccess(remoteEpisode.Release.IndexerId); + } + catch (ReleaseDownloadException ex) + { + var http429 = ex.InnerException as TooManyRequestsException; + if (http429 != null) + { + _indexerStatusService.RecordFailure(remoteEpisode.Release.IndexerId, http429.RetryAfter); + } + else + { + _indexerStatusService.RecordFailure(remoteEpisode.Release.IndexerId); + } + throw; + } + var episodeGrabbedEvent = new EpisodeGrabbedEvent(remoteEpisode); episodeGrabbedEvent.DownloadClient = downloadClient.GetType().Name; diff --git a/src/NzbDrone.Core/Download/Pending/PendingReleaseService.cs b/src/NzbDrone.Core/Download/Pending/PendingReleaseService.cs index c4b4cec0b..57cc746ca 100644 --- a/src/NzbDrone.Core/Download/Pending/PendingReleaseService.cs +++ b/src/NzbDrone.Core/Download/Pending/PendingReleaseService.cs @@ -35,6 +35,7 @@ namespace NzbDrone.Core.Download.Pending IHandle, IHandle { + private readonly IIndexerStatusService _indexerStatusService; private readonly IPendingReleaseRepository _repository; private readonly ISeriesService _seriesService; private readonly IParsingService _parsingService; @@ -44,7 +45,8 @@ namespace NzbDrone.Core.Download.Pending private readonly IEventAggregator _eventAggregator; private readonly Logger _logger; - public PendingReleaseService(IPendingReleaseRepository repository, + public PendingReleaseService(IIndexerStatusService indexerStatusService, + IPendingReleaseRepository repository, ISeriesService seriesService, IParsingService parsingService, IDelayProfileService delayProfileService, @@ -53,6 +55,7 @@ namespace NzbDrone.Core.Download.Pending IEventAggregator eventAggregator, Logger logger) { + _indexerStatusService = indexerStatusService; _repository = repository; _seriesService = seriesService; _parsingService = parsingService; @@ -86,7 +89,21 @@ namespace NzbDrone.Core.Download.Pending public List GetPending() { - return _repository.All().Select(p => p.Release).ToList(); + var releases = _repository.All().Select(p => p.Release).ToList(); + + if (releases.Any()) + { + releases = FilterBlockedIndexers(releases); + } + + return releases; + } + + private List FilterBlockedIndexers(List releases) + { + var blockedIndexers = new HashSet(_indexerStatusService.GetBlockedIndexers().Select(v => v.IndexerId)); + + return releases.Where(release => !blockedIndexers.Contains(release.IndexerId)).ToList(); } public List GetPendingRemoteEpisodes(int seriesId) diff --git a/src/NzbDrone.Core/Download/TorrentClientBase.cs b/src/NzbDrone.Core/Download/TorrentClientBase.cs index d4c2a7bc3..cbd5b36e9 100644 --- a/src/NzbDrone.Core/Download/TorrentClientBase.cs +++ b/src/NzbDrone.Core/Download/TorrentClientBase.cs @@ -126,10 +126,16 @@ namespace NzbDrone.Core.Download torrentFile = response.ResponseData; } + catch (HttpException ex) + { + _logger.ErrorException(String.Format("Downloading torrent file for episode '{0}' failed ({1})", + remoteEpisode.Release.Title, torrentUrl), ex); + throw new ReleaseDownloadException(remoteEpisode.Release, "Downloading torrent failed", ex); + } catch (WebException ex) { - _logger.ErrorException(String.Format("Downloading torrentfile for episode '{0}' failed ({1})", + _logger.ErrorException(String.Format("Downloading torrent file for episode '{0}' failed ({1})", remoteEpisode.Release.Title, torrentUrl), ex); throw new ReleaseDownloadException(remoteEpisode.Release, "Downloading torrent failed", ex); diff --git a/src/NzbDrone.Core/Download/UsenetClientBase.cs b/src/NzbDrone.Core/Download/UsenetClientBase.cs index f5011911a..b29e55fc2 100644 --- a/src/NzbDrone.Core/Download/UsenetClientBase.cs +++ b/src/NzbDrone.Core/Download/UsenetClientBase.cs @@ -49,6 +49,13 @@ namespace NzbDrone.Core.Download { nzbData = _httpClient.Get(new HttpRequest(url)).ResponseData; } + catch (HttpException ex) + { + _logger.ErrorException(String.Format("Downloading nzb for episode '{0}' failed ({1})", + remoteEpisode.Release.Title, url), ex); + + throw new ReleaseDownloadException(remoteEpisode.Release, "Downloading nzb failed", ex); + } catch (WebException ex) { _logger.ErrorException(String.Format("Downloading nzb for episode '{0}' failed ({1})", diff --git a/src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs new file mode 100644 index 000000000..2b6aafa55 --- /dev/null +++ b/src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs @@ -0,0 +1,48 @@ +using System; +using System.Linq; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.Indexers; + +namespace NzbDrone.Core.HealthCheck.Checks +{ + public class IndexerStatusCheck : HealthCheckBase + { + private readonly IIndexerFactory _indexerFactory; + private readonly IIndexerStatusService _indexerStatusService; + + public IndexerStatusCheck(IIndexerFactory indexerFactory, IIndexerStatusService indexerStatusService) + { + _indexerFactory = indexerFactory; + _indexerStatusService = indexerStatusService; + } + + public override HealthCheck Check() + { + var enabledIndexers = _indexerFactory.GetAvailableProviders(); + var backOffIndexers = enabledIndexers.Join(_indexerStatusService.GetBlockedIndexers(), + i => i.Definition.Id, + s => s.IndexerId, + (i, s) => new { Indexer = i, Status = s }) + .Where(v => (v.Status.MostRecentFailure - v.Status.InitialFailure) > TimeSpan.FromHours(1)) + .ToList(); + + if (backOffIndexers.Empty()) + { + return new HealthCheck(GetType()); + } + + if (backOffIndexers.Count == enabledIndexers.Count) + { + return new HealthCheck(GetType(), HealthCheckResult.Error, "All indexers are unavailable due to failures", "#indexers-are-unavailable-due-to-failures"); + } + + if (backOffIndexers.Count > 1) + { + return new HealthCheck(GetType(), HealthCheckResult.Warning, string.Format("{0} indexers are unavailable due to failures", backOffIndexers.Count), "#indexers-are-unavailable-due-to-failures"); + } + + var indexer = backOffIndexers.First(); + return new HealthCheck(GetType(), HealthCheckResult.Warning, string.Format("Indexer {0} is unavailable due to failures", indexer.Indexer.Definition.Name), "#indexers-are-unavailable-due-to-failures"); + } + } +} diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedIndexerStatus.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedIndexerStatus.cs new file mode 100644 index 000000000..b3cf47027 --- /dev/null +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedIndexerStatus.cs @@ -0,0 +1,26 @@ +using NzbDrone.Core.Datastore; + +namespace NzbDrone.Core.Housekeeping.Housekeepers +{ + public class CleanupOrphanedIndexerStatus : IHousekeepingTask + { + private readonly IMainDatabase _database; + + public CleanupOrphanedIndexerStatus(IMainDatabase database) + { + _database = database; + } + + public void Clean() + { + var mapper = _database.GetDataMapper(); + + mapper.ExecuteNonQuery(@"DELETE FROM IndexerStatus + WHERE Id IN ( + SELECT IndexerStatus.Id FROM IndexerStatus + LEFT OUTER JOIN Indexers + ON IndexerStatus.IndexerId = Indexers.Id + WHERE Indexers.Id IS NULL)"); + } + } +} diff --git a/src/NzbDrone.Core/IndexerSearch/EpisodeSearchService.cs b/src/NzbDrone.Core/IndexerSearch/EpisodeSearchService.cs index f58d9be7a..3ae1e6191 100644 --- a/src/NzbDrone.Core/IndexerSearch/EpisodeSearchService.cs +++ b/src/NzbDrone.Core/IndexerSearch/EpisodeSearchService.cs @@ -16,14 +16,7 @@ using NzbDrone.Core.Tv.Events; namespace NzbDrone.Core.IndexerSearch { - public interface IEpisodeSearchService - { - void MissingEpisodesAiredAfter(DateTime dateTime, IEnumerable grabbed); - } - - public class EpisodeSearchService : IEpisodeSearchService, - IExecute, - IExecute + public class EpisodeSearchService : IExecute, IExecute { private readonly ISearchForNzb _nzbSearchService; private readonly IProcessDownloadDecisions _processDownloadDecisions; @@ -44,28 +37,6 @@ namespace NzbDrone.Core.IndexerSearch _logger = logger; } - public void MissingEpisodesAiredAfter(DateTime dateTime, IEnumerable grabbed) - { - var missing = _episodeService.EpisodesBetweenDates(dateTime, DateTime.UtcNow, false) - .Where(e => !e.HasFile && - !_queueService.GetQueue().Select(q => q.Episode.Id).Contains(e.Id) && - !grabbed.Contains(e.Id)) - .ToList(); - - var downloadedCount = 0; - _logger.Info("Searching for {0} missing episodes since last RSS Sync", missing.Count); - - foreach (var episode in missing) - { - //TODO: Add a flag to the search to state it is a "scheduled" search - var decisions = _nzbSearchService.EpisodeSearch(episode); - var processed = _processDownloadDecisions.ProcessDecisions(decisions); - downloadedCount += processed.Grabbed.Count; - } - - _logger.ProgressInfo("Completed search for {0} episodes. {1} reports downloaded.", missing.Count, downloadedCount); - } - private void SearchForMissingEpisodes(List episodes) { _logger.ProgressInfo("Performing missing search for {0} episodes", episodes.Count); diff --git a/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs b/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs index e56d1c734..7211dbea1 100644 --- a/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs +++ b/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs @@ -247,7 +247,7 @@ namespace NzbDrone.Core.IndexerSearch private List Dispatch(Func> searchAction, SearchCriteriaBase criteriaBase) { - var indexers = _indexerFactory.SearchEnabled().ToList(); + var indexers = _indexerFactory.SearchEnabled(); var reports = new List(); _logger.ProgressInfo("Searching {0} indexers for {1}", indexers.Count, criteriaBase); diff --git a/src/NzbDrone.Core/Indexers/BitMeTv/BitMeTv.cs b/src/NzbDrone.Core/Indexers/BitMeTv/BitMeTv.cs index 283915ac9..0ed0cefd5 100644 --- a/src/NzbDrone.Core/Indexers/BitMeTv/BitMeTv.cs +++ b/src/NzbDrone.Core/Indexers/BitMeTv/BitMeTv.cs @@ -20,8 +20,8 @@ namespace NzbDrone.Core.Indexers.BitMeTv public override Boolean SupportsSearch { get { return false; } } public override Int32 PageSize { get { return 0; } } - public BitMeTv(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger) - : base(httpClient, configService, parsingService, logger) + public BitMeTv(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) + : base(httpClient, indexerStatusService, configService, parsingService, logger) { } diff --git a/src/NzbDrone.Core/Indexers/BroadcastheNet/BroadcastheNet.cs b/src/NzbDrone.Core/Indexers/BroadcastheNet/BroadcastheNet.cs index a36ce0040..9917c1d1d 100644 --- a/src/NzbDrone.Core/Indexers/BroadcastheNet/BroadcastheNet.cs +++ b/src/NzbDrone.Core/Indexers/BroadcastheNet/BroadcastheNet.cs @@ -20,8 +20,8 @@ namespace NzbDrone.Core.Indexers.BroadcastheNet public override bool SupportsSearch { get { return true; } } public override int PageSize { get { return 100; } } - public BroadcastheNet(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger) - : base(httpClient, configService, parsingService, logger) + public BroadcastheNet(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) + : base(httpClient, indexerStatusService, configService, parsingService, logger) { } diff --git a/src/NzbDrone.Core/Indexers/BroadcastheNet/BroadcastheNetRequestGenerator.cs b/src/NzbDrone.Core/Indexers/BroadcastheNet/BroadcastheNetRequestGenerator.cs index 37c819ef6..db41099b8 100644 --- a/src/NzbDrone.Core/Indexers/BroadcastheNet/BroadcastheNetRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/BroadcastheNet/BroadcastheNetRequestGenerator.cs @@ -23,7 +23,7 @@ namespace NzbDrone.Core.Indexers.BroadcastheNet { var pageableRequests = new List>(); - pageableRequests.AddIfNotNull(GetPagedRequests(1, null)); + pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, null)); return pageableRequests; } diff --git a/src/NzbDrone.Core/Indexers/Fanzub/Fanzub.cs b/src/NzbDrone.Core/Indexers/Fanzub/Fanzub.cs index 7c06b8d50..ad1231aeb 100644 --- a/src/NzbDrone.Core/Indexers/Fanzub/Fanzub.cs +++ b/src/NzbDrone.Core/Indexers/Fanzub/Fanzub.cs @@ -17,8 +17,8 @@ namespace NzbDrone.Core.Indexers.Fanzub public override DownloadProtocol Protocol { get { return DownloadProtocol.Usenet; } } - public Fanzub(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger) - : base(httpClient, configService, parsingService, logger) + public Fanzub(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) + : base(httpClient, indexerStatusService, configService, parsingService, logger) { } diff --git a/src/NzbDrone.Core/Indexers/FetchAndParseRssService.cs b/src/NzbDrone.Core/Indexers/FetchAndParseRssService.cs index 4e9dd1ccc..f1f0c48fd 100644 --- a/src/NzbDrone.Core/Indexers/FetchAndParseRssService.cs +++ b/src/NzbDrone.Core/Indexers/FetchAndParseRssService.cs @@ -4,7 +4,8 @@ using System.Threading.Tasks; using NLog; using NzbDrone.Core.Parser.Model; using NzbDrone.Common.TPL; - +using System.Collections; +using System; namespace NzbDrone.Core.Indexers { public interface IFetchAndParseRss @@ -27,7 +28,7 @@ namespace NzbDrone.Core.Indexers { var result = new List(); - var indexers = _indexerFactory.RssEnabled().ToList(); + var indexers = _indexerFactory.RssEnabled(); if (!indexers.Any()) { diff --git a/src/NzbDrone.Core/Indexers/HDBits/HDBits.cs b/src/NzbDrone.Core/Indexers/HDBits/HDBits.cs index 49458f969..7cdf779e9 100644 --- a/src/NzbDrone.Core/Indexers/HDBits/HDBits.cs +++ b/src/NzbDrone.Core/Indexers/HDBits/HDBits.cs @@ -14,11 +14,8 @@ namespace NzbDrone.Core.Indexers.HDBits public override bool SupportsSearch { get { return true; } } public override int PageSize { get { return 30; } } - public HDBits(IHttpClient httpClient, - IConfigService configService, - IParsingService parsingService, - Logger logger) - : base(httpClient, configService, parsingService, logger) + public HDBits(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) + : base(httpClient, indexerStatusService, configService, parsingService, logger) { } public override IIndexerRequestGenerator GetRequestGenerator() diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs index f8bb282b9..ff9d2a34c 100644 --- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs @@ -33,8 +33,8 @@ namespace NzbDrone.Core.Indexers public abstract IIndexerRequestGenerator GetRequestGenerator(); public abstract IParseIndexerResponse GetParser(); - public HttpIndexerBase(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger) - : base(configService, parsingService, logger) + public HttpIndexerBase(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) + : base(indexerStatusService, configService, parsingService, logger) { _httpClient = httpClient; } @@ -48,7 +48,7 @@ namespace NzbDrone.Core.Indexers var generator = GetRequestGenerator(); - return FetchReleases(generator.GetRecentRequests()); + return FetchReleases(generator.GetRecentRequests(), true); } public override IList Fetch(SingleEpisodeSearchCriteria searchCriteria) @@ -111,7 +111,7 @@ namespace NzbDrone.Core.Indexers return FetchReleases(generator.GetSearchRequests(searchCriteria)); } - protected virtual IList FetchReleases(IList> pageableRequests) + protected virtual IList FetchReleases(IList> pageableRequests, bool isRecent = false) { var releases = new List(); var url = String.Empty; @@ -120,6 +120,13 @@ namespace NzbDrone.Core.Indexers try { + var fullyUpdated = false; + ReleaseInfo lastReleaseInfo = null; + if (isRecent) + { + lastReleaseInfo = _indexerStatusService.GetLastRssSyncReleaseInfo(Definition.Id); + } + foreach (var pageableRequest in pageableRequests) { var pagedReleases = new List(); @@ -132,7 +139,33 @@ namespace NzbDrone.Core.Indexers pagedReleases.AddRange(page); - if (!IsFullPage(page) || pagedReleases.Count >= MaxNumResultsPerQuery) + if (isRecent) + { + if (lastReleaseInfo == null) + { + fullyUpdated = true; + break; + } + var oldestReleaseDate = page.Select(v => v.PublishDate).Min(); + if (oldestReleaseDate < lastReleaseInfo.PublishDate || page.Any(v => v.DownloadUrl == lastReleaseInfo.DownloadUrl)) + { + fullyUpdated = true; + break; + } + + if (pagedReleases.Count >= MaxNumResultsPerQuery && + oldestReleaseDate < DateTime.UtcNow - TimeSpan.FromHours(24)) + { + fullyUpdated = false; + break; + } + } + else if (pagedReleases.Count >= MaxNumResultsPerQuery) + { + break; + } + + if (!IsFullPage(page)) { break; } @@ -140,9 +173,26 @@ namespace NzbDrone.Core.Indexers releases.AddRange(pagedReleases); } + + if (isRecent && !releases.Empty()) + { + var ordered = releases.OrderByDescending(v => v.PublishDate).ToList(); + + if (!fullyUpdated && lastReleaseInfo != null) + { + var gapStart = lastReleaseInfo.PublishDate; + var gapEnd = ordered.Last(); + _logger.Warn("Indexer {0} rss sync didn't cover the period between {1} and {2} UTC. Search may be required.", Definition.Name, gapStart, gapEnd); + } + lastReleaseInfo = ordered.First(); + _indexerStatusService.UpdateRssSyncStatus(Definition.Id, lastReleaseInfo); + } + + _indexerStatusService.RecordSuccess(Definition.Id); } catch (WebException webException) { + _indexerStatusService.RecordFailure(Definition.Id); if (webException.Message.Contains("502") || webException.Message.Contains("503") || webException.Message.Contains("timed out")) { @@ -155,29 +205,36 @@ namespace NzbDrone.Core.Indexers } catch (HttpException httpException) { - if ((int) httpException.Response.StatusCode == 429) + if ((int)httpException.Response.StatusCode == 429) { + _indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1)); _logger.Warn("API Request Limit reached for {0}", this); } - - _logger.Warn("{0} {1}", this, httpException.Message); + else + { + _indexerStatusService.RecordFailure(Definition.Id); + _logger.Warn("{0} {1}", this, httpException.Message); + } } catch (RequestLimitReachedException) { - // TODO: Backoff for x period. + _indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1)); _logger.Warn("API Request Limit reached for {0}", this); } catch (ApiKeyException) { + _indexerStatusService.RecordFailure(Definition.Id); _logger.Warn("Invalid API Key for {0} {1}", this, url); } catch (IndexerException ex) { + _indexerStatusService.RecordFailure(Definition.Id); var message = String.Format("{0} - {1}", ex.Message, url); _logger.WarnException(message, ex); } catch (Exception feedEx) { + _indexerStatusService.RecordFailure(Definition.Id); feedEx.Data.Add("FeedUrl", url); _logger.ErrorException("An error occurred while processing feed. " + url, feedEx); } diff --git a/src/NzbDrone.Core/Indexers/IPTorrents/IPTorrents.cs b/src/NzbDrone.Core/Indexers/IPTorrents/IPTorrents.cs index f0ba7d4f6..edf1ddd45 100644 --- a/src/NzbDrone.Core/Indexers/IPTorrents/IPTorrents.cs +++ b/src/NzbDrone.Core/Indexers/IPTorrents/IPTorrents.cs @@ -20,8 +20,8 @@ namespace NzbDrone.Core.Indexers.IPTorrents public override Boolean SupportsSearch { get { return false; } } public override Int32 PageSize { get { return 0; } } - public IPTorrents(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger) - : base(httpClient, configService, parsingService, logger) + public IPTorrents(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) + : base(httpClient, indexerStatusService, configService, parsingService, logger) { } diff --git a/src/NzbDrone.Core/Indexers/IndexerBase.cs b/src/NzbDrone.Core/Indexers/IndexerBase.cs index 5c6f73cf3..ea23275d8 100644 --- a/src/NzbDrone.Core/Indexers/IndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/IndexerBase.cs @@ -15,6 +15,7 @@ namespace NzbDrone.Core.Indexers public abstract class IndexerBase : IIndexer where TSettings : IProviderConfig, new() { + protected readonly IIndexerStatusService _indexerStatusService; protected readonly IConfigService _configService; protected readonly IParsingService _parsingService; protected readonly Logger _logger; @@ -25,8 +26,9 @@ namespace NzbDrone.Core.Indexers public abstract Boolean SupportsRss { get; } public abstract Boolean SupportsSearch { get; } - public IndexerBase(IConfigService configService, IParsingService parsingService, Logger logger) + public IndexerBase(IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) { + _indexerStatusService = indexerStatusService; _configService = configService; _parsingService = parsingService; _logger = logger; @@ -85,6 +87,7 @@ namespace NzbDrone.Core.Indexers result.ForEach(c => { + c.IndexerId = Definition.Id; c.Indexer = Definition.Name; c.DownloadProtocol = Protocol; }); @@ -106,6 +109,11 @@ namespace NzbDrone.Core.Indexers failures.Add(new ValidationFailure(string.Empty, "Test was aborted due to an error: " + ex.Message)); } + if (Definition.Id != 0) + { + _indexerStatusService.RecordSuccess(Definition.Id); + } + return new ValidationResult(failures); } diff --git a/src/NzbDrone.Core/Indexers/IndexerDefinition.cs b/src/NzbDrone.Core/Indexers/IndexerDefinition.cs index 026939dab..29004a1ab 100644 --- a/src/NzbDrone.Core/Indexers/IndexerDefinition.cs +++ b/src/NzbDrone.Core/Indexers/IndexerDefinition.cs @@ -18,5 +18,7 @@ namespace NzbDrone.Core.Indexers return EnableRss || EnableSearch; } } + + public IndexerStatus Status { get; set; } } } diff --git a/src/NzbDrone.Core/Indexers/IndexerFactory.cs b/src/NzbDrone.Core/Indexers/IndexerFactory.cs index 775e08374..a4fab2bb5 100644 --- a/src/NzbDrone.Core/Indexers/IndexerFactory.cs +++ b/src/NzbDrone.Core/Indexers/IndexerFactory.cs @@ -15,21 +15,21 @@ namespace NzbDrone.Core.Indexers public class IndexerFactory : ProviderFactory, IIndexerFactory { + private readonly IIndexerStatusService _indexerStatusService; private readonly IIndexerRepository _providerRepository; + private readonly Logger _logger; - public IndexerFactory(IIndexerRepository providerRepository, + public IndexerFactory(IIndexerStatusService indexerStatusService, + IIndexerRepository providerRepository, IEnumerable providers, IContainer container, IEventAggregator eventAggregator, Logger logger) : base(providerRepository, providers, container, eventAggregator, logger) { + _indexerStatusService = indexerStatusService; _providerRepository = providerRepository; - } - - protected override void InitializeProviders() - { - //_providerRepository.DeleteImplementations("Animezb"); + _logger = logger; } protected override List Active() @@ -50,12 +50,37 @@ namespace NzbDrone.Core.Indexers public List RssEnabled() { - return GetAvailableProviders().Where(n => ((IndexerDefinition)n.Definition).EnableRss).ToList(); + var enabledIndexers = GetAvailableProviders().Where(n => ((IndexerDefinition)n.Definition).EnableRss); + + var indexers = FilterBlockedIndexers(enabledIndexers); + + return indexers.ToList(); } public List SearchEnabled() { - return GetAvailableProviders().Where(n => ((IndexerDefinition)n.Definition).EnableSearch).ToList(); - } + var enabledIndexers = GetAvailableProviders().Where(n => ((IndexerDefinition)n.Definition).EnableSearch); + + var indexers = FilterBlockedIndexers(enabledIndexers); + + return indexers.ToList(); + } + + private IEnumerable FilterBlockedIndexers(IEnumerable indexers) + { + var blockedIndexers = _indexerStatusService.GetBlockedIndexers().ToDictionary(v => v.IndexerId, v => v); + + foreach (var indexer in indexers) + { + IndexerStatus blockedIndexerStatus; + if (blockedIndexers.TryGetValue(indexer.Definition.Id, out blockedIndexerStatus)) + { + _logger.Debug("Temporarily ignoring indexer {0} till {1} due to recent failures.", indexer.Definition.Name, blockedIndexerStatus.DisabledTill.Value); + continue; + } + + yield return indexer; + } + } } } \ No newline at end of file diff --git a/src/NzbDrone.Core/Indexers/IndexerStatus.cs b/src/NzbDrone.Core/Indexers/IndexerStatus.cs new file mode 100644 index 000000000..8c47cf6c2 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/IndexerStatus.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.Indexers +{ + public class IndexerStatus : ModelBase + { + public int IndexerId { get; set; } + + public DateTime? InitialFailure { get; set; } + public DateTime? MostRecentFailure { get; set; } + public int EscalationLevel { get; set; } + public DateTime? DisabledTill { get; set; } + + public ReleaseInfo LastRssSyncReleaseInfo { get; set; } + + public bool IsDisabled() + { + return DisabledTill.HasValue && DisabledTill.Value > DateTime.UtcNow; + } + } +} diff --git a/src/NzbDrone.Core/Indexers/IndexerStatusRepository.cs b/src/NzbDrone.Core/Indexers/IndexerStatusRepository.cs new file mode 100644 index 000000000..8a70b790a --- /dev/null +++ b/src/NzbDrone.Core/Indexers/IndexerStatusRepository.cs @@ -0,0 +1,26 @@ +using System.Linq; +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.ThingiProvider; + + +namespace NzbDrone.Core.Indexers +{ + public interface IIndexerStatusRepository : IProviderRepository + { + IndexerStatus FindByIndexerId(int indexerId); + } + + public class IndexerStatusRepository : ProviderRepository, IIndexerStatusRepository + { + public IndexerStatusRepository(IMainDatabase database, IEventAggregator eventAggregator) + : base(database, eventAggregator) + { + } + + public IndexerStatus FindByIndexerId(int indexerId) + { + return Query.Where(c => c.IndexerId == indexerId).SingleOrDefault(); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/IndexerStatusService.cs b/src/NzbDrone.Core/Indexers/IndexerStatusService.cs new file mode 100644 index 000000000..5a43638ee --- /dev/null +++ b/src/NzbDrone.Core/Indexers/IndexerStatusService.cs @@ -0,0 +1,140 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using NLog; +using NzbDrone.Common.Cache; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.ThingiProvider.Events; + +namespace NzbDrone.Core.Indexers +{ + public interface IIndexerStatusService + { + List GetBlockedIndexers(); + ReleaseInfo GetLastRssSyncReleaseInfo(int indexerId); + void RecordSuccess(int indexerId); + void RecordFailure(int indexerId, TimeSpan minimumBackOff = default(TimeSpan)); + + void UpdateRssSyncStatus(int indexerId, ReleaseInfo releaseInfo); + } + + public class IndexerStatusService : IIndexerStatusService, IHandleAsync> + { + private static readonly int[] EscalationBackOffPeriods = { + 0, + 5 * 60, + 15 * 60, + 30 * 60, + 60 * 60, + 3 * 60 * 60, + 6 * 60 * 60, + 12 * 60 * 60, + 24 * 60 * 60 + }; + private static readonly int MaximumEscalationLevel = EscalationBackOffPeriods.Length - 1; + + private static readonly object _syncRoot = new object(); + + private readonly IIndexerStatusRepository _indexerStatusRepository; + private readonly Logger _logger; + + public IndexerStatusService(IIndexerStatusRepository indexerStatusRepository, Logger logger) + { + _indexerStatusRepository = indexerStatusRepository; + _logger = logger; + } + + public List GetBlockedIndexers() + { + return _indexerStatusRepository.All().Where(v => v.IsDisabled()).ToList(); + } + + public ReleaseInfo GetLastRssSyncReleaseInfo(int indexerId) + { + return GetIndexerStatus(indexerId).LastRssSyncReleaseInfo; + } + + private IndexerStatus GetIndexerStatus(int indexerId) + { + return _indexerStatusRepository.FindByIndexerId(indexerId) ?? new IndexerStatus { IndexerId = indexerId }; + } + + private TimeSpan CalculateBackOffPeriod(IndexerStatus status) + { + var level = Math.Min(MaximumEscalationLevel, status.EscalationLevel); + + return TimeSpan.FromSeconds(EscalationBackOffPeriods[level]); + } + + public void RecordSuccess(int indexerId) + { + lock (_syncRoot) + { + var status = GetIndexerStatus(indexerId); + + if (status.EscalationLevel == 0) + { + return; + } + + status.EscalationLevel--; + status.DisabledTill = null; + + _indexerStatusRepository.Upsert(status); + } + } + + public void RecordFailure(int indexerId, TimeSpan minimumBackOff = default(TimeSpan)) + { + lock (_syncRoot) + { + var status = GetIndexerStatus(indexerId); + + var now = DateTime.UtcNow; + + if (status.EscalationLevel == 0) + { + status.InitialFailure = now; + } + + status.MostRecentFailure = now; + status.EscalationLevel = Math.Min(MaximumEscalationLevel, status.EscalationLevel + 1); + + if (minimumBackOff != TimeSpan.Zero) + { + while (status.EscalationLevel < MaximumEscalationLevel && CalculateBackOffPeriod(status) < minimumBackOff) + { + status.EscalationLevel++; + } + } + + status.DisabledTill = now + CalculateBackOffPeriod(status); + + _indexerStatusRepository.Upsert(status); + } + } + + public void UpdateRssSyncStatus(int indexerId, ReleaseInfo releaseInfo) + { + lock (_syncRoot) + { + var status = GetIndexerStatus(indexerId); + + status.LastRssSyncReleaseInfo = releaseInfo; + + _indexerStatusRepository.Upsert(status); + } + } + + public void HandleAsync(ProviderDeletedEvent message) + { + var indexerStatus = _indexerStatusRepository.FindByIndexerId(message.ProviderId); + + if (indexerStatus != null) + { + _indexerStatusRepository.Delete(indexerStatus); + } + } + } +} diff --git a/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrents.cs b/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrents.cs index edf33570f..3532cf2f2 100644 --- a/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrents.cs +++ b/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrents.cs @@ -19,8 +19,8 @@ namespace NzbDrone.Core.Indexers.KickassTorrents public override DownloadProtocol Protocol { get { return DownloadProtocol.Torrent; } } public override Int32 PageSize { get { return 25; } } - public KickassTorrents(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger) - : base(httpClient, configService, parsingService, logger) + public KickassTorrents(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) + : base(httpClient, indexerStatusService, configService, parsingService, logger) { } diff --git a/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrentsRequestGenerator.cs b/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrentsRequestGenerator.cs index 4e07b1ba3..a310cd1f0 100644 --- a/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrentsRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrentsRequestGenerator.cs @@ -24,8 +24,7 @@ namespace NzbDrone.Core.Indexers.KickassTorrents { var pageableRequests = new List>(); - // We give kat a bit more pages to get to 100 total for recent, coz users have been missing releases. - pageableRequests.AddIfNotNull(GetPagedRequests(4, "tv")); + pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, "tv")); return pageableRequests; } diff --git a/src/NzbDrone.Core/Indexers/Newznab/Newznab.cs b/src/NzbDrone.Core/Indexers/Newznab/Newznab.cs index fbc5af2f8..96a17a769 100644 --- a/src/NzbDrone.Core/Indexers/Newznab/Newznab.cs +++ b/src/NzbDrone.Core/Indexers/Newznab/Newznab.cs @@ -50,8 +50,8 @@ namespace NzbDrone.Core.Indexers.Newznab } } - public Newznab(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger) - : base(httpClient, configService, parsingService, logger) + public Newznab(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) + : base(httpClient, indexerStatusService, configService, parsingService, logger) { } diff --git a/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs index 2c28f5a89..1091f948b 100644 --- a/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs @@ -23,8 +23,7 @@ namespace NzbDrone.Core.Indexers.Newznab { var pageableRequests = new List>(); - // TODO: We might consider getting multiple pages in the future, but atm we limit it to 1 page. - pageableRequests.AddIfNotNull(GetPagedRequests(1, Settings.Categories.Concat(Settings.AnimeCategories), "tvsearch", "")); + pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories.Concat(Settings.AnimeCategories), "tvsearch", "")); return pageableRequests; } diff --git a/src/NzbDrone.Core/Indexers/Nyaa/Nyaa.cs b/src/NzbDrone.Core/Indexers/Nyaa/Nyaa.cs index e869682c1..c5faae14b 100644 --- a/src/NzbDrone.Core/Indexers/Nyaa/Nyaa.cs +++ b/src/NzbDrone.Core/Indexers/Nyaa/Nyaa.cs @@ -19,8 +19,8 @@ namespace NzbDrone.Core.Indexers.Nyaa public override DownloadProtocol Protocol { get { return DownloadProtocol.Torrent; } } public override Int32 PageSize { get { return 100; } } - public Nyaa(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger) - : base(httpClient, configService, parsingService, logger) + public Nyaa(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) + : base(httpClient, indexerStatusService, configService, parsingService, logger) { } diff --git a/src/NzbDrone.Core/Indexers/Nyaa/NyaaRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Nyaa/NyaaRequestGenerator.cs index cb288955a..c794cfd33 100644 --- a/src/NzbDrone.Core/Indexers/Nyaa/NyaaRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Nyaa/NyaaRequestGenerator.cs @@ -23,7 +23,7 @@ namespace NzbDrone.Core.Indexers.Nyaa { var pageableRequests = new List>(); - pageableRequests.AddIfNotNull(GetPagedRequests(1, null)); + pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, null)); return pageableRequests; } diff --git a/src/NzbDrone.Core/Indexers/Omgwtfnzbs/Omgwtfnzbs.cs b/src/NzbDrone.Core/Indexers/Omgwtfnzbs/Omgwtfnzbs.cs index 2cccb026f..473802994 100644 --- a/src/NzbDrone.Core/Indexers/Omgwtfnzbs/Omgwtfnzbs.cs +++ b/src/NzbDrone.Core/Indexers/Omgwtfnzbs/Omgwtfnzbs.cs @@ -17,8 +17,8 @@ namespace NzbDrone.Core.Indexers.Omgwtfnzbs public override DownloadProtocol Protocol { get { return DownloadProtocol.Usenet; } } - public Omgwtfnzbs(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger) - : base(httpClient, configService, parsingService, logger) + public Omgwtfnzbs(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) + : base(httpClient, indexerStatusService, configService, parsingService, logger) { } diff --git a/src/NzbDrone.Core/Indexers/Rarbg/Rarbg.cs b/src/NzbDrone.Core/Indexers/Rarbg/Rarbg.cs index f84cf36e2..a24246b5e 100644 --- a/src/NzbDrone.Core/Indexers/Rarbg/Rarbg.cs +++ b/src/NzbDrone.Core/Indexers/Rarbg/Rarbg.cs @@ -15,8 +15,8 @@ namespace NzbDrone.Core.Indexers.Rarbg public override DownloadProtocol Protocol { get { return DownloadProtocol.Torrent; } } public override TimeSpan RateLimit { get { return TimeSpan.FromSeconds(2); } } - public Rarbg(IRarbgTokenProvider tokenProvider, IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger) - : base(httpClient, configService, parsingService, logger) + public Rarbg(IRarbgTokenProvider tokenProvider, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) + : base(httpClient, indexerStatusService, configService, parsingService, logger) { _tokenProvider = tokenProvider; } diff --git a/src/NzbDrone.Core/Indexers/RssSyncService.cs b/src/NzbDrone.Core/Indexers/RssSyncService.cs index 3313ebf00..b328d19b8 100644 --- a/src/NzbDrone.Core/Indexers/RssSyncService.cs +++ b/src/NzbDrone.Core/Indexers/RssSyncService.cs @@ -9,31 +9,35 @@ using NzbDrone.Core.Download.Pending; using NzbDrone.Core.IndexerSearch; using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.Indexers { public class RssSyncService : IExecute { + private readonly IIndexerStatusService _indexerStatusService; + private readonly IIndexerFactory _indexerFactory; private readonly IFetchAndParseRss _rssFetcherAndParser; private readonly IMakeDownloadDecision _downloadDecisionMaker; private readonly IProcessDownloadDecisions _processDownloadDecisions; - private readonly IEpisodeSearchService _episodeSearchService; private readonly IPendingReleaseService _pendingReleaseService; private readonly IEventAggregator _eventAggregator; private readonly Logger _logger; - public RssSyncService(IFetchAndParseRss rssFetcherAndParser, + public RssSyncService(IIndexerStatusService indexerStatusService, + IIndexerFactory indexerFactory, + IFetchAndParseRss rssFetcherAndParser, IMakeDownloadDecision downloadDecisionMaker, IProcessDownloadDecisions processDownloadDecisions, - IEpisodeSearchService episodeSearchService, IPendingReleaseService pendingReleaseService, IEventAggregator eventAggregator, Logger logger) { + _indexerStatusService = indexerStatusService; + _indexerFactory = indexerFactory; _rssFetcherAndParser = rssFetcherAndParser; _downloadDecisionMaker = downloadDecisionMaker; _processDownloadDecisions = processDownloadDecisions; - _episodeSearchService = episodeSearchService; _pendingReleaseService = pendingReleaseService; _eventAggregator = eventAggregator; _logger = logger; @@ -44,7 +48,10 @@ namespace NzbDrone.Core.Indexers { _logger.ProgressInfo("Starting RSS Sync"); - var reports = _rssFetcherAndParser.Fetch().Concat(_pendingReleaseService.GetPending()).ToList(); + var rssReleases = _rssFetcherAndParser.Fetch(); + var pendingReleases = _pendingReleaseService.GetPending(); + + var reports = rssReleases.Concat(pendingReleases).ToList(); var decisions = _downloadDecisionMaker.GetRssDecision(reports); var processed = _processDownloadDecisions.ProcessDecisions(decisions); @@ -65,12 +72,6 @@ namespace NzbDrone.Core.Indexers var processed = Sync(); var grabbedOrPending = processed.Grabbed.Concat(processed.Pending).ToList(); - if (message.LastExecutionTime.HasValue && DateTime.UtcNow.Subtract(message.LastExecutionTime.Value).TotalHours > 3) - { - _logger.Info("RSS Sync hasn't run since: {0}. Searching for any missing episodes since then.", message.LastExecutionTime.Value); - _episodeSearchService.MissingEpisodesAiredAfter(message.LastExecutionTime.Value.AddDays(-1), grabbedOrPending.SelectMany(d => d.RemoteEpisode.Episodes).Select(e => e.Id)); - } - _eventAggregator.PublishEvent(new RssSyncCompleteEvent(processed)); } } diff --git a/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssIndexer.cs b/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssIndexer.cs index be97c2b7c..b8d96e9f7 100644 --- a/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssIndexer.cs +++ b/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssIndexer.cs @@ -22,8 +22,8 @@ namespace NzbDrone.Core.Indexers.TorrentRss private readonly ITorrentRssParserFactory _torrentRssParserFactory; - public TorrentRssIndexer(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, ITorrentRssParserFactory torrentRssParserFactory, Logger logger) - : base(httpClient, configService, parsingService, logger) + public TorrentRssIndexer(ITorrentRssParserFactory torrentRssParserFactory, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) + : base(httpClient, indexerStatusService, configService, parsingService, logger) { _torrentRssParserFactory = torrentRssParserFactory; } diff --git a/src/NzbDrone.Core/Indexers/Torrentleech/Torrentleech.cs b/src/NzbDrone.Core/Indexers/Torrentleech/Torrentleech.cs index 59e5da6b2..e6f6bea9f 100644 --- a/src/NzbDrone.Core/Indexers/Torrentleech/Torrentleech.cs +++ b/src/NzbDrone.Core/Indexers/Torrentleech/Torrentleech.cs @@ -20,8 +20,8 @@ namespace NzbDrone.Core.Indexers.Torrentleech public override Boolean SupportsSearch { get { return false; } } public override Int32 PageSize { get { return 0; } } - public Torrentleech(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger) - : base(httpClient, configService, parsingService, logger) + public Torrentleech(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) + : base(httpClient, indexerStatusService, configService, parsingService, logger) { } diff --git a/src/NzbDrone.Core/Indexers/Torznab/Torznab.cs b/src/NzbDrone.Core/Indexers/Torznab/Torznab.cs index 0ea1b1c87..98d0c0f33 100644 --- a/src/NzbDrone.Core/Indexers/Torznab/Torznab.cs +++ b/src/NzbDrone.Core/Indexers/Torznab/Torznab.cs @@ -48,8 +48,8 @@ namespace NzbDrone.Core.Indexers.Torznab } } - public Torznab(ITorznabCapabilitiesProvider torznabCapabilitiesProvider, IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger) - : base(httpClient, configService, parsingService, logger) + public Torznab(ITorznabCapabilitiesProvider torznabCapabilitiesProvider, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) + : base(httpClient, indexerStatusService, configService, parsingService, logger) { _torznabCapabilitiesProvider = torznabCapabilitiesProvider; } diff --git a/src/NzbDrone.Core/Indexers/Torznab/TorznabRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Torznab/TorznabRequestGenerator.cs index abc36ca11..6d9315c12 100644 --- a/src/NzbDrone.Core/Indexers/Torznab/TorznabRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Torznab/TorznabRequestGenerator.cs @@ -70,8 +70,7 @@ namespace NzbDrone.Core.Indexers.Torznab if (capabilities.SupportedTvSearchParameters != null) { - // TODO: We might consider getting multiple pages in the future, but atm we limit it to 1 page. - pageableRequests.AddIfNotNull(GetPagedRequests(1, Settings.Categories.Concat(Settings.AnimeCategories), "tvsearch", "")); + pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories.Concat(Settings.AnimeCategories), "tvsearch", "")); } return pageableRequests; diff --git a/src/NzbDrone.Core/Indexers/Wombles/Wombles.cs b/src/NzbDrone.Core/Indexers/Wombles/Wombles.cs index 3fd1564df..dd33c38af 100644 --- a/src/NzbDrone.Core/Indexers/Wombles/Wombles.cs +++ b/src/NzbDrone.Core/Indexers/Wombles/Wombles.cs @@ -29,8 +29,8 @@ namespace NzbDrone.Core.Indexers.Wombles return new RssIndexerRequestGenerator("http://newshost.co.za/rss/?sec=TV&fr=false"); } - public Wombles(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger) - : base(httpClient, configService, parsingService, logger) + public Wombles(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) + : base(httpClient, indexerStatusService, configService, parsingService, logger) { } diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index a37a21353..d5159fbd7 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -235,37 +235,38 @@ + + + - - - - - - + + - - - + + + - + + + + + + - - - - - - + - + + + @@ -434,6 +435,7 @@ + @@ -453,6 +455,7 @@ + @@ -493,10 +496,13 @@ + + + @@ -890,6 +896,7 @@ + diff --git a/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs b/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs index a877bee08..35ab34d27 100644 --- a/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs +++ b/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs @@ -1,24 +1,24 @@ using System; +using System.Text; using NzbDrone.Core.Indexers; namespace NzbDrone.Core.Parser.Model { - using System.Text; - public class ReleaseInfo { - public String Guid { get; set; } - public String Title { get; set; } - public Int64 Size { get; set; } - public String DownloadUrl { get; set; } - public String InfoUrl { get; set; } - public String CommentUrl { get; set; } - public String Indexer { get; set; } + public string Guid { get; set; } + public string Title { get; set; } + public long Size { get; set; } + public string DownloadUrl { get; set; } + public string InfoUrl { get; set; } + public string CommentUrl { get; set; } + public int IndexerId { get; set; } + public string Indexer { get; set; } public DownloadProtocol DownloadProtocol { get; set; } - public Int32 TvRageId { get; set; } + public int TvRageId { get; set; } public DateTime PublishDate { get; set; } - public Int32 Age + public int Age { get { @@ -30,7 +30,7 @@ namespace NzbDrone.Core.Parser.Model private set { } } - public Double AgeHours + public double AgeHours { get { @@ -42,7 +42,7 @@ namespace NzbDrone.Core.Parser.Model private set { } } - public Double AgeMinutes + public double AgeMinutes { get { @@ -56,7 +56,7 @@ namespace NzbDrone.Core.Parser.Model public override string ToString() { - return String.Format("[{0}] {1} [{2}]", PublishDate, Title, Size); + return string.Format("[{0}] {1} [{2}]", PublishDate, Title, Size); } public virtual string ToString(string format) diff --git a/src/NzbDrone.Core/ThingiProvider/Events/ProviderDeletedEvent.cs b/src/NzbDrone.Core/ThingiProvider/Events/ProviderDeletedEvent.cs new file mode 100644 index 000000000..039006357 --- /dev/null +++ b/src/NzbDrone.Core/ThingiProvider/Events/ProviderDeletedEvent.cs @@ -0,0 +1,14 @@ +using NzbDrone.Common.Messaging; + +namespace NzbDrone.Core.ThingiProvider.Events +{ + public class ProviderDeletedEvent : IEvent + { + public int ProviderId { get; private set; } + + public ProviderDeletedEvent(int id) + { + ProviderId = id; + } + } +} diff --git a/src/NzbDrone.Core/ThingiProvider/Events/ProviderUpdatedEvent.cs b/src/NzbDrone.Core/ThingiProvider/Events/ProviderUpdatedEvent.cs index 9a8765cd0..66222ad52 100644 --- a/src/NzbDrone.Core/ThingiProvider/Events/ProviderUpdatedEvent.cs +++ b/src/NzbDrone.Core/ThingiProvider/Events/ProviderUpdatedEvent.cs @@ -4,5 +4,11 @@ namespace NzbDrone.Core.ThingiProvider.Events { public class ProviderUpdatedEvent : IEvent { + public ProviderDefinition Definition { get; private set; } + + public ProviderUpdatedEvent(ProviderDefinition definition) + { + Definition = definition; + } } -} +} \ No newline at end of file diff --git a/src/NzbDrone.Core/ThingiProvider/ProviderFactory.cs b/src/NzbDrone.Core/ThingiProvider/ProviderFactory.cs index 8f4776ea6..908d9f840 100644 --- a/src/NzbDrone.Core/ThingiProvider/ProviderFactory.cs +++ b/src/NzbDrone.Core/ThingiProvider/ProviderFactory.cs @@ -99,12 +99,13 @@ namespace NzbDrone.Core.ThingiProvider public virtual void Update(TProviderDefinition definition) { _providerRepository.Update(definition); - _eventAggregator.PublishEvent(new ProviderUpdatedEvent()); + _eventAggregator.PublishEvent(new ProviderUpdatedEvent(definition)); } public void Delete(int id) { _providerRepository.Delete(id); + _eventAggregator.PublishEvent(new ProviderDeletedEvent(id)); } public TProvider GetInstance(TProviderDefinition definition)