From 00283e3d6e75bb97b98e24af6b3afcf20266a5ff Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Sat, 12 Jan 2019 12:56:11 +0100 Subject: [PATCH] New: Limit indexer/download client backoff to 5 min during the first 15 min of application start. closes #2366 --- .../EnvironmentInfo/IRuntimeInfo.cs | 3 ++ .../EnvironmentInfo/RuntimeInfo.cs | 9 ++++ .../DownloadClientStatusServiceFixture.cs | 7 ++- .../IndexerStatusServiceFixture.cs | 7 ++- .../ProviderStatusServiceFixture.cs | 49 +++++++++++++++++-- .../Download/DownloadClientStatusService.cs | 7 +-- .../Indexers/IndexerStatusService.cs | 7 +-- .../Status/ProviderStatusServiceBase.cs | 18 ++++++- 8 files changed, 93 insertions(+), 14 deletions(-) diff --git a/src/NzbDrone.Common/EnvironmentInfo/IRuntimeInfo.cs b/src/NzbDrone.Common/EnvironmentInfo/IRuntimeInfo.cs index d387001ef..729b169e1 100644 --- a/src/NzbDrone.Common/EnvironmentInfo/IRuntimeInfo.cs +++ b/src/NzbDrone.Common/EnvironmentInfo/IRuntimeInfo.cs @@ -1,7 +1,10 @@ +using System; + namespace NzbDrone.Common.EnvironmentInfo { public interface IRuntimeInfo { + DateTime StartTime { get; } bool IsUserInteractive { get; } bool IsAdmin { get; } bool IsWindowsService { get; } diff --git a/src/NzbDrone.Common/EnvironmentInfo/RuntimeInfo.cs b/src/NzbDrone.Common/EnvironmentInfo/RuntimeInfo.cs index 3337b99b8..ad82d0cad 100644 --- a/src/NzbDrone.Common/EnvironmentInfo/RuntimeInfo.cs +++ b/src/NzbDrone.Common/EnvironmentInfo/RuntimeInfo.cs @@ -12,6 +12,7 @@ namespace NzbDrone.Common.EnvironmentInfo public class RuntimeInfo : IRuntimeInfo { private readonly Logger _logger; + private readonly DateTime _startTime = DateTime.UtcNow; public RuntimeInfo(IServiceProvider serviceProvider, Logger logger) { @@ -37,6 +38,14 @@ namespace NzbDrone.Common.EnvironmentInfo IsProduction = InternalIsProduction(); } + public DateTime StartTime + { + get + { + return _startTime; + } + } + public static bool IsUserInteractive => Environment.UserInteractive; bool IRuntimeInfo.IsUserInteractive => IsUserInteractive; diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs index 08aca1cdb..a608321d9 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs @@ -1,8 +1,9 @@ -using System; +using System; using System.Linq; using FluentAssertions; using Moq; using NUnit.Framework; +using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Core.Download; using NzbDrone.Core.Test.Framework; @@ -16,6 +17,10 @@ namespace NzbDrone.Core.Test.Download public void SetUp() { _epoch = DateTime.UtcNow; + + Mocker.GetMock() + .SetupGet(v => v.StartTime) + .Returns(_epoch - TimeSpan.FromHours(1)); } private DownloadClientStatus WithStatus(DownloadClientStatus status) diff --git a/src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs index 646212116..55ed8abfb 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs @@ -1,8 +1,9 @@ -using System; +using System; using System.Linq; using FluentAssertions; using Moq; using NUnit.Framework; +using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Core.Indexers; using NzbDrone.Core.Test.Framework; @@ -16,6 +17,10 @@ namespace NzbDrone.Core.Test.IndexerTests public void SetUp() { _epoch = DateTime.UtcNow; + + Mocker.GetMock() + .SetupGet(v => v.StartTime) + .Returns(_epoch - TimeSpan.FromHours(1)); } private void WithStatus(IndexerStatus status) diff --git a/src/NzbDrone.Core.Test/ThingiProviderTests/ProviderStatusServiceFixture.cs b/src/NzbDrone.Core.Test/ThingiProviderTests/ProviderStatusServiceFixture.cs index 32a9c4b7a..80d533ed2 100644 --- a/src/NzbDrone.Core.Test/ThingiProviderTests/ProviderStatusServiceFixture.cs +++ b/src/NzbDrone.Core.Test/ThingiProviderTests/ProviderStatusServiceFixture.cs @@ -1,9 +1,10 @@ -using System; +using System; using System.Linq; using FluentAssertions; using Moq; using NLog; using NUnit.Framework; +using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Test.Framework; using NzbDrone.Core.ThingiProvider; @@ -25,8 +26,8 @@ namespace NzbDrone.Core.Test.ThingiProviderTests public class MockProviderStatusService : ProviderStatusServiceBase { - public MockProviderStatusService(IMockProviderStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger) - : base(providerStatusRepository, eventAggregator, logger) + public MockProviderStatusService(IMockProviderStatusRepository providerStatusRepository, IEventAggregator eventAggregator, IRuntimeInfo runtimeInfo, Logger logger) + : base(providerStatusRepository, eventAggregator, runtimeInfo, logger) { } @@ -40,9 +41,20 @@ namespace NzbDrone.Core.Test.ThingiProviderTests public void SetUp() { _epoch = DateTime.UtcNow; + + Mocker.GetMock() + .SetupGet(v => v.StartTime) + .Returns(_epoch - TimeSpan.FromHours(1)); } - private void WithStatus(MockProviderStatus status) + private void GivenRecentStartup() + { + Mocker.GetMock() + .SetupGet(v => v.StartTime) + .Returns(_epoch - TimeSpan.FromMinutes(12)); + } + + private MockProviderStatus WithStatus(MockProviderStatus status) { Mocker.GetMock() .Setup(v => v.FindByProviderId(1)) @@ -51,6 +63,8 @@ namespace NzbDrone.Core.Test.ThingiProviderTests Mocker.GetMock() .Setup(v => v.All()) .Returns(new[] { status }); + + return status; } private void VerifyUpdate() @@ -122,5 +136,32 @@ namespace NzbDrone.Core.Test.ThingiProviderTests status.DisabledTill.Should().HaveValue(); status.DisabledTill.Value.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(15), 500); } + + [Test] + public void should_not_escalate_further_till_after_5_minutes_since_startup() + { + GivenRecentStartup(); + + var origStatus = WithStatus(new MockProviderStatus + { + InitialFailure = _epoch - TimeSpan.FromMinutes(6), + MostRecentFailure = _epoch - TimeSpan.FromSeconds(120), + EscalationLevel = 3 + }); + + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().NotBeNull(); + + origStatus.EscalationLevel.Should().Be(3); + status.DisabledTill.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(5), 500); + } } } diff --git a/src/NzbDrone.Core/Download/DownloadClientStatusService.cs b/src/NzbDrone.Core/Download/DownloadClientStatusService.cs index b4fd0f83c..11eecfe89 100644 --- a/src/NzbDrone.Core/Download/DownloadClientStatusService.cs +++ b/src/NzbDrone.Core/Download/DownloadClientStatusService.cs @@ -1,5 +1,6 @@ -using System; +using System; using NLog; +using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.ThingiProvider.Status; @@ -12,8 +13,8 @@ namespace NzbDrone.Core.Download public class DownloadClientStatusService : ProviderStatusServiceBase, IDownloadClientStatusService { - public DownloadClientStatusService(IDownloadClientStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger) - : base(providerStatusRepository, eventAggregator, logger) + public DownloadClientStatusService(IDownloadClientStatusRepository providerStatusRepository, IEventAggregator eventAggregator, IRuntimeInfo runtimeInfo, Logger logger) + : base(providerStatusRepository, eventAggregator, runtimeInfo, logger) { MinimumTimeSinceInitialFailure = TimeSpan.FromMinutes(5); MaximumEscalationLevel = 5; diff --git a/src/NzbDrone.Core/Indexers/IndexerStatusService.cs b/src/NzbDrone.Core/Indexers/IndexerStatusService.cs index 08baa51a5..1bdf81533 100644 --- a/src/NzbDrone.Core/Indexers/IndexerStatusService.cs +++ b/src/NzbDrone.Core/Indexers/IndexerStatusService.cs @@ -1,4 +1,5 @@ -using NLog; +using NLog; +using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.ThingiProvider.Status; @@ -14,8 +15,8 @@ namespace NzbDrone.Core.Indexers public class IndexerStatusService : ProviderStatusServiceBase, IIndexerStatusService { - public IndexerStatusService(IIndexerStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger) - : base(providerStatusRepository, eventAggregator, logger) + public IndexerStatusService(IIndexerStatusRepository providerStatusRepository, IEventAggregator eventAggregator, IRuntimeInfo runtimeInfo, Logger logger) + : base(providerStatusRepository, eventAggregator, runtimeInfo, logger) { } diff --git a/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs b/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs index 500559072..129617d3b 100644 --- a/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs +++ b/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Linq; using NLog; +using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.ThingiProvider.Events; @@ -24,15 +25,18 @@ namespace NzbDrone.Core.ThingiProvider.Status protected readonly IProviderStatusRepository _providerStatusRepository; protected readonly IEventAggregator _eventAggregator; + protected readonly IRuntimeInfo _runtimeInfo; protected readonly Logger _logger; protected int MaximumEscalationLevel { get; set; } = EscalationBackOff.Periods.Length - 1; protected TimeSpan MinimumTimeSinceInitialFailure { get; set; } = TimeSpan.Zero; + protected TimeSpan MinimumTimeSinceStartup { get; set; } = TimeSpan.FromMinutes(15); - public ProviderStatusServiceBase(IProviderStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger) + public ProviderStatusServiceBase(IProviderStatusRepository providerStatusRepository, IEventAggregator eventAggregator, IRuntimeInfo runtimeInfo, Logger logger) { _providerStatusRepository = providerStatusRepository; _eventAggregator = eventAggregator; + _runtimeInfo = runtimeInfo; _logger = logger; } @@ -89,9 +93,10 @@ namespace NzbDrone.Core.ThingiProvider.Status escalate = false; } + var inStartupGracePeriod = (_runtimeInfo.StartTime + MinimumTimeSinceStartup) > now; var inGracePeriod = (status.InitialFailure.Value + MinimumTimeSinceInitialFailure) > now; - if (escalate && !inGracePeriod) + if (escalate && !inGracePeriod && !inStartupGracePeriod) { status.EscalationLevel = Math.Min(MaximumEscalationLevel, status.EscalationLevel + 1); } @@ -109,6 +114,15 @@ namespace NzbDrone.Core.ThingiProvider.Status status.DisabledTill = now + CalculateBackOffPeriod(status); } + if (inStartupGracePeriod && minimumBackOff == TimeSpan.Zero && status.DisabledTill.HasValue) + { + var maximumDisabledTill = now + TimeSpan.FromSeconds(EscalationBackOff.Periods[1]); + if (maximumDisabledTill < status.DisabledTill) + { + status.DisabledTill = maximumDisabledTill; + } + } + _providerStatusRepository.Upsert(status); _eventAggregator.PublishEvent(new ProviderStatusChangedEvent(providerId, status));