New: Updated Rarbg request limits

Closes #5206
This commit is contained in:
Mark McDowall 2023-04-13 19:54:37 -07:00 committed by GitHub
parent ed3d880974
commit 47cf8e6430
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 98 additions and 9 deletions

View File

@ -20,6 +20,7 @@ namespace NzbDrone.Common.Http
public Dictionary<string, string> Segments { get; private set; } public Dictionary<string, string> Segments { get; private set; }
public HttpHeader Headers { get; private set; } public HttpHeader Headers { get; private set; }
public bool SuppressHttpError { get; set; } public bool SuppressHttpError { get; set; }
public IEnumerable<HttpStatusCode> SuppressHttpErrorStatusCodes { get; set; }
public bool LogHttpError { get; set; } public bool LogHttpError { get; set; }
public bool UseSimplifiedUserAgent { get; set; } public bool UseSimplifiedUserAgent { get; set; }
public bool AllowAutoRedirect { get; set; } public bool AllowAutoRedirect { get; set; }
@ -102,6 +103,7 @@ namespace NzbDrone.Common.Http
{ {
request.Method = Method; request.Method = Method;
request.SuppressHttpError = SuppressHttpError; request.SuppressHttpError = SuppressHttpError;
request.SuppressHttpErrorStatusCodes = SuppressHttpErrorStatusCodes;
request.LogHttpError = LogHttpError; request.LogHttpError = LogHttpError;
request.UseSimplifiedUserAgent = UseSimplifiedUserAgent; request.UseSimplifiedUserAgent = UseSimplifiedUserAgent;
request.AllowAutoRedirect = AllowAutoRedirect; request.AllowAutoRedirect = AllowAutoRedirect;

View File

@ -1,4 +1,4 @@
using System; using System;
namespace NzbDrone.Common.Http namespace NzbDrone.Common.Http
{ {

View File

@ -1,5 +1,6 @@
using System; using System;
using System.Linq; using System.Linq;
using System.Net;
using System.Net.Http; using System.Net.Http;
using FluentAssertions; using FluentAssertions;
using Moq; using Moq;
@ -86,5 +87,57 @@ namespace NzbDrone.Core.Test.IndexerTests.RarbgTests
ExceptionVerification.ExpectedWarns(1); ExceptionVerification.ExpectedWarns(1);
} }
[Test]
public void should_warn_and_record_failure_on_429_response()
{
Mocker.GetMock<IHttpClient>()
.Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.Get)))
.Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), "", HttpStatusCode.TooManyRequests));
var releases = Subject.FetchRecent();
releases.Should().HaveCount(0);
ExceptionVerification.ExpectedWarns(1);
Mocker.GetMock<IIndexerStatusService>()
.Verify(v => v.RecordFailure(It.IsAny<int>(), It.Is<TimeSpan>(t => t == TimeSpan.FromMinutes(2))));
}
[Test]
public void should_warn_and_record_failure_on_520_response()
{
Mocker.GetMock<IHttpClient>()
.Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.Get)))
.Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), "", (HttpStatusCode)520));
var releases = Subject.FetchRecent();
releases.Should().HaveCount(0);
ExceptionVerification.ExpectedWarns(1);
Mocker.GetMock<IIndexerStatusService>()
.Verify(v => v.RecordFailure(It.IsAny<int>(), It.Is<TimeSpan>(t => t == TimeSpan.FromMinutes(3))));
}
// Uncomment when RarbgParser is updated
// [Test]
// public void should_warn_and_record_failure_on_200_response_with_rate_limit()
// {
// Mocker.GetMock<IHttpClient>()
// .Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.Get)))
// .Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), "{ rate_limit: 1 }"));
//
// var releases = Subject.FetchRecent();
//
// releases.Should().HaveCount(0);
//
// ExceptionVerification.ExpectedWarns(1);
//
// Mocker.GetMock<IIndexerStatusService>()
// .Verify(v => v.RecordFailure(It.IsAny<int>(), It.Is<TimeSpan>(t => t == TimeSpan.FromMinutes(5))));
// }
} }
} }

View File

@ -1,9 +1,12 @@
using NzbDrone.Common.Exceptions; using System;
using NzbDrone.Common.Exceptions;
namespace NzbDrone.Core.Indexers.Exceptions namespace NzbDrone.Core.Indexers.Exceptions
{ {
public class RequestLimitReachedException : NzbDroneException public class RequestLimitReachedException : NzbDroneException
{ {
public TimeSpan RetryAfter { get; private set; }
public RequestLimitReachedException(string message, params object[] args) public RequestLimitReachedException(string message, params object[] args)
: base(message, args) : base(message, args)
{ {
@ -13,5 +16,11 @@ namespace NzbDrone.Core.Indexers.Exceptions
: base(message) : base(message)
{ {
} }
public RequestLimitReachedException(string message, TimeSpan retryAfter)
: base(message)
{
RetryAfter = retryAfter;
}
} }
} }

View File

@ -1,4 +1,4 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Net; using System.Net;
@ -243,9 +243,17 @@ namespace NzbDrone.Core.Indexers
_indexerStatusService.RecordFailure(Definition.Id); _indexerStatusService.RecordFailure(Definition.Id);
_logger.Warn("{0} {1}", this, ex.Message); _logger.Warn("{0} {1}", this, ex.Message);
} }
catch (RequestLimitReachedException) catch (RequestLimitReachedException ex)
{ {
_indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1)); if (ex.RetryAfter != TimeSpan.Zero)
{
_indexerStatusService.RecordFailure(Definition.Id, ex.RetryAfter);
}
else
{
_indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
}
_logger.Warn("API Request Limit reached for {0}", this); _logger.Warn("API Request Limit reached for {0}", this);
} }
catch (ApiKeyException) catch (ApiKeyException)

View File

@ -1,5 +1,7 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq;
using System.Net;
using NLog; using NLog;
using NzbDrone.Common.Extensions; using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http; using NzbDrone.Common.Http;
@ -18,7 +20,7 @@ namespace NzbDrone.Core.Indexers.Rarbg
public override string Name => "Rarbg"; public override string Name => "Rarbg";
public override DownloadProtocol Protocol => DownloadProtocol.Torrent; public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override TimeSpan RateLimit => TimeSpan.FromSeconds(2); public override TimeSpan RateLimit => TimeSpan.FromSeconds(4);
public Rarbg(IRarbgTokenProvider tokenProvider, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) public Rarbg(IRarbgTokenProvider tokenProvider, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
: base(httpClient, indexerStatusService, configService, parsingService, logger) : base(httpClient, indexerStatusService, configService, parsingService, logger)

View File

@ -1,4 +1,5 @@
using System.Collections.Generic; using System;
using System.Collections.Generic;
using System.Net; using System.Net;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using NzbDrone.Common.Http; using NzbDrone.Common.Http;
@ -17,10 +18,14 @@ namespace NzbDrone.Core.Indexers.Rarbg
switch (indexerResponse.HttpResponse.StatusCode) switch (indexerResponse.HttpResponse.StatusCode)
{ {
case HttpStatusCode.TooManyRequests:
throw new RequestLimitReachedException("Indexer API limit reached", TimeSpan.FromMinutes(2));
case (HttpStatusCode)520:
throw new RequestLimitReachedException("Indexer API error. Likely rate limited by origin server", TimeSpan.FromMinutes(3));
default: default:
if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK) if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
{ {
throw new IndexerException(indexerResponse, "Indexer API call returned an unexpected StatusCode [{0}]", indexerResponse.HttpResponse.StatusCode); throw new IndexerException(indexerResponse, "Indexer API call returned an unexpected status code [{0}]", indexerResponse.HttpResponse.StatusCode);
} }
break; break;
@ -44,6 +49,12 @@ namespace NzbDrone.Core.Indexers.Rarbg
if (jsonResponse.Resource.torrent_results == null) if (jsonResponse.Resource.torrent_results == null)
{ {
// Despite this being the requested behaviour it appears to be problematic, commenting it out for now
// if (jsonResponse.Resource.rate_limit == 1)
// {
// throw new RequestLimitReachedException("Indexer API limit reached", TimeSpan.FromMinutes(5));
// }
return results; return results;
} }

View File

@ -1,4 +1,5 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Net;
using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Common.Extensions; using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http; using NzbDrone.Common.Http;
@ -88,6 +89,8 @@ namespace NzbDrone.Core.Indexers.Rarbg
.Resource("/pubapi_v2.php") .Resource("/pubapi_v2.php")
.Accept(HttpAccept.Json); .Accept(HttpAccept.Json);
requestBuilder.SuppressHttpErrorStatusCodes = new[] { HttpStatusCode.TooManyRequests, (HttpStatusCode)520 };
if (Settings.CaptchaToken.IsNotNullOrWhiteSpace()) if (Settings.CaptchaToken.IsNotNullOrWhiteSpace())
{ {
requestBuilder.UseSimplifiedUserAgent = true; requestBuilder.UseSimplifiedUserAgent = true;

View File

@ -7,6 +7,7 @@ namespace NzbDrone.Core.Indexers.Rarbg
{ {
public string error { get; set; } public string error { get; set; }
public int? error_code { get; set; } public int? error_code { get; set; }
public int? rate_limit { get; set; }
public List<RarbgTorrent> torrent_results { get; set; } public List<RarbgTorrent> torrent_results { get; set; }
} }