mirror of https://github.com/lidarr/Lidarr
Generalized RateLimit logic to all indexers based on indexer id
Fixes #1982 Co-authored-by: Taloth Saldono <Taloth@users.noreply.github.com>
This commit is contained in:
parent
25f703e7b2
commit
0758a27d5b
|
@ -89,5 +89,38 @@ namespace NzbDrone.Common.Test.TPLTests
|
||||||
|
|
||||||
(GetRateLimitStore()["me"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(100));
|
(GetRateLimitStore()["me"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(100));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void should_extend_subkey_delay()
|
||||||
|
{
|
||||||
|
GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200));
|
||||||
|
GivenExisting("me-sub", _epoch + TimeSpan.FromMilliseconds(300));
|
||||||
|
|
||||||
|
Subject.WaitAndPulse("me", "sub", TimeSpan.FromMilliseconds(100));
|
||||||
|
|
||||||
|
(GetRateLimitStore()["me-sub"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(400));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void should_honor_basekey_delay()
|
||||||
|
{
|
||||||
|
GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200));
|
||||||
|
GivenExisting("me-sub", _epoch + TimeSpan.FromMilliseconds(0));
|
||||||
|
|
||||||
|
Subject.WaitAndPulse("me", "sub", TimeSpan.FromMilliseconds(100));
|
||||||
|
|
||||||
|
(GetRateLimitStore()["me-sub"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(200));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void should_not_extend_basekey_delay()
|
||||||
|
{
|
||||||
|
GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200));
|
||||||
|
GivenExisting("me-sub", _epoch + TimeSpan.FromMilliseconds(100));
|
||||||
|
|
||||||
|
Subject.WaitAndPulse("me", "sub", TimeSpan.FromMilliseconds(100));
|
||||||
|
|
||||||
|
(GetRateLimitStore()["me"] - _epoch).Should().BeCloseTo(TimeSpan.FromMilliseconds(200));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -112,7 +112,7 @@ namespace NzbDrone.Common.Http
|
||||||
|
|
||||||
if (request.RateLimit != TimeSpan.Zero)
|
if (request.RateLimit != TimeSpan.Zero)
|
||||||
{
|
{
|
||||||
_rateLimitService.WaitAndPulse(request.Url.Host, request.RateLimit);
|
_rateLimitService.WaitAndPulse(request.Url.Host, request.RateLimitKey, request.RateLimit);
|
||||||
}
|
}
|
||||||
|
|
||||||
_logger.Trace(request);
|
_logger.Trace(request);
|
||||||
|
|
|
@ -47,6 +47,7 @@ namespace NzbDrone.Common.Http
|
||||||
public bool StoreResponseCookie { get; set; }
|
public bool StoreResponseCookie { get; set; }
|
||||||
public TimeSpan RequestTimeout { get; set; }
|
public TimeSpan RequestTimeout { get; set; }
|
||||||
public TimeSpan RateLimit { get; set; }
|
public TimeSpan RateLimit { get; set; }
|
||||||
|
public string RateLimitKey { get; set; }
|
||||||
|
|
||||||
public override string ToString()
|
public override string ToString()
|
||||||
{
|
{
|
||||||
|
|
|
@ -2,12 +2,14 @@
|
||||||
using System.Collections.Concurrent;
|
using System.Collections.Concurrent;
|
||||||
using NLog;
|
using NLog;
|
||||||
using NzbDrone.Common.Cache;
|
using NzbDrone.Common.Cache;
|
||||||
|
using NzbDrone.Common.Extensions;
|
||||||
|
|
||||||
namespace NzbDrone.Common.TPL
|
namespace NzbDrone.Common.TPL
|
||||||
{
|
{
|
||||||
public interface IRateLimitService
|
public interface IRateLimitService
|
||||||
{
|
{
|
||||||
void WaitAndPulse(string key, TimeSpan interval);
|
void WaitAndPulse(string key, TimeSpan interval);
|
||||||
|
void WaitAndPulse(string key, string subKey, TimeSpan interval);
|
||||||
}
|
}
|
||||||
|
|
||||||
public class RateLimitService : IRateLimitService
|
public class RateLimitService : IRateLimitService
|
||||||
|
@ -23,9 +25,37 @@ namespace NzbDrone.Common.TPL
|
||||||
|
|
||||||
public void WaitAndPulse(string key, TimeSpan interval)
|
public void WaitAndPulse(string key, TimeSpan interval)
|
||||||
{
|
{
|
||||||
var waitUntil = _rateLimitStore.AddOrUpdate(key,
|
WaitAndPulse(key, null, interval);
|
||||||
(s) => DateTime.UtcNow + interval,
|
}
|
||||||
(s, i) => new DateTime(Math.Max(DateTime.UtcNow.Ticks, i.Ticks), DateTimeKind.Utc) + interval);
|
|
||||||
|
public void WaitAndPulse(string key, string subKey, TimeSpan interval)
|
||||||
|
{
|
||||||
|
var waitUntil = DateTime.UtcNow.Add(interval);
|
||||||
|
|
||||||
|
if (subKey.IsNotNullOrWhiteSpace())
|
||||||
|
{
|
||||||
|
// Expand the base key timer, but don't extend it beyond now+interval.
|
||||||
|
var baseUntil = _rateLimitStore.AddOrUpdate(key,
|
||||||
|
(s) => waitUntil,
|
||||||
|
(s, i) => new DateTime(Math.Max(waitUntil.Ticks, i.Ticks), DateTimeKind.Utc));
|
||||||
|
|
||||||
|
if (baseUntil > waitUntil)
|
||||||
|
{
|
||||||
|
waitUntil = baseUntil;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for the full key
|
||||||
|
var combinedKey = key + "-" + subKey;
|
||||||
|
waitUntil = _rateLimitStore.AddOrUpdate(combinedKey,
|
||||||
|
(s) => waitUntil,
|
||||||
|
(s, i) => new DateTime(Math.Max(waitUntil.Ticks, i.Add(interval).Ticks), DateTimeKind.Utc));
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
waitUntil = _rateLimitStore.AddOrUpdate(key,
|
||||||
|
(s) => waitUntil,
|
||||||
|
(s, i) => new DateTime(Math.Max(waitUntil.Ticks, i.Add(interval).Ticks), DateTimeKind.Utc));
|
||||||
|
}
|
||||||
|
|
||||||
waitUntil -= interval;
|
waitUntil -= interval;
|
||||||
|
|
||||||
|
|
|
@ -128,6 +128,7 @@ namespace NzbDrone.Core.Download
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var request = new HttpRequest(torrentUrl);
|
var request = new HttpRequest(torrentUrl);
|
||||||
|
request.RateLimitKey = remoteAlbum?.Release?.IndexerId.ToString();
|
||||||
request.Headers.Accept = "application/x-bittorrent";
|
request.Headers.Accept = "application/x-bittorrent";
|
||||||
request.AllowAutoRedirect = false;
|
request.AllowAutoRedirect = false;
|
||||||
|
|
||||||
|
|
|
@ -45,6 +45,7 @@ namespace NzbDrone.Core.Download
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var nzbDataRequest = new HttpRequest(url);
|
var nzbDataRequest = new HttpRequest(url);
|
||||||
|
nzbDataRequest.RateLimitKey = remoteAlbum?.Release?.IndexerId.ToString();
|
||||||
|
|
||||||
// TODO: Look into moving download request handling to indexer
|
// TODO: Look into moving download request handling to indexer
|
||||||
if (remoteAlbum.Release.BasicAuthString.IsNotNullOrWhiteSpace())
|
if (remoteAlbum.Release.BasicAuthString.IsNotNullOrWhiteSpace())
|
||||||
|
|
|
@ -280,6 +280,8 @@ namespace NzbDrone.Core.Indexers
|
||||||
request.HttpRequest.RateLimit = RateLimit;
|
request.HttpRequest.RateLimit = RateLimit;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
request.HttpRequest.RateLimitKey = Definition.Id.ToString();
|
||||||
|
|
||||||
return new IndexerResponse(request, _httpClient.Execute(request.HttpRequest));
|
return new IndexerResponse(request, _httpClient.Execute(request.HttpRequest));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue