Lidarr/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs

327 lines
14 KiB
C#
Raw Normal View History

using System;
using System.IO;
using System.Linq;
2015-10-10 20:20:17 +00:00
using System.Net;
using System.Net.Http;
using System.Net.NetworkInformation;
using System.Net.Security;
using System.Net.Sockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using NzbDrone.Common.Cache;
2015-10-10 20:20:17 +00:00
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http.Proxy;
2015-10-10 20:20:17 +00:00
namespace NzbDrone.Common.Http.Dispatchers
{
public class ManagedHttpDispatcher : IHttpDispatcher
{
private const string NO_PROXY_KEY = "no-proxy";
private const int connection_establish_timeout = 2000;
private static bool useIPv6 = Socket.OSSupportsIPv6;
private static bool hasResolvedIPv6Availability;
private readonly IHttpProxySettingsProvider _proxySettingsProvider;
private readonly ICreateManagedWebProxy _createManagedWebProxy;
private readonly ICertificateValidationService _certificateValidationService;
private readonly IUserAgentBuilder _userAgentBuilder;
private readonly ICached<System.Net.Http.HttpClient> _httpClientCache;
private readonly ICached<CredentialCache> _credentialCache;
public ManagedHttpDispatcher(IHttpProxySettingsProvider proxySettingsProvider,
ICreateManagedWebProxy createManagedWebProxy,
ICertificateValidationService certificateValidationService,
IUserAgentBuilder userAgentBuilder,
ICacheManager cacheManager)
{
_proxySettingsProvider = proxySettingsProvider;
_createManagedWebProxy = createManagedWebProxy;
_certificateValidationService = certificateValidationService;
_userAgentBuilder = userAgentBuilder;
_httpClientCache = cacheManager.GetCache<System.Net.Http.HttpClient>(typeof(ManagedHttpDispatcher));
_credentialCache = cacheManager.GetCache<CredentialCache>(typeof(ManagedHttpDispatcher), "credentialcache");
}
2023-07-16 18:07:31 +00:00
public async Task<HttpResponse> GetResponseAsync(HttpRequest request, CookieContainer cookies)
2015-10-10 20:20:17 +00:00
{
2023-07-16 18:08:03 +00:00
var requestMessage = new HttpRequestMessage(request.Method, (Uri)request.Url)
{
Version = HttpVersion.Version20,
VersionPolicy = HttpVersionPolicy.RequestVersionOrLower
};
requestMessage.Headers.UserAgent.ParseAdd(_userAgentBuilder.GetUserAgent(request.UseSimplifiedUserAgent));
requestMessage.Headers.ConnectionClose = !request.ConnectionKeepAlive;
2015-10-10 20:20:17 +00:00
var cookieHeader = cookies.GetCookieHeader((Uri)request.Url);
if (cookieHeader.IsNotNullOrWhiteSpace())
{
requestMessage.Headers.Add("Cookie", cookieHeader);
}
2015-10-10 20:20:17 +00:00
using var cts = new CancellationTokenSource();
if (request.RequestTimeout != TimeSpan.Zero)
{
cts.CancelAfter(request.RequestTimeout);
}
else
2015-10-10 20:20:17 +00:00
{
// The default for System.Net.Http.HttpClient
cts.CancelAfter(TimeSpan.FromSeconds(100));
2015-10-10 20:20:17 +00:00
}
if (request.Credentials != null)
2015-10-10 20:20:17 +00:00
{
if (request.Credentials is BasicNetworkCredential bc)
{
// Manually set header to avoid initial challenge response
var authInfo = bc.UserName + ":" + bc.Password;
authInfo = Convert.ToBase64String(Encoding.GetEncoding("ISO-8859-1").GetBytes(authInfo));
requestMessage.Headers.Add("Authorization", "Basic " + authInfo);
}
else if (request.Credentials is NetworkCredential nc)
2017-08-03 14:49:54 +00:00
{
var creds = GetCredentialCache();
foreach (var authtype in new[] { "Basic", "Digest" })
2017-08-03 14:49:54 +00:00
{
creds.Remove((Uri)request.Url, authtype);
creds.Add((Uri)request.Url, authtype, nc);
2017-08-03 14:49:54 +00:00
}
}
}
2017-08-03 14:49:54 +00:00
if (request.ContentData != null)
{
requestMessage.Content = new ByteArrayContent(request.ContentData);
2015-10-10 20:20:17 +00:00
}
if (request.Headers != null)
2015-10-10 20:20:17 +00:00
{
AddRequestHeaders(requestMessage, request.Headers);
}
2015-10-10 20:20:17 +00:00
var httpClient = GetClient(request.Url);
try
{
using var responseMessage = await httpClient.SendAsync(requestMessage, HttpCompletionOption.ResponseHeadersRead, cts.Token);
2015-10-10 20:20:17 +00:00
{
byte[] data = null;
try
2017-10-29 04:16:47 +00:00
{
if (request.ResponseStream != null && responseMessage.StatusCode == HttpStatusCode.OK)
{
await responseMessage.Content.CopyToAsync(request.ResponseStream, null, cts.Token);
}
else
{
data = await responseMessage.Content.ReadAsByteArrayAsync(cts.Token);
}
2017-10-29 04:16:47 +00:00
}
catch (Exception ex)
2017-10-29 04:16:47 +00:00
{
throw new WebException("Failed to read complete http response", ex, WebExceptionStatus.ReceiveFailure, null);
2017-10-29 04:16:47 +00:00
}
2015-10-10 20:20:17 +00:00
var headers = responseMessage.Headers.ToNameValueCollection();
headers.Add(responseMessage.Content.Headers.ToNameValueCollection());
return new HttpResponse(request, new HttpHeader(headers), data, responseMessage.StatusCode, responseMessage.Version);
}
}
catch (OperationCanceledException ex) when (cts.IsCancellationRequested)
{
throw new WebException("Http request timed out", ex.InnerException, WebExceptionStatus.Timeout, null);
}
2015-10-10 20:20:17 +00:00
}
protected virtual System.Net.Http.HttpClient GetClient(HttpUri uri)
{
var proxySettings = _proxySettingsProvider.GetProxySettings(uri);
var key = proxySettings?.Key ?? NO_PROXY_KEY;
return _httpClientCache.Get(key, () => CreateHttpClient(proxySettings));
}
protected virtual System.Net.Http.HttpClient CreateHttpClient(HttpProxySettings proxySettings)
{
var handler = new SocketsHttpHandler()
{
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Brotli,
UseCookies = false, // sic - we don't want to use a shared cookie container
AllowAutoRedirect = false,
Credentials = GetCredentialCache(),
PreAuthenticate = true,
MaxConnectionsPerServer = 12,
ConnectCallback = onConnect,
SslOptions = new SslClientAuthenticationOptions
{
RemoteCertificateValidationCallback = _certificateValidationService.ShouldByPassValidationError
}
};
if (proxySettings != null)
{
handler.Proxy = _createManagedWebProxy.GetWebProxy(proxySettings);
}
var client = new System.Net.Http.HttpClient(handler)
{
2023-07-16 18:08:03 +00:00
DefaultRequestVersion = HttpVersion.Version20,
DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrLower,
Timeout = Timeout.InfiniteTimeSpan
};
return client;
}
protected virtual void AddRequestHeaders(HttpRequestMessage webRequest, HttpHeader headers)
2015-10-10 20:20:17 +00:00
{
foreach (var header in headers)
{
switch (header.Key)
{
case "Accept":
webRequest.Headers.Accept.ParseAdd(header.Value);
2015-10-10 20:20:17 +00:00
break;
case "Connection":
webRequest.Headers.Connection.Clear();
webRequest.Headers.Connection.Add(header.Value);
2015-10-10 20:20:17 +00:00
break;
case "Content-Length":
AddContentHeader(webRequest, "Content-Length", header.Value);
2015-10-10 20:20:17 +00:00
break;
case "Content-Type":
AddContentHeader(webRequest, "Content-Type", header.Value);
2015-10-10 20:20:17 +00:00
break;
case "Content-Encoding":
AddContentHeader(webRequest, "Content-Encoding", header.Value);
break;
2015-10-10 20:20:17 +00:00
case "Date":
webRequest.Headers.Remove("Date");
webRequest.Headers.Date = HttpHeader.ParseDateTime(header.Value);
2015-10-10 20:20:17 +00:00
break;
case "Expect":
webRequest.Headers.Expect.ParseAdd(header.Value);
2015-10-10 20:20:17 +00:00
break;
case "Host":
webRequest.Headers.Host = header.Value;
2015-10-10 20:20:17 +00:00
break;
case "If-Modified-Since":
webRequest.Headers.IfModifiedSince = HttpHeader.ParseDateTime(header.Value);
2015-10-10 20:20:17 +00:00
break;
case "Range":
throw new NotImplementedException();
case "Referer":
webRequest.Headers.Add("Referer", header.Value);
2015-10-10 20:20:17 +00:00
break;
case "Transfer-Encoding":
webRequest.Headers.TransferEncoding.ParseAdd(header.Value);
2015-10-10 20:20:17 +00:00
break;
case "User-Agent":
2017-03-30 03:49:38 +00:00
throw new NotSupportedException("User-Agent other than Lidarr not allowed.");
2015-10-10 20:20:17 +00:00
case "Proxy-Connection":
throw new NotImplementedException();
default:
webRequest.Headers.Add(header.Key, header.Value);
2015-10-10 20:20:17 +00:00
break;
}
}
}
private static void AddContentHeader(HttpRequestMessage request, string header, string value)
{
var headers = request.Content?.Headers;
if (headers == null)
{
return;
}
headers.Remove(header);
headers.Add(header, value);
}
private CredentialCache GetCredentialCache()
{
return _credentialCache.Get("credentialCache", () => new CredentialCache());
}
private static bool HasRoutableIPv4Address()
{
// Get all IPv4 addresses from all interfaces and return true if there are any with non-loopback addresses
var networkInterfaces = NetworkInterface.GetAllNetworkInterfaces();
return networkInterfaces.Any(ni =>
ni.OperationalStatus == OperationalStatus.Up &&
ni.GetIPProperties().UnicastAddresses.Any(ip =>
ip.Address.AddressFamily == AddressFamily.InterNetwork &&
!IPAddress.IsLoopback(ip.Address)));
}
private static async ValueTask<Stream> onConnect(SocketsHttpConnectionContext context, CancellationToken cancellationToken)
{
// Until .NET supports an implementation of Happy Eyeballs (https://tools.ietf.org/html/rfc8305#section-2), let's make IPv4 fallback work in a simple way.
// This issue is being tracked at https://github.com/dotnet/runtime/issues/26177 and expected to be fixed in .NET 6.
if (useIPv6)
{
try
{
var localToken = cancellationToken;
if (!hasResolvedIPv6Availability)
{
// to make things move fast, use a very low timeout for the initial ipv6 attempt.
var quickFailCts = new CancellationTokenSource(connection_establish_timeout);
var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, quickFailCts.Token);
localToken = linkedTokenSource.Token;
}
return await attemptConnection(AddressFamily.InterNetworkV6, context, localToken);
}
catch
{
// Do not retry IPv6 if a routable IPv4 address is available, otherwise continue to attempt IPv6 connections.
useIPv6 = !HasRoutableIPv4Address();
}
finally
{
hasResolvedIPv6Availability = true;
}
}
// fallback to IPv4.
return await attemptConnection(AddressFamily.InterNetwork, context, cancellationToken);
}
private static async ValueTask<Stream> attemptConnection(AddressFamily addressFamily, SocketsHttpConnectionContext context, CancellationToken cancellationToken)
{
// The following socket constructor will create a dual-mode socket on systems where IPV6 is available.
var socket = new Socket(addressFamily, SocketType.Stream, ProtocolType.Tcp)
{
// Turn off Nagle's algorithm since it degrades performance in most HttpClient scenarios.
NoDelay = true
};
try
{
await socket.ConnectAsync(context.DnsEndPoint, cancellationToken).ConfigureAwait(false);
// The stream should take the ownership of the underlying socket,
// closing it when it's disposed.
return new NetworkStream(socket, ownsSocket: true);
}
catch
{
socket.Dispose();
throw;
}
}
2015-10-10 20:20:17 +00:00
}
}