2017-10-29 03:58:16 +00:00
using System ;
2018-09-15 02:44:56 +00:00
using System.IO ;
2024-03-03 05:20:36 +00:00
using System.Linq ;
2015-10-10 20:20:17 +00:00
using System.Net ;
2021-11-13 21:10:42 +00:00
using System.Net.Http ;
2024-03-03 05:20:36 +00:00
using System.Net.NetworkInformation ;
2021-11-13 21:10:42 +00:00
using System.Net.Security ;
using System.Net.Sockets ;
using System.Text ;
using System.Threading ;
using System.Threading.Tasks ;
using NzbDrone.Common.Cache ;
2015-10-10 20:20:17 +00:00
using NzbDrone.Common.Extensions ;
2016-04-25 19:53:26 +00:00
using NzbDrone.Common.Http.Proxy ;
2015-10-10 20:20:17 +00:00
namespace NzbDrone.Common.Http.Dispatchers
{
public class ManagedHttpDispatcher : IHttpDispatcher
{
2021-11-13 21:10:42 +00:00
private const string NO_PROXY_KEY = "no-proxy" ;
private const int connection_establish_timeout = 2000 ;
private static bool useIPv6 = Socket . OSSupportsIPv6 ;
private static bool hasResolvedIPv6Availability ;
2016-04-24 14:20:45 +00:00
private readonly IHttpProxySettingsProvider _proxySettingsProvider ;
2016-04-25 19:53:26 +00:00
private readonly ICreateManagedWebProxy _createManagedWebProxy ;
2021-11-13 21:10:42 +00:00
private readonly ICertificateValidationService _certificateValidationService ;
2017-01-04 02:36:47 +00:00
private readonly IUserAgentBuilder _userAgentBuilder ;
2021-11-13 21:10:42 +00:00
private readonly ICached < System . Net . Http . HttpClient > _httpClientCache ;
private readonly ICached < CredentialCache > _credentialCache ;
2016-04-24 14:20:45 +00:00
2021-11-13 21:10:42 +00:00
public ManagedHttpDispatcher ( IHttpProxySettingsProvider proxySettingsProvider ,
2021-12-15 22:04:09 +00:00
ICreateManagedWebProxy createManagedWebProxy ,
ICertificateValidationService certificateValidationService ,
IUserAgentBuilder userAgentBuilder ,
ICacheManager cacheManager )
2016-04-24 14:20:45 +00:00
{
_proxySettingsProvider = proxySettingsProvider ;
2016-04-25 19:53:26 +00:00
_createManagedWebProxy = createManagedWebProxy ;
2021-11-13 21:10:42 +00:00
_certificateValidationService = certificateValidationService ;
2017-01-04 02:36:47 +00:00
_userAgentBuilder = userAgentBuilder ;
2021-11-13 21:10:42 +00:00
_httpClientCache = cacheManager . GetCache < System . Net . Http . HttpClient > ( typeof ( ManagedHttpDispatcher ) ) ;
_credentialCache = cacheManager . GetCache < CredentialCache > ( typeof ( ManagedHttpDispatcher ) , "credentialcache" ) ;
2016-04-24 14:20:45 +00:00
}
2023-07-16 18:07:31 +00:00
public async Task < HttpResponse > GetResponseAsync ( HttpRequest request , CookieContainer cookies )
2015-10-10 20:20:17 +00:00
{
2023-07-16 18:08:03 +00:00
var requestMessage = new HttpRequestMessage ( request . Method , ( Uri ) request . Url )
{
Version = HttpVersion . Version20 ,
VersionPolicy = HttpVersionPolicy . RequestVersionOrLower
} ;
2021-11-13 21:10:42 +00:00
requestMessage . Headers . UserAgent . ParseAdd ( _userAgentBuilder . GetUserAgent ( request . UseSimplifiedUserAgent ) ) ;
requestMessage . Headers . ConnectionClose = ! request . ConnectionKeepAlive ;
2015-10-10 20:20:17 +00:00
2021-11-13 21:10:42 +00:00
var cookieHeader = cookies . GetCookieHeader ( ( Uri ) request . Url ) ;
if ( cookieHeader . IsNotNullOrWhiteSpace ( ) )
{
requestMessage . Headers . Add ( "Cookie" , cookieHeader ) ;
}
2015-10-10 20:20:17 +00:00
2021-11-13 21:10:42 +00:00
using var cts = new CancellationTokenSource ( ) ;
2016-02-28 15:41:22 +00:00
if ( request . RequestTimeout ! = TimeSpan . Zero )
{
2021-11-13 21:10:42 +00:00
cts . CancelAfter ( request . RequestTimeout ) ;
2016-02-28 15:41:22 +00:00
}
2021-11-13 21:10:42 +00:00
else
2015-10-10 20:20:17 +00:00
{
2021-11-13 21:10:42 +00:00
// The default for System.Net.Http.HttpClient
cts . CancelAfter ( TimeSpan . FromSeconds ( 100 ) ) ;
2015-10-10 20:20:17 +00:00
}
2021-11-13 21:10:42 +00:00
if ( request . Credentials ! = null )
2015-10-10 20:20:17 +00:00
{
2021-11-13 21:10:42 +00:00
if ( request . Credentials is BasicNetworkCredential bc )
{
// Manually set header to avoid initial challenge response
var authInfo = bc . UserName + ":" + bc . Password ;
authInfo = Convert . ToBase64String ( Encoding . GetEncoding ( "ISO-8859-1" ) . GetBytes ( authInfo ) ) ;
requestMessage . Headers . Add ( "Authorization" , "Basic " + authInfo ) ;
}
else if ( request . Credentials is NetworkCredential nc )
2017-08-03 14:49:54 +00:00
{
2021-11-13 21:10:42 +00:00
var creds = GetCredentialCache ( ) ;
foreach ( var authtype in new [ ] { "Basic" , "Digest" } )
2017-08-03 14:49:54 +00:00
{
2021-11-13 21:10:42 +00:00
creds . Remove ( ( Uri ) request . Url , authtype ) ;
creds . Add ( ( Uri ) request . Url , authtype , nc ) ;
2017-08-03 14:49:54 +00:00
}
}
2021-11-13 21:10:42 +00:00
}
2017-08-03 14:49:54 +00:00
2021-11-13 21:10:42 +00:00
if ( request . ContentData ! = null )
{
requestMessage . Content = new ByteArrayContent ( request . ContentData ) ;
2015-10-10 20:20:17 +00:00
}
2021-11-13 21:10:42 +00:00
if ( request . Headers ! = null )
2015-10-10 20:20:17 +00:00
{
2021-11-13 21:10:42 +00:00
AddRequestHeaders ( requestMessage , request . Headers ) ;
}
2015-10-10 20:20:17 +00:00
2021-11-13 21:10:42 +00:00
var httpClient = GetClient ( request . Url ) ;
2023-08-27 16:53:34 +00:00
try
2021-11-13 21:10:42 +00:00
{
2023-08-27 16:53:34 +00:00
using var responseMessage = await httpClient . SendAsync ( requestMessage , HttpCompletionOption . ResponseHeadersRead , cts . Token ) ;
2015-10-10 20:20:17 +00:00
{
2023-08-27 16:53:34 +00:00
byte [ ] data = null ;
try
2017-10-29 04:16:47 +00:00
{
2023-08-27 16:53:34 +00:00
if ( request . ResponseStream ! = null & & responseMessage . StatusCode = = HttpStatusCode . OK )
{
await responseMessage . Content . CopyToAsync ( request . ResponseStream , null , cts . Token ) ;
}
else
{
2023-08-30 19:11:27 +00:00
data = await responseMessage . Content . ReadAsByteArrayAsync ( cts . Token ) ;
2023-08-27 16:53:34 +00:00
}
2017-10-29 04:16:47 +00:00
}
2023-08-27 16:53:34 +00:00
catch ( Exception ex )
2017-10-29 04:16:47 +00:00
{
2023-08-27 16:53:34 +00:00
throw new WebException ( "Failed to read complete http response" , ex , WebExceptionStatus . ReceiveFailure , null ) ;
2017-10-29 04:16:47 +00:00
}
2015-10-10 20:20:17 +00:00
2023-08-27 16:53:34 +00:00
var headers = responseMessage . Headers . ToNameValueCollection ( ) ;
2021-11-13 21:10:42 +00:00
2023-08-27 16:53:34 +00:00
headers . Add ( responseMessage . Content . Headers . ToNameValueCollection ( ) ) ;
2021-11-13 21:10:42 +00:00
2023-08-27 16:53:34 +00:00
return new HttpResponse ( request , new HttpHeader ( headers ) , data , responseMessage . StatusCode , responseMessage . Version ) ;
}
}
catch ( OperationCanceledException ex ) when ( cts . IsCancellationRequested )
{
throw new WebException ( "Http request timed out" , ex . InnerException , WebExceptionStatus . Timeout , null ) ;
2021-12-15 22:04:09 +00:00
}
2015-10-10 20:20:17 +00:00
}
2021-11-13 21:10:42 +00:00
protected virtual System . Net . Http . HttpClient GetClient ( HttpUri uri )
2016-04-24 14:20:45 +00:00
{
2021-11-13 21:10:42 +00:00
var proxySettings = _proxySettingsProvider . GetProxySettings ( uri ) ;
2020-09-07 04:28:15 +00:00
2021-11-13 21:10:42 +00:00
var key = proxySettings ? . Key ? ? NO_PROXY_KEY ;
2020-09-07 04:28:15 +00:00
2021-11-13 21:10:42 +00:00
return _httpClientCache . Get ( key , ( ) = > CreateHttpClient ( proxySettings ) ) ;
2020-09-07 04:28:15 +00:00
}
2021-11-13 21:10:42 +00:00
protected virtual System . Net . Http . HttpClient CreateHttpClient ( HttpProxySettings proxySettings )
2020-09-07 04:28:15 +00:00
{
2021-11-13 21:10:42 +00:00
var handler = new SocketsHttpHandler ( )
{
AutomaticDecompression = DecompressionMethods . GZip | DecompressionMethods . Brotli ,
UseCookies = false , // sic - we don't want to use a shared cookie container
AllowAutoRedirect = false ,
Credentials = GetCredentialCache ( ) ,
PreAuthenticate = true ,
MaxConnectionsPerServer = 12 ,
ConnectCallback = onConnect ,
SslOptions = new SslClientAuthenticationOptions
{
RemoteCertificateValidationCallback = _certificateValidationService . ShouldByPassValidationError
}
} ;
2020-09-07 04:28:15 +00:00
2016-04-24 14:20:45 +00:00
if ( proxySettings ! = null )
{
2021-11-13 21:10:42 +00:00
handler . Proxy = _createManagedWebProxy . GetWebProxy ( proxySettings ) ;
2016-04-24 14:20:45 +00:00
}
2020-09-07 04:28:15 +00:00
2021-11-13 21:10:42 +00:00
var client = new System . Net . Http . HttpClient ( handler )
{
2023-07-16 18:08:03 +00:00
DefaultRequestVersion = HttpVersion . Version20 ,
DefaultVersionPolicy = HttpVersionPolicy . RequestVersionOrLower ,
2021-11-13 21:10:42 +00:00
Timeout = Timeout . InfiniteTimeSpan
} ;
return client ;
2016-04-24 14:20:45 +00:00
}
2016-09-13 20:57:07 +00:00
2021-11-13 21:10:42 +00:00
protected virtual void AddRequestHeaders ( HttpRequestMessage webRequest , HttpHeader headers )
2015-10-10 20:20:17 +00:00
{
foreach ( var header in headers )
{
switch ( header . Key )
{
case "Accept" :
2021-11-13 21:10:42 +00:00
webRequest . Headers . Accept . ParseAdd ( header . Value ) ;
2015-10-10 20:20:17 +00:00
break ;
case "Connection" :
2021-11-13 21:10:42 +00:00
webRequest . Headers . Connection . Clear ( ) ;
webRequest . Headers . Connection . Add ( header . Value ) ;
2015-10-10 20:20:17 +00:00
break ;
case "Content-Length" :
2021-11-13 21:10:42 +00:00
AddContentHeader ( webRequest , "Content-Length" , header . Value ) ;
2015-10-10 20:20:17 +00:00
break ;
case "Content-Type" :
2021-11-13 21:10:42 +00:00
AddContentHeader ( webRequest , "Content-Type" , header . Value ) ;
2015-10-10 20:20:17 +00:00
break ;
2023-02-04 04:39:41 +00:00
case "Content-Encoding" :
AddContentHeader ( webRequest , "Content-Encoding" , header . Value ) ;
break ;
2015-10-10 20:20:17 +00:00
case "Date" :
2021-11-13 21:10:42 +00:00
webRequest . Headers . Remove ( "Date" ) ;
webRequest . Headers . Date = HttpHeader . ParseDateTime ( header . Value ) ;
2015-10-10 20:20:17 +00:00
break ;
case "Expect" :
2021-11-13 21:10:42 +00:00
webRequest . Headers . Expect . ParseAdd ( header . Value ) ;
2015-10-10 20:20:17 +00:00
break ;
case "Host" :
2021-11-13 21:10:42 +00:00
webRequest . Headers . Host = header . Value ;
2015-10-10 20:20:17 +00:00
break ;
case "If-Modified-Since" :
2021-11-13 21:10:42 +00:00
webRequest . Headers . IfModifiedSince = HttpHeader . ParseDateTime ( header . Value ) ;
2015-10-10 20:20:17 +00:00
break ;
case "Range" :
throw new NotImplementedException ( ) ;
case "Referer" :
2021-11-13 21:10:42 +00:00
webRequest . Headers . Add ( "Referer" , header . Value ) ;
2015-10-10 20:20:17 +00:00
break ;
case "Transfer-Encoding" :
2021-11-13 21:10:42 +00:00
webRequest . Headers . TransferEncoding . ParseAdd ( header . Value ) ;
2015-10-10 20:20:17 +00:00
break ;
case "User-Agent" :
2017-03-30 03:49:38 +00:00
throw new NotSupportedException ( "User-Agent other than Lidarr not allowed." ) ;
2015-10-10 20:20:17 +00:00
case "Proxy-Connection" :
throw new NotImplementedException ( ) ;
default :
2016-02-28 15:41:22 +00:00
webRequest . Headers . Add ( header . Key , header . Value ) ;
2015-10-10 20:20:17 +00:00
break ;
}
}
}
2021-11-13 21:10:42 +00:00
2023-02-18 19:59:08 +00:00
private static void AddContentHeader ( HttpRequestMessage request , string header , string value )
2021-11-13 21:10:42 +00:00
{
var headers = request . Content ? . Headers ;
if ( headers = = null )
{
return ;
}
headers . Remove ( header ) ;
headers . Add ( header , value ) ;
}
private CredentialCache GetCredentialCache ( )
{
return _credentialCache . Get ( "credentialCache" , ( ) = > new CredentialCache ( ) ) ;
}
2024-03-03 05:20:36 +00:00
private static bool HasRoutableIPv4Address ( )
{
// Get all IPv4 addresses from all interfaces and return true if there are any with non-loopback addresses
var networkInterfaces = NetworkInterface . GetAllNetworkInterfaces ( ) ;
return networkInterfaces . Any ( ni = >
ni . OperationalStatus = = OperationalStatus . Up & &
ni . GetIPProperties ( ) . UnicastAddresses . Any ( ip = >
ip . Address . AddressFamily = = AddressFamily . InterNetwork & &
! IPAddress . IsLoopback ( ip . Address ) ) ) ;
}
2021-11-13 21:10:42 +00:00
private static async ValueTask < Stream > onConnect ( SocketsHttpConnectionContext context , CancellationToken cancellationToken )
{
// Until .NET supports an implementation of Happy Eyeballs (https://tools.ietf.org/html/rfc8305#section-2), let's make IPv4 fallback work in a simple way.
// This issue is being tracked at https://github.com/dotnet/runtime/issues/26177 and expected to be fixed in .NET 6.
if ( useIPv6 )
{
try
{
var localToken = cancellationToken ;
if ( ! hasResolvedIPv6Availability )
{
// to make things move fast, use a very low timeout for the initial ipv6 attempt.
var quickFailCts = new CancellationTokenSource ( connection_establish_timeout ) ;
var linkedTokenSource = CancellationTokenSource . CreateLinkedTokenSource ( cancellationToken , quickFailCts . Token ) ;
localToken = linkedTokenSource . Token ;
}
return await attemptConnection ( AddressFamily . InterNetworkV6 , context , localToken ) ;
}
catch
{
2024-03-03 05:20:36 +00:00
// Do not retry IPv6 if a routable IPv4 address is available, otherwise continue to attempt IPv6 connections.
useIPv6 = ! HasRoutableIPv4Address ( ) ;
2021-11-13 21:10:42 +00:00
}
finally
{
hasResolvedIPv6Availability = true ;
}
}
// fallback to IPv4.
return await attemptConnection ( AddressFamily . InterNetwork , context , cancellationToken ) ;
}
private static async ValueTask < Stream > attemptConnection ( AddressFamily addressFamily , SocketsHttpConnectionContext context , CancellationToken cancellationToken )
{
// The following socket constructor will create a dual-mode socket on systems where IPV6 is available.
var socket = new Socket ( addressFamily , SocketType . Stream , ProtocolType . Tcp )
{
// Turn off Nagle's algorithm since it degrades performance in most HttpClient scenarios.
NoDelay = true
} ;
try
{
await socket . ConnectAsync ( context . DnsEndPoint , cancellationToken ) . ConfigureAwait ( false ) ;
// The stream should take the ownership of the underlying socket,
// closing it when it's disposed.
return new NetworkStream ( socket , ownsSocket : true ) ;
}
catch
{
socket . Dispose ( ) ;
throw ;
}
}
2015-10-10 20:20:17 +00:00
}
}