2020-02-09 02:35:16 +00:00
using System ;
2017-04-15 08:45:10 +00:00
using System.Collections.Generic ;
2020-09-21 06:03:18 +00:00
using System.Diagnostics ;
2017-04-15 08:45:10 +00:00
using System.Linq ;
using System.Net ;
using System.Net.Http ;
2020-09-21 06:03:18 +00:00
using System.Net.Security ;
using System.Security.Cryptography.X509Certificates ;
2017-04-15 08:45:10 +00:00
using System.Text ;
2020-03-14 22:52:06 +00:00
using System.Text.RegularExpressions ;
2017-04-15 08:45:10 +00:00
using System.Threading.Tasks ;
2020-02-09 18:08:34 +00:00
using CloudflareSolverRe ;
2020-09-21 06:03:18 +00:00
using com.LandonKey.SocksWebProxy ;
using com.LandonKey.SocksWebProxy.Proxy ;
2020-02-09 18:08:34 +00:00
using Jackett.Common.Helpers ;
2017-11-06 10:51:26 +00:00
using Jackett.Common.Models.Config ;
2018-03-10 08:05:56 +00:00
using Jackett.Common.Services.Interfaces ;
using NLog ;
2017-04-15 08:45:10 +00:00
2018-03-10 08:05:56 +00:00
namespace Jackett.Common.Utils.Clients
2017-04-15 08:45:10 +00:00
{
// Compared to HttpWebClient this implementation will reuse the HttpClient instance (one per indexer).
// This should improve performance and avoid problems with too man open file handles.
2017-11-05 09:42:03 +00:00
public class HttpWebClient2 : WebClient
2017-04-15 08:45:10 +00:00
{
2020-02-10 22:16:19 +00:00
private readonly CookieContainer cookies ;
private ClearanceHandler clearanceHandlr ;
private HttpClientHandler clientHandlr ;
private HttpClient client ;
2020-09-21 06:03:18 +00:00
protected static Dictionary < string , ICollection < string > > trustedCertificates = new Dictionary < string , ICollection < string > > ( ) ;
protected static string webProxyUrl ;
protected static IWebProxy webProxy ;
[DebuggerNonUserCode] // avoid "Exception User-Unhandled" Visual Studio messages
2020-09-21 06:04:06 +00:00
public static bool ValidateCertificate ( object sender , X509Certificate certificate , X509Chain chain , SslPolicyErrors sslPolicyErrors )
2020-09-21 06:03:18 +00:00
{
2020-09-21 06:04:06 +00:00
if ( sender . GetType ( ) ! = typeof ( HttpWebRequest ) )
return sslPolicyErrors = = SslPolicyErrors . None ;
var request = ( HttpWebRequest ) sender ;
2020-09-21 06:03:18 +00:00
var hash = certificate . GetCertHashString ( ) ;
trustedCertificates . TryGetValue ( hash , out var hosts ) ;
if ( hosts ! = null )
{
2020-09-21 06:04:06 +00:00
if ( hosts . Contains ( request . Host ) )
2020-09-21 06:03:18 +00:00
return true ;
}
if ( sslPolicyErrors ! = SslPolicyErrors . None )
{
// Throw exception with certificate details, this will cause a "Exception User-Unhandled" when running it in the Visual Studio debugger.
// The certificate is only available inside this function, so we can't catch it at the calling method.
throw new Exception ( "certificate validation failed: " + certificate . ToString ( ) ) ;
}
return sslPolicyErrors = = SslPolicyErrors . None ;
}
public static void InitProxy ( ServerConfig serverConfig )
{
// dispose old SocksWebProxy
if ( webProxy is SocksWebProxy proxy )
proxy . Dispose ( ) ;
webProxy = null ;
webProxyUrl = serverConfig . GetProxyUrl ( ) ;
if ( ! string . IsNullOrWhiteSpace ( webProxyUrl ) )
{
if ( serverConfig . ProxyType ! = ProxyType . Http )
{
var addresses = Dns . GetHostAddressesAsync ( serverConfig . ProxyUrl ) . Result ;
var socksConfig = new ProxyConfig
{
SocksAddress = addresses . FirstOrDefault ( ) ,
Username = serverConfig . ProxyUsername ,
Password = serverConfig . ProxyPassword ,
Version = serverConfig . ProxyType = = ProxyType . Socks4 ?
ProxyConfig . SocksVersion . Four :
ProxyConfig . SocksVersion . Five
} ;
if ( serverConfig . ProxyPort . HasValue )
{
socksConfig . SocksPort = serverConfig . ProxyPort . Value ;
}
webProxy = new SocksWebProxy ( socksConfig , false ) ;
}
else
{
NetworkCredential creds = null ;
if ( ! serverConfig . ProxyIsAnonymous )
{
var username = serverConfig . ProxyUsername ;
var password = serverConfig . ProxyPassword ;
creds = new NetworkCredential ( username , password ) ;
}
webProxy = new WebProxy ( webProxyUrl )
{
BypassProxyOnLocal = false ,
Credentials = creds
} ;
}
}
}
2017-11-17 15:46:58 +00:00
public HttpWebClient2 ( IProcessService p , Logger l , IConfigurationService c , ServerConfig sc )
: base ( p : p ,
l : l ,
c : c ,
sc : sc )
{
2020-09-21 06:03:18 +00:00
if ( webProxyUrl = = null )
InitProxy ( sc ) ;
2017-11-17 15:46:58 +00:00
cookies = new CookieContainer ( ) ;
CreateClient ( ) ;
}
2017-04-15 08:45:10 +00:00
2017-11-17 15:46:58 +00:00
public void CreateClient ( )
{
2020-03-26 22:15:28 +00:00
clearanceHandlr = new ClearanceHandler ( BrowserUtil . ChromeUserAgent )
{
2020-05-12 22:04:47 +00:00
MaxTries = 10
2020-03-26 22:15:28 +00:00
} ;
2017-04-15 08:45:10 +00:00
clientHandlr = new HttpClientHandler
{
CookieContainer = cookies ,
AllowAutoRedirect = false , // Do not use this - Bugs ahoy! Lost cookies and more.
UseCookies = true ,
2017-11-17 15:46:58 +00:00
Proxy = webProxy ,
UseProxy = ( webProxy ! = null ) ,
2017-04-15 08:45:10 +00:00
AutomaticDecompression = DecompressionMethods . GZip | DecompressionMethods . Deflate
} ;
clearanceHandlr . InnerHandler = clientHandlr ;
client = new HttpClient ( clearanceHandlr ) ;
}
2017-11-17 15:46:58 +00:00
// Called everytime the ServerConfig changes
public override void OnNext ( ServerConfig value )
{
2020-09-21 06:03:18 +00:00
var newProxyUrl = serverConfig . GetProxyUrl ( ) ;
if ( webProxyUrl ! = newProxyUrl ) // if proxy URL changed
InitProxy ( serverConfig ) ;
2017-11-17 15:46:58 +00:00
// recreate client if needed (can't just change the proxy attribute)
if ( ! ReferenceEquals ( clientHandlr . Proxy , webProxy ) )
{
CreateClient ( ) ;
}
}
2020-02-10 22:16:19 +00:00
public override void Init ( )
2017-04-15 08:45:10 +00:00
{
2020-09-21 06:03:18 +00:00
if ( serverConfig . RuntimeSettings . IgnoreSslErrors = = true )
{
logger . Info ( string . Format ( "HttpWebClient2: Disabling certificate validation" ) ) ;
ServicePointManager . ServerCertificateValidationCallback + = ( sender , certificate , chain , sslPolicyErrors ) = > { return true ; } ;
}
2017-04-15 08:45:10 +00:00
ServicePointManager . SecurityProtocol = ( SecurityProtocolType ) 192 | ( SecurityProtocolType ) 768 | ( SecurityProtocolType ) 3072 ;
2020-09-21 06:04:06 +00:00
// custom handler for our own internal certificates
ServicePointManager . ServerCertificateValidationCallback + = ValidateCertificate ;
2017-04-15 08:45:10 +00:00
}
2020-09-21 06:02:58 +00:00
protected override async Task < WebClientByteResult > Run ( WebRequest webRequest )
2017-04-15 08:45:10 +00:00
{
HttpResponseMessage response = null ;
var request = new HttpRequestMessage ( ) ;
request . Headers . ExpectContinue = false ;
request . RequestUri = new Uri ( webRequest . Url ) ;
2017-08-11 14:52:58 +00:00
if ( webRequest . EmulateBrowser = = true )
2017-04-15 08:45:10 +00:00
request . Headers . UserAgent . ParseAdd ( BrowserUtil . ChromeUserAgent ) ;
else
request . Headers . UserAgent . ParseAdd ( "Jackett/" + configService . GetVersion ( ) ) ;
// clear cookies from cookiecontainer
var oldCookies = cookies . GetCookies ( request . RequestUri ) ;
foreach ( Cookie oldCookie in oldCookies )
oldCookie . Expired = true ;
2020-04-13 04:22:50 +00:00
// add cookies to cookiecontainer
if ( ! string . IsNullOrWhiteSpace ( webRequest . Cookies ) )
2017-04-15 08:45:10 +00:00
{
2020-04-13 04:22:50 +00:00
// don't include the path, Scheme is needed for mono compatibility
var cookieUrl = new Uri ( request . RequestUri . Scheme + "://" + request . RequestUri . Host ) ;
var cookieDictionary = CookieUtil . CookieHeaderToDictionary ( webRequest . Cookies ) ;
foreach ( var kv in cookieDictionary )
cookies . Add ( cookieUrl , new Cookie ( kv . Key , kv . Value ) ) ;
2017-04-15 08:45:10 +00:00
}
if ( webRequest . Headers ! = null )
{
foreach ( var header in webRequest . Headers )
{
if ( header . Key ! = "Content-Type" )
{
request . Headers . TryAddWithoutValidation ( header . Key , header . Value ) ;
}
}
}
if ( ! string . IsNullOrEmpty ( webRequest . Referer ) )
request . Headers . Referrer = new Uri ( webRequest . Referer ) ;
if ( ! string . IsNullOrEmpty ( webRequest . RawBody ) )
{
2017-11-06 10:51:26 +00:00
var type = webRequest . Headers . Where ( h = > h . Key = = "Content-Type" ) . Cast < KeyValuePair < string , string > ? > ( ) . FirstOrDefault ( ) ;
2017-04-15 08:45:10 +00:00
if ( type . HasValue )
{
var str = new StringContent ( webRequest . RawBody ) ;
str . Headers . Remove ( "Content-Type" ) ;
str . Headers . Add ( "Content-Type" , type . Value . Value ) ;
request . Content = str ;
}
else
request . Content = new StringContent ( webRequest . RawBody ) ;
request . Method = HttpMethod . Post ;
}
else if ( webRequest . Type = = RequestType . POST )
{
if ( webRequest . PostData ! = null )
2020-02-08 06:03:03 +00:00
request . Content = FormUrlEncodedContentWithEncoding ( webRequest . PostData , webRequest . Encoding ) ;
2017-04-15 08:45:10 +00:00
request . Method = HttpMethod . Post ;
}
else
{
request . Method = HttpMethod . Get ;
}
response = await client . SendAsync ( request ) ;
2020-09-21 06:02:58 +00:00
var result = new WebClientByteResult
2020-03-26 22:15:28 +00:00
{
2020-09-21 06:04:11 +00:00
Content = await response . Content . ReadAsByteArrayAsync ( )
2020-03-26 22:15:28 +00:00
} ;
2017-04-15 08:45:10 +00:00
foreach ( var header in response . Headers )
{
2020-02-10 22:16:19 +00:00
var value = header . Value ;
2017-04-15 08:45:10 +00:00
result . Headers [ header . Key . ToLowerInvariant ( ) ] = value . ToArray ( ) ;
}
// some cloudflare clients are using a refresh header
2020-04-13 04:22:50 +00:00
// Pull it out manually
2017-04-15 08:45:10 +00:00
if ( response . StatusCode = = System . Net . HttpStatusCode . ServiceUnavailable & & response . Headers . Contains ( "Refresh" ) )
{
var refreshHeaders = response . Headers . GetValues ( "Refresh" ) ;
var redirval = "" ;
var redirtime = 0 ;
if ( refreshHeaders ! = null )
{
foreach ( var value in refreshHeaders )
{
var start = value . IndexOf ( "=" ) ;
var end = value . IndexOf ( ";" ) ;
var len = value . Length ;
if ( start > - 1 )
{
redirval = value . Substring ( start + 1 ) ;
result . RedirectingTo = redirval ;
// normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
2020-09-21 06:02:58 +00:00
// of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
2017-04-15 08:45:10 +00:00
// it shoudln't include service unavailable..only if we have this redirect header.
response . StatusCode = System . Net . HttpStatusCode . Redirect ;
2020-02-10 22:16:19 +00:00
redirtime = int . Parse ( value . Substring ( 0 , end ) ) ;
2017-04-15 08:45:10 +00:00
System . Threading . Thread . Sleep ( redirtime * 1000 ) ;
}
}
}
}
if ( response . Headers . Location ! = null )
{
result . RedirectingTo = response . Headers . Location . ToString ( ) ;
}
2017-08-28 09:01:45 +00:00
// Mono won't add the baseurl to relative redirects.
// e.g. a "Location: /index.php" header will result in the Uri "file:///index.php"
// See issue #1200
if ( result . RedirectingTo ! = null & & result . RedirectingTo . StartsWith ( "file://" ) )
{
2018-06-14 15:28:57 +00:00
// URL decoding apparently is needed to, without it e.g. Demonoid download is broken
// TODO: is it always needed (not just for relative redirects)?
var newRedirectingTo = WebUtilityHelpers . UrlDecode ( result . RedirectingTo , webRequest . Encoding ) ;
2018-09-17 14:43:09 +00:00
if ( newRedirectingTo . StartsWith ( "file:////" ) ) // Location without protocol but with host (only add scheme)
newRedirectingTo = newRedirectingTo . Replace ( "file://" , request . RequestUri . Scheme + ":" ) ;
else
newRedirectingTo = newRedirectingTo . Replace ( "file://" , request . RequestUri . Scheme + "://" + request . RequestUri . Host ) ;
2017-08-28 09:01:45 +00:00
logger . Debug ( "[MONO relative redirect bug] Rewriting relative redirect URL from " + result . RedirectingTo + " to " + newRedirectingTo ) ;
result . RedirectingTo = newRedirectingTo ;
}
2017-04-15 08:45:10 +00:00
result . Status = response . StatusCode ;
// Compatiblity issue between the cookie format and httpclient
// Pull it out manually ignoring the expiry date then set it manually
// http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
var responseCookies = new List < Tuple < string , string > > ( ) ;
2020-02-10 22:16:19 +00:00
if ( response . Headers . TryGetValues ( "set-cookie" , out var cookieHeaders ) )
2017-04-15 08:45:10 +00:00
{
foreach ( var value in cookieHeaders )
{
logger . Debug ( value ) ;
var nameSplit = value . IndexOf ( '=' ) ;
if ( nameSplit > - 1 )
{
2017-11-06 10:51:26 +00:00
responseCookies . Add ( new Tuple < string , string > ( value . Substring ( 0 , nameSplit ) , value . Substring ( 0 , value . IndexOf ( ';' ) = = - 1 ? value . Length : ( value . IndexOf ( ';' ) ) ) + ";" ) ) ;
2017-04-15 08:45:10 +00:00
}
}
var cookieBuilder = new StringBuilder ( ) ;
foreach ( var cookieGroup in responseCookies . GroupBy ( c = > c . Item1 ) )
{
cookieBuilder . AppendFormat ( "{0} " , cookieGroup . Last ( ) . Item2 ) ;
}
result . Cookies = cookieBuilder . ToString ( ) . Trim ( ) ;
}
ServerUtil . ResureRedirectIsFullyQualified ( webRequest , result ) ;
2020-03-14 22:52:06 +00:00
Encoding encoding = null ;
if ( webRequest . Encoding ! = null )
{
encoding = webRequest . Encoding ;
}
else if ( result . Headers . ContainsKey ( "content-type" ) )
{
var CharsetRegex = new Regex ( @"charset=([\w-]+)" , RegexOptions . Compiled ) ;
var CharsetRegexMatch = CharsetRegex . Match ( result . Headers [ "content-type" ] [ 0 ] ) ;
if ( CharsetRegexMatch . Success )
{
var charset = CharsetRegexMatch . Groups [ 1 ] . Value ;
try
{
encoding = Encoding . GetEncoding ( charset ) ;
}
catch ( Exception ex )
{
logger . Error ( string . Format ( "WebClient({0}).GetString(Url:{1}): Error loading encoding {2} based on header {3}: {4}" , ClientType , webRequest . Url , charset , result . Headers [ "content-type" ] [ 0 ] , ex ) ) ;
}
}
else
{
logger . Error ( string . Format ( "WebClient({0}).GetString(Url:{1}): Got header without charset: {2}" , ClientType , webRequest . Url , result . Headers [ "content-type" ] [ 0 ] ) ) ;
}
}
if ( encoding = = null )
{
logger . Error ( string . Format ( "WebClient({0}).GetString(Url:{1}): No encoding detected, defaulting to UTF-8" , ClientType , webRequest . Url ) ) ;
encoding = Encoding . UTF8 ;
}
result . Encoding = encoding ;
2017-04-15 08:45:10 +00:00
return result ;
}
2020-09-21 06:03:18 +00:00
public override void AddTrustedCertificate ( string host , string hash )
{
hash = hash . ToUpper ( ) ;
trustedCertificates . TryGetValue ( hash . ToUpper ( ) , out var hosts ) ;
if ( hosts = = null )
{
hosts = new HashSet < string > ( ) ;
trustedCertificates [ hash ] = hosts ;
}
hosts . Add ( host ) ;
}
2017-04-15 08:45:10 +00:00
}
}