mirror/Jackett
1
0
Fork 0
mirror of https://github.com/Jackett/Jackett synced 2025-03-09 21:54:51 +00:00
Code Issues Releases Wiki Activity

Revert "core: refactor http webclient part 11 #8529 (#7728)"

Browse source 
This reverts commit e0ef6bc8a8.
This commit is contained in:
ngosang 2020-09-21 08:03:08 +02:00
parent a8d055f0d5
commit 90d31a9b8e
5 changed files with 507 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
src/Jackett.Common/Plumbing/JackettModule.cs
View file

@ -1,3 +1,5 @@
using System;
using System.Reflection;
using Autofac;
using Jackett.Common.Indexers;
using Jackett.Common.Indexers.Meta;
@ -43,14 +45,19 @@ namespace Jackett.Common.Plumbing
switch (_runtimeSettings.ClientOverride)
{
case "httpclientnetcore":
RegisterWebClient<HttpWebClientNetCore>(builder);
break;
case "httpclient2netcore":
RegisterWebClient<HttpWebClient2NetCore>(builder);
break;
case "httpclient":
RegisterWebClient<HttpWebClient>(builder);
break;
case "httpclient2netcore":
case "httpclient2":
RegisterWebClient<HttpWebClient2>(builder);
break;
default:
var usehttpclient = DetectMonoCompatabilityWithHttpClient();
RegisterWebClient<HttpWebClient>(builder);
break;
}
@ -63,5 +70,55 @@ namespace Jackett.Common.Plumbing
var configService = ctx.Resolve<IConfigurationService>();
return configService.BuildServerConfig(_runtimeSettings);
}
private static bool DetectMonoCompatabilityWithHttpClient()
{
var usehttpclient = false;
try
{
var monotype = Type.GetType("Mono.Runtime");
if (monotype != null)
{
var displayName = monotype.GetMethod("GetDisplayName", BindingFlags.NonPublic | BindingFlags.Static);
if (displayName != null)
{
var monoVersion = displayName.Invoke(null, null).ToString();
var monoVersionO = new Version(monoVersion.Split(' ')[0]);
if ((monoVersionO.Major >= 4 && monoVersionO.Minor >= 8) || monoVersionO.Major >= 5)
{
// check if btls is supported
var monoSecurity = Assembly.Load("Mono.Security");
var monoTlsProviderFactory = monoSecurity.GetType("Mono.Security.Interface.MonoTlsProviderFactory");
if (monoTlsProviderFactory != null)
{
var isProviderSupported = monoTlsProviderFactory.GetMethod("IsProviderSupported");
if (isProviderSupported != null)
{
var btlsSupported = (bool)isProviderSupported.Invoke(null, new string[] { "btls" });
if (btlsSupported)
{
// initialize btls
var initialize = monoTlsProviderFactory.GetMethod("Initialize", new[] { typeof(string) });
if (initialize != null)
{
initialize.Invoke(null, new string[] { "btls" });
usehttpclient = true;
}
}
}
}
}
}
}
}
catch (Exception e)
{
Console.Out.WriteLine("Error while deciding which HttpWebClient to use: " + e);
}
return usehttpclient;
}
}
}

1
src/Jackett.Common/Utils/Clients/HttpWebClient.cs
View file

@ -4,6 +4,7 @@ using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using CloudflareSolverRe;
using Jackett.Common.Helpers;

3
src/Jackett.Common/Utils/Clients/HttpWebClient2.cs
View file

@ -4,6 +4,7 @@ using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using CloudflareSolverRe;
using Jackett.Common.Helpers;
@ -68,8 +69,6 @@ namespace Jackett.Common.Utils.Clients
public override void Init()
{
ServicePointManager.DefaultConnectionLimit = 1000;
base.Init();
ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072;

233
src/Jackett.Common/Utils/Clients/HttpWebClient2NetCore.cs Normal file
View file

@ -0,0 +1,233 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using CloudflareSolverRe;
using Jackett.Common.Helpers;
using Jackett.Common.Models.Config;
using Jackett.Common.Services.Interfaces;
using NLog;
namespace Jackett.Common.Utils.Clients
{
// Compared to HttpWebClient this implementation will reuse the HttpClient instance (one per indexer).
// This should improve performance and avoid problems with too many open file handles.
public class HttpWebClient2NetCore : WebClient
{
private readonly CookieContainer cookies;
private ClearanceHandler clearanceHandlr;
private HttpClientHandler clientHandlr;
private HttpClient client;
public HttpWebClient2NetCore(IProcessService p, Logger l, IConfigurationService c, ServerConfig sc)
: base(p: p,
l: l,
c: c,
sc: sc)
{
cookies = new CookieContainer();
CreateClient();
}
public void CreateClient()
{
clearanceHandlr = new ClearanceHandler(BrowserUtil.ChromeUserAgent)
{
MaxTries = 10
};
clientHandlr = new HttpClientHandler
{
CookieContainer = cookies,
AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
UseCookies = true,
Proxy = webProxy,
UseProxy = (webProxy != null),
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
};
// custom certificate validation handler (netcore version)
clientHandlr.ServerCertificateCustomValidationCallback = ValidateCertificate;
clearanceHandlr.InnerHandler = clientHandlr;
client = new HttpClient(clearanceHandlr);
}
// Called everytime the ServerConfig changes
public override void OnNext(ServerConfig value)
{
base.OnNext(value);
// recreate client if needed (can't just change the proxy attribute)
if (!ReferenceEquals(clientHandlr.Proxy, webProxy))
{
CreateClient();
}
}
public override void Init()
{
ServicePointManager.DefaultConnectionLimit = 1000;
base.Init();
}
protected override async Task<WebClientByteResult> Run(WebRequest webRequest)
{
HttpResponseMessage response = null;
var request = new HttpRequestMessage();
request.Headers.ExpectContinue = false;
request.RequestUri = new Uri(webRequest.Url);
//if (webRequest.EmulateBrowser == true)
// request.Headers.UserAgent.ParseAdd(BrowserUtil.ChromeUserAgent);
//else
// request.Headers.UserAgent.ParseAdd("Jackett/" + configService.GetVersion());
// clear cookies from cookiecontainer
var oldCookies = cookies.GetCookies(request.RequestUri);
foreach (Cookie oldCookie in oldCookies)
oldCookie.Expired = true;
// add cookies to cookiecontainer
if (!string.IsNullOrWhiteSpace(webRequest.Cookies))
{
// don't include the path, Scheme is needed for mono compatibility
var cookieUrl = new Uri(request.RequestUri.Scheme + "://" + request.RequestUri.Host);
var cookieDictionary = CookieUtil.CookieHeaderToDictionary(webRequest.Cookies);
foreach (var kv in cookieDictionary)
cookies.Add(cookieUrl, new Cookie(kv.Key, kv.Value));
}
if (webRequest.Headers != null)
{
foreach (var header in webRequest.Headers)
{
if (header.Key != "Content-Type")
{
request.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
}
}
if (!string.IsNullOrEmpty(webRequest.Referer))
request.Headers.Referrer = new Uri(webRequest.Referer);
if (!string.IsNullOrEmpty(webRequest.RawBody))
{
var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast<KeyValuePair<string, string>?>().FirstOrDefault();
if (type.HasValue)
{
var str = new StringContent(webRequest.RawBody);
str.Headers.Remove("Content-Type");
str.Headers.Add("Content-Type", type.Value.Value);
request.Content = str;
}
else
request.Content = new StringContent(webRequest.RawBody);
request.Method = HttpMethod.Post;
}
else if (webRequest.Type == RequestType.POST)
{
if (webRequest.PostData != null)
request.Content = FormUrlEncodedContentWithEncoding(webRequest.PostData, webRequest.Encoding);
request.Method = HttpMethod.Post;
}
else
{
request.Method = HttpMethod.Get;
}
response = await client.SendAsync(request);
var result = new WebClientByteResult
{
ContentBytes = await response.Content.ReadAsByteArrayAsync()
};
foreach (var header in response.Headers)
{
var value = header.Value;
result.Headers[header.Key.ToLowerInvariant()] = value.ToArray();
}
// some cloudflare clients are using a refresh header
// Pull it out manually
if (response.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh"))
{
var refreshHeaders = response.Headers.GetValues("Refresh");
var redirval = "";
var redirtime = 0;
if (refreshHeaders != null)
{
foreach (var value in refreshHeaders)
{
var start = value.IndexOf("=");
var end = value.IndexOf(";");
var len = value.Length;
if (start > -1)
{
redirval = value.Substring(start + 1);
result.RedirectingTo = redirval;
// normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
// of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
// it shoudln't include service unavailable..only if we have this redirect header.
response.StatusCode = System.Net.HttpStatusCode.Redirect;
redirtime = int.Parse(value.Substring(0, end));
System.Threading.Thread.Sleep(redirtime * 1000);
}
}
}
}
if (response.Headers.Location != null)
{
result.RedirectingTo = response.Headers.Location.ToString();
}
// Mono won't add the baseurl to relative redirects.
// e.g. a "Location: /index.php" header will result in the Uri "file:///index.php"
// See issue #1200
if (result.RedirectingTo != null && result.RedirectingTo.StartsWith("file://"))
{
// URL decoding apparently is needed to, without it e.g. Demonoid download is broken
// TODO: is it always needed (not just for relative redirects)?
var newRedirectingTo = WebUtilityHelpers.UrlDecode(result.RedirectingTo, webRequest.Encoding);
if (newRedirectingTo.StartsWith("file:////")) // Location without protocol but with host (only add scheme)
newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + ":");
else
newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + "://" + request.RequestUri.Host);
logger.Debug("[MONO relative redirect bug] Rewriting relative redirect URL from " + result.RedirectingTo + " to " + newRedirectingTo);
result.RedirectingTo = newRedirectingTo;
}
result.Status = response.StatusCode;
// Compatiblity issue between the cookie format and httpclient
// Pull it out manually ignoring the expiry date then set it manually
// http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
var responseCookies = new List<Tuple<string, string>>();
if (response.Headers.TryGetValues("set-cookie", out var cookieHeaders))
{
foreach (var value in cookieHeaders)
{
logger.Debug(value);
var nameSplit = value.IndexOf('=');
if (nameSplit > -1)
{
responseCookies.Add(new Tuple<string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';'))) + ";"));
}
}
var cookieBuilder = new StringBuilder();
foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
{
cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
}
result.Cookies = cookieBuilder.ToString().Trim();
}
ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
return result;
}
}
}

214
src/Jackett.Common/Utils/Clients/HttpWebClientNetCore.cs Normal file
View file

@ -0,0 +1,214 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using CloudflareSolverRe;
using Jackett.Common.Helpers;
using Jackett.Common.Models.Config;
using Jackett.Common.Services.Interfaces;
using NLog;
namespace Jackett.Common.Utils.Clients
{
// custom HttpWebClient based WebClient for netcore (due to changed custom certificate validation API)
public class HttpWebClientNetCore : WebClient
{
public HttpWebClientNetCore(IProcessService p, Logger l, IConfigurationService c, ServerConfig sc)
: base(p: p,
l: l,
c: c,
sc: sc)
{
}
public override void Init()
{
ServicePointManager.DefaultConnectionLimit = 1000;
base.Init();
}
protected override async Task<WebClientByteResult> Run(WebRequest webRequest)
{
ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072;
var cookies = new CookieContainer();
if (!string.IsNullOrWhiteSpace(webRequest.Cookies))
{
// don't include the path, Scheme is needed for mono compatibility
var requestUri = new Uri(webRequest.Url);
var cookieUrl = new Uri(requestUri.Scheme + "://" + requestUri.Host);
var cookieDictionary = CookieUtil.CookieHeaderToDictionary(webRequest.Cookies);
foreach (var kv in cookieDictionary)
cookies.Add(cookieUrl, new Cookie(kv.Key, kv.Value));
}
var userAgent = webRequest.EmulateBrowser.Value ? BrowserUtil.ChromeUserAgent : "Jackett/" + configService.GetVersion();
using (var clearanceHandlr = new ClearanceHandler(userAgent))
{
clearanceHandlr.MaxTries = 10;
using (var clientHandlr = new HttpClientHandler
{
CookieContainer = cookies,
AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
UseCookies = true,
Proxy = webProxy,
UseProxy = (webProxy != null),
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
})
{
// custom certificate validation handler (netcore version)
clientHandlr.ServerCertificateCustomValidationCallback = ValidateCertificate;
clearanceHandlr.InnerHandler = clientHandlr;
using (var client = new HttpClient(clearanceHandlr))
{
//if (webRequest.EmulateBrowser == true)
// client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
//else
// client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion());
HttpResponseMessage response = null;
using (var request = new HttpRequestMessage())
{
request.Headers.ExpectContinue = false;
request.RequestUri = new Uri(webRequest.Url);
if (webRequest.Headers != null)
{
foreach (var header in webRequest.Headers)
{
if (header.Key != "Content-Type")
{
request.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
}
}
if (!string.IsNullOrEmpty(webRequest.Referer))
request.Headers.Referrer = new Uri(webRequest.Referer);
if (!string.IsNullOrEmpty(webRequest.RawBody))
{
var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast<KeyValuePair<string, string>?>().FirstOrDefault();
if (type.HasValue)
{
var str = new StringContent(webRequest.RawBody);
str.Headers.Remove("Content-Type");
str.Headers.Add("Content-Type", type.Value.Value);
request.Content = str;
}
else
request.Content = new StringContent(webRequest.RawBody);
request.Method = HttpMethod.Post;
}
else if (webRequest.Type == RequestType.POST)
{
if (webRequest.PostData != null)
request.Content = FormUrlEncodedContentWithEncoding(webRequest.PostData, webRequest.Encoding);
request.Method = HttpMethod.Post;
}
else
{
request.Method = HttpMethod.Get;
}
using (response = await client.SendAsync(request))
{
var result = new WebClientByteResult
{
ContentBytes = await response.Content.ReadAsByteArrayAsync()
};
foreach (var header in response.Headers)
{
var value = header.Value;
result.Headers[header.Key.ToLowerInvariant()] = value.ToArray();
}
// some cloudflare clients are using a refresh header
// Pull it out manually
if (response.StatusCode == HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh"))
{
var refreshHeaders = response.Headers.GetValues("Refresh");
var redirval = "";
var redirtime = 0;
if (refreshHeaders != null)
{
foreach (var value in refreshHeaders)
{
var start = value.IndexOf("=");
var end = value.IndexOf(";");
var len = value.Length;
if (start > -1)
{
redirval = value.Substring(start + 1);
result.RedirectingTo = redirval;
// normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
// of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
// it shoudln't include service unavailable..only if we have this redirect header.
response.StatusCode = System.Net.HttpStatusCode.Redirect;
redirtime = int.Parse(value.Substring(0, end));
System.Threading.Thread.Sleep(redirtime * 1000);
}
}
}
}
if (response.Headers.Location != null)
{
result.RedirectingTo = response.Headers.Location.ToString();
}
// Mono won't add the baseurl to relative redirects.
// e.g. a "Location: /index.php" header will result in the Uri "file:///index.php"
// See issue #1200
if (result.RedirectingTo != null && result.RedirectingTo.StartsWith("file://"))
{
// URL decoding apparently is needed to, without it e.g. Demonoid download is broken
// TODO: is it always needed (not just for relative redirects)?
var newRedirectingTo = WebUtilityHelpers.UrlDecode(result.RedirectingTo, webRequest.Encoding);
if (newRedirectingTo.StartsWith("file:////")) // Location without protocol but with host (only add scheme)
newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + ":");
else
newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + "://" + request.RequestUri.Host);
logger.Debug("[MONO relative redirect bug] Rewriting relative redirect URL from " + result.RedirectingTo + " to " + newRedirectingTo);
result.RedirectingTo = newRedirectingTo;
}
result.Status = response.StatusCode;
// Compatiblity issue between the cookie format and httpclient
// Pull it out manually ignoring the expiry date then set it manually
// http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
var responseCookies = new List<Tuple<string, string>>();
if (response.Headers.TryGetValues("set-cookie", out var cookieHeaders))
{
foreach (var value in cookieHeaders)
{
var nameSplit = value.IndexOf('=');
if (nameSplit > -1)
{
responseCookies.Add(new Tuple<string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';'))) + ";"));
}
}
var cookieBuilder = new StringBuilder();
foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
{
cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
}
result.Cookies = cookieBuilder.ToString().Trim();
}
ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
return result;
}
}
}
}
}
}
}
}