From 90d31a9b8e2ca94f175eab0f2f108eadba2265b9 Mon Sep 17 00:00:00 2001 From: ngosang Date: Mon, 21 Sep 2020 08:03:08 +0200 Subject: [PATCH] Revert "core: refactor http webclient part 11 #8529 (#7728)" This reverts commit e0ef6bc8a8553f5d470ff540d91c510b4c92e854. --- src/Jackett.Common/Plumbing/JackettModule.cs | 59 ++++- .../Utils/Clients/HttpWebClient.cs | 1 + .../Utils/Clients/HttpWebClient2.cs | 3 +- .../Utils/Clients/HttpWebClient2NetCore.cs | 233 ++++++++++++++++++ .../Utils/Clients/HttpWebClientNetCore.cs | 214 ++++++++++++++++ 5 files changed, 507 insertions(+), 3 deletions(-) create mode 100644 src/Jackett.Common/Utils/Clients/HttpWebClient2NetCore.cs create mode 100644 src/Jackett.Common/Utils/Clients/HttpWebClientNetCore.cs diff --git a/src/Jackett.Common/Plumbing/JackettModule.cs b/src/Jackett.Common/Plumbing/JackettModule.cs index 93c40a802..fcc010c36 100644 --- a/src/Jackett.Common/Plumbing/JackettModule.cs +++ b/src/Jackett.Common/Plumbing/JackettModule.cs @@ -1,3 +1,5 @@ +using System; +using System.Reflection; using Autofac; using Jackett.Common.Indexers; using Jackett.Common.Indexers.Meta; @@ -43,14 +45,19 @@ namespace Jackett.Common.Plumbing switch (_runtimeSettings.ClientOverride) { case "httpclientnetcore": + RegisterWebClient(builder); + break; + case "httpclient2netcore": + RegisterWebClient(builder); + break; case "httpclient": RegisterWebClient(builder); break; - case "httpclient2netcore": case "httpclient2": RegisterWebClient(builder); break; default: + var usehttpclient = DetectMonoCompatabilityWithHttpClient(); RegisterWebClient(builder); break; } @@ -63,5 +70,55 @@ namespace Jackett.Common.Plumbing var configService = ctx.Resolve(); return configService.BuildServerConfig(_runtimeSettings); } + + private static bool DetectMonoCompatabilityWithHttpClient() + { + var usehttpclient = false; + try + { + var monotype = Type.GetType("Mono.Runtime"); + if (monotype != null) + { + var displayName = monotype.GetMethod("GetDisplayName", BindingFlags.NonPublic | BindingFlags.Static); + if (displayName != null) + { + var monoVersion = displayName.Invoke(null, null).ToString(); + var monoVersionO = new Version(monoVersion.Split(' ')[0]); + if ((monoVersionO.Major >= 4 && monoVersionO.Minor >= 8) || monoVersionO.Major >= 5) + { + // check if btls is supported + var monoSecurity = Assembly.Load("Mono.Security"); + var monoTlsProviderFactory = monoSecurity.GetType("Mono.Security.Interface.MonoTlsProviderFactory"); + if (monoTlsProviderFactory != null) + { + var isProviderSupported = monoTlsProviderFactory.GetMethod("IsProviderSupported"); + if (isProviderSupported != null) + { + var btlsSupported = (bool)isProviderSupported.Invoke(null, new string[] { "btls" }); + if (btlsSupported) + { + // initialize btls + var initialize = monoTlsProviderFactory.GetMethod("Initialize", new[] { typeof(string) }); + if (initialize != null) + { + initialize.Invoke(null, new string[] { "btls" }); + usehttpclient = true; + } + } + } + } + } + } + } + } + catch (Exception e) + { + Console.Out.WriteLine("Error while deciding which HttpWebClient to use: " + e); + } + + return usehttpclient; + } + + } } diff --git a/src/Jackett.Common/Utils/Clients/HttpWebClient.cs b/src/Jackett.Common/Utils/Clients/HttpWebClient.cs index 024406695..567a7bb22 100644 --- a/src/Jackett.Common/Utils/Clients/HttpWebClient.cs +++ b/src/Jackett.Common/Utils/Clients/HttpWebClient.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Net; using System.Net.Http; using System.Text; +using System.Text.RegularExpressions; using System.Threading.Tasks; using CloudflareSolverRe; using Jackett.Common.Helpers; diff --git a/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs b/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs index 9eef78c2c..117020ac3 100644 --- a/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs +++ b/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Net; using System.Net.Http; using System.Text; +using System.Text.RegularExpressions; using System.Threading.Tasks; using CloudflareSolverRe; using Jackett.Common.Helpers; @@ -68,8 +69,6 @@ namespace Jackett.Common.Utils.Clients public override void Init() { - ServicePointManager.DefaultConnectionLimit = 1000; - base.Init(); ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072; diff --git a/src/Jackett.Common/Utils/Clients/HttpWebClient2NetCore.cs b/src/Jackett.Common/Utils/Clients/HttpWebClient2NetCore.cs new file mode 100644 index 000000000..b0260d3fc --- /dev/null +++ b/src/Jackett.Common/Utils/Clients/HttpWebClient2NetCore.cs @@ -0,0 +1,233 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using CloudflareSolverRe; +using Jackett.Common.Helpers; +using Jackett.Common.Models.Config; +using Jackett.Common.Services.Interfaces; +using NLog; + +namespace Jackett.Common.Utils.Clients +{ + // Compared to HttpWebClient this implementation will reuse the HttpClient instance (one per indexer). + // This should improve performance and avoid problems with too many open file handles. + public class HttpWebClient2NetCore : WebClient + { + private readonly CookieContainer cookies; + private ClearanceHandler clearanceHandlr; + private HttpClientHandler clientHandlr; + private HttpClient client; + + public HttpWebClient2NetCore(IProcessService p, Logger l, IConfigurationService c, ServerConfig sc) + : base(p: p, + l: l, + c: c, + sc: sc) + { + cookies = new CookieContainer(); + CreateClient(); + } + + public void CreateClient() + { + clearanceHandlr = new ClearanceHandler(BrowserUtil.ChromeUserAgent) + { + MaxTries = 10 + }; + clientHandlr = new HttpClientHandler + { + CookieContainer = cookies, + AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more. + UseCookies = true, + Proxy = webProxy, + UseProxy = (webProxy != null), + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate + }; + + // custom certificate validation handler (netcore version) + clientHandlr.ServerCertificateCustomValidationCallback = ValidateCertificate; + + clearanceHandlr.InnerHandler = clientHandlr; + client = new HttpClient(clearanceHandlr); + } + + // Called everytime the ServerConfig changes + public override void OnNext(ServerConfig value) + { + base.OnNext(value); + // recreate client if needed (can't just change the proxy attribute) + if (!ReferenceEquals(clientHandlr.Proxy, webProxy)) + { + CreateClient(); + } + } + + public override void Init() + { + ServicePointManager.DefaultConnectionLimit = 1000; + + base.Init(); + } + + protected override async Task Run(WebRequest webRequest) + { + HttpResponseMessage response = null; + var request = new HttpRequestMessage(); + request.Headers.ExpectContinue = false; + request.RequestUri = new Uri(webRequest.Url); + + //if (webRequest.EmulateBrowser == true) + // request.Headers.UserAgent.ParseAdd(BrowserUtil.ChromeUserAgent); + //else + // request.Headers.UserAgent.ParseAdd("Jackett/" + configService.GetVersion()); + + // clear cookies from cookiecontainer + var oldCookies = cookies.GetCookies(request.RequestUri); + foreach (Cookie oldCookie in oldCookies) + oldCookie.Expired = true; + + // add cookies to cookiecontainer + if (!string.IsNullOrWhiteSpace(webRequest.Cookies)) + { + // don't include the path, Scheme is needed for mono compatibility + var cookieUrl = new Uri(request.RequestUri.Scheme + "://" + request.RequestUri.Host); + var cookieDictionary = CookieUtil.CookieHeaderToDictionary(webRequest.Cookies); + foreach (var kv in cookieDictionary) + cookies.Add(cookieUrl, new Cookie(kv.Key, kv.Value)); + } + + if (webRequest.Headers != null) + { + foreach (var header in webRequest.Headers) + { + if (header.Key != "Content-Type") + { + request.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + } + } + + if (!string.IsNullOrEmpty(webRequest.Referer)) + request.Headers.Referrer = new Uri(webRequest.Referer); + + if (!string.IsNullOrEmpty(webRequest.RawBody)) + { + var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast?>().FirstOrDefault(); + if (type.HasValue) + { + var str = new StringContent(webRequest.RawBody); + str.Headers.Remove("Content-Type"); + str.Headers.Add("Content-Type", type.Value.Value); + request.Content = str; + } + else + request.Content = new StringContent(webRequest.RawBody); + request.Method = HttpMethod.Post; + } + else if (webRequest.Type == RequestType.POST) + { + if (webRequest.PostData != null) + request.Content = FormUrlEncodedContentWithEncoding(webRequest.PostData, webRequest.Encoding); + request.Method = HttpMethod.Post; + } + else + { + request.Method = HttpMethod.Get; + } + + response = await client.SendAsync(request); + + var result = new WebClientByteResult + { + ContentBytes = await response.Content.ReadAsByteArrayAsync() + }; + + foreach (var header in response.Headers) + { + var value = header.Value; + result.Headers[header.Key.ToLowerInvariant()] = value.ToArray(); + } + + // some cloudflare clients are using a refresh header + // Pull it out manually + if (response.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh")) + { + var refreshHeaders = response.Headers.GetValues("Refresh"); + var redirval = ""; + var redirtime = 0; + if (refreshHeaders != null) + { + foreach (var value in refreshHeaders) + { + var start = value.IndexOf("="); + var end = value.IndexOf(";"); + var len = value.Length; + if (start > -1) + { + redirval = value.Substring(start + 1); + result.RedirectingTo = redirval; + // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature + // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally + // it shoudln't include service unavailable..only if we have this redirect header. + response.StatusCode = System.Net.HttpStatusCode.Redirect; + redirtime = int.Parse(value.Substring(0, end)); + System.Threading.Thread.Sleep(redirtime * 1000); + } + } + } + } + if (response.Headers.Location != null) + { + result.RedirectingTo = response.Headers.Location.ToString(); + } + // Mono won't add the baseurl to relative redirects. + // e.g. a "Location: /index.php" header will result in the Uri "file:///index.php" + // See issue #1200 + if (result.RedirectingTo != null && result.RedirectingTo.StartsWith("file://")) + { + // URL decoding apparently is needed to, without it e.g. Demonoid download is broken + // TODO: is it always needed (not just for relative redirects)? + var newRedirectingTo = WebUtilityHelpers.UrlDecode(result.RedirectingTo, webRequest.Encoding); + if (newRedirectingTo.StartsWith("file:////")) // Location without protocol but with host (only add scheme) + newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + ":"); + else + newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + "://" + request.RequestUri.Host); + logger.Debug("[MONO relative redirect bug] Rewriting relative redirect URL from " + result.RedirectingTo + " to " + newRedirectingTo); + result.RedirectingTo = newRedirectingTo; + } + result.Status = response.StatusCode; + + // Compatiblity issue between the cookie format and httpclient + // Pull it out manually ignoring the expiry date then set it manually + // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer + var responseCookies = new List>(); + + if (response.Headers.TryGetValues("set-cookie", out var cookieHeaders)) + { + foreach (var value in cookieHeaders) + { + logger.Debug(value); + var nameSplit = value.IndexOf('='); + if (nameSplit > -1) + { + responseCookies.Add(new Tuple(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';'))) + ";")); + } + } + + var cookieBuilder = new StringBuilder(); + foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1)) + { + cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2); + } + result.Cookies = cookieBuilder.ToString().Trim(); + } + ServerUtil.ResureRedirectIsFullyQualified(webRequest, result); + return result; + } + } +} diff --git a/src/Jackett.Common/Utils/Clients/HttpWebClientNetCore.cs b/src/Jackett.Common/Utils/Clients/HttpWebClientNetCore.cs new file mode 100644 index 000000000..71cfa3b6e --- /dev/null +++ b/src/Jackett.Common/Utils/Clients/HttpWebClientNetCore.cs @@ -0,0 +1,214 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using CloudflareSolverRe; +using Jackett.Common.Helpers; +using Jackett.Common.Models.Config; +using Jackett.Common.Services.Interfaces; +using NLog; + +namespace Jackett.Common.Utils.Clients +{ + // custom HttpWebClient based WebClient for netcore (due to changed custom certificate validation API) + public class HttpWebClientNetCore : WebClient + { + public HttpWebClientNetCore(IProcessService p, Logger l, IConfigurationService c, ServerConfig sc) + : base(p: p, + l: l, + c: c, + sc: sc) + { + } + public override void Init() + { + ServicePointManager.DefaultConnectionLimit = 1000; + + base.Init(); + } + + protected override async Task Run(WebRequest webRequest) + { + ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072; + + var cookies = new CookieContainer(); + if (!string.IsNullOrWhiteSpace(webRequest.Cookies)) + { + // don't include the path, Scheme is needed for mono compatibility + var requestUri = new Uri(webRequest.Url); + var cookieUrl = new Uri(requestUri.Scheme + "://" + requestUri.Host); + var cookieDictionary = CookieUtil.CookieHeaderToDictionary(webRequest.Cookies); + foreach (var kv in cookieDictionary) + cookies.Add(cookieUrl, new Cookie(kv.Key, kv.Value)); + } + + var userAgent = webRequest.EmulateBrowser.Value ? BrowserUtil.ChromeUserAgent : "Jackett/" + configService.GetVersion(); + + using (var clearanceHandlr = new ClearanceHandler(userAgent)) + { + clearanceHandlr.MaxTries = 10; + using (var clientHandlr = new HttpClientHandler + { + CookieContainer = cookies, + AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more. + UseCookies = true, + Proxy = webProxy, + UseProxy = (webProxy != null), + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate + }) + { + // custom certificate validation handler (netcore version) + clientHandlr.ServerCertificateCustomValidationCallback = ValidateCertificate; + + clearanceHandlr.InnerHandler = clientHandlr; + using (var client = new HttpClient(clearanceHandlr)) + { + //if (webRequest.EmulateBrowser == true) + // client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent); + //else + // client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion()); + + HttpResponseMessage response = null; + using (var request = new HttpRequestMessage()) + { + request.Headers.ExpectContinue = false; + request.RequestUri = new Uri(webRequest.Url); + + if (webRequest.Headers != null) + { + foreach (var header in webRequest.Headers) + { + if (header.Key != "Content-Type") + { + request.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + } + } + + if (!string.IsNullOrEmpty(webRequest.Referer)) + request.Headers.Referrer = new Uri(webRequest.Referer); + + if (!string.IsNullOrEmpty(webRequest.RawBody)) + { + var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast?>().FirstOrDefault(); + if (type.HasValue) + { + var str = new StringContent(webRequest.RawBody); + str.Headers.Remove("Content-Type"); + str.Headers.Add("Content-Type", type.Value.Value); + request.Content = str; + } + else + request.Content = new StringContent(webRequest.RawBody); + request.Method = HttpMethod.Post; + } + else if (webRequest.Type == RequestType.POST) + { + if (webRequest.PostData != null) + request.Content = FormUrlEncodedContentWithEncoding(webRequest.PostData, webRequest.Encoding); + request.Method = HttpMethod.Post; + } + else + { + request.Method = HttpMethod.Get; + } + + using (response = await client.SendAsync(request)) + { + var result = new WebClientByteResult + { + ContentBytes = await response.Content.ReadAsByteArrayAsync() + }; + + foreach (var header in response.Headers) + { + var value = header.Value; + result.Headers[header.Key.ToLowerInvariant()] = value.ToArray(); + } + + // some cloudflare clients are using a refresh header + // Pull it out manually + if (response.StatusCode == HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh")) + { + var refreshHeaders = response.Headers.GetValues("Refresh"); + var redirval = ""; + var redirtime = 0; + if (refreshHeaders != null) + { + foreach (var value in refreshHeaders) + { + var start = value.IndexOf("="); + var end = value.IndexOf(";"); + var len = value.Length; + if (start > -1) + { + redirval = value.Substring(start + 1); + result.RedirectingTo = redirval; + // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature + // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally + // it shoudln't include service unavailable..only if we have this redirect header. + response.StatusCode = System.Net.HttpStatusCode.Redirect; + redirtime = int.Parse(value.Substring(0, end)); + System.Threading.Thread.Sleep(redirtime * 1000); + } + } + } + } + if (response.Headers.Location != null) + { + result.RedirectingTo = response.Headers.Location.ToString(); + } + // Mono won't add the baseurl to relative redirects. + // e.g. a "Location: /index.php" header will result in the Uri "file:///index.php" + // See issue #1200 + if (result.RedirectingTo != null && result.RedirectingTo.StartsWith("file://")) + { + // URL decoding apparently is needed to, without it e.g. Demonoid download is broken + // TODO: is it always needed (not just for relative redirects)? + var newRedirectingTo = WebUtilityHelpers.UrlDecode(result.RedirectingTo, webRequest.Encoding); + if (newRedirectingTo.StartsWith("file:////")) // Location without protocol but with host (only add scheme) + newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + ":"); + else + newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + "://" + request.RequestUri.Host); + logger.Debug("[MONO relative redirect bug] Rewriting relative redirect URL from " + result.RedirectingTo + " to " + newRedirectingTo); + result.RedirectingTo = newRedirectingTo; + } + result.Status = response.StatusCode; + + // Compatiblity issue between the cookie format and httpclient + // Pull it out manually ignoring the expiry date then set it manually + // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer + var responseCookies = new List>(); + + if (response.Headers.TryGetValues("set-cookie", out var cookieHeaders)) + { + foreach (var value in cookieHeaders) + { + var nameSplit = value.IndexOf('='); + if (nameSplit > -1) + { + responseCookies.Add(new Tuple(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';'))) + ";")); + } + } + + var cookieBuilder = new StringBuilder(); + foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1)) + { + cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2); + } + result.Cookies = cookieBuilder.ToString().Trim(); + } + ServerUtil.ResureRedirectIsFullyQualified(webRequest, result); + return result; + } + } + } + } + } + } + } +}