From bcc1dc194887ac4089a1a314f6464c2a83ac9da5 Mon Sep 17 00:00:00 2001 From: kaso17 Date: Mon, 2 Jul 2018 13:05:24 +0200 Subject: [PATCH] fix custom certificate validation handler (#3297) * fix netcore custom certificate validator * conditional HttpWebClientNetCore register * deprecate IgnoreSslErrors option * Use httpclient when running full framework --- src/Jackett.Common/Plumbing/JackettModule.cs | 3 + src/Jackett.Server/HttpWebClientNetCore.cs | 313 +++++++++++++++++++ src/Jackett.Server/Initialisation.cs | 4 +- src/Jackett.Server/Program.cs | 13 +- src/Jackett.Server/Startup.cs | 3 + 5 files changed, 332 insertions(+), 4 deletions(-) create mode 100644 src/Jackett.Server/HttpWebClientNetCore.cs diff --git a/src/Jackett.Common/Plumbing/JackettModule.cs b/src/Jackett.Common/Plumbing/JackettModule.cs index bebd1713c..2a405182c 100644 --- a/src/Jackett.Common/Plumbing/JackettModule.cs +++ b/src/Jackett.Common/Plumbing/JackettModule.cs @@ -53,6 +53,9 @@ namespace Jackett.Common.Plumbing // Register the best web client for the platform or the override switch (_runtimeSettings.ClientOverride) { + case "httpclientnetcore": + // do nothing, registered by the netcore app + break; case "httpclient": RegisterWebClient(builder); break; diff --git a/src/Jackett.Server/HttpWebClientNetCore.cs b/src/Jackett.Server/HttpWebClientNetCore.cs new file mode 100644 index 000000000..6f8245c34 --- /dev/null +++ b/src/Jackett.Server/HttpWebClientNetCore.cs @@ -0,0 +1,313 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Security; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using System.Threading.Tasks; +using com.LandonKey.SocksWebProxy; +using com.LandonKey.SocksWebProxy.Proxy; +using CloudFlareUtilities; +using Jackett.Common.Models.Config; +using Jackett.Common.Services.Interfaces; +using NLog; +using Jackett.Common.Helpers; + +namespace Jackett.Common.Utils.Clients +{ + // custom HttpWebClient based WebClient for netcore (due to changed custom certificate validation API) + public class HttpWebClientNetCore : WebClient + { + static protected Dictionary> trustedCertificates = new Dictionary>(); + static protected string webProxyUrl; + static protected IWebProxy webProxy; + + static public void InitProxy(ServerConfig serverConfig) + { + // dispose old SocksWebProxy + if (webProxy != null && webProxy is SocksWebProxy) + { + ((SocksWebProxy)webProxy).Dispose(); + webProxy = null; + } + + webProxyUrl = serverConfig.GetProxyUrl(); + if (!string.IsNullOrWhiteSpace(webProxyUrl)) + { + if (serverConfig.ProxyType != ProxyType.Http) + { + var addresses = Dns.GetHostAddressesAsync(serverConfig.ProxyUrl).Result; + var socksConfig = new ProxyConfig + { + SocksAddress = addresses.FirstOrDefault(), + Username = serverConfig.ProxyUsername, + Password = serverConfig.ProxyPassword, + Version = serverConfig.ProxyType == ProxyType.Socks4 ? + ProxyConfig.SocksVersion.Four : + ProxyConfig.SocksVersion.Five + }; + if (serverConfig.ProxyPort.HasValue) + { + socksConfig.SocksPort = serverConfig.ProxyPort.Value; + } + webProxy = new SocksWebProxy(socksConfig, false); + } + else + { + NetworkCredential creds = null; + if (!serverConfig.ProxyIsAnonymous) + { + var username = serverConfig.ProxyUsername; + var password = serverConfig.ProxyPassword; + creds = new NetworkCredential(username, password); + } + webProxy = new WebProxy(webProxyUrl) + { + BypassProxyOnLocal = false, + Credentials = creds + }; + } + } + } + + public HttpWebClientNetCore(IProcessService p, Logger l, IConfigurationService c, ServerConfig sc) + : base(p: p, + l: l, + c: c, + sc: sc) + { + if (webProxyUrl == null) + InitProxy(sc); + } + + // Called everytime the ServerConfig changes + public override void OnNext(ServerConfig value) + { + var newProxyUrl = serverConfig.GetProxyUrl(); + if (webProxyUrl != newProxyUrl) // if proxy URL changed + InitProxy(serverConfig); + } + + override public void Init() + { + ServicePointManager.DefaultConnectionLimit = 1000; + + if (serverConfig.RuntimeSettings.IgnoreSslErrors == true) + { + logger.Info(string.Format("HttpWebClient: Disabling certificate validation")); + ServicePointManager.ServerCertificateValidationCallback += (sender, certificate, chain, sslPolicyErrors) => { return true; }; + } + } + + override protected async Task Run(WebRequest webRequest) + { + ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072; + + var cookies = new CookieContainer(); + if (!string.IsNullOrEmpty(webRequest.Cookies)) + { + var uri = new Uri(webRequest.Url); + var cookieUrl = new Uri(uri.Scheme + "://" + uri.Host); // don't include the path, Scheme is needed for mono compatibility + foreach (var c in webRequest.Cookies.Split(';')) + { + try + { + cookies.SetCookies(cookieUrl, c.Trim()); + } + catch (CookieException ex) + { + logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message); + } + } + } + + using (ClearanceHandler clearanceHandlr = new ClearanceHandler()) + { + clearanceHandlr.MaxRetries = 30; + using (HttpClientHandler clientHandlr = new HttpClientHandler + { + CookieContainer = cookies, + AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more. + UseCookies = true, + Proxy = webProxy, + UseProxy = (webProxy != null), + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate + }) + { + // custom certificate validation handler (netcore version) + clientHandlr.ServerCertificateCustomValidationCallback = (request, certificate, chain, sslPolicyErrors) => + { + var hash = certificate.GetCertHashString(); + + ICollection hosts; + + trustedCertificates.TryGetValue(hash, out hosts); + if (hosts != null) + { + if (hosts.Contains(request.RequestUri.Host)) + return true; + } + return sslPolicyErrors == SslPolicyErrors.None; + }; + + clearanceHandlr.InnerHandler = clientHandlr; + using (var client = new HttpClient(clearanceHandlr)) + { + if (webRequest.EmulateBrowser == true) + client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent); + else + client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion()); + + HttpResponseMessage response = null; + using (var request = new HttpRequestMessage()) + { + request.Headers.ExpectContinue = false; + request.RequestUri = new Uri(webRequest.Url); + + if (webRequest.Headers != null) + { + foreach (var header in webRequest.Headers) + { + if (header.Key != "Content-Type") + { + request.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + } + } + + if (!string.IsNullOrEmpty(webRequest.Referer)) + request.Headers.Referrer = new Uri(webRequest.Referer); + + if (!string.IsNullOrEmpty(webRequest.RawBody)) + { + var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast?>().FirstOrDefault(); + if (type.HasValue) + { + var str = new StringContent(webRequest.RawBody); + str.Headers.Remove("Content-Type"); + str.Headers.Add("Content-Type", type.Value.Value); + request.Content = str; + } + else + request.Content = new StringContent(webRequest.RawBody); + request.Method = HttpMethod.Post; + } + else if (webRequest.Type == RequestType.POST) + { + if (webRequest.PostData != null) + request.Content = new FormUrlEncodedContent(webRequest.PostData); + request.Method = HttpMethod.Post; + } + else + { + request.Method = HttpMethod.Get; + } + + using (response = await client.SendAsync(request)) + { + var result = new WebClientByteResult + { + Content = await response.Content.ReadAsByteArrayAsync() + }; + + foreach (var header in response.Headers) + { + IEnumerable value = header.Value; + result.Headers[header.Key.ToLowerInvariant()] = value.ToArray(); + } + + // some cloudflare clients are using a refresh header + // Pull it out manually + if (response.StatusCode == HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh")) + { + var refreshHeaders = response.Headers.GetValues("Refresh"); + var redirval = ""; + var redirtime = 0; + if (refreshHeaders != null) + { + foreach (var value in refreshHeaders) + { + var start = value.IndexOf("="); + var end = value.IndexOf(";"); + var len = value.Length; + if (start > -1) + { + redirval = value.Substring(start + 1); + result.RedirectingTo = redirval; + // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature + // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally + // it shoudln't include service unavailable..only if we have this redirect header. + response.StatusCode = System.Net.HttpStatusCode.Redirect; + redirtime = Int32.Parse(value.Substring(0, end)); + System.Threading.Thread.Sleep(redirtime * 1000); + } + } + } + } + if (response.Headers.Location != null) + { + result.RedirectingTo = response.Headers.Location.ToString(); + } + // Mono won't add the baseurl to relative redirects. + // e.g. a "Location: /index.php" header will result in the Uri "file:///index.php" + // See issue #1200 + if (result.RedirectingTo != null && result.RedirectingTo.StartsWith("file://")) + { + // URL decoding apparently is needed to, without it e.g. Demonoid download is broken + // TODO: is it always needed (not just for relative redirects)? + var newRedirectingTo = WebUtilityHelpers.UrlDecode(result.RedirectingTo, webRequest.Encoding); + newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + "://" + request.RequestUri.Host); + logger.Debug("[MONO relative redirect bug] Rewriting relative redirect URL from " + result.RedirectingTo + " to " + newRedirectingTo); + result.RedirectingTo = newRedirectingTo; + } + result.Status = response.StatusCode; + + // Compatiblity issue between the cookie format and httpclient + // Pull it out manually ignoring the expiry date then set it manually + // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer + IEnumerable cookieHeaders; + var responseCookies = new List>(); + + if (response.Headers.TryGetValues("set-cookie", out cookieHeaders)) + { + foreach (var value in cookieHeaders) + { + var nameSplit = value.IndexOf('='); + if (nameSplit > -1) + { + responseCookies.Add(new Tuple(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';'))) + ";")); + } + } + + var cookieBuilder = new StringBuilder(); + foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1)) + { + cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2); + } + result.Cookies = cookieBuilder.ToString().Trim(); + } + ServerUtil.ResureRedirectIsFullyQualified(webRequest, result); + return result; + } + } + } + } + } + } + + override public void AddTrustedCertificate(string host, string hash) + { + hash = hash.ToUpper(); + ICollection hosts; + trustedCertificates.TryGetValue(hash.ToUpper(), out hosts); + if (hosts == null) + { + hosts = new HashSet(); + trustedCertificates[hash] = hosts; + } + hosts.Add(host); + } + } +} diff --git a/src/Jackett.Server/Initialisation.cs b/src/Jackett.Server/Initialisation.cs index c9772072b..a0b5e5f82 100644 --- a/src/Jackett.Server/Initialisation.cs +++ b/src/Jackett.Server/Initialisation.cs @@ -13,7 +13,7 @@ namespace Jackett.Server { public static void ProcessSettings(RuntimeSettings runtimeSettings, Logger logger) { - if (runtimeSettings.ClientOverride != "httpclient" && runtimeSettings.ClientOverride != "httpclient2") + if (runtimeSettings.ClientOverride != "httpclient" && runtimeSettings.ClientOverride != "httpclient2" && runtimeSettings.ClientOverride != "httpclientnetcore") { logger.Error($"Client override ({runtimeSettings.ClientOverride}) has been deprecated, please remove it from your start arguments"); Environment.Exit(1); @@ -37,7 +37,7 @@ namespace Jackett.Server if (runtimeSettings.IgnoreSslErrors == true) { - logger.Info("Jackett will ignore SSL certificate errors."); + logger.Error($"The IgnoreSslErrors option has been deprecated, please remove it from your start arguments"); } if (!string.IsNullOrWhiteSpace(runtimeSettings.CustomDataFolder)) diff --git a/src/Jackett.Server/Program.cs b/src/Jackett.Server/Program.cs index 6052841a3..73178ba42 100644 --- a/src/Jackett.Server/Program.cs +++ b/src/Jackett.Server/Program.cs @@ -14,7 +14,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; - +using System.Runtime.InteropServices; namespace Jackett.Server { @@ -48,7 +48,16 @@ namespace Jackett.Server if (string.IsNullOrEmpty(options.Client)) { //TODO: Remove libcurl once off owin - options.Client = "httpclient"; + bool runningOnDotNetCore = RuntimeInformation.FrameworkDescription.IndexOf("Core", StringComparison.OrdinalIgnoreCase) >= 0; + + if (runningOnDotNetCore) + { + options.Client = "httpclientnetcore"; + } + else + { + options.Client = "httpclient"; + } } Settings = options.ToRunTimeSettings(); diff --git a/src/Jackett.Server/Startup.cs b/src/Jackett.Server/Startup.cs index fbc8b72d3..729e84587 100644 --- a/src/Jackett.Server/Startup.cs +++ b/src/Jackett.Server/Startup.cs @@ -3,6 +3,7 @@ using Autofac.Extensions.DependencyInjection; using Jackett.Common.Models.Config; using Jackett.Common.Plumbing; using Jackett.Common.Services.Interfaces; +using Jackett.Common.Utils.Clients; using Jackett.Server.Middleware; using Jackett.Server.Services; using Microsoft.AspNetCore.Authentication.Cookies; @@ -82,6 +83,8 @@ namespace Jackett.Server builder.RegisterType().As(); builder.RegisterType().As(); builder.RegisterType().As(); + if (runtimeSettings.ClientOverride == "httpclientnetcore") + builder.RegisterType().As(); IContainer container = builder.Build(); Helper.ApplicationContainer = container;