From e180b4bfc2bb2c2dd09ed07558c03c985aca1073 Mon Sep 17 00:00:00 2001 From: flightlevel Date: Sat, 18 Aug 2018 16:44:58 +1000 Subject: [PATCH] Remove references to CurlSharp Dead code since upgrade to Jackett.Server --- src/Jackett.Common/CurlHelper.cs | 282 ------------------ src/Jackett.Common/CurlSharp.dll.config | 7 - src/Jackett.Common/Jackett.Common.csproj | 1 - .../Utils/Clients/UnixLibCurlWebClient.cs | 167 ----------- .../Utils/Clients/UnixSafeCurlWebClient.cs | 179 ----------- 5 files changed, 636 deletions(-) delete mode 100644 src/Jackett.Common/CurlHelper.cs delete mode 100644 src/Jackett.Common/CurlSharp.dll.config delete mode 100644 src/Jackett.Common/Utils/Clients/UnixLibCurlWebClient.cs delete mode 100644 src/Jackett.Common/Utils/Clients/UnixSafeCurlWebClient.cs diff --git a/src/Jackett.Common/CurlHelper.cs b/src/Jackett.Common/CurlHelper.cs deleted file mode 100644 index acde3b001..000000000 --- a/src/Jackett.Common/CurlHelper.cs +++ /dev/null @@ -1,282 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Text; -using System.Threading.Tasks; -using CurlSharp; -using CurlSharp.Enums; -using Jackett.Common.Models.Config; -using Jackett.Common.Utils; - -namespace Jackett.Common -{ - public class CurlHelper - { - private static readonly object instance = new object(); - - public class CurlRequest - { - public string Url { get; private set; } - public string Cookies { get; private set; } - public string Referer { get; private set; } - public HttpMethod Method { get; private set; } - public IEnumerable> PostData { get; set; } - public Dictionary Headers { get; set; } - public string RawPOSTDdata { get; set; } - - public CurlRequest(HttpMethod method, string url, string cookies = null, string referer = null, Dictionary headers = null, string rawPOSTData = null) - { - Method = method; - Url = url.Replace(" ", "+"); // avoids bad request to cloudflare for urls containing a space followed by H (" H") - Cookies = cookies; - Referer = referer; - Headers = headers; - RawPOSTDdata = rawPOSTData; - } - } - - public class CurlResponse - { - public List HeaderList { get; private set; } - public byte[] Content { get; private set; } - public HttpStatusCode Status { get; private set; } - public string Cookies { set; get; } - - public CurlResponse(List headers, byte[] content, HttpStatusCode s, string cookies) - { - HeaderList = headers; - Content = content; - Status = s; - Cookies = cookies; - } - } - - public static async Task GetAsync(string url, ServerConfig config, string cookies = null, string referer = null, Dictionary headers = null) - { - var curlRequest = new CurlRequest(HttpMethod.Get, url, cookies, referer, headers); - var result = await PerformCurlAsync(curlRequest, config); - return result; - } - - public static async Task PostAsync(string url, ServerConfig config, IEnumerable> formData, string cookies = null, string referer = null, Dictionary headers = null, string rawPostData = null) - { - var curlRequest = new CurlRequest(HttpMethod.Post, url, cookies, referer, headers); - curlRequest.PostData = formData; - curlRequest.RawPOSTDdata = rawPostData; - var result = await PerformCurlAsync(curlRequest, config); - return result; - } - - public static async Task PerformCurlAsync(CurlRequest curlRequest, ServerConfig config) - { - return await Task.Run(() => PerformCurl(curlRequest, config)); - } - - public delegate void ErrorMessage(string s); - public static ErrorMessage OnErrorMessage; - - public static CurlResponse PerformCurl(CurlRequest curlRequest, ServerConfig config) - { - lock (instance) - { - var headerBuffers = new List(); - var contentBuffers = new List(); - - using (var easy = new CurlEasy()) - { - easy.Url = curlRequest.Url; - easy.BufferSize = 64 * 1024; - easy.UserAgent = BrowserUtil.ChromeUserAgent; - easy.FollowLocation = false; - easy.ConnectTimeout = 20; - if (curlRequest.Headers != null) - { - CurlSlist curlHeaders = new CurlSlist(); - foreach (var header in curlRequest.Headers) - { - curlHeaders.Append(header.Key + ": " + header.Value); - } - easy.SetOpt(CurlOption.HttpHeader, curlHeaders); - } - - easy.WriteFunction = (byte[] buf, int size, int nmemb, object data) => - { - contentBuffers.Add(buf); - return size * nmemb; - }; - - easy.HeaderFunction = (byte[] buf, int size, int nmemb, object extraData) => - { - headerBuffers.Add(buf); - return size * nmemb; - }; - - if (!string.IsNullOrEmpty(curlRequest.Cookies)) - easy.Cookie = curlRequest.Cookies; - - if (!string.IsNullOrEmpty(curlRequest.Referer)) - easy.Referer = curlRequest.Referer; - - if (curlRequest.Method == HttpMethod.Post) - { - if (!string.IsNullOrEmpty(curlRequest.RawPOSTDdata)) - { - easy.Post = true; - easy.PostFields = curlRequest.RawPOSTDdata; - easy.PostFieldSize = Encoding.UTF8.GetByteCount(curlRequest.RawPOSTDdata); - } - else - { - easy.Post = true; - var postString = StringUtil.PostDataFromDict(curlRequest.PostData); - easy.PostFields = postString; - easy.PostFieldSize = Encoding.UTF8.GetByteCount(postString); - } - } - - if (config.RuntimeSettings.DoSSLFix == true) - { - // http://stackoverflow.com/questions/31107851/how-to-fix-curl-35-cannot-communicate-securely-with-peer-no-common-encryptio - // https://git.fedorahosted.org/cgit/mod_nss.git/plain/docs/mod_nss.html - easy.SslCipherList = SSLFix.CipherList; - easy.FreshConnect = true; - easy.ForbidReuse = true; - } - - if (config.RuntimeSettings.IgnoreSslErrors == true) - { - easy.SetOpt(CurlOption.SslVerifyhost, false); - easy.SetOpt(CurlOption.SslVerifyPeer, false); - } - - var proxy = config.GetProxyUrl(); - if (proxy != null) - { - easy.SetOpt(CurlOption.HttpProxyTunnel, 1); - easy.SetOpt(CurlOption.Proxy, proxy); - - var authString = config.GetProxyAuthString(); - if (authString != null) - { - easy.SetOpt(CurlOption.ProxyUserPwd, authString); - } - } - - easy.Perform(); - - if (easy.LastErrorCode != CurlCode.Ok) - { - var message = "Error " + easy.LastErrorCode.ToString() + " " + easy.LastErrorDescription + " " + easy.ErrorBuffer; - if (null != OnErrorMessage) - OnErrorMessage(message); - else - Console.WriteLine(message); - } - } - - var headerBytes = Combine(headerBuffers.ToArray()); - var headerString = Encoding.UTF8.GetString(headerBytes); - if (config.GetProxyUrl() != null) - { - var firstcrlf = headerString.IndexOf("\r\n\r\n"); - var secondcrlf = headerString.IndexOf("\r\n\r\n", firstcrlf + 1); - if (secondcrlf > 0) - { - headerString = headerString.Substring(firstcrlf + 4, secondcrlf - (firstcrlf)); - } - } - var headerParts = headerString.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries); - var headers = new List(); - var headerCount = 0; - HttpStatusCode status = HttpStatusCode.NotImplemented; - var cookieBuilder = new StringBuilder(); - var cookies = new List>(); - foreach (var headerPart in headerParts) - { - if (headerCount == 0) - { - var split = headerPart.Split(' '); - if (split.Length < 2) - throw new Exception("HTTP Header missing"); - var responseCode = int.Parse(headerPart.Split(' ')[1]); - status = (HttpStatusCode)responseCode; - } - else - { - var keyVal = headerPart.Split(new char[] { ':' }, 2); - if (keyVal.Length > 1) - { - var key = keyVal[0].ToLower().Trim(); - var value = keyVal[1].Trim(); - - if (key == "set-cookie") - { - var nameSplit = value.IndexOf('='); - if (nameSplit > -1) - { - var cKey = value.Substring(0, nameSplit); - var cVal = value.Split(';')[0] + ";"; - cookies.Add(new Tuple(cKey, cVal)); - } - } - else - { - headers.Add(new[] { key, value }); - } - } - } - - headerCount++; - } - - foreach (var cookieGroup in cookies.GroupBy(c => c.Item1)) - { - cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2); - } - - // add some debug output to track down the problem causing people getting InternalServerError results - if (status == HttpStatusCode.NotImplemented || status == HttpStatusCode.InternalServerError) - { - try - { - OnErrorMessage("got NotImplemented/InternalServerError"); - OnErrorMessage("request.Method: " + curlRequest.Method); - OnErrorMessage("request.Url: " + curlRequest.Url); - OnErrorMessage("request.Cookies: " + curlRequest.Cookies); - OnErrorMessage("request.Referer: " + curlRequest.Referer); - OnErrorMessage("request.RawPOSTDdata: " + curlRequest.RawPOSTDdata); - OnErrorMessage("cookies: " + cookieBuilder.ToString().Trim()); - OnErrorMessage("headerString:\n" + headerString); - - foreach (var headerPart in headerParts) - { - OnErrorMessage("headerParts: " + headerPart); - } - } - catch (Exception ex) - { - OnErrorMessage(string.Format("CurlHelper: error while handling NotImplemented/InternalServerError:\n{0}", ex)); - } - } - - var contentBytes = Combine(contentBuffers.ToArray()); - var curlResponse = new CurlResponse(headers, contentBytes, status, cookieBuilder.ToString().Trim()); - return curlResponse; - } - } - - public static byte[] Combine(params byte[][] arrays) - { - byte[] ret = new byte[arrays.Sum(x => x.Length)]; - int offset = 0; - foreach (byte[] data in arrays) - { - Buffer.BlockCopy(data, 0, ret, offset, data.Length); - offset += data.Length; - } - return ret; - } - } -} diff --git a/src/Jackett.Common/CurlSharp.dll.config b/src/Jackett.Common/CurlSharp.dll.config deleted file mode 100644 index eadeb9ad7..000000000 --- a/src/Jackett.Common/CurlSharp.dll.config +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - diff --git a/src/Jackett.Common/Jackett.Common.csproj b/src/Jackett.Common/Jackett.Common.csproj index 2757cf7d5..8597478bd 100644 --- a/src/Jackett.Common/Jackett.Common.csproj +++ b/src/Jackett.Common/Jackett.Common.csproj @@ -128,7 +128,6 @@ - diff --git a/src/Jackett.Common/Utils/Clients/UnixLibCurlWebClient.cs b/src/Jackett.Common/Utils/Clients/UnixLibCurlWebClient.cs deleted file mode 100644 index 5969accaa..000000000 --- a/src/Jackett.Common/Utils/Clients/UnixLibCurlWebClient.cs +++ /dev/null @@ -1,167 +0,0 @@ -using System; -using System.Linq; -using System.Net; -using System.Text; -using System.Threading.Tasks; -using CloudFlareUtilities; -using CurlSharp; -using CurlSharp.Enums; -using Jackett.Common.Models.Config; -using Jackett.Common.Services.Interfaces; -using NLog; - -namespace Jackett.Common.Utils.Clients -{ - public class UnixLibCurlWebClient : WebClient - { - public UnixLibCurlWebClient(IProcessService p, Logger l, IConfigurationService c, ServerConfig sc) - : base(p: p, - l: l, - c: c, - sc: sc) - { - } - - private string CloudFlareChallengeSolverSolve(string challengePageContent, Uri uri) - { - var solution = ChallengeSolver.Solve(challengePageContent, uri.Host); - string clearanceUri = uri.Scheme + Uri.SchemeDelimiter + uri.Host + ":" + uri.Port + solution.ClearanceQuery; - return clearanceUri; - } - - override public void Init() - { - try - { - logger.Info("LibCurl init " + Curl.GlobalInit(CurlInitFlag.All).ToString()); - CurlHelper.OnErrorMessage += (msg) => - { - logger.Error(msg); - }; - } - catch (Exception e) - { - logger.Warn("Libcurl failed to initalize. Did you install it?"); - logger.Warn("Debian: apt-get install libcurl4-openssl-dev"); - logger.Warn("Redhat: yum install libcurl-devel"); - throw e; - } - - var version = Curl.Version; - logger.Info("LibCurl version " + version); - - if (!serverConfig.RuntimeSettings.DoSSLFix.HasValue && version.IndexOf("NSS") > -1) - { - logger.Info("NSS Detected SSL ECC workaround enabled."); - serverConfig.RuntimeSettings.DoSSLFix = true; - } - } - - // Wrapper for Run which takes care of CloudFlare challenges, calls RunCurl - override protected async Task Run(WebRequest request) - { - WebClientByteResult result = await RunCurl(request); - - // check if we've received a CloudFlare challenge - string[] server; - if (result.Status == HttpStatusCode.ServiceUnavailable && result.Headers.TryGetValue("server", out server) && (server[0] == "cloudflare-nginx" || server[0] == "cloudflare")) - { - logger.Info("UnixLibCurlWebClient: Received a new CloudFlare challenge"); - - // solve the challenge - string pageContent = Encoding.UTF8.GetString(result.Content); - Uri uri = new Uri(request.Url); - string clearanceUri = CloudFlareChallengeSolverSolve(pageContent, uri); - logger.Info(string.Format("UnixLibCurlWebClient: CloudFlare clearanceUri: {0}", clearanceUri)); - - // wait... - await Task.Delay(5000); - - // request clearanceUri to get cf_clearance cookie - var response = await CurlHelper.GetAsync(clearanceUri, serverConfig, request.Cookies, request.Referer); - logger.Info(string.Format("UnixLibCurlWebClient: received CloudFlare clearance cookie: {0}", response.Cookies)); - - // add new cf_clearance cookies to the original request - request.Cookies = response.Cookies + request.Cookies; - - // re-run the original request with updated cf_clearance cookie - result = await RunCurl(request); - - // add cf_clearance cookie to the final result so we update the config for the next request - result.Cookies = response.Cookies + " " + result.Cookies; - } - return result; - } - - protected async Task RunCurl(WebRequest request) - { - CurlHelper.CurlResponse response; - if (request.Type == RequestType.GET) - { - response = await CurlHelper.GetAsync(request.Url, serverConfig, request.Cookies, request.Referer, request.Headers); - } - else - { - if (!string.IsNullOrEmpty(request.RawBody)) - { - logger.Debug("UnixLibCurlWebClient: Posting " + request.RawBody); - } - else if (request.PostData != null && request.PostData.Count() > 0) - { - logger.Debug("UnixLibCurlWebClient: Posting " + StringUtil.PostDataFromDict(request.PostData)); - } - - response = await CurlHelper.PostAsync(request.Url, serverConfig, request.PostData, request.Cookies, request.Referer, request.Headers, request.RawBody); - } - - var result = new WebClientByteResult() - { - Content = response.Content, - Cookies = response.Cookies, - Status = response.Status - }; - - if (response.HeaderList != null) - { - foreach (var header in response.HeaderList) - { - var key = header[0].ToLowerInvariant(); - - result.Headers[key] = new string[] { header[1] }; // doesn't support multiple identical headers? - - switch (key) - { - case "location": - result.RedirectingTo = header[1]; - break; - case "refresh": - if (response.Status == System.Net.HttpStatusCode.ServiceUnavailable) - { - //"Refresh: 8;URL=/cdn-cgi/l/chk_jschl?pass=1451000679.092-1vJFUJLb9R" - var redirval = ""; - var value = header[1]; - var start = value.IndexOf("="); - var end = value.IndexOf(";"); - var len = value.Length; - if (start > -1) - { - redirval = value.Substring(start + 1); - result.RedirectingTo = redirval; - // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature - // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally - // it shoudln't include service unavailable..only if we have this redirect header. - result.Status = System.Net.HttpStatusCode.Redirect; - var redirtime = Int32.Parse(value.Substring(0, end)); - System.Threading.Thread.Sleep(redirtime * 1000); - } - } - break; - } - } - } - - ServerUtil.ResureRedirectIsFullyQualified(request, result); - return result; - } - } -} diff --git a/src/Jackett.Common/Utils/Clients/UnixSafeCurlWebClient.cs b/src/Jackett.Common/Utils/Clients/UnixSafeCurlWebClient.cs deleted file mode 100644 index dece80ce7..000000000 --- a/src/Jackett.Common/Utils/Clients/UnixSafeCurlWebClient.cs +++ /dev/null @@ -1,179 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Text; -using System.Threading.Tasks; -using CurlSharp; -using Jackett.Common.Models.Config; -using Jackett.Common.Services.Interfaces; -using NLog; - -namespace Jackett.Common.Utils.Clients -{ - public class UnixSafeCurlWebClient : WebClient - { - public UnixSafeCurlWebClient(IProcessService p, Logger l, IConfigurationService c, ServerConfig sc) - : base(p: p, - l: l, - c: c, - sc: sc) - { - - } - - override public void Init() - { - } - - override protected async Task Run(WebRequest request) - { - var args = new StringBuilder(); - var proxy = serverConfig.GetProxyUrl(true); - if (proxy != null) - { - args.AppendFormat("-x '" + proxy + "' "); - } - - args.AppendFormat("--url \"{0}\" ", request.Url); - - if (request.EmulateBrowser == true) - args.AppendFormat("-i -sS --user-agent \"{0}\" ", BrowserUtil.ChromeUserAgent); - else - args.AppendFormat("-i -sS --user-agent \"{0}\" ", "Jackett/" + configService.GetVersion()); - - if (!string.IsNullOrWhiteSpace(request.Cookies)) - { - args.AppendFormat("--cookie \"{0}\" ", request.Cookies); - } - - if (!string.IsNullOrWhiteSpace(request.Referer)) - { - args.AppendFormat("--referer \"{0}\" ", request.Referer); - } - - if (!string.IsNullOrEmpty(request.RawBody)) - { - var postString = StringUtil.PostDataFromDict(request.PostData); - args.AppendFormat("--data \"{0}\" ", request.RawBody.Replace("\"", "\\\"")); - } - else if (request.PostData != null && request.PostData.Count() > 0) - { - var postString = StringUtil.PostDataFromDict(request.PostData); - args.AppendFormat("--data \"{0}\" ", postString); - } - - var tempFile = Path.GetTempFileName(); - args.AppendFormat("--output \"{0}\" ", tempFile); - - if (serverConfig.RuntimeSettings.DoSSLFix == true) - { - // http://stackoverflow.com/questions/31107851/how-to-fix-curl-35-cannot-communicate-securely-with-peer-no-common-encryptio - // https://git.fedorahosted.org/cgit/mod_nss.git/plain/docs/mod_nss.html - args.Append("--cipher " + SSLFix.CipherList); - } - if (serverConfig.RuntimeSettings.IgnoreSslErrors == true) - { - args.Append("-k "); - } - args.Append("-H \"Accept-Language: en-US,en\" "); - args.Append("-H \"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\" "); - string stdout = null; - await Task.Run(() => - { - stdout = processService.StartProcessAndGetOutput(System.Environment.OSVersion.Platform == PlatformID.Unix ? "curl" : "curl.exe", args.ToString(), true); - }); - - var outputData = File.ReadAllBytes(tempFile); - File.Delete(tempFile); - stdout = Encoding.UTF8.GetString(outputData); - var result = new WebClientByteResult(); - var headSplit = stdout.IndexOf("\r\n\r\n"); - if (headSplit < 0) - throw new Exception("Invalid response"); - var headers = stdout.Substring(0, headSplit); - if (serverConfig.RuntimeSettings.ProxyConnection != null) - { - // the proxy provided headers too so we need to split headers again - var headSplit1 = stdout.IndexOf("\r\n\r\n", headSplit + 4); - if (headSplit1 > 0) - { - headers = stdout.Substring(headSplit + 4, headSplit1 - (headSplit + 4)); - headSplit = headSplit1; - } - } - var headerCount = 0; - var cookieBuilder = new StringBuilder(); - var cookies = new List>(); - - foreach (var header in headers.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries)) - { - if (headerCount == 0) - { - var responseCode = int.Parse(header.Split(' ')[1]); - result.Status = (HttpStatusCode)responseCode; - } - else - { - var headerSplitIndex = header.IndexOf(':'); - if (headerSplitIndex > 0) - { - var name = header.Substring(0, headerSplitIndex).ToLowerInvariant(); - var value = header.Substring(headerSplitIndex + 1); - switch (name) - { - case "set-cookie": - var nameSplit = value.IndexOf('='); - if (nameSplit > -1) - { - cookies.Add(new Tuple(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') + 1))); - } - break; - case "location": - result.RedirectingTo = value.Trim(); - break; - case "refresh": - //"Refresh: 8;URL=/cdn-cgi/l/chk_jschl?pass=1451000679.092-1vJFUJLb9R" - var redirval = ""; - var start = value.IndexOf("="); - var end = value.IndexOf(";"); - var len = value.Length; - if (start > -1) - { - redirval = value.Substring(start + 1); - result.RedirectingTo = redirval; - // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature - // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally - // it shoudln't include service unavailable..only if we have this redirect header. - result.Status = System.Net.HttpStatusCode.Redirect; - var redirtime = Int32.Parse(value.Substring(0, end)); - System.Threading.Thread.Sleep(redirtime * 1000); - } - break; - } - } - } - headerCount++; - } - - foreach (var cookieGroup in cookies.GroupBy(c => c.Item1)) - { - cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2); - } - - result.Cookies = cookieBuilder.ToString().Trim(); - result.Content = new byte[outputData.Length - (headSplit + 3)]; - var dest = 0; - for (int i = headSplit + 4; i < outputData.Length; i++) - { - result.Content[dest] = outputData[i]; - dest++; - } - - logger.Debug("WebClientByteResult returned " + result.Status); - ServerUtil.ResureRedirectIsFullyQualified(request, result); - return result; - } - } -}