diff --git a/src/Jackett.Common/Utils/Clients/BaseWebResult.cs b/src/Jackett.Common/Utils/Clients/BaseWebResult.cs index cf47df862..940fb9655 100644 --- a/src/Jackett.Common/Utils/Clients/BaseWebResult.cs +++ b/src/Jackett.Common/Utils/Clients/BaseWebResult.cs @@ -1,12 +1,10 @@ using System.Collections.Generic; using System.Net; -using System.Text; namespace Jackett.Common.Utils.Clients { public abstract class BaseWebResult { - public Encoding Encoding { get; set; } public HttpStatusCode Status { get; set; } public string Cookies { get; set; } public string RedirectingTo { get; set; } diff --git a/src/Jackett.Common/Utils/Clients/HttpWebClient.cs b/src/Jackett.Common/Utils/Clients/HttpWebClient.cs index 8bc4a4185..a35e20cb2 100644 --- a/src/Jackett.Common/Utils/Clients/HttpWebClient.cs +++ b/src/Jackett.Common/Utils/Clients/HttpWebClient.cs @@ -7,7 +7,6 @@ using System.Net.Http; using System.Net.Security; using System.Security.Cryptography.X509Certificates; using System.Text; -using System.Text.RegularExpressions; using System.Threading.Tasks; using CloudflareSolverRe; using com.LandonKey.SocksWebProxy; @@ -298,40 +297,6 @@ namespace Jackett.Common.Utils.Clients result.Cookies = cookieBuilder.ToString().Trim(); } ServerUtil.ResureRedirectIsFullyQualified(webRequest, result); - Encoding encoding = null; - if (webRequest.Encoding != null) - { - encoding = webRequest.Encoding; - } - else if (result.Headers.ContainsKey("content-type")) - { - var CharsetRegex = new Regex(@"charset=([\w-]+)", RegexOptions.Compiled); - var CharsetRegexMatch = CharsetRegex.Match(result.Headers["content-type"][0]); - if (CharsetRegexMatch.Success) - { - var charset = CharsetRegexMatch.Groups[1].Value; - try - { - encoding = Encoding.GetEncoding(charset); - } - catch (Exception ex) - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Error loading encoding {2} based on header {3}: {4}", ClientType, webRequest.Url, charset, result.Headers["content-type"][0], ex)); - } - } - else - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Got header without charset: {2}", ClientType, webRequest.Url, result.Headers["content-type"][0])); - } - } - - if (encoding == null) - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): No encoding detected, defaulting to UTF-8", ClientType, webRequest.Url)); - encoding = Encoding.UTF8; - } - - result.Encoding = encoding; return result; } } diff --git a/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs b/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs index 367288e84..dfc91165d 100644 --- a/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs +++ b/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs @@ -7,7 +7,6 @@ using System.Net.Http; using System.Net.Security; using System.Security.Cryptography.X509Certificates; using System.Text; -using System.Text.RegularExpressions; using System.Threading.Tasks; using CloudflareSolverRe; using com.LandonKey.SocksWebProxy; @@ -318,40 +317,6 @@ namespace Jackett.Common.Utils.Clients result.Cookies = cookieBuilder.ToString().Trim(); } ServerUtil.ResureRedirectIsFullyQualified(webRequest, result); - Encoding encoding = null; - if (webRequest.Encoding != null) - { - encoding = webRequest.Encoding; - } - else if (result.Headers.ContainsKey("content-type")) - { - var CharsetRegex = new Regex(@"charset=([\w-]+)", RegexOptions.Compiled); - var CharsetRegexMatch = CharsetRegex.Match(result.Headers["content-type"][0]); - if (CharsetRegexMatch.Success) - { - var charset = CharsetRegexMatch.Groups[1].Value; - try - { - encoding = Encoding.GetEncoding(charset); - } - catch (Exception ex) - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Error loading encoding {2} based on header {3}: {4}", ClientType, webRequest.Url, charset, result.Headers["content-type"][0], ex)); - } - } - else - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Got header without charset: {2}", ClientType, webRequest.Url, result.Headers["content-type"][0])); - } - } - - if (encoding == null) - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): No encoding detected, defaulting to UTF-8", ClientType, webRequest.Url)); - encoding = Encoding.UTF8; - } - - result.Encoding = encoding; return result; } diff --git a/src/Jackett.Common/Utils/Clients/HttpWebClient2NetCore.cs b/src/Jackett.Common/Utils/Clients/HttpWebClient2NetCore.cs index 796b70a9c..6e6e33aab 100644 --- a/src/Jackett.Common/Utils/Clients/HttpWebClient2NetCore.cs +++ b/src/Jackett.Common/Utils/Clients/HttpWebClient2NetCore.cs @@ -7,7 +7,6 @@ using System.Net.Http; using System.Net.Security; using System.Security.Cryptography.X509Certificates; using System.Text; -using System.Text.RegularExpressions; using System.Threading.Tasks; using CloudflareSolverRe; using com.LandonKey.SocksWebProxy; @@ -314,40 +313,6 @@ namespace Jackett.Common.Utils.Clients result.Cookies = cookieBuilder.ToString().Trim(); } ServerUtil.ResureRedirectIsFullyQualified(webRequest, result); - Encoding encoding = null; - if (webRequest.Encoding != null) - { - encoding = webRequest.Encoding; - } - else if (result.Headers.ContainsKey("content-type")) - { - var CharsetRegex = new Regex(@"charset=([\w-]+)", RegexOptions.Compiled); - var CharsetRegexMatch = CharsetRegex.Match(result.Headers["content-type"][0]); - if (CharsetRegexMatch.Success) - { - var charset = CharsetRegexMatch.Groups[1].Value; - try - { - encoding = Encoding.GetEncoding(charset); - } - catch (Exception ex) - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Error loading encoding {2} based on header {3}: {4}", ClientType, webRequest.Url, charset, result.Headers["content-type"][0], ex)); - } - } - else - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Got header without charset: {2}", ClientType, webRequest.Url, result.Headers["content-type"][0])); - } - } - - if (encoding == null) - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): No encoding detected, defaulting to UTF-8", ClientType, webRequest.Url)); - encoding = Encoding.UTF8; - } - - result.Encoding = encoding; return result; } diff --git a/src/Jackett.Common/Utils/Clients/HttpWebClientNetCore.cs b/src/Jackett.Common/Utils/Clients/HttpWebClientNetCore.cs index 8262744de..60555a984 100644 --- a/src/Jackett.Common/Utils/Clients/HttpWebClientNetCore.cs +++ b/src/Jackett.Common/Utils/Clients/HttpWebClientNetCore.cs @@ -7,7 +7,6 @@ using System.Net.Http; using System.Net.Security; using System.Security.Cryptography.X509Certificates; using System.Text; -using System.Text.RegularExpressions; using System.Threading.Tasks; using CloudflareSolverRe; using com.LandonKey.SocksWebProxy; @@ -297,40 +296,6 @@ namespace Jackett.Common.Utils.Clients result.Cookies = cookieBuilder.ToString().Trim(); } ServerUtil.ResureRedirectIsFullyQualified(webRequest, result); - Encoding encoding = null; - if (webRequest.Encoding != null) - { - encoding = webRequest.Encoding; - } - else if (result.Headers.ContainsKey("content-type")) - { - var CharsetRegex = new Regex(@"charset=([\w-]+)", RegexOptions.Compiled); - var CharsetRegexMatch = CharsetRegex.Match(result.Headers["content-type"][0]); - if (CharsetRegexMatch.Success) - { - var charset = CharsetRegexMatch.Groups[1].Value; - try - { - encoding = Encoding.GetEncoding(charset); - } - catch (Exception ex) - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Error loading encoding {2} based on header {3}: {4}", ClientType, webRequest.Url, charset, result.Headers["content-type"][0], ex)); - } - } - else - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Got header without charset: {2}", ClientType, webRequest.Url, result.Headers["content-type"][0])); - } - } - - if (encoding == null) - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): No encoding detected, defaulting to UTF-8", ClientType, webRequest.Url)); - encoding = Encoding.UTF8; - } - - result.Encoding = encoding; return result; } } diff --git a/src/Jackett.Common/Utils/Clients/WebClient.cs b/src/Jackett.Common/Utils/Clients/WebClient.cs index 2b0dca34a..4eed61df6 100644 --- a/src/Jackett.Common/Utils/Clients/WebClient.cs +++ b/src/Jackett.Common/Utils/Clients/WebClient.cs @@ -115,10 +115,42 @@ namespace Jackett.Common.Utils.Clients lastRequest = DateTime.Now; result.Request = request; var stringResult = Mapper.Map(result); + Encoding encoding = null; + if (request.Encoding != null) + { + encoding = request.Encoding; + } + else if (result.Headers.ContainsKey("content-type")) + { + var CharsetRegex = new Regex(@"charset=([\w-]+)", RegexOptions.Compiled); + var CharsetRegexMatch = CharsetRegex.Match(result.Headers["content-type"][0]); + if (CharsetRegexMatch.Success) + { + var charset = CharsetRegexMatch.Groups[1].Value; + try + { + encoding = Encoding.GetEncoding(charset); + } + catch (Exception ex) + { + logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Error loading encoding {2} based on header {3}: {4}", ClientType, request.Url, charset, result.Headers["content-type"][0], ex)); + } + } + else + { + logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Got header without charset: {2}", ClientType, request.Url, result.Headers["content-type"][0])); + } + } + + if (encoding == null) + { + logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): No encoding detected, defaulting to UTF-8", ClientType, request.Url)); + encoding = Encoding.UTF8; + } string decodedContent = null; if (result.Content != null) - decodedContent = result.Encoding.GetString(result.Content); + decodedContent = encoding.GetString(result.Content); stringResult.ContentString = decodedContent; logger.Debug(string.Format("WebClient({0}): Returning {1} => {2}", ClientType, result.Status, (result.IsRedirect ? result.RedirectingTo + " " : "") + (decodedContent == null ? "" : decodedContent)));