From f7705963544d389ba852d924ab3ff7388a36e002 Mon Sep 17 00:00:00 2001 From: Diego Heras Date: Sat, 8 Feb 2020 07:03:03 +0100 Subject: [PATCH] core: fix form encoding. resolves #4346 resolves #3061 (#6994) --- .../Utils/Clients/HttpWebClient.cs | 2 +- .../Utils/Clients/HttpWebClient2.cs | 2 +- .../Utils/Clients/HttpWebClient2NetCore.cs | 4 +-- .../Utils/Clients/HttpWebClientNetCore.cs | 2 +- src/Jackett.Common/Utils/Clients/WebClient.cs | 31 +++++++++++++++++++ src/Jackett.Common/Utils/StringUtil.cs | 5 --- 6 files changed, 36 insertions(+), 10 deletions(-) diff --git a/src/Jackett.Common/Utils/Clients/HttpWebClient.cs b/src/Jackett.Common/Utils/Clients/HttpWebClient.cs index 701e3579a..fad7d5507 100644 --- a/src/Jackett.Common/Utils/Clients/HttpWebClient.cs +++ b/src/Jackett.Common/Utils/Clients/HttpWebClient.cs @@ -214,7 +214,7 @@ namespace Jackett.Common.Utils.Clients else if (webRequest.Type == RequestType.POST) { if (webRequest.PostData != null) - request.Content = new FormUrlEncodedContent(webRequest.PostData); + request.Content = FormUrlEncodedContentWithEncoding(webRequest.PostData, webRequest.Encoding); request.Method = HttpMethod.Post; } else diff --git a/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs b/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs index cbf9b70b2..dfe2c31d6 100644 --- a/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs +++ b/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs @@ -234,7 +234,7 @@ namespace Jackett.Common.Utils.Clients else if (webRequest.Type == RequestType.POST) { if (webRequest.PostData != null) - request.Content = new FormUrlEncodedContent(webRequest.PostData); + request.Content = FormUrlEncodedContentWithEncoding(webRequest.PostData, webRequest.Encoding); request.Method = HttpMethod.Post; } else diff --git a/src/Jackett.Common/Utils/Clients/HttpWebClient2NetCore.cs b/src/Jackett.Common/Utils/Clients/HttpWebClient2NetCore.cs index 4339c46ea..ebb0999c9 100644 --- a/src/Jackett.Common/Utils/Clients/HttpWebClient2NetCore.cs +++ b/src/Jackett.Common/Utils/Clients/HttpWebClient2NetCore.cs @@ -230,7 +230,7 @@ namespace Jackett.Common.Utils.Clients else if (webRequest.Type == RequestType.POST) { if (webRequest.PostData != null) - request.Content = new FormUrlEncodedContent(webRequest.PostData); + request.Content = FormUrlEncodedContentWithEncoding(webRequest.PostData, webRequest.Encoding); request.Method = HttpMethod.Post; } else @@ -250,7 +250,7 @@ namespace Jackett.Common.Utils.Clients } // some cloudflare clients are using a refresh header - // Pull it out manually + // Pull it out manually if (response.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh")) { var refreshHeaders = response.Headers.GetValues("Refresh"); diff --git a/src/Jackett.Common/Utils/Clients/HttpWebClientNetCore.cs b/src/Jackett.Common/Utils/Clients/HttpWebClientNetCore.cs index 722918991..93a6e01bb 100644 --- a/src/Jackett.Common/Utils/Clients/HttpWebClientNetCore.cs +++ b/src/Jackett.Common/Utils/Clients/HttpWebClientNetCore.cs @@ -213,7 +213,7 @@ namespace Jackett.Common.Utils.Clients else if (webRequest.Type == RequestType.POST) { if (webRequest.PostData != null) - request.Content = new FormUrlEncodedContent(webRequest.PostData); + request.Content = FormUrlEncodedContentWithEncoding(webRequest.PostData, webRequest.Encoding); request.Method = HttpMethod.Post; } else diff --git a/src/Jackett.Common/Utils/Clients/WebClient.cs b/src/Jackett.Common/Utils/Clients/WebClient.cs index 2ca428a62..1e39c30da 100644 --- a/src/Jackett.Common/Utils/Clients/WebClient.cs +++ b/src/Jackett.Common/Utils/Clients/WebClient.cs @@ -1,8 +1,11 @@ using System; using System.Collections.Generic; +using System.Net.Http; +using System.Net.Http.Headers; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; +using System.Web; using AutoMapper; using Jackett.Common.Models.Config; using Jackett.Common.Services.Interfaces; @@ -184,5 +187,33 @@ namespace Jackett.Common.Utils.Clients { // nothing by default } + + /** + * This method does the same as FormUrlEncodedContent but with custom encoding instead of utf-8 + * https://stackoverflow.com/a/13832544 + */ + protected static ByteArrayContent FormUrlEncodedContentWithEncoding( + IEnumerable> nameValueCollection, Encoding encoding) + { + // utf-8 / default + if (Encoding.UTF8.Equals(encoding) || encoding == null) + return new FormUrlEncodedContent(nameValueCollection); + + // other encodings + var builder = new StringBuilder(); + foreach (KeyValuePair pair in nameValueCollection) + { + if (builder.Length > 0) + builder.Append('&'); + builder.Append(HttpUtility.UrlEncode(pair.Key, encoding)); + builder.Append('='); + builder.Append(HttpUtility.UrlEncode(pair.Value, encoding)); + } + // HttpRuleParser.DefaultHttpEncoding == "latin1" + var data = Encoding.GetEncoding("latin1").GetBytes(builder.ToString()); + var content = new ByteArrayContent(data); + content.Headers.ContentType = new MediaTypeHeaderValue("application/x-www-form-urlencoded"); + return content; + } } } diff --git a/src/Jackett.Common/Utils/StringUtil.cs b/src/Jackett.Common/Utils/StringUtil.cs index 73c5fb83b..c19d58671 100644 --- a/src/Jackett.Common/Utils/StringUtil.cs +++ b/src/Jackett.Common/Utils/StringUtil.cs @@ -49,11 +49,6 @@ namespace Jackett.Common.Utils return Encoding.UTF8.GetString(Convert.FromBase64String(str)); } - public static string PostDataFromDict(IEnumerable> dict) - { - return new FormUrlEncodedContent(dict).ReadAsStringAsync().Result; - } - /// /// Convert an array of bytes to a string of hex digits ///