Revert "core: refactor http webclient part 10 #8529 (#7729)"

This reverts commit 3a0ece162f.

src/Jackett.Common/Indexers/Abnormal.cs

@@ -548,7 +548,7 @@ namespace Jackett.Common.Indexers
         /// <returns>Results from query</returns>
         private async Task<string> queryTracker(string request)
         {
-            BaseWebResult results = null;
+            WebClientStringResult results = null;
 
             // Cache mode not enabled or cached file didn't exist for our query
             output("\nQuerying tracker for results....");

src/Jackett.Common/Indexers/AniDUB.cs

@@ -509,7 +509,7 @@ namespace Jackett.Common.Indexers
             return domDate.NodeValue.Trim();
         }
 
-        private bool IsAuthorized(BaseWebResult result) =>
+        private bool IsAuthorized(WebClientStringResult result) =>
             result.ContentString.Contains("index.php?action=logout");
 
         private IEnumerable<int> ParseCategories(Uri showUri)

src/Jackett.Common/Indexers/BJShare.cs

@@ -219,7 +219,7 @@ namespace Jackett.Common.Indexers
             return title;
         }
 
-        private bool IsSessionIsClosed(BaseWebResult result)
+        private bool IsSessionIsClosed(WebClientStringResult result)
         {
             return result.IsRedirect && result.RedirectingTo.Contains("login.php");
         }

src/Jackett.Common/Indexers/BaseIndexer.cs

@@ -412,7 +412,7 @@ namespace Jackett.Common.Indexers
             return response.ContentBytes;
         }
 
-        protected async Task<BaseWebResult> RequestBytesWithCookiesAndRetry(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable<KeyValuePair<string, string>> data = null)
+        protected async Task<WebClientByteResult> RequestBytesWithCookiesAndRetry(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable<KeyValuePair<string, string>> data = null)
         {
             Exception lastException = null;
             for (var i = 0; i < 3; i++)
@@ -432,7 +432,7 @@ namespace Jackett.Common.Indexers
             throw lastException;
         }
 
-        protected async Task<BaseWebResult> RequestStringWithCookies(string url, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null)
+        protected async Task<WebClientStringResult> RequestStringWithCookies(string url, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null)
         {
             var request = new Utils.Clients.WebRequest()
             {
@@ -452,7 +452,7 @@ namespace Jackett.Common.Indexers
             return result;
         }
 
-        protected async Task<BaseWebResult> RequestStringWithCookiesAndRetry(string url, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null)
+        protected async Task<WebClientStringResult> RequestStringWithCookiesAndRetry(string url, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null)
         {
             Exception lastException = null;
             for (var i = 0; i < 3; i++)
@@ -472,7 +472,7 @@ namespace Jackett.Common.Indexers
             throw lastException;
         }
 
-        protected virtual async Task<BaseWebResult> RequestBytesWithCookies(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable<KeyValuePair<string, string>> data = null, Dictionary<string, string> headers = null)
+        protected virtual async Task<WebClientByteResult> RequestBytesWithCookies(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable<KeyValuePair<string, string>> data = null, Dictionary<string, string> headers = null)
         {
             var request = new Utils.Clients.WebRequest()
             {
@@ -492,7 +492,7 @@ namespace Jackett.Common.Indexers
             return result;
         }
 
-        protected async Task<BaseWebResult> PostDataWithCookies(string url, IEnumerable<KeyValuePair<string, string>> data, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null, string rawbody = null, bool? emulateBrowser = null)
+        protected async Task<WebClientStringResult> PostDataWithCookies(string url, IEnumerable<KeyValuePair<string, string>> data, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null, string rawbody = null, bool? emulateBrowser = null)
         {
             var request = new Utils.Clients.WebRequest()
             {
@@ -514,7 +514,7 @@ namespace Jackett.Common.Indexers
             return result;
         }
 
-        protected async Task<BaseWebResult> PostDataWithCookiesAndRetry(string url, IEnumerable<KeyValuePair<string, string>> data, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null, string rawbody = null, bool? emulateBrowser = null)
+        protected async Task<WebClientStringResult> PostDataWithCookiesAndRetry(string url, IEnumerable<KeyValuePair<string, string>> data, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null, string rawbody = null, bool? emulateBrowser = null)
         {
             Exception lastException = null;
             for (var i = 0; i < 3; i++)
@@ -534,7 +534,7 @@ namespace Jackett.Common.Indexers
             throw lastException;
         }
 
-        protected async Task<BaseWebResult> RequestLoginAndFollowRedirect(string url, IEnumerable<KeyValuePair<string, string>> data, string cookies, bool returnCookiesFromFirstCall, string redirectUrlOverride = null, string referer = null, bool accumulateCookies = false)
+        protected async Task<WebClientStringResult> RequestLoginAndFollowRedirect(string url, IEnumerable<KeyValuePair<string, string>> data, string cookies, bool returnCookiesFromFirstCall, string redirectUrlOverride = null, string referer = null, bool accumulateCookies = false)
         {
             var request = new Utils.Clients.WebRequest()
             {
@@ -566,7 +566,7 @@ namespace Jackett.Common.Indexers
             return response;
         }
 
-        protected void CheckTrackerDown(BaseWebResult response)
+        protected void CheckTrackerDown(WebClientStringResult response)
         {
             if (response.Status == System.Net.HttpStatusCode.BadGateway
                 || response.Status == System.Net.HttpStatusCode.GatewayTimeout

src/Jackett.Common/Indexers/BitHDTV.cs

@@ -122,7 +122,7 @@ namespace Jackett.Common.Indexers
             {
                 {"cat", MapTorznabCapsToTrackers(query, true).FirstIfSingleOrDefault("0")}
             };
-            var results = new List<BaseWebResult>();
+            var results = new List<WebClientStringResult>();
             var search = new UriBuilder(SearchUrl);
             if (query.IsImdbQuery)
             {

src/Jackett.Common/Indexers/CardigannIndexer.cs

@@ -26,7 +26,7 @@ namespace Jackett.Common.Indexers
     public class CardigannIndexer : BaseWebIndexer
     {
         protected IndexerDefinition Definition;
-        protected BaseWebResult landingResult;
+        protected WebClientStringResult landingResult;
         protected IHtmlDocument landingResultDocument;
 
         protected List<string> DefaultCategories = new List<string>();
@@ -455,7 +455,7 @@ namespace Jackett.Common.Indexers
             return template;
         }
 
-        protected bool checkForError(BaseWebResult loginResult, IList<errorBlock> errorBlocks)
+        protected bool checkForError(WebClientStringResult loginResult, IList<errorBlock> errorBlocks)
         {
             if (loginResult.Status == HttpStatusCode.Unauthorized) // e.g. used by YGGtorrent
                 throw new ExceptionWithConfigData("401 Unauthorized, check your credentials", configData);
@@ -702,7 +702,7 @@ namespace Jackett.Common.Indexers
                 landingResult = null;
                 landingResultDocument = null;
 
-                BaseWebResult loginResult = null;
+                WebClientStringResult loginResult = null;
                 var enctype = form.GetAttribute("enctype");
                 if (enctype == "multipart/form-data")
                 {
@@ -1334,7 +1334,7 @@ namespace Jackett.Common.Indexers
                 logger.Info($"Fetching: {searchUrl}");
 
                 // send HTTP request
-                BaseWebResult response = null;
+                WebClientStringResult response = null;
                 Dictionary<string, string> headers = null;
                 if (Search.Headers != null)
                 {
@@ -1721,7 +1721,7 @@ namespace Jackett.Common.Indexers
             return releases;
         }
 
-        protected async Task<BaseWebResult> handleRequest(requestBlock request, Dictionary<string, object> variables = null, string referer = null)
+        protected async Task<WebClientByteResult> handleRequest(requestBlock request, Dictionary<string, object> variables = null, string referer = null)
         {
             var requestLinkStr = resolvePath(applyGoTemplateText(request.Path, variables)).ToString();
             logger.Debug($"CardigannIndexer ({Id}): handleRequest() requestLinkStr= {requestLinkStr}");

src/Jackett.Common/Indexers/Cinecalidad.cs

@@ -142,7 +142,7 @@ namespace Jackett.Common.Indexers
             return null;
         }
 
-        private List<ReleaseInfo> ParseReleases(BaseWebResult response, TorznabQuery query)
+        private List<ReleaseInfo> ParseReleases(WebClientStringResult response, TorznabQuery query)
         {
             var releases = new List<ReleaseInfo>();
 

src/Jackett.Common/Indexers/CorsaroRed.cs

@@ -107,7 +107,7 @@ namespace Jackett.Common.Indexers
             return IndexerConfigurationStatus.Completed;
         }
 
-        private dynamic CheckResponse(BaseWebResult result)
+        private dynamic CheckResponse(WebClientStringResult result)
         {
             try
             {

src/Jackett.Common/Indexers/DanishBits.cs

@@ -65,7 +65,7 @@ namespace Jackett.Common.Indexers
             return searchString;
         }
 
-        protected override async Task<BaseWebResult> RequestBytesWithCookies(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable<KeyValuePair<string, string>> data = null, Dictionary<string, string> headers = null)
+        protected override async Task<WebClientByteResult> RequestBytesWithCookies(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable<KeyValuePair<string, string>> data = null, Dictionary<string, string> headers = null)
         {
             CookieHeader = null; // Download fill fail with cookies set
             return await base.RequestBytesWithCookies(url, cookieOverride, method, referer, data, headers);

src/Jackett.Common/Indexers/InternetArchive.cs

@@ -150,7 +150,7 @@ namespace Jackett.Common.Indexers
             return releases;
         }
 
-        private JArray ParseResponse(BaseWebResult result)
+        private JArray ParseResponse(WebClientStringResult result)
         {
             try
             {

src/Jackett.Common/Indexers/LostFilm.cs

@@ -212,7 +212,7 @@ namespace Jackett.Common.Indexers
             }
         }
 
-        private async Task<BaseWebResult> RequestStringAndRelogin(string url)
+        private async Task<WebClientStringResult> RequestStringAndRelogin(string url)
         {
             var results = await RequestStringWithCookies(url);
             if (results.ContentString.Contains("503 Service"))

src/Jackett.Common/Indexers/NCore.cs

@@ -207,7 +207,7 @@ namespace Jackett.Common.Indexers
             return releases;
         }
 
-        private List<ReleaseInfo> ParseTorrents(BaseWebResult results, string episodeString, TorznabQuery query,
+        private List<ReleaseInfo> ParseTorrents(WebClientStringResult results, string episodeString, TorznabQuery query,
                                                 int alreadyFound, int limit, int previouslyParsedOnPage)
         {
             var releases = new List<ReleaseInfo>();

src/Jackett.Common/Indexers/NorBits.cs

@@ -459,9 +459,9 @@ namespace Jackett.Common.Indexers
         /// </summary>
         /// <param name="request">URL created by Query Builder</param>
         /// <returns>Results from query</returns>
-        private async Task<BaseWebResult> QueryExec(string request)
+        private async Task<WebClientStringResult> QueryExec(string request)
         {
-            BaseWebResult results;
+            WebClientStringResult results;
 
             // Switch in we are in DEV mode with Hard Drive Cache or not
             if (DevMode && CacheMode)
@@ -482,9 +482,9 @@ namespace Jackett.Common.Indexers
         /// </summary>
         /// <param name="request">URL created by Query Builder</param>
         /// <returns>Results from query</returns>
-        private async Task<BaseWebResult> QueryCache(string request)
+        private async Task<WebClientStringResult> QueryCache(string request)
         {
-            BaseWebResult results;
+            WebClientStringResult results;
 
             // Create Directory if not exist
             System.IO.Directory.CreateDirectory(Directory);
@@ -519,7 +519,7 @@ namespace Jackett.Common.Indexers
         /// </summary>
         /// <param name="request">URL created by Query Builder</param>
         /// <returns>Results from query</returns>
-        private async Task<BaseWebResult> QueryTracker(string request)
+        private async Task<WebClientStringResult> QueryTracker(string request)
         {
             // Cache mode not enabled or cached file didn't exist for our query
             Output("\nQuerying tracker for results....");

src/Jackett.Common/Indexers/NordicBits.cs

@@ -530,9 +530,9 @@ namespace Jackett.Common.Indexers
         /// </summary>
         /// <param name="request">URL created by Query Builder</param>
         /// <returns>Results from query</returns>
-        private async Task<BaseWebResult> QueryExec(string request)
+        private async Task<WebClientStringResult> QueryExec(string request)
         {
-            BaseWebResult results;
+            WebClientStringResult results;
 
             // Switch in we are in DEV mode with Hard Drive Cache or not
             if (DevMode && CacheMode)
@@ -553,9 +553,9 @@ namespace Jackett.Common.Indexers
         /// </summary>
         /// <param name="request">URL created by Query Builder</param>
         /// <returns>Results from query</returns>
-        private async Task<BaseWebResult> QueryCache(string request)
+        private async Task<WebClientStringResult> QueryCache(string request)
         {
-            BaseWebResult results;
+            WebClientStringResult results;
 
             // Create Directory if not exist
             System.IO.Directory.CreateDirectory(Directory);
@@ -590,7 +590,7 @@ namespace Jackett.Common.Indexers
         /// </summary>
         /// <param name="request">URL created by Query Builder</param>
         /// <returns>Results from query</returns>
-        private async Task<BaseWebResult> QueryTracker(string request)
+        private async Task<WebClientStringResult> QueryTracker(string request)
         {
             // Cache mode not enabled or cached file didn't exist for our query
             Output("\nQuerying tracker for results....");

src/Jackett.Common/Indexers/Partis.cs

@@ -116,7 +116,7 @@ namespace Jackett.Common.Indexers
             var releases = new List<ReleaseInfo>();     //List of releases initialization
             var searchString = query.GetQueryString();  //get search string from query
 
-            BaseWebResult results = null;
+            WebClientStringResult results = null;
             var queryCollection = new NameValueCollection();
             var catList = MapTorznabCapsToTrackers(query);     // map categories from query to indexer specific
             var categ = string.Join(",", catList);

src/Jackett.Common/Indexers/SceneTime.cs

@@ -104,7 +104,7 @@ namespace Jackett.Common.Indexers
 
         public override async Task<ConfigurationData> GetConfigurationForSetup()
         {
-            BaseWebResult loginPage;
+            WebClientStringResult loginPage;
             try
             {
                 loginPage = await RequestStringWithCookies(StartPageUrl, string.Empty);

src/Jackett.Common/Indexers/Shazbat.cs

@@ -81,7 +81,7 @@ namespace Jackett.Common.Indexers
         {
             var releases = new List<ReleaseInfo>();
             var queryString = query.GetQueryString();
-            BaseWebResult results = null;
+            WebClientStringResult results = null;
             var searchUrls = new List<string>();
             if (!string.IsNullOrWhiteSpace(query.SanitizedSearchTerm))
             {
@@ -170,7 +170,7 @@ namespace Jackett.Common.Indexers
             return releases;
         }
 
-        private async Task<BaseWebResult> ReloginIfNecessary(BaseWebResult response)
+        private async Task<WebClientStringResult> ReloginIfNecessary(WebClientStringResult response)
         {
             if (response.ContentString.Contains("onclick=\"document.location='logout'\""))
                 return response;

src/Jackett.Common/Indexers/SolidTorrents.cs

@@ -76,7 +76,7 @@ namespace Jackett.Common.Indexers
             return IndexerConfigurationStatus.Completed;
         }
 
-        private JArray CheckResponse(BaseWebResult result)
+        private JArray CheckResponse(WebClientStringResult result)
         {
             try
             {

src/Jackett.Common/Indexers/TVStore.cs

@@ -119,7 +119,7 @@ namespace Jackett.Common.Indexers
         /// <param name="alreadyFound">Number of the already found torrents.(used for limit)</param>
         /// <param name="limit">The limit to the number of torrents to download </param>
         /// <param name="previouslyParsedOnPage">Current position in parsed results</param>
-        private async Task<List<ReleaseInfo>> ParseTorrentsAsync(BaseWebResult results, int alreadyFound, int limit,
+        private async Task<List<ReleaseInfo>> ParseTorrentsAsync(WebClientStringResult results, int alreadyFound, int limit,
                                                                  int previouslyParsedOnPage)
         {
             var releases = new List<ReleaseInfo>();

src/Jackett.Common/Indexers/TorrenTech.cs

@@ -76,7 +76,7 @@ namespace Jackett.Common.Indexers
             var releases = new List<ReleaseInfo>();
             var searchString = query.GetQueryString();
 
-            BaseWebResult results = null;
+            WebClientStringResult results = null;
             var queryCollection = new NameValueCollection
             {
                 { "act", "search" },

src/Jackett.Common/Utils/Clients/BaseWebResult.cs

@@ -8,7 +8,6 @@ namespace Jackett.Common.Utils.Clients
 {
     public abstract class BaseWebResult
     {
-        private string _contentString;
         private Encoding _encoding;
 
         public Encoding Encoding
@@ -21,8 +20,7 @@ namespace Jackett.Common.Utils.Clients
                     _encoding = Request.Encoding;
                 else if (Headers.ContainsKey("content-type"))
                 {
-                    var charsetRegexMatch = Regex.Match(
+                    var charsetRegexMatch = Regex.Match(Headers["content-type"][0], @"charset=([\w-]+)", RegexOptions.Compiled);
-                        Headers["content-type"][0], @"charset=([\w-]+)", RegexOptions.Compiled);
                     if (charsetRegexMatch.Success)
                     {
                         var charset = charsetRegexMatch.Groups[1].Value;
@@ -38,34 +36,19 @@ namespace Jackett.Common.Utils.Clients
                 }
 
                 _encoding ??= Encoding.UTF8;
+
                 return _encoding;
             }
             set => _encoding = value;
         }
 
-        public byte[] ContentBytes { get; set; }
         public HttpStatusCode Status { get; set; }
         public string Cookies { get; set; }
         public string RedirectingTo { get; set; }
         public WebRequest Request { get; set; }
-
         public Dictionary<string, string[]> Headers { get; protected set; } =
             new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase);
 
-        public string ContentString
-        {
-            get
-            {
-                if (_contentString != null)
-                    return _contentString;
-                if (ContentBytes == null)
-                    return null;
-                _contentString = Encoding.GetString(ContentBytes);
-                return _contentString;
-            }
-            set => _contentString = value;
-        }
-
         public bool IsRedirect => Status == HttpStatusCode.Redirect ||
                                   Status == HttpStatusCode.RedirectKeepVerb ||
                                   Status == HttpStatusCode.RedirectMethod ||

src/Jackett.Common/Utils/Clients/WebByteResult.cs

@@ -2,5 +2,6 @@ namespace Jackett.Common.Utils.Clients
 {
     public class WebClientByteResult : BaseWebResult
     {
+        public byte[] ContentBytes { get; set; }
     }
 }

src/Jackett.Common/Utils/Clients/WebClient.cs

@@ -186,7 +186,7 @@ namespace Jackett.Common.Utils.Clients
             return;
         }
 
-        public virtual async Task<BaseWebResult> GetBytes(WebRequest request)
+        public virtual async Task<WebClientByteResult> GetBytes(WebRequest request)
         {
             logger.Debug(string.Format("WebClient({0}).GetBytes(Url:{1})", ClientType, request.Url));
             PrepareRequest(request);
@@ -198,7 +198,7 @@ namespace Jackett.Common.Utils.Clients
             return result;
         }
 
-        public virtual async Task<BaseWebResult> GetString(WebRequest request)
+        public virtual async Task<WebClientStringResult> GetString(WebRequest request)
         {
             logger.Debug(string.Format("WebClient({0}).GetString(Url:{1})", ClientType, request.Url));
             PrepareRequest(request);
@@ -206,7 +206,7 @@ namespace Jackett.Common.Utils.Clients
             var result = await Run(request);
             lastRequest = DateTime.Now;
             result.Request = request;
-            BaseWebResult stringResult = result;
+            WebClientStringResult stringResult = result;
 
             logger.Debug(string.Format("WebClient({0}): Returning {1} => {2}", ClientType, result.Status, (result.IsRedirect ? result.RedirectingTo + " " : "") + (stringResult.ContentString ?? "<NULL>")));
 

src/Jackett.Common/Utils/Clients/WebClientResult.cs

@@ -2,9 +2,11 @@ namespace Jackett.Common.Utils.Clients
 {
     public class WebClientStringResult : BaseWebResult
     {
+        public string ContentString { get; set; }
+
         public static implicit operator WebClientStringResult(WebClientByteResult br) => new WebClientStringResult()
         {
-            ContentBytes = br.ContentBytes,
+            ContentString = br.ContentBytes == null ? null : br.Encoding.GetString(br.ContentBytes),
             Cookies = br.Cookies,
             Encoding = br.Encoding,
             Headers = br.Headers,

src/Jackett.Test/TestWebClient.cs

@@ -11,8 +11,8 @@ namespace Jackett.Test
 {
     public class TestWebClient : WebClient
     {
-        private readonly Dictionary<WebRequest, Func<WebRequest, BaseWebResult>> byteCallbacks = new Dictionary<WebRequest, Func<WebRequest, BaseWebResult>>();
+        private readonly Dictionary<WebRequest, Func<WebRequest, WebClientByteResult>> byteCallbacks = new Dictionary<WebRequest, Func<WebRequest, WebClientByteResult>>();
-        private readonly Dictionary<WebRequest, Func<WebRequest, BaseWebResult>> stringCallbacks = new Dictionary<WebRequest, Func<WebRequest, BaseWebResult>>();
+        private readonly Dictionary<WebRequest, Func<WebRequest, WebClientStringResult>> stringCallbacks = new Dictionary<WebRequest, Func<WebRequest, WebClientStringResult>>();
 
         public TestWebClient(IProcessService p, Logger l, IConfigurationService c, ServerConfig sc)
             : base(p: p,
@@ -26,9 +26,9 @@ namespace Jackett.Test
 
         public void RegisterStringCall(WebRequest req, Func<WebRequest, WebClientStringResult> f) => stringCallbacks.Add(req, f);
 
-        public override Task<BaseWebResult> GetBytes(WebRequest request) => Task.FromResult(byteCallbacks.Where(r => r.Key.Equals(request)).First().Value.Invoke(request));
+        public override Task<WebClientByteResult> GetBytes(WebRequest request) => Task.FromResult<WebClientByteResult>(byteCallbacks.Where(r => r.Key.Equals(request)).First().Value.Invoke(request));
 
-        public override Task<BaseWebResult> GetString(WebRequest request) => Task.FromResult(stringCallbacks.Where(r => r.Key.Equals(request)).First().Value.Invoke(request));
+        public override Task<WebClientStringResult> GetString(WebRequest request) => Task.FromResult<WebClientStringResult>(stringCallbacks.Where(r => r.Key.Equals(request)).First().Value.Invoke(request));
 
         public override void Init()
         {