From ca61f965e9e92e71e7966ed851fe7c9d65ef362a Mon Sep 17 00:00:00 2001 From: ngosang Date: Mon, 21 Sep 2020 08:02:53 +0200 Subject: [PATCH] Revert "core: refactor http webclient part 13 #8529 (#8941)" This reverts commit 4d63fa81d521ab36fb35615765f999aacdc38c18. --- src/Jackett.Common/Indexers/Abnormal.cs | 7 +- .../Indexers/Abstract/AvistazTracker.cs | 7 +- .../Indexers/Abstract/CouchPotatoTracker.cs | 2 +- .../Indexers/Abstract/GazelleTracker.cs | 5 +- .../Indexers/Abstract/XtremeZoneTracker.cs | 12 +- src/Jackett.Common/Indexers/AniDUB.cs | 10 +- src/Jackett.Common/Indexers/Anidex.cs | 7 +- src/Jackett.Common/Indexers/AnimeBytes.cs | 3 +- src/Jackett.Common/Indexers/AnimeTorrents.cs | 7 +- src/Jackett.Common/Indexers/Anthelion.cs | 2 +- src/Jackett.Common/Indexers/AwesomeHD.cs | 2 +- src/Jackett.Common/Indexers/BB.cs | 6 +- src/Jackett.Common/Indexers/BJShare.cs | 8 +- src/Jackett.Common/Indexers/BakaBT.cs | 11 +- src/Jackett.Common/Indexers/BaseIndexer.cs | 115 ++++++++++++++---- .../Indexers/BitCityReloaded.cs | 2 +- src/Jackett.Common/Indexers/BitHDTV.cs | 8 +- src/Jackett.Common/Indexers/BroadcasTheNet.cs | 12 +- .../Indexers/CardigannIndexer.cs | 42 ++++--- src/Jackett.Common/Indexers/Cinecalidad.cs | 6 +- src/Jackett.Common/Indexers/CorsaroRed.cs | 5 +- src/Jackett.Common/Indexers/DanishBits.cs | 7 +- src/Jackett.Common/Indexers/DigitalCore.cs | 2 +- src/Jackett.Common/Indexers/DigitalHive.cs | 6 +- src/Jackett.Common/Indexers/DivxTotal.cs | 6 +- src/Jackett.Common/Indexers/EliteTracker.cs | 6 +- src/Jackett.Common/Indexers/EpubLibre.cs | 5 +- .../Indexers/Feeds/BaseFeedIndexer.cs | 2 +- src/Jackett.Common/Indexers/FileList.cs | 2 +- src/Jackett.Common/Indexers/FunFile.cs | 4 +- src/Jackett.Common/Indexers/Fuzer.cs | 9 +- src/Jackett.Common/Indexers/GazelleGames.cs | 2 +- src/Jackett.Common/Indexers/GimmePeers.cs | 6 +- src/Jackett.Common/Indexers/HDBitsApi.cs | 16 +-- src/Jackett.Common/Indexers/HDSpace.cs | 4 +- src/Jackett.Common/Indexers/HDTorrents.cs | 4 +- src/Jackett.Common/Indexers/Hebits.cs | 2 +- src/Jackett.Common/Indexers/HorribleSubs.cs | 8 +- src/Jackett.Common/Indexers/IPTorrents.cs | 3 +- src/Jackett.Common/Indexers/ImmortalSeed.cs | 5 +- .../Indexers/InternetArchive.cs | 2 +- src/Jackett.Common/Indexers/Libble.cs | 6 +- src/Jackett.Common/Indexers/LostFilm.cs | 18 +-- src/Jackett.Common/Indexers/MejorTorrent.cs | 10 +- src/Jackett.Common/Indexers/Milkie.cs | 2 +- src/Jackett.Common/Indexers/MoreThanTV.cs | 8 +- src/Jackett.Common/Indexers/MyAmity.cs | 4 +- src/Jackett.Common/Indexers/MyAnonamouse.cs | 2 +- src/Jackett.Common/Indexers/NCore.cs | 8 +- src/Jackett.Common/Indexers/Nebulance.cs | 5 +- src/Jackett.Common/Indexers/NewPCT.cs | 10 +- src/Jackett.Common/Indexers/NewRealWorld.cs | 4 +- src/Jackett.Common/Indexers/NorBits.cs | 12 +- src/Jackett.Common/Indexers/NordicBits.cs | 12 +- src/Jackett.Common/Indexers/Partis.cs | 7 +- src/Jackett.Common/Indexers/PassThePopcorn.cs | 5 +- src/Jackett.Common/Indexers/PirateTheNet.cs | 6 +- src/Jackett.Common/Indexers/PixelHD.cs | 8 +- src/Jackett.Common/Indexers/PolishTracker.cs | 4 +- src/Jackett.Common/Indexers/PornoLab.cs | 10 +- src/Jackett.Common/Indexers/PreToMe.cs | 8 +- src/Jackett.Common/Indexers/RarBG.cs | 7 +- src/Jackett.Common/Indexers/RevolutionTT.cs | 10 +- src/Jackett.Common/Indexers/RuTracker.cs | 8 +- src/Jackett.Common/Indexers/SceneHD.cs | 2 +- src/Jackett.Common/Indexers/SceneTime.cs | 4 +- src/Jackett.Common/Indexers/Shazbat.cs | 9 +- src/Jackett.Common/Indexers/ShowRSS.cs | 2 +- src/Jackett.Common/Indexers/SolidTorrents.cs | 2 +- src/Jackett.Common/Indexers/SpeedCD.cs | 4 +- src/Jackett.Common/Indexers/SuperBits.cs | 2 +- src/Jackett.Common/Indexers/TVStore.cs | 10 +- src/Jackett.Common/Indexers/TVVault.cs | 2 +- src/Jackett.Common/Indexers/Toloka.cs | 4 +- src/Jackett.Common/Indexers/TorrenTech.cs | 6 +- src/Jackett.Common/Indexers/TorrentBytes.cs | 6 +- src/Jackett.Common/Indexers/TorrentDay.cs | 2 +- src/Jackett.Common/Indexers/TorrentHeaven.cs | 8 +- src/Jackett.Common/Indexers/TorrentLeech.cs | 7 +- src/Jackett.Common/Indexers/TorrentNetwork.cs | 3 +- src/Jackett.Common/Indexers/TorrentSeeds.cs | 7 +- .../Indexers/TorrentSyndikat.cs | 6 +- src/Jackett.Common/Indexers/TorrentsCSV.cs | 2 +- src/Jackett.Common/Indexers/XSpeeds.cs | 13 +- src/Jackett.Common/Indexers/Xthor.cs | 3 +- src/Jackett.Common/Indexers/YTS.cs | 2 +- src/Jackett.Common/Services/ImdbResolver.cs | 2 +- src/Jackett.Common/Services/UpdateService.cs | 6 +- src/Jackett.Common/Utils/Clients/WebClient.cs | 32 +++-- src/Jackett.Test/TestUtil.cs | 11 +- src/Jackett.Test/TestWebClient.cs | 18 ++- 91 files changed, 399 insertions(+), 330 deletions(-) diff --git a/src/Jackett.Common/Indexers/Abnormal.cs b/src/Jackett.Common/Indexers/Abnormal.cs index 932c7715d..941a5a889 100644 --- a/src/Jackett.Common/Indexers/Abnormal.cs +++ b/src/Jackett.Common/Indexers/Abnormal.cs @@ -21,7 +21,6 @@ using Jackett.Common.Utils.Clients; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NLog; -using WebRequest = Jackett.Common.Utils.Clients.WebRequest; namespace Jackett.Common.Indexers { @@ -167,7 +166,7 @@ namespace Jackett.Common.Indexers // Perform loggin latencyNow(); output("\nPerform loggin.. with " + LoginUrl); - var response = await webclient.GetResultAsync(request); + var response = await webclient.GetString(request); // Test if we are logged in await ConfigureIfOK(response.Cookies, response.Cookies.Contains("session="), () => @@ -549,12 +548,14 @@ namespace Jackett.Common.Indexers /// Results from query private async Task queryTracker(string request) { + WebResult results = null; + // Cache mode not enabled or cached file didn't exist for our query output("\nQuerying tracker for results...."); // Request our first page latencyNow(); - var results = await RequestWithCookiesAndRetryAsync(request, headers: emulatedBrowserHeaders); + results = await RequestStringWithCookiesAndRetry(request, null, null, emulatedBrowserHeaders); // Return results from tracker return results.ContentString; diff --git a/src/Jackett.Common/Indexers/Abstract/AvistazTracker.cs b/src/Jackett.Common/Indexers/Abstract/AvistazTracker.cs index 4caaab8a2..20f4b66af 100644 --- a/src/Jackett.Common/Indexers/Abstract/AvistazTracker.cs +++ b/src/Jackett.Common/Indexers/Abstract/AvistazTracker.cs @@ -10,7 +10,6 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; -using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using WebClient = Jackett.Common.Utils.Clients.WebClient; @@ -149,7 +148,7 @@ without this configuration the torrent download does not work.
You can find { "password", configData.Password.Value.Trim() }, { "pid", configData.Pid.Value.Trim() } }; - var result = await WebRequestWithCookiesAsync(AuthUrl, method: RequestType.POST, data: body, headers: AuthHeaders); + var result = await PostDataWithCookies(AuthUrl, body, headers: AuthHeaders); var json = JObject.Parse(result.ContentString); _token = json.Value("token"); if (_token == null) @@ -162,11 +161,11 @@ without this configuration the torrent download does not work.
You can find var qc = GetSearchQueryParameters(query); var episodeSearchUrl = SearchUrl + "?" + qc.GetQueryString(); - var response = await RequestWithCookiesAndRetryAsync(episodeSearchUrl, headers: GetSearchHeaders()); + var response = await RequestStringWithCookiesAndRetry(episodeSearchUrl, headers: GetSearchHeaders()); if (response.Status == HttpStatusCode.Unauthorized || response.Status == HttpStatusCode.PreconditionFailed) { await RenewalTokenAsync(); - response = await RequestWithCookiesAndRetryAsync(episodeSearchUrl, headers: GetSearchHeaders()); + response = await RequestStringWithCookiesAndRetry(episodeSearchUrl, headers: GetSearchHeaders()); } else if (response.Status != HttpStatusCode.OK) throw new Exception($"Unknown error: {response.ContentString}"); diff --git a/src/Jackett.Common/Indexers/Abstract/CouchPotatoTracker.cs b/src/Jackett.Common/Indexers/Abstract/CouchPotatoTracker.cs index feeb815a4..fdba46f78 100644 --- a/src/Jackett.Common/Indexers/Abstract/CouchPotatoTracker.cs +++ b/src/Jackett.Common/Indexers/Abstract/CouchPotatoTracker.cs @@ -75,7 +75,7 @@ namespace Jackett.Common.Indexers.Abstract searchUrl += "?" + queryCollection.GetQueryString(); - var response = await RequestWithCookiesAndRetryAsync(searchUrl); + var response = await RequestStringWithCookiesAndRetry(searchUrl); JObject json = null; try diff --git a/src/Jackett.Common/Indexers/Abstract/GazelleTracker.cs b/src/Jackett.Common/Indexers/Abstract/GazelleTracker.cs index f0b245827..ececf7356 100644 --- a/src/Jackett.Common/Indexers/Abstract/GazelleTracker.cs +++ b/src/Jackett.Common/Indexers/Abstract/GazelleTracker.cs @@ -12,7 +12,6 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; -using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using WebClient = Jackett.Common.Utils.Clients.WebClient; @@ -197,12 +196,12 @@ namespace Jackett.Common.Indexers.Abstract searchUrl += "?" + queryCollection.GetQueryString(); - var response = await RequestWithCookiesAndRetryAsync(searchUrl); + var response = await RequestStringWithCookiesAndRetry(searchUrl); if (response.IsRedirect) { // re-login await ApplyConfiguration(null); - response = await RequestWithCookiesAndRetryAsync(searchUrl); + response = await RequestStringWithCookiesAndRetry(searchUrl); } try diff --git a/src/Jackett.Common/Indexers/Abstract/XtremeZoneTracker.cs b/src/Jackett.Common/Indexers/Abstract/XtremeZoneTracker.cs index 7d7aa779c..ddeb717ba 100644 --- a/src/Jackett.Common/Indexers/Abstract/XtremeZoneTracker.cs +++ b/src/Jackett.Common/Indexers/Abstract/XtremeZoneTracker.cs @@ -10,7 +10,6 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; -using Jackett.Common.Utils.Clients; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NLog; @@ -75,8 +74,7 @@ namespace Jackett.Common.Indexers.Abstract { "password", configData.Password.Value.Trim() } }; var jsonData = JsonConvert.SerializeObject(body); - var result = await WebRequestWithCookiesAsync( - LoginUrl, method: RequestType.POST, headers: ApiHeaders, rawbody: jsonData); + var result = await PostDataWithCookies(LoginUrl, null, headers: ApiHeaders, rawbody: jsonData); var json = JObject.Parse(result.ContentString); _token = json.Value("token"); if (_token == null) @@ -107,11 +105,11 @@ namespace Jackett.Common.Indexers.Abstract await RenewalTokenAsync(); var searchUrl = SearchUrl + "?" + qc.GetQueryString(); - var response = await WebRequestWithCookiesAsync(searchUrl, headers: GetSearchHeaders()); + var response = await RequestStringWithCookies(searchUrl, headers: GetSearchHeaders()); if (response.Status == HttpStatusCode.Unauthorized) { await RenewalTokenAsync(); // re-login - response = await WebRequestWithCookiesAsync(searchUrl, headers: GetSearchHeaders()); + response = await RequestStringWithCookies(searchUrl, headers: GetSearchHeaders()); } else if (response.Status != HttpStatusCode.OK) throw new Exception($"Unknown error in search: {response.ContentString}"); @@ -169,11 +167,11 @@ namespace Jackett.Common.Indexers.Abstract public override async Task Download(Uri link) { - var response = await WebRequestWithCookiesAsync(link.ToString(), headers: GetSearchHeaders()); + var response = await RequestBytesWithCookies(link.ToString(), headers: GetSearchHeaders()); if (response.Status == HttpStatusCode.Unauthorized) { await RenewalTokenAsync(); - response = await WebRequestWithCookiesAsync(link.ToString(), headers: GetSearchHeaders()); + response = await RequestBytesWithCookies(link.ToString(), headers: GetSearchHeaders()); } else if (response.Status != HttpStatusCode.OK) throw new Exception($"Unknown error in download: {response.ContentBytes}"); diff --git a/src/Jackett.Common/Indexers/AniDUB.cs b/src/Jackett.Common/Indexers/AniDUB.cs index 6f86f138f..15181a204 100644 --- a/src/Jackett.Common/Indexers/AniDUB.cs +++ b/src/Jackett.Common/Indexers/AniDUB.cs @@ -145,7 +145,7 @@ namespace Jackett.Common.Indexers private async Task EnsureAuthorized() { - var result = await RequestWithCookiesAndRetryAsync(SiteLink); + var result = await RequestStringWithCookiesAndRetry(SiteLink); if (!IsAuthorized(result)) { @@ -156,7 +156,8 @@ namespace Jackett.Common.Indexers private async Task> FetchNewReleases() { const string ReleaseLinksSelector = "#dle-content > .story > .story_h > .lcol > h2 > a"; - var result = await RequestWithCookiesAndRetryAsync(SiteLink); + + var result = await RequestStringWithCookiesAndRetry(SiteLink); var releases = new List(); try @@ -194,7 +195,7 @@ namespace Jackett.Common.Indexers return releases; } - var result = await RequestWithCookiesAndRetryAsync(url); + var result = await RequestStringWithCookiesAndRetry(url); try { @@ -528,7 +529,8 @@ namespace Jackett.Common.Indexers const string searchLinkSelector = "#dle-content > .searchitem > h3 > a"; var releases = new List(); - var response = await RequestWithCookiesAndRetryAsync(SearchUrl, method: RequestType.POST, data: PreparePostData(query)); + + var response = await PostDataWithCookiesAndRetry(SearchUrl, PreparePostData(query)); try { diff --git a/src/Jackett.Common/Indexers/Anidex.cs b/src/Jackett.Common/Indexers/Anidex.cs index 79b957eea..c91f1a010 100644 --- a/src/Jackett.Common/Indexers/Anidex.cs +++ b/src/Jackett.Common/Indexers/Anidex.cs @@ -14,7 +14,6 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; -using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using static Jackett.Common.Models.IndexerConfig.ConfigurationData; @@ -151,13 +150,13 @@ namespace Jackett.Common.Indexers // Make search request var searchUri = GetAbsoluteUrl("?" + queryParameters.GetQueryString()); - var response = await RequestWithCookiesAndRetryAsync(searchUri.AbsoluteUri); + var response = await RequestStringWithCookiesAndRetry(searchUri.AbsoluteUri); // Check for DDOS Guard if (response.Status == System.Net.HttpStatusCode.Forbidden) { await ConfigureDDoSGuardCookie(); - response = await RequestWithCookiesAndRetryAsync(searchUri.AbsoluteUri); + response = await RequestStringWithCookiesAndRetry(searchUri.AbsoluteUri); } if (response.Status != System.Net.HttpStatusCode.OK) @@ -217,7 +216,7 @@ namespace Jackett.Common.Indexers private async Task ConfigureDDoSGuardCookie() { const string ddosPostUrl = "https://check.ddos-guard.net/check.js"; - var response = await WebRequestWithCookiesAsync(ddosPostUrl, string.Empty); + var response = await RequestStringWithCookies(ddosPostUrl, string.Empty); if (response.Status != System.Net.HttpStatusCode.OK) throw new WebException($"Unexpected DDOS Guard response: Status: {response.Status}", WebExceptionStatus.ProtocolError); if (response.IsRedirect) diff --git a/src/Jackett.Common/Indexers/AnimeBytes.cs b/src/Jackett.Common/Indexers/AnimeBytes.cs index 4bc5c6548..0859e81a5 100644 --- a/src/Jackett.Common/Indexers/AnimeBytes.cs +++ b/src/Jackett.Common/Indexers/AnimeBytes.cs @@ -12,7 +12,6 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig.Bespoke; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; -using Jackett.Common.Utils.Clients; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NLog; @@ -180,7 +179,7 @@ namespace Jackett.Common.Indexers } // Get the content from the tracker - var response = await RequestWithCookiesAndRetryAsync(queryUrl); + var response = await RequestStringWithCookiesAndRetry(queryUrl); if (!response.ContentString.StartsWith("{")) // not JSON => error throw new ExceptionWithConfigData("unexcepted response (not JSON)", configData); dynamic json = JsonConvert.DeserializeObject(response.ContentString); diff --git a/src/Jackett.Common/Indexers/AnimeTorrents.cs b/src/Jackett.Common/Indexers/AnimeTorrents.cs index 50b2b8ae3..97ad501e8 100644 --- a/src/Jackett.Common/Indexers/AnimeTorrents.cs +++ b/src/Jackett.Common/Indexers/AnimeTorrents.cs @@ -1,4 +1,4 @@ - using System; +using System; using System.Collections.Generic; using System.Collections.Specialized; using System.Diagnostics.CodeAnalysis; @@ -79,7 +79,7 @@ namespace Jackett.Common.Indexers { "rememberme[]", "1" } }; - var loginPage = await RequestWithCookiesAndRetryAsync(LoginUrl, "", RequestType.GET, LoginUrl); + var loginPage = await RequestStringWithCookiesAndRetry(LoginUrl, "", LoginUrl); var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true); await ConfigureIfOK(result.Cookies, result.ContentString != null && result.ContentString.Contains("logout.php"), () => @@ -117,8 +117,7 @@ namespace Jackett.Common.Indexers { "X-Requested-With", "XMLHttpRequest" } }; - var response = await RequestWithCookiesAndRetryAsync( - searchUrl, referer: SearchUrlReferer, headers: extraHeaders); + var response = await RequestStringWithCookiesAndRetry(searchUrl, null, SearchUrlReferer, extraHeaders); var results = response.ContentString; try diff --git a/src/Jackett.Common/Indexers/Anthelion.cs b/src/Jackett.Common/Indexers/Anthelion.cs index 2f2335afc..2145010cb 100644 --- a/src/Jackett.Common/Indexers/Anthelion.cs +++ b/src/Jackett.Common/Indexers/Anthelion.cs @@ -94,7 +94,7 @@ namespace Jackett.Common.Indexers qc.Add($"filter_cat[{cat}]", "1"); var searchUrl = BrowseUrl + "?" + qc.GetQueryString(); - var results = await WebRequestWithCookiesAsync(searchUrl); + var results = await RequestStringWithCookies(searchUrl); try { var parser = new HtmlParser(); diff --git a/src/Jackett.Common/Indexers/AwesomeHD.cs b/src/Jackett.Common/Indexers/AwesomeHD.cs index d91888d7d..8b06914d3 100644 --- a/src/Jackett.Common/Indexers/AwesomeHD.cs +++ b/src/Jackett.Common/Indexers/AwesomeHD.cs @@ -100,7 +100,7 @@ namespace Jackett.Common.Indexers } var searchUrl = SearchUrl + "?" + qc.GetQueryString(); - var results = await WebRequestWithCookiesAsync(searchUrl); + var results = await RequestStringWithCookies(searchUrl); if (string.IsNullOrWhiteSpace(results.ContentString)) throw new Exception("Empty response. Please, check the Passkey."); diff --git a/src/Jackett.Common/Indexers/BB.cs b/src/Jackett.Common/Indexers/BB.cs index 6614b2b52..36ed6f9fe 100644 --- a/src/Jackett.Common/Indexers/BB.cs +++ b/src/Jackett.Common/Indexers/BB.cs @@ -122,8 +122,8 @@ namespace Jackett.Common.Indexers request_urls.Add(SearchUrl + queryCollection.GetQueryString()); } - - var downloadTasksQuery = from url in request_urls select RequestWithCookiesAndRetryAsync(url); + var downloadTasksQuery = + from url in request_urls select RequestStringWithCookiesAndRetry(url); var responses = await Task.WhenAll(downloadTasksQuery.ToArray()); @@ -134,7 +134,7 @@ namespace Jackett.Common.Indexers if (results.IsRedirect) { await ApplyConfiguration(null); - results = await RequestWithCookiesAndRetryAsync(request_urls[i]); + results = await RequestStringWithCookiesAndRetry(request_urls[i]); } try { diff --git a/src/Jackett.Common/Indexers/BJShare.cs b/src/Jackett.Common/Indexers/BJShare.cs index e05405426..9ea87c84b 100644 --- a/src/Jackett.Common/Indexers/BJShare.cs +++ b/src/Jackett.Common/Indexers/BJShare.cs @@ -257,12 +257,12 @@ namespace Jackett.Common.Indexers foreach (var cat in MapTorznabCapsToTrackers(query)) queryCollection.Add("filter_cat[" + cat + "]", "1"); searchUrl += "?" + queryCollection.GetQueryString(); - var results = await WebRequestWithCookiesAsync(searchUrl); + var results = await RequestStringWithCookies(searchUrl); if (IsSessionIsClosed(results)) { // re-login await ApplyConfiguration(null); - results = await WebRequestWithCookiesAsync(searchUrl); + results = await RequestStringWithCookies(searchUrl); } try @@ -389,12 +389,12 @@ namespace Jackett.Common.Indexers private async Task> ParseLast24HoursAsync() { var releases = new List(); - var results = await WebRequestWithCookiesAsync(TodayUrl); + var results = await RequestStringWithCookies(TodayUrl); if (IsSessionIsClosed(results)) { // re-login await ApplyConfiguration(null); - results = await WebRequestWithCookiesAsync(TodayUrl); + results = await RequestStringWithCookies(TodayUrl); } try diff --git a/src/Jackett.Common/Indexers/BakaBT.cs b/src/Jackett.Common/Indexers/BakaBT.cs index 29f66a070..80efa758c 100644 --- a/src/Jackett.Common/Indexers/BakaBT.cs +++ b/src/Jackett.Common/Indexers/BakaBT.cs @@ -14,7 +14,6 @@ using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; -using WebRequest = Jackett.Common.Utils.Clients.WebRequest; namespace Jackett.Common.Indexers { @@ -69,7 +68,7 @@ namespace Jackett.Common.Indexers private async Task DoLogin() { - var loginForm = await webclient.GetResultAsync(new Utils.Clients.WebRequest() + var loginForm = await webclient.GetString(new Utils.Clients.WebRequest() { Url = LoginUrl, Type = RequestType.GET @@ -107,12 +106,12 @@ namespace Jackett.Common.Indexers var releases = new List(); var searchString = queryCopy.SanitizedSearchTerm; var episodeSearchUrl = SearchUrl + WebUtility.UrlEncode(searchString); - var response = await RequestWithCookiesAndRetryAsync(episodeSearchUrl); + var response = await RequestStringWithCookiesAndRetry(episodeSearchUrl); if (!response.ContentString.Contains(LogoutStr)) { //Cookie appears to expire after a period of time or logging in to the site via browser await DoLogin(); - response = await RequestWithCookiesAndRetryAsync(episodeSearchUrl); + response = await RequestStringWithCookiesAndRetry(episodeSearchUrl); } try @@ -247,7 +246,7 @@ namespace Jackett.Common.Indexers public override async Task Download(Uri link) { - var downloadPage = await WebRequestWithCookiesAsync(link.ToString()); + var downloadPage = await RequestStringWithCookies(link.ToString()); var parser = new HtmlParser(); var dom = parser.ParseDocument(downloadPage.ContentString); var downloadLink = dom.QuerySelectorAll(".download_link").First().GetAttribute("href"); @@ -255,7 +254,7 @@ namespace Jackett.Common.Indexers if (string.IsNullOrWhiteSpace(downloadLink)) throw new Exception("Unable to find download link."); - var response = await WebRequestWithCookiesAsync(SiteLink + downloadLink); + var response = await RequestBytesWithCookies(SiteLink + downloadLink); return response.ContentBytes; } } diff --git a/src/Jackett.Common/Indexers/BaseIndexer.cs b/src/Jackett.Common/Indexers/BaseIndexer.cs index 859669c82..5ab118370 100644 --- a/src/Jackett.Common/Indexers/BaseIndexer.cs +++ b/src/Jackett.Common/Indexers/BaseIndexer.cs @@ -392,11 +392,11 @@ namespace Jackett.Common.Indexers .Replace("(", "%28") .Replace(")", "%29") .Replace("'", "%27"); - var response = await RequestWithCookiesAndRetryAsync(requestLink, null, method, requestLink); + var response = await RequestBytesWithCookiesAndRetry(requestLink, null, method, requestLink); // if referer link is provied it will be used if (refererlink != null) - response = await RequestWithCookiesAndRetryAsync(requestLink, null, method, refererlink); + response = await RequestBytesWithCookiesAndRetry(requestLink, null, method, refererlink); if (response.IsRedirect) { await FollowIfRedirect(response); @@ -412,38 +412,69 @@ namespace Jackett.Common.Indexers return response.ContentBytes; } - protected async Task RequestWithCookiesAndRetryAsync( - string url, string cookieOverride = null, RequestType method = RequestType.GET, - string referer = null, IEnumerable> data = null, - Dictionary headers = null, string rawbody = null, bool? emulateBrowser = null) + protected async Task RequestBytesWithCookiesAndRetry(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable> data = null) { Exception lastException = null; for (var i = 0; i < 3; i++) { try { - return await WebRequestWithCookiesAsync( - url, cookieOverride, method, referer, data, headers, rawbody, emulateBrowser); + return await RequestBytesWithCookies(url, cookieOverride, method, referer, data); } catch (Exception e) { - logger.Error( - e, string.Format("On attempt {0} downloading from {1}: {2}", (i + 1), DisplayName, e.Message)); + logger.Error(e, string.Format("On attempt {0} downloading from {1}: {2}", (i + 1), DisplayName, e.Message)); lastException = e; } - await Task.Delay(500); } throw lastException; } - protected virtual async Task WebRequestWithCookiesAsync( - string url, string cookieOverride = null, RequestType method = RequestType.GET, - string referer = null, IEnumerable> data = null, - Dictionary headers = null, string rawbody = null, bool? emulateBrowser = null) + protected async Task RequestStringWithCookies(string url, string cookieOverride = null, string referer = null, Dictionary headers = null) { - var request = new WebRequest + var request = new Utils.Clients.WebRequest() + { + Url = url, + Type = RequestType.GET, + Cookies = CookieHeader, + Referer = referer, + Headers = headers, + Encoding = Encoding + }; + + if (cookieOverride != null) + request.Cookies = cookieOverride; + var result = await webclient.GetString(request); + CheckTrackerDown(result); + UpdateCookieHeader(result.Cookies, cookieOverride); + return result; + } + + protected async Task RequestStringWithCookiesAndRetry(string url, string cookieOverride = null, string referer = null, Dictionary headers = null) + { + Exception lastException = null; + for (var i = 0; i < 3; i++) + { + try + { + return await RequestStringWithCookies(url, cookieOverride, referer, headers); + } + catch (Exception e) + { + logger.Error(string.Format("On attempt {0} checking for results from {1}: {2}", (i + 1), DisplayName, e.Message)); + lastException = e; + } + await Task.Delay(500); + } + + throw lastException; + } + + protected virtual async Task RequestBytesWithCookies(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable> data = null, Dictionary headers = null) + { + var request = new Utils.Clients.WebRequest() { Url = url, Type = method, @@ -451,18 +482,58 @@ namespace Jackett.Common.Indexers PostData = data, Referer = referer, Headers = headers, + Encoding = Encoding + }; + + if (cookieOverride != null) + request.Cookies = cookieOverride; + var result = await webclient.GetBytes(request); + UpdateCookieHeader(result.Cookies, cookieOverride); + return result; + } + + protected async Task PostDataWithCookies(string url, IEnumerable> data, string cookieOverride = null, string referer = null, Dictionary headers = null, string rawbody = null, bool? emulateBrowser = null) + { + var request = new Utils.Clients.WebRequest() + { + Url = url, + Type = RequestType.POST, + Cookies = cookieOverride ?? CookieHeader, + PostData = data, + Referer = referer, + Headers = headers, RawBody = rawbody, Encoding = Encoding }; if (emulateBrowser.HasValue) request.EmulateBrowser = emulateBrowser.Value; - var result = await webclient.GetResultAsync(request); - CheckSiteDown(result); + var result = await webclient.GetString(request); + CheckTrackerDown(result); UpdateCookieHeader(result.Cookies, cookieOverride); return result; } + protected async Task PostDataWithCookiesAndRetry(string url, IEnumerable> data, string cookieOverride = null, string referer = null, Dictionary headers = null, string rawbody = null, bool? emulateBrowser = null) + { + Exception lastException = null; + for (var i = 0; i < 3; i++) + { + try + { + return await PostDataWithCookies(url, data, cookieOverride, referer, headers, rawbody, emulateBrowser); + } + catch (Exception e) + { + logger.Error(string.Format("On attempt {0} checking for results from {1}: {2}", (i + 1), DisplayName, e.Message)); + lastException = e; + } + await Task.Delay(500); + } + + throw lastException; + } + protected async Task RequestLoginAndFollowRedirect(string url, IEnumerable> data, string cookies, bool returnCookiesFromFirstCall, string redirectUrlOverride = null, string referer = null, bool accumulateCookies = false) { var request = new Utils.Clients.WebRequest() @@ -474,8 +545,8 @@ namespace Jackett.Common.Indexers PostData = data, Encoding = Encoding }; - var response = await webclient.GetResultAsync(request); - CheckSiteDown(response); + var response = await webclient.GetString(request); + CheckTrackerDown(response); if (accumulateCookies) { response.Cookies = ResolveCookies((request.Cookies == null ? "" : request.Cookies + " ") + response.Cookies); @@ -495,7 +566,7 @@ namespace Jackett.Common.Indexers return response; } - protected static void CheckSiteDown(WebResult response) + protected void CheckTrackerDown(WebResult response) { if (response.Status == System.Net.HttpStatusCode.BadGateway || response.Status == System.Net.HttpStatusCode.GatewayTimeout @@ -572,7 +643,7 @@ namespace Jackett.Common.Indexers redirRequestCookies = (overrideCookies != null ? overrideCookies : ""); } // Do redirect - var redirectedResponse = await webclient.GetResultAsync(new WebRequest() + var redirectedResponse = await webclient.GetBytes(new WebRequest() { Url = overrideRedirectUrl ?? incomingResponse.RedirectingTo, Referer = referrer, diff --git a/src/Jackett.Common/Indexers/BitCityReloaded.cs b/src/Jackett.Common/Indexers/BitCityReloaded.cs index 8f456320b..6dad73997 100644 --- a/src/Jackett.Common/Indexers/BitCityReloaded.cs +++ b/src/Jackett.Common/Indexers/BitCityReloaded.cs @@ -129,7 +129,7 @@ namespace Jackett.Common.Indexers searchUrl += "?" + queryCollection.GetQueryString(); - var response = await RequestWithCookiesAndRetryAsync(searchUrl, referer: BrowseUrl); + var response = await RequestStringWithCookiesAndRetry(searchUrl, null, BrowseUrl); var results = response.ContentString; try { diff --git a/src/Jackett.Common/Indexers/BitHDTV.cs b/src/Jackett.Common/Indexers/BitHDTV.cs index b34808e6a..9a047fd63 100644 --- a/src/Jackett.Common/Indexers/BitHDTV.cs +++ b/src/Jackett.Common/Indexers/BitHDTV.cs @@ -63,7 +63,7 @@ namespace Jackett.Common.Indexers public override async Task GetConfigurationForSetup() { var result = configData; - var loginPage = await WebRequestWithCookiesAsync(LoginUrl, configData.CookieHeader.Value); + var loginPage = await RequestStringWithCookies(LoginUrl, configData.CookieHeader.Value); if (loginPage.IsRedirect) return result; // already logged in var parser = new HtmlParser(); @@ -129,10 +129,10 @@ namespace Jackett.Common.Indexers qc.Add("search", query.ImdbID); qc.Add("options", "4"); //Search URL field for IMDB link search.Query = qc.GetQueryString(); - results.Add(await RequestWithCookiesAndRetryAsync(search.ToString())); + results.Add(await RequestStringWithCookiesAndRetry(search.ToString())); qc["Options"] = "1"; //Search Title and Description search.Query = qc.GetQueryString(); - results.Add(await RequestWithCookiesAndRetryAsync(search.ToString())); + results.Add(await RequestStringWithCookiesAndRetry(search.ToString())); } else { @@ -140,7 +140,7 @@ namespace Jackett.Common.Indexers qc.Add("search", query.GetQueryString()); qc.Add("options", "0"); //Search Title Only search.Query = qc.GetQueryString(); - results.Add(await RequestWithCookiesAndRetryAsync(search.ToString())); + results.Add(await RequestStringWithCookiesAndRetry(search.ToString())); } var parser = new HtmlParser(); diff --git a/src/Jackett.Common/Indexers/BroadcasTheNet.cs b/src/Jackett.Common/Indexers/BroadcasTheNet.cs index ba28f0c67..982d3068a 100644 --- a/src/Jackett.Common/Indexers/BroadcasTheNet.cs +++ b/src/Jackett.Common/Indexers/BroadcasTheNet.cs @@ -110,13 +110,11 @@ namespace Jackett.Common.Indexers new JValue(btnResults), new JValue(btnOffset) }; - var response = await RequestWithCookiesAndRetryAsync( - APIBASE, method: RequestType.POST, - headers: new Dictionary - { - {"Accept", "application/json-rpc, application/json"}, - {"Content-Type", "application/json-rpc"} - }, rawbody: JsonRPCRequest("getTorrents", parameters), emulateBrowser: false); + var response = await PostDataWithCookiesAndRetry(APIBASE, null, null, null, new Dictionary() + { + { "Accept", "application/json-rpc, application/json"}, + {"Content-Type", "application/json-rpc"} + }, JsonRPCRequest("getTorrents", parameters), false); try { diff --git a/src/Jackett.Common/Indexers/CardigannIndexer.cs b/src/Jackett.Common/Indexers/CardigannIndexer.cs index b29d5de58..df732db6c 100644 --- a/src/Jackett.Common/Indexers/CardigannIndexer.cs +++ b/src/Jackett.Common/Indexers/CardigannIndexer.cs @@ -532,12 +532,13 @@ namespace Jackett.Common.Indexers ["g-recaptcha-response"] = CaptchaConfigItem.Value }; var ClearanceUrl = resolvePath("/cdn-cgi/l/chk_captcha?" + CloudFlareQueryCollection.GetQueryString()); - var ClearanceResult = await WebRequestWithCookiesAsync(ClearanceUrl.ToString(), referer: SiteLink); + + var ClearanceResult = await RequestStringWithCookies(ClearanceUrl.ToString(), null, SiteLink); if (ClearanceResult.IsRedirect) // clearance successfull { // request real login page again - landingResult = await WebRequestWithCookiesAsync(LoginUrl, referer: SiteLink); + landingResult = await RequestStringWithCookies(LoginUrl, null, SiteLink); var htmlParser = new HtmlParser(); landingResultDocument = htmlParser.ParseDocument(landingResult.ContentString); } @@ -653,7 +654,7 @@ namespace Jackett.Common.Indexers if (simpleCaptchaPresent != null) { var captchaUrl = resolvePath("simpleCaptcha.php?numImages=1"); - var simpleCaptchaResult = await WebRequestWithCookiesAsync(captchaUrl.ToString(), referer: LoginUrl); + var simpleCaptchaResult = await RequestStringWithCookies(captchaUrl.ToString(), null, LoginUrl); var simpleCaptchaJSON = JObject.Parse(simpleCaptchaResult.ContentString); var captchaSelection = simpleCaptchaJSON["images"][0]["hash"].ToString(); pairs["captchaSelection"] = captchaSelection; @@ -722,9 +723,7 @@ namespace Jackett.Common.Indexers headers.Add("Content-Type", "multipart/form-data; boundary=" + boundary); var body = string.Join("\r\n", bodyParts); - loginResult = await WebRequestWithCookiesAsync( - submitUrl.ToString(), configData.CookieHeader.Value, RequestType.POST, SiteLink, pairs, headers, - body); + loginResult = await PostDataWithCookies(submitUrl.ToString(), pairs, configData.CookieHeader.Value, SiteLink, headers, body); } else { @@ -750,7 +749,7 @@ namespace Jackett.Common.Indexers var LoginUrl = resolvePath(Login.Path + "?" + queryCollection.GetQueryString()).ToString(); configData.CookieHeader.Value = null; - var loginResult = await WebRequestWithCookiesAsync(LoginUrl, referer: SiteLink); + var loginResult = await RequestStringWithCookies(LoginUrl, null, SiteLink); configData.CookieHeader.Value = loginResult.Cookies; checkForError(loginResult, Definition.Login.Error); @@ -760,7 +759,7 @@ namespace Jackett.Common.Indexers var OneUrl = applyGoTemplateText(Definition.Login.Inputs["oneurl"]); var LoginUrl = resolvePath(Login.Path + OneUrl).ToString(); configData.CookieHeader.Value = null; - var loginResult = await WebRequestWithCookiesAsync(LoginUrl, referer: SiteLink); + var loginResult = await RequestStringWithCookies(LoginUrl, null, SiteLink); configData.CookieHeader.Value = loginResult.Cookies; checkForError(loginResult, Definition.Login.Error); @@ -794,7 +793,7 @@ namespace Jackett.Common.Indexers // test if login was successful var LoginTestUrl = resolvePath(Login.Test.Path).ToString(); - var testResult = await WebRequestWithCookiesAsync(LoginTestUrl); + var testResult = await RequestStringWithCookies(LoginTestUrl); if (testResult.IsRedirect) { @@ -887,7 +886,7 @@ namespace Jackett.Common.Indexers configData.CookieHeader.Value = null; if (Login.Cookies != null) configData.CookieHeader.Value = string.Join("; ", Login.Cookies); - landingResult = await WebRequestWithCookiesAsync(LoginUrl.AbsoluteUri, referer: SiteLink); + landingResult = await RequestStringWithCookies(LoginUrl.AbsoluteUri, null, SiteLink); var htmlParser = new HtmlParser(); landingResultDocument = htmlParser.ParseDocument(landingResult.ContentString); @@ -930,8 +929,7 @@ namespace Jackett.Common.Indexers hasCaptcha = true; var CaptchaUrl = resolvePath(captchaElement.GetAttribute("src"), LoginUrl); - var captchaImageData = await WebRequestWithCookiesAsync( - CaptchaUrl.ToString(), landingResult.Cookies, referer: LoginUrl.AbsoluteUri); + var captchaImageData = await RequestBytesWithCookies(CaptchaUrl.ToString(), landingResult.Cookies, RequestType.GET, LoginUrl.AbsoluteUri); var CaptchaImage = new ImageItem { Name = "Captcha Image" }; var CaptchaText = new StringItem { Name = "Captcha Text" }; @@ -1336,6 +1334,7 @@ namespace Jackett.Common.Indexers logger.Info($"Fetching: {searchUrl}"); // send HTTP request + WebResult response = null; Dictionary headers = null; if (Search.Headers != null) { @@ -1344,9 +1343,10 @@ namespace Jackett.Common.Indexers foreach (var header in Search.Headers) headers.Add(header.Key, header.Value[0]); } - - var response = await WebRequestWithCookiesAsync( - searchUrl, method: method, headers: headers, data: queryCollection); + if (method == RequestType.POST) + response = await PostDataWithCookies(searchUrl, queryCollection, null, null, headers); + else + response = await RequestStringWithCookies(searchUrl, null, null, headers); if (response.IsRedirect && SearchPath.Followredirect) await FollowIfRedirect(response); @@ -1368,7 +1368,11 @@ namespace Jackett.Common.Indexers if (!LoginResult) throw new Exception(string.Format("Relogin failed")); await TestLogin(); - response = await WebRequestWithCookiesAsync(searchUrl, method: method, data: queryCollection); + if (method == RequestType.POST) + response = await PostDataWithCookies(searchUrl, queryCollection); + else + response = await RequestStringWithCookies(searchUrl); + if (response.IsRedirect && SearchPath.Followredirect) await FollowIfRedirect(response); @@ -1749,7 +1753,7 @@ namespace Jackett.Common.Indexers requestLinkStr += queryCollection.GetQueryString(Encoding, separator: request.Queryseparator); } - var response = await RequestWithCookiesAndRetryAsync(requestLinkStr, null, method, referer, pairs); + var response = await RequestBytesWithCookiesAndRetry(requestLinkStr, null, method, referer, pairs); logger.Debug($"CardigannIndexer ({Id}): handleRequest() remote server returned {response.Status.ToString()}" + (response.IsRedirect ? " => " + response.RedirectingTo : "")); return response; } @@ -1792,9 +1796,9 @@ namespace Jackett.Common.Indexers if (Download.Selector != null) { var selector = applyGoTemplateText(Download.Selector, variables); - var response = await WebRequestWithCookiesAsync(link.ToString()); + var response = await RequestStringWithCookies(link.ToString()); if (response.IsRedirect) - response = await WebRequestWithCookiesAsync(response.RedirectingTo); + response = await RequestStringWithCookies(response.RedirectingTo); var results = response.ContentString; var searchResultParser = new HtmlParser(); var searchResultDocument = searchResultParser.ParseDocument(results); diff --git a/src/Jackett.Common/Indexers/Cinecalidad.cs b/src/Jackett.Common/Indexers/Cinecalidad.cs index 9dfb654f7..02b829ddf 100644 --- a/src/Jackett.Common/Indexers/Cinecalidad.cs +++ b/src/Jackett.Common/Indexers/Cinecalidad.cs @@ -100,7 +100,7 @@ namespace Jackett.Common.Indexers { var pageParam = page > 1 ? $"page/{page}/" : ""; var searchUrl = string.Format(templateUrl, pageParam); - var response = await RequestWithCookiesAndRetryAsync(searchUrl); + var response = await RequestStringWithCookiesAndRetry(searchUrl); var pageReleases = ParseReleases(response, query); // publish date is not available in the torrent list, but we add a relative date so we can sort @@ -120,7 +120,7 @@ namespace Jackett.Common.Indexers public override async Task Download(Uri link) { - var results = await WebRequestWithCookiesAsync(link.ToString()); + var results = await RequestStringWithCookies(link.ToString()); try { @@ -129,7 +129,7 @@ namespace Jackett.Common.Indexers var preotectedLink = dom.QuerySelector("a[service=BitTorrent]").GetAttribute("href"); preotectedLink = SiteLink + preotectedLink.TrimStart('/'); - results = await WebRequestWithCookiesAsync(preotectedLink); + results = await RequestStringWithCookies(preotectedLink); dom = parser.ParseDocument(results.ContentString); var magnetUrl = dom.QuerySelector("a[href^=magnet]").GetAttribute("href"); return await base.Download(new Uri(magnetUrl)); diff --git a/src/Jackett.Common/Indexers/CorsaroRed.cs b/src/Jackett.Common/Indexers/CorsaroRed.cs index b87cc600a..54f641591 100644 --- a/src/Jackett.Common/Indexers/CorsaroRed.cs +++ b/src/Jackett.Common/Indexers/CorsaroRed.cs @@ -130,14 +130,13 @@ namespace Jackett.Common.Indexers private async Task SendApiRequest(IEnumerable> data) { - var result = await RequestWithCookiesAndRetryAsync( - ApiSearch, null, RequestType.POST, SiteLink, data, _apiHeaders, null, true); + var result = await PostDataWithCookiesAndRetry(ApiSearch, data, null, SiteLink, _apiHeaders, null, true); return CheckResponse(result); } private async Task SendApiRequestLatest() { - var result = await RequestWithCookiesAndRetryAsync(ApiLatest, referer: SiteLink, headers: _apiHeaders); + var result = await RequestStringWithCookiesAndRetry(ApiLatest, null, SiteLink, _apiHeaders); return CheckResponse(result); } diff --git a/src/Jackett.Common/Indexers/DanishBits.cs b/src/Jackett.Common/Indexers/DanishBits.cs index 14343eefa..342a0ed97 100644 --- a/src/Jackett.Common/Indexers/DanishBits.cs +++ b/src/Jackett.Common/Indexers/DanishBits.cs @@ -65,13 +65,10 @@ namespace Jackett.Common.Indexers return searchString; } - protected override async Task WebRequestWithCookiesAsync( - string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, - IEnumerable> data = null, Dictionary headers = null, - string rawbody = null, bool? emulateBrowser = null) + protected override async Task RequestBytesWithCookies(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable> data = null, Dictionary headers = null) { CookieHeader = null; // Download fill fail with cookies set - return await base.WebRequestWithCookiesAsync(url, cookieOverride, method, referer, data, headers); + return await base.RequestBytesWithCookies(url, cookieOverride, method, referer, data, headers); } } } diff --git a/src/Jackett.Common/Indexers/DigitalCore.cs b/src/Jackett.Common/Indexers/DigitalCore.cs index ea016a4e4..27b327e08 100644 --- a/src/Jackett.Common/Indexers/DigitalCore.cs +++ b/src/Jackett.Common/Indexers/DigitalCore.cs @@ -136,7 +136,7 @@ namespace Jackett.Common.Indexers searchUrl += "?" + queryCollection.GetQueryString(); foreach (var cat in MapTorznabCapsToTrackers(query)) searchUrl += "&categories[]=" + cat; - var results = await WebRequestWithCookiesAsync(searchUrl, referer: SiteLink); + var results = await RequestStringWithCookies(searchUrl, null, SiteLink); try { diff --git a/src/Jackett.Common/Indexers/DigitalHive.cs b/src/Jackett.Common/Indexers/DigitalHive.cs index c1bd3672c..e1c9d67da 100644 --- a/src/Jackett.Common/Indexers/DigitalHive.cs +++ b/src/Jackett.Common/Indexers/DigitalHive.cs @@ -93,7 +93,7 @@ namespace Jackett.Common.Indexers public override async Task GetConfigurationForSetup() { - var loginPage = await WebRequestWithCookiesAsync(LoginUrl, configData.CookieHeader.Value); + var loginPage = await RequestStringWithCookies(LoginUrl, configData.CookieHeader.Value); var parser = new HtmlParser(); var cq = parser.ParseDocument(loginPage.ContentString); var recaptchaSiteKey = cq.QuerySelector(".g-recaptcha")?.GetAttribute("data-sitekey"); @@ -181,12 +181,12 @@ namespace Jackett.Common.Indexers queryCollection.Add("blah", "0"); - var results = await RequestWithCookiesAndRetryAsync(searchUrl + "?" + queryCollection.GetQueryString()); + var results = await RequestStringWithCookiesAndRetry(searchUrl + "?" + queryCollection.GetQueryString()); if (results.IsRedirect) { // re-login await ApplyConfiguration(null); - results = await RequestWithCookiesAndRetryAsync(searchUrl + "?" + queryCollection.GetQueryString()); + results = await RequestStringWithCookiesAndRetry(searchUrl + "?" + queryCollection.GetQueryString()); } try { diff --git a/src/Jackett.Common/Indexers/DivxTotal.cs b/src/Jackett.Common/Indexers/DivxTotal.cs index 097df53f8..ba92496a9 100644 --- a/src/Jackett.Common/Indexers/DivxTotal.cs +++ b/src/Jackett.Common/Indexers/DivxTotal.cs @@ -100,7 +100,7 @@ namespace Jackett.Common.Indexers do { var url = SiteLink + "page/" + page + "/?" + qc.GetQueryString(); - var result = await WebRequestWithCookiesAsync(url); + var result = await RequestStringWithCookies(url); if (result.Status != HttpStatusCode.OK) throw new ExceptionWithConfigData(result.ContentString, configData); @@ -153,7 +153,7 @@ namespace Jackett.Common.Indexers // for other categories we have to do another step if (!downloadUrl.Contains(DownloadLink)) { - var result = await WebRequestWithCookiesAsync(downloadUrl); + var result = await RequestStringWithCookies(downloadUrl); if (result.Status != HttpStatusCode.OK) throw new ExceptionWithConfigData(result.ContentString, configData); @@ -205,7 +205,7 @@ namespace Jackett.Common.Indexers private async Task ParseSeriesRelease(ICollection releases, TorznabQuery query, string commentsLink, string cat, DateTime publishDate) { - var result = await WebRequestWithCookiesAsync(commentsLink); + var result = await RequestStringWithCookies(commentsLink); if (result.Status != HttpStatusCode.OK) throw new ExceptionWithConfigData(result.ContentString, configData); diff --git a/src/Jackett.Common/Indexers/EliteTracker.cs b/src/Jackett.Common/Indexers/EliteTracker.cs index 51091b265..629748796 100644 --- a/src/Jackett.Common/Indexers/EliteTracker.cs +++ b/src/Jackett.Common/Indexers/EliteTracker.cs @@ -151,7 +151,7 @@ namespace Jackett.Common.Indexers { "password", configData.Password.Value } }; - var result = await WebRequestWithCookiesAsync(LoginUrl, "", RequestType.POST, data: pairs); + var result = await PostDataWithCookies(LoginUrl, pairs, ""); await ConfigureIfOK(result.Cookies, result.Cookies != null, () => { @@ -174,12 +174,12 @@ namespace Jackett.Common.Indexers {"category", "0"} // multi cat search not supported }; - var results = await WebRequestWithCookiesAsync(BrowseUrl, method: RequestType.POST, data: pairs); + var results = await PostDataWithCookies(BrowseUrl, pairs); if (results.IsRedirect) { // re-login await ApplyConfiguration(null); - results = await WebRequestWithCookiesAsync(BrowseUrl, method: RequestType.POST, data: pairs); + results = await PostDataWithCookies(BrowseUrl, pairs); } try diff --git a/src/Jackett.Common/Indexers/EpubLibre.cs b/src/Jackett.Common/Indexers/EpubLibre.cs index feca335c6..df1a96dff 100644 --- a/src/Jackett.Common/Indexers/EpubLibre.cs +++ b/src/Jackett.Common/Indexers/EpubLibre.cs @@ -9,7 +9,6 @@ using AngleSharp.Html.Parser; using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; -using Jackett.Common.Utils.Clients; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NLog; @@ -86,7 +85,7 @@ namespace Jackett.Common.Indexers for (var page = 0; page < maxPages; page++) { var searchUrl = string.Format(SearchUrl, page * MaxItemsPerPage, searchString); - var result = await WebRequestWithCookiesAsync(searchUrl, headers: _apiHeaders); + var result = await RequestStringWithCookies(searchUrl, null, null, _apiHeaders); try { @@ -150,7 +149,7 @@ namespace Jackett.Common.Indexers public override async Task Download(Uri link) { - var result = await RequestWithCookiesAndRetryAsync(link.AbsoluteUri); + var result = await RequestStringWithCookiesAndRetry(link.AbsoluteUri); if (SobrecargaUrl.Equals(result.RedirectingTo)) throw new Exception("El servidor se encuentra sobrecargado en estos momentos. / The server is currently overloaded."); try diff --git a/src/Jackett.Common/Indexers/Feeds/BaseFeedIndexer.cs b/src/Jackett.Common/Indexers/Feeds/BaseFeedIndexer.cs index a4a8e9b1b..89c1b224c 100644 --- a/src/Jackett.Common/Indexers/Feeds/BaseFeedIndexer.cs +++ b/src/Jackett.Common/Indexers/Feeds/BaseFeedIndexer.cs @@ -55,7 +55,7 @@ namespace Jackett.Common.Indexers.Feeds Type = RequestType.GET, Encoding = Encoding }; - var result = await webclient.GetResultAsync(request); + var result = await webclient.GetString(request); var results = ParseFeedForResults(result.ContentString); diff --git a/src/Jackett.Common/Indexers/FileList.cs b/src/Jackett.Common/Indexers/FileList.cs index d0cdf8107..75a110e41 100644 --- a/src/Jackett.Common/Indexers/FileList.cs +++ b/src/Jackett.Common/Indexers/FileList.cs @@ -188,7 +188,7 @@ namespace Jackett.Common.Indexers { {"Authorization", "Basic " + auth} }; - var response = await WebRequestWithCookiesAsync(searchUrl, headers: headers); + var response = await RequestStringWithCookies(searchUrl, headers: headers); return response.ContentString; } catch (Exception inner) diff --git a/src/Jackett.Common/Indexers/FunFile.cs b/src/Jackett.Common/Indexers/FunFile.cs index c6d9aaa78..04a20fdee 100644 --- a/src/Jackett.Common/Indexers/FunFile.cs +++ b/src/Jackett.Common/Indexers/FunFile.cs @@ -97,12 +97,12 @@ namespace Jackett.Common.Indexers qc.Add("search", query.GetQueryString()); var searchUrl = SearchUrl + "?" + qc.GetQueryString(); - var results = await RequestWithCookiesAndRetryAsync(searchUrl); + var results = await RequestStringWithCookiesAndRetry(searchUrl); if (results.IsRedirect) // re-login { await ApplyConfiguration(null); - results = await RequestWithCookiesAndRetryAsync(searchUrl); + results = await RequestStringWithCookiesAndRetry(searchUrl); } try diff --git a/src/Jackett.Common/Indexers/Fuzer.cs b/src/Jackett.Common/Indexers/Fuzer.cs index 2e58bdb38..cc01934aa 100644 --- a/src/Jackett.Common/Indexers/Fuzer.cs +++ b/src/Jackett.Common/Indexers/Fuzer.cs @@ -13,7 +13,6 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; -using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; @@ -108,7 +107,7 @@ namespace Jackett.Common.Indexers public override async Task GetConfigurationForSetup() { - var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty); + var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty); var parser = new HtmlParser(); var cq = parser.ParseDocument(loginPage.ContentString); var captcha = cq.QuerySelector(".g-recaptcha"); // invisible recaptcha @@ -154,7 +153,7 @@ namespace Jackett.Common.Indexers } } - var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty); + var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty); var pairs = new Dictionary { {"vb_login_username", configData.Username.Value}, @@ -205,7 +204,7 @@ namespace Jackett.Common.Indexers } searchUrl = MapTorznabCapsToTrackers(query).Aggregate(searchUrl, (current, cat) => $"{current}&c[]={cat}"); - var data = await RequestWithCookiesAndRetryAsync(searchUrl); + var data = await RequestStringWithCookiesAndRetry(searchUrl); try { var parser = new HtmlParser(); @@ -277,7 +276,7 @@ namespace Jackett.Common.Indexers Path = "index.php", Query = queryString.GetQueryString() }; - var results = await WebRequestWithCookiesAsync(site.ToString()); + var results = await RequestStringWithCookies(site.ToString()); var parser = new HtmlParser(); var dom = parser.ParseDocument(results.ContentString); var rows = dom.QuerySelectorAll("#listtable > tbody > tr"); diff --git a/src/Jackett.Common/Indexers/GazelleGames.cs b/src/Jackett.Common/Indexers/GazelleGames.cs index 8b4f80935..b683a47e8 100644 --- a/src/Jackett.Common/Indexers/GazelleGames.cs +++ b/src/Jackett.Common/Indexers/GazelleGames.cs @@ -225,7 +225,7 @@ namespace Jackett.Common.Indexers searchUrl += "?" + queryCollection.GetQueryString(); - var results = await WebRequestWithCookiesAsync(searchUrl); + var results = await RequestStringWithCookies(searchUrl); if (results.IsRedirect && results.RedirectingTo.EndsWith("login.php")) { throw new Exception("relogin needed, please update your cookie"); diff --git a/src/Jackett.Common/Indexers/GimmePeers.cs b/src/Jackett.Common/Indexers/GimmePeers.cs index 943ad2d22..415aa424b 100644 --- a/src/Jackett.Common/Indexers/GimmePeers.cs +++ b/src/Jackett.Common/Indexers/GimmePeers.cs @@ -88,7 +88,7 @@ namespace Jackett.Common.Indexers { "login", "Log in!" } }; - var loginPage = await WebRequestWithCookiesAsync(SiteLink, string.Empty); + var loginPage = await RequestStringWithCookies(SiteLink, string.Empty); var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, SiteLink, SiteLink); await ConfigureIfOK(result.Cookies, result.ContentString?.Contains("logout.php") == true, () => @@ -123,12 +123,12 @@ namespace Jackett.Common.Indexers } searchUrl += "?" + queryCollection.GetQueryString(); - var response = await RequestWithCookiesAndRetryAsync(searchUrl, referer: BrowseUrl); + var response = await RequestStringWithCookiesAndRetry(searchUrl, null, BrowseUrl); if (response.IsRedirect) { // re login await ApplyConfiguration(null); - response = await RequestWithCookiesAndRetryAsync(searchUrl, referer: BrowseUrl); + response = await RequestStringWithCookiesAndRetry(searchUrl, null, BrowseUrl); } var results = response.ContentString; diff --git a/src/Jackett.Common/Indexers/HDBitsApi.cs b/src/Jackett.Common/Indexers/HDBitsApi.cs index f6d4f8986..478f4bd6e 100644 --- a/src/Jackett.Common/Indexers/HDBitsApi.cs +++ b/src/Jackett.Common/Indexers/HDBitsApi.cs @@ -176,14 +176,14 @@ namespace Jackett.Common.Indexers requestData["username"] = configData.Username.Value; requestData["passkey"] = configData.Passkey.Value; JObject json = null; - var response = await RequestWithCookiesAndRetryAsync( - APIUrl + url, null, RequestType.POST, null, null, - new Dictionary - { - {"Accept", "application/json"}, - {"Content-Type", "application/json"} - }, requestData.ToString(), false); - CheckSiteDown(response); + + var response = await PostDataWithCookiesAndRetry(APIUrl + url, null, null, null, new Dictionary() + { + {"Accept", "application/json"}, + {"Content-Type", "application/json"} + }, requestData.ToString(), false); + + CheckTrackerDown(response); try { diff --git a/src/Jackett.Common/Indexers/HDSpace.cs b/src/Jackett.Common/Indexers/HDSpace.cs index 549fa49bb..4cb97ed81 100644 --- a/src/Jackett.Common/Indexers/HDSpace.cs +++ b/src/Jackett.Common/Indexers/HDSpace.cs @@ -77,7 +77,7 @@ namespace Jackett.Common.Indexers public override async Task ApplyConfiguration(JToken configJson) { LoadValuesFromJson(configJson); - var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty); + var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty); var pairs = new Dictionary { {"uid", configData.Username.Value}, @@ -119,7 +119,7 @@ namespace Jackett.Common.Indexers queryCollection.Add("search", query.GetQueryString()); } - var response = await RequestWithCookiesAndRetryAsync(SearchUrl + queryCollection.GetQueryString()); + var response = await RequestStringWithCookiesAndRetry(SearchUrl + queryCollection.GetQueryString()); try { diff --git a/src/Jackett.Common/Indexers/HDTorrents.cs b/src/Jackett.Common/Indexers/HDTorrents.cs index 39a046950..705a47a42 100644 --- a/src/Jackett.Common/Indexers/HDTorrents.cs +++ b/src/Jackett.Common/Indexers/HDTorrents.cs @@ -104,7 +104,7 @@ namespace Jackett.Common.Indexers { LoadValuesFromJson(configJson); - var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty); + var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty); var pairs = new Dictionary { { "uid", configData.Username.Value }, @@ -134,7 +134,7 @@ namespace Jackett.Common.Indexers // manually url encode parenthesis to prevent "hacking" detection searchUrl += queryCollection.GetQueryString().Replace("(", "%28").Replace(")", "%29"); - var results = await RequestWithCookiesAndRetryAsync(searchUrl); + var results = await RequestStringWithCookiesAndRetry(searchUrl); try { var parser = new HtmlParser(); diff --git a/src/Jackett.Common/Indexers/Hebits.cs b/src/Jackett.Common/Indexers/Hebits.cs index 0d329112e..cea44ac72 100644 --- a/src/Jackett.Common/Indexers/Hebits.cs +++ b/src/Jackett.Common/Indexers/Hebits.cs @@ -105,7 +105,7 @@ namespace Jackett.Common.Indexers var cats = MapTorznabCapsToTrackers(query); if (cats.Count > 0) searchUrl = cats.Aggregate(searchUrl, (url, cat) => $"{url}&c{cat}=1"); - var response = await WebRequestWithCookiesAsync(searchUrl); + var response = await RequestStringWithCookies(searchUrl); try { var parser = new HtmlParser(); diff --git a/src/Jackett.Common/Indexers/HorribleSubs.cs b/src/Jackett.Common/Indexers/HorribleSubs.cs index 7fae39625..89815d95c 100644 --- a/src/Jackett.Common/Indexers/HorribleSubs.cs +++ b/src/Jackett.Common/Indexers/HorribleSubs.cs @@ -72,7 +72,7 @@ namespace Jackett.Common.Indexers }; var searchUrl = ApiEndpoint + "?" + queryCollection.GetQueryString(); - var response = await RequestWithCookiesAndRetryAsync(searchUrl); + var response = await RequestStringWithCookiesAndRetry(searchUrl); try { @@ -112,7 +112,7 @@ namespace Jackett.Common.Indexers }; var searchUrl = ApiEndpoint + "?" + queryCollection.GetQueryString(); - var response = await RequestWithCookiesAndRetryAsync(searchUrl); + var response = await RequestStringWithCookiesAndRetry(searchUrl); try { @@ -146,7 +146,7 @@ namespace Jackett.Common.Indexers var releases = new List(); var parser = new HtmlParser(); - var response = await RequestWithCookiesAndRetryAsync(resultUrl); + var response = await RequestStringWithCookiesAndRetry(resultUrl); await FollowIfRedirect(response); try @@ -168,7 +168,7 @@ namespace Jackett.Common.Indexers var nextId = 0; while (true) { - var showApiResponse = await RequestWithCookiesAndRetryAsync(apiUrl + "&nextid=" + nextId); + var showApiResponse = await RequestStringWithCookiesAndRetry(apiUrl + "&nextid=" + nextId); var showApiDom = parser.ParseDocument(showApiResponse.ContentString); var releaseRowResults = showApiDom.QuerySelectorAll("div.rls-info-container"); rows.AddRange(releaseRowResults); diff --git a/src/Jackett.Common/Indexers/IPTorrents.cs b/src/Jackett.Common/Indexers/IPTorrents.cs index 3fd4fd004..5c71c3be4 100644 --- a/src/Jackett.Common/Indexers/IPTorrents.cs +++ b/src/Jackett.Common/Indexers/IPTorrents.cs @@ -10,7 +10,6 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; -using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; @@ -180,7 +179,7 @@ namespace Jackett.Common.Indexers qc.Add(cat, string.Empty); var searchUrl = SearchUrl + "?" + qc.GetQueryString(); - var response = await RequestWithCookiesAndRetryAsync(searchUrl, referer: SearchUrl); + var response = await RequestStringWithCookiesAndRetry(searchUrl, null, SearchUrl); var results = response.ContentString; if (results == null || !results.Contains("/lout.php")) diff --git a/src/Jackett.Common/Indexers/ImmortalSeed.cs b/src/Jackett.Common/Indexers/ImmortalSeed.cs index c81f41083..819b6d495 100644 --- a/src/Jackett.Common/Indexers/ImmortalSeed.cs +++ b/src/Jackett.Common/Indexers/ImmortalSeed.cs @@ -11,7 +11,6 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; -using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; @@ -122,13 +121,13 @@ namespace Jackett.Common.Indexers if (!string.IsNullOrWhiteSpace(query.GetQueryString())) searchUrl += string.Format(QueryString, WebUtility.UrlEncode(query.GetQueryString())); - var results = await RequestWithCookiesAndRetryAsync(searchUrl); + var results = await RequestStringWithCookiesAndRetry(searchUrl); // Occasionally the cookies become invalid, login again if that happens if (results.ContentString.Contains("You do not have permission to access this page.")) { await ApplyConfiguration(null); - results = await RequestWithCookiesAndRetryAsync(searchUrl); + results = await RequestStringWithCookiesAndRetry(searchUrl); } try diff --git a/src/Jackett.Common/Indexers/InternetArchive.cs b/src/Jackett.Common/Indexers/InternetArchive.cs index 2557d0636..62600891b 100644 --- a/src/Jackett.Common/Indexers/InternetArchive.cs +++ b/src/Jackett.Common/Indexers/InternetArchive.cs @@ -143,7 +143,7 @@ namespace Jackett.Common.Indexers {"output", "json"} }; var fullSearchUrl = SearchUrl + "?" + qc.GetQueryString(); - var result = await RequestWithCookiesAndRetryAsync(fullSearchUrl); + var result = await RequestStringWithCookiesAndRetry(fullSearchUrl); foreach (var torrent in ParseResponse(result)) releases.Add(MakeRelease(torrent)); diff --git a/src/Jackett.Common/Indexers/Libble.cs b/src/Jackett.Common/Indexers/Libble.cs index 105a6c31a..3aa7931dc 100644 --- a/src/Jackett.Common/Indexers/Libble.cs +++ b/src/Jackett.Common/Indexers/Libble.cs @@ -83,7 +83,7 @@ namespace Jackett.Common.Indexers public override async Task GetConfigurationForSetup() { - await WebRequestWithCookiesAsync(LandingUrl); + await RequestStringWithCookies(LandingUrl); return configData; } @@ -155,12 +155,12 @@ namespace Jackett.Common.Indexers searchUrl += "?" + queryCollection.GetQueryString(); - var searchPage = await RequestWithCookiesAndRetryAsync(searchUrl, method: RequestType.POST, data: searchParams); + var searchPage = await PostDataWithCookiesAndRetry(searchUrl, searchParams); // Occasionally the cookies become invalid, login again if that happens if (searchPage.IsRedirect) { await ApplyConfiguration(null); - searchPage = await RequestWithCookiesAndRetryAsync(searchUrl, method: RequestType.POST, data: searchParams); + searchPage = await PostDataWithCookiesAndRetry(searchUrl, searchParams); } try diff --git a/src/Jackett.Common/Indexers/LostFilm.cs b/src/Jackett.Common/Indexers/LostFilm.cs index 9aad1eb02..ae496a08e 100644 --- a/src/Jackett.Common/Indexers/LostFilm.cs +++ b/src/Jackett.Common/Indexers/LostFilm.cs @@ -113,14 +113,14 @@ namespace Jackett.Common.Indexers public override async Task GetConfigurationForSetup() { // looks like after some failed login attempts there's a captcha - var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty); + var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty); var parser = new HtmlParser(); var document = parser.ParseDocument(loginPage.ContentString); var qCaptchaImg = document.QuerySelector("img#captcha_pictcha"); if (qCaptchaImg != null) { var captchaUrl = SiteLink + qCaptchaImg.GetAttribute("src"); - var captchaImage = await WebRequestWithCookiesAsync(captchaUrl, loginPage.Cookies); + var captchaImage = await RequestBytesWithCookies(captchaUrl, loginPage.Cookies); configData.CaptchaImage.Value = captchaImage.ContentBytes; } else @@ -183,7 +183,7 @@ namespace Jackett.Common.Indexers { "type", "logout" } }; - var response = await WebRequestWithCookiesAsync(ApiUrl, method: RequestType.POST, data: data); + var response = await PostDataWithCookies(url: ApiUrl, data: data); logger.Debug("Logout result: " + response.ContentString); var isOK = response.Status == System.Net.HttpStatusCode.OK; @@ -214,7 +214,7 @@ namespace Jackett.Common.Indexers private async Task RequestStringAndRelogin(string url) { - var results = await WebRequestWithCookiesAsync(url); + var results = await RequestStringWithCookies(url); if (results.ContentString.Contains("503 Service")) { throw new ExceptionWithConfigData(results.ContentString, configData); @@ -223,7 +223,7 @@ namespace Jackett.Common.Indexers { // Re-login await ApplyConfiguration(null); - return await WebRequestWithCookiesAsync(url); + return await RequestStringWithCookies(url); } else { @@ -287,7 +287,7 @@ namespace Jackett.Common.Indexers { "val", searchString } }; logger.Debug("> Searching: " + searchString); - var response = await WebRequestWithCookiesAsync(ApiUrl, method: RequestType.POST, data: data); + var response = await PostDataWithCookies(url: ApiUrl, data: data); if (response.ContentString == null) { logger.Debug("> Empty series response for query: " + searchString); @@ -471,7 +471,7 @@ namespace Jackett.Common.Indexers logger.Debug("FetchSeriesReleases: " + url + " S: " + query.Season.ToString() + " E: " + query.Episode + " Filter: " + filter); var releases = new List(); - var results = await WebRequestWithCookiesAsync(url); + var results = await RequestStringWithCookies(url); try { @@ -626,7 +626,7 @@ namespace Jackett.Common.Indexers logger.Debug("FetchTrackerReleases: " + url); // Get redirection page with generated link on it. This link can't be constructed manually as it contains Hash field and hashing algo is unknown. - var results = await WebRequestWithCookiesAsync(url); + var results = await RequestStringWithCookies(url); if (results.ContentString == null) { throw new ExceptionWithConfigData("Empty response from " + url, configData); @@ -659,7 +659,7 @@ namespace Jackett.Common.Indexers private async Task> FollowTrackerRedirection(string url, TrackerUrlDetails details) { logger.Debug("FollowTrackerRedirection: " + url); - var results = await WebRequestWithCookiesAsync(url); + var results = await RequestStringWithCookies(url); var releases = new List(); try diff --git a/src/Jackett.Common/Indexers/MejorTorrent.cs b/src/Jackett.Common/Indexers/MejorTorrent.cs index 6ad562d8b..2275e2504 100644 --- a/src/Jackett.Common/Indexers/MejorTorrent.cs +++ b/src/Jackett.Common/Indexers/MejorTorrent.cs @@ -101,14 +101,14 @@ namespace Jackett.Common.Indexers var downloadUrl = link.ToString(); // Eg https://www.mejortorrentt.net/peli-descargar-torrent-11995-Harry-Potter-y-la-piedra-filosofal.html - var result = await WebRequestWithCookiesAsync(downloadUrl); + var result = await RequestStringWithCookies(downloadUrl); if (result.Status != HttpStatusCode.OK) throw new ExceptionWithConfigData(result.ContentString, configData); var dom = parser.ParseDocument(result.ContentString); downloadUrl = SiteLink + dom.QuerySelector("a[href*=\"sec=descargas\"]").GetAttribute("href"); // Eg https://www.mejortorrentt.net/secciones.php?sec=descargas&ap=contar&tabla=peliculas&id=11995&link_bajar=1 - result = await WebRequestWithCookiesAsync(downloadUrl); + result = await RequestStringWithCookies(downloadUrl); if (result.Status != HttpStatusCode.OK) throw new ExceptionWithConfigData(result.ContentString, configData); dom = parser.ParseDocument(result.ContentString); @@ -123,7 +123,7 @@ namespace Jackett.Common.Indexers { var releases = new List(); var url = SiteLink + NewTorrentsUrl; - var result = await WebRequestWithCookiesAsync(url); + var result = await RequestStringWithCookies(url); if (result.Status != HttpStatusCode.OK) throw new ExceptionWithConfigData(result.ContentString, configData); try @@ -180,7 +180,7 @@ namespace Jackett.Common.Indexers var searchTerm = GetLongestWord(query.SearchTerm); var qc = new NameValueCollection { { "sec", "buscador" }, { "valor", searchTerm } }; var url = SiteLink + SearchUrl + "?" + qc.GetQueryString(); - var result = await WebRequestWithCookiesAsync(url); + var result = await RequestStringWithCookies(url); if (result.Status != HttpStatusCode.OK) throw new ExceptionWithConfigData(result.ContentString, configData); @@ -259,7 +259,7 @@ namespace Jackett.Common.Indexers private async Task ParseSeriesRelease(ICollection releases, TorznabQuery query, string title, string commentsLink, string cat, DateTime publishDate) { - var result = await WebRequestWithCookiesAsync(commentsLink); + var result = await RequestStringWithCookies(commentsLink); if (result.Status != HttpStatusCode.OK) throw new ExceptionWithConfigData(result.ContentString, configData); diff --git a/src/Jackett.Common/Indexers/Milkie.cs b/src/Jackett.Common/Indexers/Milkie.cs index 43cb15b95..385410fee 100644 --- a/src/Jackett.Common/Indexers/Milkie.cs +++ b/src/Jackett.Common/Indexers/Milkie.cs @@ -87,7 +87,7 @@ namespace Jackett.Common.Indexers { { "x-milkie-auth", configData.Key.Value } }; - var jsonResponse = await WebRequestWithCookiesAsync(endpoint, headers: headers); + var jsonResponse = await RequestStringWithCookies(endpoint, headers: headers); var releases = new List(); diff --git a/src/Jackett.Common/Indexers/MoreThanTV.cs b/src/Jackett.Common/Indexers/MoreThanTV.cs index 167698207..57b681fcc 100644 --- a/src/Jackett.Common/Indexers/MoreThanTV.cs +++ b/src/Jackett.Common/Indexers/MoreThanTV.cs @@ -13,7 +13,6 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; -using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; @@ -60,7 +59,8 @@ namespace Jackett.Common.Indexers { "login", "Log in" }, { "keeplogged", "1" } }; - var preRequest = await RequestWithCookiesAndRetryAsync(LoginUrl, string.Empty); + + var preRequest = await RequestStringWithCookiesAndRetry(LoginUrl, string.Empty); var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, preRequest.Cookies, true, SearchUrl, SiteLink); await ConfigureIfOK(result.Cookies, result.ContentString != null && result.ContentString.Contains("status\":\"success\""), () => @@ -134,12 +134,12 @@ namespace Jackett.Common.Indexers private async Task GetReleases(ICollection releases, TorznabQuery query, string searchQuery) { var searchUrl = GetTorrentSearchUrl(query, searchQuery); - var response = await RequestWithCookiesAndRetryAsync(searchUrl); + var response = await RequestStringWithCookiesAndRetry(searchUrl); if (response.IsRedirect) { // re login await ApplyConfiguration(null); - response = await RequestWithCookiesAndRetryAsync(searchUrl); + response = await RequestStringWithCookiesAndRetry(searchUrl); } try diff --git a/src/Jackett.Common/Indexers/MyAmity.cs b/src/Jackett.Common/Indexers/MyAmity.cs index 0b60daced..5eb7d5a9a 100644 --- a/src/Jackett.Common/Indexers/MyAmity.cs +++ b/src/Jackett.Common/Indexers/MyAmity.cs @@ -119,12 +119,12 @@ namespace Jackett.Common.Indexers } searchUrl += "?" + queryCollection.GetQueryString(); - var response = await WebRequestWithCookiesAsync(searchUrl); + var response = await RequestStringWithCookies(searchUrl); if (response.IsRedirect || response.Cookies != null && response.Cookies.Contains("pass=deleted;")) { // re-login await ApplyConfiguration(null); - response = await WebRequestWithCookiesAsync(searchUrl); + response = await RequestStringWithCookies(searchUrl); } var results = response.ContentString; diff --git a/src/Jackett.Common/Indexers/MyAnonamouse.cs b/src/Jackett.Common/Indexers/MyAnonamouse.cs index bd2f56422..8ef2f44a1 100644 --- a/src/Jackett.Common/Indexers/MyAnonamouse.cs +++ b/src/Jackett.Common/Indexers/MyAnonamouse.cs @@ -199,7 +199,7 @@ namespace Jackett.Common.Indexers if (qParams.Count > 0) urlSearch += $"?{qParams.GetQueryString()}"; - var response = await RequestWithCookiesAndRetryAsync(urlSearch); + var response = await RequestStringWithCookiesAndRetry(urlSearch); if (response.ContentString.StartsWith("Error")) throw new Exception(response.ContentString); diff --git a/src/Jackett.Common/Indexers/NCore.cs b/src/Jackett.Common/Indexers/NCore.cs index f1c4b5443..9d4095d18 100644 --- a/src/Jackett.Common/Indexers/NCore.cs +++ b/src/Jackett.Common/Indexers/NCore.cs @@ -99,7 +99,7 @@ namespace Jackett.Common.Indexers LoadValuesFromJson(configJson); if (configData.Hungarian.Value == false && configData.English.Value == false) throw new ExceptionWithConfigData("Please select at least one language.", configData); - var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty); + var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty); var pairs = new Dictionary { {"nev", configData.Username.Value}, @@ -166,8 +166,7 @@ namespace Jackett.Common.Indexers cats = cats.Except(_languageCats).ToList(); pairs.Add("kivalasztott_tipus[]", string.Join(",", cats)); - var results = await RequestWithCookiesAndRetryAsync( - SearchUrl, null, RequestType.POST, null, pairs.ToEnumerable(true)); + var results = await PostDataWithCookiesAndRetry(SearchUrl, pairs.ToEnumerable(true)); var parser = new HtmlParser(); var dom = parser.ParseDocument(results.ContentString); @@ -200,8 +199,7 @@ namespace Jackett.Common.Indexers for (var page = startPage; page <= pages && releases.Count < limit; page++) { pairs["oldal"] = page.ToString(); - results = await RequestWithCookiesAndRetryAsync( - SearchUrl, null, RequestType.POST, null, pairs.ToEnumerable(true)); + results = await PostDataWithCookiesAndRetry(SearchUrl, pairs.ToEnumerable(true)); releases.AddRange(ParseTorrents(results, episodeString, query, releases.Count, limit, previouslyParsedOnPage)); previouslyParsedOnPage = 0; } diff --git a/src/Jackett.Common/Indexers/Nebulance.cs b/src/Jackett.Common/Indexers/Nebulance.cs index 19255a185..9a643c681 100644 --- a/src/Jackett.Common/Indexers/Nebulance.cs +++ b/src/Jackett.Common/Indexers/Nebulance.cs @@ -11,7 +11,6 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; -using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; @@ -75,14 +74,14 @@ namespace Jackett.Common.Indexers protected override async Task> PerformQuery(TorznabQuery query) { - var loggedInCheck = await WebRequestWithCookiesAsync(SearchUrl); + var loggedInCheck = await RequestStringWithCookies(SearchUrl); if (!loggedInCheck.ContentString.Contains("logout.php")) // re-login await DoLogin(); // #6413 var url = $"{SearchUrl}&searchtext={WebUtility.UrlEncode(query.GetQueryString())}"; - var response = await RequestWithCookiesAndRetryAsync(url); + var response = await RequestStringWithCookiesAndRetry(url); var releases = ParseResponse(response.ContentString); return releases; diff --git a/src/Jackett.Common/Indexers/NewPCT.cs b/src/Jackett.Common/Indexers/NewPCT.cs index 6dbc9b776..d8c4d869e 100644 --- a/src/Jackett.Common/Indexers/NewPCT.cs +++ b/src/Jackett.Common/Indexers/NewPCT.cs @@ -163,7 +163,7 @@ namespace Jackett.Common.Indexers public override async Task Download(Uri linkParam) { - var results = await RequestWithCookiesAndRetryAsync(linkParam.AbsoluteUri); + var results = await RequestStringWithCookiesAndRetry(linkParam.AbsoluteUri); var uriLink = ExtractDownloadUri(results.ContentString, linkParam.AbsoluteUri); if (uriLink == null) @@ -211,7 +211,7 @@ namespace Jackett.Common.Indexers while (pg <= _maxDailyPages) { var pageUrl = SiteLink + string.Format(_dailyUrl, pg); - var results = await RequestWithCookiesAndRetryAsync(pageUrl); + var results = await RequestStringWithCookiesAndRetry(pageUrl); if (results == null || string.IsNullOrEmpty(results.ContentString)) break; @@ -304,13 +304,13 @@ namespace Jackett.Common.Indexers var releases = new List(); // Episodes list - var results = await RequestWithCookiesAndRetryAsync(uri.AbsoluteUri); + var results = await RequestStringWithCookiesAndRetry(uri.AbsoluteUri); var seriesEpisodesUrl = ParseSeriesListContent(results.ContentString, seriesName); // TV serie list if (!string.IsNullOrEmpty(seriesEpisodesUrl)) { - results = await RequestWithCookiesAndRetryAsync(seriesEpisodesUrl); + results = await RequestStringWithCookiesAndRetry(seriesEpisodesUrl); var items = ParseEpisodesListContent(results.ContentString); if (items != null && items.Any()) releases.AddRange(items); @@ -464,7 +464,7 @@ namespace Jackett.Common.Indexers {"pg", pg.ToString()} }; - var results = await WebRequestWithCookiesAsync(searchJsonUrl, method: RequestType.POST, data: queryCollection); + var results = await PostDataWithCookies(searchJsonUrl, queryCollection); var items = ParseSearchJsonContent(results.ContentString, year); if (!items.Any()) break; diff --git a/src/Jackett.Common/Indexers/NewRealWorld.cs b/src/Jackett.Common/Indexers/NewRealWorld.cs index 721913354..ff8393815 100644 --- a/src/Jackett.Common/Indexers/NewRealWorld.cs +++ b/src/Jackett.Common/Indexers/NewRealWorld.cs @@ -150,12 +150,12 @@ namespace Jackett.Common.Indexers searchUrl += "?" + queryCollection.GetQueryString(); - var response = await WebRequestWithCookiesAsync(searchUrl); + var response = await RequestStringWithCookies(searchUrl); if (response.IsRedirect) { // re-login await ApplyConfiguration(null); - response = await WebRequestWithCookiesAsync(searchUrl); + response = await RequestStringWithCookies(searchUrl); } try diff --git a/src/Jackett.Common/Indexers/NorBits.cs b/src/Jackett.Common/Indexers/NorBits.cs index fd9ff8c30..b3935fd03 100644 --- a/src/Jackett.Common/Indexers/NorBits.cs +++ b/src/Jackett.Common/Indexers/NorBits.cs @@ -136,7 +136,7 @@ namespace Jackett.Common.Indexers // Get index page for cookies Output("\nGetting index page (for cookies).. with " + SiteLink); - var indexPage = await webclient.GetResultAsync(myIndexRequest); + var indexPage = await webclient.GetString(myIndexRequest); // Building login form data var pairs = new Dictionary { @@ -158,7 +158,7 @@ namespace Jackett.Common.Indexers // Get login page -- (not used, but simulation needed by tracker security's checks) LatencyNow(); Output("\nGetting login page (user simulation).. with " + LoginUrl); - await webclient.GetResultAsync(myRequestLogin); + await webclient.GetString(myRequestLogin); // Build WebRequest for submitting authentification var request = new Utils.Clients.WebRequest() @@ -175,7 +175,7 @@ namespace Jackett.Common.Indexers // Perform loggin LatencyNow(); Output("\nPerform loggin.. with " + LoginCheckUrl); - var response = await webclient.GetResultAsync(request); + var response = await webclient.GetString(request); // Test if we are logged in await ConfigureIfOK(response.Cookies, response.Cookies != null && response.Cookies.Contains("uid="), () => @@ -204,7 +204,7 @@ namespace Jackett.Common.Indexers { // Checking ... Output("\n-> Checking logged-in state...."); - var loggedInCheck = await WebRequestWithCookiesAsync(SearchUrl); + var loggedInCheck = await RequestStringWithCookies(SearchUrl); if (!loggedInCheck.ContentString.Contains("logout.php")) { // Cookie expired, renew session on provider @@ -261,7 +261,7 @@ namespace Jackett.Common.Indexers var request = BuildQuery(searchTerm, query, searchUrl); // Getting results & Store content - var response = await RequestWithCookiesAndRetryAsync(request, ConfigData.CookieHeader.Value); + var response = await RequestStringWithCookiesAndRetry(request, ConfigData.CookieHeader.Value); var parser = new HtmlParser(); var dom = parser.ParseDocument(response.ContentString); @@ -526,7 +526,7 @@ namespace Jackett.Common.Indexers // Request our first page LatencyNow(); - var results = await RequestWithCookiesAndRetryAsync(request, ConfigData.CookieHeader.Value, RequestType.GET, SearchUrl, null, _emulatedBrowserHeaders); + var results = await RequestStringWithCookiesAndRetry(request, ConfigData.CookieHeader.Value, SearchUrl, _emulatedBrowserHeaders); // Return results from tracker return results; diff --git a/src/Jackett.Common/Indexers/NordicBits.cs b/src/Jackett.Common/Indexers/NordicBits.cs index 6f4f3b6e7..1b6a118f7 100644 --- a/src/Jackett.Common/Indexers/NordicBits.cs +++ b/src/Jackett.Common/Indexers/NordicBits.cs @@ -191,7 +191,7 @@ namespace Jackett.Common.Indexers // Get index page for cookies Output("\nGetting index page (for cookies).. with " + SiteLink); - var indexPage = await webclient.GetResultAsync(myIndexRequest); + var indexPage = await webclient.GetString(myIndexRequest); // Building login form data var pairs = new Dictionary { @@ -213,7 +213,7 @@ namespace Jackett.Common.Indexers // Get login page -- (not used, but simulation needed by tracker security's checks) LatencyNow(); Output("\nGetting login page (user simulation).. with " + LoginUrl); - await webclient.GetResultAsync(myRequestLogin); + await webclient.GetString(myRequestLogin); // Build WebRequest for submitting authentification var request = new Utils.Clients.WebRequest() @@ -230,7 +230,7 @@ namespace Jackett.Common.Indexers // Perform loggin LatencyNow(); Output("\nPerform loggin.. with " + LoginCheckUrl); - var response = await webclient.GetResultAsync(request); + var response = await webclient.GetString(request); // Test if we are logged in await ConfigureIfOK(response.Cookies, response.Cookies != null && response.Cookies.Contains("uid="), () => @@ -259,7 +259,7 @@ namespace Jackett.Common.Indexers { // Checking ... Output("\n-> Checking logged-in state...."); - var loggedInCheck = await WebRequestWithCookiesAsync(SearchUrl); + var loggedInCheck = await RequestStringWithCookies(SearchUrl); if (!loggedInCheck.ContentString.Contains("logout.php")) { // Cookie expired, renew session on provider @@ -316,7 +316,7 @@ namespace Jackett.Common.Indexers var request = BuildQuery(searchTerm, query, searchUrl); // Getting results & Store content - var response = await RequestWithCookiesAndRetryAsync(request, ConfigData.CookieHeader.Value); + var response = await RequestStringWithCookiesAndRetry(request, ConfigData.CookieHeader.Value); var parser = new HtmlParser(); var dom = parser.ParseDocument(response.ContentString); @@ -597,7 +597,7 @@ namespace Jackett.Common.Indexers // Request our first page LatencyNow(); - var results = await RequestWithCookiesAndRetryAsync(request, ConfigData.CookieHeader.Value, RequestType.GET, SearchUrl, null, _emulatedBrowserHeaders); + var results = await RequestStringWithCookiesAndRetry(request, ConfigData.CookieHeader.Value, SearchUrl, _emulatedBrowserHeaders); // Return results from tracker return results; diff --git a/src/Jackett.Common/Indexers/Partis.cs b/src/Jackett.Common/Indexers/Partis.cs index a2597125e..8c4a91c14 100644 --- a/src/Jackett.Common/Indexers/Partis.cs +++ b/src/Jackett.Common/Indexers/Partis.cs @@ -135,13 +135,13 @@ namespace Jackett.Common.Indexers logger.Info(string.Format("Searh URL Partis_: {0}", searchUrl)); // add necessary headers - var header = new Dictionary + var heder = new Dictionary { { "X-requested-with", "XMLHttpRequest" } }; //get results and follow redirect - results = await WebRequestWithCookiesAsync(searchUrl, referer: SearchUrl, headers: header); + results = await RequestStringWithCookies(searchUrl, null, SearchUrl, heder); await FollowIfRedirect(results, null, null, null, true); // are we logged in? @@ -150,8 +150,7 @@ namespace Jackett.Common.Indexers await ApplyConfiguration(null); } // another request with specific query - NEEDED for succesful response - return data - results = await WebRequestWithCookiesAsync( - SiteLink + "brskaj/?rs=false&offset=0", referer: SearchUrl, headers: header); + results = await RequestStringWithCookies(SiteLink + "brskaj/?rs=false&offset=0", null, SearchUrl, heder); await FollowIfRedirect(results, null, null, null, true); // parse results diff --git a/src/Jackett.Common/Indexers/PassThePopcorn.cs b/src/Jackett.Common/Indexers/PassThePopcorn.cs index af10d7701..9a68a48d1 100644 --- a/src/Jackett.Common/Indexers/PassThePopcorn.cs +++ b/src/Jackett.Common/Indexers/PassThePopcorn.cs @@ -11,7 +11,6 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; -using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; @@ -116,9 +115,9 @@ namespace Jackett.Common.Indexers { "ApiKey", configData.Key.Value } }; - var results = await RequestWithCookiesAndRetryAsync(movieListSearchUrl, headers: authHeaders); + var results = await RequestStringWithCookiesAndRetry(movieListSearchUrl, headers: authHeaders); if (results.IsRedirect) // untested - results = await RequestWithCookiesAndRetryAsync(movieListSearchUrl, headers: authHeaders); + results = await RequestStringWithCookiesAndRetry(movieListSearchUrl, headers: authHeaders); try { //Iterate over the releases for each movie diff --git a/src/Jackett.Common/Indexers/PirateTheNet.cs b/src/Jackett.Common/Indexers/PirateTheNet.cs index e7797cdea..fa54d834f 100644 --- a/src/Jackett.Common/Indexers/PirateTheNet.cs +++ b/src/Jackett.Common/Indexers/PirateTheNet.cs @@ -76,7 +76,7 @@ namespace Jackett.Common.Indexers LoadValuesFromJson(configJson); CookieHeader = ""; // clear old cookies - var result1 = await WebRequestWithCookiesAsync(CaptchaUrl); + var result1 = await RequestStringWithCookies(CaptchaUrl); var json1 = JObject.Parse(result1.ContentString); var captchaSelection = json1["images"][0]["hash"]; @@ -125,12 +125,12 @@ namespace Jackett.Common.Indexers var searchUrl = SearchUrl + "?" + qc.GetQueryString(); - var results = await RequestWithCookiesAndRetryAsync(searchUrl); + var results = await RequestStringWithCookiesAndRetry(searchUrl); if (results.IsRedirect) { // re-login await ApplyConfiguration(null); - results = await RequestWithCookiesAndRetryAsync(searchUrl); + results = await RequestStringWithCookiesAndRetry(searchUrl); } try diff --git a/src/Jackett.Common/Indexers/PixelHD.cs b/src/Jackett.Common/Indexers/PixelHD.cs index 784e7be5a..2814aa60d 100644 --- a/src/Jackett.Common/Indexers/PixelHD.cs +++ b/src/Jackett.Common/Indexers/PixelHD.cs @@ -57,7 +57,7 @@ namespace Jackett.Common.Indexers public override async Task GetConfigurationForSetup() { - var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty); + var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty); var LoginParser = new HtmlParser(); var LoginDocument = LoginParser.ParseDocument(loginPage.ContentString); @@ -69,7 +69,7 @@ namespace Jackett.Common.Indexers var catchaInput = LoginDocument.QuerySelector("input[maxlength=\"6\"]"); input_captcha = catchaInput.GetAttribute("name"); - var captchaImage = await WebRequestWithCookiesAsync(SiteLink + catchaImg.GetAttribute("src"), loginPage.Cookies, RequestType.GET, LoginUrl); + var captchaImage = await RequestBytesWithCookies(SiteLink + catchaImg.GetAttribute("src"), loginPage.Cookies, RequestType.GET, LoginUrl); configData.CaptchaImage.Value = captchaImage.ContentBytes; } else @@ -137,13 +137,13 @@ namespace Jackett.Common.Indexers var searchUrl = BrowseUrl + "?" + queryCollection.GetQueryString(); - var results = await WebRequestWithCookiesAsync(searchUrl); + var results = await RequestStringWithCookies(searchUrl); if (results.IsRedirect) { // re login await GetConfigurationForSetup(); await ApplyConfiguration(null); - results = await WebRequestWithCookiesAsync(searchUrl); + results = await RequestStringWithCookies(searchUrl); } var IMDBRegEx = new Regex(@"tt(\d+)", RegexOptions.Compiled); diff --git a/src/Jackett.Common/Indexers/PolishTracker.cs b/src/Jackett.Common/Indexers/PolishTracker.cs index 4ca0ea8fd..a035bdac1 100644 --- a/src/Jackett.Common/Indexers/PolishTracker.cs +++ b/src/Jackett.Common/Indexers/PolishTracker.cs @@ -104,12 +104,12 @@ namespace Jackett.Common.Indexers qc.Add("cat[]", cat); var searchUrl = SearchUrl + "?" + qc.GetQueryString(); - var result = await RequestWithCookiesAndRetryAsync(searchUrl, referer: SearchUrl); + var result = await RequestStringWithCookiesAndRetry(searchUrl, null, SearchUrl); if (result.IsRedirect) { // re-login await ApplyConfiguration(null); - result = await RequestWithCookiesAndRetryAsync(searchUrl, referer: SearchUrl); + result = await RequestStringWithCookiesAndRetry(searchUrl, null, SearchUrl); } if (!result.ContentString.StartsWith("{")) // not JSON => error diff --git a/src/Jackett.Common/Indexers/PornoLab.cs b/src/Jackett.Common/Indexers/PornoLab.cs index 65e8a12cd..d86068011 100644 --- a/src/Jackett.Common/Indexers/PornoLab.cs +++ b/src/Jackett.Common/Indexers/PornoLab.cs @@ -183,13 +183,13 @@ namespace Jackett.Common.Indexers public override async Task GetConfigurationForSetup() { configData.CookieHeader.Value = null; - var response = await WebRequestWithCookiesAsync(LoginUrl); + var response = await RequestStringWithCookies(LoginUrl); var LoginResultParser = new HtmlParser(); var LoginResultDocument = LoginResultParser.ParseDocument(response.ContentString); var captchaimg = LoginResultDocument.QuerySelector("img[src*=\"/captcha/\"]"); if (captchaimg != null) { - var captchaImage = await WebRequestWithCookiesAsync("https:" + captchaimg.GetAttribute("src")); + var captchaImage = await RequestBytesWithCookies("https:" + captchaimg.GetAttribute("src")); configData.CaptchaImage.Value = captchaImage.ContentBytes; var codefield = LoginResultDocument.QuerySelector("input[name^=\"cap_code_\"]"); @@ -260,12 +260,12 @@ namespace Jackett.Common.Indexers } var searchUrl = SearchUrl + "?" + queryCollection.GetQueryString(); - var results = await WebRequestWithCookiesAsync(searchUrl); + var results = await RequestStringWithCookies(searchUrl); if (!results.ContentString.Contains("Вы зашли как:")) { // re login await ApplyConfiguration(null); - results = await WebRequestWithCookiesAsync(searchUrl); + results = await RequestStringWithCookies(searchUrl); } try { @@ -337,7 +337,7 @@ namespace Jackett.Common.Indexers public override async Task Download(Uri link) { var downloadlink = link; - var response = await WebRequestWithCookiesAsync(link.ToString()); + var response = await RequestStringWithCookies(link.ToString()); var results = response.ContentString; var SearchResultParser = new HtmlParser(); var SearchResultDocument = SearchResultParser.ParseDocument(results); diff --git a/src/Jackett.Common/Indexers/PreToMe.cs b/src/Jackett.Common/Indexers/PreToMe.cs index 297a5d97b..e2d793802 100644 --- a/src/Jackett.Common/Indexers/PreToMe.cs +++ b/src/Jackett.Common/Indexers/PreToMe.cs @@ -123,7 +123,7 @@ namespace Jackett.Common.Indexers { LoadValuesFromJson(configJson); - var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty); + var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty); var pairs = new Dictionary { { "returnto", "%2F" }, @@ -134,7 +134,7 @@ namespace Jackett.Common.Indexers }; // Send Post - var result = await WebRequestWithCookiesAsync(LoginUrl, loginPage.Cookies, RequestType.POST, data: pairs); + var result = await PostDataWithCookies(LoginUrl, pairs, loginPage.Cookies); if (result.RedirectingTo == null) throw new ExceptionWithConfigData("Login failed. Did you use the PIN number that pretome emailed you?", configData); @@ -200,12 +200,12 @@ namespace Jackett.Common.Indexers } var searchUrl = SearchUrl + "?" + qc.GetQueryString(); - var response = await RequestWithCookiesAndRetryAsync(searchUrl); + var response = await RequestStringWithCookiesAndRetry(searchUrl); if (response.IsRedirect) // re-login { await ApplyConfiguration(null); - response = await RequestWithCookiesAndRetryAsync(searchUrl); + response = await RequestStringWithCookiesAndRetry(searchUrl); } try diff --git a/src/Jackett.Common/Indexers/RarBG.cs b/src/Jackett.Common/Indexers/RarBG.cs index 6df85a31d..a9c7c3def 100644 --- a/src/Jackett.Common/Indexers/RarBG.cs +++ b/src/Jackett.Common/Indexers/RarBG.cs @@ -11,7 +11,6 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; -using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using static Jackett.Common.Models.IndexerConfig.ConfigurationData; @@ -121,7 +120,7 @@ namespace Jackett.Common.Indexers // check the token and renewal if necessary await RenewalTokenAsync(); - var response = await RequestWithCookiesAndRetryAsync(BuildSearchUrl(query)); + var response = await RequestStringWithCookiesAndRetry(BuildSearchUrl(query)); var jsonContent = JObject.Parse(response.ContentString); var errorCode = jsonContent.Value("error_code"); switch (errorCode) @@ -131,7 +130,7 @@ namespace Jackett.Common.Indexers case 2: case 4: // invalid token await RenewalTokenAsync(true); // force renewal token - response = await RequestWithCookiesAndRetryAsync(BuildSearchUrl(query)); + response = await RequestStringWithCookiesAndRetry(BuildSearchUrl(query)); jsonContent = JObject.Parse(response.ContentString); break; case 10: // imdb not found, see issue #1486 @@ -264,7 +263,7 @@ namespace Jackett.Common.Indexers { "app_id", _appId } }; var tokenUrl = ApiEndpoint + "?" + qc.GetQueryString(); - var result = await RequestWithCookiesAndRetryAsync(tokenUrl); + var result = await RequestStringWithCookiesAndRetry(tokenUrl); var json = JObject.Parse(result.ContentString); _token = json.Value("token"); _lastTokenFetch = DateTime.Now; diff --git a/src/Jackett.Common/Indexers/RevolutionTT.cs b/src/Jackett.Common/Indexers/RevolutionTT.cs index 8f6799c50..ae51113fb 100644 --- a/src/Jackett.Common/Indexers/RevolutionTT.cs +++ b/src/Jackett.Common/Indexers/RevolutionTT.cs @@ -13,7 +13,6 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; -using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; @@ -208,8 +207,7 @@ namespace Jackett.Common.Indexers var rssParams = new Dictionary { { "feed", "dl" } }; - var rssPage = await WebRequestWithCookiesAsync( - GetRSSKeyUrl, result.Cookies, RequestType.POST, data: rssParams); + var rssPage = await PostDataWithCookies(GetRSSKeyUrl, rssParams, result.Cookies); var match = Regex.Match(rssPage.ContentString, "(?<=passkey\\=)([a-zA-z0-9]*)"); configData.RSSKey.Value = match.Success ? match.Value : string.Empty; if (string.IsNullOrWhiteSpace(configData.RSSKey.Value)) @@ -234,7 +232,7 @@ namespace Jackett.Common.Indexers // If query is empty, use the RSS Feed if (string.IsNullOrWhiteSpace(searchString)) { - var rssPage = await RequestWithCookiesAndRetryAsync(RSSUrl + configData.RSSKey.Value); + var rssPage = await RequestStringWithCookiesAndRetry(RSSUrl + configData.RSSKey.Value); var rssDoc = XDocument.Parse(rssPage.ContentString); foreach (var item in rssDoc.Descendants("item")) @@ -307,12 +305,12 @@ namespace Jackett.Common.Indexers } } - var results = await RequestWithCookiesAndRetryAsync(searchUrl); + var results = await RequestStringWithCookiesAndRetry(searchUrl); if (results.IsRedirect) { // re-login await ApplyConfiguration(null); - results = await RequestWithCookiesAndRetryAsync(searchUrl); + results = await RequestStringWithCookiesAndRetry(searchUrl); } try diff --git a/src/Jackett.Common/Indexers/RuTracker.cs b/src/Jackett.Common/Indexers/RuTracker.cs index e9ba74c43..d63cea89c 100644 --- a/src/Jackett.Common/Indexers/RuTracker.cs +++ b/src/Jackett.Common/Indexers/RuTracker.cs @@ -1510,13 +1510,13 @@ namespace Jackett.Common.Indexers try { configData.CookieHeader.Value = null; - var response = await WebRequestWithCookiesAsync(LoginUrl); + var response = await RequestStringWithCookies(LoginUrl); var parser = new HtmlParser(); var doc = parser.ParseDocument(response.ContentString); var captchaimg = doc.QuerySelector("img[src^=\"https://static.t-ru.org/captcha/\"]"); if (captchaimg != null) { - var captchaImage = await WebRequestWithCookiesAsync(captchaimg.GetAttribute("src")); + var captchaImage = await RequestBytesWithCookies(captchaimg.GetAttribute("src")); configData.CaptchaImage.Value = captchaImage.ContentBytes; var codefield = doc.QuerySelector("input[name^=\"cap_code_\"]"); @@ -1591,12 +1591,12 @@ namespace Jackett.Common.Indexers } var searchUrl = SearchUrl + "?" + queryCollection.GetQueryString(); - var results = await WebRequestWithCookiesAsync(searchUrl); + var results = await RequestStringWithCookies(searchUrl); if (!results.ContentString.Contains("id=\"logged-in-username\"")) { // re login await ApplyConfiguration(null); - results = await WebRequestWithCookiesAsync(searchUrl); + results = await RequestStringWithCookies(searchUrl); } try { diff --git a/src/Jackett.Common/Indexers/SceneHD.cs b/src/Jackett.Common/Indexers/SceneHD.cs index 92cf694b9..465d6fc6f 100644 --- a/src/Jackett.Common/Indexers/SceneHD.cs +++ b/src/Jackett.Common/Indexers/SceneHD.cs @@ -91,7 +91,7 @@ namespace Jackett.Common.Indexers qc.Add("categories[" + cat + "]", "1"); var searchUrl = SearchUrl + qc.GetQueryString(); - var response = await RequestWithCookiesAndRetryAsync(searchUrl); + var response = await RequestStringWithCookiesAndRetry(searchUrl); if (response.ContentString?.Contains("User not found or passkey not set") == true) throw new Exception("The passkey is invalid. Check the indexer configuration."); diff --git a/src/Jackett.Common/Indexers/SceneTime.cs b/src/Jackett.Common/Indexers/SceneTime.cs index 382c31655..77c6a7476 100644 --- a/src/Jackett.Common/Indexers/SceneTime.cs +++ b/src/Jackett.Common/Indexers/SceneTime.cs @@ -107,7 +107,7 @@ namespace Jackett.Common.Indexers WebResult loginPage; try { - loginPage = await WebRequestWithCookiesAsync(StartPageUrl, string.Empty); + loginPage = await RequestStringWithCookies(StartPageUrl, string.Empty); } catch (Exception) { @@ -198,7 +198,7 @@ namespace Jackett.Common.Indexers qParams.Add("freeleech", "on"); var searchUrl = SearchUrl + "?" + qParams.GetQueryString(); - var results = await WebRequestWithCookiesAsync(searchUrl); + var results = await RequestStringWithCookies(searchUrl); // response without results (the message is misleading) if (results.ContentString?.Contains("slow down geek!!!") == true) diff --git a/src/Jackett.Common/Indexers/Shazbat.cs b/src/Jackett.Common/Indexers/Shazbat.cs index c7336d801..d2ba1a25e 100644 --- a/src/Jackett.Common/Indexers/Shazbat.cs +++ b/src/Jackett.Common/Indexers/Shazbat.cs @@ -67,7 +67,7 @@ namespace Jackett.Common.Indexers var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, null, true, null, LoginUrl); await ConfigureIfOK(result.Cookies, result.ContentString?.Contains("glyphicon-log-out") == true, () => throw new ExceptionWithConfigData("The username and password entered do not match.", configData)); - var rssProfile = await RequestWithCookiesAndRetryAsync(RSSProfile); + var rssProfile = await RequestStringWithCookiesAndRetry(RSSProfile); var parser = new HtmlParser(); var rssDom = parser.ParseDocument(rssProfile.ContentString); configData.RSSKey.Value = rssDom.QuerySelector(".col-sm-9:nth-of-type(1)").TextContent.Trim(); @@ -89,8 +89,7 @@ namespace Jackett.Common.Indexers { {"search", query.SanitizedSearchTerm} }; - results = await RequestWithCookiesAndRetryAsync( - SearchUrl, null, RequestType.POST, TorrentsUrl, pairs); + results = await PostDataWithCookiesAndRetry(SearchUrl, pairs, null, TorrentsUrl); results = await ReloginIfNecessary(results); var parser = new HtmlParser(); var dom = parser.ParseDocument(results.ContentString); @@ -108,7 +107,7 @@ namespace Jackett.Common.Indexers { foreach (var searchUrl in searchUrls) { - results = await WebRequestWithCookiesAsync(searchUrl); + results = await RequestStringWithCookies(searchUrl); results = await ReloginIfNecessary(results); var parser = new HtmlParser(); var dom = parser.ParseDocument(results.ContentString); @@ -178,7 +177,7 @@ namespace Jackett.Common.Indexers await ApplyConfiguration(null); response.Request.Cookies = CookieHeader; - return await webclient.GetResultAsync(response.Request); + return await webclient.GetString(response.Request); } } } diff --git a/src/Jackett.Common/Indexers/ShowRSS.cs b/src/Jackett.Common/Indexers/ShowRSS.cs index c9fa9d805..15fac9e92 100644 --- a/src/Jackett.Common/Indexers/ShowRSS.cs +++ b/src/Jackett.Common/Indexers/ShowRSS.cs @@ -60,7 +60,7 @@ namespace Jackett.Common.Indexers { var releases = new List(); var episodeSearchUrl = string.Format(SearchAllUrl); - var result = await RequestWithCookiesAndRetryAsync(episodeSearchUrl); + var result = await RequestStringWithCookiesAndRetry(episodeSearchUrl); var xmlDoc = new XmlDocument(); try diff --git a/src/Jackett.Common/Indexers/SolidTorrents.cs b/src/Jackett.Common/Indexers/SolidTorrents.cs index 868dd3725..ae99b0cbe 100644 --- a/src/Jackett.Common/Indexers/SolidTorrents.cs +++ b/src/Jackett.Common/Indexers/SolidTorrents.cs @@ -103,7 +103,7 @@ namespace Jackett.Common.Indexers {"fuv", "no"} }; var fullSearchUrl = SearchUrl + "?" + queryCollection.GetQueryString(); - var result = await WebRequestWithCookiesAsync(fullSearchUrl, headers: APIHeaders); + var result = await RequestStringWithCookies(fullSearchUrl, null, null, APIHeaders); return CheckResponse(result); } diff --git a/src/Jackett.Common/Indexers/SpeedCD.cs b/src/Jackett.Common/Indexers/SpeedCD.cs index 30889a2ea..30fda5b9e 100644 --- a/src/Jackett.Common/Indexers/SpeedCD.cs +++ b/src/Jackett.Common/Indexers/SpeedCD.cs @@ -151,11 +151,11 @@ namespace Jackett.Common.Indexers } var searchUrl = SearchUrl + string.Join("/", qc); - var response = await RequestWithCookiesAndRetryAsync(searchUrl); + var response = await RequestStringWithCookiesAndRetry(searchUrl); if (!response.ContentString.Contains("/logout.php")) // re-login { await DoLogin(); - response = await RequestWithCookiesAndRetryAsync(searchUrl); + response = await RequestStringWithCookiesAndRetry(searchUrl); } try diff --git a/src/Jackett.Common/Indexers/SuperBits.cs b/src/Jackett.Common/Indexers/SuperBits.cs index 504b09753..b329eb83a 100644 --- a/src/Jackett.Common/Indexers/SuperBits.cs +++ b/src/Jackett.Common/Indexers/SuperBits.cs @@ -128,7 +128,7 @@ namespace Jackett.Common.Indexers searchUrl += "?" + queryCollection.GetQueryString(); foreach (var cat in MapTorznabCapsToTrackers(query)) searchUrl += "&categories[]=" + cat; - var results = await WebRequestWithCookiesAsync(searchUrl, referer: SiteLink); + var results = await RequestStringWithCookies(searchUrl, null, SiteLink); try { diff --git a/src/Jackett.Common/Indexers/TVStore.cs b/src/Jackett.Common/Indexers/TVStore.cs index ce285b28c..494e826cb 100644 --- a/src/Jackett.Common/Indexers/TVStore.cs +++ b/src/Jackett.Common/Indexers/TVStore.cs @@ -66,7 +66,7 @@ namespace Jackett.Common.Indexers public override async Task ApplyConfiguration(JToken configJson) { LoadValuesFromJson(configJson); - var loginPage = await WebRequestWithCookiesAsync(LoginPageUrl, string.Empty); + var loginPage = await RequestStringWithCookies(LoginPageUrl, string.Empty); var pairs = new Dictionary { {"username", configData.Username.Value}, @@ -163,7 +163,7 @@ namespace Jackett.Common.Indexers queryParams["id"] = torrentId; queryParams["now"] = DateTimeUtil.DateTimeToUnixTimestamp(DateTime.UtcNow) .ToString(CultureInfo.InvariantCulture); - var filesList = (await RequestWithCookiesAndRetryAsync(SearchUrl + "?" + queryParams.GetQueryString())) + var filesList = (await RequestStringWithCookiesAndRetry(SearchUrl + "?" + queryParams.GetQueryString())) .ContentString; var firstFileName = filesList.Split( new[] @@ -215,7 +215,7 @@ namespace Jackett.Common.Indexers /// private async Task PopulateImdbMapAsync() { - var result = await RequestWithCookiesAndRetryAsync(BrowseUrl); + var result = await RequestStringWithCookiesAndRetry(BrowseUrl); foreach (Match match in _seriesInfoMatch.Matches(result.ContentString)) { var internalId = int.Parse(match.Groups["seriesID"].Value); @@ -280,7 +280,7 @@ namespace Jackett.Common.Indexers queryParams.Add("e", query.Episode); } - var results = await RequestWithCookiesAndRetryAsync(SearchUrl + "?" + queryParams.GetQueryString()); + var results = await RequestStringWithCookiesAndRetry(SearchUrl + "?" + queryParams.GetQueryString()); // Parse page Information from result var content = results.ContentString; var splits = content.Split('\\'); @@ -302,7 +302,7 @@ namespace Jackett.Common.Indexers for (var page = startPage; page <= pages && releases.Count < query.Limit; page++) { queryParams["page"] = page.ToString(); - results = await RequestWithCookiesAndRetryAsync(SearchUrl + "?" + queryParams.GetQueryString()); + results = await RequestStringWithCookiesAndRetry(SearchUrl + "?" + queryParams.GetQueryString()); releases.AddRange(await ParseTorrentsAsync(results, releases.Count, query.Limit, previouslyParsedOnPage)); previouslyParsedOnPage = 0; } diff --git a/src/Jackett.Common/Indexers/TVVault.cs b/src/Jackett.Common/Indexers/TVVault.cs index 2df4a7d9f..f390713d4 100644 --- a/src/Jackett.Common/Indexers/TVVault.cs +++ b/src/Jackett.Common/Indexers/TVVault.cs @@ -94,7 +94,7 @@ namespace Jackett.Common.Indexers qc.Add("searchstr", StripSearchString(query.GetQueryString())); var searchUrl = BrowseUrl + "?" + qc.GetQueryString(); - var results = await WebRequestWithCookiesAsync(searchUrl); + var results = await RequestStringWithCookies(searchUrl); try { var seasonRegEx = new Regex(@$"Season\s+0*{query.Season}[^\d]", RegexOptions.IgnoreCase); diff --git a/src/Jackett.Common/Indexers/Toloka.cs b/src/Jackett.Common/Indexers/Toloka.cs index a183f3e6a..3c1a05ce4 100644 --- a/src/Jackett.Common/Indexers/Toloka.cs +++ b/src/Jackett.Common/Indexers/Toloka.cs @@ -232,12 +232,12 @@ namespace Jackett.Common.Indexers } var searchUrl = SearchUrl + "?" + queryCollection.GetQueryString(); - var results = await WebRequestWithCookiesAsync(searchUrl); + var results = await RequestStringWithCookies(searchUrl); if (!results.ContentString.Contains("logout=true")) { // re login await ApplyConfiguration(null); - results = await WebRequestWithCookiesAsync(searchUrl); + results = await RequestStringWithCookies(searchUrl); } try { diff --git a/src/Jackett.Common/Indexers/TorrenTech.cs b/src/Jackett.Common/Indexers/TorrenTech.cs index 3bc2f7972..c66d55d1c 100644 --- a/src/Jackett.Common/Indexers/TorrenTech.cs +++ b/src/Jackett.Common/Indexers/TorrenTech.cs @@ -100,10 +100,10 @@ namespace Jackett.Common.Indexers } var searchUrl = IndexUrl + "?" + queryCollection.GetQueryString(); - results = await WebRequestWithCookiesAsync(searchUrl); + results = await RequestStringWithCookies(searchUrl); if (results.IsRedirect && results.RedirectingTo.Contains("CODE=show")) { - results = await WebRequestWithCookiesAsync(results.RedirectingTo); + results = await RequestStringWithCookies(results.RedirectingTo); } try { @@ -203,7 +203,7 @@ namespace Jackett.Common.Indexers public override async Task Download(Uri link) { - var response = await WebRequestWithCookiesAsync(link.ToString()); + var response = await RequestStringWithCookies(link.ToString()); var results = response.ContentString; var SearchResultParser = new HtmlParser(); var SearchResultDocument = SearchResultParser.ParseDocument(results); diff --git a/src/Jackett.Common/Indexers/TorrentBytes.cs b/src/Jackett.Common/Indexers/TorrentBytes.cs index f826ada7a..0f35363ab 100644 --- a/src/Jackett.Common/Indexers/TorrentBytes.cs +++ b/src/Jackett.Common/Indexers/TorrentBytes.cs @@ -89,7 +89,7 @@ namespace Jackett.Common.Indexers {"returnto", "/"}, {"login", "Log in!"} }; - var loginPage = await WebRequestWithCookiesAsync(SiteLink, string.Empty); + var loginPage = await RequestStringWithCookies(SiteLink, string.Empty); var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, SiteLink, SiteLink); await ConfigureIfOK( result.Cookies, result.ContentString?.Contains("my.php") == true, () => @@ -127,12 +127,12 @@ namespace Jackett.Common.Indexers qc.Add("c" + cat, "1"); var searchUrl = SearchUrl + "?" + qc.GetQueryString(); - var response = await RequestWithCookiesAndRetryAsync(searchUrl, referer: SearchUrl); + var response = await RequestStringWithCookiesAndRetry(searchUrl, referer: SearchUrl); if (response.IsRedirect) // re-login { await ApplyConfiguration(null); - response = await RequestWithCookiesAndRetryAsync(searchUrl, referer: SearchUrl); + response = await RequestStringWithCookiesAndRetry(searchUrl, null, SearchUrl); } try diff --git a/src/Jackett.Common/Indexers/TorrentDay.cs b/src/Jackett.Common/Indexers/TorrentDay.cs index 66b8fc39b..435f724b1 100644 --- a/src/Jackett.Common/Indexers/TorrentDay.cs +++ b/src/Jackett.Common/Indexers/TorrentDay.cs @@ -155,7 +155,7 @@ namespace Jackett.Common.Indexers else searchUrl += ";q=" + WebUtilityHelpers.UrlEncode(query.GetQueryString(), Encoding); - var results = await RequestWithCookiesAndRetryAsync(searchUrl); + var results = await RequestStringWithCookiesAndRetry(searchUrl); // Check for being logged out if (results.IsRedirect) diff --git a/src/Jackett.Common/Indexers/TorrentHeaven.cs b/src/Jackett.Common/Indexers/TorrentHeaven.cs index aceab8235..be7f398e7 100644 --- a/src/Jackett.Common/Indexers/TorrentHeaven.cs +++ b/src/Jackett.Common/Indexers/TorrentHeaven.cs @@ -119,7 +119,7 @@ namespace Jackett.Common.Indexers throw new ExceptionWithConfigData(errorMessage, configData); } - var result2 = await WebRequestWithCookiesAsync(LoginCompleteUrl, result.Cookies); + var result2 = await RequestStringWithCookies(LoginCompleteUrl, result.Cookies); await ConfigureIfOK( result2.Cookies, result2.Cookies?.Contains("pass") == true, () => throw new ExceptionWithConfigData("Didn't get a user/pass cookie", configData)); @@ -128,14 +128,14 @@ namespace Jackett.Common.Indexers public override async Task GetConfigurationForSetup() { - var loginPage = await WebRequestWithCookiesAsync(IndexUrl, string.Empty); + var loginPage = await RequestStringWithCookies(IndexUrl, string.Empty); var parser = new HtmlParser(); var dom = parser.ParseDocument(loginPage.ContentString); var qCaptchaImg = dom.QuerySelector("td.tablea > img"); if (qCaptchaImg != null) { var captchaUrl = SiteLink + qCaptchaImg.GetAttribute("src"); - var captchaImage = await WebRequestWithCookiesAsync(captchaUrl, loginPage.Cookies); + var captchaImage = await RequestBytesWithCookies(captchaUrl, loginPage.Cookies); configData.CaptchaImage.Value = captchaImage.ContentBytes; } else @@ -182,7 +182,7 @@ namespace Jackett.Common.Indexers foreach (var cat in MapTorznabCapsToTrackers(query)) queryCollection.Add("dirs" + cat, "1"); searchUrl += "?" + queryCollection.GetQueryString(); - var response = await WebRequestWithCookiesAsync(searchUrl); + var response = await RequestStringWithCookies(searchUrl); var titleRegexp = new Regex(@"^return buildTable\('(.*?)',\s+"); try { diff --git a/src/Jackett.Common/Indexers/TorrentLeech.cs b/src/Jackett.Common/Indexers/TorrentLeech.cs index e5bc21bd7..5d8bd7de8 100644 --- a/src/Jackett.Common/Indexers/TorrentLeech.cs +++ b/src/Jackett.Common/Indexers/TorrentLeech.cs @@ -12,7 +12,6 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; -using Jackett.Common.Utils.Clients; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NLog; @@ -108,7 +107,7 @@ namespace Jackett.Common.Indexers public override async Task GetConfigurationForSetup() { - var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty); + var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty); var parser = new HtmlParser(); var dom = parser.ParseDocument(loginPage.ContentString); var captcha = dom.QuerySelector(".g-recaptcha"); @@ -199,12 +198,12 @@ namespace Jackett.Common.Indexers else searchUrl += "newfilter/2"; // include 0day and music - var results = await RequestWithCookiesAndRetryAsync(searchUrl); + var results = await RequestStringWithCookiesAndRetry(searchUrl); if (results.ContentString.Contains("/user/account/login")) // re-login { await DoLogin(); - results = await RequestWithCookiesAndRetryAsync(searchUrl); + results = await RequestStringWithCookiesAndRetry(searchUrl); } try diff --git a/src/Jackett.Common/Indexers/TorrentNetwork.cs b/src/Jackett.Common/Indexers/TorrentNetwork.cs index f6e31f81d..57bc20aae 100644 --- a/src/Jackett.Common/Indexers/TorrentNetwork.cs +++ b/src/Jackett.Common/Indexers/TorrentNetwork.cs @@ -124,8 +124,7 @@ namespace Jackett.Common.Indexers private async Task SendAPIRequest(string endpoint, object data) { var jsonData = JsonConvert.SerializeObject(data); - var result = await WebRequestWithCookiesAsync( - APIUrl + endpoint, method: RequestType.POST, referer: SiteLink, headers: APIHeaders, rawbody: jsonData); + var result = await PostDataWithCookies(APIUrl + endpoint, null, null, SiteLink, APIHeaders, jsonData); if (!result.ContentString.StartsWith("{")) // not JSON => error throw new ExceptionWithConfigData(result.ContentString, configData); dynamic json = JsonConvert.DeserializeObject(result.ContentString); diff --git a/src/Jackett.Common/Indexers/TorrentSeeds.cs b/src/Jackett.Common/Indexers/TorrentSeeds.cs index c79826b3a..3fea81e80 100644 --- a/src/Jackett.Common/Indexers/TorrentSeeds.cs +++ b/src/Jackett.Common/Indexers/TorrentSeeds.cs @@ -12,7 +12,6 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; -using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; @@ -105,7 +104,7 @@ namespace Jackett.Common.Indexers public override async Task ApplyConfiguration(JToken configJson) { LoadValuesFromJson(configJson); - var loginPage = await WebRequestWithCookiesAsync(TokenUrl); + var loginPage = await RequestStringWithCookies(TokenUrl); var parser = new HtmlParser(); var dom = parser.ParseDocument(loginPage.ContentString); var token = dom.QuerySelector("form.form-horizontal > span"); @@ -150,12 +149,12 @@ namespace Jackett.Common.Indexers foreach (var cat in MapTorznabCapsToTrackers(query)) queryCollection.Add($"cat[{cat}]", "1"); searchUrl += "?" + queryCollection.GetQueryString(); - var response = await RequestWithCookiesAndRetryAsync(searchUrl); + var response = await RequestStringWithCookiesAndRetry(searchUrl); var results = response.ContentString; if (!results.Contains("/logout.php?")) { await ApplyConfiguration(null); - response = await RequestWithCookiesAndRetryAsync(searchUrl); + response = await RequestStringWithCookiesAndRetry(searchUrl); results = response.ContentString; } diff --git a/src/Jackett.Common/Indexers/TorrentSyndikat.cs b/src/Jackett.Common/Indexers/TorrentSyndikat.cs index 20fc1aab4..40e791e2c 100644 --- a/src/Jackett.Common/Indexers/TorrentSyndikat.cs +++ b/src/Jackett.Common/Indexers/TorrentSyndikat.cs @@ -111,7 +111,7 @@ namespace Jackett.Common.Indexers LoadValuesFromJson(configJson); CookieHeader = ""; - var result1 = await WebRequestWithCookiesAsync(CaptchaUrl); + var result1 = await RequestStringWithCookies(CaptchaUrl); var json1 = JObject.Parse(result1.ContentString); var captchaSelection = json1["images"][0]["hash"]; @@ -172,12 +172,12 @@ namespace Jackett.Common.Indexers var searchUrl = SearchUrl + "?" + queryCollection.GetQueryString(); - var results = await RequestWithCookiesAndRetryAsync(searchUrl); + var results = await RequestStringWithCookiesAndRetry(searchUrl); if (results.IsRedirect) { await ApplyConfiguration(null); - results = await RequestWithCookiesAndRetryAsync(searchUrl); + results = await RequestStringWithCookiesAndRetry(searchUrl); } try diff --git a/src/Jackett.Common/Indexers/TorrentsCSV.cs b/src/Jackett.Common/Indexers/TorrentsCSV.cs index ec5355a9f..83bae844d 100644 --- a/src/Jackett.Common/Indexers/TorrentsCSV.cs +++ b/src/Jackett.Common/Indexers/TorrentsCSV.cs @@ -70,7 +70,7 @@ namespace Jackett.Common.Indexers var searchUrl = (string.IsNullOrWhiteSpace(searchString) ? NewEndpoint : SearchEndpoint) + "?" + qc.GetQueryString(); - var response = await RequestWithCookiesAndRetryAsync(searchUrl); + var response = await RequestStringWithCookiesAndRetry(searchUrl); try { diff --git a/src/Jackett.Common/Indexers/XSpeeds.cs b/src/Jackett.Common/Indexers/XSpeeds.cs index 72341f0ff..3a918d122 100644 --- a/src/Jackett.Common/Indexers/XSpeeds.cs +++ b/src/Jackett.Common/Indexers/XSpeeds.cs @@ -132,14 +132,14 @@ namespace Jackett.Common.Indexers public override async Task GetConfigurationForSetup() { - var loginPage = await WebRequestWithCookiesAsync(LandingUrl); + var loginPage = await RequestStringWithCookies(LandingUrl); var parser = new HtmlParser(); var dom = parser.ParseDocument(loginPage.ContentString); var qCaptchaImg = dom.QuerySelector("img#regimage"); if (qCaptchaImg != null) { var captchaUrl = qCaptchaImg.GetAttribute("src"); - var captchaImageResponse = await WebRequestWithCookiesAsync(captchaUrl, loginPage.Cookies, RequestType.GET, LandingUrl); + var captchaImageResponse = await RequestBytesWithCookies(captchaUrl, loginPage.Cookies, RequestType.GET, LandingUrl); var captchaText = new StringItem { Name = "Captcha Text" }; var captchaImage = new ImageItem {Name = "Captcha Image", Value = captchaImageResponse.ContentBytes}; @@ -189,8 +189,7 @@ namespace Jackett.Common.Indexers {"timezone", "0"}, {"showrows", "50"} }; - var rssPage = await WebRequestWithCookiesAsync( - GetRSSKeyUrl, result.Cookies, RequestType.POST, data: rssParams); + var rssPage = await PostDataWithCookies(GetRSSKeyUrl, rssParams, result.Cookies); var match = Regex.Match(rssPage.ContentString, "(?<=secret_key\\=)([a-zA-z0-9]*)"); configData.RSSKey.Value = match.Success ? match.Value : string.Empty; if (string.IsNullOrWhiteSpace(configData.RSSKey.Value)) @@ -228,14 +227,12 @@ namespace Jackett.Common.Indexers searchParams.Add("search_type", "t_name"); } - var searchPage = await RequestWithCookiesAndRetryAsync( - SearchUrl, CookieHeader, RequestType.POST, null, searchParams); + var searchPage = await PostDataWithCookiesAndRetry(SearchUrl, searchParams, CookieHeader); // Occasionally the cookies become invalid, login again if that happens if (searchPage.IsRedirect) { await ApplyConfiguration(null); - searchPage = await RequestWithCookiesAndRetryAsync( - SearchUrl, CookieHeader, RequestType.POST, null, searchParams); + searchPage = await PostDataWithCookiesAndRetry(SearchUrl, searchParams, CookieHeader); } try diff --git a/src/Jackett.Common/Indexers/Xthor.cs b/src/Jackett.Common/Indexers/Xthor.cs index adda0e449..afd9e3443 100644 --- a/src/Jackett.Common/Indexers/Xthor.cs +++ b/src/Jackett.Common/Indexers/Xthor.cs @@ -17,7 +17,6 @@ using Jackett.Common.Utils.Clients; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NLog; -using WebRequest = Jackett.Common.Utils.Clients.WebRequest; namespace Jackett.Common.Indexers { @@ -489,7 +488,7 @@ namespace Jackett.Common.Indexers }; // Request our first page - var results = await webclient.GetResultAsync(myIndexRequest); + var results = await webclient.GetString(myIndexRequest); if (results.Status == HttpStatusCode.InternalServerError) // See issue #2110 throw new Exception("Internal Server Error (" + results.ContentString + "), probably you reached the API limits, please reduce the number of queries"); diff --git a/src/Jackett.Common/Indexers/YTS.cs b/src/Jackett.Common/Indexers/YTS.cs index ab3526079..f4f3d8f73 100644 --- a/src/Jackett.Common/Indexers/YTS.cs +++ b/src/Jackett.Common/Indexers/YTS.cs @@ -99,7 +99,7 @@ namespace Jackett.Common.Indexers } var searchUrl = ApiEndpoint + "?" + queryCollection.GetQueryString(); - var response = await RequestWithCookiesAndRetryAsync(searchUrl); + var response = await RequestStringWithCookiesAndRetry(searchUrl); try { diff --git a/src/Jackett.Common/Services/ImdbResolver.cs b/src/Jackett.Common/Services/ImdbResolver.cs index d35136c7e..9b8b6785b 100644 --- a/src/Jackett.Common/Services/ImdbResolver.cs +++ b/src/Jackett.Common/Services/ImdbResolver.cs @@ -37,7 +37,7 @@ namespace Jackett.Common.Services { Encoding = Encoding.UTF8 }; - var result = await WebClient.GetResultAsync(request); + var result = await WebClient.GetString(request); var movie = JsonConvert.DeserializeObject(result.ContentString); return movie; diff --git a/src/Jackett.Common/Services/UpdateService.cs b/src/Jackett.Common/Services/UpdateService.cs index 01cb46fcd..0d4f28d60 100644 --- a/src/Jackett.Common/Services/UpdateService.cs +++ b/src/Jackett.Common/Services/UpdateService.cs @@ -114,7 +114,7 @@ namespace Jackett.Common.Services try { - var response = await client.GetResultAsync(new WebRequest() + var response = await client.GetString(new WebRequest() { Url = "https://api.github.com/repos/Jackett/Jackett/releases", Encoding = Encoding.UTF8, @@ -258,11 +258,11 @@ namespace Jackett.Common.Services var url = targetAsset.Browser_download_url; - var data = await client.GetResultAsync(SetDownloadHeaders(new WebRequest() { Url = url, EmulateBrowser = true, Type = RequestType.GET })); + var data = await client.GetBytes(SetDownloadHeaders(new WebRequest() { Url = url, EmulateBrowser = true, Type = RequestType.GET })); while (data.IsRedirect) { - data = await client.GetResultAsync(new WebRequest() { Url = data.RedirectingTo, EmulateBrowser = true, Type = RequestType.GET }); + data = await client.GetBytes(new WebRequest() { Url = data.RedirectingTo, EmulateBrowser = true, Type = RequestType.GET }); } var tempDir = Path.Combine(Path.GetTempPath(), "JackettUpdate-" + version + "-" + DateTime.Now.Ticks); diff --git a/src/Jackett.Common/Utils/Clients/WebClient.cs b/src/Jackett.Common/Utils/Clients/WebClient.cs index 507eef3a4..6350e7076 100644 --- a/src/Jackett.Common/Utils/Clients/WebClient.cs +++ b/src/Jackett.Common/Utils/Clients/WebClient.cs @@ -186,24 +186,38 @@ namespace Jackett.Common.Utils.Clients return; } - public virtual async Task GetResultAsync(WebRequest request) + public virtual async Task GetBytes(WebRequest request) { - logger.Debug(string.Format("WebClient({0}).GetResultAsync(Url:{1})", ClientType, request.Url)); + logger.Debug(string.Format("WebClient({0}).GetBytes(Url:{1})", ClientType, request.Url)); PrepareRequest(request); await DelayRequest(request); var result = await Run(request); lastRequest = DateTime.Now; result.Request = request; - logger.Debug( - string.Format( - "WebClient({0}): Returning {1} => {2} bytes", ClientType, result.Status, - (result.IsRedirect ? result.RedirectingTo + " " : "") + - (result.ContentBytes == null ? "" : result.ContentBytes.Length.ToString()))); - if (result.Headers.TryGetValue("server", out var server) && server[0] == "cloudflare-nginx") - result.ContentString = BrowserUtil.DecodeCloudFlareProtectedEmailFromHTML(result.ContentString); + logger.Debug(string.Format("WebClient({0}): Returning {1} => {2} bytes", ClientType, result.Status, (result.IsRedirect ? result.RedirectingTo + " " : "") + (result.ContentBytes == null ? "" : result.ContentBytes.Length.ToString()))); return result; } + public virtual async Task GetString(WebRequest request) + { + logger.Debug(string.Format("WebClient({0}).GetString(Url:{1})", ClientType, request.Url)); + PrepareRequest(request); + await DelayRequest(request); + var result = await Run(request); + lastRequest = DateTime.Now; + result.Request = request; + WebResult stringResult = result; + + logger.Debug(string.Format("WebClient({0}): Returning {1} => {2}", ClientType, result.Status, (result.IsRedirect ? result.RedirectingTo + " " : "") + (stringResult.ContentString ?? ""))); + + if (stringResult.Headers.TryGetValue("server", out var server)) + { + if (server[0] == "cloudflare-nginx") + stringResult.ContentString = BrowserUtil.DecodeCloudFlareProtectedEmailFromHTML(stringResult.ContentString); + } + return stringResult; + } + #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously protected virtual async Task Run(WebRequest webRequest) => throw new NotImplementedException(); #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously diff --git a/src/Jackett.Test/TestUtil.cs b/src/Jackett.Test/TestUtil.cs index 9a18e7c8a..652f8d604 100644 --- a/src/Jackett.Test/TestUtil.cs +++ b/src/Jackett.Test/TestUtil.cs @@ -33,11 +33,16 @@ namespace Jackett.Test public static IContainer Container => testContainer; - //Currently not used in any Unit Tests - public static void RegisterRequestCallback(WebRequest r, Func f) + public static void RegisterByteCall(WebRequest r, Func f) { var client = testContainer.Resolve() as TestWebClient; - client.RegisterRequestCallback(r, f); + client.RegisterByteCall(r, f); + } + + public static void RegisterStringCall(WebRequest r, Func f) + { + var client = testContainer.Resolve() as TestWebClient; + client.RegisterStringCall(r, f); } public static string GetResource(string item) diff --git a/src/Jackett.Test/TestWebClient.cs b/src/Jackett.Test/TestWebClient.cs index fe691a337..b7a51abad 100644 --- a/src/Jackett.Test/TestWebClient.cs +++ b/src/Jackett.Test/TestWebClient.cs @@ -9,20 +9,26 @@ using NLog; namespace Jackett.Test { - - // Currently not used in any Unit tests. Leaving it for potential future testing purposes. public class TestWebClient : WebClient { - private readonly Dictionary> _requestCallbacks = new Dictionary>(); + private readonly Dictionary> byteCallbacks = new Dictionary>(); + private readonly Dictionary> stringCallbacks = new Dictionary>(); public TestWebClient(IProcessService p, Logger l, IConfigurationService c, ServerConfig sc) - : base(p, l, c, sc) + : base(p: p, + l: l, + c: c, + sc: sc) { } - public void RegisterRequestCallback(WebRequest req, Func f) => _requestCallbacks.Add(req, f); + public void RegisterByteCall(WebRequest req, Func f) => byteCallbacks.Add(req, f); - public override Task GetResultAsync(WebRequest request) => Task.FromResult(_requestCallbacks.First(r => r.Key.Equals(request)).Value.Invoke(request)); + public void RegisterStringCall(WebRequest req, Func f) => stringCallbacks.Add(req, f); + + public override Task GetBytes(WebRequest request) => Task.FromResult(byteCallbacks.Where(r => r.Key.Equals(request)).First().Value.Invoke(request)); + + public override Task GetString(WebRequest request) => Task.FromResult(stringCallbacks.Where(r => r.Key.Equals(request)).First().Value.Invoke(request)); public override void Init() {