mirror
/
Jackett
mirror of https://github.com/Jackett/Jackett
1
0
Fork 0
Code Issues Releases Wiki Activity

core: refactor http webclient part 13 #8529 (#8941) 

Merge string/byte request code
This commit is contained in:
Cory 2020-06-11 10:09:27 -05:00 committed by ngosang
parent 22bced9ca3
commit 4d63fa81d5
91 changed files with 330 additions and 399 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/Jackett.Common/Indexers/Abnormal.cs
View File

@ -21,6 +21,7 @@ using Jackett.Common.Utils.Clients;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
using WebRequest = Jackett.Common.Utils.Clients.WebRequest;
namespace Jackett.Common.Indexers
{
@ -166,7 +167,7 @@ namespace Jackett.Common.Indexers
// Perform loggin
latencyNow();
output("\nPerform loggin.. with " + LoginUrl);
var response = await webclient.GetString(request);
var response = await webclient.GetResultAsync(request);
// Test if we are logged in
await ConfigureIfOK(response.Cookies, response.Cookies.Contains("session="), () =>
@ -548,14 +549,12 @@ namespace Jackett.Common.Indexers
/// <returns>Results from query</returns>
private async Task<string> queryTracker(string request)
{
WebResult results = null;
// Cache mode not enabled or cached file didn't exist for our query
output("\nQuerying tracker for results....");
// Request our first page
latencyNow();
results = await RequestStringWithCookiesAndRetry(request, null, null, emulatedBrowserHeaders);
var results = await RequestWithCookiesAndRetryAsync(request, headers: emulatedBrowserHeaders);
// Return results from tracker
return results.ContentString;

7
src/Jackett.Common/Indexers/Abstract/AvistazTracker.cs
View File

@ -10,6 +10,7 @@ using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json.Linq;
using NLog;
using WebClient = Jackett.Common.Utils.Clients.WebClient;
@ -148,7 +149,7 @@ without this configuration the torrent download does not work.<br/>You can find
{ "password", configData.Password.Value.Trim() },
{ "pid", configData.Pid.Value.Trim() }
};
var result = await PostDataWithCookies(AuthUrl, body, headers: AuthHeaders);
var result = await WebRequestWithCookiesAsync(AuthUrl, method: RequestType.POST, data: body, headers: AuthHeaders);
var json = JObject.Parse(result.ContentString);
_token = json.Value<string>("token");
if (_token == null)
@ -161,11 +162,11 @@ without this configuration the torrent download does not work.<br/>You can find
var qc = GetSearchQueryParameters(query);
var episodeSearchUrl = SearchUrl + "?" + qc.GetQueryString();
var response = await RequestStringWithCookiesAndRetry(episodeSearchUrl, headers: GetSearchHeaders());
var response = await RequestWithCookiesAndRetryAsync(episodeSearchUrl, headers: GetSearchHeaders());
if (response.Status == HttpStatusCode.Unauthorized || response.Status == HttpStatusCode.PreconditionFailed)
{
await RenewalTokenAsync();
response = await RequestStringWithCookiesAndRetry(episodeSearchUrl, headers: GetSearchHeaders());
response = await RequestWithCookiesAndRetryAsync(episodeSearchUrl, headers: GetSearchHeaders());
}
else if (response.Status != HttpStatusCode.OK)
throw new Exception($"Unknown error: {response.ContentString}");

2
src/Jackett.Common/Indexers/Abstract/CouchPotatoTracker.cs
View File

@ -75,7 +75,7 @@ namespace Jackett.Common.Indexers.Abstract
searchUrl += "?" + queryCollection.GetQueryString();
var response = await RequestStringWithCookiesAndRetry(searchUrl);
var response = await RequestWithCookiesAndRetryAsync(searchUrl);
JObject json = null;
try

5
src/Jackett.Common/Indexers/Abstract/GazelleTracker.cs
View File

@ -12,6 +12,7 @@ using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json.Linq;
using NLog;
using WebClient = Jackett.Common.Utils.Clients.WebClient;
@ -196,12 +197,12 @@ namespace Jackett.Common.Indexers.Abstract
searchUrl += "?" + queryCollection.GetQueryString();
var response = await RequestStringWithCookiesAndRetry(searchUrl);
var response = await RequestWithCookiesAndRetryAsync(searchUrl);
if (response.IsRedirect)
{
// re-login
await ApplyConfiguration(null);
response = await RequestStringWithCookiesAndRetry(searchUrl);
response = await RequestWithCookiesAndRetryAsync(searchUrl);
}
try

12
src/Jackett.Common/Indexers/Abstract/XtremeZoneTracker.cs
View File

@ -10,6 +10,7 @@ using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
@ -74,7 +75,8 @@ namespace Jackett.Common.Indexers.Abstract
{ "password", configData.Password.Value.Trim() }
};
var jsonData = JsonConvert.SerializeObject(body);
var result = await PostDataWithCookies(LoginUrl, null, headers: ApiHeaders, rawbody: jsonData);
var result = await WebRequestWithCookiesAsync(
LoginUrl, method: RequestType.POST, headers: ApiHeaders, rawbody: jsonData);
var json = JObject.Parse(result.ContentString);
_token = json.Value<string>("token");
if (_token == null)
@ -105,11 +107,11 @@ namespace Jackett.Common.Indexers.Abstract
await RenewalTokenAsync();
var searchUrl = SearchUrl + "?" + qc.GetQueryString();
var response = await RequestStringWithCookies(searchUrl, headers: GetSearchHeaders());
var response = await WebRequestWithCookiesAsync(searchUrl, headers: GetSearchHeaders());
if (response.Status == HttpStatusCode.Unauthorized)
{
await RenewalTokenAsync(); // re-login
response = await RequestStringWithCookies(searchUrl, headers: GetSearchHeaders());
response = await WebRequestWithCookiesAsync(searchUrl, headers: GetSearchHeaders());
}
else if (response.Status != HttpStatusCode.OK)
throw new Exception($"Unknown error in search: {response.ContentString}");
@ -167,11 +169,11 @@ namespace Jackett.Common.Indexers.Abstract
public override async Task<byte[]> Download(Uri link)
{
var response = await RequestBytesWithCookies(link.ToString(), headers: GetSearchHeaders());
var response = await WebRequestWithCookiesAsync(link.ToString(), headers: GetSearchHeaders());
if (response.Status == HttpStatusCode.Unauthorized)
{
await RenewalTokenAsync();
response = await RequestBytesWithCookies(link.ToString(), headers: GetSearchHeaders());
response = await WebRequestWithCookiesAsync(link.ToString(), headers: GetSearchHeaders());
}
else if (response.Status != HttpStatusCode.OK)
throw new Exception($"Unknown error in download: {response.ContentBytes}");

10
src/Jackett.Common/Indexers/AniDUB.cs
View File

@ -145,7 +145,7 @@ namespace Jackett.Common.Indexers
private async Task EnsureAuthorized()
{
var result = await RequestStringWithCookiesAndRetry(SiteLink);
var result = await RequestWithCookiesAndRetryAsync(SiteLink);
if (!IsAuthorized(result))
{
@ -156,8 +156,7 @@ namespace Jackett.Common.Indexers
private async Task<List<ReleaseInfo>> FetchNewReleases()
{
const string ReleaseLinksSelector = "#dle-content > .story > .story_h > .lcol > h2 > a";
var result = await RequestStringWithCookiesAndRetry(SiteLink);
var result = await RequestWithCookiesAndRetryAsync(SiteLink);
var releases = new List<ReleaseInfo>();
try
@ -195,7 +194,7 @@ namespace Jackett.Common.Indexers
return releases;
}
var result = await RequestStringWithCookiesAndRetry(url);
var result = await RequestWithCookiesAndRetryAsync(url);
try
{
@ -529,8 +528,7 @@ namespace Jackett.Common.Indexers
const string searchLinkSelector = "#dle-content > .searchitem > h3 > a";
var releases = new List<ReleaseInfo>();
var response = await PostDataWithCookiesAndRetry(SearchUrl, PreparePostData(query));
var response = await RequestWithCookiesAndRetryAsync(SearchUrl, method: RequestType.POST, data: PreparePostData(query));
try
{

7
src/Jackett.Common/Indexers/Anidex.cs
View File

@ -14,6 +14,7 @@ using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json.Linq;
using NLog;
using static Jackett.Common.Models.IndexerConfig.ConfigurationData;
@ -150,13 +151,13 @@ namespace Jackett.Common.Indexers
// Make search request
var searchUri = GetAbsoluteUrl("?" + queryParameters.GetQueryString());
var response = await RequestStringWithCookiesAndRetry(searchUri.AbsoluteUri);
var response = await RequestWithCookiesAndRetryAsync(searchUri.AbsoluteUri);
// Check for DDOS Guard
if (response.Status == System.Net.HttpStatusCode.Forbidden)
{
await ConfigureDDoSGuardCookie();
response = await RequestStringWithCookiesAndRetry(searchUri.AbsoluteUri);
response = await RequestWithCookiesAndRetryAsync(searchUri.AbsoluteUri);
}
if (response.Status != System.Net.HttpStatusCode.OK)
@ -216,7 +217,7 @@ namespace Jackett.Common.Indexers
private async Task ConfigureDDoSGuardCookie()
{
const string ddosPostUrl = "https://check.ddos-guard.net/check.js";
var response = await RequestStringWithCookies(ddosPostUrl, string.Empty);
var response = await WebRequestWithCookiesAsync(ddosPostUrl, string.Empty);
if (response.Status != System.Net.HttpStatusCode.OK)
throw new WebException($"Unexpected DDOS Guard response: Status: {response.Status}", WebExceptionStatus.ProtocolError);
if (response.IsRedirect)

3
src/Jackett.Common/Indexers/AnimeBytes.cs
View File

@ -12,6 +12,7 @@ using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig.Bespoke;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
@ -179,7 +180,7 @@ namespace Jackett.Common.Indexers
}
// Get the content from the tracker
var response = await RequestStringWithCookiesAndRetry(queryUrl);
var response = await RequestWithCookiesAndRetryAsync(queryUrl);
if (!response.ContentString.StartsWith("{")) // not JSON => error
throw new ExceptionWithConfigData("unexcepted response (not JSON)", configData);
dynamic json = JsonConvert.DeserializeObject<dynamic>(response.ContentString);

7
src/Jackett.Common/Indexers/AnimeTorrents.cs
View File

@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Diagnostics.CodeAnalysis;
@ -79,7 +79,7 @@ namespace Jackett.Common.Indexers
{ "rememberme[]", "1" }
};
var loginPage = await RequestStringWithCookiesAndRetry(LoginUrl, "", LoginUrl);
var loginPage = await RequestWithCookiesAndRetryAsync(LoginUrl, "", RequestType.GET, LoginUrl);
var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true);
await ConfigureIfOK(result.Cookies, result.ContentString != null && result.ContentString.Contains("logout.php"), () =>
@ -117,7 +117,8 @@ namespace Jackett.Common.Indexers
{ "X-Requested-With", "XMLHttpRequest" }
};
var response = await RequestStringWithCookiesAndRetry(searchUrl, null, SearchUrlReferer, extraHeaders);
var response = await RequestWithCookiesAndRetryAsync(
searchUrl, referer: SearchUrlReferer, headers: extraHeaders);
var results = response.ContentString;
try

2
src/Jackett.Common/Indexers/Anthelion.cs
View File

@ -94,7 +94,7 @@ namespace Jackett.Common.Indexers
qc.Add($"filter_cat[{cat}]", "1");
var searchUrl = BrowseUrl + "?" + qc.GetQueryString();
var results = await RequestStringWithCookies(searchUrl);
var results = await WebRequestWithCookiesAsync(searchUrl);
try
{
var parser = new HtmlParser();

2
src/Jackett.Common/Indexers/AwesomeHD.cs
View File

@ -100,7 +100,7 @@ namespace Jackett.Common.Indexers
}
var searchUrl = SearchUrl + "?" + qc.GetQueryString();
var results = await RequestStringWithCookies(searchUrl);
var results = await WebRequestWithCookiesAsync(searchUrl);
if (string.IsNullOrWhiteSpace(results.ContentString))
throw new Exception("Empty response. Please, check the Passkey.");

6
src/Jackett.Common/Indexers/BB.cs
View File

@ -122,8 +122,8 @@ namespace Jackett.Common.Indexers
request_urls.Add(SearchUrl + queryCollection.GetQueryString());
}
var downloadTasksQuery =
from url in request_urls select RequestStringWithCookiesAndRetry(url);
var downloadTasksQuery = from url in request_urls select RequestWithCookiesAndRetryAsync(url);
var responses = await Task.WhenAll(downloadTasksQuery.ToArray());
@ -134,7 +134,7 @@ namespace Jackett.Common.Indexers
if (results.IsRedirect)
{
await ApplyConfiguration(null);
results = await RequestStringWithCookiesAndRetry(request_urls[i]);
results = await RequestWithCookiesAndRetryAsync(request_urls[i]);
}
try
{

8
src/Jackett.Common/Indexers/BJShare.cs
View File

@ -257,12 +257,12 @@ namespace Jackett.Common.Indexers
foreach (var cat in MapTorznabCapsToTrackers(query))
queryCollection.Add("filter_cat[" + cat + "]", "1");
searchUrl += "?" + queryCollection.GetQueryString();
var results = await RequestStringWithCookies(searchUrl);
var results = await WebRequestWithCookiesAsync(searchUrl);
if (IsSessionIsClosed(results))
{
// re-login
await ApplyConfiguration(null);
results = await RequestStringWithCookies(searchUrl);
results = await WebRequestWithCookiesAsync(searchUrl);
}
try
@ -389,12 +389,12 @@ namespace Jackett.Common.Indexers
private async Task<List<ReleaseInfo>> ParseLast24HoursAsync()
{
var releases = new List<ReleaseInfo>();
var results = await RequestStringWithCookies(TodayUrl);
var results = await WebRequestWithCookiesAsync(TodayUrl);
if (IsSessionIsClosed(results))
{
// re-login
await ApplyConfiguration(null);
results = await RequestStringWithCookies(TodayUrl);
results = await WebRequestWithCookiesAsync(TodayUrl);
}
try

11
src/Jackett.Common/Indexers/BakaBT.cs
View File

@ -14,6 +14,7 @@ using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json.Linq;
using NLog;
using WebRequest = Jackett.Common.Utils.Clients.WebRequest;
namespace Jackett.Common.Indexers
{
@ -68,7 +69,7 @@ namespace Jackett.Common.Indexers
private async Task DoLogin()
{
var loginForm = await webclient.GetString(new Utils.Clients.WebRequest()
var loginForm = await webclient.GetResultAsync(new Utils.Clients.WebRequest()
{
Url = LoginUrl,
Type = RequestType.GET
@ -106,12 +107,12 @@ namespace Jackett.Common.Indexers
var releases = new List<ReleaseInfo>();
var searchString = queryCopy.SanitizedSearchTerm;
var episodeSearchUrl = SearchUrl + WebUtility.UrlEncode(searchString);
var response = await RequestStringWithCookiesAndRetry(episodeSearchUrl);
var response = await RequestWithCookiesAndRetryAsync(episodeSearchUrl);
if (!response.ContentString.Contains(LogoutStr))
{
//Cookie appears to expire after a period of time or logging in to the site via browser
await DoLogin();
response = await RequestStringWithCookiesAndRetry(episodeSearchUrl);
response = await RequestWithCookiesAndRetryAsync(episodeSearchUrl);
}
try
@ -246,7 +247,7 @@ namespace Jackett.Common.Indexers
public override async Task<byte[]> Download(Uri link)
{
var downloadPage = await RequestStringWithCookies(link.ToString());
var downloadPage = await WebRequestWithCookiesAsync(link.ToString());
var parser = new HtmlParser();
var dom = parser.ParseDocument(downloadPage.ContentString);
var downloadLink = dom.QuerySelectorAll(".download_link").First().GetAttribute("href");
@ -254,7 +255,7 @@ namespace Jackett.Common.Indexers
if (string.IsNullOrWhiteSpace(downloadLink))
throw new Exception("Unable to find download link.");
var response = await RequestBytesWithCookies(SiteLink + downloadLink);
var response = await WebRequestWithCookiesAsync(SiteLink + downloadLink);
return response.ContentBytes;
}
}

115
src/Jackett.Common/Indexers/BaseIndexer.cs
View File

@ -392,11 +392,11 @@ namespace Jackett.Common.Indexers
.Replace("(", "%28")
.Replace(")", "%29")
.Replace("'", "%27");
var response = await RequestBytesWithCookiesAndRetry(requestLink, null, method, requestLink);
var response = await RequestWithCookiesAndRetryAsync(requestLink, null, method, requestLink);
// if referer link is provied it will be used
if (refererlink != null)
response = await RequestBytesWithCookiesAndRetry(requestLink, null, method, refererlink);
response = await RequestWithCookiesAndRetryAsync(requestLink, null, method, refererlink);
if (response.IsRedirect)
{
await FollowIfRedirect(response);
@ -412,69 +412,38 @@ namespace Jackett.Common.Indexers
return response.ContentBytes;
}
protected async Task<WebResult> RequestBytesWithCookiesAndRetry(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable<KeyValuePair<string, string>> data = null)
protected async Task<WebResult> RequestWithCookiesAndRetryAsync(
string url, string cookieOverride = null, RequestType method = RequestType.GET,
string referer = null, IEnumerable<KeyValuePair<string, string>> data = null,
Dictionary<string, string> headers = null, string rawbody = null, bool? emulateBrowser = null)
{
Exception lastException = null;
for (var i = 0; i < 3; i++)
{
try
{
return await RequestBytesWithCookies(url, cookieOverride, method, referer, data);
return await WebRequestWithCookiesAsync(
url, cookieOverride, method, referer, data, headers, rawbody, emulateBrowser);
}
catch (Exception e)
{
logger.Error(e, string.Format("On attempt {0} downloading from {1}: {2}", (i + 1), DisplayName, e.Message));
logger.Error(
e, string.Format("On attempt {0} downloading from {1}: {2}", (i + 1), DisplayName, e.Message));
lastException = e;
}
await Task.Delay(500);
}
throw lastException;
}
protected async Task<WebResult> RequestStringWithCookies(string url, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null)
protected virtual async Task<WebResult> WebRequestWithCookiesAsync(
string url, string cookieOverride = null, RequestType method = RequestType.GET,
string referer = null, IEnumerable<KeyValuePair<string, string>> data = null,
Dictionary<string, string> headers = null, string rawbody = null, bool? emulateBrowser = null)
{
var request = new Utils.Clients.WebRequest()
{
Url = url,
Type = RequestType.GET,
Cookies = CookieHeader,
Referer = referer,
Headers = headers,
Encoding = Encoding
};
if (cookieOverride != null)
request.Cookies = cookieOverride;
var result = await webclient.GetString(request);
CheckTrackerDown(result);
UpdateCookieHeader(result.Cookies, cookieOverride);
return result;
}
protected async Task<WebResult> RequestStringWithCookiesAndRetry(string url, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null)
{
Exception lastException = null;
for (var i = 0; i < 3; i++)
{
try
{
return await RequestStringWithCookies(url, cookieOverride, referer, headers);
}
catch (Exception e)
{
logger.Error(string.Format("On attempt {0} checking for results from {1}: {2}", (i + 1), DisplayName, e.Message));
lastException = e;
}
await Task.Delay(500);
}
throw lastException;
}
protected virtual async Task<WebResult> RequestBytesWithCookies(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable<KeyValuePair<string, string>> data = null, Dictionary<string, string> headers = null)
{
var request = new Utils.Clients.WebRequest()
var request = new WebRequest
{
Url = url,
Type = method,
@ -482,58 +451,18 @@ namespace Jackett.Common.Indexers
PostData = data,
Referer = referer,
Headers = headers,
Encoding = Encoding
};
if (cookieOverride != null)
request.Cookies = cookieOverride;
var result = await webclient.GetBytes(request);
UpdateCookieHeader(result.Cookies, cookieOverride);
return result;
}
protected async Task<WebResult> PostDataWithCookies(string url, IEnumerable<KeyValuePair<string, string>> data, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null, string rawbody = null, bool? emulateBrowser = null)
{
var request = new Utils.Clients.WebRequest()
{
Url = url,
Type = RequestType.POST,
Cookies = cookieOverride ?? CookieHeader,
PostData = data,
Referer = referer,
Headers = headers,
RawBody = rawbody,
Encoding = Encoding
};
if (emulateBrowser.HasValue)
request.EmulateBrowser = emulateBrowser.Value;
var result = await webclient.GetString(request);
CheckTrackerDown(result);
var result = await webclient.GetResultAsync(request);
CheckSiteDown(result);
UpdateCookieHeader(result.Cookies, cookieOverride);
return result;
}
protected async Task<WebResult> PostDataWithCookiesAndRetry(string url, IEnumerable<KeyValuePair<string, string>> data, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null, string rawbody = null, bool? emulateBrowser = null)
{
Exception lastException = null;
for (var i = 0; i < 3; i++)
{
try
{
return await PostDataWithCookies(url, data, cookieOverride, referer, headers, rawbody, emulateBrowser);
}
catch (Exception e)
{
logger.Error(string.Format("On attempt {0} checking for results from {1}: {2}", (i + 1), DisplayName, e.Message));
lastException = e;
}
await Task.Delay(500);
}
throw lastException;
}
protected async Task<WebResult> RequestLoginAndFollowRedirect(string url, IEnumerable<KeyValuePair<string, string>> data, string cookies, bool returnCookiesFromFirstCall, string redirectUrlOverride = null, string referer = null, bool accumulateCookies = false)
{
var request = new Utils.Clients.WebRequest()
@ -545,8 +474,8 @@ namespace Jackett.Common.Indexers
PostData = data,
Encoding = Encoding
};
var response = await webclient.GetString(request);
CheckTrackerDown(response);
var response = await webclient.GetResultAsync(request);
CheckSiteDown(response);
if (accumulateCookies)
{
response.Cookies = ResolveCookies((request.Cookies == null ? "" : request.Cookies + " ") + response.Cookies);
@ -566,7 +495,7 @@ namespace Jackett.Common.Indexers
return response;
}
protected void CheckTrackerDown(WebResult response)
protected static void CheckSiteDown(WebResult response)
{
if (response.Status == System.Net.HttpStatusCode.BadGateway
|| response.Status == System.Net.HttpStatusCode.GatewayTimeout
@ -643,7 +572,7 @@ namespace Jackett.Common.Indexers
redirRequestCookies = (overrideCookies != null ? overrideCookies : "");
}
// Do redirect
var redirectedResponse = await webclient.GetBytes(new WebRequest()
var redirectedResponse = await webclient.GetResultAsync(new WebRequest()
{
Url = overrideRedirectUrl ?? incomingResponse.RedirectingTo,
Referer = referrer,

2
src/Jackett.Common/Indexers/BitCityReloaded.cs
View File

@ -129,7 +129,7 @@ namespace Jackett.Common.Indexers
searchUrl += "?" + queryCollection.GetQueryString();
var response = await RequestStringWithCookiesAndRetry(searchUrl, null, BrowseUrl);
var response = await RequestWithCookiesAndRetryAsync(searchUrl, referer: BrowseUrl);
var results = response.ContentString;
try
{

8
src/Jackett.Common/Indexers/BitHDTV.cs
View File

@ -63,7 +63,7 @@ namespace Jackett.Common.Indexers
public override async Task<ConfigurationData> GetConfigurationForSetup()
{
var result = configData;
var loginPage = await RequestStringWithCookies(LoginUrl, configData.CookieHeader.Value);
var loginPage = await WebRequestWithCookiesAsync(LoginUrl, configData.CookieHeader.Value);
if (loginPage.IsRedirect)
return result; // already logged in
var parser = new HtmlParser();
@ -129,10 +129,10 @@ namespace Jackett.Common.Indexers
qc.Add("search", query.ImdbID);
qc.Add("options", "4"); //Search URL field for IMDB link
search.Query = qc.GetQueryString();
results.Add(await RequestStringWithCookiesAndRetry(search.ToString()));
results.Add(await RequestWithCookiesAndRetryAsync(search.ToString()));
qc["Options"] = "1"; //Search Title and Description
search.Query = qc.GetQueryString();
results.Add(await RequestStringWithCookiesAndRetry(search.ToString()));
results.Add(await RequestWithCookiesAndRetryAsync(search.ToString()));
}
else
{
@ -140,7 +140,7 @@ namespace Jackett.Common.Indexers
qc.Add("search", query.GetQueryString());
qc.Add("options", "0"); //Search Title Only
search.Query = qc.GetQueryString();
results.Add(await RequestStringWithCookiesAndRetry(search.ToString()));
results.Add(await RequestWithCookiesAndRetryAsync(search.ToString()));
}
var parser = new HtmlParser();

12
src/Jackett.Common/Indexers/BroadcasTheNet.cs
View File

@ -110,11 +110,13 @@ namespace Jackett.Common.Indexers
new JValue(btnResults),
new JValue(btnOffset)
};
var response = await PostDataWithCookiesAndRetry(APIBASE, null, null, null, new Dictionary<string, string>()
{
{ "Accept", "application/json-rpc, application/json"},
{"Content-Type", "application/json-rpc"}
}, JsonRPCRequest("getTorrents", parameters), false);
var response = await RequestWithCookiesAndRetryAsync(
APIBASE, method: RequestType.POST,
headers: new Dictionary<string, string>
{
{"Accept", "application/json-rpc, application/json"},
{"Content-Type", "application/json-rpc"}
}, rawbody: JsonRPCRequest("getTorrents", parameters), emulateBrowser: false);
try
{

42
src/Jackett.Common/Indexers/CardigannIndexer.cs
View File

@ -532,13 +532,12 @@ namespace Jackett.Common.Indexers
["g-recaptcha-response"] = CaptchaConfigItem.Value
};
var ClearanceUrl = resolvePath("/cdn-cgi/l/chk_captcha?" + CloudFlareQueryCollection.GetQueryString());
var ClearanceResult = await RequestStringWithCookies(ClearanceUrl.ToString(), null, SiteLink);
var ClearanceResult = await WebRequestWithCookiesAsync(ClearanceUrl.ToString(), referer: SiteLink);
if (ClearanceResult.IsRedirect) // clearance successfull
{
// request real login page again
landingResult = await RequestStringWithCookies(LoginUrl, null, SiteLink);
landingResult = await WebRequestWithCookiesAsync(LoginUrl, referer: SiteLink);
var htmlParser = new HtmlParser();
landingResultDocument = htmlParser.ParseDocument(landingResult.ContentString);
}
@ -654,7 +653,7 @@ namespace Jackett.Common.Indexers
if (simpleCaptchaPresent != null)
{
var captchaUrl = resolvePath("simpleCaptcha.php?numImages=1");
var simpleCaptchaResult = await RequestStringWithCookies(captchaUrl.ToString(), null, LoginUrl);
var simpleCaptchaResult = await WebRequestWithCookiesAsync(captchaUrl.ToString(), referer: LoginUrl);
var simpleCaptchaJSON = JObject.Parse(simpleCaptchaResult.ContentString);
var captchaSelection = simpleCaptchaJSON["images"][0]["hash"].ToString();
pairs["captchaSelection"] = captchaSelection;
@ -723,7 +722,9 @@ namespace Jackett.Common.Indexers
headers.Add("Content-Type", "multipart/form-data; boundary=" + boundary);
var body = string.Join("\r\n", bodyParts);
loginResult = await PostDataWithCookies(submitUrl.ToString(), pairs, configData.CookieHeader.Value, SiteLink, headers, body);
loginResult = await WebRequestWithCookiesAsync(
submitUrl.ToString(), configData.CookieHeader.Value, RequestType.POST, SiteLink, pairs, headers,
body);
}
else
{
@ -749,7 +750,7 @@ namespace Jackett.Common.Indexers
var LoginUrl = resolvePath(Login.Path + "?" + queryCollection.GetQueryString()).ToString();
configData.CookieHeader.Value = null;
var loginResult = await RequestStringWithCookies(LoginUrl, null, SiteLink);
var loginResult = await WebRequestWithCookiesAsync(LoginUrl, referer: SiteLink);
configData.CookieHeader.Value = loginResult.Cookies;
checkForError(loginResult, Definition.Login.Error);
@ -759,7 +760,7 @@ namespace Jackett.Common.Indexers
var OneUrl = applyGoTemplateText(Definition.Login.Inputs["oneurl"]);
var LoginUrl = resolvePath(Login.Path + OneUrl).ToString();
configData.CookieHeader.Value = null;
var loginResult = await RequestStringWithCookies(LoginUrl, null, SiteLink);
var loginResult = await WebRequestWithCookiesAsync(LoginUrl, referer: SiteLink);
configData.CookieHeader.Value = loginResult.Cookies;
checkForError(loginResult, Definition.Login.Error);
@ -793,7 +794,7 @@ namespace Jackett.Common.Indexers
// test if login was successful
var LoginTestUrl = resolvePath(Login.Test.Path).ToString();
var testResult = await RequestStringWithCookies(LoginTestUrl);
var testResult = await WebRequestWithCookiesAsync(LoginTestUrl);
if (testResult.IsRedirect)
{
@ -886,7 +887,7 @@ namespace Jackett.Common.Indexers
configData.CookieHeader.Value = null;
if (Login.Cookies != null)
configData.CookieHeader.Value = string.Join("; ", Login.Cookies);
landingResult = await RequestStringWithCookies(LoginUrl.AbsoluteUri, null, SiteLink);
landingResult = await WebRequestWithCookiesAsync(LoginUrl.AbsoluteUri, referer: SiteLink);
var htmlParser = new HtmlParser();
landingResultDocument = htmlParser.ParseDocument(landingResult.ContentString);
@ -929,7 +930,8 @@ namespace Jackett.Common.Indexers
hasCaptcha = true;
var CaptchaUrl = resolvePath(captchaElement.GetAttribute("src"), LoginUrl);
var captchaImageData = await RequestBytesWithCookies(CaptchaUrl.ToString(), landingResult.Cookies, RequestType.GET, LoginUrl.AbsoluteUri);
var captchaImageData = await WebRequestWithCookiesAsync(
CaptchaUrl.ToString(), landingResult.Cookies, referer: LoginUrl.AbsoluteUri);
var CaptchaImage = new ImageItem { Name = "Captcha Image" };
var CaptchaText = new StringItem { Name = "Captcha Text" };
@ -1334,7 +1336,6 @@ namespace Jackett.Common.Indexers
logger.Info($"Fetching: {searchUrl}");
// send HTTP request
WebResult response = null;
Dictionary<string, string> headers = null;
if (Search.Headers != null)
{
@ -1343,10 +1344,9 @@ namespace Jackett.Common.Indexers
foreach (var header in Search.Headers)
headers.Add(header.Key, header.Value[0]);
}
if (method == RequestType.POST)
response = await PostDataWithCookies(searchUrl, queryCollection, null, null, headers);
else
response = await RequestStringWithCookies(searchUrl, null, null, headers);
var response = await WebRequestWithCookiesAsync(
searchUrl, method: method, headers: headers, data: queryCollection);
if (response.IsRedirect && SearchPath.Followredirect)
await FollowIfRedirect(response);
@ -1368,11 +1368,7 @@ namespace Jackett.Common.Indexers
if (!LoginResult)
throw new Exception(string.Format("Relogin failed"));
await TestLogin();
if (method == RequestType.POST)
response = await PostDataWithCookies(searchUrl, queryCollection);
else
response = await RequestStringWithCookies(searchUrl);
response = await WebRequestWithCookiesAsync(searchUrl, method: method, data: queryCollection);
if (response.IsRedirect && SearchPath.Followredirect)
await FollowIfRedirect(response);
@ -1753,7 +1749,7 @@ namespace Jackett.Common.Indexers
requestLinkStr += queryCollection.GetQueryString(Encoding, separator: request.Queryseparator);
}
var response = await RequestBytesWithCookiesAndRetry(requestLinkStr, null, method, referer, pairs);
var response = await RequestWithCookiesAndRetryAsync(requestLinkStr, null, method, referer, pairs);
logger.Debug($"CardigannIndexer ({Id}): handleRequest() remote server returned {response.Status.ToString()}" + (response.IsRedirect ? " => " + response.RedirectingTo : ""));
return response;
}
@ -1796,9 +1792,9 @@ namespace Jackett.Common.Indexers
if (Download.Selector != null)
{
var selector = applyGoTemplateText(Download.Selector, variables);
var response = await RequestStringWithCookies(link.ToString());
var response = await WebRequestWithCookiesAsync(link.ToString());
if (response.IsRedirect)
response = await RequestStringWithCookies(response.RedirectingTo);
response = await WebRequestWithCookiesAsync(response.RedirectingTo);
var results = response.ContentString;
var searchResultParser = new HtmlParser();
var searchResultDocument = searchResultParser.ParseDocument(results);

6
src/Jackett.Common/Indexers/Cinecalidad.cs
View File

@ -100,7 +100,7 @@ namespace Jackett.Common.Indexers
{
var pageParam = page > 1 ? $"page/{page}/" : "";
var searchUrl = string.Format(templateUrl, pageParam);
var response = await RequestStringWithCookiesAndRetry(searchUrl);
var response = await RequestWithCookiesAndRetryAsync(searchUrl);
var pageReleases = ParseReleases(response, query);
// publish date is not available in the torrent list, but we add a relative date so we can sort
@ -120,7 +120,7 @@ namespace Jackett.Common.Indexers
public override async Task<byte[]> Download(Uri link)
{
var results = await RequestStringWithCookies(link.ToString());
var results = await WebRequestWithCookiesAsync(link.ToString());
try
{
@ -129,7 +129,7 @@ namespace Jackett.Common.Indexers
var preotectedLink = dom.QuerySelector("a[service=BitTorrent]").GetAttribute("href");
preotectedLink = SiteLink + preotectedLink.TrimStart('/');
results = await RequestStringWithCookies(preotectedLink);
results = await WebRequestWithCookiesAsync(preotectedLink);
dom = parser.ParseDocument(results.ContentString);
var magnetUrl = dom.QuerySelector("a[href^=magnet]").GetAttribute("href");
return await base.Download(new Uri(magnetUrl));

5
src/Jackett.Common/Indexers/CorsaroRed.cs
View File

@ -130,13 +130,14 @@ namespace Jackett.Common.Indexers
private async Task<dynamic> SendApiRequest(IEnumerable<KeyValuePair<string, string>> data)
{
var result = await PostDataWithCookiesAndRetry(ApiSearch, data, null, SiteLink, _apiHeaders, null, true);
var result = await RequestWithCookiesAndRetryAsync(
ApiSearch, null, RequestType.POST, SiteLink, data, _apiHeaders, null, true);
return CheckResponse(result);
}
private async Task<dynamic> SendApiRequestLatest()
{
var result = await RequestStringWithCookiesAndRetry(ApiLatest, null, SiteLink, _apiHeaders);
var result = await RequestWithCookiesAndRetryAsync(ApiLatest, referer: SiteLink, headers: _apiHeaders);
return CheckResponse(result);
}

7
src/Jackett.Common/Indexers/DanishBits.cs
View File

@ -65,10 +65,13 @@ namespace Jackett.Common.Indexers
return searchString;
}
protected override async Task<WebResult> RequestBytesWithCookies(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable<KeyValuePair<string, string>> data = null, Dictionary<string, string> headers = null)
protected override async Task<WebResult> WebRequestWithCookiesAsync(
string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null,
IEnumerable<KeyValuePair<string, string>> data = null, Dictionary<string, string> headers = null,
string rawbody = null, bool? emulateBrowser = null)
{
CookieHeader = null; // Download fill fail with cookies set
return await base.RequestBytesWithCookies(url, cookieOverride, method, referer, data, headers);
return await base.WebRequestWithCookiesAsync(url, cookieOverride, method, referer, data, headers);
}
}
}

2
src/Jackett.Common/Indexers/DigitalCore.cs
View File

@ -136,7 +136,7 @@ namespace Jackett.Common.Indexers
searchUrl += "?" + queryCollection.GetQueryString();
foreach (var cat in MapTorznabCapsToTrackers(query))
searchUrl += "&categories[]=" + cat;
var results = await RequestStringWithCookies(searchUrl, null, SiteLink);
var results = await WebRequestWithCookiesAsync(searchUrl, referer: SiteLink);
try
{

6
src/Jackett.Common/Indexers/DigitalHive.cs
View File

@ -93,7 +93,7 @@ namespace Jackett.Common.Indexers
public override async Task<ConfigurationData> GetConfigurationForSetup()
{
var loginPage = await RequestStringWithCookies(LoginUrl, configData.CookieHeader.Value);
var loginPage = await WebRequestWithCookiesAsync(LoginUrl, configData.CookieHeader.Value);
var parser = new HtmlParser();
var cq = parser.ParseDocument(loginPage.ContentString);
var recaptchaSiteKey = cq.QuerySelector(".g-recaptcha")?.GetAttribute("data-sitekey");
@ -181,12 +181,12 @@ namespace Jackett.Common.Indexers
queryCollection.Add("blah", "0");
var results = await RequestStringWithCookiesAndRetry(searchUrl + "?" + queryCollection.GetQueryString());
var results = await RequestWithCookiesAndRetryAsync(searchUrl + "?" + queryCollection.GetQueryString());
if (results.IsRedirect)
{
// re-login
await ApplyConfiguration(null);
results = await RequestStringWithCookiesAndRetry(searchUrl + "?" + queryCollection.GetQueryString());
results = await RequestWithCookiesAndRetryAsync(searchUrl + "?" + queryCollection.GetQueryString());
}
try
{

6
src/Jackett.Common/Indexers/DivxTotal.cs
View File

@ -100,7 +100,7 @@ namespace Jackett.Common.Indexers
do
{
var url = SiteLink + "page/" + page + "/?" + qc.GetQueryString();
var result = await RequestStringWithCookies(url);
var result = await WebRequestWithCookiesAsync(url);
if (result.Status != HttpStatusCode.OK)
throw new ExceptionWithConfigData(result.ContentString, configData);
@ -153,7 +153,7 @@ namespace Jackett.Common.Indexers
// for other categories we have to do another step
if (!downloadUrl.Contains(DownloadLink))
{
var result = await RequestStringWithCookies(downloadUrl);
var result = await WebRequestWithCookiesAsync(downloadUrl);
if (result.Status != HttpStatusCode.OK)
throw new ExceptionWithConfigData(result.ContentString, configData);
@ -205,7 +205,7 @@ namespace Jackett.Common.Indexers
private async Task ParseSeriesRelease(ICollection<ReleaseInfo> releases, TorznabQuery query,
string commentsLink, string cat, DateTime publishDate)
{
var result = await RequestStringWithCookies(commentsLink);
var result = await WebRequestWithCookiesAsync(commentsLink);
if (result.Status != HttpStatusCode.OK)
throw new ExceptionWithConfigData(result.ContentString, configData);

6
src/Jackett.Common/Indexers/EliteTracker.cs
View File

@ -151,7 +151,7 @@ namespace Jackett.Common.Indexers
{ "password", configData.Password.Value }
};
var result = await PostDataWithCookies(LoginUrl, pairs, "");
var result = await WebRequestWithCookiesAsync(LoginUrl, "", RequestType.POST, data: pairs);
await ConfigureIfOK(result.Cookies, result.Cookies != null, () =>
{
@ -174,12 +174,12 @@ namespace Jackett.Common.Indexers
{"category", "0"} // multi cat search not supported
};
var results = await PostDataWithCookies(BrowseUrl, pairs);
var results = await WebRequestWithCookiesAsync(BrowseUrl, method: RequestType.POST, data: pairs);
if (results.IsRedirect)
{
// re-login
await ApplyConfiguration(null);
results = await PostDataWithCookies(BrowseUrl, pairs);
results = await WebRequestWithCookiesAsync(BrowseUrl, method: RequestType.POST, data: pairs);
}
try

5
src/Jackett.Common/Indexers/EpubLibre.cs
View File

@ -9,6 +9,7 @@ using AngleSharp.Html.Parser;
using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
@ -85,7 +86,7 @@ namespace Jackett.Common.Indexers
for (var page = 0; page < maxPages; page++)
{
var searchUrl = string.Format(SearchUrl, page * MaxItemsPerPage, searchString);
var result = await RequestStringWithCookies(searchUrl, null, null, _apiHeaders);
var result = await WebRequestWithCookiesAsync(searchUrl, headers: _apiHeaders);
try
{
@ -149,7 +150,7 @@ namespace Jackett.Common.Indexers
public override async Task<byte[]> Download(Uri link)
{
var result = await RequestStringWithCookiesAndRetry(link.AbsoluteUri);
var result = await RequestWithCookiesAndRetryAsync(link.AbsoluteUri);
if (SobrecargaUrl.Equals(result.RedirectingTo))
throw new Exception("El servidor se encuentra sobrecargado en estos momentos. / The server is currently overloaded.");
try

2
src/Jackett.Common/Indexers/Feeds/BaseFeedIndexer.cs
View File

@ -55,7 +55,7 @@ namespace Jackett.Common.Indexers.Feeds
Type = RequestType.GET,
Encoding = Encoding
};
var result = await webclient.GetString(request);
var result = await webclient.GetResultAsync(request);
var results = ParseFeedForResults(result.ContentString);

2
src/Jackett.Common/Indexers/FileList.cs
View File

@ -188,7 +188,7 @@ namespace Jackett.Common.Indexers
{
{"Authorization", "Basic " + auth}
};
var response = await RequestStringWithCookies(searchUrl, headers: headers);
var response = await WebRequestWithCookiesAsync(searchUrl, headers: headers);
return response.ContentString;
}
catch (Exception inner)

4
src/Jackett.Common/Indexers/FunFile.cs
View File

@ -97,12 +97,12 @@ namespace Jackett.Common.Indexers
qc.Add("search", query.GetQueryString());
var searchUrl = SearchUrl + "?" + qc.GetQueryString();
var results = await RequestStringWithCookiesAndRetry(searchUrl);
var results = await RequestWithCookiesAndRetryAsync(searchUrl);
if (results.IsRedirect) // re-login
{
await ApplyConfiguration(null);
results = await RequestStringWithCookiesAndRetry(searchUrl);
results = await RequestWithCookiesAndRetryAsync(searchUrl);
}
try

9
src/Jackett.Common/Indexers/Fuzer.cs
View File

@ -13,6 +13,7 @@ using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json.Linq;
using NLog;
@ -107,7 +108,7 @@ namespace Jackett.Common.Indexers
public override async Task<ConfigurationData> GetConfigurationForSetup()
{
var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty);
var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty);
var parser = new HtmlParser();
var cq = parser.ParseDocument(loginPage.ContentString);
var captcha = cq.QuerySelector(".g-recaptcha"); // invisible recaptcha
@ -153,7 +154,7 @@ namespace Jackett.Common.Indexers
}
}
var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty);
var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty);
var pairs = new Dictionary<string, string>
{
{"vb_login_username", configData.Username.Value},
@ -204,7 +205,7 @@ namespace Jackett.Common.Indexers
}
searchUrl = MapTorznabCapsToTrackers(query).Aggregate(searchUrl, (current, cat) => $"{current}&c[]={cat}");
var data = await RequestStringWithCookiesAndRetry(searchUrl);
var data = await RequestWithCookiesAndRetryAsync(searchUrl);
try
{
var parser = new HtmlParser();
@ -276,7 +277,7 @@ namespace Jackett.Common.Indexers
Path = "index.php",
Query = queryString.GetQueryString()
};
var results = await RequestStringWithCookies(site.ToString());
var results = await WebRequestWithCookiesAsync(site.ToString());
var parser = new HtmlParser();
var dom = parser.ParseDocument(results.ContentString);
var rows = dom.QuerySelectorAll("#listtable > tbody > tr");

2
src/Jackett.Common/Indexers/GazelleGames.cs
View File

@ -225,7 +225,7 @@ namespace Jackett.Common.Indexers
searchUrl += "?" + queryCollection.GetQueryString();
var results = await RequestStringWithCookies(searchUrl);
var results = await WebRequestWithCookiesAsync(searchUrl);
if (results.IsRedirect && results.RedirectingTo.EndsWith("login.php"))
{
throw new Exception("relogin needed, please update your cookie");

6
src/Jackett.Common/Indexers/GimmePeers.cs
View File

@ -88,7 +88,7 @@ namespace Jackett.Common.Indexers
{ "login", "Log in!" }
};
var loginPage = await RequestStringWithCookies(SiteLink, string.Empty);
var loginPage = await WebRequestWithCookiesAsync(SiteLink, string.Empty);
var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, SiteLink, SiteLink);
await ConfigureIfOK(result.Cookies, result.ContentString?.Contains("logout.php") == true, () =>
@ -123,12 +123,12 @@ namespace Jackett.Common.Indexers
}
searchUrl += "?" + queryCollection.GetQueryString();
var response = await RequestStringWithCookiesAndRetry(searchUrl, null, BrowseUrl);
var response = await RequestWithCookiesAndRetryAsync(searchUrl, referer: BrowseUrl);
if (response.IsRedirect)
{
// re login
await ApplyConfiguration(null);
response = await RequestStringWithCookiesAndRetry(searchUrl, null, BrowseUrl);
response = await RequestWithCookiesAndRetryAsync(searchUrl, referer: BrowseUrl);
}
var results = response.ContentString;

16
src/Jackett.Common/Indexers/HDBitsApi.cs
View File

@ -176,14 +176,14 @@ namespace Jackett.Common.Indexers
requestData["username"] = configData.Username.Value;
requestData["passkey"] = configData.Passkey.Value;
JObject json = null;
var response = await PostDataWithCookiesAndRetry(APIUrl + url, null, null, null, new Dictionary<string, string>()
{
{"Accept", "application/json"},
{"Content-Type", "application/json"}
}, requestData.ToString(), false);
CheckTrackerDown(response);
var response = await RequestWithCookiesAndRetryAsync(
APIUrl + url, null, RequestType.POST, null, null,
new Dictionary<string, string>
{
{"Accept", "application/json"},
{"Content-Type", "application/json"}
}, requestData.ToString(), false);
CheckSiteDown(response);
try
{

4
src/Jackett.Common/Indexers/HDSpace.cs
View File

@ -77,7 +77,7 @@ namespace Jackett.Common.Indexers
public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
{
LoadValuesFromJson(configJson);
var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty);
var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty);
var pairs = new Dictionary<string, string>
{
{"uid", configData.Username.Value},
@ -119,7 +119,7 @@ namespace Jackett.Common.Indexers
queryCollection.Add("search", query.GetQueryString());
}
var response = await RequestStringWithCookiesAndRetry(SearchUrl + queryCollection.GetQueryString());
var response = await RequestWithCookiesAndRetryAsync(SearchUrl + queryCollection.GetQueryString());
try
{

4
src/Jackett.Common/Indexers/HDTorrents.cs
View File

@ -104,7 +104,7 @@ namespace Jackett.Common.Indexers
{
LoadValuesFromJson(configJson);
var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty);
var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty);
var pairs = new Dictionary<string, string> {
{ "uid", configData.Username.Value },
@ -134,7 +134,7 @@ namespace Jackett.Common.Indexers
// manually url encode parenthesis to prevent "hacking" detection
searchUrl += queryCollection.GetQueryString().Replace("(", "%28").Replace(")", "%29");
var results = await RequestStringWithCookiesAndRetry(searchUrl);
var results = await RequestWithCookiesAndRetryAsync(searchUrl);
try
{
var parser = new HtmlParser();

2
src/Jackett.Common/Indexers/Hebits.cs
View File

@ -105,7 +105,7 @@ namespace Jackett.Common.Indexers
var cats = MapTorznabCapsToTrackers(query);
if (cats.Count > 0)
searchUrl = cats.Aggregate(searchUrl, (url, cat) => $"{url}&c{cat}=1");
var response = await RequestStringWithCookies(searchUrl);
var response = await WebRequestWithCookiesAsync(searchUrl);
try
{
var parser = new HtmlParser();

8
src/Jackett.Common/Indexers/HorribleSubs.cs
View File

@ -72,7 +72,7 @@ namespace Jackett.Common.Indexers
};
var searchUrl = ApiEndpoint + "?" + queryCollection.GetQueryString();
var response = await RequestStringWithCookiesAndRetry(searchUrl);
var response = await RequestWithCookiesAndRetryAsync(searchUrl);
try
{
@ -112,7 +112,7 @@ namespace Jackett.Common.Indexers
};
var searchUrl = ApiEndpoint + "?" + queryCollection.GetQueryString();
var response = await RequestStringWithCookiesAndRetry(searchUrl);
var response = await RequestWithCookiesAndRetryAsync(searchUrl);
try
{
@ -146,7 +146,7 @@ namespace Jackett.Common.Indexers
var releases = new List<ReleaseInfo>();
var parser = new HtmlParser();
var response = await RequestStringWithCookiesAndRetry(resultUrl);
var response = await RequestWithCookiesAndRetryAsync(resultUrl);
await FollowIfRedirect(response);
try
@ -168,7 +168,7 @@ namespace Jackett.Common.Indexers
var nextId = 0;
while (true)
{
var showApiResponse = await RequestStringWithCookiesAndRetry(apiUrl + "&nextid=" + nextId);
var showApiResponse = await RequestWithCookiesAndRetryAsync(apiUrl + "&nextid=" + nextId);
var showApiDom = parser.ParseDocument(showApiResponse.ContentString);
var releaseRowResults = showApiDom.QuerySelectorAll("div.rls-info-container");
rows.AddRange(releaseRowResults);

3
src/Jackett.Common/Indexers/IPTorrents.cs
View File

@ -10,6 +10,7 @@ using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json.Linq;
using NLog;
@ -179,7 +180,7 @@ namespace Jackett.Common.Indexers
qc.Add(cat, string.Empty);
var searchUrl = SearchUrl + "?" + qc.GetQueryString();
var response = await RequestStringWithCookiesAndRetry(searchUrl, null, SearchUrl);
var response = await RequestWithCookiesAndRetryAsync(searchUrl, referer: SearchUrl);
var results = response.ContentString;
if (results == null || !results.Contains("/lout.php"))

5
src/Jackett.Common/Indexers/ImmortalSeed.cs
View File

@ -11,6 +11,7 @@ using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json.Linq;
using NLog;
@ -121,13 +122,13 @@ namespace Jackett.Common.Indexers
if (!string.IsNullOrWhiteSpace(query.GetQueryString()))
searchUrl += string.Format(QueryString, WebUtility.UrlEncode(query.GetQueryString()));
var results = await RequestStringWithCookiesAndRetry(searchUrl);
var results = await RequestWithCookiesAndRetryAsync(searchUrl);
// Occasionally the cookies become invalid, login again if that happens
if (results.ContentString.Contains("You do not have permission to access this page."))
{
await ApplyConfiguration(null);
results = await RequestStringWithCookiesAndRetry(searchUrl);
results = await RequestWithCookiesAndRetryAsync(searchUrl);
}
try

2
src/Jackett.Common/Indexers/InternetArchive.cs
View File

@ -143,7 +143,7 @@ namespace Jackett.Common.Indexers
{"output", "json"}
};
var fullSearchUrl = SearchUrl + "?" + qc.GetQueryString();
var result = await RequestStringWithCookiesAndRetry(fullSearchUrl);
var result = await RequestWithCookiesAndRetryAsync(fullSearchUrl);
foreach (var torrent in ParseResponse(result))
releases.Add(MakeRelease(torrent));

6
src/Jackett.Common/Indexers/Libble.cs
View File

@ -83,7 +83,7 @@ namespace Jackett.Common.Indexers
public override async Task<ConfigurationData> GetConfigurationForSetup()
{
await RequestStringWithCookies(LandingUrl);
await WebRequestWithCookiesAsync(LandingUrl);
return configData;
}
@ -155,12 +155,12 @@ namespace Jackett.Common.Indexers
searchUrl += "?" + queryCollection.GetQueryString();
var searchPage = await PostDataWithCookiesAndRetry(searchUrl, searchParams);
var searchPage = await RequestWithCookiesAndRetryAsync(searchUrl, method: RequestType.POST, data: searchParams);
// Occasionally the cookies become invalid, login again if that happens
if (searchPage.IsRedirect)
{
await ApplyConfiguration(null);
searchPage = await PostDataWithCookiesAndRetry(searchUrl, searchParams);
searchPage = await RequestWithCookiesAndRetryAsync(searchUrl, method: RequestType.POST, data: searchParams);
}
try

18
src/Jackett.Common/Indexers/LostFilm.cs
View File

@ -113,14 +113,14 @@ namespace Jackett.Common.Indexers
public override async Task<ConfigurationData> GetConfigurationForSetup()
{
// looks like after some failed login attempts there's a captcha
var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty);
var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty);
var parser = new HtmlParser();
var document = parser.ParseDocument(loginPage.ContentString);
var qCaptchaImg = document.QuerySelector("img#captcha_pictcha");
if (qCaptchaImg != null)
{
var captchaUrl = SiteLink + qCaptchaImg.GetAttribute("src");
var captchaImage = await RequestBytesWithCookies(captchaUrl, loginPage.Cookies);
var captchaImage = await WebRequestWithCookiesAsync(captchaUrl, loginPage.Cookies);
configData.CaptchaImage.Value = captchaImage.ContentBytes;
}
else
@ -183,7 +183,7 @@ namespace Jackett.Common.Indexers
{ "type", "logout" }
};
var response = await PostDataWithCookies(url: ApiUrl, data: data);
var response = await WebRequestWithCookiesAsync(ApiUrl, method: RequestType.POST, data: data);
logger.Debug("Logout result: " + response.ContentString);
var isOK = response.Status == System.Net.HttpStatusCode.OK;
@ -214,7 +214,7 @@ namespace Jackett.Common.Indexers
private async Task<WebResult> RequestStringAndRelogin(string url)
{
var results = await RequestStringWithCookies(url);
var results = await WebRequestWithCookiesAsync(url);
if (results.ContentString.Contains("503 Service"))
{
throw new ExceptionWithConfigData(results.ContentString, configData);
@ -223,7 +223,7 @@ namespace Jackett.Common.Indexers
{
// Re-login
await ApplyConfiguration(null);
return await RequestStringWithCookies(url);
return await WebRequestWithCookiesAsync(url);
}
else
{
@ -287,7 +287,7 @@ namespace Jackett.Common.Indexers
{ "val", searchString }
};
logger.Debug("> Searching: " + searchString);
var response = await PostDataWithCookies(url: ApiUrl, data: data);
var response = await WebRequestWithCookiesAsync(ApiUrl, method: RequestType.POST, data: data);
if (response.ContentString == null)
{
logger.Debug("> Empty series response for query: " + searchString);
@ -471,7 +471,7 @@ namespace Jackett.Common.Indexers
logger.Debug("FetchSeriesReleases: " + url + " S: " + query.Season.ToString() + " E: " + query.Episode + " Filter: " + filter);
var releases = new List<ReleaseInfo>();
var results = await RequestStringWithCookies(url);
var results = await WebRequestWithCookiesAsync(url);
try
{
@ -626,7 +626,7 @@ namespace Jackett.Common.Indexers
logger.Debug("FetchTrackerReleases: " + url);
// Get redirection page with generated link on it. This link can't be constructed manually as it contains Hash field and hashing algo is unknown.
var results = await RequestStringWithCookies(url);
var results = await WebRequestWithCookiesAsync(url);
if (results.ContentString == null)
{
throw new ExceptionWithConfigData("Empty response from " + url, configData);
@ -659,7 +659,7 @@ namespace Jackett.Common.Indexers
private async Task<List<ReleaseInfo>> FollowTrackerRedirection(string url, TrackerUrlDetails details)
{
logger.Debug("FollowTrackerRedirection: " + url);
var results = await RequestStringWithCookies(url);
var results = await WebRequestWithCookiesAsync(url);
var releases = new List<ReleaseInfo>();
try

10
src/Jackett.Common/Indexers/MejorTorrent.cs
View File

@ -101,14 +101,14 @@ namespace Jackett.Common.Indexers
var downloadUrl = link.ToString();
// Eg https://www.mejortorrentt.net/peli-descargar-torrent-11995-Harry-Potter-y-la-piedra-filosofal.html
var result = await RequestStringWithCookies(downloadUrl);
var result = await WebRequestWithCookiesAsync(downloadUrl);
if (result.Status != HttpStatusCode.OK)
throw new ExceptionWithConfigData(result.ContentString, configData);
var dom = parser.ParseDocument(result.ContentString);
downloadUrl = SiteLink + dom.QuerySelector("a[href*=\"sec=descargas\"]").GetAttribute("href");
// Eg https://www.mejortorrentt.net/secciones.php?sec=descargas&ap=contar&tabla=peliculas&id=11995&link_bajar=1
result = await RequestStringWithCookies(downloadUrl);
result = await WebRequestWithCookiesAsync(downloadUrl);
if (result.Status != HttpStatusCode.OK)
throw new ExceptionWithConfigData(result.ContentString, configData);
dom = parser.ParseDocument(result.ContentString);
@ -123,7 +123,7 @@ namespace Jackett.Common.Indexers
{
var releases = new List<ReleaseInfo>();
var url = SiteLink + NewTorrentsUrl;
var result = await RequestStringWithCookies(url);
var result = await WebRequestWithCookiesAsync(url);
if (result.Status != HttpStatusCode.OK)
throw new ExceptionWithConfigData(result.ContentString, configData);
try
@ -180,7 +180,7 @@ namespace Jackett.Common.Indexers
var searchTerm = GetLongestWord(query.SearchTerm);
var qc = new NameValueCollection { { "sec", "buscador" }, { "valor", searchTerm } };
var url = SiteLink + SearchUrl + "?" + qc.GetQueryString();
var result = await RequestStringWithCookies(url);
var result = await WebRequestWithCookiesAsync(url);
if (result.Status != HttpStatusCode.OK)
throw new ExceptionWithConfigData(result.ContentString, configData);
@ -259,7 +259,7 @@ namespace Jackett.Common.Indexers
private async Task ParseSeriesRelease(ICollection<ReleaseInfo> releases, TorznabQuery query, string title,
string commentsLink, string cat, DateTime publishDate)
{
var result = await RequestStringWithCookies(commentsLink);
var result = await WebRequestWithCookiesAsync(commentsLink);
if (result.Status != HttpStatusCode.OK)
throw new ExceptionWithConfigData(result.ContentString, configData);

2
src/Jackett.Common/Indexers/Milkie.cs
View File

@ -87,7 +87,7 @@ namespace Jackett.Common.Indexers
{
{ "x-milkie-auth", configData.Key.Value }
};
var jsonResponse = await RequestStringWithCookies(endpoint, headers: headers);
var jsonResponse = await WebRequestWithCookiesAsync(endpoint, headers: headers);
var releases = new List<ReleaseInfo>();

8
src/Jackett.Common/Indexers/MoreThanTV.cs
View File

@ -13,6 +13,7 @@ using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json.Linq;
using NLog;
@ -59,8 +60,7 @@ namespace Jackett.Common.Indexers
{ "login", "Log in" },
{ "keeplogged", "1" }
};
var preRequest = await RequestStringWithCookiesAndRetry(LoginUrl, string.Empty);
var preRequest = await RequestWithCookiesAndRetryAsync(LoginUrl, string.Empty);
var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, preRequest.Cookies, true, SearchUrl, SiteLink);
await ConfigureIfOK(result.Cookies, result.ContentString != null && result.ContentString.Contains("status\":\"success\""), () =>
@ -134,12 +134,12 @@ namespace Jackett.Common.Indexers
private async Task GetReleases(ICollection<ReleaseInfo> releases, TorznabQuery query, string searchQuery)
{
var searchUrl = GetTorrentSearchUrl(query, searchQuery);
var response = await RequestStringWithCookiesAndRetry(searchUrl);
var response = await RequestWithCookiesAndRetryAsync(searchUrl);
if (response.IsRedirect)
{
// re login
await ApplyConfiguration(null);
response = await RequestStringWithCookiesAndRetry(searchUrl);
response = await RequestWithCookiesAndRetryAsync(searchUrl);
}
try

4
src/Jackett.Common/Indexers/MyAmity.cs
View File

@ -119,12 +119,12 @@ namespace Jackett.Common.Indexers
}
searchUrl += "?" + queryCollection.GetQueryString();
var response = await RequestStringWithCookies(searchUrl);
var response = await WebRequestWithCookiesAsync(searchUrl);
if (response.IsRedirect || response.Cookies != null && response.Cookies.Contains("pass=deleted;"))
{
// re-login
await ApplyConfiguration(null);
response = await RequestStringWithCookies(searchUrl);
response = await WebRequestWithCookiesAsync(searchUrl);
}
var results = response.ContentString;

2
src/Jackett.Common/Indexers/MyAnonamouse.cs
View File

@ -199,7 +199,7 @@ namespace Jackett.Common.Indexers
if (qParams.Count > 0)
urlSearch += $"?{qParams.GetQueryString()}";
var response = await RequestStringWithCookiesAndRetry(urlSearch);
var response = await RequestWithCookiesAndRetryAsync(urlSearch);
if (response.ContentString.StartsWith("Error"))
throw new Exception(response.ContentString);

8
src/Jackett.Common/Indexers/NCore.cs
View File

@ -99,7 +99,7 @@ namespace Jackett.Common.Indexers
LoadValuesFromJson(configJson);
if (configData.Hungarian.Value == false && configData.English.Value == false)
throw new ExceptionWithConfigData("Please select at least one language.", configData);
var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty);
var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty);
var pairs = new Dictionary<string, string>
{
{"nev", configData.Username.Value},
@ -166,7 +166,8 @@ namespace Jackett.Common.Indexers
cats = cats.Except(_languageCats).ToList();
pairs.Add("kivalasztott_tipus[]", string.Join(",", cats));
var results = await PostDataWithCookiesAndRetry(SearchUrl, pairs.ToEnumerable(true));
var results = await RequestWithCookiesAndRetryAsync(
SearchUrl, null, RequestType.POST, null, pairs.ToEnumerable(true));
var parser = new HtmlParser();
var dom = parser.ParseDocument(results.ContentString);
@ -199,7 +200,8 @@ namespace Jackett.Common.Indexers
for (var page = startPage; page <= pages && releases.Count < limit; page++)
{
pairs["oldal"] = page.ToString();
results = await PostDataWithCookiesAndRetry(SearchUrl, pairs.ToEnumerable(true));
results = await RequestWithCookiesAndRetryAsync(
SearchUrl, null, RequestType.POST, null, pairs.ToEnumerable(true));
releases.AddRange(ParseTorrents(results, episodeString, query, releases.Count, limit, previouslyParsedOnPage));
previouslyParsedOnPage = 0;
}

5
src/Jackett.Common/Indexers/Nebulance.cs
View File

@ -11,6 +11,7 @@ using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json.Linq;
using NLog;
@ -74,14 +75,14 @@ namespace Jackett.Common.Indexers
protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
{
var loggedInCheck = await RequestStringWithCookies(SearchUrl);
var loggedInCheck = await WebRequestWithCookiesAsync(SearchUrl);
if (!loggedInCheck.ContentString.Contains("logout.php")) // re-login
await DoLogin();
// #6413
var url = $"{SearchUrl}&searchtext={WebUtility.UrlEncode(query.GetQueryString())}";
var response = await RequestStringWithCookiesAndRetry(url);
var response = await RequestWithCookiesAndRetryAsync(url);
var releases = ParseResponse(response.ContentString);
return releases;

10
src/Jackett.Common/Indexers/NewPCT.cs
View File

@ -163,7 +163,7 @@ namespace Jackett.Common.Indexers
public override async Task<byte[]> Download(Uri linkParam)
{
var results = await RequestStringWithCookiesAndRetry(linkParam.AbsoluteUri);
var results = await RequestWithCookiesAndRetryAsync(linkParam.AbsoluteUri);
var uriLink = ExtractDownloadUri(results.ContentString, linkParam.AbsoluteUri);
if (uriLink == null)
@ -211,7 +211,7 @@ namespace Jackett.Common.Indexers
while (pg <= _maxDailyPages)
{
var pageUrl = SiteLink + string.Format(_dailyUrl, pg);
var results = await RequestStringWithCookiesAndRetry(pageUrl);
var results = await RequestWithCookiesAndRetryAsync(pageUrl);
if (results == null || string.IsNullOrEmpty(results.ContentString))
break;
@ -304,13 +304,13 @@ namespace Jackett.Common.Indexers
var releases = new List<ReleaseInfo>();
// Episodes list
var results = await RequestStringWithCookiesAndRetry(uri.AbsoluteUri);
var results = await RequestWithCookiesAndRetryAsync(uri.AbsoluteUri);
var seriesEpisodesUrl = ParseSeriesListContent(results.ContentString, seriesName);
// TV serie list
if (!string.IsNullOrEmpty(seriesEpisodesUrl))
{
results = await RequestStringWithCookiesAndRetry(seriesEpisodesUrl);
results = await RequestWithCookiesAndRetryAsync(seriesEpisodesUrl);
var items = ParseEpisodesListContent(results.ContentString);
if (items != null && items.Any())
releases.AddRange(items);
@ -464,7 +464,7 @@ namespace Jackett.Common.Indexers
{"pg", pg.ToString()}
};
var results = await PostDataWithCookies(searchJsonUrl, queryCollection);
var results = await WebRequestWithCookiesAsync(searchJsonUrl, method: RequestType.POST, data: queryCollection);
var items = ParseSearchJsonContent(results.ContentString, year);
if (!items.Any())
break;

4
src/Jackett.Common/Indexers/NewRealWorld.cs
View File

@ -150,12 +150,12 @@ namespace Jackett.Common.Indexers
searchUrl += "?" + queryCollection.GetQueryString();
var response = await RequestStringWithCookies(searchUrl);
var response = await WebRequestWithCookiesAsync(searchUrl);
if (response.IsRedirect)
{
// re-login
await ApplyConfiguration(null);
response = await RequestStringWithCookies(searchUrl);
response = await WebRequestWithCookiesAsync(searchUrl);
}
try

12
src/Jackett.Common/Indexers/NorBits.cs
View File

@ -136,7 +136,7 @@ namespace Jackett.Common.Indexers
// Get index page for cookies
Output("\nGetting index page (for cookies).. with " + SiteLink);
var indexPage = await webclient.GetString(myIndexRequest);
var indexPage = await webclient.GetResultAsync(myIndexRequest);
// Building login form data
var pairs = new Dictionary<string, string> {
@ -158,7 +158,7 @@ namespace Jackett.Common.Indexers
// Get login page -- (not used, but simulation needed by tracker security's checks)
LatencyNow();
Output("\nGetting login page (user simulation).. with " + LoginUrl);
await webclient.GetString(myRequestLogin);
await webclient.GetResultAsync(myRequestLogin);
// Build WebRequest for submitting authentification
var request = new Utils.Clients.WebRequest()
@ -175,7 +175,7 @@ namespace Jackett.Common.Indexers
// Perform loggin
LatencyNow();
Output("\nPerform loggin.. with " + LoginCheckUrl);
var response = await webclient.GetString(request);
var response = await webclient.GetResultAsync(request);
// Test if we are logged in
await ConfigureIfOK(response.Cookies, response.Cookies != null && response.Cookies.Contains("uid="), () =>
@ -204,7 +204,7 @@ namespace Jackett.Common.Indexers
{
// Checking ...
Output("\n-> Checking logged-in state....");
var loggedInCheck = await RequestStringWithCookies(SearchUrl);
var loggedInCheck = await WebRequestWithCookiesAsync(SearchUrl);
if (!loggedInCheck.ContentString.Contains("logout.php"))
{
// Cookie expired, renew session on provider
@ -261,7 +261,7 @@ namespace Jackett.Common.Indexers
var request = BuildQuery(searchTerm, query, searchUrl);
// Getting results & Store content
var response = await RequestStringWithCookiesAndRetry(request, ConfigData.CookieHeader.Value);
var response = await RequestWithCookiesAndRetryAsync(request, ConfigData.CookieHeader.Value);
var parser = new HtmlParser();
var dom = parser.ParseDocument(response.ContentString);
@ -526,7 +526,7 @@ namespace Jackett.Common.Indexers
// Request our first page
LatencyNow();
var results = await RequestStringWithCookiesAndRetry(request, ConfigData.CookieHeader.Value, SearchUrl, _emulatedBrowserHeaders);
var results = await RequestWithCookiesAndRetryAsync(request, ConfigData.CookieHeader.Value, RequestType.GET, SearchUrl, null, _emulatedBrowserHeaders);
// Return results from tracker
return results;

12
src/Jackett.Common/Indexers/NordicBits.cs
View File

@ -191,7 +191,7 @@ namespace Jackett.Common.Indexers
// Get index page for cookies
Output("\nGetting index page (for cookies).. with " + SiteLink);
var indexPage = await webclient.GetString(myIndexRequest);
var indexPage = await webclient.GetResultAsync(myIndexRequest);
// Building login form data
var pairs = new Dictionary<string, string> {
@ -213,7 +213,7 @@ namespace Jackett.Common.Indexers
// Get login page -- (not used, but simulation needed by tracker security's checks)
LatencyNow();
Output("\nGetting login page (user simulation).. with " + LoginUrl);
await webclient.GetString(myRequestLogin);
await webclient.GetResultAsync(myRequestLogin);
// Build WebRequest for submitting authentification
var request = new Utils.Clients.WebRequest()
@ -230,7 +230,7 @@ namespace Jackett.Common.Indexers
// Perform loggin
LatencyNow();
Output("\nPerform loggin.. with " + LoginCheckUrl);
var response = await webclient.GetString(request);
var response = await webclient.GetResultAsync(request);
// Test if we are logged in
await ConfigureIfOK(response.Cookies, response.Cookies != null && response.Cookies.Contains("uid="), () =>
@ -259,7 +259,7 @@ namespace Jackett.Common.Indexers
{
// Checking ...
Output("\n-> Checking logged-in state....");
var loggedInCheck = await RequestStringWithCookies(SearchUrl);
var loggedInCheck = await WebRequestWithCookiesAsync(SearchUrl);
if (!loggedInCheck.ContentString.Contains("logout.php"))
{
// Cookie expired, renew session on provider
@ -316,7 +316,7 @@ namespace Jackett.Common.Indexers
var request = BuildQuery(searchTerm, query, searchUrl);
// Getting results & Store content
var response = await RequestStringWithCookiesAndRetry(request, ConfigData.CookieHeader.Value);
var response = await RequestWithCookiesAndRetryAsync(request, ConfigData.CookieHeader.Value);
var parser = new HtmlParser();
var dom = parser.ParseDocument(response.ContentString);
@ -597,7 +597,7 @@ namespace Jackett.Common.Indexers
// Request our first page
LatencyNow();
var results = await RequestStringWithCookiesAndRetry(request, ConfigData.CookieHeader.Value, SearchUrl, _emulatedBrowserHeaders);
var results = await RequestWithCookiesAndRetryAsync(request, ConfigData.CookieHeader.Value, RequestType.GET, SearchUrl, null, _emulatedBrowserHeaders);
// Return results from tracker
return results;

7
src/Jackett.Common/Indexers/Partis.cs
View File

@ -135,13 +135,13 @@ namespace Jackett.Common.Indexers
logger.Info(string.Format("Searh URL Partis_: {0}", searchUrl));
// add necessary headers
var heder = new Dictionary<string, string>
var header = new Dictionary<string, string>
{
{ "X-requested-with", "XMLHttpRequest" }
};
//get results and follow redirect
results = await RequestStringWithCookies(searchUrl, null, SearchUrl, heder);
results = await WebRequestWithCookiesAsync(searchUrl, referer: SearchUrl, headers: header);
await FollowIfRedirect(results, null, null, null, true);
// are we logged in?
@ -150,7 +150,8 @@ namespace Jackett.Common.Indexers
await ApplyConfiguration(null);
}
// another request with specific query - NEEDED for succesful response - return data
results = await RequestStringWithCookies(SiteLink + "brskaj/?rs=false&offset=0", null, SearchUrl, heder);
results = await WebRequestWithCookiesAsync(
SiteLink + "brskaj/?rs=false&offset=0", referer: SearchUrl, headers: header);
await FollowIfRedirect(results, null, null, null, true);
// parse results

5
src/Jackett.Common/Indexers/PassThePopcorn.cs
View File

@ -11,6 +11,7 @@ using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json.Linq;
using NLog;
@ -115,9 +116,9 @@ namespace Jackett.Common.Indexers
{ "ApiKey", configData.Key.Value }
};
var results = await RequestStringWithCookiesAndRetry(movieListSearchUrl, headers: authHeaders);
var results = await RequestWithCookiesAndRetryAsync(movieListSearchUrl, headers: authHeaders);
if (results.IsRedirect) // untested
results = await RequestStringWithCookiesAndRetry(movieListSearchUrl, headers: authHeaders);
results = await RequestWithCookiesAndRetryAsync(movieListSearchUrl, headers: authHeaders);
try
{
//Iterate over the releases for each movie

6
src/Jackett.Common/Indexers/PirateTheNet.cs
View File

@ -76,7 +76,7 @@ namespace Jackett.Common.Indexers
LoadValuesFromJson(configJson);
CookieHeader = ""; // clear old cookies
var result1 = await RequestStringWithCookies(CaptchaUrl);
var result1 = await WebRequestWithCookiesAsync(CaptchaUrl);
var json1 = JObject.Parse(result1.ContentString);
var captchaSelection = json1["images"][0]["hash"];
@ -125,12 +125,12 @@ namespace Jackett.Common.Indexers
var searchUrl = SearchUrl + "?" + qc.GetQueryString();
var results = await RequestStringWithCookiesAndRetry(searchUrl);
var results = await RequestWithCookiesAndRetryAsync(searchUrl);
if (results.IsRedirect)
{
// re-login
await ApplyConfiguration(null);
results = await RequestStringWithCookiesAndRetry(searchUrl);
results = await RequestWithCookiesAndRetryAsync(searchUrl);
}
try

8
src/Jackett.Common/Indexers/PixelHD.cs
View File

@ -57,7 +57,7 @@ namespace Jackett.Common.Indexers
public override async Task<ConfigurationData> GetConfigurationForSetup()
{
var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty);
var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty);
var LoginParser = new HtmlParser();
var LoginDocument = LoginParser.ParseDocument(loginPage.ContentString);
@ -69,7 +69,7 @@ namespace Jackett.Common.Indexers
var catchaInput = LoginDocument.QuerySelector("input[maxlength=\"6\"]");
input_captcha = catchaInput.GetAttribute("name");
var captchaImage = await RequestBytesWithCookies(SiteLink + catchaImg.GetAttribute("src"), loginPage.Cookies, RequestType.GET, LoginUrl);
var captchaImage = await WebRequestWithCookiesAsync(SiteLink + catchaImg.GetAttribute("src"), loginPage.Cookies, RequestType.GET, LoginUrl);
configData.CaptchaImage.Value = captchaImage.ContentBytes;
}
else
@ -137,13 +137,13 @@ namespace Jackett.Common.Indexers
var searchUrl = BrowseUrl + "?" + queryCollection.GetQueryString();
var results = await RequestStringWithCookies(searchUrl);
var results = await WebRequestWithCookiesAsync(searchUrl);
if (results.IsRedirect)
{
// re login
await GetConfigurationForSetup();
await ApplyConfiguration(null);
results = await RequestStringWithCookies(searchUrl);
results = await WebRequestWithCookiesAsync(searchUrl);
}
var IMDBRegEx = new Regex(@"tt(\d+)", RegexOptions.Compiled);

4
src/Jackett.Common/Indexers/PolishTracker.cs
View File

@ -104,12 +104,12 @@ namespace Jackett.Common.Indexers
qc.Add("cat[]", cat);
var searchUrl = SearchUrl + "?" + qc.GetQueryString();
var result = await RequestStringWithCookiesAndRetry(searchUrl, null, SearchUrl);
var result = await RequestWithCookiesAndRetryAsync(searchUrl, referer: SearchUrl);
if (result.IsRedirect)
{
// re-login
await ApplyConfiguration(null);
result = await RequestStringWithCookiesAndRetry(searchUrl, null, SearchUrl);
result = await RequestWithCookiesAndRetryAsync(searchUrl, referer: SearchUrl);
}
if (!result.ContentString.StartsWith("{")) // not JSON => error

10
src/Jackett.Common/Indexers/PornoLab.cs
View File

@ -183,13 +183,13 @@ namespace Jackett.Common.Indexers
public override async Task<ConfigurationData> GetConfigurationForSetup()
{
configData.CookieHeader.Value = null;
var response = await RequestStringWithCookies(LoginUrl);
var response = await WebRequestWithCookiesAsync(LoginUrl);
var LoginResultParser = new HtmlParser();
var LoginResultDocument = LoginResultParser.ParseDocument(response.ContentString);
var captchaimg = LoginResultDocument.QuerySelector("img[src*=\"/captcha/\"]");
if (captchaimg != null)
{
var captchaImage = await RequestBytesWithCookies("https:" + captchaimg.GetAttribute("src"));
var captchaImage = await WebRequestWithCookiesAsync("https:" + captchaimg.GetAttribute("src"));
configData.CaptchaImage.Value = captchaImage.ContentBytes;
var codefield = LoginResultDocument.QuerySelector("input[name^=\"cap_code_\"]");
@ -260,12 +260,12 @@ namespace Jackett.Common.Indexers
}
var searchUrl = SearchUrl + "?" + queryCollection.GetQueryString();
var results = await RequestStringWithCookies(searchUrl);
var results = await WebRequestWithCookiesAsync(searchUrl);
if (!results.ContentString.Contains("Вы зашли как:"))
{
// re login
await ApplyConfiguration(null);
results = await RequestStringWithCookies(searchUrl);
results = await WebRequestWithCookiesAsync(searchUrl);
}
try
{
@ -337,7 +337,7 @@ namespace Jackett.Common.Indexers
public override async Task<byte[]> Download(Uri link)
{
var downloadlink = link;
var response = await RequestStringWithCookies(link.ToString());
var response = await WebRequestWithCookiesAsync(link.ToString());
var results = response.ContentString;
var SearchResultParser = new HtmlParser();
var SearchResultDocument = SearchResultParser.ParseDocument(results);

8
src/Jackett.Common/Indexers/PreToMe.cs
View File

@ -123,7 +123,7 @@ namespace Jackett.Common.Indexers
{
LoadValuesFromJson(configJson);
var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty);
var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty);
var pairs = new Dictionary<string, string> {
{ "returnto", "%2F" },
@ -134,7 +134,7 @@ namespace Jackett.Common.Indexers
};
// Send Post
var result = await PostDataWithCookies(LoginUrl, pairs, loginPage.Cookies);
var result = await WebRequestWithCookiesAsync(LoginUrl, loginPage.Cookies, RequestType.POST, data: pairs);
if (result.RedirectingTo == null)
throw new ExceptionWithConfigData("Login failed. Did you use the PIN number that pretome emailed you?", configData);
@ -200,12 +200,12 @@ namespace Jackett.Common.Indexers
}
var searchUrl = SearchUrl + "?" + qc.GetQueryString();
var response = await RequestStringWithCookiesAndRetry(searchUrl);
var response = await RequestWithCookiesAndRetryAsync(searchUrl);
if (response.IsRedirect) // re-login
{
await ApplyConfiguration(null);
response = await RequestStringWithCookiesAndRetry(searchUrl);
response = await RequestWithCookiesAndRetryAsync(searchUrl);
}
try

7
src/Jackett.Common/Indexers/RarBG.cs
View File

@ -11,6 +11,7 @@ using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json.Linq;
using NLog;
using static Jackett.Common.Models.IndexerConfig.ConfigurationData;
@ -120,7 +121,7 @@ namespace Jackett.Common.Indexers
// check the token and renewal if necessary
await RenewalTokenAsync();
var response = await RequestStringWithCookiesAndRetry(BuildSearchUrl(query));
var response = await RequestWithCookiesAndRetryAsync(BuildSearchUrl(query));
var jsonContent = JObject.Parse(response.ContentString);
var errorCode = jsonContent.Value<int>("error_code");
switch (errorCode)
@ -130,7 +131,7 @@ namespace Jackett.Common.Indexers
case 2:
case 4: // invalid token
await RenewalTokenAsync(true); // force renewal token
response = await RequestStringWithCookiesAndRetry(BuildSearchUrl(query));
response = await RequestWithCookiesAndRetryAsync(BuildSearchUrl(query));
jsonContent = JObject.Parse(response.ContentString);
break;
case 10: // imdb not found, see issue #1486
@ -263,7 +264,7 @@ namespace Jackett.Common.Indexers
{ "app_id", _appId }
};
var tokenUrl = ApiEndpoint + "?" + qc.GetQueryString();
var result = await RequestStringWithCookiesAndRetry(tokenUrl);
var result = await RequestWithCookiesAndRetryAsync(tokenUrl);
var json = JObject.Parse(result.ContentString);
_token = json.Value<string>("token");
_lastTokenFetch = DateTime.Now;

10
src/Jackett.Common/Indexers/RevolutionTT.cs
View File

@ -13,6 +13,7 @@ using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json.Linq;
using NLog;
@ -207,7 +208,8 @@ namespace Jackett.Common.Indexers
var rssParams = new Dictionary<string, string> {
{ "feed", "dl" }
};
var rssPage = await PostDataWithCookies(GetRSSKeyUrl, rssParams, result.Cookies);
var rssPage = await WebRequestWithCookiesAsync(
GetRSSKeyUrl, result.Cookies, RequestType.POST, data: rssParams);
var match = Regex.Match(rssPage.ContentString, "(?<=passkey\\=)([a-zA-z0-9]*)");
configData.RSSKey.Value = match.Success ? match.Value : string.Empty;
if (string.IsNullOrWhiteSpace(configData.RSSKey.Value))
@ -232,7 +234,7 @@ namespace Jackett.Common.Indexers
// If query is empty, use the RSS Feed
if (string.IsNullOrWhiteSpace(searchString))
{
var rssPage = await RequestStringWithCookiesAndRetry(RSSUrl + configData.RSSKey.Value);
var rssPage = await RequestWithCookiesAndRetryAsync(RSSUrl + configData.RSSKey.Value);
var rssDoc = XDocument.Parse(rssPage.ContentString);
foreach (var item in rssDoc.Descendants("item"))
@ -305,12 +307,12 @@ namespace Jackett.Common.Indexers
}
}
var results = await RequestStringWithCookiesAndRetry(searchUrl);
var results = await RequestWithCookiesAndRetryAsync(searchUrl);
if (results.IsRedirect)
{
// re-login
await ApplyConfiguration(null);
results = await RequestStringWithCookiesAndRetry(searchUrl);
results = await RequestWithCookiesAndRetryAsync(searchUrl);
}
try

8
src/Jackett.Common/Indexers/RuTracker.cs
View File

@ -1510,13 +1510,13 @@ namespace Jackett.Common.Indexers
try
{
configData.CookieHeader.Value = null;
var response = await RequestStringWithCookies(LoginUrl);
var response = await WebRequestWithCookiesAsync(LoginUrl);
var parser = new HtmlParser();
var doc = parser.ParseDocument(response.ContentString);
var captchaimg = doc.QuerySelector("img[src^=\"https://static.t-ru.org/captcha/\"]");
if (captchaimg != null)
{
var captchaImage = await RequestBytesWithCookies(captchaimg.GetAttribute("src"));
var captchaImage = await WebRequestWithCookiesAsync(captchaimg.GetAttribute("src"));
configData.CaptchaImage.Value = captchaImage.ContentBytes;
var codefield = doc.QuerySelector("input[name^=\"cap_code_\"]");
@ -1591,12 +1591,12 @@ namespace Jackett.Common.Indexers
}
var searchUrl = SearchUrl + "?" + queryCollection.GetQueryString();
var results = await RequestStringWithCookies(searchUrl);
var results = await WebRequestWithCookiesAsync(searchUrl);
if (!results.ContentString.Contains("id=\"logged-in-username\""))
{
// re login
await ApplyConfiguration(null);
results = await RequestStringWithCookies(searchUrl);
results = await WebRequestWithCookiesAsync(searchUrl);
}
try
{

2
src/Jackett.Common/Indexers/SceneHD.cs
View File

@ -91,7 +91,7 @@ namespace Jackett.Common.Indexers
qc.Add("categories[" + cat + "]", "1");
var searchUrl = SearchUrl + qc.GetQueryString();
var response = await RequestStringWithCookiesAndRetry(searchUrl);
var response = await RequestWithCookiesAndRetryAsync(searchUrl);
if (response.ContentString?.Contains("User not found or passkey not set") == true)
throw new Exception("The passkey is invalid. Check the indexer configuration.");

4
src/Jackett.Common/Indexers/SceneTime.cs
View File

@ -107,7 +107,7 @@ namespace Jackett.Common.Indexers
WebResult loginPage;
try
{
loginPage = await RequestStringWithCookies(StartPageUrl, string.Empty);
loginPage = await WebRequestWithCookiesAsync(StartPageUrl, string.Empty);
}
catch (Exception)
{
@ -198,7 +198,7 @@ namespace Jackett.Common.Indexers
qParams.Add("freeleech", "on");
var searchUrl = SearchUrl + "?" + qParams.GetQueryString();
var results = await RequestStringWithCookies(searchUrl);
var results = await WebRequestWithCookiesAsync(searchUrl);
// response without results (the message is misleading)
if (results.ContentString?.Contains("slow down geek!!!") == true)

9
src/Jackett.Common/Indexers/Shazbat.cs
View File

@ -67,7 +67,7 @@ namespace Jackett.Common.Indexers
var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, null, true, null, LoginUrl);
await ConfigureIfOK(result.Cookies, result.ContentString?.Contains("glyphicon-log-out") == true,
() => throw new ExceptionWithConfigData("The username and password entered do not match.", configData));
var rssProfile = await RequestStringWithCookiesAndRetry(RSSProfile);
var rssProfile = await RequestWithCookiesAndRetryAsync(RSSProfile);
var parser = new HtmlParser();
var rssDom = parser.ParseDocument(rssProfile.ContentString);
configData.RSSKey.Value = rssDom.QuerySelector(".col-sm-9:nth-of-type(1)").TextContent.Trim();
@ -89,7 +89,8 @@ namespace Jackett.Common.Indexers
{
{"search", query.SanitizedSearchTerm}
};
results = await PostDataWithCookiesAndRetry(SearchUrl, pairs, null, TorrentsUrl);
results = await RequestWithCookiesAndRetryAsync(
SearchUrl, null, RequestType.POST, TorrentsUrl, pairs);
results = await ReloginIfNecessary(results);
var parser = new HtmlParser();
var dom = parser.ParseDocument(results.ContentString);
@ -107,7 +108,7 @@ namespace Jackett.Common.Indexers
{
foreach (var searchUrl in searchUrls)
{
results = await RequestStringWithCookies(searchUrl);
results = await WebRequestWithCookiesAsync(searchUrl);
results = await ReloginIfNecessary(results);
var parser = new HtmlParser();
var dom = parser.ParseDocument(results.ContentString);
@ -177,7 +178,7 @@ namespace Jackett.Common.Indexers
await ApplyConfiguration(null);
response.Request.Cookies = CookieHeader;
return await webclient.GetString(response.Request);
return await webclient.GetResultAsync(response.Request);
}
}
}

2
src/Jackett.Common/Indexers/ShowRSS.cs
View File

@ -60,7 +60,7 @@ namespace Jackett.Common.Indexers
{
var releases = new List<ReleaseInfo>();
var episodeSearchUrl = string.Format(SearchAllUrl);
var result = await RequestStringWithCookiesAndRetry(episodeSearchUrl);
var result = await RequestWithCookiesAndRetryAsync(episodeSearchUrl);
var xmlDoc = new XmlDocument();
try

2
src/Jackett.Common/Indexers/SolidTorrents.cs
View File

@ -103,7 +103,7 @@ namespace Jackett.Common.Indexers
{"fuv", "no"}
};
var fullSearchUrl = SearchUrl + "?" + queryCollection.GetQueryString();
var result = await RequestStringWithCookies(fullSearchUrl, null, null, APIHeaders);
var result = await WebRequestWithCookiesAsync(fullSearchUrl, headers: APIHeaders);
return CheckResponse(result);
}

4
src/Jackett.Common/Indexers/SpeedCD.cs
View File

@ -151,11 +151,11 @@ namespace Jackett.Common.Indexers
}
var searchUrl = SearchUrl + string.Join("/", qc);
var response = await RequestStringWithCookiesAndRetry(searchUrl);
var response = await RequestWithCookiesAndRetryAsync(searchUrl);
if (!response.ContentString.Contains("/logout.php")) // re-login
{
await DoLogin();
response = await RequestStringWithCookiesAndRetry(searchUrl);
response = await RequestWithCookiesAndRetryAsync(searchUrl);
}
try

2
src/Jackett.Common/Indexers/SuperBits.cs
View File

@ -128,7 +128,7 @@ namespace Jackett.Common.Indexers
searchUrl += "?" + queryCollection.GetQueryString();
foreach (var cat in MapTorznabCapsToTrackers(query))
searchUrl += "&categories[]=" + cat;
var results = await RequestStringWithCookies(searchUrl, null, SiteLink);
var results = await WebRequestWithCookiesAsync(searchUrl, referer: SiteLink);
try
{

10
src/Jackett.Common/Indexers/TVStore.cs
View File

@ -66,7 +66,7 @@ namespace Jackett.Common.Indexers
public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
{
LoadValuesFromJson(configJson);
var loginPage = await RequestStringWithCookies(LoginPageUrl, string.Empty);
var loginPage = await WebRequestWithCookiesAsync(LoginPageUrl, string.Empty);
var pairs = new Dictionary<string, string>
{
{"username", configData.Username.Value},
@ -163,7 +163,7 @@ namespace Jackett.Common.Indexers
queryParams["id"] = torrentId;
queryParams["now"] = DateTimeUtil.DateTimeToUnixTimestamp(DateTime.UtcNow)
.ToString(CultureInfo.InvariantCulture);
var filesList = (await RequestStringWithCookiesAndRetry(SearchUrl + "?" + queryParams.GetQueryString()))
var filesList = (await RequestWithCookiesAndRetryAsync(SearchUrl + "?" + queryParams.GetQueryString()))
.ContentString;
var firstFileName = filesList.Split(
new[]
@ -215,7 +215,7 @@ namespace Jackett.Common.Indexers
/// </summary>
private async Task PopulateImdbMapAsync()
{
var result = await RequestStringWithCookiesAndRetry(BrowseUrl);
var result = await RequestWithCookiesAndRetryAsync(BrowseUrl);
foreach (Match match in _seriesInfoMatch.Matches(result.ContentString))
{
var internalId = int.Parse(match.Groups["seriesID"].Value);
@ -280,7 +280,7 @@ namespace Jackett.Common.Indexers
queryParams.Add("e", query.Episode);
}
var results = await RequestStringWithCookiesAndRetry(SearchUrl + "?" + queryParams.GetQueryString());
var results = await RequestWithCookiesAndRetryAsync(SearchUrl + "?" + queryParams.GetQueryString());
// Parse page Information from result
var content = results.ContentString;
var splits = content.Split('\\');
@ -302,7 +302,7 @@ namespace Jackett.Common.Indexers
for (var page = startPage; page <= pages && releases.Count < query.Limit; page++)
{
queryParams["page"] = page.ToString();
results = await RequestStringWithCookiesAndRetry(SearchUrl + "?" + queryParams.GetQueryString());
results = await RequestWithCookiesAndRetryAsync(SearchUrl + "?" + queryParams.GetQueryString());
releases.AddRange(await ParseTorrentsAsync(results, releases.Count, query.Limit, previouslyParsedOnPage));
previouslyParsedOnPage = 0;
}

2
src/Jackett.Common/Indexers/TVVault.cs
View File

@ -94,7 +94,7 @@ namespace Jackett.Common.Indexers
qc.Add("searchstr", StripSearchString(query.GetQueryString()));
var searchUrl = BrowseUrl + "?" + qc.GetQueryString();
var results = await RequestStringWithCookies(searchUrl);
var results = await WebRequestWithCookiesAsync(searchUrl);
try
{
var seasonRegEx = new Regex(@$"Season\s+0*{query.Season}[^\d]", RegexOptions.IgnoreCase);

4
src/Jackett.Common/Indexers/Toloka.cs
View File

@ -232,12 +232,12 @@ namespace Jackett.Common.Indexers
}
var searchUrl = SearchUrl + "?" + queryCollection.GetQueryString();
var results = await RequestStringWithCookies(searchUrl);
var results = await WebRequestWithCookiesAsync(searchUrl);
if (!results.ContentString.Contains("logout=true"))
{
// re login
await ApplyConfiguration(null);
results = await RequestStringWithCookies(searchUrl);
results = await WebRequestWithCookiesAsync(searchUrl);
}
try
{

6
src/Jackett.Common/Indexers/TorrenTech.cs
View File

@ -100,10 +100,10 @@ namespace Jackett.Common.Indexers
}
var searchUrl = IndexUrl + "?" + queryCollection.GetQueryString();
results = await RequestStringWithCookies(searchUrl);
results = await WebRequestWithCookiesAsync(searchUrl);
if (results.IsRedirect && results.RedirectingTo.Contains("CODE=show"))
{
results = await RequestStringWithCookies(results.RedirectingTo);
results = await WebRequestWithCookiesAsync(results.RedirectingTo);
}
try
{
@ -203,7 +203,7 @@ namespace Jackett.Common.Indexers
public override async Task<byte[]> Download(Uri link)
{
var response = await RequestStringWithCookies(link.ToString());
var response = await WebRequestWithCookiesAsync(link.ToString());
var results = response.ContentString;
var SearchResultParser = new HtmlParser();
var SearchResultDocument = SearchResultParser.ParseDocument(results);

6
src/Jackett.Common/Indexers/TorrentBytes.cs
View File

@ -89,7 +89,7 @@ namespace Jackett.Common.Indexers
{"returnto", "/"},
{"login", "Log in!"}
};
var loginPage = await RequestStringWithCookies(SiteLink, string.Empty);
var loginPage = await WebRequestWithCookiesAsync(SiteLink, string.Empty);
var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, SiteLink, SiteLink);
await ConfigureIfOK(
result.Cookies, result.ContentString?.Contains("my.php") == true, () =>
@ -127,12 +127,12 @@ namespace Jackett.Common.Indexers
qc.Add("c" + cat, "1");
var searchUrl = SearchUrl + "?" + qc.GetQueryString();
var response = await RequestStringWithCookiesAndRetry(searchUrl, referer: SearchUrl);
var response = await RequestWithCookiesAndRetryAsync(searchUrl, referer: SearchUrl);
if (response.IsRedirect) // re-login
{
await ApplyConfiguration(null);
response = await RequestStringWithCookiesAndRetry(searchUrl, null, SearchUrl);
response = await RequestWithCookiesAndRetryAsync(searchUrl, referer: SearchUrl);
}
try

2
src/Jackett.Common/Indexers/TorrentDay.cs
View File

@ -155,7 +155,7 @@ namespace Jackett.Common.Indexers
else
searchUrl += ";q=" + WebUtilityHelpers.UrlEncode(query.GetQueryString(), Encoding);
var results = await RequestStringWithCookiesAndRetry(searchUrl);
var results = await RequestWithCookiesAndRetryAsync(searchUrl);
// Check for being logged out
if (results.IsRedirect)

8
src/Jackett.Common/Indexers/TorrentHeaven.cs
View File

@ -119,7 +119,7 @@ namespace Jackett.Common.Indexers
throw new ExceptionWithConfigData(errorMessage, configData);
}
var result2 = await RequestStringWithCookies(LoginCompleteUrl, result.Cookies);
var result2 = await WebRequestWithCookiesAsync(LoginCompleteUrl, result.Cookies);
await ConfigureIfOK(
result2.Cookies, result2.Cookies?.Contains("pass") == true,
() => throw new ExceptionWithConfigData("Didn't get a user/pass cookie", configData));
@ -128,14 +128,14 @@ namespace Jackett.Common.Indexers
public override async Task<ConfigurationData> GetConfigurationForSetup()
{
var loginPage = await RequestStringWithCookies(IndexUrl, string.Empty);
var loginPage = await WebRequestWithCookiesAsync(IndexUrl, string.Empty);
var parser = new HtmlParser();
var dom = parser.ParseDocument(loginPage.ContentString);
var qCaptchaImg = dom.QuerySelector("td.tablea > img");
if (qCaptchaImg != null)
{
var captchaUrl = SiteLink + qCaptchaImg.GetAttribute("src");
var captchaImage = await RequestBytesWithCookies(captchaUrl, loginPage.Cookies);
var captchaImage = await WebRequestWithCookiesAsync(captchaUrl, loginPage.Cookies);
configData.CaptchaImage.Value = captchaImage.ContentBytes;
}
else
@ -182,7 +182,7 @@ namespace Jackett.Common.Indexers
foreach (var cat in MapTorznabCapsToTrackers(query))
queryCollection.Add("dirs" + cat, "1");
searchUrl += "?" + queryCollection.GetQueryString();
var response = await RequestStringWithCookies(searchUrl);
var response = await WebRequestWithCookiesAsync(searchUrl);
var titleRegexp = new Regex(@"^return buildTable\('(.*?)',\s+");
try
{

7
src/Jackett.Common/Indexers/TorrentLeech.cs
View File

@ -12,6 +12,7 @@ using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
@ -107,7 +108,7 @@ namespace Jackett.Common.Indexers
public override async Task<ConfigurationData> GetConfigurationForSetup()
{
var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty);
var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty);
var parser = new HtmlParser();
var dom = parser.ParseDocument(loginPage.ContentString);
var captcha = dom.QuerySelector(".g-recaptcha");
@ -198,12 +199,12 @@ namespace Jackett.Common.Indexers
else
searchUrl += "newfilter/2"; // include 0day and music
var results = await RequestStringWithCookiesAndRetry(searchUrl);
var results = await RequestWithCookiesAndRetryAsync(searchUrl);
if (results.ContentString.Contains("/user/account/login")) // re-login
{
await DoLogin();
results = await RequestStringWithCookiesAndRetry(searchUrl);
results = await RequestWithCookiesAndRetryAsync(searchUrl);
}
try

3
src/Jackett.Common/Indexers/TorrentNetwork.cs
View File

@ -124,7 +124,8 @@ namespace Jackett.Common.Indexers
private async Task<dynamic> SendAPIRequest(string endpoint, object data)
{
var jsonData = JsonConvert.SerializeObject(data);
var result = await PostDataWithCookies(APIUrl + endpoint, null, null, SiteLink, APIHeaders, jsonData);
var result = await WebRequestWithCookiesAsync(
APIUrl + endpoint, method: RequestType.POST, referer: SiteLink, headers: APIHeaders, rawbody: jsonData);
if (!result.ContentString.StartsWith("{")) // not JSON => error
throw new ExceptionWithConfigData(result.ContentString, configData);
dynamic json = JsonConvert.DeserializeObject<dynamic>(result.ContentString);

7
src/Jackett.Common/Indexers/TorrentSeeds.cs
View File

@ -12,6 +12,7 @@ using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json.Linq;
using NLog;
@ -104,7 +105,7 @@ namespace Jackett.Common.Indexers
public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
{
LoadValuesFromJson(configJson);
var loginPage = await RequestStringWithCookies(TokenUrl);
var loginPage = await WebRequestWithCookiesAsync(TokenUrl);
var parser = new HtmlParser();
var dom = parser.ParseDocument(loginPage.ContentString);
var token = dom.QuerySelector("form.form-horizontal > span");
@ -149,12 +150,12 @@ namespace Jackett.Common.Indexers
foreach (var cat in MapTorznabCapsToTrackers(query))
queryCollection.Add($"cat[{cat}]", "1");
searchUrl += "?" + queryCollection.GetQueryString();
var response = await RequestStringWithCookiesAndRetry(searchUrl);
var response = await RequestWithCookiesAndRetryAsync(searchUrl);
var results = response.ContentString;
if (!results.Contains("/logout.php?"))
{
await ApplyConfiguration(null);
response = await RequestStringWithCookiesAndRetry(searchUrl);
response = await RequestWithCookiesAndRetryAsync(searchUrl);
results = response.ContentString;
}

6
src/Jackett.Common/Indexers/TorrentSyndikat.cs
View File

@ -111,7 +111,7 @@ namespace Jackett.Common.Indexers
LoadValuesFromJson(configJson);
CookieHeader = "";
var result1 = await RequestStringWithCookies(CaptchaUrl);
var result1 = await WebRequestWithCookiesAsync(CaptchaUrl);
var json1 = JObject.Parse(result1.ContentString);
var captchaSelection = json1["images"][0]["hash"];
@ -172,12 +172,12 @@ namespace Jackett.Common.Indexers
var searchUrl = SearchUrl + "?" + queryCollection.GetQueryString();
var results = await RequestStringWithCookiesAndRetry(searchUrl);
var results = await RequestWithCookiesAndRetryAsync(searchUrl);
if (results.IsRedirect)
{
await ApplyConfiguration(null);
results = await RequestStringWithCookiesAndRetry(searchUrl);
results = await RequestWithCookiesAndRetryAsync(searchUrl);
}
try

2
src/Jackett.Common/Indexers/TorrentsCSV.cs
View File

@ -70,7 +70,7 @@ namespace Jackett.Common.Indexers
var searchUrl = (string.IsNullOrWhiteSpace(searchString) ? NewEndpoint : SearchEndpoint)
+ "?" + qc.GetQueryString();
var response = await RequestStringWithCookiesAndRetry(searchUrl);
var response = await RequestWithCookiesAndRetryAsync(searchUrl);
try
{

13
src/Jackett.Common/Indexers/XSpeeds.cs
View File

@ -132,14 +132,14 @@ namespace Jackett.Common.Indexers
public override async Task<ConfigurationData> GetConfigurationForSetup()
{
var loginPage = await RequestStringWithCookies(LandingUrl);
var loginPage = await WebRequestWithCookiesAsync(LandingUrl);
var parser = new HtmlParser();
var dom = parser.ParseDocument(loginPage.ContentString);
var qCaptchaImg = dom.QuerySelector("img#regimage");
if (qCaptchaImg != null)
{
var captchaUrl = qCaptchaImg.GetAttribute("src");
var captchaImageResponse = await RequestBytesWithCookies(captchaUrl, loginPage.Cookies, RequestType.GET, LandingUrl);
var captchaImageResponse = await WebRequestWithCookiesAsync(captchaUrl, loginPage.Cookies, RequestType.GET, LandingUrl);
var captchaText = new StringItem { Name = "Captcha Text" };
var captchaImage = new ImageItem {Name = "Captcha Image", Value = captchaImageResponse.ContentBytes};
@ -189,7 +189,8 @@ namespace Jackett.Common.Indexers
{"timezone", "0"},
{"showrows", "50"}
};
var rssPage = await PostDataWithCookies(GetRSSKeyUrl, rssParams, result.Cookies);
var rssPage = await WebRequestWithCookiesAsync(
GetRSSKeyUrl, result.Cookies, RequestType.POST, data: rssParams);
var match = Regex.Match(rssPage.ContentString, "(?<=secret_key\\=)([a-zA-z0-9]*)");
configData.RSSKey.Value = match.Success ? match.Value : string.Empty;
if (string.IsNullOrWhiteSpace(configData.RSSKey.Value))
@ -227,12 +228,14 @@ namespace Jackett.Common.Indexers
searchParams.Add("search_type", "t_name");
}
var searchPage = await PostDataWithCookiesAndRetry(SearchUrl, searchParams, CookieHeader);
var searchPage = await RequestWithCookiesAndRetryAsync(
SearchUrl, CookieHeader, RequestType.POST, null, searchParams);
// Occasionally the cookies become invalid, login again if that happens
if (searchPage.IsRedirect)
{
await ApplyConfiguration(null);
searchPage = await PostDataWithCookiesAndRetry(SearchUrl, searchParams, CookieHeader);
searchPage = await RequestWithCookiesAndRetryAsync(
SearchUrl, CookieHeader, RequestType.POST, null, searchParams);
}
try

3
src/Jackett.Common/Indexers/Xthor.cs
View File

@ -17,6 +17,7 @@ using Jackett.Common.Utils.Clients;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
using WebRequest = Jackett.Common.Utils.Clients.WebRequest;
namespace Jackett.Common.Indexers
{
@ -488,7 +489,7 @@ namespace Jackett.Common.Indexers
};
// Request our first page
var results = await webclient.GetString(myIndexRequest);
var results = await webclient.GetResultAsync(myIndexRequest);
if (results.Status == HttpStatusCode.InternalServerError) // See issue #2110
throw new Exception("Internal Server Error (" + results.ContentString + "), probably you reached the API limits, please reduce the number of queries");

2
src/Jackett.Common/Indexers/YTS.cs
View File

@ -99,7 +99,7 @@ namespace Jackett.Common.Indexers
}
var searchUrl = ApiEndpoint + "?" + queryCollection.GetQueryString();
var response = await RequestStringWithCookiesAndRetry(searchUrl);
var response = await RequestWithCookiesAndRetryAsync(searchUrl);
try
{

2
src/Jackett.Common/Services/ImdbResolver.cs
View File

@ -37,7 +37,7 @@ namespace Jackett.Common.Services
{
Encoding = Encoding.UTF8
};
var result = await WebClient.GetString(request);
var result = await WebClient.GetResultAsync(request);
var movie = JsonConvert.DeserializeObject<Movie>(result.ContentString);
return movie;

6
src/Jackett.Common/Services/UpdateService.cs
View File

@ -114,7 +114,7 @@ namespace Jackett.Common.Services
try
{
var response = await client.GetString(new WebRequest()
var response = await client.GetResultAsync(new WebRequest()
{
Url = "https://api.github.com/repos/Jackett/Jackett/releases",
Encoding = Encoding.UTF8,
@ -258,11 +258,11 @@ namespace Jackett.Common.Services
var url = targetAsset.Browser_download_url;
var data = await client.GetBytes(SetDownloadHeaders(new WebRequest() { Url = url, EmulateBrowser = true, Type = RequestType.GET }));
var data = await client.GetResultAsync(SetDownloadHeaders(new WebRequest() { Url = url, EmulateBrowser = true, Type = RequestType.GET }));
while (data.IsRedirect)
{
data = await client.GetBytes(new WebRequest() { Url = data.RedirectingTo, EmulateBrowser = true, Type = RequestType.GET });
data = await client.GetResultAsync(new WebRequest() { Url = data.RedirectingTo, EmulateBrowser = true, Type = RequestType.GET });
}
var tempDir = Path.Combine(Path.GetTempPath(), "JackettUpdate-" + version + "-" + DateTime.Now.Ticks);

32
src/Jackett.Common/Utils/Clients/WebClient.cs
View File

@ -186,38 +186,24 @@ namespace Jackett.Common.Utils.Clients
return;
}
public virtual async Task<WebResult> GetBytes(WebRequest request)
public virtual async Task<WebResult> GetResultAsync(WebRequest request)
{
logger.Debug(string.Format("WebClient({0}).GetBytes(Url:{1})", ClientType, request.Url));
logger.Debug(string.Format("WebClient({0}).GetResultAsync(Url:{1})", ClientType, request.Url));
PrepareRequest(request);
await DelayRequest(request);
var result = await Run(request);
lastRequest = DateTime.Now;
result.Request = request;
logger.Debug(string.Format("WebClient({0}): Returning {1} => {2} bytes", ClientType, result.Status, (result.IsRedirect ? result.RedirectingTo + " " : "") + (result.ContentBytes == null ? "<NULL>" : result.ContentBytes.Length.ToString())));
logger.Debug(
string.Format(
"WebClient({0}): Returning {1} => {2} bytes", ClientType, result.Status,
(result.IsRedirect ? result.RedirectingTo + " " : "") +
(result.ContentBytes == null ? "<NULL>" : result.ContentBytes.Length.ToString())));
if (result.Headers.TryGetValue("server", out var server) && server[0] == "cloudflare-nginx")
result.ContentString = BrowserUtil.DecodeCloudFlareProtectedEmailFromHTML(result.ContentString);
return result;
}
public virtual async Task<WebResult> GetString(WebRequest request)
{
logger.Debug(string.Format("WebClient({0}).GetString(Url:{1})", ClientType, request.Url));
PrepareRequest(request);
await DelayRequest(request);
var result = await Run(request);
lastRequest = DateTime.Now;
result.Request = request;
WebResult stringResult = result;
logger.Debug(string.Format("WebClient({0}): Returning {1} => {2}", ClientType, result.Status, (result.IsRedirect ? result.RedirectingTo + " " : "") + (stringResult.ContentString ?? "<NULL>")));
if (stringResult.Headers.TryGetValue("server", out var server))
{
if (server[0] == "cloudflare-nginx")
stringResult.ContentString = BrowserUtil.DecodeCloudFlareProtectedEmailFromHTML(stringResult.ContentString);
}
return stringResult;
}
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
protected virtual async Task<WebResult> Run(WebRequest webRequest) => throw new NotImplementedException();
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously

11
src/Jackett.Test/TestUtil.cs
View File

@ -33,16 +33,11 @@ namespace Jackett.Test
public static IContainer Container => testContainer;
public static void RegisterByteCall(WebRequest r, Func<WebRequest, WebResult> f)
//Currently not used in any Unit Tests
public static void RegisterRequestCallback(WebRequest r, Func<WebRequest, WebResult> f)
{
var client = testContainer.Resolve<WebClient>() as TestWebClient;
client.RegisterByteCall(r, f);
}
public static void RegisterStringCall(WebRequest r, Func<WebRequest, WebResult> f)
{
var client = testContainer.Resolve<WebClient>() as TestWebClient;
client.RegisterStringCall(r, f);
client.RegisterRequestCallback(r, f);
}
public static string GetResource(string item)

18
src/Jackett.Test/TestWebClient.cs
View File

@ -9,26 +9,20 @@ using NLog;
namespace Jackett.Test
{
// Currently not used in any Unit tests. Leaving it for potential future testing purposes.
public class TestWebClient : WebClient
{
private readonly Dictionary<WebRequest, Func<WebRequest, WebResult>> byteCallbacks = new Dictionary<WebRequest, Func<WebRequest, WebResult>>();
private readonly Dictionary<WebRequest, Func<WebRequest, WebResult>> stringCallbacks = new Dictionary<WebRequest, Func<WebRequest, WebResult>>();
private readonly Dictionary<WebRequest, Func<WebRequest, WebResult>> _requestCallbacks = new Dictionary<WebRequest, Func<WebRequest, WebResult>>();
public TestWebClient(IProcessService p, Logger l, IConfigurationService c, ServerConfig sc)
: base(p: p,
l: l,
c: c,
sc: sc)
: base(p, l, c, sc)
{
}
public void RegisterByteCall(WebRequest req, Func<WebRequest, WebResult> f) => byteCallbacks.Add(req, f);
public void RegisterRequestCallback(WebRequest req, Func<WebRequest, WebResult> f) => _requestCallbacks.Add(req, f);
public void RegisterStringCall(WebRequest req, Func<WebRequest, WebResult> f) => stringCallbacks.Add(req, f);
public override Task<WebResult> GetBytes(WebRequest request) => Task.FromResult(byteCallbacks.Where(r => r.Key.Equals(request)).First().Value.Invoke(request));
public override Task<WebResult> GetString(WebRequest request) => Task.FromResult(stringCallbacks.Where(r => r.Key.Equals(request)).First().Value.Invoke(request));
public override Task<WebResult> GetResultAsync(WebRequest request) => Task.FromResult(_requestCallbacks.First(r => r.Key.Equals(request)).Value.Invoke(request));
public override void Init()
{