mirror
/
Jackett
mirror of https://github.com/Jackett/Jackett
1
0
Fork 0
Code Issues Releases Wiki Activity

core: refactor http webclient part 10 #8529 (#7729) 

Move WebResult String and Byte properties to base class
This commit is contained in:
Cory 2020-03-20 14:22:56 -05:00 committed by ngosang
parent ab74421a81
commit 125298425a
25 changed files with 67 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/Jackett.Common/Indexers/Abnormal.cs
View File

@ -548,7 +548,7 @@ namespace Jackett.Common.Indexers
/// <returns>Results from query</returns>
private async Task<string> queryTracker(string request)
{
WebClientStringResult results = null;
BaseWebResult results = null;
// Cache mode not enabled or cached file didn't exist for our query
output("\nQuerying tracker for results....");

2
src/Jackett.Common/Indexers/AniDUB.cs
View File

@ -509,7 +509,7 @@ namespace Jackett.Common.Indexers
return domDate.NodeValue.Trim();
}
private bool IsAuthorized(WebClientStringResult result) =>
private bool IsAuthorized(BaseWebResult result) =>
result.ContentString.Contains("index.php?action=logout");
private IEnumerable<int> ParseCategories(Uri showUri)

2
src/Jackett.Common/Indexers/BJShare.cs
View File

@ -219,7 +219,7 @@ namespace Jackett.Common.Indexers
return title;
}
private bool IsSessionIsClosed(WebClientStringResult result)
private bool IsSessionIsClosed(BaseWebResult result)
{
return result.IsRedirect && result.RedirectingTo.Contains("login.php");
}

16
src/Jackett.Common/Indexers/BaseIndexer.cs
View File

@ -412,7 +412,7 @@ namespace Jackett.Common.Indexers
return response.ContentBytes;
}
protected async Task<WebClientByteResult> RequestBytesWithCookiesAndRetry(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable<KeyValuePair<string, string>> data = null)
protected async Task<BaseWebResult> RequestBytesWithCookiesAndRetry(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable<KeyValuePair<string, string>> data = null)
{
Exception lastException = null;
for (var i = 0; i < 3; i++)
@ -432,7 +432,7 @@ namespace Jackett.Common.Indexers
throw lastException;
}
protected async Task<WebClientStringResult> RequestStringWithCookies(string url, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null)
protected async Task<BaseWebResult> RequestStringWithCookies(string url, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null)
{
var request = new Utils.Clients.WebRequest()
{
@ -452,7 +452,7 @@ namespace Jackett.Common.Indexers
return result;
}
protected async Task<WebClientStringResult> RequestStringWithCookiesAndRetry(string url, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null)
protected async Task<BaseWebResult> RequestStringWithCookiesAndRetry(string url, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null)
{
Exception lastException = null;
for (var i = 0; i < 3; i++)
@ -472,7 +472,7 @@ namespace Jackett.Common.Indexers
throw lastException;
}
protected virtual async Task<WebClientByteResult> RequestBytesWithCookies(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable<KeyValuePair<string, string>> data = null, Dictionary<string, string> headers = null)
protected virtual async Task<BaseWebResult> RequestBytesWithCookies(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable<KeyValuePair<string, string>> data = null, Dictionary<string, string> headers = null)
{
var request = new Utils.Clients.WebRequest()
{
@ -492,7 +492,7 @@ namespace Jackett.Common.Indexers
return result;
}
protected async Task<WebClientStringResult> PostDataWithCookies(string url, IEnumerable<KeyValuePair<string, string>> data, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null, string rawbody = null, bool? emulateBrowser = null)
protected async Task<BaseWebResult> PostDataWithCookies(string url, IEnumerable<KeyValuePair<string, string>> data, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null, string rawbody = null, bool? emulateBrowser = null)
{
var request = new Utils.Clients.WebRequest()
{
@ -514,7 +514,7 @@ namespace Jackett.Common.Indexers
return result;
}
protected async Task<WebClientStringResult> PostDataWithCookiesAndRetry(string url, IEnumerable<KeyValuePair<string, string>> data, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null, string rawbody = null, bool? emulateBrowser = null)
protected async Task<BaseWebResult> PostDataWithCookiesAndRetry(string url, IEnumerable<KeyValuePair<string, string>> data, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null, string rawbody = null, bool? emulateBrowser = null)
{
Exception lastException = null;
for (var i = 0; i < 3; i++)
@ -534,7 +534,7 @@ namespace Jackett.Common.Indexers
throw lastException;
}
protected async Task<WebClientStringResult> RequestLoginAndFollowRedirect(string url, IEnumerable<KeyValuePair<string, string>> data, string cookies, bool returnCookiesFromFirstCall, string redirectUrlOverride = null, string referer = null, bool accumulateCookies = false)
protected async Task<BaseWebResult> RequestLoginAndFollowRedirect(string url, IEnumerable<KeyValuePair<string, string>> data, string cookies, bool returnCookiesFromFirstCall, string redirectUrlOverride = null, string referer = null, bool accumulateCookies = false)
{
var request = new Utils.Clients.WebRequest()
{
@ -566,7 +566,7 @@ namespace Jackett.Common.Indexers
return response;
}
protected void CheckTrackerDown(WebClientStringResult response)
protected void CheckTrackerDown(BaseWebResult response)
{
if (response.Status == System.Net.HttpStatusCode.BadGateway
|| response.Status == System.Net.HttpStatusCode.GatewayTimeout

2
src/Jackett.Common/Indexers/BitHDTV.cs
View File

@ -122,7 +122,7 @@ namespace Jackett.Common.Indexers
{
{"cat", MapTorznabCapsToTrackers(query, true).FirstIfSingleOrDefault("0")}
};
var results = new List<WebClientStringResult>();
var results = new List<BaseWebResult>();
var search = new UriBuilder(SearchUrl);
if (query.IsImdbQuery)
{

10
src/Jackett.Common/Indexers/CardigannIndexer.cs
View File

@ -26,7 +26,7 @@ namespace Jackett.Common.Indexers
public class CardigannIndexer : BaseWebIndexer
{
protected IndexerDefinition Definition;
protected WebClientStringResult landingResult;
protected BaseWebResult landingResult;
protected IHtmlDocument landingResultDocument;
protected List<string> DefaultCategories = new List<string>();
@ -455,7 +455,7 @@ namespace Jackett.Common.Indexers
return template;
}
protected bool checkForError(WebClientStringResult loginResult, IList<errorBlock> errorBlocks)
protected bool checkForError(BaseWebResult loginResult, IList<errorBlock> errorBlocks)
{
if (loginResult.Status == HttpStatusCode.Unauthorized) // e.g. used by YGGtorrent
throw new ExceptionWithConfigData("401 Unauthorized, check your credentials", configData);
@ -702,7 +702,7 @@ namespace Jackett.Common.Indexers
landingResult = null;
landingResultDocument = null;
WebClientStringResult loginResult = null;
BaseWebResult loginResult = null;
var enctype = form.GetAttribute("enctype");
if (enctype == "multipart/form-data")
{
@ -1334,7 +1334,7 @@ namespace Jackett.Common.Indexers
logger.Info($"Fetching: {searchUrl}");
// send HTTP request
WebClientStringResult response = null;
BaseWebResult response = null;
Dictionary<string, string> headers = null;
if (Search.Headers != null)
{
@ -1721,7 +1721,7 @@ namespace Jackett.Common.Indexers
return releases;
}
protected async Task<WebClientByteResult> handleRequest(requestBlock request, Dictionary<string, object> variables = null, string referer = null)
protected async Task<BaseWebResult> handleRequest(requestBlock request, Dictionary<string, object> variables = null, string referer = null)
{
var requestLinkStr = resolvePath(applyGoTemplateText(request.Path, variables)).ToString();
logger.Debug($"CardigannIndexer ({Id}): handleRequest() requestLinkStr= {requestLinkStr}");

2
src/Jackett.Common/Indexers/Cinecalidad.cs
View File

@ -142,7 +142,7 @@ namespace Jackett.Common.Indexers
return null;
}
private List<ReleaseInfo> ParseReleases(WebClientStringResult response, TorznabQuery query)
private List<ReleaseInfo> ParseReleases(BaseWebResult response, TorznabQuery query)
{
var releases = new List<ReleaseInfo>();

2
src/Jackett.Common/Indexers/CorsaroRed.cs
View File

@ -107,7 +107,7 @@ namespace Jackett.Common.Indexers
return IndexerConfigurationStatus.Completed;
}
private dynamic CheckResponse(WebClientStringResult result)
private dynamic CheckResponse(BaseWebResult result)
{
try
{

2
src/Jackett.Common/Indexers/DanishBits.cs
View File

@ -65,7 +65,7 @@ namespace Jackett.Common.Indexers
return searchString;
}
protected override async Task<WebClientByteResult> RequestBytesWithCookies(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable<KeyValuePair<string, string>> data = null, Dictionary<string, string> headers = null)
protected override async Task<BaseWebResult> RequestBytesWithCookies(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable<KeyValuePair<string, string>> data = null, Dictionary<string, string> headers = null)
{
CookieHeader = null; // Download fill fail with cookies set
return await base.RequestBytesWithCookies(url, cookieOverride, method, referer, data, headers);

2
src/Jackett.Common/Indexers/InternetArchive.cs
View File

@ -150,7 +150,7 @@ namespace Jackett.Common.Indexers
return releases;
}
private JArray ParseResponse(WebClientStringResult result)
private JArray ParseResponse(BaseWebResult result)
{
try
{

2
src/Jackett.Common/Indexers/LostFilm.cs
View File

@ -212,7 +212,7 @@ namespace Jackett.Common.Indexers
}
}
private async Task<WebClientStringResult> RequestStringAndRelogin(string url)
private async Task<BaseWebResult> RequestStringAndRelogin(string url)
{
var results = await RequestStringWithCookies(url);
if (results.ContentString.Contains("503 Service"))

2
src/Jackett.Common/Indexers/NCore.cs
View File

@ -207,7 +207,7 @@ namespace Jackett.Common.Indexers
return releases;
}
private List<ReleaseInfo> ParseTorrents(WebClientStringResult results, string episodeString, TorznabQuery query,
private List<ReleaseInfo> ParseTorrents(BaseWebResult results, string episodeString, TorznabQuery query,
int alreadyFound, int limit, int previouslyParsedOnPage)
{
var releases = new List<ReleaseInfo>();

10
src/Jackett.Common/Indexers/NorBits.cs
View File

@ -459,9 +459,9 @@ namespace Jackett.Common.Indexers
/// </summary>
/// <param name="request">URL created by Query Builder</param>
/// <returns>Results from query</returns>
private async Task<WebClientStringResult> QueryExec(string request)
private async Task<BaseWebResult> QueryExec(string request)
{
WebClientStringResult results;
BaseWebResult results;
// Switch in we are in DEV mode with Hard Drive Cache or not
if (DevMode && CacheMode)
@ -482,9 +482,9 @@ namespace Jackett.Common.Indexers
/// </summary>
/// <param name="request">URL created by Query Builder</param>
/// <returns>Results from query</returns>
private async Task<WebClientStringResult> QueryCache(string request)
private async Task<BaseWebResult> QueryCache(string request)
{
WebClientStringResult results;
BaseWebResult results;
// Create Directory if not exist
System.IO.Directory.CreateDirectory(Directory);
@ -519,7 +519,7 @@ namespace Jackett.Common.Indexers
/// </summary>
/// <param name="request">URL created by Query Builder</param>
/// <returns>Results from query</returns>
private async Task<WebClientStringResult> QueryTracker(string request)
private async Task<BaseWebResult> QueryTracker(string request)
{
// Cache mode not enabled or cached file didn't exist for our query
Output("\nQuerying tracker for results....");

10
src/Jackett.Common/Indexers/NordicBits.cs
View File

@ -530,9 +530,9 @@ namespace Jackett.Common.Indexers
/// </summary>
/// <param name="request">URL created by Query Builder</param>
/// <returns>Results from query</returns>
private async Task<WebClientStringResult> QueryExec(string request)
private async Task<BaseWebResult> QueryExec(string request)
{
WebClientStringResult results;
BaseWebResult results;
// Switch in we are in DEV mode with Hard Drive Cache or not
if (DevMode && CacheMode)
@ -553,9 +553,9 @@ namespace Jackett.Common.Indexers
/// </summary>
/// <param name="request">URL created by Query Builder</param>
/// <returns>Results from query</returns>
private async Task<WebClientStringResult> QueryCache(string request)
private async Task<BaseWebResult> QueryCache(string request)
{
WebClientStringResult results;
BaseWebResult results;
// Create Directory if not exist
System.IO.Directory.CreateDirectory(Directory);
@ -590,7 +590,7 @@ namespace Jackett.Common.Indexers
/// </summary>
/// <param name="request">URL created by Query Builder</param>
/// <returns>Results from query</returns>
private async Task<WebClientStringResult> QueryTracker(string request)
private async Task<BaseWebResult> QueryTracker(string request)
{
// Cache mode not enabled or cached file didn't exist for our query
Output("\nQuerying tracker for results....");

2
src/Jackett.Common/Indexers/Partis.cs
View File

@ -116,7 +116,7 @@ namespace Jackett.Common.Indexers
var releases = new List<ReleaseInfo>(); //List of releases initialization
var searchString = query.GetQueryString(); //get search string from query
WebClientStringResult results = null;
BaseWebResult results = null;
var queryCollection = new NameValueCollection();
var catList = MapTorznabCapsToTrackers(query); // map categories from query to indexer specific
var categ = string.Join(",", catList);

2
src/Jackett.Common/Indexers/SceneTime.cs
View File

@ -104,7 +104,7 @@ namespace Jackett.Common.Indexers
public override async Task<ConfigurationData> GetConfigurationForSetup()
{
WebClientStringResult loginPage;
BaseWebResult loginPage;
try
{
loginPage = await RequestStringWithCookies(StartPageUrl, string.Empty);

4
src/Jackett.Common/Indexers/Shazbat.cs
View File

@ -81,7 +81,7 @@ namespace Jackett.Common.Indexers
{
var releases = new List<ReleaseInfo>();
var queryString = query.GetQueryString();
WebClientStringResult results = null;
BaseWebResult results = null;
var searchUrls = new List<string>();
if (!string.IsNullOrWhiteSpace(query.SanitizedSearchTerm))
{
@ -170,7 +170,7 @@ namespace Jackett.Common.Indexers
return releases;
}
private async Task<WebClientStringResult> ReloginIfNecessary(WebClientStringResult response)
private async Task<BaseWebResult> ReloginIfNecessary(BaseWebResult response)
{
if (response.ContentString.Contains("onclick=\"document.location='logout'\""))
return response;

2
src/Jackett.Common/Indexers/SolidTorrents.cs
View File

@ -76,7 +76,7 @@ namespace Jackett.Common.Indexers
return IndexerConfigurationStatus.Completed;
}
private JArray CheckResponse(WebClientStringResult result)
private JArray CheckResponse(BaseWebResult result)
{
try
{

2
src/Jackett.Common/Indexers/TVStore.cs
View File

@ -119,7 +119,7 @@ namespace Jackett.Common.Indexers
/// <param name="alreadyFound">Number of the already found torrents.(used for limit)</param>
/// <param name="limit">The limit to the number of torrents to download </param>
/// <param name="previouslyParsedOnPage">Current position in parsed results</param>
private async Task<List<ReleaseInfo>> ParseTorrentsAsync(WebClientStringResult results, int alreadyFound, int limit,
private async Task<List<ReleaseInfo>> ParseTorrentsAsync(BaseWebResult results, int alreadyFound, int limit,
int previouslyParsedOnPage)
{
var releases = new List<ReleaseInfo>();

2
src/Jackett.Common/Indexers/TorrenTech.cs
View File

@ -76,7 +76,7 @@ namespace Jackett.Common.Indexers
var releases = new List<ReleaseInfo>();
var searchString = query.GetQueryString();
WebClientStringResult results = null;
BaseWebResult results = null;
var queryCollection = new NameValueCollection
{
{ "act", "search" },

21
src/Jackett.Common/Utils/Clients/BaseWebResult.cs
View File

@ -8,6 +8,7 @@ namespace Jackett.Common.Utils.Clients
{
public abstract class BaseWebResult
{
private string _contentString;
private Encoding _encoding;
public Encoding Encoding
@ -20,7 +21,8 @@ namespace Jackett.Common.Utils.Clients
_encoding = Request.Encoding;
else if (Headers.ContainsKey("content-type"))
{
var charsetRegexMatch = Regex.Match(Headers["content-type"][0], @"charset=([\w-]+)", RegexOptions.Compiled);
var charsetRegexMatch = Regex.Match(
Headers["content-type"][0], @"charset=([\w-]+)", RegexOptions.Compiled);
if (charsetRegexMatch.Success)
{
var charset = charsetRegexMatch.Groups[1].Value;
@ -36,19 +38,34 @@ namespace Jackett.Common.Utils.Clients
}
_encoding ??= Encoding.UTF8;
return _encoding;
}
set => _encoding = value;
}
public byte[] ContentBytes { get; set; }
public HttpStatusCode Status { get; set; }
public string Cookies { get; set; }
public string RedirectingTo { get; set; }
public WebRequest Request { get; set; }
public Dictionary<string, string[]> Headers { get; protected set; } =
new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase);
public string ContentString
{
get
{
if (_contentString != null)
return _contentString;
if (ContentBytes == null)
return null;
_contentString = Encoding.GetString(ContentBytes);
return _contentString;
}
set => _contentString = value;
}
public bool IsRedirect => Status == HttpStatusCode.Redirect ||
Status == HttpStatusCode.RedirectKeepVerb ||
Status == HttpStatusCode.RedirectMethod ||

1
src/Jackett.Common/Utils/Clients/WebByteResult.cs
View File

@ -2,6 +2,5 @@ namespace Jackett.Common.Utils.Clients
{
public class WebClientByteResult : BaseWebResult
{
public byte[] ContentBytes { get; set; }
}
}

6
src/Jackett.Common/Utils/Clients/WebClient.cs
View File

@ -186,7 +186,7 @@ namespace Jackett.Common.Utils.Clients
return;
}
public virtual async Task<WebClientByteResult> GetBytes(WebRequest request)
public virtual async Task<BaseWebResult> GetBytes(WebRequest request)
{
logger.Debug(string.Format("WebClient({0}).GetBytes(Url:{1})", ClientType, request.Url));
PrepareRequest(request);
@ -198,7 +198,7 @@ namespace Jackett.Common.Utils.Clients
return result;
}
public virtual async Task<WebClientStringResult> GetString(WebRequest request)
public virtual async Task<BaseWebResult> GetString(WebRequest request)
{
logger.Debug(string.Format("WebClient({0}).GetString(Url:{1})", ClientType, request.Url));
PrepareRequest(request);
@ -206,7 +206,7 @@ namespace Jackett.Common.Utils.Clients
var result = await Run(request);
lastRequest = DateTime.Now;
result.Request = request;
WebClientStringResult stringResult = result;
BaseWebResult stringResult = result;
logger.Debug(string.Format("WebClient({0}): Returning {1} => {2}", ClientType, result.Status, (result.IsRedirect ? result.RedirectingTo + " " : "") + (stringResult.ContentString ?? "<NULL>")));

4
src/Jackett.Common/Utils/Clients/WebClientResult.cs
View File

@ -2,11 +2,9 @@ namespace Jackett.Common.Utils.Clients
{
public class WebClientStringResult : BaseWebResult
{
public string ContentString { get; set; }
public static implicit operator WebClientStringResult(WebClientByteResult br) => new WebClientStringResult()
{
ContentString = br.ContentBytes == null ? null : br.Encoding.GetString(br.ContentBytes),
ContentBytes = br.ContentBytes,
Cookies = br.Cookies,
Encoding = br.Encoding,
Headers = br.Headers,

8
src/Jackett.Test/TestWebClient.cs
View File

@ -11,8 +11,8 @@ namespace Jackett.Test
{
public class TestWebClient : WebClient
{
private readonly Dictionary<WebRequest, Func<WebRequest, WebClientByteResult>> byteCallbacks = new Dictionary<WebRequest, Func<WebRequest, WebClientByteResult>>();
private readonly Dictionary<WebRequest, Func<WebRequest, WebClientStringResult>> stringCallbacks = new Dictionary<WebRequest, Func<WebRequest, WebClientStringResult>>();
private readonly Dictionary<WebRequest, Func<WebRequest, BaseWebResult>> byteCallbacks = new Dictionary<WebRequest, Func<WebRequest, BaseWebResult>>();
private readonly Dictionary<WebRequest, Func<WebRequest, BaseWebResult>> stringCallbacks = new Dictionary<WebRequest, Func<WebRequest, BaseWebResult>>();
public TestWebClient(IProcessService p, Logger l, IConfigurationService c, ServerConfig sc)
: base(p: p,
@ -26,9 +26,9 @@ namespace Jackett.Test
public void RegisterStringCall(WebRequest req, Func<WebRequest, WebClientStringResult> f) => stringCallbacks.Add(req, f);
public override Task<WebClientByteResult> GetBytes(WebRequest request) => Task.FromResult<WebClientByteResult>(byteCallbacks.Where(r => r.Key.Equals(request)).First().Value.Invoke(request));
public override Task<BaseWebResult> GetBytes(WebRequest request) => Task.FromResult(byteCallbacks.Where(r => r.Key.Equals(request)).First().Value.Invoke(request));
public override Task<WebClientStringResult> GetString(WebRequest request) => Task.FromResult<WebClientStringResult>(stringCallbacks.Where(r => r.Key.Equals(request)).First().Value.Invoke(request));
public override Task<BaseWebResult> GetString(WebRequest request) => Task.FromResult(stringCallbacks.Where(r => r.Key.Equals(request)).First().Value.Invoke(request));
public override void Init()
{