2016-12-06 13:56:47 +00:00
using AutoMapper ;
using Jackett.Models ;
using Jackett.Services ;
using NLog ;
2015-07-22 22:00:52 +00:00
using System ;
using System.Collections.Generic ;
using System.Linq ;
using System.Text ;
2016-12-06 13:56:47 +00:00
using System.Text.RegularExpressions ;
2015-07-22 22:00:52 +00:00
using System.Threading.Tasks ;
namespace Jackett.Utils.Clients
{
2016-12-06 13:56:47 +00:00
public abstract class IWebClient
2015-07-22 22:00:52 +00:00
{
2016-12-06 13:56:47 +00:00
protected Logger logger ;
protected IConfigurationService configService ;
protected IProcessService processService ;
2017-03-09 14:04:05 +00:00
protected DateTime lastRequest = DateTime . MinValue ;
protected TimeSpan requestDelayTimeSpan ;
public double requestDelay
{
get { return requestDelayTimeSpan . TotalSeconds ; }
set
{
requestDelayTimeSpan = TimeSpan . FromSeconds ( value ) ;
}
}
2016-12-06 13:56:47 +00:00
2017-02-28 19:05:57 +00:00
virtual public void AddTrustedCertificate ( string host , string hash )
{
// not implemented by default
}
2016-12-06 13:56:47 +00:00
public IWebClient ( IProcessService p , Logger l , IConfigurationService c )
{
processService = p ;
logger = l ;
configService = c ;
}
2017-03-09 14:04:05 +00:00
async protected void DelayRequest ( WebRequest request )
{
if ( requestDelay ! = 0 )
{
var timeElapsed = DateTime . Now - lastRequest ;
if ( timeElapsed < requestDelayTimeSpan )
{
var delay = requestDelayTimeSpan - timeElapsed ;
logger . Debug ( string . Format ( "IWebClient: delaying request for {0} by {1} seconds" , request . Url , delay . TotalSeconds . ToString ( ) ) ) ;
await Task . Delay ( delay ) ;
}
lastRequest = DateTime . Now ;
}
}
2017-02-07 16:06:17 +00:00
virtual protected void PrepareRequest ( WebRequest request )
{
// add accept header if not set
if ( request . Headers = = null )
request . Headers = new Dictionary < string , string > ( StringComparer . InvariantCultureIgnoreCase ) ;
var hasAccept = false ;
foreach ( var header in request . Headers )
{
var key = header . Key . ToLower ( ) ;
if ( key = = "accept" )
{
hasAccept = true ;
}
}
if ( ! hasAccept )
request . Headers . Add ( "Accept" , "*/*" ) ;
return ;
}
2016-12-06 13:56:47 +00:00
virtual public async Task < WebClientByteResult > GetBytes ( WebRequest request )
{
logger . Debug ( string . Format ( "IWebClient.GetBytes(Url:{0})" , request . Url ) ) ;
2017-02-07 16:06:17 +00:00
PrepareRequest ( request ) ;
2017-03-09 14:04:05 +00:00
DelayRequest ( request ) ;
2016-12-06 13:56:47 +00:00
var result = await Run ( request ) ;
2017-03-07 10:28:18 +00:00
result . Request = request ;
2016-12-11 04:57:39 +00:00
logger . Debug ( string . Format ( "IWebClient: Returning {0} => {1} bytes" , result . Status , ( result . IsRedirect ? result . RedirectingTo + " " : "" ) + ( result . Content = = null ? "<NULL>" : result . Content . Length . ToString ( ) ) ) ) ;
2016-12-06 13:56:47 +00:00
return result ;
}
virtual public async Task < WebClientStringResult > GetString ( WebRequest request )
{
logger . Debug ( string . Format ( "IWebClient.GetString(Url:{0})" , request . Url ) ) ;
2017-02-07 16:06:17 +00:00
PrepareRequest ( request ) ;
2017-03-09 14:04:05 +00:00
DelayRequest ( request ) ;
2016-12-06 13:56:47 +00:00
var result = await Run ( request ) ;
2017-03-07 10:28:18 +00:00
result . Request = request ;
2016-12-06 13:56:47 +00:00
WebClientStringResult stringResult = Mapper . Map < WebClientStringResult > ( result ) ;
Encoding encoding = null ;
if ( request . Encoding ! = null )
{
encoding = request . Encoding ;
}
else if ( result . Headers . ContainsKey ( "content-type" ) )
{
Regex CharsetRegex = new Regex ( @"charset=([\w-]+)" , RegexOptions . Compiled ) ;
var CharsetRegexMatch = CharsetRegex . Match ( result . Headers [ "content-type" ] [ 0 ] ) ;
if ( CharsetRegexMatch . Success )
{
var charset = CharsetRegexMatch . Groups [ 1 ] . Value ;
try
{
encoding = Encoding . GetEncoding ( charset ) ;
}
catch ( Exception ex )
{
logger . Error ( string . Format ( "IWebClient.GetString(Url:{0}): Error loading encoding {0} based on header {1}: {2}" , request . Url , charset , result . Headers [ "content-type" ] [ 0 ] , ex ) ) ;
}
}
else
{
logger . Error ( string . Format ( "IWebClient.GetString(Url:{0}): Got header without charset: {0}" , request . Url , result . Headers [ "content-type" ] [ 0 ] ) ) ;
}
}
if ( encoding = = null )
{
logger . Error ( string . Format ( "IWebClient.GetString(Url:{0}): No encoding detected, defaulting to UTF-8" , request . Url ) ) ;
encoding = Encoding . UTF8 ;
}
string decodedContent = null ;
if ( result . Content ! = null )
decodedContent = encoding . GetString ( result . Content ) ;
stringResult . Content = decodedContent ;
2016-12-11 04:57:39 +00:00
logger . Debug ( string . Format ( "IWebClient: Returning {0} => {1}" , result . Status , ( result . IsRedirect ? result . RedirectingTo + " " : "" ) + ( decodedContent = = null ? "<NULL>" : decodedContent ) ) ) ;
2016-12-06 13:56:47 +00:00
string [ ] server ;
if ( stringResult . Headers . TryGetValue ( "server" , out server ) )
{
if ( server [ 0 ] = = "cloudflare-nginx" )
stringResult . Content = BrowserUtil . DecodeCloudFlareProtectedEmailFromHTML ( stringResult . Content ) ;
}
return stringResult ;
}
2016-12-10 10:17:56 +00:00
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
virtual protected async Task < WebClientByteResult > Run ( WebRequest webRequest ) { throw new NotImplementedException ( ) ; }
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
2016-12-06 13:56:47 +00:00
abstract public void Init ( ) ;
2015-07-22 22:00:52 +00:00
}
}