2017-04-15 08:45:10 +00:00
using AutoMapper ;
2016-12-06 13:56:47 +00:00
using Jackett.Models ;
2017-04-15 08:45:10 +00:00
using Jackett.Services ;
using NLog ;
2015-07-22 22:00:52 +00:00
using System ;
using System.Collections.Generic ;
using System.Linq ;
using System.Text ;
2017-04-15 08:45:10 +00:00
using System.Text.RegularExpressions ;
2015-07-22 22:00:52 +00:00
using System.Threading.Tasks ;
2017-10-29 10:19:09 +00:00
using Jackett.Services.Interfaces ;
2017-11-05 09:42:03 +00:00
using Jackett.Models.Config ;
2015-07-22 22:00:52 +00:00
namespace Jackett.Utils.Clients
{
2017-11-17 15:46:58 +00:00
public abstract class WebClient : IObserver < ServerConfig >
2015-07-22 22:00:52 +00:00
{
2017-11-17 15:46:58 +00:00
protected IDisposable ServerConfigUnsubscriber ;
2017-04-15 08:45:10 +00:00
protected Logger logger ;
2016-12-06 13:56:47 +00:00
protected IConfigurationService configService ;
2017-11-05 09:42:03 +00:00
protected readonly ServerConfig serverConfig ;
2016-12-06 13:56:47 +00:00
protected IProcessService processService ;
2017-03-09 14:04:05 +00:00
protected DateTime lastRequest = DateTime . MinValue ;
protected TimeSpan requestDelayTimeSpan ;
2017-11-15 18:00:27 +00:00
protected string ClientType ;
2017-08-11 14:52:58 +00:00
public bool EmulateBrowser = true ;
2017-04-15 08:45:10 +00:00
public double requestDelay
{
get { return requestDelayTimeSpan . TotalSeconds ; }
set
{
requestDelayTimeSpan = TimeSpan . FromSeconds ( value ) ;
}
2017-03-09 14:04:05 +00:00
}
2016-12-06 13:56:47 +00:00
2017-11-17 15:46:58 +00:00
virtual protected void OnConfigChange ( )
{
}
2017-04-15 08:45:10 +00:00
virtual public void AddTrustedCertificate ( string host , string hash )
{
// not implemented by default
2017-02-28 19:05:57 +00:00
}
2017-11-05 09:42:03 +00:00
public WebClient ( IProcessService p , Logger l , IConfigurationService c , ServerConfig sc )
2017-04-15 08:45:10 +00:00
{
processService = p ;
logger = l ;
configService = c ;
2017-11-05 09:42:03 +00:00
serverConfig = sc ;
2017-11-15 18:00:27 +00:00
ClientType = GetType ( ) . Name ;
2017-11-17 15:46:58 +00:00
ServerConfigUnsubscriber = serverConfig . Subscribe ( this ) ;
2016-12-06 13:56:47 +00:00
}
2017-10-03 13:32:34 +00:00
async protected Task DelayRequest ( WebRequest request )
2017-04-15 08:45:10 +00:00
{
2017-08-11 14:52:58 +00:00
if ( request . EmulateBrowser = = null )
request . EmulateBrowser = EmulateBrowser ;
2017-04-15 08:45:10 +00:00
if ( requestDelay ! = 0 )
{
var timeElapsed = DateTime . Now - lastRequest ;
if ( timeElapsed < requestDelayTimeSpan )
{
var delay = requestDelayTimeSpan - timeElapsed ;
2017-11-15 18:00:27 +00:00
logger . Debug ( string . Format ( "WebClient({0}): delaying request for {1} by {2} seconds" , ClientType , request . Url , delay . TotalSeconds . ToString ( ) ) ) ;
2017-04-15 08:45:10 +00:00
await Task . Delay ( delay ) ;
}
}
2017-03-09 14:04:05 +00:00
}
2017-04-15 08:45:10 +00:00
virtual protected void PrepareRequest ( WebRequest request )
{
2017-11-07 15:10:12 +00:00
// add Accept/Accept-Language header if not set
// some webservers won't accept requests without accept
// e.g. elittracker requieres the Accept-Language header
2017-04-15 08:45:10 +00:00
if ( request . Headers = = null )
request . Headers = new Dictionary < string , string > ( StringComparer . InvariantCultureIgnoreCase ) ;
var hasAccept = false ;
2017-11-07 15:10:12 +00:00
var hasAcceptLanguage = false ;
2017-04-15 08:45:10 +00:00
foreach ( var header in request . Headers )
{
var key = header . Key . ToLower ( ) ;
if ( key = = "accept" )
{
hasAccept = true ;
}
2017-11-07 15:10:12 +00:00
else if ( key = = "accept-language" )
{
hasAcceptLanguage = true ;
}
2017-04-15 08:45:10 +00:00
}
if ( ! hasAccept )
request . Headers . Add ( "Accept" , "*/*" ) ;
2017-11-07 15:10:12 +00:00
if ( ! hasAcceptLanguage )
request . Headers . Add ( "Accept-Language" , "*" ) ;
2017-04-15 08:45:10 +00:00
return ;
2017-02-07 16:06:17 +00:00
}
2017-04-15 08:45:10 +00:00
virtual public async Task < WebClientByteResult > GetBytes ( WebRequest request )
{
2017-11-15 18:00:27 +00:00
logger . Debug ( string . Format ( "WebClient({0}).GetBytes(Url:{1})" , ClientType , request . Url ) ) ;
2017-04-15 08:45:10 +00:00
PrepareRequest ( request ) ;
2017-10-03 13:32:34 +00:00
await DelayRequest ( request ) ;
2017-04-15 08:45:10 +00:00
var result = await Run ( request ) ;
2017-12-13 08:37:09 +00:00
lastRequest = DateTime . Now ;
2017-04-15 08:45:10 +00:00
result . Request = request ;
2017-11-15 18:00:27 +00:00
logger . Debug ( string . Format ( "WebClient({0}): Returning {1} => {2} bytes" , ClientType , result . Status , ( result . IsRedirect ? result . RedirectingTo + " " : "" ) + ( result . Content = = null ? "<NULL>" : result . Content . Length . ToString ( ) ) ) ) ;
2017-04-15 08:45:10 +00:00
return result ;
}
virtual public async Task < WebClientStringResult > GetString ( WebRequest request )
{
2017-11-15 18:00:27 +00:00
logger . Debug ( string . Format ( "WebClient({0}).GetString(Url:{1})" , ClientType , request . Url ) ) ;
2017-04-15 08:45:10 +00:00
PrepareRequest ( request ) ;
2017-10-03 13:32:34 +00:00
await DelayRequest ( request ) ;
2017-04-15 08:45:10 +00:00
var result = await Run ( request ) ;
2017-12-13 08:37:09 +00:00
lastRequest = DateTime . Now ;
2017-04-15 08:45:10 +00:00
result . Request = request ;
WebClientStringResult stringResult = Mapper . Map < WebClientStringResult > ( result ) ;
Encoding encoding = null ;
if ( request . Encoding ! = null )
{
encoding = request . Encoding ;
}
else if ( result . Headers . ContainsKey ( "content-type" ) )
{
Regex CharsetRegex = new Regex ( @"charset=([\w-]+)" , RegexOptions . Compiled ) ;
var CharsetRegexMatch = CharsetRegex . Match ( result . Headers [ "content-type" ] [ 0 ] ) ;
if ( CharsetRegexMatch . Success )
{
var charset = CharsetRegexMatch . Groups [ 1 ] . Value ;
try
{
encoding = Encoding . GetEncoding ( charset ) ;
}
catch ( Exception ex )
{
2017-11-15 18:00:27 +00:00
logger . Error ( string . Format ( "WebClient({0}).GetString(Url:{1}): Error loading encoding {2} based on header {3}: {4}" , ClientType , request . Url , charset , result . Headers [ "content-type" ] [ 0 ] , ex ) ) ;
2017-04-15 08:45:10 +00:00
}
}
else
{
2017-11-15 18:00:27 +00:00
logger . Error ( string . Format ( "WebClient({0}).GetString(Url:{1}): Got header without charset: {2}" , ClientType , request . Url , result . Headers [ "content-type" ] [ 0 ] ) ) ;
2017-04-15 08:45:10 +00:00
}
}
if ( encoding = = null )
{
2017-11-15 18:00:27 +00:00
logger . Error ( string . Format ( "WebClient({0}).GetString(Url:{1}): No encoding detected, defaulting to UTF-8" , ClientType , request . Url ) ) ;
2017-04-15 08:45:10 +00:00
encoding = Encoding . UTF8 ;
}
string decodedContent = null ;
if ( result . Content ! = null )
decodedContent = encoding . GetString ( result . Content ) ;
stringResult . Content = decodedContent ;
2017-11-15 18:00:27 +00:00
logger . Debug ( string . Format ( "WebClient({0}): Returning {1} => {2}" , ClientType , result . Status , ( result . IsRedirect ? result . RedirectingTo + " " : "" ) + ( decodedContent = = null ? "<NULL>" : decodedContent ) ) ) ;
2017-04-15 08:45:10 +00:00
string [ ] server ;
if ( stringResult . Headers . TryGetValue ( "server" , out server ) )
{
if ( server [ 0 ] = = "cloudflare-nginx" )
stringResult . Content = BrowserUtil . DecodeCloudFlareProtectedEmailFromHTML ( stringResult . Content ) ;
}
return stringResult ;
}
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
virtual protected async Task < WebClientByteResult > Run ( WebRequest webRequest ) { throw new NotImplementedException ( ) ; }
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
2016-12-06 13:56:47 +00:00
abstract public void Init ( ) ;
2017-11-17 15:46:58 +00:00
public virtual void OnCompleted ( )
{
throw new NotImplementedException ( ) ;
}
public virtual void OnError ( Exception error )
{
throw new NotImplementedException ( ) ;
}
public virtual void OnNext ( ServerConfig value )
{
// nothing by default
}
2015-07-22 22:00:52 +00:00
}
}