2017-04-15 08:45:10 +00:00
using System ;
using System.Collections.Generic ;
2018-05-30 11:28:20 +00:00
using System.Diagnostics ;
2017-04-15 08:45:10 +00:00
using System.Linq ;
2018-05-30 11:28:20 +00:00
using System.Reflection ;
2017-04-15 08:45:10 +00:00
using System.Text ;
using System.Threading.Tasks ;
2017-10-29 06:21:18 +00:00
using AutoMapper ;
2018-03-10 08:05:56 +00:00
using Jackett.Common.Models ;
using Jackett.Common.Models.IndexerConfig ;
using Jackett.Common.Services.Interfaces ;
using Jackett.Common.Utils ;
using Jackett.Common.Utils.Clients ;
2017-07-11 20:32:56 +00:00
using Newtonsoft.Json ;
2017-10-29 06:21:18 +00:00
using Newtonsoft.Json.Linq ;
using NLog ;
2017-04-15 08:45:10 +00:00
2018-03-10 08:05:56 +00:00
namespace Jackett.Common.Indexers
2017-04-15 08:45:10 +00:00
{
2017-06-28 05:31:38 +00:00
public abstract class BaseIndexer : IIndexer
2017-04-15 08:45:10 +00:00
{
2017-07-10 20:58:44 +00:00
public static string GetIndexerID ( Type type )
{
return type . Name . ToLowerInvariant ( ) . StripNonAlphaNumeric ( ) ;
}
2017-04-15 08:45:10 +00:00
public string SiteLink { get ; protected set ; }
2017-09-11 13:10:54 +00:00
public virtual string [ ] LegacySiteLinks { get ; protected set ; }
2017-04-15 08:45:10 +00:00
public string DefaultSiteLink { get ; protected set ; }
2017-08-31 08:50:47 +00:00
public virtual string [ ] AlternativeSiteLinks { get ; protected set ; } = new string [ ] { } ;
2017-04-15 08:45:10 +00:00
public string DisplayDescription { get ; protected set ; }
public string DisplayName { get ; protected set ; }
public string Language { get ; protected set ; }
public string Type { get ; protected set ; }
2017-07-10 20:58:44 +00:00
public virtual string ID { get { return GetIndexerID ( GetType ( ) ) ; } }
2017-04-15 08:45:10 +00:00
2017-07-10 20:58:44 +00:00
public virtual bool IsConfigured { get ; protected set ; }
2017-04-15 08:45:10 +00:00
protected Logger logger ;
2017-07-10 20:58:44 +00:00
protected IIndexerConfigurationService configurationService ;
2017-04-15 08:45:10 +00:00
protected IProtectionService protectionService ;
2017-07-10 20:58:44 +00:00
protected ConfigurationData configData ;
2017-04-15 08:45:10 +00:00
protected string CookieHeader
{
get { return configData . CookieHeader . Value ; }
set { configData . CookieHeader . Value = value ; }
}
public string LastError
{
get { return configData . LastError . Value ; }
set
{
bool SaveNeeded = configData . LastError . Value ! = value & & IsConfigured ;
configData . LastError . Value = value ;
if ( SaveNeeded )
SaveConfig ( ) ;
}
}
2017-07-10 20:58:44 +00:00
public abstract TorznabCapabilities TorznabCaps { get ; protected set ; }
2017-04-15 08:45:10 +00:00
// standard constructor used by most indexers
2017-07-10 20:58:44 +00:00
public BaseIndexer ( string name , string link , string description , IIndexerConfigurationService configService , Logger logger , ConfigurationData configData , IProtectionService p )
2017-04-15 08:45:10 +00:00
{
2017-07-10 20:58:44 +00:00
this . logger = logger ;
configurationService = configService ;
protectionService = p ;
2017-06-28 05:31:38 +00:00
if ( ! link . EndsWith ( "/" , StringComparison . Ordinal ) )
2017-04-15 08:45:10 +00:00
throw new Exception ( "Site link must end with a slash." ) ;
DisplayName = name ;
DisplayDescription = description ;
SiteLink = link ;
DefaultSiteLink = link ;
this . configData = configData ;
2017-07-10 20:58:44 +00:00
if ( configData ! = null )
LoadValuesFromJson ( null ) ;
2017-04-15 08:45:10 +00:00
}
public virtual Task < ConfigurationData > GetConfigurationForSetup ( )
{
return Task . FromResult < ConfigurationData > ( configData ) ;
}
public virtual void ResetBaseConfig ( )
{
CookieHeader = string . Empty ;
IsConfigured = false ;
}
public virtual void SaveConfig ( )
{
2017-07-10 20:58:44 +00:00
configurationService . Save ( this as IIndexer , configData . ToJson ( protectionService , forDisplay : false ) ) ;
2017-04-15 08:45:10 +00:00
}
protected void LoadLegacyCookieConfig ( JToken jsonConfig )
{
string legacyCookieHeader = ( string ) jsonConfig [ "cookie_header" ] ;
if ( ! string . IsNullOrEmpty ( legacyCookieHeader ) )
{
CookieHeader = legacyCookieHeader ;
}
else
{
// Legacy cookie key
var jcookies = jsonConfig [ "cookies" ] ;
if ( jcookies is JArray )
{
var array = ( JArray ) jcookies ;
legacyCookieHeader = string . Empty ;
for ( int i = 0 ; i < array . Count ; i + + )
{
if ( i ! = 0 )
legacyCookieHeader + = "; " ;
legacyCookieHeader + = array [ i ] ;
}
CookieHeader = legacyCookieHeader ;
}
else if ( jcookies ! = null )
{
CookieHeader = ( string ) jcookies ;
}
}
}
virtual public void LoadValuesFromJson ( JToken jsonConfig , bool useProtectionService = false )
{
IProtectionService ps = null ;
if ( useProtectionService )
ps = protectionService ;
configData . LoadValuesFromJson ( jsonConfig , ps ) ;
if ( string . IsNullOrWhiteSpace ( configData . SiteLink . Value ) )
{
configData . SiteLink . Value = DefaultSiteLink ;
}
2017-08-30 16:46:36 +00:00
2017-06-28 05:31:38 +00:00
if ( ! configData . SiteLink . Value . EndsWith ( "/" , StringComparison . Ordinal ) )
2017-04-15 08:45:10 +00:00
configData . SiteLink . Value + = "/" ;
2017-08-30 16:46:36 +00:00
// reset site link to default if it's a legacy (defunc link)
if ( LegacySiteLinks ! = null & & LegacySiteLinks . Contains ( configData . SiteLink . Value ) )
{
logger . Debug ( string . Format ( "changing legacy site link from {0} to {1}" , configData . SiteLink . Value , DefaultSiteLink ) ) ;
configData . SiteLink . Value = DefaultSiteLink ;
}
2017-07-19 17:24:03 +00:00
// check whether the site link is well-formatted
var siteUri = new Uri ( configData . SiteLink . Value ) ;
2017-04-15 08:45:10 +00:00
SiteLink = configData . SiteLink . Value ;
}
2017-07-10 20:58:44 +00:00
public void LoadFromSavedConfiguration ( JToken jsonConfig )
2017-04-15 08:45:10 +00:00
{
if ( jsonConfig is JArray )
{
2018-05-30 11:28:20 +00:00
if ( ! MigratedFromDPAPI ( jsonConfig ) )
{
LoadValuesFromJson ( jsonConfig , true ) ;
IsConfigured = true ;
}
2017-04-15 08:45:10 +00:00
}
// read and upgrade old settings file format
else if ( jsonConfig is Object )
{
LoadLegacyCookieConfig ( jsonConfig ) ;
SaveConfig ( ) ;
IsConfigured = true ;
}
}
2018-05-30 11:28:20 +00:00
//TODO: Remove this section once users have moved off DPAPI
private bool MigratedFromDPAPI ( JToken jsonConfig )
{
var currentAssembly = Assembly . GetExecutingAssembly ( ) ;
bool runningLegacyOwin = new StackTrace ( ) . GetFrames ( )
. Select ( x = > x . GetMethod ( ) . ReflectedType . Assembly ) . Distinct ( )
. Where ( x = > x . GetReferencedAssemblies ( ) . Any ( y = > y . FullName = = currentAssembly . FullName ) )
. Where ( x = > x . ManifestModule . Name = = "Jackett.dll" | | x . ManifestModule . Name = = "JackettConsole.exe" )
. Count ( ) = = 2 ;
if ( runningLegacyOwin )
{
//Still running legacy Owin and using the DPAPI, we don't want to migrate
2018-06-02 07:42:01 +00:00
logger . Debug ( ID + " - Running Owin, no need to migrate from DPAPI" ) ;
2018-05-30 11:28:20 +00:00
return false ;
}
Version dotNetVersion = Microsoft . Extensions . PlatformAbstractions . PlatformServices . Default . Application . RuntimeFramework . Version ;
2018-06-05 11:47:13 +00:00
bool isWindows = Environment . OSVersion . Platform = = PlatformID . Win32NT ;
2018-05-30 11:28:20 +00:00
if ( ! isWindows & & dotNetVersion . Major < 4 )
{
// User isn't running Windows, but is running on .NET Core framewrok, no access to the DPAPI, so don't bother trying to migrate
return false ;
}
LoadValuesFromJson ( jsonConfig , false ) ;
object passwordPropertyValue = null ;
string passwordValue = "" ;
try
{
passwordPropertyValue = configData . GetType ( ) . GetProperty ( "Password" ) . GetValue ( configData , null ) ;
passwordValue = passwordPropertyValue . GetType ( ) . GetProperty ( "Value" ) . GetValue ( passwordPropertyValue , null ) . ToString ( ) ;
}
2018-06-05 11:47:13 +00:00
catch ( Exception )
2018-05-30 11:28:20 +00:00
{
2018-06-05 11:47:13 +00:00
logger . Debug ( $"Unable to source password for [{ID}] while attempting migration, likely a public tracker" ) ;
2018-05-30 11:28:20 +00:00
return false ;
}
if ( ! string . IsNullOrEmpty ( passwordValue ) )
{
try
{
protectionService . UnProtect ( passwordValue ) ;
//Password successfully unprotected using Microsoft.AspNetCore.DataProtection, no further action needed as we've already converted the password previously
return false ;
}
catch ( Exception ex )
{
2018-06-05 11:14:32 +00:00
logger . Info ( "Password could not be unprotected using Microsoft.AspNetCore.DataProtection, trying legacy: " + ex . ToString ( ) ) ;
2018-05-30 11:28:20 +00:00
try
{
string unprotectedPassword = protectionService . LegacyUnProtect ( passwordValue ) ;
//Password successfully unprotected using Windows/Mono DPAPI
passwordPropertyValue . GetType ( ) . GetProperty ( "Value" ) . SetValue ( passwordPropertyValue , unprotectedPassword ) ;
SaveConfig ( ) ;
IsConfigured = true ;
return true ;
}
catch ( Exception exception )
{
2018-06-05 11:14:32 +00:00
logger . Info ( "Password could not be unprotected using legacy DPAPI: " + exception . ToString ( ) ) ;
2018-05-30 11:28:20 +00:00
}
}
}
return false ;
}
2017-07-10 20:58:44 +00:00
protected async Task ConfigureIfOK ( string cookies , bool isLoggedin , Func < Task > onError )
{
if ( isLoggedin )
{
CookieHeader = cookies ;
IsConfigured = true ;
SaveConfig ( ) ;
}
else
{
await onError ( ) ;
}
}
protected virtual IEnumerable < ReleaseInfo > FilterResults ( TorznabQuery query , IEnumerable < ReleaseInfo > results )
{
if ( query . Categories . Length = = 0 )
return results ;
var filteredResults = results . Where ( result = >
{
return result . Category . IsEmptyOrNull ( ) | | query . Categories . Intersect ( result . Category ) . Any ( ) | | TorznabCatType . QueryContainsParentCategory ( query . Categories , result . Category ) ;
} ) ;
return filteredResults ;
}
2018-04-06 15:43:18 +00:00
public virtual bool CanHandleQuery ( TorznabQuery query )
2017-07-10 20:58:44 +00:00
{
if ( query = = null )
return false ;
2017-08-11 22:30:43 +00:00
if ( query . QueryType = = "caps" )
return true ;
2017-07-10 20:58:44 +00:00
var caps = TorznabCaps ;
if ( query . HasSpecifiedCategories )
if ( ! caps . SupportsCategories ( query . Categories ) )
return false ;
2018-04-17 10:43:00 +00:00
if ( caps . SupportsImdbSearch & & query . IsImdbQuery )
return true ;
2018-04-25 18:35:35 +00:00
else if ( ! caps . SupportsImdbSearch & & query . IsImdbQuery & & query . QueryType ! = "TorrentPotato" ) // potato query should always contain imdb+search term
2018-04-17 10:43:00 +00:00
return false ;
2017-08-11 16:13:22 +00:00
if ( caps . SearchAvailable & & query . IsSearch )
return true ;
if ( caps . TVSearchAvailable & & query . IsTVSearch )
return true ;
if ( caps . MovieSearchAvailable & & query . IsMovieSearch )
return true ;
2017-10-18 16:30:41 +00:00
if ( caps . MusicSearchAvailable & & query . IsMusicSearch )
return true ;
2017-08-11 16:13:22 +00:00
if ( caps . SupportsTVRageSearch & & query . IsTVRageSearch )
return true ;
if ( caps . SupportsImdbSearch & & query . IsImdbQuery )
return true ;
return false ;
2017-07-10 20:58:44 +00:00
}
public void Unconfigure ( )
{
IsConfigured = false ;
2017-11-13 11:55:54 +00:00
SiteLink = DefaultSiteLink ;
2017-11-13 15:55:02 +00:00
CookieHeader = "" ; // clear cookies
2017-07-10 20:58:44 +00:00
}
public abstract Task < IndexerConfigurationStatus > ApplyConfiguration ( JToken configJson ) ;
2017-08-24 10:28:41 +00:00
public virtual async Task < IndexerResult > ResultsForQuery ( TorznabQuery query )
2017-07-10 20:58:44 +00:00
{
2017-08-24 10:28:41 +00:00
try
2017-07-10 20:58:44 +00:00
{
2017-08-24 10:28:41 +00:00
if ( ! CanHandleQuery ( query ) )
return new IndexerResult ( this , new ReleaseInfo [ 0 ] ) ;
var results = await PerformQuery ( query ) ;
results = FilterResults ( query , results ) ;
2018-05-14 14:48:57 +00:00
if ( query . Limit > 0 )
{
results = results . Take ( query . Limit ) ;
}
2017-08-24 10:28:41 +00:00
results = results . Select ( r = >
{
r . Origin = this ;
2017-08-08 15:02:16 +00:00
2017-08-24 10:28:41 +00:00
// Some trackers do not keep their clocks up to date and can be ~20 minutes out!
if ( r . PublishDate > DateTime . Now )
r . PublishDate = DateTime . Now ;
return r ;
} ) ;
2017-07-10 20:58:44 +00:00
2017-08-24 10:28:41 +00:00
return new IndexerResult ( this , results ) ;
}
catch ( Exception ex )
{
throw new IndexerException ( this , ex ) ;
}
2017-07-10 20:58:44 +00:00
}
2017-10-29 06:21:18 +00:00
2017-07-10 20:58:44 +00:00
protected abstract Task < IEnumerable < ReleaseInfo > > PerformQuery ( TorznabQuery query ) ;
}
public abstract class BaseWebIndexer : BaseIndexer , IWebIndexer
{
2017-11-05 09:42:03 +00:00
protected BaseWebIndexer ( string name , string link , string description , IIndexerConfigurationService configService , WebClient client , Logger logger , ConfigurationData configData , IProtectionService p , TorznabCapabilities caps = null , string downloadBase = null )
2017-07-10 20:58:44 +00:00
: base ( name , link , description , configService , logger , configData , p )
{
this . webclient = client ;
this . downloadUrlBase = downloadBase ;
if ( caps = = null )
caps = TorznabUtil . CreateDefaultTorznabTVCaps ( ) ;
TorznabCaps = caps ;
}
// minimal constructor used by e.g. cardigann generic indexer
2017-11-05 09:42:03 +00:00
protected BaseWebIndexer ( IIndexerConfigurationService configService , WebClient client , Logger logger , IProtectionService p )
2017-07-10 20:58:44 +00:00
: base ( "" , "/" , "" , configService , logger , null , p )
{
this . webclient = client ;
}
public async virtual Task < byte [ ] > Download ( Uri link )
{
var uncleanLink = UncleanLink ( link ) ;
return await Download ( uncleanLink , RequestType . GET ) ;
2017-04-15 08:45:10 +00:00
}
2017-06-28 05:31:38 +00:00
protected async Task < byte [ ] > Download ( Uri link , RequestType method )
2017-04-15 08:45:10 +00:00
{
2017-11-09 12:28:15 +00:00
// return magnet link
if ( link . Scheme = = "magnet" )
return Encoding . UTF8 . GetBytes ( link . OriginalString ) ;
2017-04-15 08:45:10 +00:00
// do some extra escaping, needed for HD-Torrents
var requestLink = link . ToString ( )
. Replace ( "(" , "%28" )
. Replace ( ")" , "%29" )
. Replace ( "'" , "%27" ) ;
var response = await RequestBytesWithCookiesAndRetry ( requestLink , null , method , requestLink ) ;
2017-11-30 11:11:42 +00:00
if ( response . IsRedirect )
{
await FollowIfRedirect ( response ) ;
}
2017-04-15 08:45:10 +00:00
if ( response . Status ! = System . Net . HttpStatusCode . OK & & response . Status ! = System . Net . HttpStatusCode . Continue & & response . Status ! = System . Net . HttpStatusCode . PartialContent )
{
logger . Error ( "Failed download cookies: " + this . CookieHeader ) ;
if ( response . Content ! = null )
logger . Error ( "Failed download response:\n" + Encoding . UTF8 . GetString ( response . Content ) ) ;
2017-07-10 20:58:44 +00:00
throw new Exception ( $"Remote server returned {response.Status.ToString()}" + ( response . IsRedirect ? " => " + response . RedirectingTo : "" ) ) ;
2017-04-15 08:45:10 +00:00
}
return response . Content ;
}
protected async Task < WebClientByteResult > RequestBytesWithCookiesAndRetry ( string url , string cookieOverride = null , RequestType method = RequestType . GET , string referer = null , IEnumerable < KeyValuePair < string , string > > data = null )
{
Exception lastException = null ;
for ( int i = 0 ; i < 3 ; i + + )
{
try
{
return await RequestBytesWithCookies ( url , cookieOverride , method , referer , data ) ;
}
catch ( Exception e )
{
logger . Error ( e , string . Format ( "On attempt {0} downloading from {1}: {2}" , ( i + 1 ) , DisplayName , e . Message ) ) ;
lastException = e ;
}
await Task . Delay ( 500 ) ;
}
throw lastException ;
}
protected async Task < WebClientStringResult > RequestStringWithCookies ( string url , string cookieOverride = null , string referer = null , Dictionary < string , string > headers = null )
{
var request = new Utils . Clients . WebRequest ( )
{
Url = url ,
Type = RequestType . GET ,
Cookies = CookieHeader ,
Referer = referer ,
Headers = headers ,
Encoding = Encoding
} ;
if ( cookieOverride ! = null )
request . Cookies = cookieOverride ;
WebClientStringResult result = await webclient . GetString ( request ) ;
2017-09-13 07:57:39 +00:00
CheckTrackerDown ( result ) ;
2017-04-15 08:45:10 +00:00
UpdateCookieHeader ( result . Cookies , cookieOverride ) ;
return result ;
}
protected async Task < WebClientStringResult > RequestStringWithCookiesAndRetry ( string url , string cookieOverride = null , string referer = null , Dictionary < string , string > headers = null )
{
Exception lastException = null ;
for ( int i = 0 ; i < 3 ; i + + )
{
try
{
return await RequestStringWithCookies ( url , cookieOverride , referer , headers ) ;
}
catch ( Exception e )
{
logger . Error ( string . Format ( "On attempt {0} checking for results from {1}: {2}" , ( i + 1 ) , DisplayName , e . Message ) ) ;
lastException = e ;
}
await Task . Delay ( 500 ) ;
}
throw lastException ;
}
2017-10-03 12:23:31 +00:00
protected virtual async Task < WebClientByteResult > RequestBytesWithCookies ( string url , string cookieOverride = null , RequestType method = RequestType . GET , string referer = null , IEnumerable < KeyValuePair < string , string > > data = null , Dictionary < string , string > headers = null )
2017-04-15 08:45:10 +00:00
{
var request = new Utils . Clients . WebRequest ( )
{
Url = url ,
Type = method ,
Cookies = cookieOverride ? ? CookieHeader ,
PostData = data ,
Referer = referer ,
Headers = headers ,
Encoding = Encoding
} ;
if ( cookieOverride ! = null )
request . Cookies = cookieOverride ;
2017-10-25 15:20:39 +00:00
var result = await webclient . GetBytes ( request ) ;
UpdateCookieHeader ( result . Cookies , cookieOverride ) ;
return result ;
2017-04-15 08:45:10 +00:00
}
protected async Task < WebClientStringResult > PostDataWithCookies ( string url , IEnumerable < KeyValuePair < string , string > > data , string cookieOverride = null , string referer = null , Dictionary < string , string > headers = null , string rawbody = null , bool? emulateBrowser = null )
{
var request = new Utils . Clients . WebRequest ( )
{
Url = url ,
Type = RequestType . POST ,
Cookies = cookieOverride ? ? CookieHeader ,
PostData = data ,
Referer = referer ,
Headers = headers ,
RawBody = rawbody ,
Encoding = Encoding
} ;
if ( emulateBrowser . HasValue )
request . EmulateBrowser = emulateBrowser . Value ;
WebClientStringResult result = await webclient . GetString ( request ) ;
2017-09-13 07:57:39 +00:00
CheckTrackerDown ( result ) ;
2017-04-15 08:45:10 +00:00
UpdateCookieHeader ( result . Cookies , cookieOverride ) ;
return result ;
}
protected async Task < WebClientStringResult > PostDataWithCookiesAndRetry ( string url , IEnumerable < KeyValuePair < string , string > > data , string cookieOverride = null , string referer = null , Dictionary < string , string > headers = null , string rawbody = null , bool? emulateBrowser = null )
{
Exception lastException = null ;
for ( int i = 0 ; i < 3 ; i + + )
{
try
{
return await PostDataWithCookies ( url , data , cookieOverride , referer , headers , rawbody , emulateBrowser ) ;
}
catch ( Exception e )
{
logger . Error ( string . Format ( "On attempt {0} checking for results from {1}: {2}" , ( i + 1 ) , DisplayName , e . Message ) ) ;
lastException = e ;
}
await Task . Delay ( 500 ) ;
}
throw lastException ;
}
protected async Task < WebClientStringResult > RequestLoginAndFollowRedirect ( string url , IEnumerable < KeyValuePair < string , string > > data , string cookies , bool returnCookiesFromFirstCall , string redirectUrlOverride = null , string referer = null , bool accumulateCookies = false )
{
var request = new Utils . Clients . WebRequest ( )
{
Url = url ,
Type = RequestType . POST ,
Cookies = cookies ,
Referer = referer ,
PostData = data ,
Encoding = Encoding
} ;
var response = await webclient . GetString ( request ) ;
2017-09-13 07:57:39 +00:00
CheckTrackerDown ( response ) ;
2017-04-15 08:45:10 +00:00
if ( accumulateCookies )
{
response . Cookies = ResolveCookies ( ( request . Cookies = = null ? "" : request . Cookies + " " ) + response . Cookies ) ;
}
var firstCallCookies = response . Cookies ;
if ( response . IsRedirect )
{
await FollowIfRedirect ( response , request . Url , redirectUrlOverride , response . Cookies , accumulateCookies ) ;
}
if ( returnCookiesFromFirstCall )
{
response . Cookies = ResolveCookies ( firstCallCookies + ( accumulateCookies ? " " + response . Cookies : "" ) ) ;
}
2017-07-10 20:58:44 +00:00
2017-04-15 08:45:10 +00:00
return response ;
}
2017-09-13 07:57:39 +00:00
protected void CheckTrackerDown ( WebClientStringResult response )
{
if ( response . Status = = System . Net . HttpStatusCode . BadGateway
| | response . Status = = System . Net . HttpStatusCode . GatewayTimeout
| | ( int ) response . Status = = 521 // used by cloudflare to signal the original webserver is refusing the connection
| | ( int ) response . Status = = 522 // used by cloudflare to signal the original webserver is not reachable at all (timeout)
2018-04-01 13:40:43 +00:00
| | ( int ) response . Status = = 523 // used by cloudflare to signal the original webserver is not reachable at all (Origin is unreachable)
2017-10-29 06:21:18 +00:00
)
2017-09-13 07:57:39 +00:00
{
throw new Exception ( "Request to " + response . Request . Url + " failed (Error " + response . Status + ") - The tracker seems to be down." ) ;
}
}
2017-07-10 20:58:44 +00:00
protected async Task FollowIfRedirect ( WebClientStringResult response , string referrer = null , string overrideRedirectUrl = null , string overrideCookies = null , bool accumulateCookies = false )
2017-04-15 08:45:10 +00:00
{
2017-07-10 20:58:44 +00:00
var byteResult = new WebClientByteResult ( ) ;
// Map to byte
Mapper . Map ( response , byteResult ) ;
await FollowIfRedirect ( byteResult , referrer , overrideRedirectUrl , overrideCookies , accumulateCookies ) ;
// Map to string
Mapper . Map ( byteResult , response ) ;
}
protected async Task FollowIfRedirect ( WebClientByteResult response , string referrer = null , string overrideRedirectUrl = null , string overrideCookies = null , bool accumulateCookies = false )
{
// Follow up to 5 redirects
for ( int i = 0 ; i < 5 ; i + + )
2017-04-15 08:45:10 +00:00
{
2017-07-10 20:58:44 +00:00
if ( ! response . IsRedirect )
break ;
await DoFollowIfRedirect ( response , referrer , overrideRedirectUrl , overrideCookies , accumulateCookies ) ;
if ( accumulateCookies )
{
CookieHeader = ResolveCookies ( ( CookieHeader ! = null & & CookieHeader ! = "" ? CookieHeader + " " : "" ) + ( overrideCookies ! = null & & overrideCookies ! = "" ? overrideCookies + " " : "" ) + response . Cookies ) ;
overrideCookies = response . Cookies = CookieHeader ;
}
if ( overrideCookies ! = null & & response . Cookies = = null )
{
response . Cookies = overrideCookies ;
}
2017-04-15 08:45:10 +00:00
}
2017-07-10 20:58:44 +00:00
}
private String ResolveCookies ( String incomingCookies = "" )
{
var redirRequestCookies = ( CookieHeader ! = null & & CookieHeader ! = "" ? CookieHeader + " " : "" ) + incomingCookies ;
System . Text . RegularExpressions . Regex expression = new System . Text . RegularExpressions . Regex ( @"([^\\,;\s]+)=([^=\\,;\s]*)" ) ;
Dictionary < string , string > cookieDIctionary = new Dictionary < string , string > ( ) ;
var matches = expression . Match ( redirRequestCookies ) ;
while ( matches . Success )
2017-04-15 08:45:10 +00:00
{
2017-07-10 20:58:44 +00:00
if ( matches . Groups . Count > 2 ) cookieDIctionary [ matches . Groups [ 1 ] . Value ] = matches . Groups [ 2 ] . Value ;
matches = matches . NextMatch ( ) ;
2017-04-15 08:45:10 +00:00
}
2017-12-27 18:05:19 +00:00
return string . Join ( "; " , cookieDIctionary
. Where ( kv = > kv . Key ! = "cf_use_ob" & & kv . Key ! = "cf_ob_info" ) // These cookies are causing BadGateway errors, so we drop them, see issue #2306
. Select ( kv = > kv . Key . ToString ( ) + "=" + kv . Value . ToString ( ) ) . ToArray ( ) ) ;
2017-04-15 08:45:10 +00:00
}
2017-07-10 20:58:44 +00:00
// Update CookieHeader with new cookies and save the config if something changed (e.g. a new CloudFlare clearance cookie was issued)
2017-10-03 12:23:31 +00:00
protected virtual void UpdateCookieHeader ( string newCookies , string cookieOverride = null )
2017-04-15 08:45:10 +00:00
{
2017-07-10 20:58:44 +00:00
string newCookieHeader = ResolveCookies ( ( cookieOverride ! = null & & cookieOverride ! = "" ? cookieOverride + " " : "" ) + newCookies ) ;
if ( CookieHeader ! = newCookieHeader )
2017-04-15 08:45:10 +00:00
{
2017-07-10 20:58:44 +00:00
logger . Debug ( string . Format ( "updating Cookies {0} => {1}" , CookieHeader , newCookieHeader ) ) ;
CookieHeader = newCookieHeader ;
if ( IsConfigured )
SaveConfig ( ) ;
}
}
private async Task DoFollowIfRedirect ( WebClientByteResult incomingResponse , string referrer = null , string overrideRedirectUrl = null , string overrideCookies = null , bool accumulateCookies = false )
{
if ( incomingResponse . IsRedirect )
{
var redirRequestCookies = "" ;
if ( accumulateCookies )
2017-04-15 08:45:10 +00:00
{
2017-07-10 20:58:44 +00:00
redirRequestCookies = ResolveCookies ( ( CookieHeader ! = "" ? CookieHeader + " " : "" ) + ( overrideCookies ! = null ? overrideCookies : "" ) ) ;
2017-04-15 08:45:10 +00:00
}
2017-07-10 20:58:44 +00:00
else
{
redirRequestCookies = ( overrideCookies ! = null ? overrideCookies : "" ) ;
}
// Do redirect
var redirectedResponse = await webclient . GetBytes ( new WebRequest ( )
{
Url = overrideRedirectUrl ? ? incomingResponse . RedirectingTo ,
Referer = referrer ,
Cookies = redirRequestCookies ,
Encoding = Encoding
} ) ;
Mapper . Map ( redirectedResponse , incomingResponse ) ;
2017-04-15 08:45:10 +00:00
}
}
protected List < string > GetAllTrackerCategories ( )
{
return categoryMapping . Select ( x = > x . TrackerCategory ) . ToList ( ) ;
}
protected void AddCategoryMapping ( string trackerCategory , TorznabCategory newznabCategory , string trackerCategoryDesc = null )
{
categoryMapping . Add ( new CategoryMapping ( trackerCategory , trackerCategoryDesc , newznabCategory . ID ) ) ;
if ( ! TorznabCaps . Categories . Contains ( newznabCategory ) )
{
TorznabCaps . Categories . Add ( newznabCategory ) ;
if ( TorznabCatType . Movies . Contains ( newznabCategory ) )
TorznabCaps . MovieSearchAvailable = true ;
}
// add 1:1 categories
if ( trackerCategoryDesc ! = null & & trackerCategory ! = null )
{
try
{
var trackerCategoryInt = int . Parse ( trackerCategory ) ;
var CustomCat = new TorznabCategory ( trackerCategoryInt + 100000 , trackerCategoryDesc ) ;
if ( ! TorznabCaps . Categories . Contains ( CustomCat ) )
TorznabCaps . Categories . Add ( CustomCat ) ;
}
catch ( FormatException )
{
// trackerCategory is not an integer, continue
}
}
}
protected void AddCategoryMapping ( int trackerCategory , TorznabCategory newznabCategory , string trackerCategoryDesc = null )
{
AddCategoryMapping ( trackerCategory . ToString ( ) , newznabCategory , trackerCategoryDesc ) ;
}
protected void AddMultiCategoryMapping ( TorznabCategory newznabCategory , params int [ ] trackerCategories )
{
foreach ( var trackerCat in trackerCategories )
{
AddCategoryMapping ( trackerCat , newznabCategory ) ;
}
}
protected virtual List < string > MapTorznabCapsToTrackers ( TorznabQuery query , bool mapChildrenCatsToParent = false )
{
var result = new List < string > ( ) ;
foreach ( var cat in query . Categories )
{
// use 1:1 mapping to tracker categories for newznab categories >= 100000
if ( cat > = 100000 )
{
result . Add ( ( cat - 100000 ) . ToString ( ) ) ;
continue ;
}
var queryCats = new List < int > { cat } ;
var newznabCat = TorznabCatType . AllCats . FirstOrDefault ( c = > c . ID = = cat ) ;
if ( newznabCat ! = null )
{
queryCats . AddRange ( newznabCat . SubCategories . Select ( c = > c . ID ) ) ;
}
if ( mapChildrenCatsToParent )
{
var parentNewznabCat = TorznabCatType . AllCats . FirstOrDefault ( c = > c . SubCategories . Contains ( newznabCat ) ) ;
if ( parentNewznabCat ! = null )
{
queryCats . Add ( parentNewznabCat . ID ) ;
}
}
foreach ( var mapping in categoryMapping . Where ( c = > queryCats . Contains ( c . NewzNabCategory ) ) )
{
result . Add ( mapping . TrackerCategory ) ;
}
}
return result . Distinct ( ) . ToList ( ) ;
}
2017-05-14 16:55:36 +00:00
2017-07-10 20:58:44 +00:00
protected ICollection < int > MapTrackerCatToNewznab ( string input )
2017-05-14 16:55:36 +00:00
{
2017-07-10 20:58:44 +00:00
var cats = new List < int > ( ) ;
if ( null ! = input )
{
var mapping = categoryMapping . Where ( m = > m . TrackerCategory ! = null & & m . TrackerCategory . ToLowerInvariant ( ) = = input . ToLowerInvariant ( ) ) . FirstOrDefault ( ) ;
if ( mapping ! = null )
{
cats . Add ( mapping . NewzNabCategory ) ;
}
2017-05-14 16:55:36 +00:00
2017-07-10 20:58:44 +00:00
// 1:1 category mapping
try
{
var trackerCategoryInt = int . Parse ( input ) ;
cats . Add ( trackerCategoryInt + 100000 ) ;
}
catch ( FormatException )
{
// input is not an integer, continue
}
}
return cats ;
2017-05-14 16:55:36 +00:00
}
2017-06-28 05:31:38 +00:00
2017-07-10 20:58:44 +00:00
protected ICollection < int > MapTrackerCatDescToNewznab ( string input )
{
var cats = new List < int > ( ) ;
if ( null ! = input )
{
var mapping = categoryMapping . Where ( m = > m . TrackerCategoryDesc ! = null & & m . TrackerCategoryDesc . ToLowerInvariant ( ) = = input . ToLowerInvariant ( ) ) . FirstOrDefault ( ) ;
if ( mapping ! = null )
{
cats . Add ( mapping . NewzNabCategory ) ;
2017-07-03 05:15:47 +00:00
2017-07-10 20:58:44 +00:00
if ( mapping . TrackerCategory ! = null )
{
// 1:1 category mapping
try
{
var trackerCategoryInt = int . Parse ( mapping . TrackerCategory ) ;
cats . Add ( trackerCategoryInt + 100000 ) ;
}
catch ( FormatException )
{
// mapping.TrackerCategory is not an integer, continue
}
}
}
}
return cats ;
}
private IEnumerable < ReleaseInfo > CleanLinks ( IEnumerable < ReleaseInfo > releases )
2017-07-03 05:15:47 +00:00
{
2017-07-10 20:58:44 +00:00
if ( string . IsNullOrEmpty ( downloadUrlBase ) )
return releases ;
foreach ( var release in releases )
2017-07-03 05:15:47 +00:00
{
2017-07-10 20:58:44 +00:00
if ( release . Link . ToString ( ) . StartsWith ( downloadUrlBase , StringComparison . Ordinal ) )
{
release . Link = new Uri ( release . Link . ToString ( ) . Substring ( downloadUrlBase . Length ) , UriKind . Relative ) ;
}
2017-07-03 05:15:47 +00:00
}
2017-07-10 20:58:44 +00:00
return releases ;
}
2017-08-24 10:28:41 +00:00
public override async Task < IndexerResult > ResultsForQuery ( TorznabQuery query )
2017-07-10 20:58:44 +00:00
{
2017-08-24 10:28:41 +00:00
var result = await base . ResultsForQuery ( query ) ;
result . Releases = CleanLinks ( result . Releases ) ;
2017-07-10 20:58:44 +00:00
2017-08-24 10:28:41 +00:00
return result ;
2017-07-03 05:15:47 +00:00
}
2017-07-10 20:58:44 +00:00
protected virtual Uri UncleanLink ( Uri link )
{
if ( string . IsNullOrWhiteSpace ( downloadUrlBase ) )
{
return link ;
}
if ( link . ToString ( ) . StartsWith ( downloadUrlBase , StringComparison . Ordinal ) )
{
return link ;
}
return new Uri ( downloadUrlBase + link . ToString ( ) , UriKind . RelativeOrAbsolute ) ;
}
protected void OnParseError ( string results , Exception ex )
{
var fileName = string . Format ( "Error on {0} for {1}.txt" , DateTime . Now . ToString ( "yyyyMMddHHmmss" ) , DisplayName ) ;
var spacing = string . Join ( "" , Enumerable . Repeat ( Environment . NewLine , 5 ) ) ;
var fileContents = string . Format ( "{0}{1}{2}" , ex , spacing , results ) ;
logger . Error ( fileName + fileContents ) ;
2017-09-13 09:49:24 +00:00
throw new Exception ( "Parse error" , ex ) ;
2017-07-10 20:58:44 +00:00
}
public override TorznabCapabilities TorznabCaps { get ; protected set ; }
2017-07-11 20:32:56 +00:00
[JsonConverter(typeof(EncodingJsonConverter))]
2017-07-10 20:58:44 +00:00
public Encoding Encoding { get ; protected set ; }
private List < CategoryMapping > categoryMapping = new List < CategoryMapping > ( ) ;
2017-11-05 09:42:03 +00:00
protected WebClient webclient ;
2017-07-10 20:58:44 +00:00
protected readonly string downloadUrlBase = "" ;
}
public abstract class BaseCachingWebIndexer : BaseWebIndexer
{
2017-11-05 09:42:03 +00:00
protected BaseCachingWebIndexer ( string name , string link , string description , IIndexerConfigurationService configService , WebClient client , Logger logger , ConfigurationData configData , IProtectionService p , TorznabCapabilities caps = null , string downloadBase = null )
2017-07-10 20:58:44 +00:00
: base ( name , link , description , configService , client , logger , configData , p , caps , downloadBase )
{
}
protected void CleanCache ( )
{
foreach ( var expired in cache . Where ( i = > DateTime . Now - i . Created > cacheTime ) . ToList ( ) )
{
cache . Remove ( expired ) ;
}
}
protected static List < CachedQueryResult > cache = new List < CachedQueryResult > ( ) ;
protected static readonly TimeSpan cacheTime = new TimeSpan ( 0 , 9 , 0 ) ;
2017-04-15 08:45:10 +00:00
}
}