2020-02-09 18:08:34 +00:00
using System ;
using System.Collections.Generic ;
using System.Linq ;
using System.Text ;
using System.Threading.Tasks ;
2020-02-09 02:35:16 +00:00
using AutoMapper ;
2018-03-10 08:05:56 +00:00
using Jackett.Common.Models ;
using Jackett.Common.Models.IndexerConfig ;
using Jackett.Common.Services.Interfaces ;
using Jackett.Common.Utils ;
using Jackett.Common.Utils.Clients ;
2017-07-11 20:32:56 +00:00
using Newtonsoft.Json ;
2017-10-29 06:21:18 +00:00
using Newtonsoft.Json.Linq ;
using NLog ;
2021-05-16 18:13:54 +00:00
using Polly ;
2018-06-26 15:47:19 +00:00
using static Jackett . Common . Models . IndexerConfig . ConfigurationData ;
2017-04-15 08:45:10 +00:00
2018-03-10 08:05:56 +00:00
namespace Jackett.Common.Indexers
2017-04-15 08:45:10 +00:00
{
2017-06-28 05:31:38 +00:00
public abstract class BaseIndexer : IIndexer
2017-04-15 08:45:10 +00:00
{
2020-05-11 19:59:28 +00:00
public string Id { get ; protected set ; }
2017-04-15 08:45:10 +00:00
public string SiteLink { get ; protected set ; }
2017-09-11 13:10:54 +00:00
public virtual string [ ] LegacySiteLinks { get ; protected set ; }
2017-04-15 08:45:10 +00:00
public string DefaultSiteLink { get ; protected set ; }
2020-10-19 21:19:10 +00:00
public virtual string [ ] AlternativeSiteLinks { get ; protected set ; } = { } ;
2017-04-15 08:45:10 +00:00
public string DisplayDescription { get ; protected set ; }
public string DisplayName { get ; protected set ; }
public string Language { get ; protected set ; }
public string Type { get ; protected set ; }
2020-05-11 19:59:28 +00:00
2017-04-15 08:45:10 +00:00
2018-06-15 09:12:03 +00:00
[JsonConverter(typeof(EncodingJsonConverter))]
public Encoding Encoding { get ; protected set ; }
2017-07-10 20:58:44 +00:00
public virtual bool IsConfigured { get ; protected set ; }
2021-05-08 20:24:18 +00:00
public virtual string [ ] Tags { get ; protected set ; }
2017-04-15 08:45:10 +00:00
protected Logger logger ;
2017-07-10 20:58:44 +00:00
protected IIndexerConfigurationService configurationService ;
2017-04-15 08:45:10 +00:00
protected IProtectionService protectionService ;
2020-12-11 22:14:21 +00:00
protected ICacheService cacheService ;
2017-07-10 20:58:44 +00:00
protected ConfigurationData configData ;
2017-04-15 08:45:10 +00:00
protected string CookieHeader
{
2020-02-25 16:08:03 +00:00
get = > configData . CookieHeader . Value ;
set = > configData . CookieHeader . Value = value ;
2017-04-15 08:45:10 +00:00
}
public string LastError
{
2020-02-25 16:08:03 +00:00
get = > configData . LastError . Value ;
2017-04-15 08:45:10 +00:00
set
{
2020-02-10 22:16:19 +00:00
var SaveNeeded = configData . LastError . Value ! = value & & IsConfigured ;
2017-04-15 08:45:10 +00:00
configData . LastError . Value = value ;
if ( SaveNeeded )
SaveConfig ( ) ;
}
}
2017-07-10 20:58:44 +00:00
public abstract TorznabCapabilities TorznabCaps { get ; protected set ; }
2017-04-15 08:45:10 +00:00
// standard constructor used by most indexers
2020-05-11 19:59:28 +00:00
public BaseIndexer ( string link , string id , string name , string description ,
IIndexerConfigurationService configService , Logger logger , ConfigurationData configData ,
2020-12-11 22:14:21 +00:00
IProtectionService p , ICacheService cs )
2017-04-15 08:45:10 +00:00
{
2017-07-10 20:58:44 +00:00
this . logger = logger ;
configurationService = configService ;
protectionService = p ;
2020-12-11 22:14:21 +00:00
cacheService = cs ;
2017-07-10 20:58:44 +00:00
2017-06-28 05:31:38 +00:00
if ( ! link . EndsWith ( "/" , StringComparison . Ordinal ) )
2017-04-15 08:45:10 +00:00
throw new Exception ( "Site link must end with a slash." ) ;
2020-05-11 19:59:28 +00:00
Id = id ;
2017-04-15 08:45:10 +00:00
DisplayName = name ;
DisplayDescription = description ;
SiteLink = link ;
DefaultSiteLink = link ;
this . configData = configData ;
2017-07-10 20:58:44 +00:00
if ( configData ! = null )
LoadValuesFromJson ( null ) ;
2017-04-15 08:45:10 +00:00
}
2020-02-25 16:08:03 +00:00
public virtual Task < ConfigurationData > GetConfigurationForSetup ( ) = > Task . FromResult < ConfigurationData > ( configData ) ;
2017-04-15 08:45:10 +00:00
public virtual void ResetBaseConfig ( )
{
CookieHeader = string . Empty ;
IsConfigured = false ;
}
2020-02-25 16:08:03 +00:00
public virtual void SaveConfig ( ) = > configurationService . Save ( this as IIndexer , configData . ToJson ( protectionService , forDisplay : false ) ) ;
2017-04-15 08:45:10 +00:00
protected void LoadLegacyCookieConfig ( JToken jsonConfig )
{
2020-02-10 22:16:19 +00:00
var legacyCookieHeader = ( string ) jsonConfig [ "cookie_header" ] ;
2017-04-15 08:45:10 +00:00
if ( ! string . IsNullOrEmpty ( legacyCookieHeader ) )
{
CookieHeader = legacyCookieHeader ;
}
else
{
// Legacy cookie key
var jcookies = jsonConfig [ "cookies" ] ;
if ( jcookies is JArray )
{
var array = ( JArray ) jcookies ;
legacyCookieHeader = string . Empty ;
2020-02-10 22:16:19 +00:00
for ( var i = 0 ; i < array . Count ; i + + )
2017-04-15 08:45:10 +00:00
{
if ( i ! = 0 )
legacyCookieHeader + = "; " ;
legacyCookieHeader + = array [ i ] ;
}
CookieHeader = legacyCookieHeader ;
}
else if ( jcookies ! = null )
{
CookieHeader = ( string ) jcookies ;
}
}
}
2020-02-10 22:16:19 +00:00
public virtual void LoadValuesFromJson ( JToken jsonConfig , bool useProtectionService = false )
2017-04-15 08:45:10 +00:00
{
IProtectionService ps = null ;
if ( useProtectionService )
ps = protectionService ;
2020-12-11 23:12:40 +00:00
configData . LoadConfigDataValuesFromJson ( jsonConfig , ps ) ;
2017-04-15 08:45:10 +00:00
if ( string . IsNullOrWhiteSpace ( configData . SiteLink . Value ) )
{
configData . SiteLink . Value = DefaultSiteLink ;
}
2017-08-30 16:46:36 +00:00
2017-06-28 05:31:38 +00:00
if ( ! configData . SiteLink . Value . EndsWith ( "/" , StringComparison . Ordinal ) )
2017-04-15 08:45:10 +00:00
configData . SiteLink . Value + = "/" ;
2017-08-30 16:46:36 +00:00
// reset site link to default if it's a legacy (defunc link)
if ( LegacySiteLinks ! = null & & LegacySiteLinks . Contains ( configData . SiteLink . Value ) )
{
logger . Debug ( string . Format ( "changing legacy site link from {0} to {1}" , configData . SiteLink . Value , DefaultSiteLink ) ) ;
configData . SiteLink . Value = DefaultSiteLink ;
}
2017-07-19 17:24:03 +00:00
// check whether the site link is well-formatted
var siteUri = new Uri ( configData . SiteLink . Value ) ;
2017-04-15 08:45:10 +00:00
SiteLink = configData . SiteLink . Value ;
2021-05-08 20:24:18 +00:00
Tags = configData . Tags . Values . Select ( t = > t . ToLowerInvariant ( ) ) . ToArray ( ) ;
2017-04-15 08:45:10 +00:00
}
2017-07-10 20:58:44 +00:00
public void LoadFromSavedConfiguration ( JToken jsonConfig )
2017-04-15 08:45:10 +00:00
{
if ( jsonConfig is JArray )
{
2018-05-30 11:28:20 +00:00
if ( ! MigratedFromDPAPI ( jsonConfig ) )
{
LoadValuesFromJson ( jsonConfig , true ) ;
IsConfigured = true ;
}
2017-04-15 08:45:10 +00:00
}
// read and upgrade old settings file format
2020-02-10 22:16:19 +00:00
else if ( jsonConfig is object )
2017-04-15 08:45:10 +00:00
{
LoadLegacyCookieConfig ( jsonConfig ) ;
SaveConfig ( ) ;
IsConfigured = true ;
}
}
2018-05-30 11:28:20 +00:00
//TODO: Remove this section once users have moved off DPAPI
private bool MigratedFromDPAPI ( JToken jsonConfig )
{
2020-02-10 22:16:19 +00:00
var isWindows = Environment . OSVersion . Platform = = PlatformID . Win32NT ;
2018-05-30 11:28:20 +00:00
2018-09-24 09:24:17 +00:00
if ( ! isWindows & & DotNetCoreUtil . IsRunningOnDotNetCore )
2018-05-30 11:28:20 +00:00
{
2018-06-10 02:51:34 +00:00
// User isn't running Windows, but is running on .NET Core framework, no access to the DPAPI, so don't bother trying to migrate
2018-05-30 11:28:20 +00:00
return false ;
}
LoadValuesFromJson ( jsonConfig , false ) ;
2021-03-16 23:29:26 +00:00
StringConfigurationItem passwordPropertyValue = null ;
2020-02-10 22:16:19 +00:00
var passwordValue = "" ;
2018-05-30 11:28:20 +00:00
try
{
2018-06-26 15:47:19 +00:00
// try dynamic items first (e.g. all cardigann indexers)
2021-03-16 23:29:26 +00:00
passwordPropertyValue = ( StringConfigurationItem ) configData . GetDynamicByName ( "password" ) ;
2018-06-26 15:47:19 +00:00
if ( passwordPropertyValue = = null ) // if there's no dynamic password try the static property
{
2021-03-16 23:29:26 +00:00
passwordPropertyValue = ( StringConfigurationItem ) configData . GetType ( ) . GetProperty ( "Password" ) . GetValue ( configData , null ) ;
2018-06-26 15:47:19 +00:00
// protection is based on the item.Name value (property name might be different, example: Abnormal), so check the Name again
if ( ! string . Equals ( passwordPropertyValue . Name , "password" , StringComparison . InvariantCultureIgnoreCase ) )
{
2020-05-11 19:59:28 +00:00
logger . Debug ( $"Skipping non default password property (unencrpyted password) for [{Id}] while attempting migration" ) ;
2018-06-26 15:47:19 +00:00
return false ;
}
}
passwordValue = passwordPropertyValue . Value ;
2018-05-30 11:28:20 +00:00
}
2018-06-05 11:47:13 +00:00
catch ( Exception )
2018-05-30 11:28:20 +00:00
{
2020-05-11 19:59:28 +00:00
logger . Debug ( $"Unable to source password for [{Id}] while attempting migration, likely a tracker without a password setting" ) ;
2018-05-30 11:28:20 +00:00
return false ;
}
if ( ! string . IsNullOrEmpty ( passwordValue ) )
{
try
{
protectionService . UnProtect ( passwordValue ) ;
//Password successfully unprotected using Microsoft.AspNetCore.DataProtection, no further action needed as we've already converted the password previously
return false ;
}
catch ( Exception ex )
{
2018-06-10 02:51:34 +00:00
if ( ex . Message ! = "The provided payload cannot be decrypted because it was not protected with this protection provider." )
2020-05-11 19:59:28 +00:00
logger . Info ( $"Password could not be unprotected using Microsoft.AspNetCore.DataProtection - {Id} : " + ex ) ;
2018-05-30 11:28:20 +00:00
}
}
return false ;
}
2017-07-10 20:58:44 +00:00
protected async Task ConfigureIfOK ( string cookies , bool isLoggedin , Func < Task > onError )
{
if ( isLoggedin )
{
CookieHeader = cookies ;
IsConfigured = true ;
SaveConfig ( ) ;
}
else
{
await onError ( ) ;
}
}
protected virtual IEnumerable < ReleaseInfo > FilterResults ( TorznabQuery query , IEnumerable < ReleaseInfo > results )
{
2020-11-02 13:20:13 +00:00
var filteredResults = results ;
2017-07-10 20:58:44 +00:00
2020-11-02 13:20:13 +00:00
// filter results with wrong categories
if ( query . Categories . Length > 0 )
{
// expand parent categories from the query
var expandedQueryCats = TorznabCaps . Categories . ExpandTorznabQueryCategories ( query ) ;
2020-11-01 11:07:24 +00:00
2020-11-02 13:20:13 +00:00
filteredResults = filteredResults . Where ( result = >
result . Category ? . Any ( ) ! = true | |
expandedQueryCats . Intersect ( result . Category ) . Any ( )
2020-11-01 11:07:24 +00:00
) ;
2020-11-02 13:20:13 +00:00
}
// eliminate excess results
if ( query . Limit > 0 )
filteredResults = filteredResults . Take ( query . Limit ) ;
2017-07-10 20:58:44 +00:00
return filteredResults ;
}
2020-11-02 13:20:13 +00:00
protected virtual IEnumerable < ReleaseInfo > FixResults ( TorznabQuery query , IEnumerable < ReleaseInfo > results )
{
var fixedResults = results . Select ( r = >
{
// add origin
r . Origin = this ;
// fix publish date
// some trackers do not keep their clocks up to date and can be ~20 minutes out!
if ( r . PublishDate > DateTime . Now )
r . PublishDate = DateTime . Now ;
2020-11-02 16:43:13 +00:00
// generate magnet link from info hash (not allowed for private sites)
if ( r . MagnetUri = = null & & ! string . IsNullOrWhiteSpace ( r . InfoHash ) & & Type ! = "private" )
r . MagnetUri = MagnetUtil . InfoHashToPublicMagnet ( r . InfoHash , r . Title ) ;
// generate info hash from magnet link
if ( r . MagnetUri ! = null & & string . IsNullOrWhiteSpace ( r . InfoHash ) )
r . InfoHash = MagnetUtil . MagnetToInfoHash ( r . MagnetUri ) ;
2020-12-22 17:49:59 +00:00
// set guid
if ( r . Guid = = null )
{
if ( r . Details ! = null )
r . Guid = r . Details ;
else if ( r . Link ! = null )
r . Guid = r . Link ;
else if ( r . MagnetUri ! = null )
r . Guid = r . MagnetUri ;
}
2020-11-02 13:20:13 +00:00
return r ;
} ) ;
return fixedResults ;
}
2018-04-06 15:43:18 +00:00
public virtual bool CanHandleQuery ( TorznabQuery query )
2017-07-10 20:58:44 +00:00
{
if ( query = = null )
return false ;
2017-08-11 22:30:43 +00:00
if ( query . QueryType = = "caps" )
return true ;
2017-07-10 20:58:44 +00:00
var caps = TorznabCaps ;
2020-10-18 20:47:36 +00:00
if ( caps . TvSearchImdbAvailable & & query . IsImdbQuery & & query . IsTVSearch )
2020-01-09 03:32:02 +00:00
return true ;
2020-10-18 17:26:22 +00:00
if ( caps . MovieSearchImdbAvailable & & query . IsImdbQuery & & query . IsMovieSearch )
2018-04-17 10:43:00 +00:00
return true ;
2020-11-08 22:27:54 +00:00
if ( ! caps . MovieSearchImdbAvailable & & query . IsImdbQuery & & query . QueryType ! = "TorrentPotato" ) // potato query should always contain imdb+search term
2018-04-17 10:43:00 +00:00
return false ;
2017-08-11 16:13:22 +00:00
if ( caps . SearchAvailable & & query . IsSearch )
return true ;
2020-10-18 20:47:36 +00:00
if ( caps . TvSearchAvailable & & query . IsTVSearch )
2017-08-11 16:13:22 +00:00
return true ;
if ( caps . MovieSearchAvailable & & query . IsMovieSearch )
return true ;
2017-10-18 16:30:41 +00:00
if ( caps . MusicSearchAvailable & & query . IsMusicSearch )
return true ;
2020-08-16 21:44:12 +00:00
if ( caps . BookSearchAvailable & & query . IsBookSearch )
return true ;
2020-10-18 20:47:36 +00:00
if ( caps . TvSearchTvRageAvailable & & query . IsTVRageSearch )
2017-08-11 16:13:22 +00:00
return true ;
2020-10-18 20:47:36 +00:00
if ( caps . TvSearchTvdbAvailable & & query . IsTvdbSearch )
2020-08-16 22:07:04 +00:00
return true ;
2020-10-18 17:26:22 +00:00
if ( caps . MovieSearchImdbAvailable & & query . IsImdbQuery )
2017-08-11 16:13:22 +00:00
return true ;
2020-10-18 17:26:22 +00:00
if ( caps . MovieSearchTmdbAvailable & & query . IsTmdbQuery )
2020-08-16 22:07:04 +00:00
return true ;
2017-08-11 16:13:22 +00:00
return false ;
2017-07-10 20:58:44 +00:00
}
2020-11-08 22:27:54 +00:00
protected bool CanHandleCategories ( TorznabQuery query , bool isMetaIndexer = false )
{
// https://torznab.github.io/spec-1.3-draft/torznab/Specification-v1.3.html#cat-parameter
if ( query . HasSpecifiedCategories )
{
var supportedCats = TorznabCaps . Categories . SupportedCategories ( query . Categories ) ;
if ( supportedCats . Length = = 0 )
{
if ( ! isMetaIndexer )
logger . Error ( $"All categories provided are unsupported in {DisplayName}: {string.Join(" , ", query.Categories)}" ) ;
return false ;
}
if ( supportedCats . Length ! = query . Categories . Length & & ! isMetaIndexer )
{
var unsupportedCats = query . Categories . Except ( supportedCats ) ;
logger . Warn ( $"Some of the categories provided are unsupported in {DisplayName}: {string.Join(" , ", unsupportedCats)}" ) ;
}
}
return true ;
}
2017-07-10 20:58:44 +00:00
public void Unconfigure ( )
{
IsConfigured = false ;
2017-11-13 11:55:54 +00:00
SiteLink = DefaultSiteLink ;
2017-11-13 15:55:02 +00:00
CookieHeader = "" ; // clear cookies
2017-07-10 20:58:44 +00:00
}
public abstract Task < IndexerConfigurationStatus > ApplyConfiguration ( JToken configJson ) ;
2020-11-08 22:27:54 +00:00
public virtual async Task < IndexerResult > ResultsForQuery ( TorznabQuery query , bool isMetaIndexer )
2017-07-10 20:58:44 +00:00
{
2020-11-08 22:27:54 +00:00
if ( ! CanHandleQuery ( query ) | | ! CanHandleCategories ( query , isMetaIndexer ) )
2020-12-11 22:14:21 +00:00
return new IndexerResult ( this , new ReleaseInfo [ 0 ] , false ) ;
var cachedReleases = cacheService . Search ( this , query ) ;
if ( cachedReleases ! = null )
return new IndexerResult ( this , cachedReleases , true ) ;
2020-11-02 13:20:13 +00:00
2017-08-24 10:28:41 +00:00
try
2017-07-10 20:58:44 +00:00
{
2017-08-24 10:28:41 +00:00
var results = await PerformQuery ( query ) ;
results = FilterResults ( query , results ) ;
2020-11-02 13:20:13 +00:00
results = FixResults ( query , results ) ;
2020-12-11 22:14:21 +00:00
cacheService . CacheResults ( this , query , results . ToList ( ) ) ;
return new IndexerResult ( this , results , false ) ;
2017-08-24 10:28:41 +00:00
}
catch ( Exception ex )
{
throw new IndexerException ( this , ex ) ;
}
2017-07-10 20:58:44 +00:00
}
2017-10-29 06:21:18 +00:00
2017-07-10 20:58:44 +00:00
protected abstract Task < IEnumerable < ReleaseInfo > > PerformQuery ( TorznabQuery query ) ;
}
public abstract class BaseWebIndexer : BaseIndexer , IWebIndexer
{
2020-05-11 19:59:28 +00:00
protected BaseWebIndexer ( string link , string id , string name , string description ,
IIndexerConfigurationService configService , WebClient client , Logger logger ,
2020-12-11 22:14:21 +00:00
ConfigurationData configData , IProtectionService p , ICacheService cacheService ,
TorznabCapabilities caps , string downloadBase = null )
: base ( link , id , name , description , configService , logger , configData , p , cacheService )
2017-07-10 20:58:44 +00:00
{
2020-02-10 22:16:19 +00:00
webclient = client ;
downloadUrlBase = downloadBase ;
2017-07-10 20:58:44 +00:00
TorznabCaps = caps ;
}
// minimal constructor used by e.g. cardigann generic indexer
2020-12-11 22:14:21 +00:00
protected BaseWebIndexer ( IIndexerConfigurationService configService , WebClient client , Logger logger ,
IProtectionService p , ICacheService cacheService )
: base ( "/" , "" , "" , "" , configService , logger , null , p , cacheService ) = > webclient = client ;
2017-07-10 20:58:44 +00:00
2021-03-15 06:27:18 +00:00
protected virtual int DefaultNumberOfRetryAttempts = > 2 ;
/// <summary>
/// Number of retry attempts to make if a web request fails.
/// </summary>
/// <remarks>
/// Number of retries can be overridden for unstable indexers by overriding this property. Note that retry attempts include an
/// exponentially increasing delay.
///
/// Alternatively, <see cref="EnableConfigurableRetryAttempts()" /> can be called in the constructor to add user configurable options.
/// </remarks>
protected virtual int NumberOfRetryAttempts
{
get
{
var configItem = configData . GetDynamic ( "retryAttempts" ) ;
if ( configItem = = null )
{
// No config specified so use the default.
return DefaultNumberOfRetryAttempts ;
}
2021-03-16 23:29:26 +00:00
var configValue = ( ( SingleSelectConfigurationItem ) configItem ) . Value ;
2021-03-15 06:27:18 +00:00
2021-03-16 23:29:26 +00:00
if ( int . TryParse ( configValue , out var parsedConfigValue ) & & parsedConfigValue > 0 )
2021-03-15 06:27:18 +00:00
{
return parsedConfigValue ;
}
else
{
// No config specified so use the default.
return DefaultNumberOfRetryAttempts ;
}
}
}
private AsyncPolicy < WebResult > RetryPolicy
{
get
{
// Configure the retry policy
int attemptNumber = 1 ;
var retryPolicy = Policy
. HandleResult < WebResult > ( r = > ( int ) r . Status > = 500 )
. Or < Exception > ( )
. WaitAndRetryAsync (
NumberOfRetryAttempts ,
retryAttempt = > TimeSpan . FromSeconds ( Math . Pow ( 2 , retryAttempt ) / 4 ) ,
onRetry : ( exception , timeSpan , context ) = >
{
2021-03-27 04:36:57 +00:00
if ( exception . Result = = null )
{
logger . Warn ( $"Request to {DisplayName} failed with exception '{exception.Exception.Message}'. Retrying in {timeSpan.TotalSeconds}s... (Attempt {attemptNumber} of {NumberOfRetryAttempts})." ) ;
}
else
{
logger . Warn ( $"Request to {DisplayName} failed with status {exception.Result.Status}. Retrying in {timeSpan.TotalSeconds}s... (Attempt {attemptNumber} of {NumberOfRetryAttempts})." ) ;
}
2021-03-15 06:27:18 +00:00
attemptNumber + + ;
} ) ;
return retryPolicy ;
}
}
/// <summary>
/// Adds configuration options to allow the user to manually configure request retries.
/// </summary>
/// <remarks>
/// This should only be enabled for indexers known to be unstable. To control the default value, override <see cref="DefaultNumberOfRetryAttempts" />.
/// </remarks>
protected void EnableConfigurableRetryAttempts ( )
{
2021-03-16 23:29:26 +00:00
var attemptSelect = new SingleSelectConfigurationItem (
"Number of retries" ,
2021-03-15 06:27:18 +00:00
new Dictionary < string , string >
{
{ "0" , "No retries (fail fast)" } ,
{ "1" , "1 retry (0.5s delay)" } ,
{ "2" , "2 retries (1s delay)" } ,
{ "3" , "3 retries (2s delay)" } ,
{ "4" , "4 retries (4s delay)" } ,
{ "5" , "5 retries (8s delay)" }
} )
{
Value = DefaultNumberOfRetryAttempts . ToString ( )
} ;
configData . AddDynamic ( "retryAttempts" , attemptSelect ) ;
}
2020-02-10 22:16:19 +00:00
public virtual async Task < byte [ ] > Download ( Uri link )
2017-07-10 20:58:44 +00:00
{
var uncleanLink = UncleanLink ( link ) ;
return await Download ( uncleanLink , RequestType . GET ) ;
2017-04-15 08:45:10 +00:00
}
2021-05-16 18:13:54 +00:00
protected async Task < byte [ ] > Download ( Uri link , RequestType method , string referer = null , Dictionary < string , string > headers = null )
2017-04-15 08:45:10 +00:00
{
2017-11-09 12:28:15 +00:00
// return magnet link
if ( link . Scheme = = "magnet" )
return Encoding . UTF8 . GetBytes ( link . OriginalString ) ;
2017-04-15 08:45:10 +00:00
// do some extra escaping, needed for HD-Torrents
var requestLink = link . ToString ( )
. Replace ( "(" , "%28" )
. Replace ( ")" , "%29" )
. Replace ( "'" , "%27" ) ;
2020-10-08 20:21:40 +00:00
var response = await RequestWithCookiesAndRetryAsync ( requestLink , null , method , referer , null , headers ) ;
2019-12-05 03:26:59 +00:00
2017-11-30 11:11:42 +00:00
if ( response . IsRedirect )
{
await FollowIfRedirect ( response ) ;
}
2017-04-15 08:45:10 +00:00
if ( response . Status ! = System . Net . HttpStatusCode . OK & & response . Status ! = System . Net . HttpStatusCode . Continue & & response . Status ! = System . Net . HttpStatusCode . PartialContent )
{
2020-02-10 22:16:19 +00:00
logger . Error ( "Failed download cookies: " + CookieHeader ) ;
2020-03-14 23:58:50 +00:00
if ( response . ContentBytes ! = null )
logger . Error ( "Failed download response:\n" + Encoding . UTF8 . GetString ( response . ContentBytes ) ) ;
2017-07-10 20:58:44 +00:00
throw new Exception ( $"Remote server returned {response.Status.ToString()}" + ( response . IsRedirect ? " => " + response . RedirectingTo : "" ) ) ;
2017-04-15 08:45:10 +00:00
}
2020-03-14 23:58:50 +00:00
return response . ContentBytes ;
2017-04-15 08:45:10 +00:00
}
2020-06-11 15:09:27 +00:00
protected async Task < WebResult > RequestWithCookiesAndRetryAsync (
string url , string cookieOverride = null , RequestType method = RequestType . GET ,
string referer = null , IEnumerable < KeyValuePair < string , string > > data = null ,
Dictionary < string , string > headers = null , string rawbody = null , bool? emulateBrowser = null )
2017-04-15 08:45:10 +00:00
{
2021-03-15 06:27:18 +00:00
return await RetryPolicy . ExecuteAsync ( async ( ) = >
await RequestWithCookiesAsync ( url , cookieOverride , method , referer , data , headers , rawbody , emulateBrowser )
) ;
2017-04-15 08:45:10 +00:00
}
2020-09-21 16:39:47 +00:00
protected virtual async Task < WebResult > RequestWithCookiesAsync (
2020-06-11 15:09:27 +00:00
string url , string cookieOverride = null , RequestType method = RequestType . GET ,
string referer = null , IEnumerable < KeyValuePair < string , string > > data = null ,
Dictionary < string , string > headers = null , string rawbody = null , bool? emulateBrowser = null )
2017-04-15 08:45:10 +00:00
{
2020-06-11 15:09:27 +00:00
var request = new WebRequest
2017-04-15 08:45:10 +00:00
{
Url = url ,
Type = method ,
Cookies = cookieOverride ? ? CookieHeader ,
PostData = data ,
Referer = referer ,
Headers = headers ,
RawBody = rawbody ,
Encoding = Encoding
} ;
if ( emulateBrowser . HasValue )
request . EmulateBrowser = emulateBrowser . Value ;
2020-06-11 15:09:27 +00:00
var result = await webclient . GetResultAsync ( request ) ;
CheckSiteDown ( result ) ;
2017-04-15 08:45:10 +00:00
UpdateCookieHeader ( result . Cookies , cookieOverride ) ;
return result ;
}
2021-03-14 21:19:48 +00:00
protected async Task < WebResult > RequestLoginAndFollowRedirect ( string url , IEnumerable < KeyValuePair < string , string > > data , string cookies , bool returnCookiesFromFirstCall , string redirectUrlOverride = null , string referer = null , bool accumulateCookies = false , Dictionary < string , string > headers = null )
2017-04-15 08:45:10 +00:00
{
2020-10-19 21:19:10 +00:00
var request = new WebRequest
2017-04-15 08:45:10 +00:00
{
Url = url ,
Type = RequestType . POST ,
Cookies = cookies ,
Referer = referer ,
PostData = data ,
2021-03-14 21:19:48 +00:00
Encoding = Encoding ,
Headers = headers ,
2017-04-15 08:45:10 +00:00
} ;
2020-06-11 15:09:27 +00:00
var response = await webclient . GetResultAsync ( request ) ;
CheckSiteDown ( response ) ;
2017-04-15 08:45:10 +00:00
if ( accumulateCookies )
{
response . Cookies = ResolveCookies ( ( request . Cookies = = null ? "" : request . Cookies + " " ) + response . Cookies ) ;
}
var firstCallCookies = response . Cookies ;
if ( response . IsRedirect )
{
await FollowIfRedirect ( response , request . Url , redirectUrlOverride , response . Cookies , accumulateCookies ) ;
}
if ( returnCookiesFromFirstCall )
{
response . Cookies = ResolveCookies ( firstCallCookies + ( accumulateCookies ? " " + response . Cookies : "" ) ) ;
}
2017-07-10 20:58:44 +00:00
2017-04-15 08:45:10 +00:00
return response ;
}
2020-06-11 15:09:27 +00:00
protected static void CheckSiteDown ( WebResult response )
2017-09-13 07:57:39 +00:00
{
if ( response . Status = = System . Net . HttpStatusCode . BadGateway
| | response . Status = = System . Net . HttpStatusCode . GatewayTimeout
| | ( int ) response . Status = = 521 // used by cloudflare to signal the original webserver is refusing the connection
| | ( int ) response . Status = = 522 // used by cloudflare to signal the original webserver is not reachable at all (timeout)
2018-04-01 13:40:43 +00:00
| | ( int ) response . Status = = 523 // used by cloudflare to signal the original webserver is not reachable at all (Origin is unreachable)
2017-10-29 06:21:18 +00:00
)
2017-09-13 07:57:39 +00:00
{
throw new Exception ( "Request to " + response . Request . Url + " failed (Error " + response . Status + ") - The tracker seems to be down." ) ;
}
}
2020-06-10 21:22:29 +00:00
protected async Task FollowIfRedirect ( WebResult response , string referrer = null , string overrideRedirectUrl = null , string overrideCookies = null , bool accumulateCookies = false )
2017-07-10 20:58:44 +00:00
{
// Follow up to 5 redirects
2020-02-10 22:16:19 +00:00
for ( var i = 0 ; i < 5 ; i + + )
2017-04-15 08:45:10 +00:00
{
2017-07-10 20:58:44 +00:00
if ( ! response . IsRedirect )
break ;
await DoFollowIfRedirect ( response , referrer , overrideRedirectUrl , overrideCookies , accumulateCookies ) ;
if ( accumulateCookies )
{
CookieHeader = ResolveCookies ( ( CookieHeader ! = null & & CookieHeader ! = "" ? CookieHeader + " " : "" ) + ( overrideCookies ! = null & & overrideCookies ! = "" ? overrideCookies + " " : "" ) + response . Cookies ) ;
overrideCookies = response . Cookies = CookieHeader ;
}
if ( overrideCookies ! = null & & response . Cookies = = null )
{
response . Cookies = overrideCookies ;
}
2017-04-15 08:45:10 +00:00
}
2017-07-10 20:58:44 +00:00
}
2020-02-10 22:16:19 +00:00
private string ResolveCookies ( string incomingCookies = "" )
2017-07-10 20:58:44 +00:00
{
2020-04-12 13:00:14 +00:00
var redirRequestCookies = string . IsNullOrWhiteSpace ( CookieHeader ) ? incomingCookies : CookieHeader + " " + incomingCookies ;
2020-04-13 04:22:50 +00:00
var cookieDictionary = CookieUtil . CookieHeaderToDictionary ( redirRequestCookies ) ;
// These cookies are causing BadGateway errors, so we drop them, see issue #2306
cookieDictionary . Remove ( "cf_use_ob" ) ;
cookieDictionary . Remove ( "cf_ob_info" ) ;
return CookieUtil . CookieDictionaryToHeader ( cookieDictionary ) ;
2017-04-15 08:45:10 +00:00
}
2017-07-10 20:58:44 +00:00
// Update CookieHeader with new cookies and save the config if something changed (e.g. a new CloudFlare clearance cookie was issued)
2017-10-03 12:23:31 +00:00
protected virtual void UpdateCookieHeader ( string newCookies , string cookieOverride = null )
2017-04-15 08:45:10 +00:00
{
2020-02-10 22:16:19 +00:00
var newCookieHeader = ResolveCookies ( ( cookieOverride ! = null & & cookieOverride ! = "" ? cookieOverride + " " : "" ) + newCookies ) ;
2017-07-10 20:58:44 +00:00
if ( CookieHeader ! = newCookieHeader )
2017-04-15 08:45:10 +00:00
{
2017-07-10 20:58:44 +00:00
logger . Debug ( string . Format ( "updating Cookies {0} => {1}" , CookieHeader , newCookieHeader ) ) ;
CookieHeader = newCookieHeader ;
if ( IsConfigured )
SaveConfig ( ) ;
}
}
2020-06-10 21:22:29 +00:00
private async Task DoFollowIfRedirect ( WebResult incomingResponse , string referrer = null , string overrideRedirectUrl = null , string overrideCookies = null , bool accumulateCookies = false )
2017-07-10 20:58:44 +00:00
{
if ( incomingResponse . IsRedirect )
{
var redirRequestCookies = "" ;
if ( accumulateCookies )
2017-04-15 08:45:10 +00:00
{
2017-07-10 20:58:44 +00:00
redirRequestCookies = ResolveCookies ( ( CookieHeader ! = "" ? CookieHeader + " " : "" ) + ( overrideCookies ! = null ? overrideCookies : "" ) ) ;
2017-04-15 08:45:10 +00:00
}
2017-07-10 20:58:44 +00:00
else
{
redirRequestCookies = ( overrideCookies ! = null ? overrideCookies : "" ) ;
}
// Do redirect
2020-10-19 21:19:10 +00:00
var redirectedResponse = await webclient . GetResultAsync ( new WebRequest
2017-07-10 20:58:44 +00:00
{
Url = overrideRedirectUrl ? ? incomingResponse . RedirectingTo ,
Referer = referrer ,
Cookies = redirRequestCookies ,
Encoding = Encoding
} ) ;
Mapper . Map ( redirectedResponse , incomingResponse ) ;
2017-04-15 08:45:10 +00:00
}
}
2020-10-27 21:17:03 +00:00
protected List < string > GetAllTrackerCategories ( ) = >
TorznabCaps . Categories . GetTrackerCategories ( ) ;
2017-04-15 08:45:10 +00:00
2020-10-27 21:17:03 +00:00
protected void AddCategoryMapping ( string trackerCategory , TorznabCategory newznabCategory , string trackerCategoryDesc = null ) = >
TorznabCaps . Categories . AddCategoryMapping ( trackerCategory , newznabCategory , trackerCategoryDesc ) ;
2017-04-15 08:45:10 +00:00
2020-10-31 00:12:35 +00:00
// TODO: remove this method ?
2020-10-27 21:17:03 +00:00
protected void AddCategoryMapping ( int trackerCategory , TorznabCategory newznabCategory , string trackerCategoryDesc = null ) = >
AddCategoryMapping ( trackerCategory . ToString ( ) , newznabCategory , trackerCategoryDesc ) ;
2017-04-15 08:45:10 +00:00
2020-10-31 00:12:35 +00:00
// TODO: remove this method and use AddCategoryMapping instead. this method doesn't allow to create custom cats
2017-04-15 08:45:10 +00:00
protected void AddMultiCategoryMapping ( TorznabCategory newznabCategory , params int [ ] trackerCategories )
{
foreach ( var trackerCat in trackerCategories )
AddCategoryMapping ( trackerCat , newznabCategory ) ;
}
2020-10-27 21:17:03 +00:00
protected List < string > MapTorznabCapsToTrackers ( TorznabQuery query , bool mapChildrenCatsToParent = false ) = >
TorznabCaps . Categories . MapTorznabCapsToTrackers ( query , mapChildrenCatsToParent ) ;
2017-04-15 08:45:10 +00:00
2020-10-27 21:17:03 +00:00
protected ICollection < int > MapTrackerCatToNewznab ( string input ) = >
TorznabCaps . Categories . MapTrackerCatToNewznab ( input ) ;
2017-04-15 08:45:10 +00:00
2020-10-27 21:17:03 +00:00
protected ICollection < int > MapTrackerCatDescToNewznab ( string input ) = >
TorznabCaps . Categories . MapTrackerCatDescToNewznab ( input ) ;
2017-07-10 20:58:44 +00:00
private IEnumerable < ReleaseInfo > CleanLinks ( IEnumerable < ReleaseInfo > releases )
2017-07-03 05:15:47 +00:00
{
2017-07-10 20:58:44 +00:00
if ( string . IsNullOrEmpty ( downloadUrlBase ) )
return releases ;
foreach ( var release in releases )
2017-07-03 05:15:47 +00:00
{
2017-07-10 20:58:44 +00:00
if ( release . Link . ToString ( ) . StartsWith ( downloadUrlBase , StringComparison . Ordinal ) )
{
release . Link = new Uri ( release . Link . ToString ( ) . Substring ( downloadUrlBase . Length ) , UriKind . Relative ) ;
}
2017-07-03 05:15:47 +00:00
}
2017-07-10 20:58:44 +00:00
return releases ;
}
2020-11-08 22:27:54 +00:00
public override async Task < IndexerResult > ResultsForQuery ( TorznabQuery query , bool isMetaIndexer )
2017-07-10 20:58:44 +00:00
{
2020-11-08 22:27:54 +00:00
var result = await base . ResultsForQuery ( query , isMetaIndexer ) ;
2017-08-24 10:28:41 +00:00
result . Releases = CleanLinks ( result . Releases ) ;
return result ;
2017-07-03 05:15:47 +00:00
}
2017-07-10 20:58:44 +00:00
protected virtual Uri UncleanLink ( Uri link )
{
if ( string . IsNullOrWhiteSpace ( downloadUrlBase ) )
{
return link ;
}
if ( link . ToString ( ) . StartsWith ( downloadUrlBase , StringComparison . Ordinal ) )
{
return link ;
}
return new Uri ( downloadUrlBase + link . ToString ( ) , UriKind . RelativeOrAbsolute ) ;
}
protected void OnParseError ( string results , Exception ex )
{
var fileName = string . Format ( "Error on {0} for {1}.txt" , DateTime . Now . ToString ( "yyyyMMddHHmmss" ) , DisplayName ) ;
var spacing = string . Join ( "" , Enumerable . Repeat ( Environment . NewLine , 5 ) ) ;
var fileContents = string . Format ( "{0}{1}{2}" , ex , spacing , results ) ;
logger . Error ( fileName + fileContents ) ;
2017-09-13 09:49:24 +00:00
throw new Exception ( "Parse error" , ex ) ;
2017-07-10 20:58:44 +00:00
}
public override TorznabCapabilities TorznabCaps { get ; protected set ; }
2017-07-11 20:32:56 +00:00
2017-11-05 09:42:03 +00:00
protected WebClient webclient ;
2017-07-10 20:58:44 +00:00
protected readonly string downloadUrlBase = "" ;
}
public abstract class BaseCachingWebIndexer : BaseWebIndexer
{
2021-05-16 18:13:54 +00:00
protected BaseCachingWebIndexer ( string link , string id , string name , string description ,
2020-05-11 19:59:28 +00:00
IIndexerConfigurationService configService , WebClient client , Logger logger ,
2020-12-11 22:14:21 +00:00
ConfigurationData configData , IProtectionService p , ICacheService cacheService ,
TorznabCapabilities caps = null , string downloadBase = null )
: base ( link , id , name , description , configService , client , logger , configData , p , cacheService , caps , downloadBase )
2017-07-10 20:58:44 +00:00
{
}
protected void CleanCache ( )
{
foreach ( var expired in cache . Where ( i = > DateTime . Now - i . Created > cacheTime ) . ToList ( ) )
{
cache . Remove ( expired ) ;
}
}
2020-12-11 22:14:21 +00:00
// TODO: remove this implementation and use gloal cache
2017-07-10 20:58:44 +00:00
protected static List < CachedQueryResult > cache = new List < CachedQueryResult > ( ) ;
protected static readonly TimeSpan cacheTime = new TimeSpan ( 0 , 9 , 0 ) ;
2017-04-15 08:45:10 +00:00
}
}