2016-10-27 07:30:03 +00:00
using Jackett.Utils.Clients ;
using NLog ;
using Jackett.Services ;
using Jackett.Utils ;
using Jackett.Models ;
using System.Threading.Tasks ;
using Newtonsoft.Json.Linq ;
using System.Collections.Generic ;
using System ;
using Jackett.Models.IndexerConfig ;
using System.Collections.Specialized ;
using System.Text ;
using YamlDotNet.Serialization ;
using YamlDotNet.Serialization.NamingConventions ;
using static Jackett . Models . IndexerConfig . ConfigurationData ;
using AngleSharp.Parser.Html ;
using System.Text.RegularExpressions ;
using System.Web ;
2016-10-30 15:15:01 +00:00
using AngleSharp.Dom ;
2016-11-26 17:00:53 +00:00
using AngleSharp.Dom.Html ;
2016-12-05 14:09:43 +00:00
using System.Linq ;
2016-10-27 07:30:03 +00:00
namespace Jackett.Indexers
{
public class CardigannIndexer : BaseIndexer , IIndexer
{
2016-12-10 17:20:04 +00:00
public string DefinitionString { get ; protected set ; }
2016-10-27 07:30:03 +00:00
protected IndexerDefinition Definition ;
public new string ID { get { return ( Definition ! = null ? Definition . Site : GetIndexerID ( GetType ( ) ) ) ; } }
2016-11-29 18:32:50 +00:00
protected WebClientStringResult landingResult ;
protected IHtmlDocument landingResultDocument ;
2016-10-27 07:30:03 +00:00
new ConfigurationData configData
{
get { return ( ConfigurationData ) base . configData ; }
set { base . configData = value ; }
}
// Cardigann yaml classes
public class IndexerDefinition {
public string Site { get ; set ; }
public List < settingsField > Settings { get ; set ; }
public string Name { get ; set ; }
public string Description { get ; set ; }
public string Language { get ; set ; }
2016-11-28 18:31:12 +00:00
public string Encoding { get ; set ; }
2016-10-27 07:30:03 +00:00
public List < string > Links { get ; set ; }
public capabilitiesBlock Caps { get ; set ; }
public loginBlock Login { get ; set ; }
public ratioBlock Ratio { get ; set ; }
public searchBlock Search { get ; set ; }
2016-12-15 08:12:28 +00:00
public downloadBlock Download { get ; set ; }
2016-10-27 07:30:03 +00:00
// IndexerDefinitionStats not needed/implemented
}
public class settingsField
{
public string Name { get ; set ; }
public string Type { get ; set ; }
public string Label { get ; set ; }
}
public class capabilitiesBlock
{
public Dictionary < string , string > Categories { get ; set ; }
public Dictionary < string , List < string > > Modes { get ; set ; }
}
2016-12-23 16:18:37 +00:00
public class captchaBlock
{
public string Type { get ; set ; }
public string Image { get ; set ; }
public string Input { get ; set ; }
}
2016-10-27 07:30:03 +00:00
public class loginBlock
{
public string Path { get ; set ; }
2017-01-18 20:41:57 +00:00
public string Submitpath { get ; set ; }
2016-10-27 07:30:03 +00:00
public string Method { get ; set ; }
public string Form { get ; set ; }
2017-01-16 11:24:45 +00:00
public bool Selectors { get ; set ; } = false ;
2016-10-27 07:30:03 +00:00
public Dictionary < string , string > Inputs { get ; set ; }
2017-01-18 20:17:45 +00:00
public Dictionary < string , selectorBlock > Selectorinputs { get ; set ; }
2016-10-27 07:30:03 +00:00
public List < errorBlock > Error { get ; set ; }
public pageTestBlock Test { get ; set ; }
2016-12-23 16:18:37 +00:00
public captchaBlock Captcha { get ; set ; }
2016-10-27 07:30:03 +00:00
}
public class errorBlock
{
public string Path { get ; set ; }
public string Selector { get ; set ; }
public selectorBlock Message { get ; set ; }
}
public class selectorBlock
{
public string Selector { get ; set ; }
public string Text { get ; set ; }
public string Attribute { get ; set ; }
public string Remove { get ; set ; }
public List < filterBlock > Filters { get ; set ; }
public Dictionary < string , string > Case { get ; set ; }
}
public class filterBlock
{
public string Name { get ; set ; }
public dynamic Args { get ; set ; }
}
public class pageTestBlock
{
public string Path { get ; set ; }
public string Selector { get ; set ; }
}
public class ratioBlock : selectorBlock
{
public string Path { get ; set ; }
}
public class searchBlock
{
public string Path { get ; set ; }
2017-01-27 11:14:49 +00:00
public List < filterBlock > Keywordsfilters { get ; set ; }
2016-10-27 07:30:03 +00:00
public Dictionary < string , string > Inputs { get ; set ; }
public rowsBlock Rows { get ; set ; }
public Dictionary < string , selectorBlock > Fields { get ; set ; }
}
public class rowsBlock : selectorBlock
{
public int After { get ; set ; }
//public string Remove { get; set; } // already inherited
2016-11-21 17:48:36 +00:00
public selectorBlock Dateheaders { get ; set ; }
2016-10-27 07:30:03 +00:00
}
2017-01-27 11:14:49 +00:00
public class requestBlock
{
public string Path { get ; set ; }
public string Method { get ; set ; }
public Dictionary < string , string > Inputs { get ; set ; }
}
2016-12-15 08:12:28 +00:00
public class downloadBlock
{
public string Selector { get ; set ; }
2017-01-03 12:19:23 +00:00
public string Method { get ; set ; }
2017-01-27 11:14:49 +00:00
public requestBlock Before { get ; set ; }
2016-12-15 08:12:28 +00:00
}
2016-12-20 08:38:01 +00:00
protected readonly string [ ] OptionalFileds = new string [ ] { "imdb" , "rageid" , "tvdbid" , "banner" } ;
2016-12-05 14:09:43 +00:00
2016-10-27 07:30:03 +00:00
public CardigannIndexer ( IIndexerManagerService i , IWebClient wc , Logger l , IProtectionService ps )
: base ( manager : i ,
client : wc ,
logger : l ,
p : ps )
{
}
public CardigannIndexer ( IIndexerManagerService i , IWebClient wc , Logger l , IProtectionService ps , string DefinitionString )
: base ( manager : i ,
client : wc ,
logger : l ,
p : ps )
{
Init ( DefinitionString ) ;
}
protected void Init ( string DefinitionString )
{
2016-12-10 17:20:04 +00:00
this . DefinitionString = DefinitionString ;
2016-10-27 07:30:03 +00:00
var deserializer = new DeserializerBuilder ( )
. WithNamingConvention ( new CamelCaseNamingConvention ( ) )
. IgnoreUnmatchedProperties ( )
. Build ( ) ;
Definition = deserializer . Deserialize < IndexerDefinition > ( DefinitionString ) ;
// Add default data if necessary
if ( Definition . Settings = = null )
Definition . Settings = new List < settingsField > ( ) ;
if ( Definition . Settings . Count = = 0 )
{
Definition . Settings . Add ( new settingsField { Name = "username" , Label = "Username" , Type = "text" } ) ;
Definition . Settings . Add ( new settingsField { Name = "password" , Label = "Password" , Type = "password" } ) ;
}
2016-11-28 18:31:12 +00:00
if ( Definition . Encoding = = null )
Definition . Encoding = "iso-8859-1" ;
2016-11-29 18:32:50 +00:00
if ( Definition . Login ! = null & & Definition . Login . Method = = null )
Definition . Login . Method = "form" ;
// init missing mandatory attributes
2016-10-27 07:30:03 +00:00
DisplayName = Definition . Name ;
DisplayDescription = Definition . Description ;
2017-01-02 20:39:28 +00:00
DefaultSiteLink = Definition . Links [ 0 ] ; // TODO: implement alternative links
2016-12-06 13:56:47 +00:00
Encoding = Encoding . GetEncoding ( Definition . Encoding ) ;
2017-01-02 20:39:28 +00:00
if ( ! DefaultSiteLink . EndsWith ( "/" ) )
DefaultSiteLink + = "/" ;
2016-12-02 17:58:10 +00:00
Language = Definition . Language ;
2016-12-27 18:06:37 +00:00
TorznabCaps = new TorznabCapabilities ( ) ;
2016-10-27 07:30:03 +00:00
2017-01-25 09:56:06 +00:00
TorznabCaps . SupportsImdbSearch = Definition . Caps . Modes . Where ( c = > c . Key = = "movie-search" & & c . Value . Contains ( "imdbid" ) ) . Any ( ) ;
2016-10-27 07:30:03 +00:00
// init config Data
configData = new ConfigurationData ( ) ;
foreach ( var Setting in Definition . Settings )
{
configData . AddDynamic ( Setting . Name , new StringItem { Name = Setting . Label } ) ;
}
foreach ( var Category in Definition . Caps . Categories )
{
var cat = TorznabCatType . GetCatByName ( Category . Value ) ;
if ( cat = = null )
{
logger . Error ( string . Format ( "CardigannIndexer ({0}): Can't find a category for {1}" , ID , Category . Value ) ) ;
continue ;
}
AddCategoryMapping ( Category . Key , TorznabCatType . GetCatByName ( Category . Value ) ) ;
}
2017-01-02 20:39:28 +00:00
LoadValuesFromJson ( null ) ;
2016-10-27 07:30:03 +00:00
}
protected Dictionary < string , object > getTemplateVariablesFromConfigData ( )
{
Dictionary < string , object > variables = new Dictionary < string , object > ( ) ;
foreach ( settingsField Setting in Definition . Settings )
{
variables [ ".Config." + Setting . Name ] = ( ( StringItem ) configData . GetDynamic ( Setting . Name ) ) . Value ;
}
return variables ;
}
// A very bad implementation of the golang template/text templating engine.
// But it should work for most basic constucts used by Cardigann definitions.
protected string applyGoTemplateText ( string template , Dictionary < string , object > variables = null )
{
if ( variables = = null )
{
variables = getTemplateVariablesFromConfigData ( ) ;
}
2016-12-18 11:20:33 +00:00
// handle re_replace expression
// Example: {{ re_replace .Query.Keywords "[^a-zA-Z0-9]+" "%" }}
Regex ReReplaceRegex = new Regex ( @"{{\s*re_replace\s+(\..+?)\s+""(.+)""\s+""(.+?)""\s*}}" ) ;
var ReReplaceRegexMatches = ReReplaceRegex . Match ( template ) ;
while ( ReReplaceRegexMatches . Success )
{
string all = ReReplaceRegexMatches . Groups [ 0 ] . Value ;
string variable = ReReplaceRegexMatches . Groups [ 1 ] . Value ;
string regexp = ReReplaceRegexMatches . Groups [ 2 ] . Value ;
string newvalue = ReReplaceRegexMatches . Groups [ 3 ] . Value ;
Regex ReplaceRegex = new Regex ( regexp ) ;
var input = ( string ) variables [ variable ] ;
var expanded = ReplaceRegex . Replace ( input , newvalue ) ;
template = template . Replace ( all , expanded ) ;
ReReplaceRegexMatches = ReReplaceRegexMatches . NextMatch ( ) ;
}
2016-10-27 07:30:03 +00:00
// handle if ... else ... expression
Regex IfElseRegex = new Regex ( @"{{if\s*(.+?)\s*}}(.*?){{\s*else\s*}}(.*?){{\s*end\s*}}" ) ;
var IfElseRegexMatches = IfElseRegex . Match ( template ) ;
while ( IfElseRegexMatches . Success )
{
string conditionResult = null ;
string all = IfElseRegexMatches . Groups [ 0 ] . Value ;
string condition = IfElseRegexMatches . Groups [ 1 ] . Value ;
string onTrue = IfElseRegexMatches . Groups [ 2 ] . Value ;
string onFalse = IfElseRegexMatches . Groups [ 3 ] . Value ;
if ( condition . StartsWith ( "." ) )
{
string value = ( string ) variables [ condition ] ;
if ( ! string . IsNullOrWhiteSpace ( value ) )
{
conditionResult = onTrue ;
}
else
{
conditionResult = onFalse ;
}
}
else
{
throw new NotImplementedException ( "CardigannIndexer: Condition operation '" + condition + "' not implemented" ) ;
}
template = template . Replace ( all , conditionResult ) ;
IfElseRegexMatches = IfElseRegexMatches . NextMatch ( ) ;
}
// handle range expression
Regex RangeRegex = new Regex ( @"{{\s*range\s*(.+?)\s*}}(.*?){{\.}}(.*?){{end}}" ) ;
var RangeRegexMatches = RangeRegex . Match ( template ) ;
while ( RangeRegexMatches . Success )
{
string expanded = string . Empty ;
string all = RangeRegexMatches . Groups [ 0 ] . Value ;
string variable = RangeRegexMatches . Groups [ 1 ] . Value ;
string prefix = RangeRegexMatches . Groups [ 2 ] . Value ;
string postfix = RangeRegexMatches . Groups [ 3 ] . Value ;
foreach ( string value in ( List < string > ) variables [ variable ] )
{
expanded + = prefix + value + postfix ;
}
template = template . Replace ( all , expanded ) ;
RangeRegexMatches = RangeRegexMatches . NextMatch ( ) ;
}
// handle simple variables
Regex VariablesRegEx = new Regex ( @"{{\s*(\..+?)\s*}}" ) ;
var VariablesRegExMatches = VariablesRegEx . Match ( template ) ;
while ( VariablesRegExMatches . Success )
{
string expanded = string . Empty ;
string all = VariablesRegExMatches . Groups [ 0 ] . Value ;
string variable = VariablesRegExMatches . Groups [ 1 ] . Value ;
string value = ( string ) variables [ variable ] ;
template = template . Replace ( all , value ) ;
VariablesRegExMatches = VariablesRegExMatches . NextMatch ( ) ;
}
return template ;
2016-11-19 11:46:31 +00:00
}
protected bool checkForLoginError ( WebClientStringResult loginResult )
{
var ErrorBlocks = Definition . Login . Error ;
if ( ErrorBlocks = = null )
return true ; // no error
var loginResultParser = new HtmlParser ( ) ;
var loginResultDocument = loginResultParser . Parse ( loginResult . Content ) ;
foreach ( errorBlock error in ErrorBlocks )
{
var selection = loginResultDocument . QuerySelector ( error . Selector ) ;
if ( selection ! = null )
{
string errorMessage = selection . TextContent ;
if ( error . Message ! = null )
{
2017-01-17 14:00:59 +00:00
errorMessage = handleSelector ( error . Message , loginResultDocument . FirstElementChild ) ;
2016-11-19 11:46:31 +00:00
}
throw new ExceptionWithConfigData ( string . Format ( "Login failed: {0}" , errorMessage . Trim ( ) ) , configData ) ;
}
}
return true ; // no error
2016-10-27 07:30:03 +00:00
}
2016-11-29 18:32:50 +00:00
2016-10-27 07:30:03 +00:00
protected async Task < bool > DoLogin ( )
{
var Login = Definition . Login ;
if ( Login = = null )
return false ;
2016-11-29 18:32:50 +00:00
if ( Login . Method = = "post" )
2016-10-27 07:30:03 +00:00
{
var pairs = new Dictionary < string , string > ( ) ;
foreach ( var Input in Definition . Login . Inputs )
{
var value = applyGoTemplateText ( Input . Value ) ;
pairs . Add ( Input . Key , value ) ;
}
2016-10-30 14:16:28 +00:00
2017-01-06 14:05:51 +00:00
var LoginUrl = resolvePath ( Login . Path ) . ToString ( ) ;
2016-10-27 07:30:03 +00:00
configData . CookieHeader . Value = null ;
var loginResult = await RequestLoginAndFollowRedirect ( LoginUrl , pairs , null , true , null , SiteLink , true ) ;
configData . CookieHeader . Value = loginResult . Cookies ;
2016-11-19 11:46:31 +00:00
checkForLoginError ( loginResult ) ;
}
else if ( Login . Method = = "form" )
{
2017-01-06 14:05:51 +00:00
var LoginUrl = resolvePath ( Login . Path ) . ToString ( ) ;
2016-11-19 11:46:31 +00:00
var pairs = new Dictionary < string , string > ( ) ;
2016-11-29 18:32:50 +00:00
var CaptchaConfigItem = ( RecaptchaItem ) configData . GetDynamic ( "Captcha" ) ;
if ( CaptchaConfigItem ! = null )
{
if ( ! string . IsNullOrWhiteSpace ( CaptchaConfigItem . Cookie ) )
{
// for remote users just set the cookie and return
CookieHeader = CaptchaConfigItem . Cookie ;
return true ;
}
var CloudFlareCaptchaChallenge = landingResultDocument . QuerySelector ( "script[src=\"/cdn-cgi/scripts/cf.challenge.js\"]" ) ;
if ( CloudFlareCaptchaChallenge ! = null )
{
var CloudFlareQueryCollection = new NameValueCollection ( ) ;
CloudFlareQueryCollection [ "id" ] = CloudFlareCaptchaChallenge . GetAttribute ( "data-ray" ) ;
CloudFlareQueryCollection [ "g-recaptcha-response" ] = CaptchaConfigItem . Value ;
var ClearanceUrl = resolvePath ( "/cdn-cgi/l/chk_captcha?" + CloudFlareQueryCollection . GetQueryString ( ) ) ;
var ClearanceResult = await RequestStringWithCookies ( ClearanceUrl . ToString ( ) , null , SiteLink ) ;
if ( ClearanceResult . IsRedirect ) // clearance successfull
{
// request real login page again
landingResult = await RequestStringWithCookies ( LoginUrl , null , SiteLink ) ;
var htmlParser = new HtmlParser ( ) ;
landingResultDocument = htmlParser . Parse ( landingResult . Content ) ;
}
else
{
throw new ExceptionWithConfigData ( string . Format ( "Login failed: Cloudflare clearance failed using cookies {0}: {1}" , CookieHeader , ClearanceResult . Content ) , configData ) ;
}
}
else
{
pairs . Add ( "g-recaptcha-response" , CaptchaConfigItem . Value ) ;
}
}
2016-11-19 11:46:31 +00:00
2016-11-29 18:32:50 +00:00
var FormSelector = Login . Form ;
if ( FormSelector = = null )
FormSelector = "form" ;
2016-11-19 11:46:31 +00:00
2016-12-04 19:58:50 +00:00
// landingResultDocument might not be initiated if the login is caused by a relogin during a query
if ( landingResultDocument = = null )
{
await GetConfigurationForSetup ( ) ;
}
2016-11-19 11:46:31 +00:00
var form = landingResultDocument . QuerySelector ( FormSelector ) ;
if ( form = = null )
2016-10-27 07:30:03 +00:00
{
2016-11-19 11:46:31 +00:00
throw new ExceptionWithConfigData ( string . Format ( "Login failed: No form found on {0} using form selector {1}" , LoginUrl , FormSelector ) , configData ) ;
}
var inputs = form . QuerySelectorAll ( "input" ) ;
if ( inputs = = null )
{
throw new ExceptionWithConfigData ( string . Format ( "Login failed: No inputs found on {0} using form selector {1}" , LoginUrl , FormSelector ) , configData ) ;
}
var submitUrl = resolvePath ( form . GetAttribute ( "action" ) ) ;
2017-01-18 20:41:57 +00:00
if ( Login . Submitpath ! = null )
submitUrl = resolvePath ( Login . Submitpath ) ;
2016-11-19 11:46:31 +00:00
foreach ( var input in inputs )
{
var name = input . GetAttribute ( "name" ) ;
if ( name = = null )
continue ;
var value = input . GetAttribute ( "value" ) ;
if ( value = = null )
value = "" ;
pairs [ name ] = value ;
}
foreach ( var Input in Definition . Login . Inputs )
{
var value = applyGoTemplateText ( Input . Value ) ;
2017-01-16 11:24:45 +00:00
var input = Input . Key ;
if ( Login . Selectors )
{
var inputElement = landingResultDocument . QuerySelector ( Input . Key ) ;
if ( inputElement = = null )
throw new ExceptionWithConfigData ( string . Format ( "Login failed: No input found using selector {0}" , Input . Key ) , configData ) ;
input = inputElement . GetAttribute ( "name" ) ;
}
pairs [ input ] = value ;
2016-11-19 11:46:31 +00:00
}
2016-11-20 10:24:29 +00:00
2017-01-18 20:17:45 +00:00
// selector inputs
2017-01-19 13:22:26 +00:00
if ( Login . Selectorinputs ! = null )
2017-01-18 20:17:45 +00:00
{
2017-01-19 13:22:26 +00:00
foreach ( var Selectorinput in Login . Selectorinputs )
2017-01-18 20:17:45 +00:00
{
2017-01-19 13:22:26 +00:00
string value = null ;
try
{
value = handleSelector ( Selectorinput . Value , landingResultDocument . FirstElementChild ) ;
pairs [ Selectorinput . Key ] = value ;
}
catch ( Exception ex )
{
throw new Exception ( string . Format ( "Error while parsing selector input={0}, selector={1}, value={2}: {3}" , Selectorinput . Key , Selectorinput . Value . Selector , value , ex . Message ) ) ;
}
2017-01-18 20:17:45 +00:00
}
}
2016-11-20 10:24:29 +00:00
// automatically solve simpleCaptchas, if used
var simpleCaptchaPresent = landingResultDocument . QuerySelector ( "script[src*=\"simpleCaptcha\"]" ) ;
if ( simpleCaptchaPresent ! = null )
{
var captchaUrl = resolvePath ( "simpleCaptcha.php?numImages=1" ) ;
2016-11-29 18:32:50 +00:00
var simpleCaptchaResult = await RequestStringWithCookies ( captchaUrl . ToString ( ) , null , LoginUrl ) ;
2016-11-20 10:24:29 +00:00
var simpleCaptchaJSON = JObject . Parse ( simpleCaptchaResult . Content ) ;
var captchaSelection = simpleCaptchaJSON [ "images" ] [ 0 ] [ "hash" ] . ToString ( ) ;
pairs [ "captchaSelection" ] = captchaSelection ;
pairs [ "submitme" ] = "X" ;
}
2016-12-10 10:17:21 +00:00
2016-12-23 16:18:37 +00:00
if ( Login . Captcha ! = null )
{
var Captcha = Login . Captcha ;
if ( Captcha . Type = = "image" )
{
var CaptchaText = ( StringItem ) configData . GetDynamic ( "CaptchaText" ) ;
if ( CaptchaText ! = null )
2017-01-16 11:24:45 +00:00
{
var input = Captcha . Input ;
if ( Login . Selectors )
{
var inputElement = landingResultDocument . QuerySelector ( Captcha . Input ) ;
if ( inputElement = = null )
throw new ExceptionWithConfigData ( string . Format ( "Login failed: No captcha input found using {0}" , Captcha . Input ) , configData ) ;
input = inputElement . GetAttribute ( "name" ) ;
}
pairs [ input ] = CaptchaText . Value ;
}
2016-12-23 16:18:37 +00:00
}
}
2016-12-14 07:44:56 +00:00
// clear landingResults/Document, otherwise we might use an old version for a new relogin (if GetConfigurationForSetup() wasn't called before)
landingResult = null ;
landingResultDocument = null ;
2016-12-10 10:17:21 +00:00
WebClientStringResult loginResult = null ;
var enctype = form . GetAttribute ( "enctype" ) ;
if ( enctype = = "multipart/form-data" )
{
var headers = new Dictionary < string , string > ( ) ;
var boundary = "---------------------------" + ( DateTime . UtcNow . Subtract ( new DateTime ( 1970 , 1 , 1 ) ) ) . TotalSeconds . ToString ( ) . Replace ( "." , "" ) ;
var bodyParts = new List < string > ( ) ;
foreach ( var pair in pairs )
{
var part = "--" + boundary + "\r\n" +
"Content-Disposition: form-data; name=\"" + pair . Key + "\"\r\n" +
"\r\n" +
pair . Value ;
bodyParts . Add ( part ) ;
}
bodyParts . Add ( "--" + boundary + "--" ) ;
headers . Add ( "Content-Type" , "multipart/form-data; boundary=" + boundary ) ;
var body = string . Join ( "\r\n" , bodyParts ) ;
loginResult = await PostDataWithCookies ( submitUrl . ToString ( ) , pairs , configData . CookieHeader . Value , SiteLink , headers , body ) ;
} else {
loginResult = await RequestLoginAndFollowRedirect ( submitUrl . ToString ( ) , pairs , configData . CookieHeader . Value , true , null , SiteLink , true ) ;
}
2016-11-19 11:46:31 +00:00
configData . CookieHeader . Value = loginResult . Cookies ;
checkForLoginError ( loginResult ) ;
2016-10-27 07:30:03 +00:00
}
else if ( Login . Method = = "cookie" )
{
configData . CookieHeader . Value = ( ( StringItem ) configData . GetDynamic ( "cookie" ) ) . Value ;
}
2017-01-06 18:04:26 +00:00
else if ( Login . Method = = "get" )
{
var queryCollection = new NameValueCollection ( ) ;
foreach ( var Input in Definition . Login . Inputs )
{
var value = applyGoTemplateText ( Input . Value ) ;
queryCollection . Add ( Input . Key , value ) ;
}
var LoginUrl = resolvePath ( Login . Path + "?" + queryCollection . GetQueryString ( ) ) . ToString ( ) ;
configData . CookieHeader . Value = null ;
var loginResult = await RequestStringWithCookies ( LoginUrl , null , SiteLink ) ;
configData . CookieHeader . Value = loginResult . Cookies ;
checkForLoginError ( loginResult ) ;
}
2016-10-27 07:30:03 +00:00
else
{
throw new NotImplementedException ( "Login method " + Definition . Login . Method + " not implemented" ) ;
}
return true ;
}
protected async Task < bool > TestLogin ( )
{
var Login = Definition . Login ;
if ( Login = = null | | Login . Test = = null )
return false ;
// test if login was successful
2017-01-06 14:05:51 +00:00
var LoginTestUrl = resolvePath ( Login . Test . Path ) . ToString ( ) ;
2016-10-27 07:30:03 +00:00
var testResult = await RequestStringWithCookies ( LoginTestUrl ) ;
if ( testResult . IsRedirect )
{
throw new ExceptionWithConfigData ( "Login Failed, got redirected" , configData ) ;
}
if ( Login . Test . Selector ! = null )
{
var testResultParser = new HtmlParser ( ) ;
var testResultDocument = testResultParser . Parse ( testResult . Content ) ;
var selection = testResultDocument . QuerySelectorAll ( Login . Test . Selector ) ;
if ( selection . Length = = 0 )
{
throw new ExceptionWithConfigData ( string . Format ( "Login failed: Selector \"{0}\" didn't match" , Login . Test . Selector ) , configData ) ;
}
}
return true ;
}
2016-12-06 13:56:47 +00:00
protected bool CheckIfLoginIsNeeded ( WebClientStringResult Result , IHtmlDocument document )
2016-11-26 17:00:53 +00:00
{
if ( Result . IsRedirect )
{
return true ;
}
2016-12-04 19:49:07 +00:00
if ( Definition . Login = = null | | Definition . Login . Test = = null )
return false ;
2016-11-26 17:00:53 +00:00
if ( Definition . Login . Test . Selector ! = null )
{
var selection = document . QuerySelectorAll ( Definition . Login . Test . Selector ) ;
if ( selection . Length = = 0 )
{
return true ;
}
}
return false ;
2016-11-29 18:32:50 +00:00
}
public override async Task < ConfigurationData > GetConfigurationForSetup ( )
{
var Login = Definition . Login ;
if ( Login = = null | | Login . Method ! = "form" )
return configData ;
2017-01-06 14:05:51 +00:00
var LoginUrl = resolvePath ( Login . Path ) . ToString ( ) ;
2016-11-29 18:32:50 +00:00
configData . CookieHeader . Value = null ;
landingResult = await RequestStringWithCookies ( LoginUrl , null , SiteLink ) ;
var htmlParser = new HtmlParser ( ) ;
landingResultDocument = htmlParser . Parse ( landingResult . Content ) ;
var grecaptcha = landingResultDocument . QuerySelector ( ".g-recaptcha" ) ;
if ( grecaptcha ! = null )
{
var CaptchaItem = new RecaptchaItem ( ) ;
CaptchaItem . Name = "Captcha" ;
CaptchaItem . Version = "2" ;
CaptchaItem . SiteKey = grecaptcha . GetAttribute ( "data-sitekey" ) ;
if ( CaptchaItem . SiteKey = = null ) // some sites don't store the sitekey in the .g-recaptcha div (e.g. cloudflare captcha challenge page)
CaptchaItem . SiteKey = landingResultDocument . QuerySelector ( "[data-sitekey]" ) . GetAttribute ( "data-sitekey" ) ;
configData . AddDynamic ( "Captcha" , CaptchaItem ) ;
}
2016-12-23 16:18:37 +00:00
if ( Login . Captcha ! = null )
{
var Captcha = Login . Captcha ;
if ( Captcha . Type = = "image" )
{
var captchaElement = landingResultDocument . QuerySelector ( Captcha . Image ) ;
if ( captchaElement ! = null ) {
var CaptchaUrl = resolvePath ( captchaElement . GetAttribute ( "src" ) ) ;
2017-01-17 11:32:11 +00:00
var captchaImageData = await RequestBytesWithCookies ( CaptchaUrl . ToString ( ) , landingResult . Cookies , RequestType . GET , LoginUrl . ToString ( ) ) ;
2016-12-23 16:18:37 +00:00
var CaptchaImage = new ImageItem { Name = "Captcha Image" } ;
var CaptchaText = new StringItem { Name = "Captcha Text" } ;
2017-01-16 11:24:45 +00:00
2016-12-23 16:18:37 +00:00
CaptchaImage . Value = captchaImageData . Content ;
configData . AddDynamic ( "CaptchaImage" , CaptchaImage ) ;
configData . AddDynamic ( "CaptchaText" , CaptchaText ) ;
}
else
{
logger . Debug ( string . Format ( "CardigannIndexer ({0}): No captcha image found" , ID ) ) ;
}
}
else
{
throw new NotImplementedException ( string . Format ( "Captcha type \"{0}\" is not implemented" , Captcha . Type ) ) ;
}
}
2016-11-29 18:32:50 +00:00
return configData ;
2016-11-26 17:00:53 +00:00
}
2016-10-27 07:30:03 +00:00
public async Task < IndexerConfigurationStatus > ApplyConfiguration ( JToken configJson )
{
configData . LoadValuesFromJson ( configJson ) ;
await DoLogin ( ) ;
await TestLogin ( ) ;
SaveConfig ( ) ;
IsConfigured = true ;
return IndexerConfigurationStatus . Completed ;
}
protected string applyFilters ( string Data , List < filterBlock > Filters )
{
if ( Filters = = null )
return Data ;
foreach ( filterBlock Filter in Filters )
{
switch ( Filter . Name )
{
case "querystring" :
var param = ( string ) Filter . Args ;
var qsStr = Data . Split ( new char [ ] { '?' } , 2 ) [ 1 ] ;
2016-11-01 21:45:02 +00:00
qsStr = qsStr . Split ( new char [ ] { '#' } , 2 ) [ 0 ] ;
2016-10-27 07:30:03 +00:00
var qs = HttpUtility . ParseQueryString ( qsStr ) ;
Data = qs . Get ( param ) ;
break ;
case "timeparse" :
case "dateparse" :
2016-11-16 07:45:32 +00:00
var layout = ( string ) Filter . Args ;
try
{
var Date = DateTimeUtil . ParseDateTimeGoLang ( Data , layout ) ;
Data = Date . ToString ( DateTimeUtil . RFC1123ZPattern ) ;
}
2017-01-26 10:15:30 +00:00
catch ( FormatException ex )
2016-11-16 07:45:32 +00:00
{
2017-01-26 10:15:30 +00:00
logger . Debug ( ex . Message ) ;
2016-11-16 07:45:32 +00:00
}
break ;
2016-10-27 07:30:03 +00:00
case "regexp" :
var pattern = ( string ) Filter . Args ;
var Regexp = new Regex ( pattern ) ;
var Match = Regexp . Match ( Data ) ;
Data = Match . Groups [ 1 ] . Value ;
break ;
2017-01-27 08:48:15 +00:00
case "re_replace" :
2017-01-27 08:46:40 +00:00
var regexpreplace_pattern = ( string ) Filter . Args [ 0 ] ;
var regexpreplace_replacement = ( string ) Filter . Args [ 1 ] ;
Regex regexpreplace_regex = new Regex ( regexpreplace_pattern ) ;
Data = regexpreplace_regex . Replace ( Data , regexpreplace_replacement ) ;
break ;
2016-10-27 07:30:03 +00:00
case "split" :
var sep = ( string ) Filter . Args [ 0 ] ;
var pos = ( string ) Filter . Args [ 1 ] ;
var posInt = int . Parse ( pos ) ;
var strParts = Data . Split ( sep [ 0 ] ) ;
if ( posInt < 0 )
{
posInt + = strParts . Length ;
}
Data = strParts [ posInt ] ;
break ;
case "replace" :
var from = ( string ) Filter . Args [ 0 ] ;
var to = ( string ) Filter . Args [ 1 ] ;
Data = Data . Replace ( from , to ) ;
break ;
case "trim" :
var cutset = ( string ) Filter . Args ;
2017-01-24 14:49:10 +00:00
if ( cutset ! = null )
Data = Data . Trim ( cutset [ 0 ] ) ;
else
Data = Data . Trim ( ) ;
2016-10-27 07:30:03 +00:00
break ;
2017-01-17 16:10:50 +00:00
case "prepend" :
var prependstr = ( string ) Filter . Args ;
Data = prependstr + Data ;
break ;
2016-10-27 07:30:03 +00:00
case "append" :
var str = ( string ) Filter . Args ;
Data + = str ;
break ;
case "timeago" :
case "fuzzytime" :
case "reltime" :
var timestr = ( string ) Filter . Args ;
Data = DateTimeUtil . FromUnknown ( timestr ) . ToString ( DateTimeUtil . RFC1123ZPattern ) ;
break ;
2017-01-26 09:34:28 +00:00
case "hexdump" :
// this is mainly for debugging invisible special char related issues
Data = string . Join ( "" , Data . Select ( c = > c + "(" + ( ( int ) c ) . ToString ( "X2" ) + ")" ) ) ;
break ;
2016-10-27 07:30:03 +00:00
default :
break ;
}
}
return Data ;
}
2016-12-02 12:05:00 +00:00
protected IElement QuerySelector ( IElement Element , string Selector )
{
// AngleSharp doesn't support the :root pseudo selector, so we check for it manually
if ( Selector . StartsWith ( ":root" ) )
{
Selector = Selector . Substring ( 5 ) ;
while ( Element . ParentElement ! = null )
{
Element = Element . ParentElement ;
}
}
return Element . QuerySelector ( Selector ) ;
}
2016-10-30 15:15:01 +00:00
protected string handleSelector ( selectorBlock Selector , IElement Dom )
2016-10-27 07:30:03 +00:00
{
if ( Selector . Text ! = null )
{
return applyFilters ( Selector . Text , Selector . Filters ) ;
}
2016-10-30 15:15:01 +00:00
IElement selection = Dom ;
2016-10-27 07:30:03 +00:00
string value = null ;
2016-10-30 15:15:01 +00:00
2016-10-27 07:30:03 +00:00
if ( Selector . Selector ! = null )
{
2016-12-02 12:05:00 +00:00
selection = QuerySelector ( Dom , Selector . Selector ) ;
2016-10-27 07:30:03 +00:00
if ( selection = = null )
{
throw new Exception ( string . Format ( "Selector \"{0}\" didn't match {1}" , Selector . Selector , Dom . OuterHtml ) ) ;
}
2016-10-30 15:15:01 +00:00
}
if ( Selector . Remove ! = null )
{
foreach ( var i in selection . QuerySelectorAll ( Selector . Remove ) )
2016-10-27 07:30:03 +00:00
{
2016-10-30 15:15:01 +00:00
i . Remove ( ) ;
2016-10-27 07:30:03 +00:00
}
2016-10-30 15:15:01 +00:00
}
if ( Selector . Case ! = null )
{
foreach ( var Case in Selector . Case )
2016-10-27 07:30:03 +00:00
{
2016-12-02 12:05:00 +00:00
if ( selection . Matches ( Case . Key ) | | QuerySelector ( selection , Case . Key ) ! = null )
2016-10-30 15:15:01 +00:00
{
value = Case . Value ;
break ;
}
2016-10-27 07:30:03 +00:00
}
2016-10-30 15:15:01 +00:00
if ( value = = null )
throw new Exception ( string . Format ( "None of the case selectors \"{0}\" matched {1}" , string . Join ( "," , Selector . Case ) , selection . OuterHtml ) ) ;
2016-10-27 07:30:03 +00:00
}
2016-10-30 15:15:01 +00:00
else if ( Selector . Attribute ! = null )
{
value = selection . GetAttribute ( Selector . Attribute ) ;
2017-01-05 18:15:39 +00:00
if ( value = = null )
throw new Exception ( string . Format ( "Attribute \"{0}\" is not set for element {1}" , Selector . Attribute , selection . OuterHtml ) ) ;
2016-10-30 15:15:01 +00:00
}
else
{
value = selection . TextContent ;
}
2017-01-11 17:16:25 +00:00
return applyFilters ( ParseUtil . NormalizeSpace ( value ) , Selector . Filters ) ;
2016-10-27 07:30:03 +00:00
}
protected Uri resolvePath ( string path )
{
2016-11-02 18:02:39 +00:00
if ( path . StartsWith ( "http" ) )
{
return new Uri ( path ) ;
}
2017-01-06 14:05:51 +00:00
else if ( path . StartsWith ( "//" ) )
{
var basepath = new Uri ( SiteLink ) ;
return new Uri ( basepath . Scheme + ":" + path ) ;
}
2016-11-29 18:32:50 +00:00
else if ( path . StartsWith ( "/" ) )
{
var basepath = new Uri ( SiteLink ) ;
return new Uri ( basepath . Scheme + "://" + basepath . Host + path ) ;
}
2016-11-02 18:02:39 +00:00
else
{
return new Uri ( SiteLink + path ) ;
}
2016-10-27 07:30:03 +00:00
}
public async Task < IEnumerable < ReleaseInfo > > PerformQuery ( TorznabQuery query )
{
var releases = new List < ReleaseInfo > ( ) ;
searchBlock Search = Definition . Search ;
// init template context
var variables = getTemplateVariablesFromConfigData ( ) ;
variables [ ".Query.Type" ] = query . QueryType ;
variables [ ".Query.Q" ] = query . SearchTerm ;
variables [ ".Query.Series" ] = null ;
variables [ ".Query.Ep" ] = query . Episode ;
variables [ ".Query.Season" ] = query . Season ;
variables [ ".Query.Movie" ] = null ;
variables [ ".Query.Year" ] = null ;
variables [ ".Query.Limit" ] = query . Limit ;
variables [ ".Query.Offset" ] = query . Offset ;
variables [ ".Query.Extended" ] = query . Extended ;
variables [ ".Query.Categories" ] = query . Categories ;
variables [ ".Query.APIKey" ] = query . ApiKey ;
variables [ ".Query.TVDBID" ] = null ;
variables [ ".Query.TVRageID" ] = query . RageID ;
variables [ ".Query.IMDBID" ] = query . ImdbID ;
2017-01-25 15:59:36 +00:00
variables [ ".Query.IMDBIDShort" ] = query . ImdbIDShort ;
2016-10-27 07:30:03 +00:00
variables [ ".Query.TVMazeID" ] = null ;
variables [ ".Query.TraktID" ] = null ;
variables [ ".Query.Episode" ] = query . GetEpisodeSearchString ( ) ;
variables [ ".Categories" ] = MapTorznabCapsToTrackers ( query ) ;
var KeywordTokens = new List < string > ( ) ;
var KeywordTokenKeys = new List < string > { "Q" , "Series" , "Movie" , "Year" } ;
foreach ( var key in KeywordTokenKeys )
{
var Value = ( string ) variables [ ".Query." + key ] ;
if ( ! string . IsNullOrWhiteSpace ( Value ) )
KeywordTokens . Add ( Value ) ;
}
if ( ! string . IsNullOrWhiteSpace ( ( string ) variables [ ".Query.Episode" ] ) )
KeywordTokens . Add ( ( string ) variables [ ".Query.Episode" ] ) ;
variables [ ".Query.Keywords" ] = string . Join ( " " , KeywordTokens ) ;
2017-01-27 11:14:49 +00:00
variables [ ".Keywords" ] = applyFilters ( ( string ) variables [ ".Query.Keywords" ] , Search . Keywordsfilters ) ;
2016-10-27 07:30:03 +00:00
// build search URL
2017-01-06 14:05:51 +00:00
var searchUrl = resolvePath ( applyGoTemplateText ( Search . Path , variables ) + "?" ) . ToString ( ) ;
2016-10-27 07:30:03 +00:00
var queryCollection = new NameValueCollection ( ) ;
if ( Search . Inputs ! = null )
{
foreach ( var Input in Search . Inputs )
{
var value = applyGoTemplateText ( Input . Value , variables ) ;
if ( Input . Key = = "$raw" )
searchUrl + = value ;
else
queryCollection . Add ( Input . Key , value ) ;
}
}
2017-01-16 11:25:16 +00:00
if ( queryCollection . Count > 0 )
2017-01-23 17:14:20 +00:00
searchUrl + = "&" + queryCollection . GetQueryString ( Encoding ) ;
2017-01-16 11:25:16 +00:00
// in case no args are added remove ? again (needed for KAT)
searchUrl = searchUrl . TrimEnd ( '?' ) ;
2016-10-27 07:30:03 +00:00
// send HTTP request
2016-12-06 13:56:47 +00:00
var response = await RequestStringWithCookies ( searchUrl ) ;
var results = response . Content ;
2016-10-27 07:30:03 +00:00
try
{
var SearchResultParser = new HtmlParser ( ) ;
var SearchResultDocument = SearchResultParser . Parse ( results ) ;
2016-11-26 17:00:53 +00:00
// check if we need to login again
var loginNeeded = CheckIfLoginIsNeeded ( response , SearchResultDocument ) ;
if ( loginNeeded )
{
logger . Info ( string . Format ( "CardigannIndexer ({0}): Relogin required" , ID ) ) ;
await DoLogin ( ) ;
await TestLogin ( ) ;
2016-12-06 13:56:47 +00:00
response = await RequestStringWithCookies ( searchUrl ) ;
results = results = response . Content ;
2016-11-26 17:00:53 +00:00
SearchResultDocument = SearchResultParser . Parse ( results ) ;
}
2016-11-20 11:49:56 +00:00
var RowsDom = SearchResultDocument . QuerySelectorAll ( Search . Rows . Selector ) ;
List < IElement > Rows = new List < IElement > ( ) ;
foreach ( var RowDom in RowsDom )
{
Rows . Add ( RowDom ) ;
}
// merge following rows for After selector
var After = Definition . Search . Rows . After ;
if ( After > 0 )
{
for ( int i = 0 ; i < Rows . Count ; i + = 1 )
{
var CurrentRow = Rows [ i ] ;
for ( int j = 0 ; j < After ; j + = 1 )
{
var MergeRowIndex = i + j + 1 ;
var MergeRow = Rows [ MergeRowIndex ] ;
List < INode > MergeNodes = new List < INode > ( ) ;
foreach ( var node in MergeRow . QuerySelectorAll ( "td" ) )
{
MergeNodes . Add ( node ) ;
}
CurrentRow . Append ( MergeNodes . ToArray ( ) ) ;
}
Rows . RemoveRange ( i + 1 , After ) ;
}
}
2016-10-27 07:30:03 +00:00
foreach ( var Row in Rows )
{
try
{
var release = new ReleaseInfo ( ) ;
release . MinimumRatio = 1 ;
release . MinimumSeedTime = 48 * 60 * 60 ;
// Parse fields
foreach ( var Field in Search . Fields )
{
2017-01-05 18:15:39 +00:00
var FieldParts = Field . Key . Split ( '|' ) ;
var FieldName = FieldParts [ 0 ] ;
2017-01-11 17:16:25 +00:00
var FieldModifiers = new List < string > ( ) ;
for ( var i = 1 ; i < FieldParts . Length ; i + + )
FieldModifiers . Add ( FieldParts [ i ] ) ;
2017-01-05 18:15:39 +00:00
2016-12-05 14:09:43 +00:00
string value = null ;
2016-10-27 07:30:03 +00:00
try
{
2016-12-05 14:09:43 +00:00
value = handleSelector ( Field . Value , Row ) ;
2017-01-05 18:15:39 +00:00
switch ( FieldName )
2016-10-27 07:30:03 +00:00
{
case "download" :
2016-11-24 07:31:39 +00:00
if ( value . StartsWith ( "magnet:" ) )
{
release . MagnetUri = new Uri ( value ) ;
release . Link = release . MagnetUri ;
}
else
{
release . Link = resolvePath ( value ) ;
}
2016-10-27 07:30:03 +00:00
break ;
case "details" :
var url = resolvePath ( value ) ;
release . Guid = url ;
2016-11-28 07:50:08 +00:00
release . Comments = url ;
if ( release . Guid = = null )
release . Guid = url ;
2016-10-27 07:30:03 +00:00
break ;
case "comments" :
2016-11-20 09:15:48 +00:00
var CommentsUrl = resolvePath ( value ) ;
2016-11-28 07:50:08 +00:00
if ( release . Comments = = null )
release . Comments = CommentsUrl ;
2016-11-20 09:15:48 +00:00
if ( release . Guid = = null )
release . Guid = CommentsUrl ;
2016-10-27 07:30:03 +00:00
break ;
case "title" :
2017-01-11 17:16:25 +00:00
if ( FieldModifiers . Contains ( "append" ) )
release . Title + = value ;
else
release . Title = value ;
2016-10-27 07:30:03 +00:00
break ;
case "description" :
2017-01-11 17:16:25 +00:00
if ( FieldModifiers . Contains ( "append" ) )
release . Description + = value ;
else
release . Description = value ;
2016-10-27 07:30:03 +00:00
break ;
case "category" :
release . Category = MapTrackerCatToNewznab ( value ) ;
break ;
case "size" :
release . Size = ReleaseInfo . GetBytes ( value ) ;
break ;
case "leechers" :
if ( release . Peers = = null )
release . Peers = ParseUtil . CoerceInt ( value ) ;
else
release . Peers + = ParseUtil . CoerceInt ( value ) ;
break ;
case "seeders" :
release . Seeders = ParseUtil . CoerceInt ( value ) ;
if ( release . Peers = = null )
release . Peers = release . Seeders ;
else
release . Peers + = release . Seeders ;
break ;
case "date" :
release . PublishDate = DateTimeUtil . FromUnknown ( value ) ;
break ;
2016-10-30 14:50:53 +00:00
case "files" :
release . Files = ParseUtil . CoerceLong ( value ) ;
break ;
case "grabs" :
release . Grabs = ParseUtil . CoerceLong ( value ) ;
break ;
case "downloadvolumefactor" :
release . DownloadVolumeFactor = ParseUtil . CoerceDouble ( value ) ;
break ;
case "uploadvolumefactor" :
2016-10-30 15:22:26 +00:00
release . UploadVolumeFactor = ParseUtil . CoerceDouble ( value ) ;
2016-10-30 14:50:53 +00:00
break ;
2016-12-20 08:38:01 +00:00
case "minimumratio" :
release . MinimumRatio = ParseUtil . CoerceDouble ( value ) ;
break ;
case "minimumseedtime" :
release . MinimumSeedTime = ParseUtil . CoerceLong ( value ) ;
break ;
2016-12-05 14:09:43 +00:00
case "imdb" :
Regex IMDBRegEx = new Regex ( @"(\d+)" , RegexOptions . Compiled ) ;
var IMDBMatch = IMDBRegEx . Match ( value ) ;
var IMDBId = IMDBMatch . Groups [ 1 ] . Value ;
release . Imdb = ParseUtil . CoerceLong ( IMDBId ) ;
break ;
2016-12-20 08:38:01 +00:00
case "rageid" :
Regex RageIDRegEx = new Regex ( @"(\d+)" , RegexOptions . Compiled ) ;
var RageIDMatch = RageIDRegEx . Match ( value ) ;
var RageID = RageIDMatch . Groups [ 1 ] . Value ;
release . RageID = ParseUtil . CoerceLong ( RageID ) ;
break ;
case "tvdbid" :
Regex TVDBIdRegEx = new Regex ( @"(\d+)" , RegexOptions . Compiled ) ;
var TVDBIdMatch = TVDBIdRegEx . Match ( value ) ;
var TVDBId = TVDBIdMatch . Groups [ 1 ] . Value ;
release . TVDBId = ParseUtil . CoerceLong ( TVDBId ) ;
break ;
case "banner" :
2016-12-20 12:18:48 +00:00
if ( ! string . IsNullOrWhiteSpace ( value ) ) {
var bannerurl = resolvePath ( value ) ;
release . BannerUrl = bannerurl ;
}
2016-12-20 08:38:01 +00:00
break ;
2016-10-27 07:30:03 +00:00
default :
break ;
}
}
catch ( Exception ex )
{
2017-01-11 17:16:25 +00:00
if ( OptionalFileds . Contains ( Field . Key ) | | FieldModifiers . Contains ( "optional" ) )
2016-12-05 14:09:43 +00:00
continue ;
2017-01-24 14:49:10 +00:00
throw new Exception ( string . Format ( "Error while parsing field={0}, selector={1}, value={2}: {3}" , Field . Key , Field . Value . Selector , ( value = = null ? "<null>" : value ) , ex . Message ) ) ;
2016-10-27 07:30:03 +00:00
}
}
2016-12-29 11:19:49 +00:00
var Filters = Definition . Search . Rows . Filters ;
var SkipRelease = false ;
if ( Filters ! = null )
{
foreach ( filterBlock Filter in Filters )
{
switch ( Filter . Name )
{
case "andmatch" :
int CharacterLimit = - 1 ;
if ( Filter . Args ! = null )
CharacterLimit = int . Parse ( Filter . Args ) ;
if ( ! query . MatchQueryStringAND ( release . Title , CharacterLimit ) )
{
logger . Debug ( string . Format ( "CardigannIndexer ({0}): skipping {1} (andmatch filter)" , ID , release . Title ) ) ;
SkipRelease = true ;
}
break ;
default :
logger . Error ( string . Format ( "CardigannIndexer ({0}): Unsupported rows filter: {1}" , ID , Filter . Name ) ) ;
break ;
}
}
}
if ( SkipRelease )
continue ;
2016-11-21 17:48:36 +00:00
// if DateHeaders is set go through the previous rows and look for the header selector
var DateHeaders = Definition . Search . Rows . Dateheaders ;
2016-12-05 17:01:11 +00:00
if ( release . PublishDate = = null & & DateHeaders ! = null )
2016-11-21 17:48:36 +00:00
{
var PrevRow = Row . PreviousElementSibling ;
string value = null ;
while ( PrevRow ! = null )
{
try
{
value = handleSelector ( DateHeaders , PrevRow ) ;
break ;
}
2016-11-29 18:32:50 +00:00
catch ( Exception )
2016-11-21 17:48:36 +00:00
{
// do nothing
}
PrevRow = PrevRow . PreviousElementSibling ;
}
if ( value = = null )
throw new Exception ( string . Format ( "No date header row found for {0}" , release . ToString ( ) ) ) ;
release . PublishDate = DateTimeUtil . FromUnknown ( value ) ;
}
2016-10-27 07:30:03 +00:00
releases . Add ( release ) ;
}
catch ( Exception ex )
{
2016-12-27 18:07:32 +00:00
logger . Error ( string . Format ( "CardigannIndexer ({0}): Error while parsing row '{1}':\n\n{2}" , ID , Row . OuterHtml , ex ) ) ;
2016-10-27 07:30:03 +00:00
}
}
}
catch ( Exception ex )
{
OnParseError ( results , ex ) ;
}
return releases ;
}
2016-12-15 08:12:28 +00:00
2017-01-27 11:14:49 +00:00
protected async Task < WebClientByteResult > handleRequest ( requestBlock request , Dictionary < string , object > variables = null , string referer = null )
{
var requestLinkStr = resolvePath ( applyGoTemplateText ( request . Path , variables ) ) . ToString ( ) ;
Dictionary < string , string > pairs = null ;
var queryCollection = new NameValueCollection ( ) ;
RequestType method = RequestType . GET ;
if ( String . Equals ( request . Method , "post" , StringComparison . OrdinalIgnoreCase ) )
{
method = RequestType . POST ;
pairs = new Dictionary < string , string > ( ) ;
}
foreach ( var Input in request . Inputs )
{
var value = applyGoTemplateText ( Input . Value , variables ) ;
if ( method = = RequestType . GET )
queryCollection . Add ( Input . Key , value ) ;
else if ( method = = RequestType . POST )
pairs . Add ( Input . Key , value ) ;
}
if ( queryCollection . Count > 0 )
{
if ( ! requestLinkStr . Contains ( "?" ) )
requestLinkStr + = "?" + queryCollection . GetQueryString ( Encoding ) . Substring ( 1 ) ;
else
requestLinkStr + = queryCollection . GetQueryString ( Encoding ) ;
}
var response = await RequestBytesWithCookiesAndRetry ( requestLinkStr , null , method , referer , pairs ) ;
logger . Debug ( $"CardigannIndexer ({ID}): handleRequest() remote server returned {response.Status.ToString()}" + ( response . IsRedirect ? " => " + response . RedirectingTo : "" ) ) ;
return response ;
}
protected IDictionary < string , object > AddTemplateVariablesFromUri ( IDictionary < string , object > variables , Uri uri , string prefix = "" )
{
variables [ prefix + ".AbsoluteUri" ] = uri . AbsoluteUri ;
variables [ prefix + ".AbsolutePath" ] = uri . AbsolutePath ;
variables [ prefix + ".Scheme" ] = uri . Scheme ;
variables [ prefix + ".Host" ] = uri . Host ;
variables [ prefix + ".Port" ] = uri . Port . ToString ( ) ;
variables [ prefix + ".PathAndQuery" ] = uri . PathAndQuery ;
variables [ prefix + ".Query" ] = uri . Query ;
var queryString = HttpUtility . ParseQueryString ( uri . Query ) ;
foreach ( string key in queryString . Keys )
{
variables [ prefix + ".Query." + key ] = queryString . Get ( key ) ;
}
return variables ;
}
2016-12-15 08:12:28 +00:00
public override async Task < byte [ ] > Download ( Uri link )
{
2017-01-03 12:19:23 +00:00
var method = RequestType . GET ;
if ( Definition . Download ! = null )
2016-12-15 08:12:28 +00:00
{
var Download = Definition . Download ;
2017-01-27 11:14:49 +00:00
if ( Download . Before ! = null )
{
var beforeVariables = getTemplateVariablesFromConfigData ( ) ;
AddTemplateVariablesFromUri ( beforeVariables , link , ".DownloadUri" ) ;
var beforeresult = await handleRequest ( Download . Before , beforeVariables , link . ToString ( ) ) ;
}
2017-01-03 12:19:23 +00:00
if ( Download . Method ! = null )
{
if ( Download . Method = = "post" )
method = RequestType . POST ;
}
2016-12-15 08:12:28 +00:00
if ( Download . Selector ! = null )
{
var response = await RequestStringWithCookies ( link . ToString ( ) ) ;
2017-01-12 16:18:50 +00:00
if ( response . IsRedirect )
response = await RequestStringWithCookies ( response . RedirectingTo ) ;
2016-12-15 08:12:28 +00:00
var results = response . Content ;
var SearchResultParser = new HtmlParser ( ) ;
var SearchResultDocument = SearchResultParser . Parse ( results ) ;
var DlUri = SearchResultDocument . QuerySelector ( Download . Selector ) ;
if ( DlUri ! = null )
{
logger . Debug ( string . Format ( "CardigannIndexer ({0}): Download selector {1} matched:{2}" , ID , Download . Selector , DlUri . OuterHtml ) ) ;
var href = DlUri . GetAttribute ( "href" ) ;
link = resolvePath ( href ) ;
}
else
{
logger . Error ( string . Format ( "CardigannIndexer ({0}): Download selector {1} didn't match:\n{2}" , ID , Download . Selector , results ) ) ;
throw new Exception ( string . Format ( "Download selector {0} didn't match" , Download . Selector ) ) ;
}
}
}
2017-01-03 12:19:23 +00:00
return await base . Download ( link , method ) ;
2016-12-15 08:12:28 +00:00
}
2016-10-27 07:30:03 +00:00
}
}