mirror
/
Jackett
mirror of https://github.com/Jackett/Jackett
1
0
Fork 0
Code Issues Releases Wiki Activity

Add basic support for Cardigann definitions (#571) 

* Add basic support for Cardigann definitions

* Add HDME definition

* Fix tests

* support split with negative indexes

* allow FromTimeAgo formats without spaces betwen value and unit

Example: 2h 3m

* Add basic support for Cardigann definitions

* Add HDME definition

* Fix tests

* support split with negative indexes

* allow FromTimeAgo formats without spaces betwen value and unit

Example: 2h 3m
This commit is contained in:
kaso17 2016-10-27 09:30:03 +02:00 committed by flightlevel
parent fa9bbaa18c
commit 21cffe2d35
13 changed files with 987 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/Jackett.Test/TestIIndexerManagerServiceHelper.cs
View File

@ -42,5 +42,10 @@ namespace JackettTest
{
throw new NotImplementedException();
}
public void InitCardigannIndexers(string path)
{
throw new NotImplementedException();
}
}
}

101
src/Jackett/Definitions/hdme.yml Normal file
View File

@ -0,0 +1,101 @@
---
site: hdme
name: HDME
language: en-us
links:
- https://hdme.eu
caps:
categories:
24: TV/Anime # Anime
25: PC/0day # Appz
47: Movies/HD # AVCHD
26: Movies/BluRay # Bluray
54: Movies/HD # dbREMUX
41: Movies/HD # Documentaries
50: Movies/HD # FourGHD
44: Movies/HD # HDME
28: Audio/Lossless # HQ Music
48: Movies/HD # iCandy
45: Movies/HD # INtL
29: Other # Misc
49: PC/Phone-Other # Mobile
30: Movies/HD # Movie 1080i
31: Movies/HD # Movie 1080p
32: Movies/HD # Movie 720p
33: Audio/Video # Music Videos
34: TV # Packs
53: Movies/HD # Remux
56: Movies/HD # RUXi
55: Movies/HD # SiNiSteR
36: TV/Sport # Sports
37: TV/HD # TV Series 1080i
38: TV/HD # TV Series 1080p
39: TV/HD # TV Series 720p
57: Movies # UHD 2160p
40: XXX # XXX
modes:
search: [q]
tv-search: [q, season, ep]
login:
path: /takelogin.php
method: post
form: form
inputs:
username: "{{ .Config.username }}"
password: "{{ .Config.password }}"
error:
- selector: td.embedded
message:
selector: td.text
test:
path: /my.php
ratio:
path: /my.php
selector: span.smallfont > font
filters:
- name: regexp
args: "Ratio:(.+?)Uploaded"
- name: replace
args: [",", ""]
search:
path: /browse.php
inputs:
$raw: "{{range .Categories}}c{{.}}=1&{{end}}"
search: "{{ .Keywords }}"
incldead: "1"
blah: "0"
rows:
selector: table[width="100%"] > tbody > tr:has(td.bottom[background="_images/bg_torrent.jpg"])
fields:
category:
selector: td:nth-child(2) a
attribute: href
filters:
- name: querystring
args: cat
title:
selector: td:nth-child(3) > a
attribute: title
comments:
selector: td:nth-child(3) > a
attribute: href
download:
selector: td:nth-child(11) > a
attribute: href
size:
selector: td:nth-child(6)
remove: br
date:
selector: td:nth-child(3)
filters:
- name: regexp
args: "Added: (.+?)\n"
seeders:
selector: td:nth-child(8)
leechers:
selector: td:nth-child(9)

24
src/Jackett/Indexers/BaseIndexer.cs
View File

@ -17,13 +17,13 @@ namespace Jackett.Indexers
{
public abstract class BaseIndexer
{
public string SiteLink { get; private set; }
public string DisplayDescription { get; private set; }
public string DisplayName { get; private set; }
public string SiteLink { get; protected set; }
public string DisplayDescription { get; protected set; }
public string DisplayName { get; protected set; }
public string ID { get { return GetIndexerID(GetType()); } }
public bool IsConfigured { get; protected set; }
public TorznabCapabilities TorznabCaps { get; private set; }
public TorznabCapabilities TorznabCaps { get; protected set; }
protected Logger logger;
protected IIndexerManagerService indexerService;
protected static List<CachedQueryResult> cache = new List<CachedQueryResult>();
@ -44,7 +44,9 @@ namespace Jackett.Indexers
private List<CategoryMapping> categoryMapping = new List<CategoryMapping>();
// standard constructor used by most indexers
public BaseIndexer(string name, string link, string description, IIndexerManagerService manager, IWebClient client, Logger logger, ConfigurationData configData, IProtectionService p, TorznabCapabilities caps = null, string downloadBase = null)
: this(manager, client, logger, p)
{
if (!link.EndsWith("/"))
throw new Exception("Site link must end with a slash.");
@ -52,12 +54,7 @@ namespace Jackett.Indexers
DisplayName = name;
DisplayDescription = description;
SiteLink = link;
this.logger = logger;
indexerService = manager;
webclient = client;
protectionService = p;
this.downloadUrlBase = downloadBase;
this.configData = configData;
if (caps == null)
@ -66,6 +63,15 @@ namespace Jackett.Indexers
}
// minimal constructor used by e.g. cardigann generic indexer
public BaseIndexer(IIndexerManagerService manager, IWebClient client, Logger logger, IProtectionService p)
{
this.logger = logger;
indexerService = manager;
webclient = client;
protectionService = p;
}
public IEnumerable<ReleaseInfo> CleanLinks(IEnumerable<ReleaseInfo> releases)
{
if (string.IsNullOrEmpty(downloadUrlBase))

636
src/Jackett/Indexers/CardigannIndexer.cs Normal file
View File

@ -0,0 +1,636 @@
using Jackett.Utils.Clients;
using NLog;
using Jackett.Services;
using Jackett.Utils;
using Jackett.Models;
using System.Threading.Tasks;
using Newtonsoft.Json.Linq;
using System.Collections.Generic;
using System;
using Jackett.Models.IndexerConfig;
using System.Collections.Specialized;
using System.Text;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions;
using static Jackett.Models.IndexerConfig.ConfigurationData;
using AngleSharp.Parser.Html;
using System.Text.RegularExpressions;
using System.Web;
namespace Jackett.Indexers
{
public class CardigannIndexer : BaseIndexer, IIndexer
{
protected IndexerDefinition Definition;
public new string ID { get { return (Definition != null ? Definition.Site : GetIndexerID(GetType())); } }
new ConfigurationData configData
{
get { return (ConfigurationData)base.configData; }
set { base.configData = value; }
}
// Cardigann yaml classes
public class IndexerDefinition {
public string Site { get; set; }
public List<settingsField> Settings { get; set; }
public string Name { get; set; }
public string Description { get; set; }
public string Language { get; set; }
public List<string> Links { get; set; }
public capabilitiesBlock Caps { get; set; }
public loginBlock Login { get; set; }
public ratioBlock Ratio { get; set; }
public searchBlock Search { get; set; }
// IndexerDefinitionStats not needed/implemented
}
public class settingsField
{
public string Name { get; set; }
public string Type { get; set; }
public string Label { get; set; }
}
public class capabilitiesBlock
{
public Dictionary<string, string> Categories { get; set; }
public Dictionary<string, List<string>> Modes { get; set; }
}
public class loginBlock
{
public string Path { get; set; }
public string Method { get; set; }
public string Form { get; set; }
public Dictionary<string, string> Inputs { get; set; }
public List<errorBlock> Error { get; set; }
public pageTestBlock Test { get; set; }
}
public class errorBlock
{
public string Path { get; set; }
public string Selector { get; set; }
public selectorBlock Message { get; set; }
}
public class selectorBlock
{
public string Selector { get; set; }
public string Text { get; set; }
public string Attribute { get; set; }
public string Remove { get; set; }
public List<filterBlock> Filters { get; set; }
public Dictionary<string, string> Case { get; set; }
}
public class filterBlock
{
public string Name { get; set; }
public dynamic Args { get; set; }
}
public class pageTestBlock
{
public string Path { get; set; }
public string Selector { get; set; }
}
public class ratioBlock : selectorBlock
{
public string Path { get; set; }
}
public class searchBlock
{
public string Path { get; set; }
public Dictionary<string, string> Inputs { get; set; }
public rowsBlock Rows { get; set; }
public Dictionary<string, selectorBlock> Fields { get; set; }
}
public class rowsBlock : selectorBlock
{
public int After { get; set; }
//public string Remove { get; set; } // already inherited
public string Dateheaders { get; set; }
}
public CardigannIndexer(IIndexerManagerService i, IWebClient wc, Logger l, IProtectionService ps)
: base(manager: i,
client: wc,
logger: l,
p: ps)
{
}
public CardigannIndexer(IIndexerManagerService i, IWebClient wc, Logger l, IProtectionService ps, string DefinitionString)
: base(manager: i,
client: wc,
logger: l,
p: ps)
{
Init(DefinitionString);
}
protected void Init(string DefinitionString)
{
var deserializer = new DeserializerBuilder()
.WithNamingConvention(new CamelCaseNamingConvention())
.IgnoreUnmatchedProperties()
.Build();
Definition = deserializer.Deserialize<IndexerDefinition>(DefinitionString);
// Add default data if necessary
if (Definition.Settings == null)
Definition.Settings = new List<settingsField>();
if (Definition.Settings.Count == 0)
{
Definition.Settings.Add(new settingsField { Name = "username", Label = "Username", Type = "text" });
Definition.Settings.Add(new settingsField { Name = "password", Label = "Password", Type = "password" });
}
// init missing mandatory attributes
DisplayName = Definition.Name;
DisplayDescription = Definition.Description;
SiteLink = Definition.Links[0]; // TODO: implement alternative links
if (!SiteLink.EndsWith("/"))
SiteLink += "/";
TorznabCaps = TorznabUtil.CreateDefaultTorznabTVCaps(); // TODO implement caps
// init config Data
configData = new ConfigurationData();
foreach (var Setting in Definition.Settings)
{
configData.AddDynamic(Setting.Name, new StringItem { Name = Setting.Label });
}
foreach (var Category in Definition.Caps.Categories)
{
var cat = TorznabCatType.GetCatByName(Category.Value);
if (cat == null)
{
logger.Error(string.Format("CardigannIndexer ({0}): Can't find a category for {1}", ID, Category.Value));
continue;
}
AddCategoryMapping(Category.Key, TorznabCatType.GetCatByName(Category.Value));
}
}
protected Dictionary<string, object> getTemplateVariablesFromConfigData()
{
Dictionary<string, object> variables = new Dictionary<string, object>();
foreach (settingsField Setting in Definition.Settings)
{
variables[".Config."+Setting.Name] = ((StringItem)configData.GetDynamic(Setting.Name)).Value;
}
return variables;
}
// A very bad implementation of the golang template/text templating engine.
// But it should work for most basic constucts used by Cardigann definitions.
protected string applyGoTemplateText(string template, Dictionary<string, object> variables = null)
{
if (variables == null)
{
variables = getTemplateVariablesFromConfigData();
}
// handle if ... else ... expression
Regex IfElseRegex = new Regex(@"{{if\s*(.+?)\s*}}(.*?){{\s*else\s*}}(.*?){{\s*end\s*}}");
var IfElseRegexMatches = IfElseRegex.Match(template);
while (IfElseRegexMatches.Success)
{
string conditionResult = null;
string all = IfElseRegexMatches.Groups[0].Value;
string condition = IfElseRegexMatches.Groups[1].Value;
string onTrue = IfElseRegexMatches.Groups[2].Value;
string onFalse = IfElseRegexMatches.Groups[3].Value;
if (condition.StartsWith("."))
{
string value = (string)variables[condition];
if (!string.IsNullOrWhiteSpace(value))
{
conditionResult = onTrue;
}
else
{
conditionResult = onFalse;
}
}
else
{
throw new NotImplementedException("CardigannIndexer: Condition operation '" + condition + "' not implemented");
}
template = template.Replace(all, conditionResult);
IfElseRegexMatches = IfElseRegexMatches.NextMatch();
}
// handle range expression
Regex RangeRegex = new Regex(@"{{\s*range\s*(.+?)\s*}}(.*?){{\.}}(.*?){{end}}");
var RangeRegexMatches = RangeRegex.Match(template);
while (RangeRegexMatches.Success)
{
string expanded = string.Empty;
string all = RangeRegexMatches.Groups[0].Value;
string variable = RangeRegexMatches.Groups[1].Value;
string prefix = RangeRegexMatches.Groups[2].Value;
string postfix = RangeRegexMatches.Groups[3].Value;
foreach (string value in (List<string>)variables[variable])
{
expanded += prefix + value + postfix;
}
template = template.Replace(all, expanded);
RangeRegexMatches = RangeRegexMatches.NextMatch();
}
// handle simple variables
Regex VariablesRegEx = new Regex(@"{{\s*(\..+?)\s*}}");
var VariablesRegExMatches = VariablesRegEx.Match(template);
while (VariablesRegExMatches.Success)
{
string expanded = string.Empty;
string all = VariablesRegExMatches.Groups[0].Value;
string variable = VariablesRegExMatches.Groups[1].Value;
string value = (string)variables[variable];
template = template.Replace(all, value);
VariablesRegExMatches = VariablesRegExMatches.NextMatch();
}
return template;
}
protected async Task<bool> DoLogin()
{
var Login = Definition.Login;
if (Login == null)
return false;
if (Login.Method == "post")
{
var pairs = new Dictionary<string, string>();
foreach (var Input in Definition.Login.Inputs)
{
var value = applyGoTemplateText(Input.Value);
pairs.Add(Input.Key, value);
}
foreach (var x in pairs)
{
logger.Error(x.Key + ": " + x.Value);
}
var LoginUrl = SiteLink + Login.Path;
configData.CookieHeader.Value = null;
var loginResult = await RequestLoginAndFollowRedirect(LoginUrl, pairs, null, true, null, SiteLink, true);
configData.CookieHeader.Value = loginResult.Cookies;
if (Login.Error != null)
{
var loginResultParser = new HtmlParser();
var loginResultDocument = loginResultParser.Parse(loginResult.Content);
foreach (errorBlock error in Login.Error)
{
var selection = loginResultDocument.QuerySelector(error.Selector);
if (selection != null)
{
string errorMessage = selection.TextContent;
if (error.Message != null)
{
var errorSubMessage = loginResultDocument.QuerySelector(error.Message.Selector);
errorMessage = errorSubMessage.TextContent;
}
throw new ExceptionWithConfigData(string.Format("Login failed: {0}", errorMessage.Trim()), configData);
}
}
}
}
else if (Login.Method == "cookie")
{
configData.CookieHeader.Value = ((StringItem)configData.GetDynamic("cookie")).Value;
}
else
{
throw new NotImplementedException("Login method " + Definition.Login.Method + " not implemented");
}
return true;
}
protected async Task<bool> TestLogin()
{
var Login = Definition.Login;
if (Login == null || Login.Test == null)
return false;
// test if login was successful
var LoginTestUrl = SiteLink + Login.Test.Path;
var testResult = await RequestStringWithCookies(LoginTestUrl);
if (testResult.IsRedirect)
{
throw new ExceptionWithConfigData("Login Failed, got redirected", configData);
}
if (Login.Test.Selector != null)
{
var testResultParser = new HtmlParser();
var testResultDocument = testResultParser.Parse(testResult.Content);
var selection = testResultDocument.QuerySelectorAll(Login.Test.Selector);
if (selection.Length == 0)
{
throw new ExceptionWithConfigData(string.Format("Login failed: Selector \"{0}\" didn't match", Login.Test.Selector), configData);
}
}
return true;
}
public async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
{
configData.LoadValuesFromJson(configJson);
await DoLogin();
await TestLogin();
SaveConfig();
IsConfigured = true;
return IndexerConfigurationStatus.Completed;
}
protected string applyFilters(string Data, List<filterBlock> Filters)
{
if (Filters == null)
return Data;
foreach(filterBlock Filter in Filters)
{
switch (Filter.Name)
{
case "querystring":
var param = (string)Filter.Args;
var qsStr = Data.Split(new char[] { '?' }, 2)[1];
qsStr = Data.Split(new char[] { '#' }, 2)[0];
var qs = HttpUtility.ParseQueryString(qsStr);
Data = qs.Get(param);
break;
case "timeparse":
case "dateparse":
throw new NotImplementedException("Filter " + Filter.Name + " not implemented");
/*
TODO: implement golang time format conversion, see http://fuckinggodateformat.com/
if args == nil {
return filterDateParse(nil, value)
}
if layout, ok := args.(string); ok {
return filterDateParse([]string{layout}, value)
}
return "", fmt.Errorf("Filter argument type %T was invalid", args)
*/
break;
case "regexp":
var pattern = (string)Filter.Args;
var Regexp = new Regex(pattern);
var Match = Regexp.Match(Data);
Data = Match.Groups[1].Value;
break;
case "split":
var sep = (string)Filter.Args[0];
var pos = (string)Filter.Args[1];
var posInt = int.Parse(pos);
var strParts = Data.Split(sep[0]);
if (posInt < 0)
{
posInt += strParts.Length;
}
Data = strParts[posInt];
break;
case "replace":
var from = (string)Filter.Args[0];
var to = (string)Filter.Args[1];
Data = Data.Replace(from, to);
break;
case "trim":
var cutset = (string)Filter.Args;
Data = Data.Trim(cutset[0]);
break;
case "append":
var str = (string)Filter.Args;
Data += str;
break;
case "timeago":
case "fuzzytime":
case "reltime":
var timestr = (string)Filter.Args;
Data = DateTimeUtil.FromUnknown(timestr).ToString(DateTimeUtil.RFC1123ZPattern);
break;
default:
break;
}
}
return Data;
}
protected string handleSelector(selectorBlock Selector, AngleSharp.Dom.IElement Dom)
{
if (Selector.Text != null)
{
return applyFilters(Selector.Text, Selector.Filters);
}
string value = null;
if (Selector.Selector != null)
{
AngleSharp.Dom.IElement selection = Dom.QuerySelector(Selector.Selector);
if (selection == null)
{
throw new Exception(string.Format("Selector \"{0}\" didn't match {1}", Selector.Selector, Dom.OuterHtml));
}
if (Selector.Remove != null)
{
foreach(var i in selection.QuerySelectorAll(Selector.Remove))
{
i.Remove();
}
}
if (Selector.Attribute != null)
{
value = selection.GetAttribute(Selector.Attribute);
}
else
{
value = selection.TextContent;
}
}
return applyFilters(value, Selector.Filters); ;
}
protected Uri resolvePath(string path)
{
return new Uri(SiteLink + path);
}
public async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
{
var releases = new List<ReleaseInfo>();
searchBlock Search = Definition.Search;
// init template context
var variables = getTemplateVariablesFromConfigData();
variables[".Query.Type"] = query.QueryType;
variables[".Query.Q"] = query.SearchTerm;
variables[".Query.Series"] = null;
variables[".Query.Ep"] = query.Episode;
variables[".Query.Season"] = query.Season;
variables[".Query.Movie"] = null;
variables[".Query.Year"] = null;
variables[".Query.Limit"] = query.Limit;
variables[".Query.Offset"] = query.Offset;
variables[".Query.Extended"] = query.Extended;
variables[".Query.Categories"] = query.Categories;
variables[".Query.APIKey"] = query.ApiKey;
variables[".Query.TVDBID"] = null;
variables[".Query.TVRageID"] = query.RageID;
variables[".Query.IMDBID"] = query.ImdbID;
variables[".Query.TVMazeID"] = null;
variables[".Query.TraktID"] = null;
variables[".Query.Episode"] = query.GetEpisodeSearchString();
variables[".Categories"] = MapTorznabCapsToTrackers(query);
var KeywordTokens = new List<string>();
var KeywordTokenKeys = new List<string> { "Q", "Series", "Movie", "Year" };
foreach (var key in KeywordTokenKeys)
{
var Value = (string)variables[".Query." + key];
if (!string.IsNullOrWhiteSpace(Value))
KeywordTokens.Add(Value);
}
if (!string.IsNullOrWhiteSpace((string)variables[".Query.Episode"]))
KeywordTokens.Add((string)variables[".Query.Episode"]);
variables[".Query.Keywords"] = string.Join(" ", KeywordTokens);
variables[".Keywords"] = variables[".Query.Keywords"];
// build search URL
var searchUrl = SiteLink + applyGoTemplateText(Search.Path, variables) + "?";
var queryCollection = new NameValueCollection();
if (Search.Inputs != null)
{
foreach (var Input in Search.Inputs)
{
var value = applyGoTemplateText(Input.Value, variables);
if (Input.Key == "$raw")
searchUrl += value;
else
queryCollection.Add(Input.Key, value);
}
}
searchUrl += "&" + queryCollection.GetQueryString();
// send HTTP request
var response = await RequestBytesWithCookies(searchUrl);
var results = Encoding.GetEncoding("iso-8859-1").GetString(response.Content);
try
{
var SearchResultParser = new HtmlParser();
var SearchResultDocument = SearchResultParser.Parse(results);
var Rows = SearchResultDocument.QuerySelectorAll(Search.Rows.Selector);
foreach (var Row in Rows)
{
try
{
var release = new ReleaseInfo();
release.MinimumRatio = 1;
release.MinimumSeedTime = 48 * 60 * 60;
// Parse fields
foreach (var Field in Search.Fields)
{
string value = handleSelector(Field.Value, Row);
try
{
switch (Field.Key)
{
case "download":
release.Link = resolvePath(value);
break;
case "details":
var url = resolvePath(value);
release.Guid = url;
if (release.Comments == null)
release.Comments = url;
break;
case "comments":
release.Comments = resolvePath(value);
break;
case "title":
release.Title = value;
break;
case "description":
release.Description = value;
break;
case "category":
release.Category = MapTrackerCatToNewznab(value);
break;
case "size":
release.Size = ReleaseInfo.GetBytes(value);
break;
case "leechers":
if (release.Peers == null)
release.Peers = ParseUtil.CoerceInt(value);
else
release.Peers += ParseUtil.CoerceInt(value);
break;
case "seeders":
release.Seeders = ParseUtil.CoerceInt(value);
if (release.Peers == null)
release.Peers = release.Seeders;
else
release.Peers += release.Seeders;
break;
case "date":
release.PublishDate = DateTimeUtil.FromUnknown(value);
break;
default:
break;
}
}
catch (Exception ex)
{
throw new Exception(string.Format("Error while parsing field={0}, selector={1}, value={2}: {3}", Field.Key, Field.Value.Selector, value, ex.Message));
}
}
releases.Add(release);
}
catch (Exception ex)
{
logger.Error(string.Format("CardigannIndexer ({0}): Error while parsing row '{1}': {2}", ID, Row.OuterHtml, ex.Message));
}
}
}
catch (Exception ex)
{
OnParseError(results, ex);
}
return releases;
}
}
}

13
src/Jackett/Jackett.csproj
View File

@ -54,6 +54,10 @@
<HintPath>..\packages\CloudFlareUtilities.0.3.2-alpha\lib\portable45-net45+win8+wpa81\CloudFlareUtilities.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="DateTimeRoutines, Version=1.0.0.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\DateTimeRoutines.1.0.16\lib\net40\DateTimeRoutines.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System" />
<Reference Include="System.Configuration.Install" />
<Reference Include="System.Core" />
@ -147,6 +151,10 @@
<Reference Include="System.Web.Http.Tracing, Version=5.2.3.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35">
<HintPath>..\packages\Microsoft.AspNet.WebApi.Tracing.5.2.3\lib\net45\System.Web.Http.Tracing.dll</HintPath>
</Reference>
<Reference Include="YamlDotNet, Version=4.0.0.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\YamlDotNet.4.0.1-pre288\lib\net35\YamlDotNet.dll</HintPath>
<Private>True</Private>
</Reference>
</ItemGroup>
<ItemGroup>
<Compile Include="AuthenticationException.cs" />
@ -156,6 +164,7 @@
<Compile Include="Controllers\TorznabController.cs" />
<Compile Include="Controllers\DownloadController.cs" />
<Compile Include="Engine.cs" />
<Compile Include="Indexers\CardigannIndexer.cs" />
<Compile Include="Indexers\myAmity.cs" />
<Compile Include="Indexers\TorrentNetwork.cs" />
<Compile Include="Indexers\Andraste.cs" />
@ -361,6 +370,9 @@
<None Include="CurlSharp.dll.config">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Include="Definitions\hdme.yml">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Include="packages.config">
<SubType>Designer</SubType>
</None>
@ -687,6 +699,7 @@
<ItemGroup>
<Service Include="{508349B6-6B84-4DF5-91F0-309BEEBAD82D}" />
</ItemGroup>
<ItemGroup />
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.

15
src/Jackett/Models/IndexerConfig/ConfigurationData.cs
View File

@ -10,9 +10,10 @@ using System.Threading.Tasks;
namespace Jackett.Models.IndexerConfig
{
public abstract class ConfigurationData
public class ConfigurationData
{
const string PASSWORD_REPLACEMENT = "|||%%PREVJACKPASSWD%%|||";
protected Dictionary<string, Item> dynamics = new Dictionary<string, Item>(); // list for dynamic items
public enum ItemType
{
@ -132,6 +133,8 @@ namespace Jackett.Models.IndexerConfig
.Where(p => p.PropertyType.IsSubclassOf(typeof(Item)))
.Select(p => (Item)p.GetValue(this));
properties = properties.Concat(dynamics.Values).ToArray();
if (!forDisplay)
{
properties = properties
@ -142,6 +145,16 @@ namespace Jackett.Models.IndexerConfig
return properties.ToArray();
}
public void AddDynamic(string ID, Item item)
{
dynamics.Add(ID, item);
}
public Item GetDynamic(string ID)
{
return dynamics[ID];
}
public class Item
{
public ItemType ItemType { get; set; }

16
src/Jackett/Models/TorznabCatType.cs
View File

@ -27,5 +27,21 @@ namespace Jackett.Models
return string.Empty;
}
public static string NormalizeCatName(string name)
{
return name.Replace(" ", "").ToLower();
}
public static TorznabCategory GetCatByName(string name)
{
var cat = AllCats.FirstOrDefault(c => NormalizeCatName(c.Name) == NormalizeCatName(name));
if (cat != null)
{
return cat;
}
return null;
}
}
}

17
src/Jackett/Services/ConfigurationService.cs
View File

@ -25,6 +25,7 @@ namespace Jackett.Services
T GetConfig<T>();
void SaveConfig<T>(T config);
string ApplicationFolder();
string GetCardigannDefinitionsFolder();
void CreateOrMigrateSettings();
void PerformMigration();
}
@ -193,6 +194,22 @@ namespace Jackett.Services
{
dir = sourcePath;
}
#endif
return dir;
}
public string GetCardigannDefinitionsFolder()
{
// If we are debugging we can use the non copied definitions.
string dir = Path.Combine(ApplicationFolder(), "Definitions"); ;
#if DEBUG
// When we are running in debug use the source files
var sourcePath = Path.GetFullPath(Path.Combine(ApplicationFolder(), "..\\..\\..\\Jackett\\Definitions"));
if (Directory.Exists(sourcePath))
{
dir = sourcePath;
}
#endif
return dir;
}

58
src/Jackett/Services/IndexerManagerService.cs
View File

@ -22,6 +22,7 @@ namespace Jackett.Services
IEnumerable<IIndexer> GetAllIndexers();
void SaveConfig(IIndexer indexer, JToken obj);
void InitIndexers();
void InitCardigannIndexers(string path);
}
public class IndexerManagerService : IIndexerManagerService
@ -40,27 +41,54 @@ namespace Jackett.Services
cacheService = cache;
}
protected void LoadIndexerConfig(IIndexer idx)
{
var configFilePath = GetIndexerConfigFilePath(idx);
if (File.Exists(configFilePath))
{
var fileStr = File.ReadAllText(configFilePath);
var jsonString = JToken.Parse(fileStr);
try
{
idx.LoadFromSavedConfiguration(jsonString);
}
catch (Exception ex)
{
logger.Error(ex, "Failed loading configuration for {0}, you must reconfigure this indexer", idx.DisplayName);
}
}
}
public void InitIndexers()
{
logger.Info("Using HTTP Client: " + container.Resolve<IWebClient>().GetType().Name);
foreach (var idx in container.Resolve<IEnumerable<IIndexer>>().OrderBy(_ => _.DisplayName))
foreach (var idx in container.Resolve<IEnumerable<IIndexer>>().Where(p => p.ID != "cardigannindexer").OrderBy(_ => _.DisplayName))
{
indexers.Add(idx.ID, idx);
var configFilePath = GetIndexerConfigFilePath(idx);
if (File.Exists(configFilePath))
{
var fileStr = File.ReadAllText(configFilePath);
var jsonString = JToken.Parse(fileStr);
try
{
idx.LoadFromSavedConfiguration(jsonString);
}
catch (Exception ex)
{
logger.Error(ex, "Failed loading configuration for {0}, you must reconfigure this indexer", idx.DisplayName);
}
}
LoadIndexerConfig(idx);
}
}
public void InitCardigannIndexers(string path)
{
logger.Info("Loading Cardigann definitions from: " + path);
DirectoryInfo d = new DirectoryInfo(path);
foreach (var file in d.GetFiles("*.yml"))
{
string DefinitionString = File.ReadAllText(file.FullName);
CardigannIndexer idx = new CardigannIndexer(this, container.Resolve<IWebClient>(), logger, container.Resolve<IProtectionService>(), DefinitionString);
if (indexers.ContainsKey(idx.ID))
{
logger.Debug(string.Format("Ignoring definition ID={0}, file={1}: Indexer already exists", idx.ID, file.FullName));
}
else
{
indexers.Add(idx.ID, idx);
LoadIndexerConfig(idx);
}
}
}

1
src/Jackett/Services/ServerService.cs
View File

@ -150,6 +150,7 @@ namespace Jackett.Services
CultureInfo.DefaultThreadCurrentCulture = new CultureInfo("en-US");
// Load indexers
indexerService.InitIndexers();
indexerService.InitCardigannIndexers(configService.GetCardigannDefinitionsFolder());
client.Init();
}

118
src/Jackett/Utils/DateTimeUtil.cs
View File

@ -1,13 +1,17 @@
using System;
using Cliver;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace Jackett.Utils
{
public static class DateTimeUtil
{
public static string RFC1123ZPattern = "ddd, dd MMM yyyy HH':'mm':'ss z";
public static DateTime UnixTimestampToDateTime(double unixTime)
{
DateTime unixStart = new DateTime(1970, 1, 1, 0, 0, 0, 0, System.DateTimeKind.Utc);
@ -32,33 +36,117 @@ namespace Jackett.Utils
return DateTime.SpecifyKind(DateTime.Now, DateTimeKind.Local);
}
var dateParts = str.Split(new char[0], StringSplitOptions.RemoveEmptyEntries);
str = str.Replace(",", "");
str = str.Replace("ago", "");
str = str.Replace("and", "");
TimeSpan timeAgo = TimeSpan.Zero;
for (var i = 0; i < dateParts.Length / 2; i++)
{
var val = ParseUtil.CoerceFloat(dateParts[i * 2]);
var unit = dateParts[i * 2 + 1];
if (unit.Contains("sec"))
Regex TimeagoRegex = new Regex(@"\s*?([\d\.]+)\s*?([^\d\s\.]+)\s*?");
var TimeagoMatches = TimeagoRegex.Match(str);
while (TimeagoMatches.Success)
{
string expanded = string.Empty;
var val = ParseUtil.CoerceFloat(TimeagoMatches.Groups[1].Value);
var unit = TimeagoMatches.Groups[2].Value;
TimeagoMatches = TimeagoMatches.NextMatch();
if (unit.Contains("sec") || unit == "s")
timeAgo += TimeSpan.FromSeconds(val);
else if (unit.Contains("min"))
else if (unit.Contains("min") || unit == "m")
timeAgo += TimeSpan.FromMinutes(val);
else if (unit.Contains("hour") || unit.Contains("hr"))
else if (unit.Contains("hour") || unit.Contains("hr") || unit == "h")
timeAgo += TimeSpan.FromHours(val);
else if (unit.Contains("day"))
else if (unit.Contains("day") ||unit == "d")
timeAgo += TimeSpan.FromDays(val);
else if (unit.Contains("week") || unit.Contains("wk"))
else if (unit.Contains("week") || unit.Contains("wk") || unit == "w")
timeAgo += TimeSpan.FromDays(val * 7);
else if (unit.Contains("month"))
else if (unit.Contains("month") || unit == "mo")
timeAgo += TimeSpan.FromDays(val * 30);
else if (unit.Contains("year"))
else if (unit.Contains("year") || unit == "y")
timeAgo += TimeSpan.FromDays(val * 365);
else
{
throw new Exception("TimeAgo parsing failed");
{
throw new Exception("TimeAgo parsing failed, unknown unit: "+unit);
}
}
return DateTime.SpecifyKind(DateTime.Now - timeAgo, DateTimeKind.Local);
}
// Uses the DateTimeRoutines library to parse the date
// http://www.codeproject.com/Articles/33298/C-Date-Time-Parser
public static DateTime FromFuzzyTime(string str, DateTimeRoutines.DateTimeFormat format = DateTimeRoutines.DateTimeFormat.USA_DATE)
{
DateTimeRoutines.ParsedDateTime dt;
if (DateTimeRoutines.TryParseDateOrTime(str, format, out dt))
{
return dt.DateTime;
}
throw new Exception("FromFuzzyTime parsing failed");
}
public static Regex timeAgoRegexp = new Regex(@"(?i)\bago", RegexOptions.Compiled);
public static Regex todayRegexp = new Regex(@"(?i)\btoday([\s,]+|$)", RegexOptions.Compiled);
public static Regex tomorrowRegexp = new Regex(@"(?i)\btomorrow([\s,]+|$)", RegexOptions.Compiled);
public static Regex yesterdayRegexp = new Regex(@"(?i)\byesterday([\s,]+|$)", RegexOptions.Compiled);
public static Regex missingYearRegexp = new Regex(@"^\d{1,2}-\d{1,2}\b", RegexOptions.Compiled);
public static DateTime FromUnknown(string str)
{
str = ParseUtil.NormalizeSpace(str);
Match match;
// ... ago
match = timeAgoRegexp.Match(str);
if (match.Success)
{
var timeago = str;
return FromTimeAgo(timeago);
}
// Today ...
match = todayRegexp.Match(str);
if (match.Success)
{
var time = str.Replace(match.Groups[0].Value, "");
DateTime dt = DateTime.SpecifyKind(DateTime.UtcNow.Date, DateTimeKind.Unspecified);
dt += TimeSpan.Parse(time);
return dt;
}
// Yesterday ...
match = yesterdayRegexp.Match(str);
if (match.Success)
{
var time = str.Replace(match.Groups[0].Value, "");
DateTime dt = DateTime.SpecifyKind(DateTime.UtcNow.Date, DateTimeKind.Unspecified);
dt += TimeSpan.Parse(time);
dt -= TimeSpan.FromDays(1);
return dt;
}
// Tomorrow ...
match = tomorrowRegexp.Match(str);
if (match.Success)
{
var time = str.Replace(match.Groups[0].Value, "");
DateTime dt = DateTime.SpecifyKind(DateTime.UtcNow.Date, DateTimeKind.Unspecified);
dt += TimeSpan.Parse(time);
dt += TimeSpan.FromDays(1);
return dt;
}
// add missing year
match = missingYearRegexp.Match(str);
if (match.Success)
{
var date = match.Groups[0].Value;
string newDate = date+"-"+DateTime.Now.Year.ToString();
str = str.Replace(date, newDate);
}
return FromFuzzyTime(str);
}
}
}

29
src/Jackett/Utils/ParseUtil.cs
View File

@ -9,45 +9,58 @@ namespace Jackett.Utils
{
public static class ParseUtil
{
public static string NormalizeSpace(string s)
{
return s.Trim();
}
public static string NormalizeNumber(string s)
{
string normalized = NormalizeSpace(s);
normalized = normalized.Replace("-", "0");
normalized = normalized.Replace(",", "");
return normalized;
}
public static double CoerceDouble(string str)
{
return double.Parse(str.Trim(), NumberStyles.Any, CultureInfo.InvariantCulture);
return double.Parse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture);
}
public static float CoerceFloat(string str)
{
return float.Parse(str.Trim(), NumberStyles.Any, CultureInfo.InvariantCulture);
return float.Parse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture);
}
public static int CoerceInt(string str)
{
return int.Parse(str.Trim(), NumberStyles.Any, CultureInfo.InvariantCulture);
return int.Parse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture);
}
public static long CoerceLong(string str)
{
return long.Parse(str.Trim(), NumberStyles.Any, CultureInfo.InvariantCulture);
return long.Parse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture);
}
public static bool TryCoerceDouble(string str, out double result)
{
return double.TryParse(str.Trim(), NumberStyles.Any, CultureInfo.InvariantCulture, out result);
return double.TryParse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture, out result);
}
public static bool TryCoerceFloat(string str, out float result)
{
return float.TryParse(str.Trim(), NumberStyles.Any, CultureInfo.InvariantCulture, out result);
return float.TryParse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture, out result);
}
public static bool TryCoerceInt(string str, out int result)
{
return int.TryParse(str.Trim(), NumberStyles.Any, CultureInfo.InvariantCulture, out result);
return int.TryParse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture, out result);
}
public static bool TryCoerceLong(string str, out long result)
{
return long.TryParse(str.Trim(), NumberStyles.Any, CultureInfo.InvariantCulture, out result);
return long.TryParse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture, out result);
}
}

2
src/Jackett/packages.config
View File

@ -9,6 +9,7 @@
<package id="AutoMapper" version="4.1.1" targetFramework="net45" />
<package id="CloudFlareUtilities" version="0.3.2-alpha" targetFramework="net45" />
<package id="CsQuery" version="1.3.4" targetFramework="net45" />
<package id="DateTimeRoutines" version="1.0.16" targetFramework="net45" />
<package id="Microsoft.AspNet.Identity.Core" version="2.2.1" targetFramework="net45" />
<package id="Microsoft.AspNet.WebApi.Client" version="5.2.3" targetFramework="net45" />
<package id="Microsoft.AspNet.WebApi.Core" version="5.2.3" targetFramework="net45" />
@ -30,4 +31,5 @@
<package id="NLog.Windows.Forms" version="4.2.3" targetFramework="net45" />
<package id="Owin" version="1.0" targetFramework="net45" />
<package id="SharpZipLib" version="0.86.0" targetFramework="net45" />
<package id="YamlDotNet" version="4.0.1-pre288" targetFramework="net45" />
</packages>