From 27d4f2108e810caf185e89a7c5cd423c4d0de0b9 Mon Sep 17 00:00:00 2001 From: chibidev Date: Sun, 14 May 2017 18:55:36 +0200 Subject: [PATCH] Feature/aggregate performance improvement (#1349) * Checking capabilities before executing a query * Introduce base class for meta indexers * Build fix - I seriously do not know how I missed that * Moving things to the appropriate place * Simplifying things as much as possible and moving once again * Build fix? --- src/Jackett/Indexers/AggregateIndexer.cs | 99 ------------------- src/Jackett/Indexers/BaseIndexer.cs | 20 ++++ src/Jackett/Indexers/IIndexer.cs | 2 + src/Jackett/Indexers/Meta/BaseMetaIndexer.cs | 85 ++++++++++++++++ src/Jackett/Indexers/Meta/MetaIndexers.cs | 21 ++++ src/Jackett/Jackett.csproj | 6 +- src/Jackett/JackettModule.cs | 5 +- src/Jackett/Models/TorznabCapabilities.cs | 16 +++ src/Jackett/Models/TorznabQuery.cs | 48 +++++++++ src/Jackett/Services/IndexerManagerService.cs | 5 +- 10 files changed, 202 insertions(+), 105 deletions(-) delete mode 100644 src/Jackett/Indexers/AggregateIndexer.cs create mode 100644 src/Jackett/Indexers/Meta/BaseMetaIndexer.cs create mode 100644 src/Jackett/Indexers/Meta/MetaIndexers.cs diff --git a/src/Jackett/Indexers/AggregateIndexer.cs b/src/Jackett/Indexers/AggregateIndexer.cs deleted file mode 100644 index 15d0e8887..000000000 --- a/src/Jackett/Indexers/AggregateIndexer.cs +++ /dev/null @@ -1,99 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Jackett.Models; -using Newtonsoft.Json.Linq; -using Jackett.Services; -using Jackett.Utils.Clients; -using NLog; - -namespace Jackett.Indexers -{ - class AggregateIndexer : BaseIndexer, IIndexer - { - private IEnumerable Indexers; - public AggregateIndexer(IIndexerManagerService i, IWebClient wc, Logger l, IProtectionService ps) - : base("AggregateSearch", "http://127.0.0.1/", "This feed includes all configured trackers", i, wc, l, new Models.IndexerConfig.ConfigurationData(), ps) - { - } - - public void SetIndexers(IEnumerable indexers) - { - Indexers = indexers.Where(i => i.IsConfigured); - - var caps = new TorznabCapabilities(); - foreach (var indexer in indexers) { - var indexerCaps = indexer.TorznabCaps; - caps.SearchAvailable = caps.SearchAvailable || indexerCaps.SearchAvailable; - caps.TVSearchAvailable = caps.TVSearchAvailable || indexerCaps.TVSearchAvailable; - caps.MovieSearchAvailable = caps.MovieSearchAvailable || indexerCaps.MovieSearchAvailable; - caps.SupportsTVRageSearch = caps.SupportsTVRageSearch || indexerCaps.SupportsTVRageSearch; - caps.SupportsImdbSearch = caps.SupportsImdbSearch || indexerCaps.SupportsImdbSearch; - caps.Categories.AddRange(indexerCaps.Categories.Except (caps.Categories)); - } - - base.TorznabCaps = caps; - base.IsConfigured = true; - } - - public async Task ApplyConfiguration(JToken configJson) - { - return IndexerConfigurationStatus.Completed; - } - - public async Task> PerformQuery(TorznabQuery query) - { - var tasks = new List>>(); - foreach (var indexer in Indexers) - tasks.Add(indexer.PerformQuery(query)); - - var t = Task.WhenAll>(tasks); - try - { - t.Wait(); - } - catch (AggregateException exception) - { - logger.Error(exception, "Error during request from Aggregate"); - } - - IEnumerable result = tasks.Where(x => x.Status == TaskStatus.RanToCompletion).SelectMany(x => x.Result).OrderByDescending(r => r.PublishDate); - // Limiting the response size might be interesting for use-cases where there are - // tons of trackers configured in Jackett. For now just use the limit param if - // someone wants to do that. - if (query.Limit > 0) - result = result.Take(query.Limit); - return result; - } - - public override Uri UncleanLink(Uri link) - { - var indexer = GetOriginalIndexerForLink(link); - if (indexer != null) - return indexer.UncleanLink(link); - - return base.UncleanLink(link); - } - - public override Task Download(Uri link) - { - var indexer = GetOriginalIndexerForLink(link); - if (indexer != null) - return indexer.Download(link); - - return base.Download(link); - } - - private IIndexer GetOriginalIndexerForLink(Uri link) - { - var prefix = string.Format("{0}://{1}", link.Scheme, link.Host); - var validIndexers = Indexers.Where(i => i.SiteLink.StartsWith(prefix)); - if (validIndexers.Count() > 0) - return validIndexers.First(); - - return null; - } - } -} \ No newline at end of file diff --git a/src/Jackett/Indexers/BaseIndexer.cs b/src/Jackett/Indexers/BaseIndexer.cs index e438e1264..05ef60d42 100644 --- a/src/Jackett/Indexers/BaseIndexer.cs +++ b/src/Jackett/Indexers/BaseIndexer.cs @@ -641,5 +641,25 @@ namespace Jackett.Indexers return result.Distinct().ToList(); } + + public bool CanHandleQuery(TorznabQuery query) + { + var caps = TorznabCaps; + if (!caps.SearchAvailable && query.IsSearch) + return false; + if (!caps.TVSearchAvailable && query.IsTVSearch) + return false; + if (!caps.MovieSearchAvailable && query.IsMovieSearch) + return false; + if (!caps.SupportsTVRageSearch && query.IsTVRageSearch) + return false; + if (!caps.SupportsImdbSearch && query.IsImdbQuery) + return false; + + if (query.HasSpecifiedCategories) + if (!caps.SupportsCategories(query.Categories)) + return false; + return true; + } } } diff --git a/src/Jackett/Indexers/IIndexer.cs b/src/Jackett/Indexers/IIndexer.cs index a9f97681a..2c0897246 100644 --- a/src/Jackett/Indexers/IIndexer.cs +++ b/src/Jackett/Indexers/IIndexer.cs @@ -45,5 +45,7 @@ namespace Jackett.Indexers IEnumerable CleanLinks(IEnumerable releases); Uri UncleanLink(Uri link); + + bool CanHandleQuery(TorznabQuery query); } } diff --git a/src/Jackett/Indexers/Meta/BaseMetaIndexer.cs b/src/Jackett/Indexers/Meta/BaseMetaIndexer.cs new file mode 100644 index 000000000..8e80744e8 --- /dev/null +++ b/src/Jackett/Indexers/Meta/BaseMetaIndexer.cs @@ -0,0 +1,85 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Jackett.Models; +using Jackett.Models.IndexerConfig; +using Jackett.Services; +using Jackett.Utils.Clients; +using Newtonsoft.Json.Linq; +using NLog; + +namespace Jackett.Indexers.Meta +{ + public abstract class BaseMetaIndexer : BaseIndexer, IIndexer + { + protected BaseMetaIndexer(string name, string description, IIndexerManagerService manager, Logger logger, ConfigurationData configData, IProtectionService p, Func filter) + : base(name, "http://127.0.0.1/", description, manager, null, logger, configData, p, null, null) + { + filterFunc = filter; + } + + public Task ApplyConfiguration(JToken configJson) + { + return Task.FromResult(IndexerConfigurationStatus.Completed); + } + + public virtual async Task> PerformQuery(TorznabQuery query) + { + var tasks = Indexers.Where(i => i.CanHandleQuery(query)).Select(i => i.PerformQuery(query)).ToList(); // explicit conversion to List to execute LINQ query + var aggregateTask = Task.WhenAll>(tasks); + await aggregateTask; + if (aggregateTask.Exception != null) + logger.Error(aggregateTask.Exception, "Error during request in metaindexer " + ID); + + IEnumerable result = tasks.Where(x => x.Status == TaskStatus.RanToCompletion).SelectMany(x => x.Result).OrderByDescending(r => r.PublishDate); // Ordering by the number of seeders might be useful as well. + // Limiting the response size might be interesting for use-cases where there are + // tons of trackers configured in Jackett. For now just use the limit param if + // someone wants to do that. + if (query.Limit > 0) + result = result.Take(query.Limit); + return result; + } + + public override Uri UncleanLink(Uri link) + { + var indexer = GetOriginalIndexerForLink(link); + if (indexer != null) + return indexer.UncleanLink(link); + + return base.UncleanLink(link); + } + + public override Task Download(Uri link) + { + var indexer = GetOriginalIndexerForLink(link); + if (indexer != null) + return indexer.Download(link); + + return base.Download(link); + } + + private IIndexer GetOriginalIndexerForLink(Uri link) + { + var prefix = string.Format("{0}://{1}", link.Scheme, link.Host); + var validIndexers = Indexers.Where(i => i.SiteLink.StartsWith(prefix, StringComparison.CurrentCulture)); + if (validIndexers.Count() > 0) + return validIndexers.First(); + + return null; + } + + private Func filterFunc; + private IEnumerable indexers; + public IEnumerable Indexers { + get { + return indexers; + } + set { + indexers = value.Where(i => i.IsConfigured && filterFunc(i)); + TorznabCaps = value.Select(i => i.TorznabCaps).Aggregate(new TorznabCapabilities(), TorznabCapabilities.Concat); ; + IsConfigured = true; + } + } + } +} diff --git a/src/Jackett/Indexers/Meta/MetaIndexers.cs b/src/Jackett/Indexers/Meta/MetaIndexers.cs new file mode 100644 index 000000000..15a1391cf --- /dev/null +++ b/src/Jackett/Indexers/Meta/MetaIndexers.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Jackett.Models; +using Newtonsoft.Json.Linq; +using Jackett.Services; +using Jackett.Utils.Clients; +using NLog; + +namespace Jackett.Indexers.Meta +{ + class AggregateIndexer : BaseMetaIndexer, IIndexer + { + public AggregateIndexer(IIndexerManagerService i, Logger l, IProtectionService ps) + : base("AggregateSearch", "This feed includes all configured trackers", i, l, new Models.IndexerConfig.ConfigurationData(), ps, x => true) + { + } + } +} \ No newline at end of file diff --git a/src/Jackett/Jackett.csproj b/src/Jackett/Jackett.csproj index 5249f1aa5..62bd6ce66 100644 --- a/src/Jackett/Jackett.csproj +++ b/src/Jackett/Jackett.csproj @@ -178,7 +178,6 @@ - @@ -371,6 +370,8 @@ + + @@ -932,6 +933,9 @@ + + +