Feature/aggregate performance improvement (#1349)

* Checking capabilities before executing a query

* Introduce base class for meta indexers

* Build fix - I seriously do not know how I missed that

* Moving things to the appropriate place

* Simplifying things as much as possible and moving once again

* Build fix?

src/Jackett/Indexers/AggregateIndexer.cs

@@ -1,99 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using Jackett.Models;
-using Newtonsoft.Json.Linq;
-using Jackett.Services;
-using Jackett.Utils.Clients;
-using NLog;
-
-namespace Jackett.Indexers
-{
-    class AggregateIndexer : BaseIndexer, IIndexer
-    {
-        private IEnumerable<IIndexer> Indexers;
-        public AggregateIndexer(IIndexerManagerService i, IWebClient wc, Logger l, IProtectionService ps)
-            : base("AggregateSearch", "http://127.0.0.1/", "This feed includes all configured trackers", i, wc, l, new Models.IndexerConfig.ConfigurationData(), ps)
-        {
-        }
-
-        public void SetIndexers(IEnumerable<IIndexer> indexers)
-        {
-            Indexers = indexers.Where(i => i.IsConfigured);
-
-            var caps = new TorznabCapabilities();
-            foreach (var indexer in indexers) {
-                var indexerCaps = indexer.TorznabCaps;
-                caps.SearchAvailable = caps.SearchAvailable || indexerCaps.SearchAvailable;
-                caps.TVSearchAvailable = caps.TVSearchAvailable || indexerCaps.TVSearchAvailable;
-                caps.MovieSearchAvailable = caps.MovieSearchAvailable || indexerCaps.MovieSearchAvailable;
-                caps.SupportsTVRageSearch = caps.SupportsTVRageSearch || indexerCaps.SupportsTVRageSearch;
-                caps.SupportsImdbSearch = caps.SupportsImdbSearch || indexerCaps.SupportsImdbSearch;
-                caps.Categories.AddRange(indexerCaps.Categories.Except (caps.Categories));
-            }
-
-            base.TorznabCaps = caps;
-            base.IsConfigured = true;
-        }
-
-        public async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
-        {
-            return IndexerConfigurationStatus.Completed;
-        }
-
-        public async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
-        {
-            var tasks = new List<Task<IEnumerable<ReleaseInfo>>>();
-            foreach (var indexer in Indexers)
-                tasks.Add(indexer.PerformQuery(query));
-
-            var t = Task.WhenAll<IEnumerable<ReleaseInfo>>(tasks);
-            try
-            {
-                t.Wait();
-            }
-            catch (AggregateException exception)
-            {
-                logger.Error(exception, "Error during request from Aggregate");
-            }
-
-            IEnumerable<ReleaseInfo> result = tasks.Where(x => x.Status == TaskStatus.RanToCompletion).SelectMany(x => x.Result).OrderByDescending(r => r.PublishDate);
-            // Limiting the response size might be interesting for use-cases where there are
-            // tons of trackers configured in Jackett. For now just use the limit param if
-            // someone wants to do that.
-            if (query.Limit > 0)
-                result = result.Take(query.Limit);
-            return result;
-        }
-
-        public override Uri UncleanLink(Uri link)
-        {
-            var indexer = GetOriginalIndexerForLink(link);
-            if (indexer != null)
-                return indexer.UncleanLink(link);
-
-            return base.UncleanLink(link);
-        }
-
-        public override Task<byte[]> Download(Uri link)
-        {
-            var indexer = GetOriginalIndexerForLink(link);
-            if (indexer != null)
-                return indexer.Download(link);
-
-            return base.Download(link);
-        }
-
-        private IIndexer GetOriginalIndexerForLink(Uri link)
-        {
-            var prefix = string.Format("{0}://{1}", link.Scheme, link.Host);
-            var validIndexers = Indexers.Where(i => i.SiteLink.StartsWith(prefix));
-            if (validIndexers.Count() > 0)
-                return validIndexers.First();
-
-            return null;
-        }
-    }
-}

src/Jackett/Indexers/BaseIndexer.cs

@@ -641,5 +641,25 @@ namespace Jackett.Indexers
 
             return result.Distinct().ToList();
         }
+
+        public bool CanHandleQuery(TorznabQuery query)
+        {
+            var caps = TorznabCaps;
+            if (!caps.SearchAvailable && query.IsSearch)
+                return false;
+            if (!caps.TVSearchAvailable && query.IsTVSearch)
+                return false;
+            if (!caps.MovieSearchAvailable && query.IsMovieSearch)
+                return false;
+            if (!caps.SupportsTVRageSearch && query.IsTVRageSearch)
+                return false;
+            if (!caps.SupportsImdbSearch && query.IsImdbQuery)
+                return false;
+
+            if (query.HasSpecifiedCategories)
+                if (!caps.SupportsCategories(query.Categories))
+                    return false;
+            return true;
+        }
     }
 }

src/Jackett/Indexers/IIndexer.cs

@@ -45,5 +45,7 @@ namespace Jackett.Indexers
 
         IEnumerable<ReleaseInfo> CleanLinks(IEnumerable<ReleaseInfo> releases);
         Uri UncleanLink(Uri link);
+
+        bool CanHandleQuery(TorznabQuery query);
     }
 }

src/Jackett/Indexers/Meta/BaseMetaIndexer.cs

@@ -0,0 +1,85 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Jackett.Models;
+using Jackett.Models.IndexerConfig;
+using Jackett.Services;
+using Jackett.Utils.Clients;
+using Newtonsoft.Json.Linq;
+using NLog;
+
+namespace Jackett.Indexers.Meta
+{
+    public abstract class BaseMetaIndexer : BaseIndexer, IIndexer
+    {
+        protected BaseMetaIndexer(string name, string description, IIndexerManagerService manager, Logger logger, ConfigurationData configData, IProtectionService p, Func<IIndexer, bool> filter)
+            : base(name, "http://127.0.0.1/", description, manager, null, logger, configData, p, null, null)
+        {
+            filterFunc = filter;
+        }
+
+        public Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
+        {
+            return Task.FromResult(IndexerConfigurationStatus.Completed);
+        }
+
+        public virtual async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
+        {
+            var tasks = Indexers.Where(i => i.CanHandleQuery(query)).Select(i => i.PerformQuery(query)).ToList(); // explicit conversion to List to execute LINQ query
+            var aggregateTask = Task.WhenAll<IEnumerable<ReleaseInfo>>(tasks);
+            await aggregateTask;
+            if (aggregateTask.Exception != null)
+                logger.Error(aggregateTask.Exception, "Error during request in metaindexer " + ID);
+
+            IEnumerable<ReleaseInfo> result = tasks.Where(x => x.Status == TaskStatus.RanToCompletion).SelectMany(x => x.Result).OrderByDescending(r => r.PublishDate); // Ordering by the number of seeders might be useful as well.
+            // Limiting the response size might be interesting for use-cases where there are
+            // tons of trackers configured in Jackett. For now just use the limit param if
+            // someone wants to do that.
+            if (query.Limit > 0)
+                result = result.Take(query.Limit);
+            return result;
+        }
+
+        public override Uri UncleanLink(Uri link)
+        {
+            var indexer = GetOriginalIndexerForLink(link);
+            if (indexer != null)
+                return indexer.UncleanLink(link);
+
+            return base.UncleanLink(link);
+        }
+
+        public override Task<byte[]> Download(Uri link)
+        {
+            var indexer = GetOriginalIndexerForLink(link);
+            if (indexer != null)
+                return indexer.Download(link);
+
+            return base.Download(link);
+        }
+
+        private IIndexer GetOriginalIndexerForLink(Uri link)
+        {
+            var prefix = string.Format("{0}://{1}", link.Scheme, link.Host);
+            var validIndexers = Indexers.Where(i => i.SiteLink.StartsWith(prefix, StringComparison.CurrentCulture));
+            if (validIndexers.Count() > 0)
+                return validIndexers.First();
+
+            return null;
+        }
+
+        private Func<IIndexer, bool> filterFunc;
+        private IEnumerable<IIndexer> indexers;
+        public IEnumerable<IIndexer> Indexers {
+            get {
+                return indexers;
+            }
+            set {
+                indexers = value.Where(i => i.IsConfigured && filterFunc(i));
+                TorznabCaps = value.Select(i => i.TorznabCaps).Aggregate(new TorznabCapabilities(), TorznabCapabilities.Concat); ;
+                IsConfigured = true;
+            }
+        }
+    }
+}

src/Jackett/Indexers/Meta/MetaIndexers.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Jackett.Models;
+using Newtonsoft.Json.Linq;
+using Jackett.Services;
+using Jackett.Utils.Clients;
+using NLog;
+
+namespace Jackett.Indexers.Meta
+{
+    class AggregateIndexer : BaseMetaIndexer, IIndexer
+    {
+        public AggregateIndexer(IIndexerManagerService i, Logger l, IProtectionService ps)
+            : base("AggregateSearch", "This feed includes all configured trackers", i, l, new Models.IndexerConfig.ConfigurationData(), ps, x => true)
+        {
+        }
+    }
+}

src/Jackett/Jackett.csproj

@@ -178,7 +178,6 @@
     <Compile Include="Controllers\TorznabController.cs" />
     <Compile Include="Controllers\DownloadController.cs" />
     <Compile Include="Engine.cs" />
-    <Compile Include="Indexers\AggregateIndexer.cs" />
     <Compile Include="Indexers\ArcheTorrent.cs" />
     <Compile Include="Indexers\HDOnly.cs" />
     <Compile Include="Indexers\cgpeers.cs" />
@@ -371,6 +370,8 @@
     <Compile Include="WebAPIExceptionHandler.cs" />
     <Compile Include="WebAPIExceptionLogger.cs" />
     <Compile Include="Indexers\BakaBT.cs" />
+    <Compile Include="Indexers\Meta\MetaIndexers.cs" />
+    <Compile Include="Indexers\Meta\BaseMetaIndexer.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="App.config">
@@ -932,6 +933,9 @@
     <Service Include="{508349B6-6B84-4DF5-91F0-309BEEBAD82D}" />
   </ItemGroup>
   <ItemGroup />
+  <ItemGroup>
+    <Folder Include="Indexers\Meta\" />
+  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
        Other similar extension points exist, see Microsoft.Common.targets.

src/Jackett/JackettModule.cs

@@ -98,9 +98,8 @@ namespace Jackett
             }
 
             // Register indexers
-            foreach (var indexer in thisAssembly.GetTypes()
+            var indexerTypes = thisAssembly.GetTypes().Where(p => typeof (IIndexer).IsAssignableFrom (p) && !p.IsInterface && !p.IsInNamespace("Jackett.Indexers.Meta"));
-                .Where(p => typeof(IIndexer).IsAssignableFrom(p) && !p.IsInterface)
+            foreach (var indexer in indexerTypes)
-                .ToArray())
             {
                 builder.RegisterType(indexer).Named<IIndexer>(BaseIndexer.GetIndexerID(indexer));
             }

src/Jackett/Models/TorznabCapabilities.cs

@@ -66,6 +66,11 @@ namespace Jackett.Models
             }
         }
 
+        public bool SupportsCategories (int[] categories)
+        {
+            return Categories.Count(i => categories.Any(c => c == i.ID)) > 0;
+        }
+
         public JArray CapsToJson()
         {
             var jArray = new JArray();
@@ -112,5 +117,16 @@ namespace Jackett.Models
 
             return xdoc.Declaration.ToString() + Environment.NewLine + xdoc.ToString();
         }
+        public static TorznabCapabilities Concat (TorznabCapabilities lhs, TorznabCapabilities rhs)
+        {
+            lhs.SearchAvailable = lhs.SearchAvailable || rhs.SearchAvailable;
+            lhs.TVSearchAvailable = lhs.TVSearchAvailable || rhs.TVSearchAvailable;
+            lhs.MovieSearchAvailable = lhs.MovieSearchAvailable || rhs.MovieSearchAvailable;
+            lhs.SupportsTVRageSearch = lhs.SupportsTVRageSearch || rhs.SupportsTVRageSearch;
+            lhs.SupportsImdbSearch = lhs.SupportsImdbSearch || rhs.SupportsImdbSearch;
+            lhs.Categories.AddRange (rhs.Categories.Except (lhs.Categories));
+
+            return lhs;
+        }
     }
 }

src/Jackett/Models/TorznabQuery.cs

@@ -31,6 +31,54 @@ namespace Jackett.Models
 
         protected string[] QueryStringParts = null;
 
+        public bool IsSearch
+        {
+            get
+            {
+                return (QueryStringParts != null && QueryStringParts.Length > 0);
+            }
+        }
+
+        public bool IsTVSearch
+        {
+            get
+            {
+                return QueryType == "tvsearch";
+            }
+        }
+
+        public bool IsMovieSearch
+        {
+            get
+            {
+                return QueryType == "movie";
+            }
+        }
+
+        public bool IsTVRageSearch
+        {
+            get
+            {
+                return RageID != null;
+            }
+        }
+
+        public bool IsImdbQuery
+        {
+            get
+            {
+                return ImdbID != null;
+            }
+        }
+
+        public bool HasSpecifiedCategories
+        {
+            get
+            {
+                return (Categories != null && Categories.Length > 0);
+            }
+        }
+
         public string SanitizedSearchTerm
         {
             get

src/Jackett/Services/IndexerManagerService.cs

@@ -11,6 +11,7 @@ using System.IO;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using Jackett.Indexers.Meta;
 
 namespace Jackett.Services
 {
@@ -131,7 +132,7 @@ namespace Jackett.Services
         public void InitAggregateIndexer()
         {
             logger.Info("Adding aggregate indexer");
-            AggregateIndexer aggregateIndexer = new AggregateIndexer(this, container.Resolve<IWebClient>(), logger, container.Resolve<IProtectionService>());
+            AggregateIndexer aggregateIndexer = new AggregateIndexer(this, logger, container.Resolve<IProtectionService>());
             this.aggregateIndexer = aggregateIndexer;
             UpdateAggregateIndexer();
         }
@@ -266,7 +267,7 @@ namespace Jackett.Services
 
         private void UpdateAggregateIndexer()
         {
-            aggregateIndexer.SetIndexers(indexers.Where (p => p.Value.IsConfigured).Select(p => p.Value));
+            aggregateIndexer.Indexers = indexers.Where(p => p.Value.IsConfigured).Select(p => p.Value);
         }
     }
 }