Adding an aggregate torznab feed for all configured trackers (#1312)

* Adding an aggregate torznab feed for all configured trackers

This adds just a basic first implementation of a special
torznab feed that will make all configured trackers available
with all the supported query params.
- This should close #1247 
- This also contributes to #921 

* Adding missing file

* Add missing implementation from Test project

src/Jackett.Test/TestIIndexerManagerServiceHelper.cs

@@ -52,5 +52,10 @@ namespace JackettTest
         {
             throw new NotImplementedException();
         }
+
+        public void InitAggregateIndexer()
+        {
+            throw new NotImplementedException();
+        }
     }
 }

src/Jackett/Indexers/AggregateIndexer.cs

@@ -0,0 +1,51 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Jackett.Models;
+using Newtonsoft.Json.Linq;
+using Jackett.Services;
+using Jackett.Utils.Clients;
+using NLog;
+
+namespace Jackett.Indexers
+{
+    class AggregateIndexer : BaseIndexer, IIndexer
+    {
+        private IEnumerable<IIndexer> Indexers;
+        public AggregateIndexer(IIndexerManagerService i, IWebClient wc, Logger l, IProtectionService ps)
+            : base("AggregateSearch", "http://127.0.0.1/", "This feed includes all configured trackers", i, wc, l, new Models.IndexerConfig.ConfigurationData(), ps)
+        {
+        }
+
+        public void SetIndexers(IEnumerable<IIndexer> indexers)
+        {
+            Indexers = indexers;
+            base.IsConfigured = true;
+        }
+
+        public async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
+        {
+            return IndexerConfigurationStatus.Completed;
+        }
+
+        public async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
+        {
+            var tasks = new List<Task<IEnumerable<ReleaseInfo>>>();
+            foreach (var indexer in Indexers.Where(i => i.IsConfigured))
+                tasks.Add(indexer.PerformQuery(query));
+
+            var t = Task.WhenAll<IEnumerable<ReleaseInfo>>(tasks);
+            t.Wait();
+
+            IEnumerable<ReleaseInfo> result = t.Result.SelectMany(x => x).OrderByDescending(r => r.PublishDate);
+            // Limiting the response size might be interesting for use-cases where there are
+            // tons of trackers configured in Jackett. For now just use the limit param if
+            // someone wants to do that.
+            if (query.Limit > 0)
+                result = result.Take(query.Limit);
+            return result;
+        }
+    }
+}

src/Jackett/Jackett.csproj

@@ -178,6 +178,7 @@
     <Compile Include="Controllers\TorznabController.cs" />
     <Compile Include="Controllers\DownloadController.cs" />
     <Compile Include="Engine.cs" />
+    <Compile Include="Indexers\AggregateIndexer.cs" />
     <Compile Include="Indexers\HDOnly.cs" />
     <Compile Include="Indexers\cgpeers.cs" />
     <Compile Include="Indexers\PiXELHD.cs" />

src/Jackett/Services/IndexerManagerService.cs

@@ -23,6 +23,7 @@ namespace Jackett.Services
         void SaveConfig(IIndexer indexer, JToken obj);
         void InitIndexers();
         void InitCardigannIndexers(string path);
+        void InitAggregateIndexer();
         void SortIndexers();
     }
 
@@ -35,6 +36,7 @@ namespace Jackett.Services
         private Logger logger;
         private Dictionary<string, IIndexer> indexers = new Dictionary<string, IIndexer>();
         private ICacheService cacheService;
+        private IIndexer aggregateIndexer;
 
         public IndexerManagerService(IContainer c, IConfigurationService config, Logger l, ICacheService cache)
         {
@@ -86,7 +88,7 @@ namespace Jackett.Services
         {
             logger.Info("Using HTTP Client: " + container.Resolve<IWebClient>().GetType().Name);
 
-            foreach (var idx in container.Resolve<IEnumerable<IIndexer>>().Where(p => p.ID != "cardigannindexer").OrderBy(_ => _.DisplayName))
+            foreach (var idx in container.Resolve<IEnumerable<IIndexer>>().Where(p => p.ID != "cardigannindexer" && p.ID != "aggregateindexer").OrderBy(_ => _.DisplayName))
             {
                 indexers.Add(idx.ID, idx);
                 LoadIndexerConfig(idx);
@@ -126,12 +128,25 @@ namespace Jackett.Services
             }
         }
 
+        public void InitAggregateIndexer()
+        {
+            logger.Info("Adding aggregate indexer");
+            AggregateIndexer aggregateIndexer = new AggregateIndexer(this, container.Resolve<IWebClient>(), logger, container.Resolve<IProtectionService>());
+            aggregateIndexer.SetIndexers(indexers.Values);
+
+            this.aggregateIndexer = aggregateIndexer;
+        }
+
         public IIndexer GetIndexer(string name)
         {
             if (indexers.ContainsKey(name))
             {
                 return indexers[name];
             }
+            else if (name == "all")
+            {
+                return aggregateIndexer;
+            }
             else
             {
                 logger.Error("Request for unknown indexer: " + name);

src/Jackett/Services/ServerService.cs

@@ -287,6 +287,7 @@ namespace Jackett.Services
             {
                 indexerService.InitCardigannIndexers(dir);
             }
+            indexerService.InitAggregateIndexer();
             indexerService.SortIndexers();
             client.Init();
             updater.CleanupTempDir();