mirror of https://github.com/Jackett/Jackett
Feature/aggregate performance improvement (#1349)
* Checking capabilities before executing a query * Introduce base class for meta indexers * Build fix - I seriously do not know how I missed that * Moving things to the appropriate place * Simplifying things as much as possible and moving once again * Build fix?
This commit is contained in:
parent
2fb045e94a
commit
27d4f2108e
|
@ -1,99 +0,0 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Jackett.Models;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Jackett.Services;
|
||||
using Jackett.Utils.Clients;
|
||||
using NLog;
|
||||
|
||||
namespace Jackett.Indexers
|
||||
{
|
||||
class AggregateIndexer : BaseIndexer, IIndexer
|
||||
{
|
||||
private IEnumerable<IIndexer> Indexers;
|
||||
public AggregateIndexer(IIndexerManagerService i, IWebClient wc, Logger l, IProtectionService ps)
|
||||
: base("AggregateSearch", "http://127.0.0.1/", "This feed includes all configured trackers", i, wc, l, new Models.IndexerConfig.ConfigurationData(), ps)
|
||||
{
|
||||
}
|
||||
|
||||
public void SetIndexers(IEnumerable<IIndexer> indexers)
|
||||
{
|
||||
Indexers = indexers.Where(i => i.IsConfigured);
|
||||
|
||||
var caps = new TorznabCapabilities();
|
||||
foreach (var indexer in indexers) {
|
||||
var indexerCaps = indexer.TorznabCaps;
|
||||
caps.SearchAvailable = caps.SearchAvailable || indexerCaps.SearchAvailable;
|
||||
caps.TVSearchAvailable = caps.TVSearchAvailable || indexerCaps.TVSearchAvailable;
|
||||
caps.MovieSearchAvailable = caps.MovieSearchAvailable || indexerCaps.MovieSearchAvailable;
|
||||
caps.SupportsTVRageSearch = caps.SupportsTVRageSearch || indexerCaps.SupportsTVRageSearch;
|
||||
caps.SupportsImdbSearch = caps.SupportsImdbSearch || indexerCaps.SupportsImdbSearch;
|
||||
caps.Categories.AddRange(indexerCaps.Categories.Except (caps.Categories));
|
||||
}
|
||||
|
||||
base.TorznabCaps = caps;
|
||||
base.IsConfigured = true;
|
||||
}
|
||||
|
||||
public async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
|
||||
{
|
||||
return IndexerConfigurationStatus.Completed;
|
||||
}
|
||||
|
||||
public async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
|
||||
{
|
||||
var tasks = new List<Task<IEnumerable<ReleaseInfo>>>();
|
||||
foreach (var indexer in Indexers)
|
||||
tasks.Add(indexer.PerformQuery(query));
|
||||
|
||||
var t = Task.WhenAll<IEnumerable<ReleaseInfo>>(tasks);
|
||||
try
|
||||
{
|
||||
t.Wait();
|
||||
}
|
||||
catch (AggregateException exception)
|
||||
{
|
||||
logger.Error(exception, "Error during request from Aggregate");
|
||||
}
|
||||
|
||||
IEnumerable<ReleaseInfo> result = tasks.Where(x => x.Status == TaskStatus.RanToCompletion).SelectMany(x => x.Result).OrderByDescending(r => r.PublishDate);
|
||||
// Limiting the response size might be interesting for use-cases where there are
|
||||
// tons of trackers configured in Jackett. For now just use the limit param if
|
||||
// someone wants to do that.
|
||||
if (query.Limit > 0)
|
||||
result = result.Take(query.Limit);
|
||||
return result;
|
||||
}
|
||||
|
||||
public override Uri UncleanLink(Uri link)
|
||||
{
|
||||
var indexer = GetOriginalIndexerForLink(link);
|
||||
if (indexer != null)
|
||||
return indexer.UncleanLink(link);
|
||||
|
||||
return base.UncleanLink(link);
|
||||
}
|
||||
|
||||
public override Task<byte[]> Download(Uri link)
|
||||
{
|
||||
var indexer = GetOriginalIndexerForLink(link);
|
||||
if (indexer != null)
|
||||
return indexer.Download(link);
|
||||
|
||||
return base.Download(link);
|
||||
}
|
||||
|
||||
private IIndexer GetOriginalIndexerForLink(Uri link)
|
||||
{
|
||||
var prefix = string.Format("{0}://{1}", link.Scheme, link.Host);
|
||||
var validIndexers = Indexers.Where(i => i.SiteLink.StartsWith(prefix));
|
||||
if (validIndexers.Count() > 0)
|
||||
return validIndexers.First();
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -641,5 +641,25 @@ namespace Jackett.Indexers
|
|||
|
||||
return result.Distinct().ToList();
|
||||
}
|
||||
|
||||
public bool CanHandleQuery(TorznabQuery query)
|
||||
{
|
||||
var caps = TorznabCaps;
|
||||
if (!caps.SearchAvailable && query.IsSearch)
|
||||
return false;
|
||||
if (!caps.TVSearchAvailable && query.IsTVSearch)
|
||||
return false;
|
||||
if (!caps.MovieSearchAvailable && query.IsMovieSearch)
|
||||
return false;
|
||||
if (!caps.SupportsTVRageSearch && query.IsTVRageSearch)
|
||||
return false;
|
||||
if (!caps.SupportsImdbSearch && query.IsImdbQuery)
|
||||
return false;
|
||||
|
||||
if (query.HasSpecifiedCategories)
|
||||
if (!caps.SupportsCategories(query.Categories))
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -45,5 +45,7 @@ namespace Jackett.Indexers
|
|||
|
||||
IEnumerable<ReleaseInfo> CleanLinks(IEnumerable<ReleaseInfo> releases);
|
||||
Uri UncleanLink(Uri link);
|
||||
|
||||
bool CanHandleQuery(TorznabQuery query);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,85 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Jackett.Models;
|
||||
using Jackett.Models.IndexerConfig;
|
||||
using Jackett.Services;
|
||||
using Jackett.Utils.Clients;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using NLog;
|
||||
|
||||
namespace Jackett.Indexers.Meta
|
||||
{
|
||||
public abstract class BaseMetaIndexer : BaseIndexer, IIndexer
|
||||
{
|
||||
protected BaseMetaIndexer(string name, string description, IIndexerManagerService manager, Logger logger, ConfigurationData configData, IProtectionService p, Func<IIndexer, bool> filter)
|
||||
: base(name, "http://127.0.0.1/", description, manager, null, logger, configData, p, null, null)
|
||||
{
|
||||
filterFunc = filter;
|
||||
}
|
||||
|
||||
public Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
|
||||
{
|
||||
return Task.FromResult(IndexerConfigurationStatus.Completed);
|
||||
}
|
||||
|
||||
public virtual async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
|
||||
{
|
||||
var tasks = Indexers.Where(i => i.CanHandleQuery(query)).Select(i => i.PerformQuery(query)).ToList(); // explicit conversion to List to execute LINQ query
|
||||
var aggregateTask = Task.WhenAll<IEnumerable<ReleaseInfo>>(tasks);
|
||||
await aggregateTask;
|
||||
if (aggregateTask.Exception != null)
|
||||
logger.Error(aggregateTask.Exception, "Error during request in metaindexer " + ID);
|
||||
|
||||
IEnumerable<ReleaseInfo> result = tasks.Where(x => x.Status == TaskStatus.RanToCompletion).SelectMany(x => x.Result).OrderByDescending(r => r.PublishDate); // Ordering by the number of seeders might be useful as well.
|
||||
// Limiting the response size might be interesting for use-cases where there are
|
||||
// tons of trackers configured in Jackett. For now just use the limit param if
|
||||
// someone wants to do that.
|
||||
if (query.Limit > 0)
|
||||
result = result.Take(query.Limit);
|
||||
return result;
|
||||
}
|
||||
|
||||
public override Uri UncleanLink(Uri link)
|
||||
{
|
||||
var indexer = GetOriginalIndexerForLink(link);
|
||||
if (indexer != null)
|
||||
return indexer.UncleanLink(link);
|
||||
|
||||
return base.UncleanLink(link);
|
||||
}
|
||||
|
||||
public override Task<byte[]> Download(Uri link)
|
||||
{
|
||||
var indexer = GetOriginalIndexerForLink(link);
|
||||
if (indexer != null)
|
||||
return indexer.Download(link);
|
||||
|
||||
return base.Download(link);
|
||||
}
|
||||
|
||||
private IIndexer GetOriginalIndexerForLink(Uri link)
|
||||
{
|
||||
var prefix = string.Format("{0}://{1}", link.Scheme, link.Host);
|
||||
var validIndexers = Indexers.Where(i => i.SiteLink.StartsWith(prefix, StringComparison.CurrentCulture));
|
||||
if (validIndexers.Count() > 0)
|
||||
return validIndexers.First();
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private Func<IIndexer, bool> filterFunc;
|
||||
private IEnumerable<IIndexer> indexers;
|
||||
public IEnumerable<IIndexer> Indexers {
|
||||
get {
|
||||
return indexers;
|
||||
}
|
||||
set {
|
||||
indexers = value.Where(i => i.IsConfigured && filterFunc(i));
|
||||
TorznabCaps = value.Select(i => i.TorznabCaps).Aggregate(new TorznabCapabilities(), TorznabCapabilities.Concat); ;
|
||||
IsConfigured = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,21 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Jackett.Models;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Jackett.Services;
|
||||
using Jackett.Utils.Clients;
|
||||
using NLog;
|
||||
|
||||
namespace Jackett.Indexers.Meta
|
||||
{
|
||||
class AggregateIndexer : BaseMetaIndexer, IIndexer
|
||||
{
|
||||
public AggregateIndexer(IIndexerManagerService i, Logger l, IProtectionService ps)
|
||||
: base("AggregateSearch", "This feed includes all configured trackers", i, l, new Models.IndexerConfig.ConfigurationData(), ps, x => true)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
|
@ -178,7 +178,6 @@
|
|||
<Compile Include="Controllers\TorznabController.cs" />
|
||||
<Compile Include="Controllers\DownloadController.cs" />
|
||||
<Compile Include="Engine.cs" />
|
||||
<Compile Include="Indexers\AggregateIndexer.cs" />
|
||||
<Compile Include="Indexers\ArcheTorrent.cs" />
|
||||
<Compile Include="Indexers\HDOnly.cs" />
|
||||
<Compile Include="Indexers\cgpeers.cs" />
|
||||
|
@ -371,6 +370,8 @@
|
|||
<Compile Include="WebAPIExceptionHandler.cs" />
|
||||
<Compile Include="WebAPIExceptionLogger.cs" />
|
||||
<Compile Include="Indexers\BakaBT.cs" />
|
||||
<Compile Include="Indexers\Meta\MetaIndexers.cs" />
|
||||
<Compile Include="Indexers\Meta\BaseMetaIndexer.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="App.config">
|
||||
|
@ -932,6 +933,9 @@
|
|||
<Service Include="{508349B6-6B84-4DF5-91F0-309BEEBAD82D}" />
|
||||
</ItemGroup>
|
||||
<ItemGroup />
|
||||
<ItemGroup>
|
||||
<Folder Include="Indexers\Meta\" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Microsoft.Common.targets.
|
||||
|
|
|
@ -98,9 +98,8 @@ namespace Jackett
|
|||
}
|
||||
|
||||
// Register indexers
|
||||
foreach (var indexer in thisAssembly.GetTypes()
|
||||
.Where(p => typeof(IIndexer).IsAssignableFrom(p) && !p.IsInterface)
|
||||
.ToArray())
|
||||
var indexerTypes = thisAssembly.GetTypes().Where(p => typeof (IIndexer).IsAssignableFrom (p) && !p.IsInterface && !p.IsInNamespace("Jackett.Indexers.Meta"));
|
||||
foreach (var indexer in indexerTypes)
|
||||
{
|
||||
builder.RegisterType(indexer).Named<IIndexer>(BaseIndexer.GetIndexerID(indexer));
|
||||
}
|
||||
|
|
|
@ -66,6 +66,11 @@ namespace Jackett.Models
|
|||
}
|
||||
}
|
||||
|
||||
public bool SupportsCategories (int[] categories)
|
||||
{
|
||||
return Categories.Count(i => categories.Any(c => c == i.ID)) > 0;
|
||||
}
|
||||
|
||||
public JArray CapsToJson()
|
||||
{
|
||||
var jArray = new JArray();
|
||||
|
@ -112,5 +117,16 @@ namespace Jackett.Models
|
|||
|
||||
return xdoc.Declaration.ToString() + Environment.NewLine + xdoc.ToString();
|
||||
}
|
||||
public static TorznabCapabilities Concat (TorznabCapabilities lhs, TorznabCapabilities rhs)
|
||||
{
|
||||
lhs.SearchAvailable = lhs.SearchAvailable || rhs.SearchAvailable;
|
||||
lhs.TVSearchAvailable = lhs.TVSearchAvailable || rhs.TVSearchAvailable;
|
||||
lhs.MovieSearchAvailable = lhs.MovieSearchAvailable || rhs.MovieSearchAvailable;
|
||||
lhs.SupportsTVRageSearch = lhs.SupportsTVRageSearch || rhs.SupportsTVRageSearch;
|
||||
lhs.SupportsImdbSearch = lhs.SupportsImdbSearch || rhs.SupportsImdbSearch;
|
||||
lhs.Categories.AddRange (rhs.Categories.Except (lhs.Categories));
|
||||
|
||||
return lhs;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -31,6 +31,54 @@ namespace Jackett.Models
|
|||
|
||||
protected string[] QueryStringParts = null;
|
||||
|
||||
public bool IsSearch
|
||||
{
|
||||
get
|
||||
{
|
||||
return (QueryStringParts != null && QueryStringParts.Length > 0);
|
||||
}
|
||||
}
|
||||
|
||||
public bool IsTVSearch
|
||||
{
|
||||
get
|
||||
{
|
||||
return QueryType == "tvsearch";
|
||||
}
|
||||
}
|
||||
|
||||
public bool IsMovieSearch
|
||||
{
|
||||
get
|
||||
{
|
||||
return QueryType == "movie";
|
||||
}
|
||||
}
|
||||
|
||||
public bool IsTVRageSearch
|
||||
{
|
||||
get
|
||||
{
|
||||
return RageID != null;
|
||||
}
|
||||
}
|
||||
|
||||
public bool IsImdbQuery
|
||||
{
|
||||
get
|
||||
{
|
||||
return ImdbID != null;
|
||||
}
|
||||
}
|
||||
|
||||
public bool HasSpecifiedCategories
|
||||
{
|
||||
get
|
||||
{
|
||||
return (Categories != null && Categories.Length > 0);
|
||||
}
|
||||
}
|
||||
|
||||
public string SanitizedSearchTerm
|
||||
{
|
||||
get
|
||||
|
|
|
@ -11,6 +11,7 @@ using System.IO;
|
|||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Jackett.Indexers.Meta;
|
||||
|
||||
namespace Jackett.Services
|
||||
{
|
||||
|
@ -131,7 +132,7 @@ namespace Jackett.Services
|
|||
public void InitAggregateIndexer()
|
||||
{
|
||||
logger.Info("Adding aggregate indexer");
|
||||
AggregateIndexer aggregateIndexer = new AggregateIndexer(this, container.Resolve<IWebClient>(), logger, container.Resolve<IProtectionService>());
|
||||
AggregateIndexer aggregateIndexer = new AggregateIndexer(this, logger, container.Resolve<IProtectionService>());
|
||||
this.aggregateIndexer = aggregateIndexer;
|
||||
UpdateAggregateIndexer();
|
||||
}
|
||||
|
@ -266,7 +267,7 @@ namespace Jackett.Services
|
|||
|
||||
private void UpdateAggregateIndexer()
|
||||
{
|
||||
aggregateIndexer.SetIndexers(indexers.Where (p => p.Value.IsConfigured).Select(p => p.Value));
|
||||
aggregateIndexer.Indexers = indexers.Where(p => p.Value.IsConfigured).Select(p => p.Value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue