2020-02-09 02:35:16 +00:00
|
|
|
using System;
|
2015-07-19 00:27:41 +00:00
|
|
|
using System.Collections.Generic;
|
|
|
|
using System.IO;
|
|
|
|
using System.Linq;
|
|
|
|
using System.Threading.Tasks;
|
2018-03-10 08:05:56 +00:00
|
|
|
using Jackett.Common.Indexers;
|
|
|
|
using Jackett.Common.Indexers.Meta;
|
|
|
|
using Jackett.Common.Models;
|
|
|
|
using Jackett.Common.Models.Config;
|
|
|
|
using Jackett.Common.Services.Interfaces;
|
|
|
|
using Jackett.Common.Utils.Clients;
|
|
|
|
using NLog;
|
2017-07-10 20:58:44 +00:00
|
|
|
using YamlDotNet.Serialization;
|
|
|
|
using YamlDotNet.Serialization.NamingConventions;
|
2015-07-19 00:27:41 +00:00
|
|
|
|
2018-03-10 08:05:56 +00:00
|
|
|
namespace Jackett.Common.Services
|
2015-07-19 00:27:41 +00:00
|
|
|
{
|
|
|
|
|
|
|
|
public class IndexerManagerService : IIndexerManagerService
|
|
|
|
{
|
2020-02-10 22:16:19 +00:00
|
|
|
private readonly ICacheService cacheService;
|
|
|
|
private readonly IIndexerConfigurationService configService;
|
|
|
|
private readonly IProtectionService protectionService;
|
|
|
|
private readonly WebClient webClient;
|
|
|
|
private readonly IProcessService processService;
|
|
|
|
private readonly IConfigurationService globalConfigService;
|
|
|
|
private readonly ServerConfig serverConfig;
|
|
|
|
private readonly Logger logger;
|
2017-07-14 05:39:52 +00:00
|
|
|
|
2020-02-10 22:16:19 +00:00
|
|
|
private readonly Dictionary<string, IIndexer> indexers = new Dictionary<string, IIndexer>();
|
2017-05-07 07:10:57 +00:00
|
|
|
private AggregateIndexer aggregateIndexer;
|
2015-07-19 00:27:41 +00:00
|
|
|
|
2017-11-05 09:42:03 +00:00
|
|
|
public IndexerManagerService(IIndexerConfigurationService config, IProtectionService protectionService, WebClient webClient, Logger l, ICacheService cache, IProcessService processService, IConfigurationService globalConfigService, ServerConfig serverConfig)
|
2015-07-19 00:27:41 +00:00
|
|
|
{
|
|
|
|
configService = config;
|
2017-07-14 05:39:52 +00:00
|
|
|
this.protectionService = protectionService;
|
|
|
|
this.webClient = webClient;
|
2017-08-11 14:53:27 +00:00
|
|
|
this.processService = processService;
|
|
|
|
this.globalConfigService = globalConfigService;
|
2017-11-05 09:42:03 +00:00
|
|
|
this.serverConfig = serverConfig;
|
2015-07-19 00:27:41 +00:00
|
|
|
logger = l;
|
2015-07-28 23:56:08 +00:00
|
|
|
cacheService = cache;
|
2015-07-19 00:27:41 +00:00
|
|
|
}
|
|
|
|
|
2017-07-14 05:39:52 +00:00
|
|
|
public void InitIndexers(IEnumerable<string> path)
|
2015-07-19 00:27:41 +00:00
|
|
|
{
|
2017-07-14 05:39:52 +00:00
|
|
|
InitIndexers();
|
|
|
|
InitCardigannIndexers(path);
|
|
|
|
InitAggregateIndexer();
|
|
|
|
}
|
2015-07-28 19:22:23 +00:00
|
|
|
|
2017-07-14 05:39:52 +00:00
|
|
|
private void InitIndexers()
|
|
|
|
{
|
|
|
|
logger.Info("Using HTTP Client: " + webClient.GetType().Name);
|
|
|
|
|
|
|
|
var allTypes = GetType().Assembly.GetTypes();
|
|
|
|
var allIndexerTypes = allTypes.Where(p => typeof(IIndexer).IsAssignableFrom(p));
|
|
|
|
var allInstantiatableIndexerTypes = allIndexerTypes.Where(p => !p.IsInterface && !p.IsAbstract);
|
|
|
|
var allNonMetaInstantiatableIndexerTypes = allInstantiatableIndexerTypes.Where(p => !typeof(BaseMetaIndexer).IsAssignableFrom(p));
|
|
|
|
var indexerTypes = allNonMetaInstantiatableIndexerTypes.Where(p => p.Name != "CardigannIndexer");
|
|
|
|
var ixs = indexerTypes.Select(type =>
|
2015-07-19 13:22:50 +00:00
|
|
|
{
|
2017-11-05 09:42:03 +00:00
|
|
|
var constructorArgumentTypes = new Type[] { typeof(IIndexerConfigurationService), typeof(WebClient), typeof(Logger), typeof(IProtectionService) };
|
2017-07-14 05:39:52 +00:00
|
|
|
var constructor = type.GetConstructor(constructorArgumentTypes);
|
|
|
|
if (constructor != null)
|
|
|
|
{
|
2017-08-11 14:53:27 +00:00
|
|
|
// create own webClient instance for each indexer (seperate cookies stores, etc.)
|
2017-11-05 09:42:03 +00:00
|
|
|
var indexerWebClientInstance = (WebClient)Activator.CreateInstance(webClient.GetType(), processService, logger, globalConfigService, serverConfig);
|
2017-08-11 14:53:27 +00:00
|
|
|
|
|
|
|
var arguments = new object[] { configService, indexerWebClientInstance, logger, protectionService };
|
2017-07-14 05:39:52 +00:00
|
|
|
var indexer = (IIndexer)constructor.Invoke(arguments);
|
|
|
|
return indexer;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
logger.Error("Cannot instantiate " + type.Name);
|
|
|
|
}
|
|
|
|
return null;
|
|
|
|
});
|
|
|
|
|
|
|
|
foreach (var idx in ixs)
|
|
|
|
{
|
|
|
|
if (idx == null)
|
|
|
|
continue;
|
2020-05-11 19:59:28 +00:00
|
|
|
indexers.Add(idx.Id, idx);
|
2017-07-10 20:58:44 +00:00
|
|
|
configService.Load(idx);
|
2016-10-27 07:30:03 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-07-14 05:39:52 +00:00
|
|
|
private void InitCardigannIndexers(IEnumerable<string> path)
|
2016-10-27 07:30:03 +00:00
|
|
|
{
|
2017-07-14 05:39:52 +00:00
|
|
|
logger.Info("Loading Cardigann definitions from: " + string.Join(", ", path));
|
|
|
|
|
|
|
|
var deserializer = new DeserializerBuilder()
|
2020-01-11 18:04:24 +00:00
|
|
|
.WithNamingConvention(CamelCaseNamingConvention.Instance)
|
2020-04-12 21:45:57 +00:00
|
|
|
// .IgnoreUnmatchedProperties()
|
2017-07-14 05:39:52 +00:00
|
|
|
.Build();
|
2016-10-27 07:30:03 +00:00
|
|
|
|
2017-04-15 08:45:10 +00:00
|
|
|
try
|
|
|
|
{
|
2017-07-14 05:39:52 +00:00
|
|
|
var directoryInfos = path.Select(p => new DirectoryInfo(p));
|
|
|
|
var existingDirectories = directoryInfos.Where(d => d.Exists);
|
|
|
|
var files = existingDirectories.SelectMany(d => d.GetFiles("*.yml"));
|
|
|
|
var definitions = files.Select(file =>
|
2017-04-15 08:45:10 +00:00
|
|
|
{
|
2020-02-23 20:38:59 +00:00
|
|
|
logger.Debug("Loading Cardigann definition " + file.FullName);
|
2020-02-09 02:35:16 +00:00
|
|
|
try
|
|
|
|
{
|
2020-02-10 22:16:19 +00:00
|
|
|
var DefinitionString = File.ReadAllText(file.FullName);
|
2017-09-13 09:33:17 +00:00
|
|
|
var definition = deserializer.Deserialize<IndexerDefinition>(DefinitionString);
|
|
|
|
return definition;
|
|
|
|
}
|
|
|
|
catch (Exception ex)
|
|
|
|
{
|
|
|
|
logger.Error(ex, "Error while parsing Cardigann definition " + file.FullName + ": " + ex.Message);
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
}).Where(definition => definition != null);
|
2017-07-10 20:58:44 +00:00
|
|
|
|
2020-02-10 22:16:19 +00:00
|
|
|
var cardigannIndexers = definitions.Select(definition =>
|
2017-07-14 05:39:52 +00:00
|
|
|
{
|
2017-09-13 09:33:17 +00:00
|
|
|
try
|
|
|
|
{
|
|
|
|
// create own webClient instance for each indexer (seperate cookies stores, etc.)
|
2017-11-05 09:42:03 +00:00
|
|
|
var indexerWebClientInstance = (WebClient)Activator.CreateInstance(webClient.GetType(), processService, logger, globalConfigService, serverConfig);
|
2017-08-11 14:53:27 +00:00
|
|
|
|
2017-09-13 09:33:17 +00:00
|
|
|
IIndexer indexer = new CardigannIndexer(configService, indexerWebClientInstance, logger, protectionService, definition);
|
|
|
|
configService.Load(indexer);
|
|
|
|
return indexer;
|
|
|
|
}
|
|
|
|
catch (Exception ex)
|
|
|
|
{
|
|
|
|
logger.Error(ex, "Error while creating Cardigann instance from Definition: " + ex.Message);
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
}).Where(cardigannIndexer => cardigannIndexer != null).ToList(); // Explicit conversion to list to avoid repeated resource loading
|
2017-07-14 05:39:52 +00:00
|
|
|
|
|
|
|
foreach (var indexer in cardigannIndexers)
|
|
|
|
{
|
2020-05-11 19:59:28 +00:00
|
|
|
if (indexers.ContainsKey(indexer.Id))
|
2017-04-15 08:45:10 +00:00
|
|
|
{
|
2020-05-11 19:59:28 +00:00
|
|
|
logger.Debug(string.Format("Ignoring definition ID={0}: Indexer already exists", indexer.Id));
|
2017-07-14 05:39:52 +00:00
|
|
|
continue;
|
2017-04-15 08:45:10 +00:00
|
|
|
}
|
2017-07-14 05:39:52 +00:00
|
|
|
|
2020-05-11 19:59:28 +00:00
|
|
|
indexers.Add(indexer.Id, indexer);
|
2016-10-29 16:01:43 +00:00
|
|
|
}
|
2020-02-23 20:38:59 +00:00
|
|
|
logger.Info("Cardigann definitions loaded: " + string.Join(", ", indexers.Keys));
|
2017-04-15 08:45:10 +00:00
|
|
|
}
|
|
|
|
catch (Exception ex)
|
|
|
|
{
|
2017-07-10 20:58:44 +00:00
|
|
|
logger.Error(ex, "Error while loading Cardigann definitions: " + ex.Message);
|
2015-07-19 13:22:50 +00:00
|
|
|
}
|
2015-07-19 00:27:41 +00:00
|
|
|
}
|
|
|
|
|
2017-04-30 08:06:29 +00:00
|
|
|
public void InitAggregateIndexer()
|
|
|
|
{
|
2017-11-05 09:42:03 +00:00
|
|
|
var omdbApiKey = serverConfig.OmdbApiKey;
|
2017-07-01 16:23:57 +00:00
|
|
|
IFallbackStrategyProvider fallbackStrategyProvider = null;
|
|
|
|
IResultFilterProvider resultFilterProvider = null;
|
2020-03-25 02:39:38 +00:00
|
|
|
if (!string.IsNullOrWhiteSpace(omdbApiKey))
|
2017-07-01 16:23:57 +00:00
|
|
|
{
|
2020-04-03 00:31:39 +00:00
|
|
|
var imdbResolver = new OmdbResolver(webClient, omdbApiKey, serverConfig.OmdbApiUrl);
|
2017-07-01 16:23:57 +00:00
|
|
|
fallbackStrategyProvider = new ImdbFallbackStrategyProvider(imdbResolver);
|
|
|
|
resultFilterProvider = new ImdbTitleResultFilterProvider(imdbResolver);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
fallbackStrategyProvider = new NoFallbackStrategyProvider();
|
|
|
|
resultFilterProvider = new NoResultFilterProvider();
|
|
|
|
}
|
2017-06-28 05:31:38 +00:00
|
|
|
|
2017-04-30 08:06:29 +00:00
|
|
|
logger.Info("Adding aggregate indexer");
|
2020-03-26 22:15:28 +00:00
|
|
|
aggregateIndexer = new AggregateIndexer(fallbackStrategyProvider, resultFilterProvider, configService, webClient, logger, protectionService)
|
|
|
|
{
|
|
|
|
Indexers = indexers.Values
|
|
|
|
};
|
2017-04-30 08:06:29 +00:00
|
|
|
}
|
|
|
|
|
2015-07-19 13:22:50 +00:00
|
|
|
public IIndexer GetIndexer(string name)
|
2015-07-19 00:27:41 +00:00
|
|
|
{
|
2015-07-22 22:06:46 +00:00
|
|
|
if (indexers.ContainsKey(name))
|
2015-07-19 17:18:54 +00:00
|
|
|
{
|
2015-07-22 22:06:46 +00:00
|
|
|
return indexers[name];
|
2015-07-19 17:18:54 +00:00
|
|
|
}
|
2017-04-30 08:06:29 +00:00
|
|
|
else if (name == "all")
|
|
|
|
{
|
|
|
|
return aggregateIndexer;
|
|
|
|
}
|
2015-07-19 17:18:54 +00:00
|
|
|
else
|
2015-07-19 13:22:50 +00:00
|
|
|
{
|
|
|
|
logger.Error("Request for unknown indexer: " + name);
|
|
|
|
throw new Exception("Unknown indexer: " + name);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-07-10 20:58:44 +00:00
|
|
|
public IWebIndexer GetWebIndexer(string name)
|
|
|
|
{
|
|
|
|
if (indexers.ContainsKey(name))
|
2017-10-16 16:22:16 +00:00
|
|
|
{
|
2017-07-10 20:58:44 +00:00
|
|
|
return indexers[name] as IWebIndexer;
|
2017-10-16 16:22:16 +00:00
|
|
|
}
|
|
|
|
else if (name == "all")
|
|
|
|
{
|
|
|
|
return aggregateIndexer as IWebIndexer;
|
|
|
|
}
|
2017-07-10 20:58:44 +00:00
|
|
|
|
|
|
|
logger.Error("Request for unknown indexer: " + name);
|
|
|
|
throw new Exception("Unknown indexer: " + name);
|
|
|
|
}
|
|
|
|
|
2020-02-25 16:08:03 +00:00
|
|
|
public IEnumerable<IIndexer> GetAllIndexers() => indexers.Values.OrderBy(_ => _.DisplayName);
|
2015-07-19 00:27:41 +00:00
|
|
|
|
2015-07-19 18:25:47 +00:00
|
|
|
public async Task TestIndexer(string name)
|
2015-07-19 00:27:41 +00:00
|
|
|
{
|
|
|
|
var indexer = GetIndexer(name);
|
2020-03-26 22:15:28 +00:00
|
|
|
var browseQuery = new TorznabQuery
|
|
|
|
{
|
|
|
|
QueryType = "search",
|
|
|
|
SearchTerm = "",
|
|
|
|
IsTest = true
|
|
|
|
};
|
2017-08-24 10:28:41 +00:00
|
|
|
var result = await indexer.ResultsForQuery(browseQuery);
|
|
|
|
logger.Info(string.Format("Found {0} releases from {1}", result.Releases.Count(), indexer.DisplayName));
|
|
|
|
if (result.Releases.Count() == 0)
|
2015-07-19 00:27:41 +00:00
|
|
|
throw new Exception("Found no results while trying to browse this tracker");
|
2017-08-24 10:28:41 +00:00
|
|
|
cacheService.CacheRssResults(indexer, result.Releases);
|
2015-07-19 00:27:41 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
public void DeleteIndexer(string name)
|
|
|
|
{
|
|
|
|
var indexer = GetIndexer(name);
|
2017-07-10 20:58:44 +00:00
|
|
|
configService.Delete(indexer);
|
|
|
|
indexer.Unconfigure();
|
2017-04-15 08:45:10 +00:00
|
|
|
}
|
2015-07-19 00:27:41 +00:00
|
|
|
}
|
|
|
|
}
|