mirror
/
Jackett
mirror of https://github.com/Jackett/Jackett
1
0
Fork 0
Code Issues Releases Wiki Activity

core: clean up indexer manager code (#13867)

This commit is contained in:
Diego Heras 2023-01-07 17:18:24 +01:00 committed by GitHub
parent e7384b73e4
commit 034ca38368
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 82 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

157
src/Jackett.Common/Services/IndexerManagerService.cs
View File

@ -9,7 +9,6 @@ using Jackett.Common.Indexers.Meta;
using Jackett.Common.Models;
using Jackett.Common.Models.Config;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using NLog;
using YamlDotNet.Serialization;
@ -21,24 +20,24 @@ namespace Jackett.Common.Services
public class IndexerManagerService : IIndexerManagerService
{
private readonly ICacheService cacheService;
private readonly IIndexerConfigurationService configService;
private readonly IProtectionService protectionService;
private readonly WebClient webClient;
private readonly IProcessService processService;
private readonly IConfigurationService globalConfigService;
private readonly ServerConfig serverConfig;
private readonly Logger logger;
private readonly ICacheService _cacheService;
private readonly IIndexerConfigurationService _configService;
private readonly IProtectionService _protectionService;
private readonly WebClient _webClient;
private readonly IProcessService _processService;
private readonly IConfigurationService _globalConfigService;
private readonly ServerConfig _serverConfig;
private readonly Logger _logger;
private readonly Dictionary<string, IIndexer> indexers = new Dictionary<string, IIndexer>();
private AggregateIndexer aggregateIndexer;
private ConcurrentDictionary<string, IWebIndexer> availableFilters = new ConcurrentDictionary<string, IWebIndexer>();
private readonly Dictionary<string, IIndexer> _indexers = new Dictionary<string, IIndexer>();
private AggregateIndexer _aggregateIndexer;
private ConcurrentDictionary<string, IWebIndexer> _availableFilters = new ConcurrentDictionary<string, IWebIndexer>();
// this map is used to maintain backward compatibility when renaming the id of an indexer
// (the id is used in the torznab/download/search urls and in the indexer configuration file)
// if the indexer is removed, remove it from this list too
// use: {"<old id>", "<new id>"}
private readonly Dictionary<string, string> renamedIndexers = new Dictionary<string, string>
private readonly Dictionary<string, string> _renamedIndexers = new Dictionary<string, string>
{
{"audiobooktorrents", "abtorrents"},
{"baibako", "rudub"},
@ -69,19 +68,19 @@ namespace Jackett.Common.Services
public IndexerManagerService(IIndexerConfigurationService config, IProtectionService protectionService, WebClient webClient, Logger l, ICacheService cache, IProcessService processService, IConfigurationService globalConfigService, ServerConfig serverConfig)
{
configService = config;
this.protectionService = protectionService;
this.webClient = webClient;
this.processService = processService;
this.globalConfigService = globalConfigService;
this.serverConfig = serverConfig;
logger = l;
cacheService = cache;
_configService = config;
_protectionService = protectionService;
_webClient = webClient;
_processService = processService;
_globalConfigService = globalConfigService;
_serverConfig = serverConfig;
_logger = l;
_cacheService = cache;
}
public void InitIndexers(IEnumerable<string> path)
public void InitIndexers(List<string> path)
{
logger.Info($"Using HTTP Client: {webClient.GetType().Name}");
_logger.Info($"Using HTTP Client: {_webClient.GetType().Name}");
MigrateRenamedIndexers();
InitIndexers();
@ -92,14 +91,14 @@ namespace Jackett.Common.Services
private void MigrateRenamedIndexers()
{
foreach (var oldId in renamedIndexers.Keys)
foreach (var oldId in _renamedIndexers.Keys)
{
var oldPath = configService.GetIndexerConfigFilePath(oldId);
var oldPath = _configService.GetIndexerConfigFilePath(oldId);
if (File.Exists(oldPath))
{
// if the old configuration exists, we rename it to be used by the renamed indexer
logger.Info($"Old configuration detected: {oldPath}");
var newPath = configService.GetIndexerConfigFilePath(renamedIndexers[oldId]);
_logger.Info($"Old configuration detected: {oldPath}");
var newPath = _configService.GetIndexerConfigFilePath(_renamedIndexers[oldId]);
if (File.Exists(newPath))
File.Delete(newPath);
File.Move(oldPath, newPath);
@ -112,14 +111,14 @@ namespace Jackett.Common.Services
File.Delete(newPathBak);
File.Move(oldPathBak, newPathBak);
}
logger.Info($"Configuration renamed: {oldPath} => {newPath}");
_logger.Info($"Configuration renamed: {oldPath} => {newPath}");
}
}
}
private void InitIndexers()
{
logger.Info("Loading Native indexers ...");
_logger.Info("Loading Native indexers ...");
var allTypes = GetType().Assembly.GetTypes();
var allIndexerTypes = allTypes.Where(p => typeof(IIndexer).IsAssignableFrom(p));
@ -133,29 +132,29 @@ namespace Jackett.Common.Services
if (constructor != null)
{
// create own webClient instance for each indexer (separate cookies stores, etc.)
var indexerWebClientInstance = (WebClient)Activator.CreateInstance(webClient.GetType(), processService, logger, globalConfigService, serverConfig);
var indexerWebClientInstance = (WebClient)Activator.CreateInstance(_webClient.GetType(), _processService, _logger, _globalConfigService, _serverConfig);
var arguments = new object[] { configService, indexerWebClientInstance, logger, protectionService, cacheService };
var arguments = new object[] { _configService, indexerWebClientInstance, _logger, _protectionService, _cacheService };
var indexer = (IIndexer)constructor.Invoke(arguments);
return indexer;
}
logger.Error($"Cannot instantiate Native indexer: {type.Name}");
_logger.Error($"Cannot instantiate Native indexer: {type.Name}");
return null;
}).Where(indexer => indexer != null).ToList();
foreach (var indexer in nativeIndexers)
{
indexers.Add(indexer.Id, indexer);
configService.Load(indexer);
_indexers.Add(indexer.Id, indexer);
_configService.Load(indexer);
}
logger.Info($"Loaded {nativeIndexers.Count} Native indexers: {string.Join(", ", nativeIndexers.Select(i => i.Id))}");
_logger.Info($"Loaded {nativeIndexers.Count} Native indexers: {string.Join(", ", nativeIndexers.Select(i => i.Id))}");
}
private void InitCardigannIndexers(IEnumerable<string> path)
private void InitCardigannIndexers(List<string> path)
{
logger.Info("Loading Cardigann indexers from: " + string.Join(", ", path));
_logger.Info("Loading Cardigann indexers from: " + string.Join(", ", path));
var deserializer = new DeserializerBuilder()
.WithNamingConvention(CamelCaseNamingConvention.Instance)
@ -169,7 +168,7 @@ namespace Jackett.Common.Services
var files = existingDirectories.SelectMany(d => d.GetFiles("*.yml"));
var definitions = files.Select(file =>
{
logger.Debug("Loading Cardigann definition " + file.FullName);
_logger.Debug("Loading Cardigann definition " + file.FullName);
try
{
var definitionString = File.ReadAllText(file.FullName);
@ -178,7 +177,7 @@ namespace Jackett.Common.Services
}
catch (Exception e)
{
logger.Error($"Error while parsing Cardigann definition {file.FullName}\n{e}");
_logger.Error($"Error while parsing Cardigann definition {file.FullName}\n{e}");
return null;
}
}).Where(definition => definition != null);
@ -187,16 +186,16 @@ namespace Jackett.Common.Services
{
try
{
// create own webClient instance for each indexer (seperate cookies stores, etc.)
var indexerWebClientInstance = (WebClient)Activator.CreateInstance(webClient.GetType(), processService, logger, globalConfigService, serverConfig);
// create own webClient instance for each indexer (separate cookies stores, etc.)
var indexerWebClientInstance = (WebClient)Activator.CreateInstance(_webClient.GetType(), _processService, _logger, _globalConfigService, _serverConfig);
IIndexer indexer = new CardigannIndexer(configService, indexerWebClientInstance, logger, protectionService, cacheService, definition);
configService.Load(indexer);
IIndexer indexer = new CardigannIndexer(_configService, indexerWebClientInstance, _logger, _protectionService, _cacheService, definition);
_configService.Load(indexer);
return indexer;
}
catch (Exception e)
{
logger.Error($"Error while creating Cardigann instance from definition ID={definition.Id}: {e}");
_logger.Error($"Error while creating Cardigann instance from definition ID={definition.Id}: {e}");
return null;
}
}).Where(cardigannIndexer => cardigannIndexer != null).ToList(); // Explicit conversion to list to avoid repeated resource loading
@ -205,59 +204,59 @@ namespace Jackett.Common.Services
var cardiganIds = new List<string>();
foreach (var indexer in cardigannIndexers)
{
if (indexers.ContainsKey(indexer.Id))
if (_indexers.ContainsKey(indexer.Id))
{
logger.Warn($"Ignoring definition ID={indexer.Id}: Indexer already exists");
_logger.Warn($"Ignoring definition ID={indexer.Id}: Indexer already exists");
continue;
}
indexers.Add(indexer.Id, indexer);
_indexers.Add(indexer.Id, indexer);
cardigannCounter++;
cardiganIds.Add(indexer.Id);
}
logger.Info($"Loaded {cardigannCounter} Cardigann indexers: {string.Join(", ", cardiganIds)}");
_logger.Info($"Loaded {cardigannCounter} Cardigann indexers: {string.Join(", ", cardiganIds)}");
}
catch (Exception e)
{
logger.Error($"Error while loading Cardigann definitions: {e}");
_logger.Error($"Error while loading Cardigann definitions: {e}");
}
logger.Info($"Loaded {indexers.Count} indexers in total");
_logger.Info($"Loaded {_indexers.Count} indexers in total");
}
public void InitMetaIndexers()
{
var (fallbackStrategyProvider, resultFilterProvider) = GetStrategyProviders();
logger.Info("Adding aggregate indexer ('all' indexer) ...");
aggregateIndexer = new AggregateIndexer(fallbackStrategyProvider, resultFilterProvider, configService, webClient, logger, protectionService, cacheService)
_logger.Info("Adding aggregate indexer ('all' indexer) ...");
_aggregateIndexer = new AggregateIndexer(fallbackStrategyProvider, resultFilterProvider, _configService, _webClient, _logger, _protectionService, _cacheService)
{
Indexers = indexers.Values
Indexers = _indexers.Values
};
var predefinedFilters =
new[] { "public", "private", "semi-public" }
.Select(type => (filter: FilterFunc.Type.ToFilter(type), func: FilterFunc.Type.ToFunc(type)))
.Concat(
indexers.Values.SelectMany(x => x.Tags).Distinct()
_indexers.Values.SelectMany(x => x.Tags).Distinct()
.Select(tag => (filter: FilterFunc.Tag.ToFilter(tag), func: FilterFunc.Tag.ToFunc(tag)))
).Select(x => new KeyValuePair<string, IWebIndexer>(x.filter, CreateFilterIndexer(x.filter, x.func)));
availableFilters = new ConcurrentDictionary<string, IWebIndexer>(predefinedFilters);
_availableFilters = new ConcurrentDictionary<string, IWebIndexer>(predefinedFilters);
}
public void RemoveLegacyConfigurations()
{
var directoryInfo = new DirectoryInfo(globalConfigService.GetIndexerConfigDir());
var directoryInfo = new DirectoryInfo(_globalConfigService.GetIndexerConfigDir());
if (!directoryInfo.Exists)
return; // the directory does not exist the first start
var files = directoryInfo.GetFiles("*.json*");
foreach (var file in files)
{
var indexerId = file.Name.Replace(".bak", "").Replace(".json", "");
if (!indexers.ContainsKey(indexerId) && File.Exists(file.FullName))
if (!_indexers.ContainsKey(indexerId) && File.Exists(file.FullName))
{
logger.Info($"Removing old configuration file: {file.FullName}");
_logger.Info($"Removing old configuration file: {file.FullName}");
File.Delete(file.FullName);
}
}
@ -268,10 +267,10 @@ namespace Jackett.Common.Services
// old id of renamed indexer is used to maintain backward compatibility
// both, the old id and the new one can be used until we remove it from renamedIndexers
var realName = name;
if (renamedIndexers.ContainsKey(name))
if (_renamedIndexers.ContainsKey(name))
{
realName = renamedIndexers[name];
logger.Warn($@"Indexer {name} has been renamed to {realName}. Please, update the URL of the feeds.
realName = _renamedIndexers[name];
_logger.Warn($@"Indexer {name} has been renamed to {realName}. Please, update the URL of the feeds.
This may stop working in the future.");
}
@ -281,23 +280,23 @@ namespace Jackett.Common.Services
public IWebIndexer GetWebIndexer(string name)
{
if (indexers.ContainsKey(name))
return indexers[name] as IWebIndexer;
if (_indexers.ContainsKey(name))
return _indexers[name] as IWebIndexer;
if (name == "all")
return aggregateIndexer;
return _aggregateIndexer;
if (availableFilters.TryGetValue(name, out var indexer))
if (_availableFilters.TryGetValue(name, out var indexer))
return indexer;
if (FilterFunc.TryParse(name, out var filterFunc))
return availableFilters.GetOrAdd(name, x => CreateFilterIndexer(name, filterFunc));
return _availableFilters.GetOrAdd(name, x => CreateFilterIndexer(name, filterFunc));
logger.Error($"Request for unknown indexer: {name}");
_logger.Error($"Request for unknown indexer: {name.Replace(Environment.NewLine, "")}");
throw new Exception($"Unknown indexer: {name}");
}
public IEnumerable<IIndexer> GetAllIndexers() => indexers.Values.OrderBy(_ => _.DisplayName);
public List<IIndexer> GetAllIndexers() => _indexers.Values.OrderBy(_ => _.DisplayName).ToList();
public async Task TestIndexer(string name)
{
@ -310,7 +309,7 @@ namespace Jackett.Common.Services
};
var result = await indexer.ResultsForQuery(query);
logger.Info($"Test search in {indexer.DisplayName} => Found {result.Releases.Count()} releases");
_logger.Info($"Test search in {indexer.DisplayName} => Found {result.Releases.Count()} releases");
if (!result.Releases.Any())
throw new Exception($"Test search in {indexer.DisplayName} => Found no results while trying to browse this tracker");
@ -319,39 +318,39 @@ namespace Jackett.Common.Services
public void DeleteIndexer(string name)
{
var indexer = GetIndexer(name);
configService.Delete(indexer);
_configService.Delete(indexer);
indexer.Unconfigure();
}
private IWebIndexer CreateFilterIndexer(string filter, Func<IIndexer, bool> filterFunc)
{
var (fallbackStrategyProvider, resultFilterProvider) = GetStrategyProviders();
logger.Info($"Adding filter indexer ('{filter}' indexer) ...");
_logger.Info($"Adding filter indexer ('{filter.Replace(Environment.NewLine, "")}' indexer) ...");
return new FilterIndexer(
filter,
fallbackStrategyProvider,
resultFilterProvider,
configService,
webClient,
logger,
protectionService,
cacheService,
_configService,
_webClient,
_logger,
_protectionService,
_cacheService,
filterFunc
)
{
Indexers = indexers.Values
Indexers = _indexers.Values
};
}
private (IFallbackStrategyProvider fallbackStrategyProvider, IResultFilterProvider resultFilterProvider)
GetStrategyProviders()
{
var omdbApiKey = serverConfig.OmdbApiKey;
var omdbApiKey = _serverConfig.OmdbApiKey;
IFallbackStrategyProvider fallbackStrategyProvider;
IResultFilterProvider resultFilterProvider;
if (!string.IsNullOrWhiteSpace(omdbApiKey))
{
var imdbResolver = new OmdbResolver(webClient, omdbApiKey, serverConfig.OmdbApiUrl);
var imdbResolver = new OmdbResolver(_webClient, omdbApiKey, _serverConfig.OmdbApiUrl);
fallbackStrategyProvider = new ImdbFallbackStrategyProvider(imdbResolver);
resultFilterProvider = new ImdbTitleResultFilterProvider(imdbResolver);
}

4
src/Jackett.Common/Services/Interfaces/IIndexerManagerService.cs
View File

@ -10,9 +10,9 @@ namespace Jackett.Common.Services.Interfaces
void DeleteIndexer(string name);
IIndexer GetIndexer(string name);
IWebIndexer GetWebIndexer(string name);
IEnumerable<IIndexer> GetAllIndexers();
List<IIndexer> GetAllIndexers();
void InitIndexers(IEnumerable<string> path);
void InitIndexers(List<string> path);
void InitMetaIndexers();
}
}

4
src/Jackett.Test/TestHelpers/TestIIndexerManagerServiceHelper.cs
View File

@ -13,13 +13,13 @@ namespace Jackett.Test.TestHelpers
public void DeleteIndexer(string name) => throw new NotImplementedException();
public IEnumerable<IIndexer> GetAllIndexers() => throw new NotImplementedException();
public List<IIndexer> GetAllIndexers() => throw new NotImplementedException();
public IIndexer GetIndexer(string name) => throw new NotImplementedException();
public IWebIndexer GetWebIndexer(string name) => throw new NotImplementedException();
public void InitIndexers(IEnumerable<string> path) => throw new NotImplementedException();
public void InitIndexers(List<string> path) => throw new NotImplementedException();
public void SaveConfig(IIndexer indexer, JToken obj) => LastSavedConfig = obj;