mirror
/
Jackett
mirror of https://github.com/Jackett/Jackett
1
0
Fork 0
Code Issues Releases Wiki Activity

core: search cache optimizations #10382 (#10484) 

* Empty cache when user changes proxy configuration
* Reduce the CPU needed to clean up results that exceed the limit per indexer
This commit is contained in:
Diego Heras 2020-12-12 18:44:08 +01:00 committed by GitHub
parent b39fd35cf9
commit f8d3781f7a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 43 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
src/Jackett.Common/Services/CacheService.cs
View File

@ -25,6 +25,8 @@ namespace Jackett.Common.Services
/// before testing the configuration
/// * When there is some error/exception in the indexer => The results are not cached so we can retry in the
/// next request
/// * When the user changes proxy configuration => We call CleanCache to remove all cached results. The user will
/// be able to test the proxy
/// * We want to limit the memory usage, so we try to remove elements from cache ASAP:
/// * Each indexer can have a maximum number of results in memory. If the limit is exceeded we remove old results
/// * Cached results expire after some time
@ -78,7 +80,7 @@ namespace Jackett.Common.Services
_logger.Debug($"CACHE CacheResults / Indexer: {trackerCache.TrackerId} / Added: {releases.Count} releases");
PruneCacheByMaxResultsPerIndexer(); // remove old results if we exceed the maximum limit
PruneCacheByMaxResultsPerIndexer(trackerCache); // remove old results if we exceed the maximum limit
}
}
@ -159,6 +161,18 @@ namespace Jackett.Common.Services
}
}
public void CleanCache()
{
lock (_cache)
{
if (!IsCacheEnabled())
return;
_cache.Clear();
_logger.Debug("CACHE CleanCache");
}
}
private bool IsCacheEnabled()
{
if (!_serverConfig.CacheEnabled)
@ -191,27 +205,26 @@ namespace Jackett.Common.Services
}
}
private void PruneCacheByMaxResultsPerIndexer()
private void PruneCacheByMaxResultsPerIndexer(TrackerCache trackerCache)
{
// Remove queries exceeding max results per indexer
var resultsPerQuery = trackerCache.Queries
.OrderByDescending(q => q.Value.Created) // newest first
.Select(q => new Tuple<string, int>(q.Key, q.Value.Results.Count)).ToList();
var prunedCounter = 0;
foreach (var trackerCache in _cache.Values)
while (true)
{
// Remove queries exceeding max results per indexer
var resultsPerQuery = trackerCache.Queries
.OrderByDescending(q => q.Value.Created) // newest first
.Select(q => new Tuple<string, int>(q.Key, q.Value.Results.Count)).ToList();
while (true)
{
var total = resultsPerQuery.Select(q => q.Item2).Sum();
if (total <= _serverConfig.CacheMaxResultsPerIndexer)
break;
trackerCache.Queries.Remove(resultsPerQuery.Pop().Item1); // remove the older
prunedCounter++;
}
var total = resultsPerQuery.Select(q => q.Item2).Sum();
if (total <= _serverConfig.CacheMaxResultsPerIndexer)
break;
trackerCache.Queries.Remove(resultsPerQuery.Pop().Item1); // remove the older
prunedCounter++;
}
if (_logger.IsDebugEnabled)
{
_logger.Debug($"CACHE PruneCacheByMaxResultsPerIndexer / Pruned queries: {prunedCounter}");
_logger.Debug($"CACHE PruneCacheByMaxResultsPerIndexer / Indexer: {trackerCache.TrackerId} / Pruned queries: {prunedCounter}");
PrintCacheStatus();
}
}

1
src/Jackett.Common/Services/Interfaces/ICacheService.cs
View File

@ -10,5 +10,6 @@ namespace Jackett.Common.Services.Interfaces
void CacheResults(IIndexer indexer, TorznabQuery query, List<ReleaseInfo> releases);
List<TrackerCacheResult> GetCachedResults();
void CleanIndexerCache(IIndexer indexer);
void CleanCache();
}
}

19
src/Jackett.Server/Controllers/ServerConfigurationController.cs
View File

@ -22,11 +22,14 @@ namespace Jackett.Server.Controllers
private readonly IProcessService processService;
private readonly IIndexerManagerService indexerService;
private readonly ISecuityService securityService;
private readonly ICacheService cacheService;
private readonly IUpdateService updater;
private readonly ILogCacheService logCache;
private readonly Logger logger;
public ServerConfigurationController(IConfigurationService c, IServerService s, IProcessService p, IIndexerManagerService i, ISecuityService ss, IUpdateService u, ILogCacheService lc, Logger l, ServerConfig sc)
public ServerConfigurationController(IConfigurationService c, IServerService s, IProcessService p,
IIndexerManagerService i, ISecuityService ss, ICacheService cs, IUpdateService u, ILogCacheService lc,
Logger l, ServerConfig sc)
{
configService = c;
serverConfig = sc;
@ -34,6 +37,7 @@ namespace Jackett.Server.Controllers
processService = p;
indexerService = i;
securityService = ss;
cacheService = cs;
updater = u;
logCache = lc;
logger = l;
@ -134,6 +138,9 @@ namespace Jackett.Server.Controllers
serverConfig.ProxyPassword = config.proxy_password;
configService.SaveConfig(serverConfig);
webHostRestartNeeded = true;
// Remove all results from cache so we can test the new proxy
cacheService.CleanCache();
}
if (port != serverConfig.Port || external != serverConfig.AllowExternal)
@ -150,7 +157,7 @@ namespace Jackett.Server.Controllers
configService.SaveConfig(serverConfig);
// On Windows change the url reservations
if (System.Environment.OSVersion.Platform != PlatformID.Unix)
if (Environment.OSVersion.Platform != PlatformID.Unix)
{
if (!ServerUtil.IsUserAdministrator())
{
@ -170,7 +177,7 @@ namespace Jackett.Server.Controllers
}
else
{
serverService.ReserveUrls(true);
serverService.ReserveUrls();
}
}
@ -196,14 +203,14 @@ namespace Jackett.Server.Controllers
// we have to restore log level when the server restarts because we are not saving the state in the
// configuration. when the server restarts the UI is inconsistent with the active log level
// https://github.com/Jackett/Jackett/issues/8315
setEnhancedLogLevel(false);
SetEnhancedLogLevel(false);
Thread.Sleep(500);
logger.Info("Restarting webhost due to configuration change");
Helper.RestartWebHost();
}
else
setEnhancedLogLevel(enhancedLogging);
SetEnhancedLogLevel(enhancedLogging);
serverConfig.ConfigChanged();
@ -213,7 +220,7 @@ namespace Jackett.Server.Controllers
[HttpGet]
public List<CachedLog> Logs() => logCache.Logs;
private void setEnhancedLogLevel(bool enabled)
private void SetEnhancedLogLevel(bool enabled)
{
Helper.SetLogLevel(enabled ? LogLevel.Debug : LogLevel.Info);
serverConfig.RuntimeSettings.TracingEnabled = enabled;