2020-02-09 18:08:34 +00:00
|
|
|
using System;
|
|
|
|
using System.Collections.Generic;
|
2024-02-28 23:11:02 +00:00
|
|
|
using System.Diagnostics;
|
2020-02-09 18:08:34 +00:00
|
|
|
using System.Linq;
|
|
|
|
using System.Text;
|
|
|
|
using System.Threading.Tasks;
|
2023-02-19 11:26:49 +00:00
|
|
|
using Jackett.Common.Exceptions;
|
2023-03-09 14:06:12 +00:00
|
|
|
using Jackett.Common.Extensions;
|
2018-03-10 08:05:56 +00:00
|
|
|
using Jackett.Common.Models;
|
|
|
|
using Jackett.Common.Models.IndexerConfig;
|
|
|
|
using Jackett.Common.Services.Interfaces;
|
|
|
|
using Jackett.Common.Utils;
|
|
|
|
using Jackett.Common.Utils.Clients;
|
2017-07-11 20:32:56 +00:00
|
|
|
using Newtonsoft.Json;
|
2017-10-29 06:21:18 +00:00
|
|
|
using Newtonsoft.Json.Linq;
|
|
|
|
using NLog;
|
2021-05-16 18:13:54 +00:00
|
|
|
using Polly;
|
2023-12-10 15:52:12 +00:00
|
|
|
using Polly.Retry;
|
2018-06-26 15:47:19 +00:00
|
|
|
using static Jackett.Common.Models.IndexerConfig.ConfigurationData;
|
2017-04-15 08:45:10 +00:00
|
|
|
|
2018-03-10 08:05:56 +00:00
|
|
|
namespace Jackett.Common.Indexers
|
2017-04-15 08:45:10 +00:00
|
|
|
{
|
2017-06-28 05:31:38 +00:00
|
|
|
public abstract class BaseIndexer : IIndexer
|
2017-04-15 08:45:10 +00:00
|
|
|
{
|
2023-03-09 14:06:12 +00:00
|
|
|
public virtual string Id { get; protected set; }
|
|
|
|
public virtual string Name { get; protected set; }
|
|
|
|
public virtual string Description { get; protected set; }
|
2020-05-11 19:59:28 +00:00
|
|
|
|
2023-03-09 14:06:12 +00:00
|
|
|
public virtual string SiteLink { get; protected set; }
|
|
|
|
public string DefaultSiteLink { get; protected set; }
|
|
|
|
public virtual string[] AlternativeSiteLinks { get; protected set; } = Array.Empty<string>();
|
|
|
|
public virtual string[] LegacySiteLinks { get; protected set; } = Array.Empty<string>();
|
2017-04-15 08:45:10 +00:00
|
|
|
|
2018-06-15 09:12:03 +00:00
|
|
|
[JsonConverter(typeof(EncodingJsonConverter))]
|
2023-03-09 14:06:12 +00:00
|
|
|
public virtual Encoding Encoding { get; protected set; }
|
|
|
|
public virtual string Language { get; protected set; } = "en-US";
|
|
|
|
public virtual string Type { get; protected set; }
|
|
|
|
|
|
|
|
public virtual bool SupportsPagination => false;
|
2018-06-15 09:12:03 +00:00
|
|
|
|
2017-07-10 20:58:44 +00:00
|
|
|
public virtual bool IsConfigured { get; protected set; }
|
2021-05-08 20:24:18 +00:00
|
|
|
public virtual string[] Tags { get; protected set; }
|
|
|
|
|
2021-06-24 02:37:27 +00:00
|
|
|
// https://github.com/Jackett/Jackett/issues/3292#issuecomment-838586679
|
|
|
|
private TimeSpan HealthyStatusValidity => cacheService.CacheTTL + cacheService.CacheTTL;
|
|
|
|
private static readonly TimeSpan ErrorStatusValidity = TimeSpan.FromMinutes(10);
|
|
|
|
private static readonly TimeSpan MaxStatusValidity = TimeSpan.FromDays(1);
|
|
|
|
|
|
|
|
private int errorCount;
|
|
|
|
private DateTime expireAt;
|
|
|
|
|
2017-04-15 08:45:10 +00:00
|
|
|
protected Logger logger;
|
2017-07-10 20:58:44 +00:00
|
|
|
protected IIndexerConfigurationService configurationService;
|
2017-04-15 08:45:10 +00:00
|
|
|
protected IProtectionService protectionService;
|
2020-12-11 22:14:21 +00:00
|
|
|
protected ICacheService cacheService;
|
2017-07-10 20:58:44 +00:00
|
|
|
|
|
|
|
protected ConfigurationData configData;
|
2017-04-15 08:45:10 +00:00
|
|
|
|
|
|
|
protected string CookieHeader
|
|
|
|
{
|
2020-02-25 16:08:03 +00:00
|
|
|
get => configData.CookieHeader.Value;
|
|
|
|
set => configData.CookieHeader.Value = value;
|
2017-04-15 08:45:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
public string LastError
|
|
|
|
{
|
2020-02-25 16:08:03 +00:00
|
|
|
get => configData.LastError.Value;
|
2017-04-15 08:45:10 +00:00
|
|
|
set
|
|
|
|
{
|
2020-02-10 22:16:19 +00:00
|
|
|
var SaveNeeded = configData.LastError.Value != value && IsConfigured;
|
2017-04-15 08:45:10 +00:00
|
|
|
configData.LastError.Value = value;
|
|
|
|
if (SaveNeeded)
|
|
|
|
SaveConfig();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-06-24 02:37:27 +00:00
|
|
|
public virtual bool IsHealthy => errorCount == 0 && expireAt > DateTime.Now;
|
|
|
|
public virtual bool IsFailing => errorCount > 0 && expireAt > DateTime.Now;
|
|
|
|
|
|
|
|
|
2017-07-10 20:58:44 +00:00
|
|
|
public abstract TorznabCapabilities TorznabCaps { get; protected set; }
|
2017-04-15 08:45:10 +00:00
|
|
|
|
|
|
|
// standard constructor used by most indexers
|
2023-03-09 14:06:12 +00:00
|
|
|
public BaseIndexer(IIndexerConfigurationService configService, Logger logger, ConfigurationData configData, IProtectionService p, ICacheService cs)
|
2017-04-15 08:45:10 +00:00
|
|
|
{
|
2017-07-10 20:58:44 +00:00
|
|
|
this.logger = logger;
|
|
|
|
configurationService = configService;
|
|
|
|
protectionService = p;
|
2020-12-11 22:14:21 +00:00
|
|
|
cacheService = cs;
|
2017-07-10 20:58:44 +00:00
|
|
|
|
2023-03-09 14:06:12 +00:00
|
|
|
if (SiteLink.IsNotNullOrWhiteSpace() && !SiteLink.EndsWith("/", StringComparison.Ordinal))
|
2017-04-15 08:45:10 +00:00
|
|
|
throw new Exception("Site link must end with a slash.");
|
|
|
|
|
2023-03-09 14:06:12 +00:00
|
|
|
DefaultSiteLink = SiteLink;
|
|
|
|
|
2017-04-15 08:45:10 +00:00
|
|
|
this.configData = configData;
|
2017-07-10 20:58:44 +00:00
|
|
|
if (configData != null)
|
|
|
|
LoadValuesFromJson(null);
|
2017-04-15 08:45:10 +00:00
|
|
|
}
|
|
|
|
|
2020-02-25 16:08:03 +00:00
|
|
|
public virtual Task<ConfigurationData> GetConfigurationForSetup() => Task.FromResult<ConfigurationData>(configData);
|
2017-04-15 08:45:10 +00:00
|
|
|
|
|
|
|
public virtual void ResetBaseConfig()
|
|
|
|
{
|
|
|
|
CookieHeader = string.Empty;
|
|
|
|
IsConfigured = false;
|
2021-06-24 02:37:27 +00:00
|
|
|
errorCount = 0;
|
|
|
|
expireAt = DateTime.MinValue;
|
2017-04-15 08:45:10 +00:00
|
|
|
}
|
|
|
|
|
2020-02-25 16:08:03 +00:00
|
|
|
public virtual void SaveConfig() => configurationService.Save(this as IIndexer, configData.ToJson(protectionService, forDisplay: false));
|
2017-04-15 08:45:10 +00:00
|
|
|
|
2020-02-10 22:16:19 +00:00
|
|
|
public virtual void LoadValuesFromJson(JToken jsonConfig, bool useProtectionService = false)
|
2017-04-15 08:45:10 +00:00
|
|
|
{
|
|
|
|
IProtectionService ps = null;
|
|
|
|
if (useProtectionService)
|
|
|
|
ps = protectionService;
|
2020-12-11 23:12:40 +00:00
|
|
|
configData.LoadConfigDataValuesFromJson(jsonConfig, ps);
|
2017-04-15 08:45:10 +00:00
|
|
|
if (string.IsNullOrWhiteSpace(configData.SiteLink.Value))
|
|
|
|
{
|
|
|
|
configData.SiteLink.Value = DefaultSiteLink;
|
|
|
|
}
|
2017-08-30 16:46:36 +00:00
|
|
|
|
2017-06-28 05:31:38 +00:00
|
|
|
if (!configData.SiteLink.Value.EndsWith("/", StringComparison.Ordinal))
|
2017-04-15 08:45:10 +00:00
|
|
|
configData.SiteLink.Value += "/";
|
|
|
|
|
2017-08-30 16:46:36 +00:00
|
|
|
// reset site link to default if it's a legacy (defunc link)
|
|
|
|
if (LegacySiteLinks != null && LegacySiteLinks.Contains(configData.SiteLink.Value))
|
|
|
|
{
|
|
|
|
logger.Debug(string.Format("changing legacy site link from {0} to {1}", configData.SiteLink.Value, DefaultSiteLink));
|
|
|
|
configData.SiteLink.Value = DefaultSiteLink;
|
|
|
|
}
|
|
|
|
|
2017-07-19 17:24:03 +00:00
|
|
|
// check whether the site link is well-formatted
|
|
|
|
var siteUri = new Uri(configData.SiteLink.Value);
|
2017-04-15 08:45:10 +00:00
|
|
|
SiteLink = configData.SiteLink.Value;
|
2021-05-08 20:24:18 +00:00
|
|
|
|
|
|
|
Tags = configData.Tags.Values.Select(t => t.ToLowerInvariant()).ToArray();
|
2017-04-15 08:45:10 +00:00
|
|
|
}
|
|
|
|
|
2017-07-10 20:58:44 +00:00
|
|
|
public void LoadFromSavedConfiguration(JToken jsonConfig)
|
2017-04-15 08:45:10 +00:00
|
|
|
{
|
|
|
|
if (jsonConfig is JArray)
|
|
|
|
{
|
2022-04-24 20:09:08 +00:00
|
|
|
LoadValuesFromJson(jsonConfig, true);
|
2017-04-15 08:45:10 +00:00
|
|
|
IsConfigured = true;
|
|
|
|
}
|
2022-04-24 20:09:08 +00:00
|
|
|
else
|
|
|
|
logger.Warn("Some of the configuration files (.json) are in the old format. Please, update Jackett.");
|
2018-05-30 11:28:20 +00:00
|
|
|
}
|
|
|
|
|
2017-07-10 20:58:44 +00:00
|
|
|
protected async Task ConfigureIfOK(string cookies, bool isLoggedin, Func<Task> onError)
|
|
|
|
{
|
|
|
|
if (isLoggedin)
|
|
|
|
{
|
|
|
|
CookieHeader = cookies;
|
|
|
|
IsConfigured = true;
|
|
|
|
SaveConfig();
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
await onError();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
protected virtual IEnumerable<ReleaseInfo> FilterResults(TorznabQuery query, IEnumerable<ReleaseInfo> results)
|
|
|
|
{
|
2024-04-13 20:59:27 +00:00
|
|
|
var filteredResults = results.Where(r => IsValidRelease(r, query.InteractiveSearch)).ToList();
|
2017-07-10 20:58:44 +00:00
|
|
|
|
2020-11-02 13:20:13 +00:00
|
|
|
// filter results with wrong categories
|
|
|
|
if (query.Categories.Length > 0)
|
|
|
|
{
|
|
|
|
// expand parent categories from the query
|
|
|
|
var expandedQueryCats = TorznabCaps.Categories.ExpandTorznabQueryCategories(query);
|
2020-11-01 11:07:24 +00:00
|
|
|
|
2020-11-02 13:20:13 +00:00
|
|
|
filteredResults = filteredResults.Where(result =>
|
|
|
|
result.Category?.Any() != true ||
|
|
|
|
expandedQueryCats.Intersect(result.Category).Any()
|
2023-11-16 13:11:59 +00:00
|
|
|
).ToList();
|
2020-11-02 13:20:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// eliminate excess results
|
|
|
|
if (query.Limit > 0)
|
2023-11-06 14:03:24 +00:00
|
|
|
{
|
2023-11-16 13:11:59 +00:00
|
|
|
filteredResults = filteredResults.Take(query.Limit).ToList();
|
2023-11-06 14:03:24 +00:00
|
|
|
}
|
2017-07-10 20:58:44 +00:00
|
|
|
|
|
|
|
return filteredResults;
|
|
|
|
}
|
|
|
|
|
2020-11-02 13:20:13 +00:00
|
|
|
protected virtual IEnumerable<ReleaseInfo> FixResults(TorznabQuery query, IEnumerable<ReleaseInfo> results)
|
|
|
|
{
|
|
|
|
var fixedResults = results.Select(r =>
|
|
|
|
{
|
|
|
|
// add origin
|
|
|
|
r.Origin = this;
|
|
|
|
|
|
|
|
// fix publish date
|
|
|
|
// some trackers do not keep their clocks up to date and can be ~20 minutes out!
|
2023-03-05 20:48:08 +00:00
|
|
|
if (!EnvironmentUtil.IsDebug && r.PublishDate > DateTime.Now)
|
2023-11-06 14:03:24 +00:00
|
|
|
{
|
2020-11-02 13:20:13 +00:00
|
|
|
r.PublishDate = DateTime.Now;
|
2023-11-06 14:03:24 +00:00
|
|
|
}
|
2020-11-02 13:20:13 +00:00
|
|
|
|
2020-11-02 16:43:13 +00:00
|
|
|
// generate magnet link from info hash (not allowed for private sites)
|
|
|
|
if (r.MagnetUri == null && !string.IsNullOrWhiteSpace(r.InfoHash) && Type != "private")
|
2023-11-06 14:03:24 +00:00
|
|
|
{
|
2020-11-02 16:43:13 +00:00
|
|
|
r.MagnetUri = MagnetUtil.InfoHashToPublicMagnet(r.InfoHash, r.Title);
|
2023-11-06 14:03:24 +00:00
|
|
|
}
|
2020-11-02 16:43:13 +00:00
|
|
|
|
|
|
|
// generate info hash from magnet link
|
|
|
|
if (r.MagnetUri != null && string.IsNullOrWhiteSpace(r.InfoHash))
|
2023-11-06 14:03:24 +00:00
|
|
|
{
|
2020-11-02 16:43:13 +00:00
|
|
|
r.InfoHash = MagnetUtil.MagnetToInfoHash(r.MagnetUri);
|
2023-11-06 14:03:24 +00:00
|
|
|
}
|
2020-11-02 16:43:13 +00:00
|
|
|
|
2020-12-22 17:49:59 +00:00
|
|
|
// set guid
|
|
|
|
if (r.Guid == null)
|
|
|
|
{
|
2021-10-26 09:06:33 +00:00
|
|
|
if (r.Link != null)
|
2023-11-06 14:03:24 +00:00
|
|
|
{
|
2020-12-22 17:49:59 +00:00
|
|
|
r.Guid = r.Link;
|
2023-11-06 14:03:24 +00:00
|
|
|
}
|
2020-12-22 17:49:59 +00:00
|
|
|
else if (r.MagnetUri != null)
|
2023-11-06 14:03:24 +00:00
|
|
|
{
|
2020-12-22 17:49:59 +00:00
|
|
|
r.Guid = r.MagnetUri;
|
2023-11-06 14:03:24 +00:00
|
|
|
}
|
2021-10-26 09:06:33 +00:00
|
|
|
else if (r.Details != null)
|
2023-11-06 14:03:24 +00:00
|
|
|
{
|
2021-10-26 09:06:33 +00:00
|
|
|
r.Guid = r.Details;
|
2023-11-06 14:03:24 +00:00
|
|
|
}
|
2020-12-22 17:49:59 +00:00
|
|
|
}
|
|
|
|
|
2020-11-02 13:20:13 +00:00
|
|
|
return r;
|
|
|
|
});
|
2023-11-06 14:03:24 +00:00
|
|
|
|
2020-11-02 13:20:13 +00:00
|
|
|
return fixedResults;
|
|
|
|
}
|
|
|
|
|
2024-04-13 20:59:27 +00:00
|
|
|
protected virtual bool IsValidRelease(ReleaseInfo release, bool interactiveSearch)
|
2023-11-06 14:03:24 +00:00
|
|
|
{
|
|
|
|
if (release.Title.IsNullOrWhiteSpace())
|
|
|
|
{
|
2024-04-01 00:46:50 +00:00
|
|
|
logger.Error("[{0}] Invalid Release: '{1}'. No title provided.", Id, release.Details);
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2024-04-13 20:59:27 +00:00
|
|
|
if (interactiveSearch)
|
|
|
|
{
|
|
|
|
// Show releases with issues in the interactive search
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2024-04-01 00:46:50 +00:00
|
|
|
if (release.Size == null)
|
|
|
|
{
|
|
|
|
logger.Warn("[{0}] Invalid Release: '{1}'. No size provided.", Id, release.Details);
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (release.Category == null || !release.Category.Any())
|
|
|
|
{
|
|
|
|
logger.Warn("[{0}] Invalid Release: '{1}'. No categories provided.", Id, release.Details);
|
2023-11-06 14:03:24 +00:00
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2018-04-06 15:43:18 +00:00
|
|
|
public virtual bool CanHandleQuery(TorznabQuery query)
|
2017-07-10 20:58:44 +00:00
|
|
|
{
|
|
|
|
if (query == null)
|
|
|
|
return false;
|
2017-08-11 22:30:43 +00:00
|
|
|
if (query.QueryType == "caps")
|
|
|
|
return true;
|
|
|
|
|
2017-07-10 20:58:44 +00:00
|
|
|
var caps = TorznabCaps;
|
2020-10-18 20:47:36 +00:00
|
|
|
if (caps.TvSearchImdbAvailable && query.IsImdbQuery && query.IsTVSearch)
|
2020-01-09 03:32:02 +00:00
|
|
|
return true;
|
2020-10-18 17:26:22 +00:00
|
|
|
if (caps.MovieSearchImdbAvailable && query.IsImdbQuery && query.IsMovieSearch)
|
2018-04-17 10:43:00 +00:00
|
|
|
return true;
|
2020-11-08 22:27:54 +00:00
|
|
|
if (!caps.MovieSearchImdbAvailable && query.IsImdbQuery && query.QueryType != "TorrentPotato") // potato query should always contain imdb+search term
|
2018-04-17 10:43:00 +00:00
|
|
|
return false;
|
2017-08-11 16:13:22 +00:00
|
|
|
if (caps.SearchAvailable && query.IsSearch)
|
|
|
|
return true;
|
2020-10-18 20:47:36 +00:00
|
|
|
if (caps.TvSearchAvailable && query.IsTVSearch)
|
2017-08-11 16:13:22 +00:00
|
|
|
return true;
|
|
|
|
if (caps.MovieSearchAvailable && query.IsMovieSearch)
|
|
|
|
return true;
|
2017-10-18 16:30:41 +00:00
|
|
|
if (caps.MusicSearchAvailable && query.IsMusicSearch)
|
|
|
|
return true;
|
2020-08-16 21:44:12 +00:00
|
|
|
if (caps.BookSearchAvailable && query.IsBookSearch)
|
|
|
|
return true;
|
2023-11-07 19:29:16 +00:00
|
|
|
if (caps.TvSearchTvRageAvailable && query.IsTVRageQuery)
|
2017-08-11 16:13:22 +00:00
|
|
|
return true;
|
2023-11-07 19:29:16 +00:00
|
|
|
if (caps.TvSearchTvdbAvailable && query.IsTvdbQuery)
|
2020-08-16 22:07:04 +00:00
|
|
|
return true;
|
2020-10-18 17:26:22 +00:00
|
|
|
if (caps.MovieSearchImdbAvailable && query.IsImdbQuery)
|
2017-08-11 16:13:22 +00:00
|
|
|
return true;
|
2022-06-12 07:58:12 +00:00
|
|
|
if (caps.MovieSearchTmdbAvailable && query.IsTmdbQuery && query.IsMovieSearch)
|
|
|
|
return true;
|
|
|
|
if (caps.TvSearchTmdbAvailable && query.IsTmdbQuery && query.IsTVSearch)
|
2020-08-16 22:07:04 +00:00
|
|
|
return true;
|
2017-08-11 16:13:22 +00:00
|
|
|
|
|
|
|
return false;
|
2017-07-10 20:58:44 +00:00
|
|
|
}
|
|
|
|
|
2020-11-08 22:27:54 +00:00
|
|
|
protected bool CanHandleCategories(TorznabQuery query, bool isMetaIndexer = false)
|
|
|
|
{
|
|
|
|
// https://torznab.github.io/spec-1.3-draft/torznab/Specification-v1.3.html#cat-parameter
|
|
|
|
if (query.HasSpecifiedCategories)
|
|
|
|
{
|
|
|
|
var supportedCats = TorznabCaps.Categories.SupportedCategories(query.Categories);
|
|
|
|
if (supportedCats.Length == 0)
|
|
|
|
{
|
|
|
|
if (!isMetaIndexer)
|
2023-03-09 14:06:12 +00:00
|
|
|
logger.Error($"All categories provided are unsupported in {Name}: {string.Join(",", query.Categories)}");
|
2020-11-08 22:27:54 +00:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
if (supportedCats.Length != query.Categories.Length && !isMetaIndexer)
|
|
|
|
{
|
|
|
|
var unsupportedCats = query.Categories.Except(supportedCats);
|
2023-03-09 14:06:12 +00:00
|
|
|
logger.Warn($"Some of the categories provided are unsupported in {Name}: {string.Join(",", unsupportedCats)}");
|
2020-11-08 22:27:54 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2017-07-10 20:58:44 +00:00
|
|
|
public void Unconfigure()
|
|
|
|
{
|
|
|
|
IsConfigured = false;
|
2017-11-13 11:55:54 +00:00
|
|
|
SiteLink = DefaultSiteLink;
|
2017-11-13 15:55:02 +00:00
|
|
|
CookieHeader = ""; // clear cookies
|
2017-07-10 20:58:44 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
public abstract Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson);
|
|
|
|
|
2020-11-08 22:27:54 +00:00
|
|
|
public virtual async Task<IndexerResult> ResultsForQuery(TorznabQuery query, bool isMetaIndexer)
|
2017-07-10 20:58:44 +00:00
|
|
|
{
|
2022-04-24 21:11:44 +00:00
|
|
|
// we make a copy just in case some C# indexer modifies the object.
|
|
|
|
// without the copy, if you make a request with several indexers, all indexers share the query object.
|
|
|
|
var queryCopy = query.Clone();
|
|
|
|
|
|
|
|
if (!CanHandleQuery(queryCopy) || !CanHandleCategories(queryCopy, isMetaIndexer))
|
2024-02-28 23:11:02 +00:00
|
|
|
return new IndexerResult(this, Array.Empty<ReleaseInfo>(), 0, false);
|
2023-03-05 12:30:40 +00:00
|
|
|
|
|
|
|
if (!SupportsPagination && queryCopy.Offset > 0)
|
2024-02-28 23:11:02 +00:00
|
|
|
return new IndexerResult(this, Array.Empty<ReleaseInfo>(), 0, false);
|
2020-12-11 22:14:21 +00:00
|
|
|
|
2022-04-24 21:11:44 +00:00
|
|
|
if (queryCopy.Cache)
|
2021-08-30 03:09:48 +00:00
|
|
|
{
|
2022-04-24 21:11:44 +00:00
|
|
|
var cachedReleases = cacheService.Search(this, queryCopy);
|
2021-08-30 03:09:48 +00:00
|
|
|
if (cachedReleases != null)
|
2024-02-28 23:11:02 +00:00
|
|
|
return new IndexerResult(this, cachedReleases, 0, true);
|
2021-08-30 03:09:48 +00:00
|
|
|
}
|
2020-11-02 13:20:13 +00:00
|
|
|
|
2017-08-24 10:28:41 +00:00
|
|
|
try
|
2017-07-10 20:58:44 +00:00
|
|
|
{
|
2024-02-28 23:11:02 +00:00
|
|
|
var sw = new Stopwatch();
|
|
|
|
|
|
|
|
sw.Start();
|
|
|
|
|
2022-04-24 21:11:44 +00:00
|
|
|
var results = await PerformQuery(queryCopy);
|
2024-02-28 23:11:02 +00:00
|
|
|
|
|
|
|
sw.Stop();
|
|
|
|
|
|
|
|
results = FilterResults(queryCopy, results).ToList();
|
|
|
|
results = FixResults(queryCopy, results).ToList();
|
2022-04-24 21:11:44 +00:00
|
|
|
cacheService.CacheResults(this, queryCopy, results.ToList());
|
2021-06-24 02:37:27 +00:00
|
|
|
errorCount = 0;
|
|
|
|
expireAt = DateTime.Now.Add(HealthyStatusValidity);
|
2024-02-28 23:11:02 +00:00
|
|
|
return new IndexerResult(this, results, sw.ElapsedMilliseconds, false);
|
2017-08-24 10:28:41 +00:00
|
|
|
}
|
2023-02-19 11:26:49 +00:00
|
|
|
catch (TooManyRequestsException ex)
|
|
|
|
{
|
|
|
|
var delay = ex.RetryAfter.TotalSeconds;
|
|
|
|
expireAt = DateTime.Now.AddSeconds(delay);
|
|
|
|
throw new IndexerException(this, ex);
|
|
|
|
}
|
2017-08-24 10:28:41 +00:00
|
|
|
catch (Exception ex)
|
|
|
|
{
|
2021-06-24 02:37:27 +00:00
|
|
|
var delay = Math.Min(MaxStatusValidity.TotalSeconds, ErrorStatusValidity.TotalSeconds * Math.Pow(2, errorCount++));
|
|
|
|
expireAt = DateTime.Now.AddSeconds(delay);
|
2017-08-24 10:28:41 +00:00
|
|
|
throw new IndexerException(this, ex);
|
|
|
|
}
|
2017-07-10 20:58:44 +00:00
|
|
|
}
|
2017-10-29 06:21:18 +00:00
|
|
|
|
2017-07-10 20:58:44 +00:00
|
|
|
protected abstract Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query);
|
|
|
|
}
|
|
|
|
|
|
|
|
public abstract class BaseWebIndexer : BaseIndexer, IWebIndexer
|
|
|
|
{
|
2023-03-09 14:06:12 +00:00
|
|
|
protected BaseWebIndexer(IIndexerConfigurationService configService, WebClient client, Logger logger,
|
2020-12-11 22:14:21 +00:00
|
|
|
ConfigurationData configData, IProtectionService p, ICacheService cacheService,
|
2023-03-10 11:20:29 +00:00
|
|
|
string downloadBase = null)
|
2023-03-09 14:06:12 +00:00
|
|
|
: base(configService: configService, logger: logger, configData: configData, p: p, cs: cacheService)
|
2017-07-10 20:58:44 +00:00
|
|
|
{
|
2020-02-10 22:16:19 +00:00
|
|
|
webclient = client;
|
|
|
|
downloadUrlBase = downloadBase;
|
2017-07-10 20:58:44 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// minimal constructor used by e.g. cardigann generic indexer
|
2020-12-11 22:14:21 +00:00
|
|
|
protected BaseWebIndexer(IIndexerConfigurationService configService, WebClient client, Logger logger,
|
|
|
|
IProtectionService p, ICacheService cacheService)
|
2023-03-09 14:06:12 +00:00
|
|
|
: base(configService: configService, logger: logger, configData: null, p: p, cs: cacheService)
|
2023-03-07 18:21:46 +00:00
|
|
|
{
|
|
|
|
webclient = client;
|
|
|
|
}
|
2017-07-10 20:58:44 +00:00
|
|
|
|
2021-03-15 06:27:18 +00:00
|
|
|
protected virtual int DefaultNumberOfRetryAttempts => 2;
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
/// Number of retry attempts to make if a web request fails.
|
|
|
|
/// </summary>
|
|
|
|
/// <remarks>
|
|
|
|
/// Number of retries can be overridden for unstable indexers by overriding this property. Note that retry attempts include an
|
2021-11-16 18:17:10 +00:00
|
|
|
/// exponentially increasing delay.
|
|
|
|
///
|
2021-03-15 06:27:18 +00:00
|
|
|
/// Alternatively, <see cref="EnableConfigurableRetryAttempts()" /> can be called in the constructor to add user configurable options.
|
|
|
|
/// </remarks>
|
|
|
|
protected virtual int NumberOfRetryAttempts
|
|
|
|
{
|
|
|
|
get
|
|
|
|
{
|
|
|
|
var configItem = configData.GetDynamic("retryAttempts");
|
|
|
|
if (configItem == null)
|
|
|
|
{
|
|
|
|
// No config specified so use the default.
|
|
|
|
return DefaultNumberOfRetryAttempts;
|
|
|
|
}
|
|
|
|
|
2021-03-16 23:29:26 +00:00
|
|
|
var configValue = ((SingleSelectConfigurationItem)configItem).Value;
|
2021-03-15 06:27:18 +00:00
|
|
|
|
2021-03-16 23:29:26 +00:00
|
|
|
if (int.TryParse(configValue, out var parsedConfigValue) && parsedConfigValue > 0)
|
2021-03-15 06:27:18 +00:00
|
|
|
{
|
|
|
|
return parsedConfigValue;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
// No config specified so use the default.
|
|
|
|
return DefaultNumberOfRetryAttempts;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-12-10 15:52:12 +00:00
|
|
|
private ResiliencePipeline<WebResult> RetryStrategy
|
2021-03-15 06:27:18 +00:00
|
|
|
{
|
|
|
|
get
|
|
|
|
{
|
2023-12-10 15:52:12 +00:00
|
|
|
var retryPipeline = new ResiliencePipelineBuilder<WebResult>()
|
|
|
|
.AddRetry(new RetryStrategyOptions<WebResult>
|
|
|
|
{
|
|
|
|
ShouldHandle = args => args.Outcome switch
|
|
|
|
{
|
|
|
|
{ Result: { HasHttpServerError: true } } => PredicateResult.True(),
|
|
|
|
{ Result: { Status: System.Net.HttpStatusCode.RequestTimeout } } => PredicateResult.True(),
|
|
|
|
{ Exception: { } } => PredicateResult.True(),
|
|
|
|
_ => PredicateResult.False()
|
|
|
|
},
|
|
|
|
Delay = TimeSpan.FromSeconds(2),
|
|
|
|
MaxRetryAttempts = NumberOfRetryAttempts,
|
|
|
|
BackoffType = DelayBackoffType.Exponential,
|
|
|
|
UseJitter = true,
|
|
|
|
OnRetry = args =>
|
2021-03-15 06:27:18 +00:00
|
|
|
{
|
2023-12-10 15:52:12 +00:00
|
|
|
if (args.Outcome.Exception != null)
|
2021-03-27 04:36:57 +00:00
|
|
|
{
|
2023-12-10 15:52:12 +00:00
|
|
|
logger.Warn("Request to {0} failed with exception '{1}'. Retrying in {2}s.", Name, args.Outcome.Exception.Message, args.RetryDelay.TotalSeconds);
|
2021-03-27 04:36:57 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2023-12-10 15:52:12 +00:00
|
|
|
logger.Warn("Request to {0} failed with status {1}. Retrying in {2}s.", Name, args.Outcome.Result?.Status, args.RetryDelay.TotalSeconds);
|
2021-03-27 04:36:57 +00:00
|
|
|
}
|
2023-12-10 15:52:12 +00:00
|
|
|
|
|
|
|
return default;
|
|
|
|
}
|
|
|
|
})
|
|
|
|
.Build();
|
|
|
|
|
|
|
|
return retryPipeline;
|
2021-03-15 06:27:18 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
/// Adds configuration options to allow the user to manually configure request retries.
|
|
|
|
/// </summary>
|
|
|
|
/// <remarks>
|
|
|
|
/// This should only be enabled for indexers known to be unstable. To control the default value, override <see cref="DefaultNumberOfRetryAttempts" />.
|
|
|
|
/// </remarks>
|
|
|
|
protected void EnableConfigurableRetryAttempts()
|
|
|
|
{
|
2021-03-16 23:29:26 +00:00
|
|
|
var attemptSelect = new SingleSelectConfigurationItem(
|
|
|
|
"Number of retries",
|
2021-03-15 06:27:18 +00:00
|
|
|
new Dictionary<string, string>
|
|
|
|
{
|
|
|
|
{"0", "No retries (fail fast)"},
|
|
|
|
{"1", "1 retry (0.5s delay)"},
|
|
|
|
{"2", "2 retries (1s delay)"},
|
|
|
|
{"3", "3 retries (2s delay)"},
|
|
|
|
{"4", "4 retries (4s delay)"},
|
|
|
|
{"5", "5 retries (8s delay)"}
|
|
|
|
})
|
|
|
|
{
|
|
|
|
Value = DefaultNumberOfRetryAttempts.ToString()
|
|
|
|
};
|
|
|
|
configData.AddDynamic("retryAttempts", attemptSelect);
|
|
|
|
}
|
|
|
|
|
2020-02-10 22:16:19 +00:00
|
|
|
public virtual async Task<byte[]> Download(Uri link)
|
2017-07-10 20:58:44 +00:00
|
|
|
{
|
|
|
|
var uncleanLink = UncleanLink(link);
|
|
|
|
return await Download(uncleanLink, RequestType.GET);
|
2017-04-15 08:45:10 +00:00
|
|
|
}
|
|
|
|
|
2021-05-16 18:13:54 +00:00
|
|
|
protected async Task<byte[]> Download(Uri link, RequestType method, string referer = null, Dictionary<string, string> headers = null)
|
2017-04-15 08:45:10 +00:00
|
|
|
{
|
2017-11-09 12:28:15 +00:00
|
|
|
// return magnet link
|
|
|
|
if (link.Scheme == "magnet")
|
|
|
|
return Encoding.UTF8.GetBytes(link.OriginalString);
|
|
|
|
|
2017-04-15 08:45:10 +00:00
|
|
|
// do some extra escaping, needed for HD-Torrents
|
|
|
|
var requestLink = link.ToString()
|
|
|
|
.Replace("(", "%28")
|
|
|
|
.Replace(")", "%29")
|
|
|
|
.Replace("'", "%27");
|
2020-10-08 20:21:40 +00:00
|
|
|
var response = await RequestWithCookiesAndRetryAsync(requestLink, null, method, referer, null, headers);
|
2019-12-05 03:26:59 +00:00
|
|
|
|
2017-11-30 11:11:42 +00:00
|
|
|
if (response.IsRedirect)
|
|
|
|
{
|
2024-06-04 08:30:49 +00:00
|
|
|
response = await FollowIfRedirect(response);
|
2017-11-30 11:11:42 +00:00
|
|
|
}
|
2022-09-25 04:11:26 +00:00
|
|
|
|
|
|
|
if (response.IsRedirect)
|
|
|
|
{
|
|
|
|
var redirectingTo = new Uri(response.RedirectingTo);
|
|
|
|
if (redirectingTo.Scheme == "magnet")
|
|
|
|
return Encoding.UTF8.GetBytes(redirectingTo.OriginalString);
|
|
|
|
|
2024-06-04 08:30:49 +00:00
|
|
|
response = await FollowIfRedirect(response);
|
2022-09-25 04:11:26 +00:00
|
|
|
}
|
|
|
|
|
2017-04-15 08:45:10 +00:00
|
|
|
if (response.Status != System.Net.HttpStatusCode.OK && response.Status != System.Net.HttpStatusCode.Continue && response.Status != System.Net.HttpStatusCode.PartialContent)
|
|
|
|
{
|
2020-02-10 22:16:19 +00:00
|
|
|
logger.Error("Failed download cookies: " + CookieHeader);
|
2020-03-14 23:58:50 +00:00
|
|
|
if (response.ContentBytes != null)
|
|
|
|
logger.Error("Failed download response:\n" + Encoding.UTF8.GetString(response.ContentBytes));
|
2017-07-10 20:58:44 +00:00
|
|
|
throw new Exception($"Remote server returned {response.Status.ToString()}" + (response.IsRedirect ? " => " + response.RedirectingTo : ""));
|
2017-04-15 08:45:10 +00:00
|
|
|
}
|
|
|
|
|
2020-03-14 23:58:50 +00:00
|
|
|
return response.ContentBytes;
|
2017-04-15 08:45:10 +00:00
|
|
|
}
|
|
|
|
|
2022-01-16 23:38:58 +00:00
|
|
|
public virtual async Task<WebResult> DownloadImage(Uri link)
|
|
|
|
{
|
|
|
|
var uncleanLink = UncleanLink(link);
|
|
|
|
var requestLink = uncleanLink.ToString();
|
|
|
|
var referer = SiteLink;
|
|
|
|
|
|
|
|
var response = await RequestWithCookiesAsync(requestLink, null, RequestType.GET, referer);
|
|
|
|
if (response.IsRedirect)
|
2024-06-04 08:30:49 +00:00
|
|
|
response = await FollowIfRedirect(response);
|
2022-01-16 23:38:58 +00:00
|
|
|
|
|
|
|
return response;
|
|
|
|
}
|
|
|
|
|
2020-06-11 15:09:27 +00:00
|
|
|
protected async Task<WebResult> RequestWithCookiesAndRetryAsync(
|
|
|
|
string url, string cookieOverride = null, RequestType method = RequestType.GET,
|
|
|
|
string referer = null, IEnumerable<KeyValuePair<string, string>> data = null,
|
|
|
|
Dictionary<string, string> headers = null, string rawbody = null, bool? emulateBrowser = null)
|
2017-04-15 08:45:10 +00:00
|
|
|
{
|
2023-12-10 15:52:12 +00:00
|
|
|
return await RetryStrategy
|
|
|
|
.ExecuteAsync(async _ => await RequestWithCookiesAsync(url, cookieOverride, method, referer, data, headers, rawbody, emulateBrowser))
|
|
|
|
.ConfigureAwait(false);
|
2017-04-15 08:45:10 +00:00
|
|
|
}
|
|
|
|
|
2020-09-21 16:39:47 +00:00
|
|
|
protected virtual async Task<WebResult> RequestWithCookiesAsync(
|
2020-06-11 15:09:27 +00:00
|
|
|
string url, string cookieOverride = null, RequestType method = RequestType.GET,
|
|
|
|
string referer = null, IEnumerable<KeyValuePair<string, string>> data = null,
|
|
|
|
Dictionary<string, string> headers = null, string rawbody = null, bool? emulateBrowser = null)
|
2017-04-15 08:45:10 +00:00
|
|
|
{
|
2020-06-11 15:09:27 +00:00
|
|
|
var request = new WebRequest
|
2017-04-15 08:45:10 +00:00
|
|
|
{
|
|
|
|
Url = url,
|
|
|
|
Type = method,
|
|
|
|
Cookies = cookieOverride ?? CookieHeader,
|
|
|
|
PostData = data,
|
|
|
|
Referer = referer,
|
|
|
|
Headers = headers,
|
|
|
|
RawBody = rawbody,
|
|
|
|
Encoding = Encoding
|
|
|
|
};
|
|
|
|
|
|
|
|
if (emulateBrowser.HasValue)
|
|
|
|
request.EmulateBrowser = emulateBrowser.Value;
|
2020-06-11 15:09:27 +00:00
|
|
|
var result = await webclient.GetResultAsync(request);
|
|
|
|
CheckSiteDown(result);
|
2017-04-15 08:45:10 +00:00
|
|
|
UpdateCookieHeader(result.Cookies, cookieOverride);
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2021-03-14 21:19:48 +00:00
|
|
|
protected async Task<WebResult> RequestLoginAndFollowRedirect(string url, IEnumerable<KeyValuePair<string, string>> data, string cookies, bool returnCookiesFromFirstCall, string redirectUrlOverride = null, string referer = null, bool accumulateCookies = false, Dictionary<string, string> headers = null)
|
2017-04-15 08:45:10 +00:00
|
|
|
{
|
2020-10-19 21:19:10 +00:00
|
|
|
var request = new WebRequest
|
2017-04-15 08:45:10 +00:00
|
|
|
{
|
|
|
|
Url = url,
|
|
|
|
Type = RequestType.POST,
|
2023-02-20 20:48:36 +00:00
|
|
|
Cookies = cookies ?? CookieHeader,
|
2017-04-15 08:45:10 +00:00
|
|
|
Referer = referer,
|
|
|
|
PostData = data,
|
2021-03-14 21:19:48 +00:00
|
|
|
Encoding = Encoding,
|
|
|
|
Headers = headers,
|
2017-04-15 08:45:10 +00:00
|
|
|
};
|
2020-06-11 15:09:27 +00:00
|
|
|
var response = await webclient.GetResultAsync(request);
|
|
|
|
CheckSiteDown(response);
|
2017-04-15 08:45:10 +00:00
|
|
|
if (accumulateCookies)
|
|
|
|
{
|
|
|
|
response.Cookies = ResolveCookies((request.Cookies == null ? "" : request.Cookies + " ") + response.Cookies);
|
|
|
|
}
|
|
|
|
var firstCallCookies = response.Cookies;
|
|
|
|
|
|
|
|
if (response.IsRedirect)
|
|
|
|
{
|
2024-06-04 08:30:49 +00:00
|
|
|
response = await FollowIfRedirect(response, request.Url, redirectUrlOverride, response.Cookies, accumulateCookies);
|
2017-04-15 08:45:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (returnCookiesFromFirstCall)
|
|
|
|
{
|
|
|
|
response.Cookies = ResolveCookies(firstCallCookies + (accumulateCookies ? " " + response.Cookies : ""));
|
|
|
|
}
|
2017-07-10 20:58:44 +00:00
|
|
|
|
2017-04-15 08:45:10 +00:00
|
|
|
return response;
|
|
|
|
}
|
|
|
|
|
2020-06-11 15:09:27 +00:00
|
|
|
protected static void CheckSiteDown(WebResult response)
|
2017-09-13 07:57:39 +00:00
|
|
|
{
|
|
|
|
if (response.Status == System.Net.HttpStatusCode.BadGateway
|
|
|
|
|| response.Status == System.Net.HttpStatusCode.GatewayTimeout
|
|
|
|
|| (int)response.Status == 521 // used by cloudflare to signal the original webserver is refusing the connection
|
|
|
|
|| (int)response.Status == 522 // used by cloudflare to signal the original webserver is not reachable at all (timeout)
|
2018-04-01 13:40:43 +00:00
|
|
|
|| (int)response.Status == 523 // used by cloudflare to signal the original webserver is not reachable at all (Origin is unreachable)
|
2017-10-29 06:21:18 +00:00
|
|
|
)
|
2017-09-13 07:57:39 +00:00
|
|
|
{
|
|
|
|
throw new Exception("Request to " + response.Request.Url + " failed (Error " + response.Status + ") - The tracker seems to be down.");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-06-04 08:30:49 +00:00
|
|
|
protected async Task<WebResult> FollowIfRedirect(WebResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null, bool accumulateCookies = false, int maxRedirects = 5)
|
2017-07-10 20:58:44 +00:00
|
|
|
{
|
2023-07-07 22:37:22 +00:00
|
|
|
for (var i = 0; i < maxRedirects; i++)
|
2017-04-15 08:45:10 +00:00
|
|
|
{
|
2017-07-10 20:58:44 +00:00
|
|
|
if (!response.IsRedirect)
|
2023-07-07 22:37:22 +00:00
|
|
|
{
|
2017-07-10 20:58:44 +00:00
|
|
|
break;
|
2023-07-07 22:37:22 +00:00
|
|
|
}
|
2022-09-25 04:11:26 +00:00
|
|
|
|
|
|
|
var redirectingTo = new Uri(response.RedirectingTo);
|
|
|
|
if (redirectingTo.Scheme == "magnet")
|
2023-07-07 22:37:22 +00:00
|
|
|
{
|
2022-09-25 04:11:26 +00:00
|
|
|
break;
|
2023-07-07 22:37:22 +00:00
|
|
|
}
|
2022-09-25 04:11:26 +00:00
|
|
|
|
2024-06-02 07:01:21 +00:00
|
|
|
response = await DoFollowIfRedirect(response, referrer, overrideRedirectUrl, overrideCookies, accumulateCookies);
|
2023-07-07 22:37:22 +00:00
|
|
|
|
2017-07-10 20:58:44 +00:00
|
|
|
if (accumulateCookies)
|
|
|
|
{
|
|
|
|
CookieHeader = ResolveCookies((CookieHeader != null && CookieHeader != "" ? CookieHeader + " " : "") + (overrideCookies != null && overrideCookies != "" ? overrideCookies + " " : "") + response.Cookies);
|
|
|
|
overrideCookies = response.Cookies = CookieHeader;
|
|
|
|
}
|
2023-07-07 22:37:22 +00:00
|
|
|
|
2017-07-10 20:58:44 +00:00
|
|
|
if (overrideCookies != null && response.Cookies == null)
|
|
|
|
{
|
|
|
|
response.Cookies = overrideCookies;
|
|
|
|
}
|
2017-04-15 08:45:10 +00:00
|
|
|
}
|
2024-06-04 08:30:49 +00:00
|
|
|
|
|
|
|
return response;
|
2017-07-10 20:58:44 +00:00
|
|
|
}
|
|
|
|
|
2024-02-08 20:49:51 +00:00
|
|
|
protected virtual string ResolveCookies(string incomingCookies = "")
|
2017-07-10 20:58:44 +00:00
|
|
|
{
|
2020-04-12 13:00:14 +00:00
|
|
|
var redirRequestCookies = string.IsNullOrWhiteSpace(CookieHeader) ? incomingCookies : CookieHeader + " " + incomingCookies;
|
2020-04-13 04:22:50 +00:00
|
|
|
var cookieDictionary = CookieUtil.CookieHeaderToDictionary(redirRequestCookies);
|
|
|
|
|
|
|
|
// These cookies are causing BadGateway errors, so we drop them, see issue #2306
|
|
|
|
cookieDictionary.Remove("cf_use_ob");
|
|
|
|
cookieDictionary.Remove("cf_ob_info");
|
|
|
|
|
|
|
|
return CookieUtil.CookieDictionaryToHeader(cookieDictionary);
|
2017-04-15 08:45:10 +00:00
|
|
|
}
|
|
|
|
|
2017-07-10 20:58:44 +00:00
|
|
|
// Update CookieHeader with new cookies and save the config if something changed (e.g. a new CloudFlare clearance cookie was issued)
|
2017-10-03 12:23:31 +00:00
|
|
|
protected virtual void UpdateCookieHeader(string newCookies, string cookieOverride = null)
|
2017-04-15 08:45:10 +00:00
|
|
|
{
|
2020-02-10 22:16:19 +00:00
|
|
|
var newCookieHeader = ResolveCookies((cookieOverride != null && cookieOverride != "" ? cookieOverride + " " : "") + newCookies);
|
2017-07-10 20:58:44 +00:00
|
|
|
if (CookieHeader != newCookieHeader)
|
2017-04-15 08:45:10 +00:00
|
|
|
{
|
2017-07-10 20:58:44 +00:00
|
|
|
logger.Debug(string.Format("updating Cookies {0} => {1}", CookieHeader, newCookieHeader));
|
|
|
|
CookieHeader = newCookieHeader;
|
|
|
|
if (IsConfigured)
|
|
|
|
SaveConfig();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-06-02 07:01:21 +00:00
|
|
|
private async Task<WebResult> DoFollowIfRedirect(WebResult incomingResponse, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null, bool accumulateCookies = false)
|
2017-07-10 20:58:44 +00:00
|
|
|
{
|
|
|
|
if (incomingResponse.IsRedirect)
|
|
|
|
{
|
|
|
|
var redirRequestCookies = "";
|
|
|
|
if (accumulateCookies)
|
2017-04-15 08:45:10 +00:00
|
|
|
{
|
2017-07-10 20:58:44 +00:00
|
|
|
redirRequestCookies = ResolveCookies((CookieHeader != "" ? CookieHeader + " " : "") + (overrideCookies != null ? overrideCookies : ""));
|
2017-04-15 08:45:10 +00:00
|
|
|
}
|
2017-07-10 20:58:44 +00:00
|
|
|
else
|
|
|
|
{
|
|
|
|
redirRequestCookies = (overrideCookies != null ? overrideCookies : "");
|
|
|
|
}
|
|
|
|
// Do redirect
|
2020-10-19 21:19:10 +00:00
|
|
|
var redirectedResponse = await webclient.GetResultAsync(new WebRequest
|
2017-07-10 20:58:44 +00:00
|
|
|
{
|
|
|
|
Url = overrideRedirectUrl ?? incomingResponse.RedirectingTo,
|
|
|
|
Referer = referrer,
|
|
|
|
Cookies = redirRequestCookies,
|
|
|
|
Encoding = Encoding
|
|
|
|
});
|
2024-06-02 07:01:21 +00:00
|
|
|
|
|
|
|
return redirectedResponse;
|
2017-04-15 08:45:10 +00:00
|
|
|
}
|
2024-06-02 07:01:21 +00:00
|
|
|
|
|
|
|
return incomingResponse;
|
2017-04-15 08:45:10 +00:00
|
|
|
}
|
|
|
|
|
2020-10-27 21:17:03 +00:00
|
|
|
protected List<string> GetAllTrackerCategories() =>
|
|
|
|
TorznabCaps.Categories.GetTrackerCategories();
|
2017-04-15 08:45:10 +00:00
|
|
|
|
2020-10-27 21:17:03 +00:00
|
|
|
protected void AddCategoryMapping(string trackerCategory, TorznabCategory newznabCategory, string trackerCategoryDesc = null) =>
|
|
|
|
TorznabCaps.Categories.AddCategoryMapping(trackerCategory, newznabCategory, trackerCategoryDesc);
|
2017-04-15 08:45:10 +00:00
|
|
|
|
2020-10-31 00:12:35 +00:00
|
|
|
// TODO: remove this method ?
|
2020-10-27 21:17:03 +00:00
|
|
|
protected void AddCategoryMapping(int trackerCategory, TorznabCategory newznabCategory, string trackerCategoryDesc = null) =>
|
|
|
|
AddCategoryMapping(trackerCategory.ToString(), newznabCategory, trackerCategoryDesc);
|
2017-04-15 08:45:10 +00:00
|
|
|
|
2020-10-31 00:12:35 +00:00
|
|
|
// TODO: remove this method and use AddCategoryMapping instead. this method doesn't allow to create custom cats
|
2017-04-15 08:45:10 +00:00
|
|
|
protected void AddMultiCategoryMapping(TorznabCategory newznabCategory, params int[] trackerCategories)
|
|
|
|
{
|
|
|
|
foreach (var trackerCat in trackerCategories)
|
|
|
|
AddCategoryMapping(trackerCat, newznabCategory);
|
|
|
|
}
|
|
|
|
|
2020-10-27 21:17:03 +00:00
|
|
|
protected List<string> MapTorznabCapsToTrackers(TorznabQuery query, bool mapChildrenCatsToParent = false) =>
|
|
|
|
TorznabCaps.Categories.MapTorznabCapsToTrackers(query, mapChildrenCatsToParent);
|
2017-04-15 08:45:10 +00:00
|
|
|
|
2020-10-27 21:17:03 +00:00
|
|
|
protected ICollection<int> MapTrackerCatToNewznab(string input) =>
|
|
|
|
TorznabCaps.Categories.MapTrackerCatToNewznab(input);
|
2017-04-15 08:45:10 +00:00
|
|
|
|
2020-10-27 21:17:03 +00:00
|
|
|
protected ICollection<int> MapTrackerCatDescToNewznab(string input) =>
|
|
|
|
TorznabCaps.Categories.MapTrackerCatDescToNewznab(input);
|
2017-07-10 20:58:44 +00:00
|
|
|
|
|
|
|
private IEnumerable<ReleaseInfo> CleanLinks(IEnumerable<ReleaseInfo> releases)
|
2017-07-03 05:15:47 +00:00
|
|
|
{
|
2017-07-10 20:58:44 +00:00
|
|
|
if (string.IsNullOrEmpty(downloadUrlBase))
|
|
|
|
return releases;
|
|
|
|
foreach (var release in releases)
|
2017-07-03 05:15:47 +00:00
|
|
|
{
|
2017-07-10 20:58:44 +00:00
|
|
|
if (release.Link.ToString().StartsWith(downloadUrlBase, StringComparison.Ordinal))
|
|
|
|
{
|
|
|
|
release.Link = new Uri(release.Link.ToString().Substring(downloadUrlBase.Length), UriKind.Relative);
|
|
|
|
}
|
2017-07-03 05:15:47 +00:00
|
|
|
}
|
|
|
|
|
2017-07-10 20:58:44 +00:00
|
|
|
return releases;
|
|
|
|
}
|
|
|
|
|
2020-11-08 22:27:54 +00:00
|
|
|
public override async Task<IndexerResult> ResultsForQuery(TorznabQuery query, bool isMetaIndexer)
|
2017-07-10 20:58:44 +00:00
|
|
|
{
|
2020-11-08 22:27:54 +00:00
|
|
|
var result = await base.ResultsForQuery(query, isMetaIndexer);
|
2017-08-24 10:28:41 +00:00
|
|
|
result.Releases = CleanLinks(result.Releases);
|
|
|
|
return result;
|
2017-07-03 05:15:47 +00:00
|
|
|
}
|
2017-07-10 20:58:44 +00:00
|
|
|
|
|
|
|
protected virtual Uri UncleanLink(Uri link)
|
|
|
|
{
|
|
|
|
if (string.IsNullOrWhiteSpace(downloadUrlBase))
|
|
|
|
{
|
|
|
|
return link;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (link.ToString().StartsWith(downloadUrlBase, StringComparison.Ordinal))
|
|
|
|
{
|
|
|
|
return link;
|
|
|
|
}
|
|
|
|
|
|
|
|
return new Uri(downloadUrlBase + link.ToString(), UriKind.RelativeOrAbsolute);
|
|
|
|
}
|
|
|
|
|
|
|
|
protected void OnParseError(string results, Exception ex)
|
|
|
|
{
|
2023-03-09 14:06:12 +00:00
|
|
|
var fileName = string.Format("Error on {0} for {1}.txt", DateTime.Now.ToString("yyyyMMddHHmmss"), Name);
|
2017-07-10 20:58:44 +00:00
|
|
|
var spacing = string.Join("", Enumerable.Repeat(Environment.NewLine, 5));
|
|
|
|
var fileContents = string.Format("{0}{1}{2}", ex, spacing, results);
|
|
|
|
logger.Error(fileName + fileContents);
|
2017-09-13 09:49:24 +00:00
|
|
|
throw new Exception("Parse error", ex);
|
2017-07-10 20:58:44 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
public override TorznabCapabilities TorznabCaps { get; protected set; }
|
2017-07-11 20:32:56 +00:00
|
|
|
|
2017-11-05 09:42:03 +00:00
|
|
|
protected WebClient webclient;
|
2017-07-10 20:58:44 +00:00
|
|
|
protected readonly string downloadUrlBase = "";
|
|
|
|
}
|
|
|
|
|
|
|
|
public abstract class BaseCachingWebIndexer : BaseWebIndexer
|
|
|
|
{
|
2023-03-09 14:06:12 +00:00
|
|
|
protected BaseCachingWebIndexer(IIndexerConfigurationService configService, WebClient client, Logger logger,
|
2020-12-11 22:14:21 +00:00
|
|
|
ConfigurationData configData, IProtectionService p, ICacheService cacheService,
|
2023-03-10 11:20:29 +00:00
|
|
|
string downloadBase = null)
|
|
|
|
: base(configService: configService, client: client, logger: logger, configData: configData, p: p, cacheService: cacheService, downloadBase: downloadBase)
|
2017-07-10 20:58:44 +00:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
protected void CleanCache()
|
|
|
|
{
|
|
|
|
foreach (var expired in cache.Where(i => DateTime.Now - i.Created > cacheTime).ToList())
|
|
|
|
{
|
|
|
|
cache.Remove(expired);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-12-11 22:14:21 +00:00
|
|
|
// TODO: remove this implementation and use gloal cache
|
2017-07-10 20:58:44 +00:00
|
|
|
protected static List<CachedQueryResult> cache = new List<CachedQueryResult>();
|
2023-04-09 01:21:09 +00:00
|
|
|
protected static readonly TimeSpan cacheTime = TimeSpan.FromMinutes(9);
|
2017-04-15 08:45:10 +00:00
|
|
|
}
|
|
|
|
}
|