mirror
/
Jackett
mirror of https://github.com/Jackett/Jackett
1
0
Fork 0
Code Issues Releases Wiki Activity

xthor-api: massive improvements (#11690)

This commit is contained in:
JigSaw 2021-05-07 21:32:07 +02:00 committed by GitHub
parent 6740c7c40f
commit d34dbcb626
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 189 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

257
src/Jackett.Common/Indexers/Xthor.cs
View File

@ -8,13 +8,14 @@ using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig.Bespoke;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
using static Jackett.Common.Models.IndexerConfig.ConfigurationData;
using WebRequest = Jackett.Common.Utils.Clients.WebRequest;
namespace Jackett.Common.Indexers
@ -23,21 +24,27 @@ namespace Jackett.Common.Indexers
public class Xthor : BaseCachingWebIndexer
{
private static string ApiEndpoint => "https://api.xthor.tk/";
private int MaxPagesHardLimit => 4;
private string TorrentDetailsUrl => SiteLink + "details.php?id={id}";
private string ReplaceMulti => ConfigData.ReplaceMulti.Value;
private bool EnhancedAnime => ConfigData.EnhancedAnime.Value;
private static int MaxPageLoads => 4;
private string WebRequestDelay => ((SingleSelectConfigurationItem)configData.GetDynamic("webRequestDelay")).Value;
private int MaxPages => Convert.ToInt32(((SingleSelectConfigurationItem)configData.GetDynamic("maxPages")).Value);
private bool MaxPagesBypassForTMDB => ((BoolConfigurationItem)configData.GetDynamic("maxPagesBypassForTMDB")).Value;
private string MultiReplacement => ((StringConfigurationItem)configData.GetDynamic("multiReplacement")).Value;
private bool SubReplacement => ((BoolConfigurationItem)configData.GetDynamic("subReplacement")).Value;
private bool EnhancedAnimeSearch => ((BoolConfigurationItem)configData.GetDynamic("enhancedAnimeSearch")).Value;
private string SpecificLanguageAccent => ((SingleSelectConfigurationItem)configData.GetDynamic("specificLanguageAccent")).Value;
private bool FreeleechOnly => ((BoolConfigurationItem)configData.GetDynamic("freeleechOnly")).Value;
public override string[] LegacySiteLinks { get; protected set; } = {
"https://xthor.bz/",
"https://xthor.to"
};
private ConfigurationDataXthor ConfigData => (ConfigurationDataXthor)configData;
private ConfigurationDataPasskey ConfigData => (ConfigurationDataPasskey)configData;
public Xthor(IIndexerConfigurationService configService, Utils.Clients.WebClient w, Logger l,
IProtectionService ps, ICacheService cs)
: base(id: "xthor",
name: "Xthor",
: base(id: "xthor-api",
name: "Xthor API",
description: "General French Private Tracker",
link: "https://xthor.tk/",
caps: new TorznabCapabilities
@ -48,7 +55,7 @@ namespace Jackett.Common.Indexers
},
MovieSearchParams = new List<MovieSearchParam>
{
MovieSearchParam.Q
MovieSearchParam.Q, MovieSearchParam.TmdbId
},
MusicSearchParams = new List<MusicSearchParam>
{
@ -65,15 +72,13 @@ namespace Jackett.Common.Indexers
p: ps,
cacheService: cs,
downloadBase: "https://xthor.tk/download.php?torrent=",
configData: new ConfigurationDataXthor())
configData: new ConfigurationDataPasskey()
)
{
Encoding = Encoding.UTF8;
Language = "fr-fr";
Type = "private";
// Api has 1req/2s limit
webclient.requestDelay = 2.1;
// Movies / Films
AddCategoryMapping(118, TorznabCatType.MoviesBluRay, "Films 2160p/Bluray");
AddCategoryMapping(119, TorznabCatType.MoviesBluRay, "Films 2160p/Remux");
@ -142,6 +147,106 @@ namespace Jackett.Common.Indexers
AddCategoryMapping(21, TorznabCatType.PC, "Logiciels Applis PC");
AddCategoryMapping(22, TorznabCatType.PCMac, "Logiciels Applis Mac");
AddCategoryMapping(23, TorznabCatType.PCMobileAndroid, "Logiciels Smartphone");
// Dynamic Configuration
ConfigData.AddDynamic("optionsConfigurationWarning", new DisplayInfoConfigurationItem(string.Empty, "<center><b>Available Options</b></center>,<br /><br /> <ul><li><b>Freeleech Only</b>: (<i>Restrictive</i>) If you want to discover only freeleech torrents to not impact your ratio, check the related box. So only torrents marked as freeleech will be returned instead of all.</li><br /><li><b>Specific Language</b>: (<i>Restrictive</i>) You can scope your searches with a specific language / accent.</li></ul>"));
var ConfigFreeleechOnly = new BoolConfigurationItem("Do you want to discover only freeleech tagged torrents ?");
ConfigData.AddDynamic("freeleechOnly", ConfigFreeleechOnly);
var ConfigSpecificLanguageAccent = new SingleSelectConfigurationItem("Do you want to scope your searches with a specific language ? (Accent)", new Dictionary<string, string>
{
{"0", "All Voices (default)"},
{"1", "Françaises"},
{"2", "Quebecoises"},
{"47", "Françaises et Québécoises"},
{"3", "Anglaises"},
{"4", "Japonaises"},
{"5", "Espagnoles"},
{"6", "Allemandes"},
{"7", "Chinoises"},
{"8", "Italiennes"},
{"9", "Coréennes"},
{"10", "Danoises"},
{"11", "Russes"},
{"12", "Portugaises"},
{"13", "Hindi"},
{"14", "Hollandaises"},
{"15", "Suédoises"},
{"16", "Norvégiennes"},
{"17", "Thaïlandaises"},
{"18", "Hébreu"},
{"19", "Persanes"},
{"20", "Arabes"},
{"21", "Turques"},
{"22", "Hongroises"},
{"23", "Polonaises"},
{"24", "Finnoises"},
{"25", "Indonésiennes"},
{"26", "Roumaines"},
{"27", "Malaisiennes"},
{"28", "Estoniennes"},
{"29", "Islandaises"},
{"30", "Grecques"},
{"31", "Serbes"},
{"32", "Norvégiennes"},
{"33", "Ukrainiennes"},
{"34", "Bulgares"},
{"35", "Tagalogues"},
{"36", "Xhosa"},
{"37", "Kurdes"},
{"38", "Bengali"},
{"39", "Amhariques"},
{"40", "Bosniaques"},
{"41", "Malayalam"},
{"42", "Télougou"},
{"43", "Bambara"},
{"44", "Catalanes"},
{"45", "Tchèques"},
{"46", "Afrikaans"}
})
{ Value = "0" };
ConfigData.AddDynamic("specificLanguageAccent", ConfigSpecificLanguageAccent);
ConfigData.AddDynamic("advancedConfigurationWarning", new DisplayInfoConfigurationItem(string.Empty, "<center><b>Advanced Configuration</b></center>,<br /><br /> <center><b><u>WARNING !</u></b> <i>Be sure to read instructions before editing options bellow, you can <b>drastically reduce performance</b> of queries or have <b>non-accurate results</b>.</i></center><br/><br/><ul><li><b>Delay betwwen Requests</b>: (<i>not recommended</i>) you can increase delay to requests made to the tracker, but a minimum of 2.1s is enforced as there is an anti-spam protection.</li><br /><li><b>Max Pages</b>: (<i>not recommended</i>) you can increase max pages to follow when making a request. But be aware that others apps can consider this indexer not working if jackett take too many times to return results. Another thing is that API is very buggy on tracker side, most of time, results of next pages are same ... as the first page. Even if we deduplicate rows, you will loose performance for the same results. You can check logs to see if an higher pages following is not benefical, you will see an error percentage (duplicates) with recommandations.</li><br /><li><b>Bypass for TMDB</b>: (<i>recommended</i>) this indexer is compatible with TMDB queries (<i>for movies only</i>), so when requesting content with an TMDB ID, we will search directly ID on API instead of name. Results will be more accurate, so you can enable a max pages bypass for this query type. You will be at least limited by the hard limit of 4 pages.</li><br /><li><b>Enhanced Anime</b>: if you have \"Anime\", this will improve queries made to this tracker related to this type when making searches.</li><br /><li><b>Multi Replacement</b>: you can dynamically replace the word \"MULTI\" with another of your choice like \"MULTI.FRENCH\" for better analysis of 3rd party softwares.</li><li><b>Sub Replacement</b>: you can dynamically replace the word \"VOSTFR\" or \"SUBFRENCH\" with the word \"ENGLISH\" for better analysis of 3rd party softwares.</li></ul>"));
var ConfigWebRequestDelay = new SingleSelectConfigurationItem("Which delay do you want to apply between each requests made to tracker ?", new Dictionary<string, string>
{
{"2.1", "2.1s (minimum)"},
{"2.2", "2.2s"},
{"2.3", "2.3s"},
{"2.4", "2.4s" },
{"2.5", "2.5s"},
{"2.6", "2.6s"}
})
{ Value = "2.1" };
ConfigData.AddDynamic("webRequestDelay", ConfigWebRequestDelay);
var ConfigMaxPages = new SingleSelectConfigurationItem("How many pages do you want to follow ?", new Dictionary<string, string>
{
{"1", "1 (32 results - default / best perf.)"},
{"2", "2 (64 results)"},
{"3", "3 (96 results)"},
{"4", "4 (128 results - hard limit max)" },
})
{ Value = "1" };
ConfigData.AddDynamic("maxPages", ConfigMaxPages);
var ConfigMaxPagesBypassForTMDB = new BoolConfigurationItem("Do you want to bypass max pages for TMDB searches ? (Radarr) - Hard limit of 4") { Value = true };
ConfigData.AddDynamic("maxPagesBypassForTMDB", ConfigMaxPagesBypassForTMDB);
var ConfigEnhancedAnimeSearch = new BoolConfigurationItem("Do you want to use enhanced ANIME search ?") { Value = false };
ConfigData.AddDynamic("enhancedAnimeSearch", ConfigEnhancedAnimeSearch);
var ConfigMultiReplacement = new StringConfigurationItem("Do you want to replace \"MULTI\" keyword in release title by another word ?") { Value = "MULTI.FRENCH" };
ConfigData.AddDynamic("multiReplacement", ConfigMultiReplacement);
var ConfigSubReplacement = new BoolConfigurationItem("Do you want to replace \"VOSTFR\" and \"SUBFRENCH\" with \"ENGLISH\" word ?") { Value = false };
ConfigData.AddDynamic("subReplacement", ConfigSubReplacement);
// Api has 1req/2s limit (minimum)
webclient.requestDelay = Convert.ToDouble(WebRequestDelay);
}
/// <summary>
@ -153,10 +258,9 @@ namespace Jackett.Common.Indexers
// Warning 1998 is async method with no await calls inside
// TODO: Remove pragma by wrapping return in Task.FromResult and removing async
#pragma warning disable 1998
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
#pragma warning restore 1998
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
{
// Provider not yet configured
IsConfigured = false;
@ -164,8 +268,18 @@ namespace Jackett.Common.Indexers
// Retrieve config values set by Jackett's user
LoadValuesFromJson(configJson);
// Check & Validate Config
ValidateConfig();
logger.Debug("\nXthor - Validating Settings ... \n");
// Check Passkey Setting
if (string.IsNullOrEmpty(ConfigData.Passkey.Value))
{
throw new ExceptionWithConfigData("You must provide your passkey for this tracker to be allowed to use API !", ConfigData);
}
else
{
logger.Debug("Xthor - Validated Setting -- PassKey (auth) => " + ConfigData.Passkey.Value);
}
// Tracker is now configured
IsConfigured = true;
@ -189,14 +303,12 @@ namespace Jackett.Common.Indexers
searchTerm = searchTerm.ToLower();
searchTerm = searchTerm.Replace(" ", ".");
if (EnhancedAnime && query.HasSpecifiedCategories && (query.Categories.Contains(TorznabCatType.TVAnime.ID) || query.Categories.Contains(100032) || query.Categories.Contains(100101) || query.Categories.Contains(100110)))
if (EnhancedAnimeSearch && query.HasSpecifiedCategories && (query.Categories.Contains(TorznabCatType.TVAnime.ID) || query.Categories.Contains(100032) || query.Categories.Contains(100101) || query.Categories.Contains(100110)))
{
var regex = new Regex(" ([0-9]+)");
searchTerm = regex.Replace(searchTerm, " E$1");
}
logger.Info("\nXthor - Search requested for \"" + searchTerm + "\"");
// Multiple page support
var nextPage = 1; var followingPages = true;
do
@ -233,14 +345,14 @@ namespace Jackett.Common.Indexers
.Where(torrent => torrent.Category != 106).Select(torrent =>
{
//issue #3847 replace multi keyword
if (!string.IsNullOrEmpty(ReplaceMulti))
if (!string.IsNullOrEmpty(MultiReplacement))
{
var regex = new Regex("(?i)([\\.\\- ])MULTI([\\.\\- ])");
torrent.Name = regex.Replace(torrent.Name, "$1" + ReplaceMulti + "$2");
torrent.Name = regex.Replace(torrent.Name, "$1" + MultiReplacement + "$2");
}
// issue #8759 replace vostfr and subfrench with English
if (ConfigData.Vostfr.Value)
if (SubReplacement)
torrent.Name = torrent.Name.Replace("VOSTFR", "ENGLISH").Replace("SUBFRENCH", "ENGLISH");
var publishDate = DateTimeUtil.UnixTimestampToDateTime(torrent.Added);
@ -275,9 +387,10 @@ namespace Jackett.Common.Indexers
}
else
{
logger.Info("\nXthor - No results found on page " + (nextPage -1) + ", stopping follow of next page.");
logger.Info("\nXthor - No results found on page " + nextPage + ", stopping follow of next page.");
// No results or no more results available
followingPages = false;
break;
}
}
catch (Exception ex)
@ -286,16 +399,42 @@ namespace Jackett.Common.Indexers
}
// Stop ?
if(nextPage > MaxPageLoads | torrentsCount < 32 | string.IsNullOrWhiteSpace(searchTerm))
if(query.IsTmdbQuery && MaxPagesBypassForTMDB)
{
logger.Info("\nXthor - Stopping follow of next page " + nextPage + " due to page limit or max available results reached or indexer test.");
followingPages = false;
if(nextPage > MaxPagesHardLimit)
{
logger.Info("\nXthor - Stopping follow of next page " + nextPage + " due to page hard limit reached.");
break;
}
logger.Info("\nXthor - Continue to next page " + nextPage + " due to TMDB request and activated max page bypass for this type of query. Max page hard limit: 4.");
continue;
}
else
{
if(torrentsCount < 32)
{
logger.Info("\nXthor - Stopping follow of next page " + nextPage + " due max available results reached.");
break;
} else if(nextPage > MaxPages)
{
logger.Info("\nXthor - Stopping follow of next page " + nextPage + " due to page limit reached.");
break;
} else if (query.IsTest)
{
logger.Info("\nXthor - Stopping follow of next page " + nextPage + " due to index test query.");
break;
}
}
} while (followingPages);
// Check if there is duplicate and return unique rows - Xthor API can be very buggy !
var uniqReleases = releases.GroupBy(x => x.Guid).Select(x => x.First()).ToList();
var errorPercentage = 1 - ((double) uniqReleases.Count() / releases.Count());
if(errorPercentage >= 0.25)
{
logger.Warn("\nXthor - High percentage error detected: " + string.Format("{0:0.0%}", errorPercentage) + "\nWe strongly recommend that you lower max page to 1, as there is no benefit to grab additionnals.\nTracker API sent us duplicated pages with same results, even if we deduplicate returned rows, please consider to lower as it's unnecessary and increase time used for query for the same result.");
}
// Return found releases
return uniqReleases;
}
@ -369,19 +508,27 @@ namespace Jackett.Common.Indexers
var categoriesList = MapTorznabCapsToTrackers(query);
// Passkey
parameters.Add("passkey", ConfigData.PassKey.Value);
parameters.Add("passkey", ConfigData.Passkey.Value);
// If search term provided
if (!string.IsNullOrWhiteSpace(term))
if (query.IsTmdbQuery)
{
// Add search term
// ReSharper disable once AssignNullToNotNullAttribute
parameters.Add("search", WebUtility.UrlEncode(term));
logger.Info("\nXthor - Search requested for movie with TMDB ID n°" + query.TmdbID.ToString());
parameters.Add("tmdbid", query.TmdbID.ToString());
}
else
{
parameters.Add("search", string.Empty);
// Showing all torrents
if (!string.IsNullOrWhiteSpace(term))
{
// Add search term
logger.Info("\nXthor - Search requested for movie with title \"" + term + "\"");
parameters.Add("search", WebUtility.UrlEncode(term));
}
else
{
logger.Info("\nXthor - Global search requested without term");
parameters.Add("search", string.Empty);
// Showing all torrents
}
}
// Loop on Categories needed
@ -391,18 +538,19 @@ namespace Jackett.Common.Indexers
}
// If Only Freeleech Enabled
if (ConfigData.Freeleech.Value)
if (FreeleechOnly)
{
parameters.Add("freeleech", "1");
}
if (!string.IsNullOrEmpty(ConfigData.Accent.Value))
// If Specific Language Accent Requested
if (!string.IsNullOrEmpty(SpecificLanguageAccent) && SpecificLanguageAccent != "0")
{
parameters.Add("accent", ConfigData.Accent.Value);
parameters.Add("accent", SpecificLanguageAccent);
}
// Pages handling
if (page > 1 && !string.IsNullOrWhiteSpace(term))
if (page > 1 && !query.IsTest)
{
parameters.Add("page", page.ToString());
}
@ -410,7 +558,7 @@ namespace Jackett.Common.Indexers
// Building our query -- Cannot use GetQueryString due to UrlEncode (generating wrong category param)
url += "?" + string.Join("&", parameters.AllKeys.Select(a => a + "=" + parameters[a]));
logger.Info("\nXthor - Builded query for \"" + term + "\"... " + url);
logger.Info("\nXthor - Builded query: " + url);
// Return our search url
return url;
@ -454,7 +602,7 @@ namespace Jackett.Common.Indexers
{
case 0:
// Everything OK
logger.Info("\nXthor - API State : Everything OK ... -> " + state.Descr);
logger.Debug("\nXthor - API State : Everything OK ... -> " + state.Descr);
break;
case 1:
@ -485,32 +633,5 @@ namespace Jackett.Common.Indexers
throw new Exception("Unknown state, aborting querying");
}
}
/// <summary>
/// Validate Config entered by user on Jackett
/// </summary>
private void ValidateConfig()
{
logger.Debug("\nXthor - Validating Settings ... \n");
// Check Passkey Setting
if (string.IsNullOrEmpty(ConfigData.PassKey.Value))
{
throw new ExceptionWithConfigData("You must provide your passkey for this tracker to be allowed to use API !", ConfigData);
}
else
{
logger.Debug("Xthor - Validated Setting -- PassKey (auth) => " + ConfigData.PassKey.Value);
}
if (!string.IsNullOrEmpty(ConfigData.Accent.Value) && !string.Equals(ConfigData.Accent.Value, "1") && !string.Equals(ConfigData.Accent.Value, "2"))
{
throw new ExceptionWithConfigData("Only '1' or '2' are available in the Accent parameter.", ConfigData);
}
else
{
logger.Debug("Xthor - Validated Setting -- Accent (audio) => " + ConfigData.Accent.Value);
}
}
}
}

31
src/Jackett.Common/Models/IndexerConfig/Bespoke/ConfigurationDataXthor.cs
View File

@ -1,31 +0,0 @@
using System.Diagnostics.CodeAnalysis;
namespace Jackett.Common.Models.IndexerConfig.Bespoke
{
[ExcludeFromCodeCoverage]
internal class ConfigurationDataXthor : ConfigurationData
{
public DisplayInfoConfigurationItem CredentialsWarning { get; private set; }
public StringConfigurationItem PassKey { get; set; }
public DisplayInfoConfigurationItem PagesWarning { get; private set; }
public StringConfigurationItem Accent { get; set; }
public BoolConfigurationItem Freeleech { get; private set; }
public StringConfigurationItem ReplaceMulti { get; private set; }
public BoolConfigurationItem EnhancedAnime { get; private set; }
public BoolConfigurationItem Vostfr { get; private set; }
public ConfigurationDataXthor()
: base()
{
CredentialsWarning = new DisplayInfoConfigurationItem("Credentials", "<b>Credentials Configuration</b> (<i>Private Tracker</i>),<br /><br /> <ul><li><b>PassKey</b> is your private key on your account</li></ul>");
PassKey = new StringConfigurationItem("PassKey") { Value = "" };
Accent = new StringConfigurationItem("Accent") { Value = "" };
PagesWarning = new DisplayInfoConfigurationItem("Preferences", "<b>Preferences Configuration</b> (<i>Tweak your search settings</i>),<br /><br /> <ul><li><b>Freeleech Only</b> let you search <u>only</u> for torrents which are marked Freeleech.</li><li><b>Replace MULTI</b>, replace multi keyword in the resultset (leave empty to deactivate)</li><li><b>Enhanced anime search</b>, Enhance sonarr compatibility with Xthor. Only effective on requests with the <u>TVAnime Torznab category</u>.</li><li><b>Accent</b> is the french accent you want. 1 for VFF (Truefrench) 2 for VFQ (FRENCH, canada). When one is selected, the other will not be searched.</li></ul>");
Freeleech = new BoolConfigurationItem("Freeleech Only (Optional)") { Value = false };
ReplaceMulti = new StringConfigurationItem("Replace MULTI") { Value = "MULTI.FRENCH" };
EnhancedAnime = new BoolConfigurationItem("Enhanced anime search") { Value = false };
Vostfr = new BoolConfigurationItem("Replace VOSTFR or SUBFRENCH with ENGLISH") { Value = false };
}
}
}