mirror
/
Jackett
mirror of https://github.com/Jackett/Jackett
1
0
Fork 0
Code Issues Releases Wiki Activity

xthor: pagination support, cleaning, dedup of results, resolves #10635 #6392 (#11675)

This commit is contained in:
JigSaw 2021-05-04 18:38:37 +02:00 committed by GitHub
parent 993116c96f
commit ca3466050c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 116 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

194
src/Jackett.Common/Indexers/Xthor.cs
View File

@ -2,10 +2,8 @@ using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Collections.Specialized; using System.Collections.Specialized;
using System.Diagnostics.CodeAnalysis; using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq; using System.Linq;
using System.Net; using System.Net;
using System.Reflection;
using System.Text; using System.Text;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System.Threading.Tasks; using System.Threading.Tasks;
@ -25,15 +23,15 @@ namespace Jackett.Common.Indexers
public class Xthor : BaseCachingWebIndexer public class Xthor : BaseCachingWebIndexer
{ {
private static string ApiEndpoint => "https://api.xthor.tk/"; private static string ApiEndpoint => "https://api.xthor.tk/";
private string TorrentDetailsUrl => SiteLink + "details.php?id={id}";
private string ReplaceMulti => ConfigData.ReplaceMulti.Value;
private bool EnhancedAnime => ConfigData.EnhancedAnime.Value;
private static int MaxPageLoads => 4;
public override string[] LegacySiteLinks { get; protected set; } = { public override string[] LegacySiteLinks { get; protected set; } = {
"https://xthor.bz/", "https://xthor.bz/",
"https://xthor.to" "https://xthor.to"
}; };
private string TorrentDetailsUrl => SiteLink + "details.php?id={id}";
private string ReplaceMulti => ConfigData.ReplaceMulti.Value;
private bool EnhancedAnime => ConfigData.EnhancedAnime.Value;
private ConfigurationDataXthor ConfigData => (ConfigurationDataXthor)configData; private ConfigurationDataXthor ConfigData => (ConfigurationDataXthor)configData;
public Xthor(IIndexerConfigurationService configService, Utils.Clients.WebClient w, Logger l, public Xthor(IIndexerConfigurationService configService, Utils.Clients.WebClient w, Logger l,
@ -186,9 +184,10 @@ namespace Jackett.Common.Indexers
protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query) protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
{ {
var releases = new List<ReleaseInfo>(); var releases = new List<ReleaseInfo>();
var searchTerm = query.GetEpisodeSearchString() + " " + query.SanitizedSearchTerm; // use episode search string first, see issue #1202 var searchTerm = query.SanitizedSearchTerm + " " + query.GetEpisodeSearchString();
searchTerm = searchTerm.Trim(); searchTerm = searchTerm.Trim();
searchTerm = searchTerm.ToLower(); searchTerm = searchTerm.ToLower();
searchTerm = searchTerm.Replace(" ", ".");
if (EnhancedAnime && query.HasSpecifiedCategories && (query.Categories.Contains(TorznabCatType.TVAnime.ID) || query.Categories.Contains(100032) || query.Categories.Contains(100101) || query.Categories.Contains(100110))) if (EnhancedAnime && query.HasSpecifiedCategories && (query.Categories.Contains(TorznabCatType.TVAnime.ID) || query.Categories.Contains(100032) || query.Categories.Contains(100101) || query.Categories.Contains(100110)))
{ {
@ -196,75 +195,109 @@ namespace Jackett.Common.Indexers
searchTerm = regex.Replace(searchTerm, " E$1"); searchTerm = regex.Replace(searchTerm, " E$1");
} }
// Build our query logger.Info("\nXthor - Search requested for \"" + searchTerm + "\"");
var request = BuildQuery(searchTerm, query, ApiEndpoint);
// Getting results // Multiple page support
var results = await QueryTrackerAsync(request); var nextPage = 1; var followingPages = true;
do
try
{ {
// Deserialize our Json Response
var xthorResponse = JsonConvert.DeserializeObject<XthorResponse>(results);
// Check Tracker's State // Build our query
CheckApiState(xthorResponse.Error); var request = BuildQuery(searchTerm, query, ApiEndpoint, nextPage);
// If contains torrents // Getting results
if (xthorResponse.Torrents != null) logger.Info("\nXthor - Querying API page " + nextPage);
var results = await QueryTrackerAsync(request);
// Torrents Result Count
var torrentsCount = 0;
try
{ {
// Adding each torrent row to releases // Deserialize our Json Response
// Exclude hidden torrents (category 106, example => search 'yoda' in the API) #10407 var xthorResponse = JsonConvert.DeserializeObject<XthorResponse>(results);
releases.AddRange(xthorResponse.Torrents
.Where(torrent => torrent.Category != 106).Select(torrent => // Check Tracker's State
CheckApiState(xthorResponse.Error);
// If contains torrents
if (xthorResponse.Torrents != null)
{ {
//issue #3847 replace multi keyword // Store torrents rows count result
if (!string.IsNullOrEmpty(ReplaceMulti)) torrentsCount = xthorResponse.Torrents.Count();
{ logger.Info("\nXthor - Found " + torrentsCount + " torrents on current page.");
var regex = new Regex("(?i)([\\.\\- ])MULTI([\\.\\- ])");
torrent.Name = regex.Replace(torrent.Name, "$1" + ReplaceMulti + "$2");
}
// issue #8759 replace vostfr and subfrench with English // Adding each torrent row to releases
if (ConfigData.Vostfr.Value) torrent.Name = torrent.Name.Replace("VOSTFR","ENGLISH").Replace("SUBFRENCH","ENGLISH"); // Exclude hidden torrents (category 106, example => search 'yoda' in the API) #10407
releases.AddRange(xthorResponse.Torrents
.Where(torrent => torrent.Category != 106).Select(torrent =>
{
//issue #3847 replace multi keyword
if (!string.IsNullOrEmpty(ReplaceMulti))
{
var regex = new Regex("(?i)([\\.\\- ])MULTI([\\.\\- ])");
torrent.Name = regex.Replace(torrent.Name, "$1" + ReplaceMulti + "$2");
}
var publishDate = DateTimeUtil.UnixTimestampToDateTime(torrent.Added); // issue #8759 replace vostfr and subfrench with English
//TODO replace with download link? if (ConfigData.Vostfr.Value)
var guid = new Uri(TorrentDetailsUrl.Replace("{id}", torrent.Id.ToString())); torrent.Name = torrent.Name.Replace("VOSTFR", "ENGLISH").Replace("SUBFRENCH", "ENGLISH");
var details = new Uri(TorrentDetailsUrl.Replace("{id}", torrent.Id.ToString()));
var link = new Uri(torrent.Download_link);
var release = new ReleaseInfo
{
// Mapping data
Category = MapTrackerCatToNewznab(torrent.Category.ToString()),
Title = torrent.Name,
Seeders = torrent.Seeders,
Peers = torrent.Seeders + torrent.Leechers,
MinimumRatio = 1,
MinimumSeedTime = 345600,
PublishDate = publishDate,
Size = torrent.Size,
Grabs = torrent.Times_completed,
Files = torrent.Numfiles,
UploadVolumeFactor = 1,
DownloadVolumeFactor = (torrent.Freeleech == 1 ? 0 : 1),
Guid = guid,
Details = details,
Link = link,
TMDb = torrent.Tmdb_id
};
return release; var publishDate = DateTimeUtil.UnixTimestampToDateTime(torrent.Added);
})); //TODO replace with download link?
var guid = new Uri(TorrentDetailsUrl.Replace("{id}", torrent.Id.ToString()));
var details = new Uri(TorrentDetailsUrl.Replace("{id}", torrent.Id.ToString()));
var link = new Uri(torrent.Download_link);
var release = new ReleaseInfo
{
// Mapping data
Category = MapTrackerCatToNewznab(torrent.Category.ToString()),
Title = torrent.Name,
Seeders = torrent.Seeders,
Peers = torrent.Seeders + torrent.Leechers,
MinimumRatio = 1,
MinimumSeedTime = 345600,
PublishDate = publishDate,
Size = torrent.Size,
Grabs = torrent.Times_completed,
Files = torrent.Numfiles,
UploadVolumeFactor = 1,
DownloadVolumeFactor = (torrent.Freeleech == 1 ? 0 : 1),
Guid = guid,
Details = details,
Link = link,
TMDb = torrent.Tmdb_id
};
return release;
}));
nextPage++;
}
else
{
logger.Info("\nXthor - No results found on page " + (nextPage -1) + ", stopping follow of next page.");
// No results or no more results available
followingPages = false;
}
}
catch (Exception ex)
{
OnParseError("Unable to parse result \n" + ex.StackTrace, ex);
} }
}
catch (Exception ex)
{
OnParseError("Unable to parse result \n" + ex.StackTrace, ex);
}
// Stop ?
if(nextPage > MaxPageLoads | torrentsCount < 32 | string.IsNullOrWhiteSpace(searchTerm))
{
logger.Info("\nXthor - Stopping follow of next page " + nextPage + " due to page limit or max available results reached or indexer test.");
followingPages = false;
}
} while (followingPages);
// Check if there is duplicate and return unique rows - Xthor API can be very buggy !
var uniqReleases = releases.GroupBy(x => x.Guid).Select(x => x.First()).ToList();
// Return found releases // Return found releases
return releases; return uniqReleases;
} }
/// <summary> /// <summary>
@ -330,7 +363,7 @@ namespace Jackett.Common.Indexers
/// <param name="query">Torznab Query for categories mapping</param> /// <param name="query">Torznab Query for categories mapping</param>
/// <param name="url">Search url for provider</param> /// <param name="url">Search url for provider</param>
/// <returns>URL to query for parsing and processing results</returns> /// <returns>URL to query for parsing and processing results</returns>
private string BuildQuery(string term, TorznabQuery query, string url) private string BuildQuery(string term, TorznabQuery query, string url, int page = 1)
{ {
var parameters = new NameValueCollection(); var parameters = new NameValueCollection();
var categoriesList = MapTorznabCapsToTrackers(query); var categoriesList = MapTorznabCapsToTrackers(query);
@ -348,8 +381,7 @@ namespace Jackett.Common.Indexers
else else
{ {
parameters.Add("search", string.Empty); parameters.Add("search", string.Empty);
// Showing all torrents (just for output function) // Showing all torrents
term = "all";
} }
// Loop on Categories needed // Loop on Categories needed
@ -369,10 +401,16 @@ namespace Jackett.Common.Indexers
parameters.Add("accent", ConfigData.Accent.Value); parameters.Add("accent", ConfigData.Accent.Value);
} }
// Pages handling
if (page > 1 && !string.IsNullOrWhiteSpace(term))
{
parameters.Add("page", page.ToString());
}
// Building our query -- Cannot use GetQueryString due to UrlEncode (generating wrong category param) // Building our query -- Cannot use GetQueryString due to UrlEncode (generating wrong category param)
url += "?" + string.Join("&", parameters.AllKeys.Select(a => a + "=" + parameters[a])); url += "?" + string.Join("&", parameters.AllKeys.Select(a => a + "=" + parameters[a]));
logger.Debug("\nBuilded query for \"" + term + "\"... " + url); logger.Info("\nXthor - Builded query for \"" + term + "\"... " + url);
// Return our search url // Return our search url
return url; return url;
@ -416,34 +454,34 @@ namespace Jackett.Common.Indexers
{ {
case 0: case 0:
// Everything OK // Everything OK
logger.Debug("\nAPI State : Everything OK ... -> " + state.Descr); logger.Info("\nXthor - API State : Everything OK ... -> " + state.Descr);
break; break;
case 1: case 1:
// Passkey not found // Passkey not found
logger.Debug("\nAPI State : Error, Passkey not found in tracker's database, aborting... -> " + state.Descr); logger.Error("\nXthor - API State : Error, Passkey not found in tracker's database, aborting... -> " + state.Descr);
throw new Exception("Passkey not found in tracker's database"); throw new Exception("Passkey not found in tracker's database");
case 2: case 2:
// No results // No results
logger.Debug("\nAPI State : No results for query ... -> " + state.Descr); logger.Info("\nXthor - API State : No results for query ... -> " + state.Descr);
break; break;
case 3: case 3:
// Power Saver // Power Saver
logger.Debug("\nAPI State : Power Saver mode, only cached query with no parameters available ... -> " + state.Descr); logger.Warn("\nXthor - API State : Power Saver mode, only cached query with no parameters available ... -> " + state.Descr);
break; break;
case 4: case 4:
// DDOS Attack, API disabled // DDOS Attack, API disabled
logger.Debug("\nAPI State : Tracker is under DDOS attack, API disabled, aborting ... -> " + state.Descr); logger.Error("\nXthor - API State : Tracker is under DDOS attack, API disabled, aborting ... -> " + state.Descr);
throw new Exception("Tracker is under DDOS attack, API disabled"); throw new Exception("Tracker is under DDOS attack, API disabled");
case 8: case 8:
// AntiSpam Protection // AntiSpam Protection
logger.Debug("\nAPI State : Triggered AntiSpam Protection -> " + state.Descr); logger.Warn("\nXthor - API State : Triggered AntiSpam Protection -> " + state.Descr);
throw new Exception("Triggered AntiSpam Protection, please delay your requests !"); throw new Exception("Triggered AntiSpam Protection, please delay your requests !");
default: default:
// Unknown state // Unknown state
logger.Debug("\nAPI State : Unknown state, aborting querying ... -> " + state.Descr); logger.Error("\nXthor - API State : Unknown state, aborting querying ... -> " + state.Descr);
throw new Exception("Unknown state, aborting querying"); throw new Exception("Unknown state, aborting querying");
} }
} }
@ -453,7 +491,7 @@ namespace Jackett.Common.Indexers
/// </summary> /// </summary>
private void ValidateConfig() private void ValidateConfig()
{ {
logger.Debug("\nValidating Settings ... \n"); logger.Debug("\nXthor - Validating Settings ... \n");
// Check Passkey Setting // Check Passkey Setting
if (string.IsNullOrEmpty(ConfigData.PassKey.Value)) if (string.IsNullOrEmpty(ConfigData.PassKey.Value))
@ -462,7 +500,7 @@ namespace Jackett.Common.Indexers
} }
else else
{ {
logger.Debug("Validated Setting -- PassKey (auth) => " + ConfigData.PassKey.Value); logger.Debug("Xthor - Validated Setting -- PassKey (auth) => " + ConfigData.PassKey.Value);
} }
if (!string.IsNullOrEmpty(ConfigData.Accent.Value) && !string.Equals(ConfigData.Accent.Value, "1") && !string.Equals(ConfigData.Accent.Value, "2")) if (!string.IsNullOrEmpty(ConfigData.Accent.Value) && !string.Equals(ConfigData.Accent.Value, "1") && !string.Equals(ConfigData.Accent.Value, "2"))
@ -471,7 +509,7 @@ namespace Jackett.Common.Indexers
} }
else else
{ {
logger.Debug("Validated Setting -- Accent (audio) => " + ConfigData.Accent.Value); logger.Debug("Xthor - Validated Setting -- Accent (audio) => " + ConfigData.Accent.Value);
} }
} }
} }