Jackett/src/Jackett.Common/Indexers/Xthor.cs

684 lines
28 KiB
C#
Raw Normal View History

using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.IO;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig.Bespoke;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
namespace Jackett.Common.Indexers
{
/// <summary>
/// Provider for Xthor Private French Tracker
/// </summary>
public class Xthor : BaseCachingWebIndexer
{
2017-11-28 18:53:59 +00:00
private static string ApiEndpoint => "https://api.xthor.to/";
2017-11-29 18:18:08 +00:00
public override string[] LegacySiteLinks { get; protected set; } = new string[] {
"https://xthor.bz/",
};
private string TorrentCommentUrl => TorrentDescriptionUrl;
private string TorrentDescriptionUrl => SiteLink + "details.php?id={id}";
private string ReplaceMulti => ConfigData.ReplaceMulti.Value;
private bool EnhancedAnime => ConfigData.EnhancedAnime.Value;
private bool DevMode => ConfigData.DevMode.Value;
private bool CacheMode => ConfigData.HardDriveCache.Value;
private static string Directory => Path.Combine(Path.GetTempPath(), Assembly.GetExecutingAssembly().GetName().Name.ToLower(), MethodBase.GetCurrentMethod().DeclaringType?.Name.ToLower());
public Dictionary<string, string> EmulatedBrowserHeaders { get; } = new Dictionary<string, string>();
private ConfigurationDataXthor ConfigData => (ConfigurationDataXthor)configData;
public Xthor(IIndexerConfigurationService configService, Utils.Clients.WebClient w, Logger l, IProtectionService ps)
: base(
name: "Xthor",
description: "General French Private Tracker",
2017-11-28 18:53:59 +00:00
link: "https://xthor.to/",
caps: new TorznabCapabilities(),
configService: configService,
client: w,
logger: l,
p: ps,
2017-11-28 18:53:59 +00:00
downloadBase: "https://xthor.to/download.php?torrent=",
configData: new ConfigurationDataXthor())
{
Encoding = Encoding.UTF8;
Language = "fr-fr";
Type = "private";
// Clean capabilities
TorznabCaps.Categories.Clear();
// Movies
2019-01-10 14:54:28 +00:00
AddCategoryMapping(118, TorznabCatType.MoviesBluRay, "UHD FULL BLURAY");
AddCategoryMapping(119, TorznabCatType.MoviesBluRay, "UHD BLURAY REMUX");
AddCategoryMapping(107, TorznabCatType.MoviesUHD, "UHD 2160P X265");
AddCategoryMapping(1, TorznabCatType.MoviesBluRay, "FULL BLURAY");
AddCategoryMapping(2, TorznabCatType.MoviesBluRay, "BLURAY REMUX");
AddCategoryMapping(100, TorznabCatType.MoviesHD, "HD 1080P X265");
AddCategoryMapping(4, TorznabCatType.MoviesHD, "HD 1080P X264");
AddCategoryMapping(5, TorznabCatType.MoviesHD, "HD 720P X264");
AddCategoryMapping(7, TorznabCatType.MoviesSD, "SD X264");
AddCategoryMapping(8, TorznabCatType.MoviesDVD, "FULL DVD");
AddCategoryMapping(3, TorznabCatType.Movies3D, "3D");
AddCategoryMapping(6, TorznabCatType.MoviesSD, "XVID");
AddCategoryMapping(122, TorznabCatType.MoviesHD, "HDTV");
AddCategoryMapping(94, TorznabCatType.MoviesWEBDL, "WEBDL");
AddCategoryMapping(95, TorznabCatType.MoviesWEBDL, "WEBRIP");
AddCategoryMapping(12, TorznabCatType.TVDocumentary, "DOCS");
AddCategoryMapping(33, TorznabCatType.MoviesOther, "SPECTACLE");
AddCategoryMapping(31, TorznabCatType.MoviesOther, "ANIMATION");
AddCategoryMapping(9, TorznabCatType.MoviesOther, "VOSTFR");
// Series
2019-01-10 14:54:28 +00:00
AddCategoryMapping(104, TorznabCatType.TVOTHER, "BLURAY");
AddCategoryMapping(13, TorznabCatType.TVOTHER, "PACK VF");
AddCategoryMapping(15, TorznabCatType.TVHD, "HD VF");
AddCategoryMapping(14, TorznabCatType.TVSD, "SD VF");
AddCategoryMapping(98, TorznabCatType.TVOTHER, "PACK VOSTFR");
AddCategoryMapping(17, TorznabCatType.TVHD, "HD VF VOSTFR");
AddCategoryMapping(16, TorznabCatType.TVSD, "SD VF VOSTFR");
AddCategoryMapping(101, TorznabCatType.TVAnime, "PACK ANIME");
AddCategoryMapping(32, TorznabCatType.TVAnime, "ANIME VF");
AddCategoryMapping(110, TorznabCatType.TVAnime, "ANIME VOSTFR");
AddCategoryMapping(123, TorznabCatType.TVOTHER, "ANIMATION");
AddCategoryMapping(109, TorznabCatType.TVDocumentary, "DOCS");
AddCategoryMapping(30, TorznabCatType.TVOTHER, "EMISSIONS");
AddCategoryMapping(34, TorznabCatType.TVOTHER, "SPORT");
// Music
2019-01-10 14:54:28 +00:00
AddCategoryMapping(20, TorznabCatType.AudioVideo, "CONCERT");
2019-01-10 14:54:28 +00:00
// Books
AddCategoryMapping(24, TorznabCatType.BooksEbook, "ENOOKS NOVEL");
AddCategoryMapping(96, TorznabCatType.BooksMagazines, "EBOOKS MAGAZINES");
AddCategoryMapping(116, TorznabCatType.BooksEbook, "EBOOKS NOVEL JUNIOR");
AddCategoryMapping(99, TorznabCatType.BooksOther, "EBOOKS BD");
AddCategoryMapping(102, TorznabCatType.BooksComics, "EBOOKS COMICS");
AddCategoryMapping(103, TorznabCatType.BooksOther, "EBOOKS MANGA");
// SOFTWARE
AddCategoryMapping(25, TorznabCatType.PCGames, "PC GAMES");
AddCategoryMapping(27, TorznabCatType.ConsolePS3, "PS GAMES");
AddCategoryMapping(111, TorznabCatType.PCMac, "MAC GAMES");
AddCategoryMapping(112, TorznabCatType.PC, "LINUX GAMES");
AddCategoryMapping(26, TorznabCatType.ConsoleXbox360, "XBOX GAMES");
AddCategoryMapping(28, TorznabCatType.ConsoleWii, "WII GAMES");
AddCategoryMapping(29, TorznabCatType.ConsoleNDS, "NDS GAMES");
AddCategoryMapping(117, TorznabCatType.PC, "ROM");
AddCategoryMapping(21, TorznabCatType.PC, "PC SOFTWARE");
AddCategoryMapping(22, TorznabCatType.PCMac, "MAC SOFTWARE");
AddCategoryMapping(23, TorznabCatType.PCPhoneAndroid, "ANDROID");
// XxX
AddCategoryMapping(36, TorznabCatType.XXX, "XxX / Films");
AddCategoryMapping(105, TorznabCatType.XXX, "XxX / Séries");
AddCategoryMapping(114, TorznabCatType.XXX, "XxX / Lesbiennes");
AddCategoryMapping(115, TorznabCatType.XXX, "XxX / Gays");
AddCategoryMapping(113, TorznabCatType.XXX, "XxX / Hentai");
AddCategoryMapping(120, TorznabCatType.XXX, "XxX / Magazines");
}
/// <summary>
/// Configure our Provider
/// </summary>
/// <param name="configJson">Our params in Json</param>
/// <returns>Configuration state</returns>
#pragma warning disable 1998
2017-10-29 06:50:47 +00:00
public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
#pragma warning restore 1998
{
// Provider not yet configured
IsConfigured = false;
// Retrieve config values set by Jackett's user
LoadValuesFromJson(configJson);
// Check & Validate Config
ValidateConfig();
// Setting our data for a better emulated browser (maximum security)
// TODO: Encoded Content not supported by Jackett at this time
// EmulatedBrowserHeaders.Add("Accept-Encoding", "gzip, deflate");
// Clean headers
EmulatedBrowserHeaders.Clear();
// Inject headers
EmulatedBrowserHeaders.Add("Accept", "application/json-rpc, application/json");
EmulatedBrowserHeaders.Add("Content-Type", "application/json-rpc");
// Tracker is now configured
IsConfigured = true;
// Saving data
SaveConfig();
return IndexerConfigurationStatus.RequiresTesting;
}
/// <summary>
/// Execute our search query
/// </summary>
/// <param name="query">Query</param>
/// <returns>Releases</returns>
protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
{
var releases = new List<ReleaseInfo>();
var searchTerm = query.GetEpisodeSearchString() + " " + query.SanitizedSearchTerm; // use episode search string first, see issue #1202
searchTerm = searchTerm.Trim();
searchTerm = searchTerm.ToLower();
if (EnhancedAnime && query.HasSpecifiedCategories && query.Categories.Contains(TorznabCatType.TVAnime.ID))
{
System.Text.RegularExpressions.Regex regex = new Regex(" ([0-9]+)");
searchTerm = regex.Replace(searchTerm, " E$1");
}
// Check cache first so we don't query the server (if search term used or not in dev mode)
if (!DevMode && !string.IsNullOrEmpty(searchTerm))
{
lock (cache)
{
// Remove old cache items
CleanCache();
// Search in cache
var cachedResult = cache.FirstOrDefault(i => i.Query == searchTerm);
if (cachedResult != null)
return cachedResult.Results.Select(s => (ReleaseInfo)s.Clone()).ToArray();
}
}
// Build our query
var request = BuildQuery(searchTerm, query, ApiEndpoint);
// Getting results & Store content
var results = await QueryExec(request);
try
{
// Deserialize our Json Response
var xthorResponse = JsonConvert.DeserializeObject<XthorResponse>(results);
// Check Tracker's State
CheckApiState(xthorResponse.error);
// If contains torrents
if (xthorResponse.torrents != null)
{
// Adding each torrent row to releases
releases.AddRange(xthorResponse.torrents.Select(torrent =>
{
//issue #3847 replace multi keyword
if (!string.IsNullOrEmpty(ReplaceMulti))
{
System.Text.RegularExpressions.Regex regex = new Regex("(?i)([\\.\\- ])MULTI([\\.\\- ])");
torrent.name = regex.Replace(torrent.name, "$1" + ReplaceMulti + "$2");
}
var release = new ReleaseInfo
{
// Mapping data
Category = MapTrackerCatToNewznab(torrent.category.ToString()),
Title = torrent.name,
Seeders = torrent.seeders,
Peers = torrent.seeders + torrent.leechers,
MinimumRatio = 1,
MinimumSeedTime = 345600,
PublishDate = DateTimeUtil.UnixTimestampToDateTime(torrent.added),
Size = torrent.size,
Grabs = torrent.times_completed,
Files = torrent.numfiles,
UploadVolumeFactor = 1,
DownloadVolumeFactor = (torrent.freeleech == 1 ? 0 : 1),
Guid = new Uri(TorrentDescriptionUrl.Replace("{id}", torrent.id.ToString())),
Comments = new Uri(TorrentCommentUrl.Replace("{id}", torrent.id.ToString())),
Link = new Uri(torrent.download_link),
TMDb = torrent.tmdb_id
};
if (DevMode)
{
Output(release.ToString());
}
return release;
}));
}
}
catch (Exception ex)
{
OnParseError("Unable to parse result \n" + ex.StackTrace, ex);
}
// Return found releases
return releases;
}
/// <summary>
/// Response from Tracker's API
/// </summary>
public class XthorResponse
{
public XthorError error { get; set; }
public XthorUser user { get; set; }
public List<XthorTorrent> torrents { get; set; }
}
/// <summary>
/// State of API
/// </summary>
public class XthorError
{
public int code { get; set; }
public string descr { get; set; }
}
/// <summary>
/// User Informations
/// </summary>
public class XthorUser
{
public int id { get; set; }
public string username { get; set; }
public long uploaded { get; set; }
public long downloaded { get; set; }
public int uclass { get; set; } // Class is a reserved keyword.
public decimal bonus_point { get; set; }
public int hits_and_run { get; set; }
public string avatar_url { get; set; }
}
/// <summary>
/// Torrent Informations
/// </summary>
public class XthorTorrent
{
public int id { get; set; }
public int category { get; set; }
public int seeders { get; set; }
public int leechers { get; set; }
public string name { get; set; }
public int times_completed { get; set; }
public long size { get; set; }
public int added { get; set; }
public int freeleech { get; set; }
public int numfiles { get; set; }
public string release_group { get; set; }
public string download_link { get; set; }
public int tmdb_id { get; set; }
public override string ToString()
{
return string.Format("[XthorTorrent: id={0}, category={1}, seeders={2}, leechers={3}, name={4}, times_completed={5}, size={6}, added={7}, freeleech={8}, numfiles={9}, release_group={10}, download_link={11}, tmdb_id={12}]", id, category, seeders, leechers, name, times_completed, size, added, freeleech, numfiles, release_group, download_link, tmdb_id);
}
}
/// <summary>
/// Build query to process
/// </summary>
/// <param name="term">Term to search</param>
/// <param name="query">Torznab Query for categories mapping</param>
/// <param name="url">Search url for provider</param>
/// <returns>URL to query for parsing and processing results</returns>
private string BuildQuery(string term, TorznabQuery query, string url)
{
var parameters = new NameValueCollection();
var categoriesList = MapTorznabCapsToTrackers(query);
// Passkey
parameters.Add("passkey", ConfigData.PassKey.Value);
// If search term provided
if (!string.IsNullOrWhiteSpace(term))
{
// Add search term
// ReSharper disable once AssignNullToNotNullAttribute
parameters.Add("search", WebUtility.UrlEncode(term));
}
else
{
parameters.Add("search", string.Empty);
// Showing all torrents (just for output function)
term = "all";
}
// Loop on Categories needed
if (categoriesList.Count > 0)
{
2017-12-04 20:38:21 +00:00
parameters.Add("category", string.Join("+", categoriesList));
}
// If Only Freeleech Enabled
if (ConfigData.Freeleech.Value)
{
parameters.Add("freeleech", "1");
}
if (!string.IsNullOrEmpty(ConfigData.Accent.Value))
{
parameters.Add("accent", ConfigData.Accent.Value);
}
// Building our query -- Cannot use GetQueryString due to UrlEncode (generating wrong category param)
url += "?" + string.Join("&", parameters.AllKeys.Select(a => a + "=" + parameters[a]));
Output("\nBuilded query for \"" + term + "\"... " + url);
// Return our search url
return url;
}
/// <summary>
/// Switch Method for Querying
/// </summary>
/// <param name="request">URL created by Query Builder</param>
/// <returns>Results from query</returns>
private async Task<String> QueryExec(string request)
{
String results;
// Switch in we are in DEV mode with Hard Drive Cache or not
if (DevMode && CacheMode)
{
// Check Cache before querying and load previous results if available
results = await QueryCache(request);
}
else
{
// Querying tracker directly
results = await QueryTracker(request);
}
return results;
}
/// <summary>
/// Get Torrents Page from Cache by Query Provided
/// </summary>
/// <param name="request">URL created by Query Builder</param>
/// <returns>Results from query</returns>
private async Task<String> QueryCache(string request)
{
String results;
// Create Directory if not exist
System.IO.Directory.CreateDirectory(Directory);
// Clean Storage Provider Directory from outdated cached queries
CleanCacheStorage();
// File Name
string fileName = StringUtil.HashSHA1(request) + ".json";
// Create fingerprint for request
string file = Path.Combine(Directory, fileName);
// Checking modes states
if (File.Exists(file))
{
// File exist... loading it right now !
Output("Loading results from hard drive cache ..." + fileName);
try
{
using (StreamReader fileReader = File.OpenText(file))
{
JsonSerializer serializer = new JsonSerializer();
results = (String)serializer.Deserialize(fileReader, typeof(String));
}
}
catch (Exception e)
{
Output("Error loading cached results ! " + e.Message, "error");
results = null;
}
}
else
{
// No cached file found, querying tracker directly
results = await QueryTracker(request);
// Cached file didn't exist for our query, writing it right now !
Output("Writing results to hard drive cache ..." + fileName);
using (StreamWriter fileWriter = File.CreateText(file))
{
JsonSerializer serializer = new JsonSerializer();
serializer.Serialize(fileWriter, results);
}
}
return results;
}
/// <summary>
/// Get Torrents Page from Tracker by Query Provided
/// </summary>
/// <param name="request">URL created by Query Builder</param>
/// <returns>Results from query</returns>
private async Task<String> QueryTracker(string request)
{
// Cache mode not enabled or cached file didn't exist for our query
Output("\nQuerying tracker for results....");
// Build WebRequest for index
var myIndexRequest = new Utils.Clients.WebRequest()
{
Type = RequestType.GET,
Url = request,
Encoding = Encoding,
Headers = EmulatedBrowserHeaders
};
// Request our first page
var results = await webclient.GetString(myIndexRequest);
2017-12-01 07:02:52 +00:00
if (results.Status == HttpStatusCode.InternalServerError) // See issue #2110
throw new Exception("Internal Server Error (" + results.Content + "), probably you reached the API limits, please reduce the number of queries");
// Return results from tracker
return results.Content;
}
/// <summary>
/// Check API's state
/// </summary>
/// <param name="state">State of API</param>
private void CheckApiState(XthorError state)
{
// Switch on state
switch (state.code)
{
case 0:
// Everything OK
Output("\nAPI State : Everything OK ... -> " + state.descr);
break;
2017-10-29 06:50:47 +00:00
case 1:
// Passkey not found
Output("\nAPI State : Error, Passkey not found in tracker's database, aborting... -> " + state.descr);
throw new Exception("Passkey not found in tracker's database");
case 2:
// No results
Output("\nAPI State : No results for query ... -> " + state.descr);
break;
2017-10-29 06:50:47 +00:00
case 3:
// Power Saver
Output("\nAPI State : Power Saver mode, only cached query with no parameters available ... -> " + state.descr);
break;
2017-10-29 06:50:47 +00:00
case 4:
// DDOS Attack, API disabled
Output("\nAPI State : Tracker is under DDOS attack, API disabled, aborting ... -> " + state.descr);
throw new Exception("Tracker is under DDOS attack, API disabled");
default:
// Unknown state
Output("\nAPI State : Unknown state, aborting querying ... -> " + state.descr);
throw new Exception("Unknown state, aborting querying");
}
}
/// <summary>
/// Clean Hard Drive Cache Storage
/// </summary>
/// <param name="force">Force Provider Folder deletion</param>
private void CleanCacheStorage(bool force = false)
{
// Check cleaning method
if (force)
{
// Deleting Provider Storage folder and all files recursively
Output("\nDeleting Provider Storage folder and all files recursively ...");
// Check if directory exist
if (System.IO.Directory.Exists(Directory))
{
// Delete storage directory of provider
System.IO.Directory.Delete(Directory, true);
Output("-> Storage folder deleted successfully.");
}
else
{
// No directory, so nothing to do
Output("-> No Storage folder found for this provider !");
}
}
else
{
var i = 0;
// Check if there is file older than ... and delete them
Output("\nCleaning Provider Storage folder... in progress.");
System.IO.Directory.GetFiles(Directory)
.Select(f => new FileInfo(f))
.Where(f => f.LastAccessTime < DateTime.Now.AddMilliseconds(-Convert.ToInt32(ConfigData.HardDriveCacheKeepTime.Value)))
.ToList()
.ForEach(f =>
{
Output("Deleting cached file << " + f.Name + " >> ... done.");
f.Delete();
i++;
});
// Inform on what was cleaned during process
if (i > 0)
{
Output("-> Deleted " + i + " cached files during cleaning.");
}
else
{
Output("-> Nothing deleted during cleaning.");
}
}
}
/// <summary>
/// Output message for logging or developpment (console)
/// </summary>
/// <param name="message">Message to output</param>
/// <param name="level">Level for Logger</param>
private void Output(string message, string level = "debug")
{
// Check if we are in dev mode
if (DevMode)
{
// Output message to console
Console.WriteLine(message);
}
else
{
// Send message to logger with level
switch (level)
{
default:
goto case "debug";
case "debug":
// Only if Debug Level Enabled on Jackett
if (logger.IsDebugEnabled)
{
logger.Debug(message);
}
break;
2017-10-29 06:50:47 +00:00
case "info":
logger.Info(message);
break;
2017-10-29 06:50:47 +00:00
case "error":
logger.Error(message);
break;
}
}
}
/// <summary>
/// Validate Config entered by user on Jackett
/// </summary>
private void ValidateConfig()
{
Output("\nValidating Settings ... \n");
// Check Passkey Setting
if (string.IsNullOrEmpty(ConfigData.PassKey.Value))
{
throw new ExceptionWithConfigData("You must provide your passkey for this tracker to be allowed to use API !", ConfigData);
}
else
{
Output("Validated Setting -- PassKey (auth) => " + ConfigData.PassKey.Value);
}
if (!string.IsNullOrEmpty(ConfigData.Accent.Value) && !string.Equals(ConfigData.Accent.Value, "1") && !string.Equals(ConfigData.Accent.Value, "2"))
{
throw new ExceptionWithConfigData("Only '1' or '2' are available in the Accent parameter.", ConfigData);
}
else
{
Output("Validated Setting -- Accent (audio) => " + ConfigData.Accent.Value);
}
// Check Dev Cache Settings
if (ConfigData.HardDriveCache.Value)
{
Output("\nValidated Setting -- DEV Hard Drive Cache enabled");
// Check if Dev Mode enabled !
if (!ConfigData.DevMode.Value)
{
throw new ExceptionWithConfigData("Hard Drive is enabled but not in DEV MODE, Please enable DEV MODE !", ConfigData);
}
// Check Cache Keep Time Setting
if (!string.IsNullOrEmpty(ConfigData.HardDriveCacheKeepTime.Value))
{
try
{
Output("Validated Setting -- Cache Keep Time (ms) => " + Convert.ToInt32(ConfigData.HardDriveCacheKeepTime.Value));
}
catch (Exception)
{
throw new ExceptionWithConfigData("Please enter a numeric hard drive keep time in ms !", ConfigData);
}
}
else
{
throw new ExceptionWithConfigData("Hard Drive Cache enabled, Please enter a maximum keep time for cache !", ConfigData);
}
}
else
{
// Delete cache if previously existed
CleanCacheStorage(true);
}
}
}
2017-11-28 18:53:59 +00:00
}