using System; using System.Collections.Generic; using System.Collections.Specialized; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using System.Net; using System.Reflection; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig.Bespoke; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; using Jackett.Common.Utils.Clients; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NLog; using WebRequest = Jackett.Common.Utils.Clients.WebRequest; namespace Jackett.Common.Indexers { [ExcludeFromCodeCoverage] public class Xthor : BaseCachingWebIndexer { private static string ApiEndpoint => "https://api.xthor.tk/"; public override string[] LegacySiteLinks { get; protected set; } = new string[] { "https://xthor.bz/", "https://xthor.to", }; private string TorrentCommentUrl => TorrentDescriptionUrl; private string TorrentDescriptionUrl => SiteLink + "details.php?id={id}"; private string ReplaceMulti => ConfigData.ReplaceMulti.Value; private bool EnhancedAnime => ConfigData.EnhancedAnime.Value; private bool DevMode => ConfigData.DevMode.Value; private bool CacheMode => ConfigData.HardDriveCache.Value; private static string Directory => Path.Combine(Path.GetTempPath(), Assembly.GetExecutingAssembly().GetName().Name.ToLower(), MethodBase.GetCurrentMethod().DeclaringType?.Name.ToLower()); public Dictionary EmulatedBrowserHeaders { get; } = new Dictionary(); private ConfigurationDataXthor ConfigData => (ConfigurationDataXthor)configData; public Xthor(IIndexerConfigurationService configService, Utils.Clients.WebClient w, Logger l, IProtectionService ps) : base(id: "xthor", name: "Xthor", description: "General French Private Tracker", link: "https://xthor.tk/", caps: new TorznabCapabilities { MovieSearchParams = new List { MovieSearchParam.Q } }, configService: configService, client: w, logger: l, p: ps, downloadBase: "https://xthor.tk/download.php?torrent=", configData: new ConfigurationDataXthor()) { Encoding = Encoding.UTF8; Language = "fr-fr"; Type = "private"; // Movies AddCategoryMapping(118, TorznabCatType.MoviesBluRay, "UHD FULL BLURAY"); AddCategoryMapping(119, TorznabCatType.MoviesBluRay, "UHD BLURAY REMUX"); AddCategoryMapping(107, TorznabCatType.MoviesUHD, "UHD 2160P X265"); AddCategoryMapping(1, TorznabCatType.MoviesBluRay, "FULL BLURAY"); AddCategoryMapping(2, TorznabCatType.MoviesBluRay, "BLURAY REMUX"); AddCategoryMapping(100, TorznabCatType.MoviesHD, "HD 1080P X265"); AddCategoryMapping(4, TorznabCatType.MoviesHD, "HD 1080P X264"); AddCategoryMapping(5, TorznabCatType.MoviesHD, "HD 720P X264"); AddCategoryMapping(7, TorznabCatType.MoviesSD, "SD X264"); AddCategoryMapping(8, TorznabCatType.MoviesDVD, "FULL DVD"); AddCategoryMapping(3, TorznabCatType.Movies3D, "3D"); AddCategoryMapping(6, TorznabCatType.MoviesSD, "XVID"); AddCategoryMapping(122, TorznabCatType.MoviesHD, "HDTV"); AddCategoryMapping(94, TorznabCatType.MoviesWEBDL, "WEBDL"); AddCategoryMapping(95, TorznabCatType.MoviesWEBDL, "WEBRIP"); AddCategoryMapping(12, TorznabCatType.TVDocumentary, "DOCS"); AddCategoryMapping(33, TorznabCatType.MoviesOther, "SPECTACLE"); AddCategoryMapping(31, TorznabCatType.MoviesOther, "ANIMATION"); AddCategoryMapping(9, TorznabCatType.MoviesOther, "VOSTFR"); // Series AddCategoryMapping(104, TorznabCatType.TVOTHER, "BLURAY"); AddCategoryMapping(13, TorznabCatType.TVOTHER, "PACK VF"); AddCategoryMapping(15, TorznabCatType.TVHD, "HD VF"); AddCategoryMapping(14, TorznabCatType.TVSD, "SD VF"); AddCategoryMapping(98, TorznabCatType.TVOTHER, "PACK VOSTFR"); AddCategoryMapping(17, TorznabCatType.TVHD, "HD VF VOSTFR"); AddCategoryMapping(16, TorznabCatType.TVSD, "SD VF VOSTFR"); AddCategoryMapping(101, TorznabCatType.TVAnime, "PACK ANIME"); AddCategoryMapping(32, TorznabCatType.TVAnime, "ANIME VF"); AddCategoryMapping(110, TorznabCatType.TVAnime, "ANIME VOSTFR"); AddCategoryMapping(123, TorznabCatType.TVOTHER, "ANIMATION"); AddCategoryMapping(109, TorznabCatType.TVDocumentary, "DOCS"); AddCategoryMapping(30, TorznabCatType.TVOTHER, "EMISSIONS"); AddCategoryMapping(34, TorznabCatType.TVOTHER, "SPORT"); // Music AddCategoryMapping(20, TorznabCatType.AudioVideo, "CONCERT"); // Books AddCategoryMapping(24, TorznabCatType.BooksEbook, "ENOOKS NOVEL"); AddCategoryMapping(96, TorznabCatType.BooksMagazines, "EBOOKS MAGAZINES"); AddCategoryMapping(116, TorznabCatType.BooksEbook, "EBOOKS NOVEL JUNIOR"); AddCategoryMapping(99, TorznabCatType.BooksOther, "EBOOKS BD"); AddCategoryMapping(102, TorznabCatType.BooksComics, "EBOOKS COMICS"); AddCategoryMapping(103, TorznabCatType.BooksOther, "EBOOKS MANGA"); // SOFTWARE AddCategoryMapping(25, TorznabCatType.PCGames, "PC GAMES"); AddCategoryMapping(27, TorznabCatType.ConsolePS3, "PS GAMES"); AddCategoryMapping(111, TorznabCatType.PCMac, "MAC GAMES"); AddCategoryMapping(112, TorznabCatType.PC, "LINUX GAMES"); AddCategoryMapping(26, TorznabCatType.ConsoleXbox360, "XBOX GAMES"); AddCategoryMapping(28, TorznabCatType.ConsoleWii, "WII GAMES"); AddCategoryMapping(29, TorznabCatType.ConsoleNDS, "NDS GAMES"); AddCategoryMapping(117, TorznabCatType.PC, "ROM"); AddCategoryMapping(21, TorznabCatType.PC, "PC SOFTWARE"); AddCategoryMapping(22, TorznabCatType.PCMac, "MAC SOFTWARE"); AddCategoryMapping(23, TorznabCatType.PCPhoneAndroid, "ANDROID"); // XxX AddCategoryMapping(36, TorznabCatType.XXX, "XxX / Films"); AddCategoryMapping(105, TorznabCatType.XXX, "XxX / Séries"); AddCategoryMapping(114, TorznabCatType.XXX, "XxX / Lesbiennes"); AddCategoryMapping(115, TorznabCatType.XXX, "XxX / Gays"); AddCategoryMapping(113, TorznabCatType.XXX, "XxX / Hentai"); AddCategoryMapping(120, TorznabCatType.XXX, "XxX / Magazines"); } /// /// Configure our Provider /// /// Our params in Json /// Configuration state // Warning 1998 is async method with no await calls inside // TODO: Remove pragma by wrapping return in Task.FromResult and removing async #pragma warning disable 1998 public override async Task ApplyConfiguration(JToken configJson) #pragma warning restore 1998 { // Provider not yet configured IsConfigured = false; // Retrieve config values set by Jackett's user LoadValuesFromJson(configJson); // Check & Validate Config ValidateConfig(); // Setting our data for a better emulated browser (maximum security) // TODO: Encoded Content not supported by Jackett at this time // EmulatedBrowserHeaders.Add("Accept-Encoding", "gzip, deflate"); // Clean headers EmulatedBrowserHeaders.Clear(); // Inject headers EmulatedBrowserHeaders.Add("Accept", "application/json-rpc, application/json"); EmulatedBrowserHeaders.Add("Content-Type", "application/json-rpc"); // Tracker is now configured IsConfigured = true; // Saving data SaveConfig(); return IndexerConfigurationStatus.RequiresTesting; } /// /// Execute our search query /// /// Query /// Releases protected override async Task> PerformQuery(TorznabQuery query) { var releases = new List(); var searchTerm = query.GetEpisodeSearchString() + " " + query.SanitizedSearchTerm; // use episode search string first, see issue #1202 searchTerm = searchTerm.Trim(); searchTerm = searchTerm.ToLower(); if (EnhancedAnime && query.HasSpecifiedCategories && (query.Categories.Contains(TorznabCatType.TVAnime.ID) || query.Categories.Contains(100032) || query.Categories.Contains(100101) || query.Categories.Contains(100110))) { var regex = new Regex(" ([0-9]+)"); searchTerm = regex.Replace(searchTerm, " E$1"); } // Check cache first so we don't query the server (if search term used or not in dev mode) if (!DevMode && !string.IsNullOrEmpty(searchTerm)) { lock (cache) { // Remove old cache items CleanCache(); // Search in cache var cachedResult = cache.FirstOrDefault(i => i.Query == searchTerm); if (cachedResult != null) return cachedResult.Results.Select(s => (ReleaseInfo)s.Clone()).ToArray(); } } // Build our query var request = BuildQuery(searchTerm, query, ApiEndpoint); // Getting results & Store content var results = await QueryExec(request); try { // Deserialize our Json Response var xthorResponse = JsonConvert.DeserializeObject(results); // Check Tracker's State CheckApiState(xthorResponse.error); // If contains torrents if (xthorResponse.torrents != null) { // Adding each torrent row to releases releases.AddRange(xthorResponse.torrents.Select(torrent => { //issue #3847 replace multi keyword if (!string.IsNullOrEmpty(ReplaceMulti)) { var regex = new Regex("(?i)([\\.\\- ])MULTI([\\.\\- ])"); torrent.name = regex.Replace(torrent.name, "$1" + ReplaceMulti + "$2"); } // issue #8759 replace vostfr and subfrench with English if (ConfigData.Vostfr.Value) torrent.name = torrent.name.Replace("VOSTFR","ENGLISH").Replace("SUBFRENCH","ENGLISH"); var publishDate = DateTimeUtil.UnixTimestampToDateTime(torrent.added); //TODO replace with download link? var guid = new Uri(TorrentDescriptionUrl.Replace("{id}", torrent.id.ToString())); var comments = new Uri(TorrentCommentUrl.Replace("{id}", torrent.id.ToString())); var link = new Uri(torrent.download_link); var release = new ReleaseInfo { // Mapping data Category = MapTrackerCatToNewznab(torrent.category.ToString()), Title = torrent.name, Seeders = torrent.seeders, Peers = torrent.seeders + torrent.leechers, MinimumRatio = 1, MinimumSeedTime = 345600, PublishDate = publishDate, Size = torrent.size, Grabs = torrent.times_completed, Files = torrent.numfiles, UploadVolumeFactor = 1, DownloadVolumeFactor = (torrent.freeleech == 1 ? 0 : 1), Guid = guid, Comments = comments, Link = link, TMDb = torrent.tmdb_id }; //TODO make consistent with other trackers if (DevMode) { Output(release.ToString()); } return release; })); } } catch (Exception ex) { OnParseError("Unable to parse result \n" + ex.StackTrace, ex); } // Return found releases return releases; } /// /// Response from Tracker's API /// public class XthorResponse { public XthorError error { get; set; } public XthorUser user { get; set; } public List torrents { get; set; } } /// /// State of API /// public class XthorError { public int code { get; set; } public string descr { get; set; } } /// /// User Informations /// public class XthorUser { public int id { get; set; } public string username { get; set; } public long uploaded { get; set; } public long downloaded { get; set; } public int uclass { get; set; } // Class is a reserved keyword. public decimal bonus_point { get; set; } public int hits_and_run { get; set; } public string avatar_url { get; set; } } /// /// Torrent Informations /// public class XthorTorrent { public int id { get; set; } public int category { get; set; } public int seeders { get; set; } public int leechers { get; set; } public string name { get; set; } public int times_completed { get; set; } public long size { get; set; } public int added { get; set; } public int freeleech { get; set; } public int numfiles { get; set; } public string release_group { get; set; } public string download_link { get; set; } public int tmdb_id { get; set; } public override string ToString() => string.Format("[XthorTorrent: id={0}, category={1}, seeders={2}, leechers={3}, name={4}, times_completed={5}, size={6}, added={7}, freeleech={8}, numfiles={9}, release_group={10}, download_link={11}, tmdb_id={12}]", id, category, seeders, leechers, name, times_completed, size, added, freeleech, numfiles, release_group, download_link, tmdb_id); } /// /// Build query to process /// /// Term to search /// Torznab Query for categories mapping /// Search url for provider /// URL to query for parsing and processing results private string BuildQuery(string term, TorznabQuery query, string url) { var parameters = new NameValueCollection(); var categoriesList = MapTorznabCapsToTrackers(query); // Passkey parameters.Add("passkey", ConfigData.PassKey.Value); // If search term provided if (!string.IsNullOrWhiteSpace(term)) { // Add search term // ReSharper disable once AssignNullToNotNullAttribute parameters.Add("search", WebUtility.UrlEncode(term)); } else { parameters.Add("search", string.Empty); // Showing all torrents (just for output function) term = "all"; } // Loop on Categories needed if (categoriesList.Count > 0) { parameters.Add("category", string.Join("+", categoriesList)); } // If Only Freeleech Enabled if (ConfigData.Freeleech.Value) { parameters.Add("freeleech", "1"); } if (!string.IsNullOrEmpty(ConfigData.Accent.Value)) { parameters.Add("accent", ConfigData.Accent.Value); } // Building our query -- Cannot use GetQueryString due to UrlEncode (generating wrong category param) url += "?" + string.Join("&", parameters.AllKeys.Select(a => a + "=" + parameters[a])); Output("\nBuilded query for \"" + term + "\"... " + url); // Return our search url return url; } /// /// Switch Method for Querying /// /// URL created by Query Builder /// Results from query private async Task QueryExec(string request) { string results; // Switch in we are in DEV mode with Hard Drive Cache or not if (DevMode && CacheMode) { // Check Cache before querying and load previous results if available results = await QueryCache(request); } else { // Querying tracker directly results = await QueryTracker(request); } return results; } /// /// Get Torrents Page from Cache by Query Provided /// /// URL created by Query Builder /// Results from query private async Task QueryCache(string request) { string results; // Create Directory if not exist System.IO.Directory.CreateDirectory(Directory); // Clean Storage Provider Directory from outdated cached queries CleanCacheStorage(); // File Name var fileName = StringUtil.HashSHA1(request) + ".json"; // Create fingerprint for request var file = Path.Combine(Directory, fileName); // Checking modes states if (File.Exists(file)) { // File exist... loading it right now ! Output("Loading results from hard drive cache ..." + fileName); try { using (var fileReader = File.OpenText(file)) { var serializer = new JsonSerializer(); results = (string)serializer.Deserialize(fileReader, typeof(string)); } } catch (Exception e) { Output("Error loading cached results ! " + e.Message, "error"); results = null; } } else { // No cached file found, querying tracker directly results = await QueryTracker(request); // Cached file didn't exist for our query, writing it right now ! Output("Writing results to hard drive cache ..." + fileName); using (var fileWriter = File.CreateText(file)) { var serializer = new JsonSerializer(); serializer.Serialize(fileWriter, results); } } return results; } /// /// Get Torrents Page from Tracker by Query Provided /// /// URL created by Query Builder /// Results from query private async Task QueryTracker(string request) { // Cache mode not enabled or cached file didn't exist for our query Output("\nQuerying tracker for results...."); // Build WebRequest for index var myIndexRequest = new Utils.Clients.WebRequest() { Type = RequestType.GET, Url = request, Encoding = Encoding, Headers = EmulatedBrowserHeaders }; // Request our first page var results = await webclient.GetResultAsync(myIndexRequest); if (results.Status == HttpStatusCode.InternalServerError) // See issue #2110 throw new Exception("Internal Server Error (" + results.ContentString + "), probably you reached the API limits, please reduce the number of queries"); // Return results from tracker return results.ContentString; } /// /// Check API's state /// /// State of API private void CheckApiState(XthorError state) { // Switch on state switch (state.code) { case 0: // Everything OK Output("\nAPI State : Everything OK ... -> " + state.descr); break; case 1: // Passkey not found Output("\nAPI State : Error, Passkey not found in tracker's database, aborting... -> " + state.descr); throw new Exception("Passkey not found in tracker's database"); case 2: // No results Output("\nAPI State : No results for query ... -> " + state.descr); break; case 3: // Power Saver Output("\nAPI State : Power Saver mode, only cached query with no parameters available ... -> " + state.descr); break; case 4: // DDOS Attack, API disabled Output("\nAPI State : Tracker is under DDOS attack, API disabled, aborting ... -> " + state.descr); throw new Exception("Tracker is under DDOS attack, API disabled"); default: // Unknown state Output("\nAPI State : Unknown state, aborting querying ... -> " + state.descr); throw new Exception("Unknown state, aborting querying"); } } /// /// Clean Hard Drive Cache Storage /// /// Force Provider Folder deletion private void CleanCacheStorage(bool force = false) { // Check cleaning method if (force) { // Deleting Provider Storage folder and all files recursively Output("\nDeleting Provider Storage folder and all files recursively ..."); // Check if directory exist if (System.IO.Directory.Exists(Directory)) { // Delete storage directory of provider System.IO.Directory.Delete(Directory, true); Output("-> Storage folder deleted successfully."); } else { // No directory, so nothing to do Output("-> No Storage folder found for this provider !"); } } else { var i = 0; // Check if there is file older than ... and delete them Output("\nCleaning Provider Storage folder... in progress."); System.IO.Directory.GetFiles(Directory) .Select(f => new FileInfo(f)) .Where(f => f.LastAccessTime < DateTime.Now.AddMilliseconds(-Convert.ToInt32(ConfigData.HardDriveCacheKeepTime.Value))) .ToList() .ForEach(f => { Output("Deleting cached file << " + f.Name + " >> ... done."); f.Delete(); i++; }); // Inform on what was cleaned during process if (i > 0) { Output("-> Deleted " + i + " cached files during cleaning."); } else { Output("-> Nothing deleted during cleaning."); } } } /// /// Output message for logging or developpment (console) /// /// Message to output /// Level for Logger private void Output(string message, string level = "debug") { // Check if we are in dev mode if (DevMode) { // Output message to console Console.WriteLine(message); } else { // Send message to logger with level switch (level) { default: goto case "debug"; case "debug": // Only if Debug Level Enabled on Jackett if (logger.IsDebugEnabled) { logger.Debug(message); } break; case "info": logger.Info(message); break; case "error": logger.Error(message); break; } } } /// /// Validate Config entered by user on Jackett /// private void ValidateConfig() { Output("\nValidating Settings ... \n"); // Check Passkey Setting if (string.IsNullOrEmpty(ConfigData.PassKey.Value)) { throw new ExceptionWithConfigData("You must provide your passkey for this tracker to be allowed to use API !", ConfigData); } else { Output("Validated Setting -- PassKey (auth) => " + ConfigData.PassKey.Value); } if (!string.IsNullOrEmpty(ConfigData.Accent.Value) && !string.Equals(ConfigData.Accent.Value, "1") && !string.Equals(ConfigData.Accent.Value, "2")) { throw new ExceptionWithConfigData("Only '1' or '2' are available in the Accent parameter.", ConfigData); } else { Output("Validated Setting -- Accent (audio) => " + ConfigData.Accent.Value); } // Check Dev Cache Settings if (ConfigData.HardDriveCache.Value) { Output("\nValidated Setting -- DEV Hard Drive Cache enabled"); // Check if Dev Mode enabled ! if (!ConfigData.DevMode.Value) { throw new ExceptionWithConfigData("Hard Drive is enabled but not in DEV MODE, Please enable DEV MODE !", ConfigData); } // Check Cache Keep Time Setting if (!string.IsNullOrEmpty(ConfigData.HardDriveCacheKeepTime.Value)) { try { Output("Validated Setting -- Cache Keep Time (ms) => " + Convert.ToInt32(ConfigData.HardDriveCacheKeepTime.Value)); } catch (Exception) { throw new ExceptionWithConfigData("Please enter a numeric hard drive keep time in ms !", ConfigData); } } else { throw new ExceptionWithConfigData("Hard Drive Cache enabled, Please enter a maximum keep time for cache !", ConfigData); } } else { // Delete cache if previously existed CleanCacheStorage(true); } } } }