From 8707e6b2e9668421e7bc0e1826685f10b60c1ebd Mon Sep 17 00:00:00 2001 From: JigSaw Date: Sun, 13 Aug 2017 20:49:03 +0200 Subject: [PATCH] fix(indexers): fixed and optimized xthor indexer * feat(utils): added sha1 hash function and refactored md5 hash function * fix(indexers): now use cross plateform path building for dev mode * fix(indexers): fix output log of xthor for dev mode * feat(release): added ToString method * refactor(dev): optimized dev mode * style(clean): cleanup code * feat(indexer): added tmdb info to releases * fix(cat): enabled categories on xthor --- src/Jackett/Indexers/Xthor.cs | 107 ++++++++++++++++++++---------- src/Jackett/Models/ReleaseInfo.cs | 5 ++ src/Jackett/Utils/StringUtil.cs | 44 +++++++++--- 3 files changed, 112 insertions(+), 44 deletions(-) diff --git a/src/Jackett/Indexers/Xthor.cs b/src/Jackett/Indexers/Xthor.cs index a80be3380..63a6541b2 100644 --- a/src/Jackett/Indexers/Xthor.cs +++ b/src/Jackett/Indexers/Xthor.cs @@ -1,9 +1,10 @@ using System; using System.Collections.Generic; using System.Collections.Specialized; +using System.IO; using System.Linq; -using System.Text; using System.Reflection; +using System.Text; using System.Threading.Tasks; using System.Web; using Jackett.Models; @@ -27,7 +28,7 @@ namespace Jackett.Indexers private string TorrentDescriptionUrl => SiteLink + "details.php?id={id}"; private bool DevMode => ConfigData.DevMode.Value; private bool CacheMode => ConfigData.HardDriveCache.Value; - private static string Directory => System.IO.Path.GetTempPath() + "Jackett\\" + MethodBase.GetCurrentMethod().DeclaringType?.Name + "\\"; + private static string Directory => Path.Combine(Path.GetTempPath(), Assembly.GetExecutingAssembly().GetName().Name.ToLower(), MethodBase.GetCurrentMethod().DeclaringType?.Name.ToLower()); public Dictionary EmulatedBrowserHeaders { get; } = new Dictionary(); private ConfigurationDataXthor ConfigData => (ConfigurationDataXthor)configData; @@ -130,7 +131,7 @@ namespace Jackett.Indexers // Setting our data for a better emulated browser (maximum security) // TODO: Encoded Content not supported by Jackett at this time - // emulatedBrowserHeaders.Add("Accept-Encoding", "gzip, deflate"); + // EmulatedBrowserHeaders.Add("Accept-Encoding", "gzip, deflate"); // Clean headers EmulatedBrowserHeaders.Clear(); @@ -183,7 +184,7 @@ namespace Jackett.Indexers try { // Deserialize our Json Response - var xthorResponse = JsonConvert.DeserializeObject(results.Content); + var xthorResponse = JsonConvert.DeserializeObject(results); // Check Tracker's State CheckApiState(xthorResponse.error); @@ -192,24 +193,35 @@ namespace Jackett.Indexers if (xthorResponse.torrents != null) { // Adding each torrent row to releases - releases.AddRange(xthorResponse.torrents.Select(torrent => new ReleaseInfo + releases.AddRange(xthorResponse.torrents.Select(torrent => { - // Mapping data - Category = MapTrackerCatToNewznab(torrent.category.ToString()), - Title = torrent.name, - Seeders = torrent.seeders, - Peers = torrent.seeders + torrent.leechers, - MinimumRatio = 1, - MinimumSeedTime = 345600, - PublishDate = DateTimeUtil.UnixTimestampToDateTime(torrent.added), - Size = torrent.size, - Grabs = torrent.times_completed, - Files = torrent.numfiles, - UploadVolumeFactor = 1, - DownloadVolumeFactor = (torrent.freeleech == 1 ? 0 : 1), - Guid = new Uri(TorrentDescriptionUrl.Replace("{id}", torrent.id.ToString())), - Comments = new Uri(TorrentCommentUrl.Replace("{id}", torrent.id.ToString())), - Link = new Uri(torrent.download_link) + var release = new ReleaseInfo + { + // Mapping data + Category = MapTrackerCatToNewznab(torrent.category.ToString()), + Title = torrent.name, + Seeders = torrent.seeders, + Peers = torrent.seeders + torrent.leechers, + MinimumRatio = 1, + MinimumSeedTime = 345600, + PublishDate = DateTimeUtil.UnixTimestampToDateTime(torrent.added), + Size = torrent.size, + Grabs = torrent.times_completed, + Files = torrent.numfiles, + UploadVolumeFactor = 1, + DownloadVolumeFactor = (torrent.freeleech == 1 ? 0 : 1), + Guid = new Uri(TorrentDescriptionUrl.Replace("{id}", torrent.id.ToString())), + Comments = new Uri(TorrentCommentUrl.Replace("{id}", torrent.id.ToString())), + Link = new Uri(torrent.download_link), + TMDb = torrent.tmdb_id + }; + + if (DevMode) + { + Output(release.ToString()); + } + + return release; })); } } @@ -273,6 +285,12 @@ namespace Jackett.Indexers public int numfiles { get; set; } public string release_group { get; set; } public string download_link { get; set; } + public int tmdb_id { get; set; } + + public override string ToString() + { + return string.Format("[XthorTorrent: id={0}, category={1}, seeders={2}, leechers={3}, name={4}, times_completed={5}, size={6}, added={7}, freeleech={8}, numfiles={9}, release_group={10}, download_link={11}, tmdb_id={12}]", id, category, seeders, leechers, name, times_completed, size, added, freeleech, numfiles, release_group, download_link, tmdb_id); + } } /// @@ -308,7 +326,7 @@ namespace Jackett.Indexers if (categoriesList.Count > 0) { // ignore categories for now, something changed or is buggy, needs investigation - //parameters.Add("category", string.Join("+", categoriesList)); + parameters.Add("category", string.Join("+", categoriesList)); } // If Only Freeleech Enabled @@ -331,9 +349,9 @@ namespace Jackett.Indexers /// /// URL created by Query Builder /// Results from query - private async Task QueryExec(string request) + private async Task QueryExec(string request) { - WebClientStringResult results; + String results; // Switch in we are in DEV mode with Hard Drive Cache or not if (DevMode && CacheMode) @@ -354,9 +372,9 @@ namespace Jackett.Indexers /// /// URL created by Query Builder /// Results from query - private async Task QueryCache(string request) + private async Task QueryCache(string request) { - WebClientStringResult results; + String results; // Create Directory if not exist System.IO.Directory.CreateDirectory(Directory); @@ -364,15 +382,30 @@ namespace Jackett.Indexers // Clean Storage Provider Directory from outdated cached queries CleanCacheStorage(); + // File Name + string fileName = StringUtil.HashSHA1(request) + ".json"; + // Create fingerprint for request - string file = Directory + request.GetHashCode() + ".json"; + string file = Path.Combine(Directory, fileName); // Checking modes states - if (System.IO.File.Exists(file)) + if (File.Exists(file)) { // File exist... loading it right now ! - Output("Loading results from hard drive cache ..." + request.GetHashCode() + ".json"); - results = JsonConvert.DeserializeObject(System.IO.File.ReadAllText(file)); + Output("Loading results from hard drive cache ..." + fileName); + try + { + using (StreamReader fileReader = File.OpenText(file)) + { + JsonSerializer serializer = new JsonSerializer(); + results = (String)serializer.Deserialize(fileReader, typeof(String)); + } + } + catch (Exception e) + { + Output("Error loading cached results ! " + e.Message, "error"); + results = null; + } } else { @@ -380,8 +413,12 @@ namespace Jackett.Indexers results = await QueryTracker(request); // Cached file didn't exist for our query, writing it right now ! - Output("Writing results to hard drive cache ..." + request.GetHashCode() + ".json"); - System.IO.File.WriteAllText(file, JsonConvert.SerializeObject(results)); + Output("Writing results to hard drive cache ..." + fileName); + using (StreamWriter fileWriter = File.CreateText(file)) + { + JsonSerializer serializer = new JsonSerializer(); + serializer.Serialize(fileWriter, results); + } } return results; } @@ -391,7 +428,7 @@ namespace Jackett.Indexers /// /// URL created by Query Builder /// Results from query - private async Task QueryTracker(string request) + private async Task QueryTracker(string request) { // Cache mode not enabled or cached file didn't exist for our query Output("\nQuerying tracker for results...."); @@ -409,7 +446,7 @@ namespace Jackett.Indexers var results = await webclient.GetString(myIndexRequest); // Return results from tracker - return results; + return results.Content; } /// @@ -479,7 +516,7 @@ namespace Jackett.Indexers // Check if there is file older than ... and delete them Output("\nCleaning Provider Storage folder... in progress."); System.IO.Directory.GetFiles(Directory) - .Select(f => new System.IO.FileInfo(f)) + .Select(f => new FileInfo(f)) .Where(f => f.LastAccessTime < DateTime.Now.AddMilliseconds(-Convert.ToInt32(ConfigData.HardDriveCacheKeepTime.Value))) .ToList() .ForEach(f => diff --git a/src/Jackett/Models/ReleaseInfo.cs b/src/Jackett/Models/ReleaseInfo.cs index 80c0c7e4e..5e7c8da9c 100644 --- a/src/Jackett/Models/ReleaseInfo.cs +++ b/src/Jackett/Models/ReleaseInfo.cs @@ -114,5 +114,10 @@ namespace Jackett.Models { return (long)(kb * 1024f); } + + public override string ToString() + { + return string.Format("[ReleaseInfo: Title={0}, Guid={1}, Link={2}, Comments={3}, PublishDate={4}, Category={5}, Size={6}, Files={7}, Grabs={8}, Description={9}, RageID={10}, TVDBId={11}, Imdb={12}, TMDb={13}, Seeders={14}, Peers={15}, BannerUrl={16}, InfoHash={17}, MagnetUri={18}, MinimumRatio={19}, MinimumSeedTime={20}, DownloadVolumeFactor={21}, UploadVolumeFactor={22}, Gain={23}]", Title, Guid, Link, Comments, PublishDate, Category, Size, Files, Grabs, Description, RageID, TVDBId, Imdb, TMDb, Seeders, Peers, BannerUrl, InfoHash, MagnetUri, MinimumRatio, MinimumSeedTime, DownloadVolumeFactor, UploadVolumeFactor, Gain); + } } } diff --git a/src/Jackett/Utils/StringUtil.cs b/src/Jackett/Utils/StringUtil.cs index 5d96fa6e4..7ffda6c99 100644 --- a/src/Jackett/Utils/StringUtil.cs +++ b/src/Jackett/Utils/StringUtil.cs @@ -55,20 +55,46 @@ namespace Jackett.Utils return new FormUrlEncodedContent(dict).ReadAsStringAsync().Result; } - public static string Hash(string input) + /// + /// Convert an array of bytes to a string of hex digits + /// + /// array of bytes + /// String of hex digits + public static string HexStringFromBytes(byte[] bytes) + { + var sb = new StringBuilder(); + foreach (byte b in bytes) + { + var hex = b.ToString("x2"); + sb.Append(hex); + } + return sb.ToString(); + } + + /// + /// Compute hash for string encoded as UTF8 + /// + /// String to be hashed + /// 40-character hex string + public static string HashSHA1(string s) + { + var sha1 = SHA1.Create(); + + byte[] bytes = Encoding.UTF8.GetBytes(s); + byte[] hashBytes = sha1.ComputeHash(bytes); + + return HexStringFromBytes(hashBytes); + } + + public static string Hash(string s) { // Use input string to calculate MD5 hash MD5 md5 = System.Security.Cryptography.MD5.Create(); - byte[] inputBytes = System.Text.Encoding.ASCII.GetBytes(input); + + byte[] inputBytes = System.Text.Encoding.ASCII.GetBytes(s); byte[] hashBytes = md5.ComputeHash(inputBytes); - // Convert the byte array to hexadecimal string - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < hashBytes.Length; i++) - { - sb.Append(hashBytes[i].ToString("X2")); - } - return sb.ToString(); + return HexStringFromBytes(hashBytes); } public static string GetExceptionDetails(this Exception exception)