mirror
/
Jackett
mirror of https://github.com/Jackett/Jackett
1
0
Fork 0
Code Issues Releases Wiki Activity

fix(indexers): fixed and optimized xthor indexer 

* feat(utils): added sha1 hash function and refactored md5 hash function

* fix(indexers): now use cross plateform path building for dev mode

* fix(indexers): fix output log of xthor for dev mode

* feat(release): added ToString method

* refactor(dev): optimized dev mode

* style(clean): cleanup code

* feat(indexer): added tmdb info to releases

* fix(cat): enabled categories on xthor
This commit is contained in:
JigSaw 2017-08-13 20:49:03 +02:00 committed by GitHub
parent 7a8d83b693
commit 8707e6b2e9
3 changed files with 112 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
src/Jackett/Indexers/Xthor.cs
View File

@ -1,9 +1,10 @@
using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.IO;
using System.Linq;
using System.Text;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using System.Web;
using Jackett.Models;
@ -27,7 +28,7 @@ namespace Jackett.Indexers
private string TorrentDescriptionUrl => SiteLink + "details.php?id={id}";
private bool DevMode => ConfigData.DevMode.Value;
private bool CacheMode => ConfigData.HardDriveCache.Value;
private static string Directory => System.IO.Path.GetTempPath() + "Jackett\\" + MethodBase.GetCurrentMethod().DeclaringType?.Name + "\\";
private static string Directory => Path.Combine(Path.GetTempPath(), Assembly.GetExecutingAssembly().GetName().Name.ToLower(), MethodBase.GetCurrentMethod().DeclaringType?.Name.ToLower());
public Dictionary<string, string> EmulatedBrowserHeaders { get; } = new Dictionary<string, string>();
private ConfigurationDataXthor ConfigData => (ConfigurationDataXthor)configData;
@ -130,7 +131,7 @@ namespace Jackett.Indexers
// Setting our data for a better emulated browser (maximum security)
// TODO: Encoded Content not supported by Jackett at this time
// emulatedBrowserHeaders.Add("Accept-Encoding", "gzip, deflate");
// EmulatedBrowserHeaders.Add("Accept-Encoding", "gzip, deflate");
// Clean headers
EmulatedBrowserHeaders.Clear();
@ -183,7 +184,7 @@ namespace Jackett.Indexers
try
{
// Deserialize our Json Response
var xthorResponse = JsonConvert.DeserializeObject<XthorResponse>(results.Content);
var xthorResponse = JsonConvert.DeserializeObject<XthorResponse>(results);
// Check Tracker's State
CheckApiState(xthorResponse.error);
@ -192,24 +193,35 @@ namespace Jackett.Indexers
if (xthorResponse.torrents != null)
{
// Adding each torrent row to releases
releases.AddRange(xthorResponse.torrents.Select(torrent => new ReleaseInfo
releases.AddRange(xthorResponse.torrents.Select(torrent =>
{
// Mapping data
Category = MapTrackerCatToNewznab(torrent.category.ToString()),
Title = torrent.name,
Seeders = torrent.seeders,
Peers = torrent.seeders + torrent.leechers,
MinimumRatio = 1,
MinimumSeedTime = 345600,
PublishDate = DateTimeUtil.UnixTimestampToDateTime(torrent.added),
Size = torrent.size,
Grabs = torrent.times_completed,
Files = torrent.numfiles,
UploadVolumeFactor = 1,
DownloadVolumeFactor = (torrent.freeleech == 1 ? 0 : 1),
Guid = new Uri(TorrentDescriptionUrl.Replace("{id}", torrent.id.ToString())),
Comments = new Uri(TorrentCommentUrl.Replace("{id}", torrent.id.ToString())),
Link = new Uri(torrent.download_link)
var release = new ReleaseInfo
{
// Mapping data
Category = MapTrackerCatToNewznab(torrent.category.ToString()),
Title = torrent.name,
Seeders = torrent.seeders,
Peers = torrent.seeders + torrent.leechers,
MinimumRatio = 1,
MinimumSeedTime = 345600,
PublishDate = DateTimeUtil.UnixTimestampToDateTime(torrent.added),
Size = torrent.size,
Grabs = torrent.times_completed,
Files = torrent.numfiles,
UploadVolumeFactor = 1,
DownloadVolumeFactor = (torrent.freeleech == 1 ? 0 : 1),
Guid = new Uri(TorrentDescriptionUrl.Replace("{id}", torrent.id.ToString())),
Comments = new Uri(TorrentCommentUrl.Replace("{id}", torrent.id.ToString())),
Link = new Uri(torrent.download_link),
TMDb = torrent.tmdb_id
};
if (DevMode)
{
Output(release.ToString());
}
return release;
}));
}
}
@ -273,6 +285,12 @@ namespace Jackett.Indexers
public int numfiles { get; set; }
public string release_group { get; set; }
public string download_link { get; set; }
public int tmdb_id { get; set; }
public override string ToString()
{
return string.Format("[XthorTorrent: id={0}, category={1}, seeders={2}, leechers={3}, name={4}, times_completed={5}, size={6}, added={7}, freeleech={8}, numfiles={9}, release_group={10}, download_link={11}, tmdb_id={12}]", id, category, seeders, leechers, name, times_completed, size, added, freeleech, numfiles, release_group, download_link, tmdb_id);
}
}
/// <summary>
@ -308,7 +326,7 @@ namespace Jackett.Indexers
if (categoriesList.Count > 0)
{
// ignore categories for now, something changed or is buggy, needs investigation
//parameters.Add("category", string.Join("+", categoriesList));
parameters.Add("category", string.Join("+", categoriesList));
}
// If Only Freeleech Enabled
@ -331,9 +349,9 @@ namespace Jackett.Indexers
/// </summary>
/// <param name="request">URL created by Query Builder</param>
/// <returns>Results from query</returns>
private async Task<WebClientStringResult> QueryExec(string request)
private async Task<String> QueryExec(string request)
{
WebClientStringResult results;
String results;
// Switch in we are in DEV mode with Hard Drive Cache or not
if (DevMode && CacheMode)
@ -354,9 +372,9 @@ namespace Jackett.Indexers
/// </summary>
/// <param name="request">URL created by Query Builder</param>
/// <returns>Results from query</returns>
private async Task<WebClientStringResult> QueryCache(string request)
private async Task<String> QueryCache(string request)
{
WebClientStringResult results;
String results;
// Create Directory if not exist
System.IO.Directory.CreateDirectory(Directory);
@ -364,15 +382,30 @@ namespace Jackett.Indexers
// Clean Storage Provider Directory from outdated cached queries
CleanCacheStorage();
// File Name
string fileName = StringUtil.HashSHA1(request) + ".json";
// Create fingerprint for request
string file = Directory + request.GetHashCode() + ".json";
string file = Path.Combine(Directory, fileName);
// Checking modes states
if (System.IO.File.Exists(file))
if (File.Exists(file))
{
// File exist... loading it right now !
Output("Loading results from hard drive cache ..." + request.GetHashCode() + ".json");
results = JsonConvert.DeserializeObject<WebClientStringResult>(System.IO.File.ReadAllText(file));
Output("Loading results from hard drive cache ..." + fileName);
try
{
using (StreamReader fileReader = File.OpenText(file))
{
JsonSerializer serializer = new JsonSerializer();
results = (String)serializer.Deserialize(fileReader, typeof(String));
}
}
catch (Exception e)
{
Output("Error loading cached results ! " + e.Message, "error");
results = null;
}
}
else
{
@ -380,8 +413,12 @@ namespace Jackett.Indexers
results = await QueryTracker(request);
// Cached file didn't exist for our query, writing it right now !
Output("Writing results to hard drive cache ..." + request.GetHashCode() + ".json");
System.IO.File.WriteAllText(file, JsonConvert.SerializeObject(results));
Output("Writing results to hard drive cache ..." + fileName);
using (StreamWriter fileWriter = File.CreateText(file))
{
JsonSerializer serializer = new JsonSerializer();
serializer.Serialize(fileWriter, results);
}
}
return results;
}
@ -391,7 +428,7 @@ namespace Jackett.Indexers
/// </summary>
/// <param name="request">URL created by Query Builder</param>
/// <returns>Results from query</returns>
private async Task<WebClientStringResult> QueryTracker(string request)
private async Task<String> QueryTracker(string request)
{
// Cache mode not enabled or cached file didn't exist for our query
Output("\nQuerying tracker for results....");
@ -409,7 +446,7 @@ namespace Jackett.Indexers
var results = await webclient.GetString(myIndexRequest);
// Return results from tracker
return results;
return results.Content;
}
/// <summary>
@ -479,7 +516,7 @@ namespace Jackett.Indexers
// Check if there is file older than ... and delete them
Output("\nCleaning Provider Storage folder... in progress.");
System.IO.Directory.GetFiles(Directory)
.Select(f => new System.IO.FileInfo(f))
.Select(f => new FileInfo(f))
.Where(f => f.LastAccessTime < DateTime.Now.AddMilliseconds(-Convert.ToInt32(ConfigData.HardDriveCacheKeepTime.Value)))
.ToList()
.ForEach(f =>

5
src/Jackett/Models/ReleaseInfo.cs
View File

@ -114,5 +114,10 @@ namespace Jackett.Models
{
return (long)(kb * 1024f);
}
public override string ToString()
{
return string.Format("[ReleaseInfo: Title={0}, Guid={1}, Link={2}, Comments={3}, PublishDate={4}, Category={5}, Size={6}, Files={7}, Grabs={8}, Description={9}, RageID={10}, TVDBId={11}, Imdb={12}, TMDb={13}, Seeders={14}, Peers={15}, BannerUrl={16}, InfoHash={17}, MagnetUri={18}, MinimumRatio={19}, MinimumSeedTime={20}, DownloadVolumeFactor={21}, UploadVolumeFactor={22}, Gain={23}]", Title, Guid, Link, Comments, PublishDate, Category, Size, Files, Grabs, Description, RageID, TVDBId, Imdb, TMDb, Seeders, Peers, BannerUrl, InfoHash, MagnetUri, MinimumRatio, MinimumSeedTime, DownloadVolumeFactor, UploadVolumeFactor, Gain);
}
}
}

44
src/Jackett/Utils/StringUtil.cs
View File

@ -55,20 +55,46 @@ namespace Jackett.Utils
return new FormUrlEncodedContent(dict).ReadAsStringAsync().Result;
}
public static string Hash(string input)
/// <summary>
/// Convert an array of bytes to a string of hex digits
/// </summary>
/// <param name="bytes">array of bytes</param>
/// <returns>String of hex digits</returns>
public static string HexStringFromBytes(byte[] bytes)
{
var sb = new StringBuilder();
foreach (byte b in bytes)
{
var hex = b.ToString("x2");
sb.Append(hex);
}
return sb.ToString();
}
/// <summary>
/// Compute hash for string encoded as UTF8
/// </summary>
/// <param name="s">String to be hashed</param>
/// <returns>40-character hex string</returns>
public static string HashSHA1(string s)
{
var sha1 = SHA1.Create();
byte[] bytes = Encoding.UTF8.GetBytes(s);
byte[] hashBytes = sha1.ComputeHash(bytes);
return HexStringFromBytes(hashBytes);
}
public static string Hash(string s)
{
// Use input string to calculate MD5 hash
MD5 md5 = System.Security.Cryptography.MD5.Create();
byte[] inputBytes = System.Text.Encoding.ASCII.GetBytes(input);
byte[] inputBytes = System.Text.Encoding.ASCII.GetBytes(s);
byte[] hashBytes = md5.ComputeHash(inputBytes);
// Convert the byte array to hexadecimal string
StringBuilder sb = new StringBuilder();
for (int i = 0; i < hashBytes.Length; i++)
{
sb.Append(hashBytes[i].ToString("X2"));
}
return sb.ToString();
return HexStringFromBytes(hashBytes);
}
public static string GetExceptionDetails(this Exception exception)