mirror/Jackett
1
0
Fork 0
mirror of https://github.com/Jackett/Jackett synced 2025-02-24 15:21:06 +00:00
Code Issues Releases Wiki Activity

Update CloudFlareUtilities to version 0.3.2-alpha and remove the UnixLibCurlWebClient reflections and add DigitalHive tracker (#553)

Browse source 
* Update CloudFlareUtilities to 0.3.2-alpha

* Remove CloudFlareUtilities reflections

With CloudFlareUtilities version 0.3.2-alpha reflections are no longer needed

* Add DigitalHive tracker
This commit is contained in:
kaso17 2016-10-16 16:09:49 +02:00 committed by JigSaw
parent 5384f85b5a
commit 5991fd62c1
6 changed files with 218 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
README.md
View file

@ -32,6 +32,7 @@ Developer note: The software implements the [Torznab](https://github.com/Sonarr/
* CinemaZ
* DanishBits
* Demonoid
* DigitalHive
* FileList
* Freshon
* FunFile

BIN
src/Jackett/Content/logos/digitalhive.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 9 KiB

204
src/Jackett/Indexers/DigitalHive.cs Normal file
View file

@ -0,0 +1,204 @@
using CsQuery;
using Jackett.Models;
using Jackett.Services;
using Jackett.Utils;
using Jackett.Utils.Clients;
using Newtonsoft.Json.Linq;
using NLog;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Jackett.Models.IndexerConfig;
using System.Collections.Specialized;
using System.Text.RegularExpressions;
namespace Jackett.Indexers
{
public class DigitalHive : BaseIndexer, IIndexer
{
private string SearchUrl { get { return SiteLink + "browse.php"; } }
private string LoginUrl { get { return SiteLink + "login.php?returnto=%2F"; } }
private string AjaxLoginUrl { get { return SiteLink + "takelogin.php"; } }
new ConfigurationDataRecaptchaLogin configData
{
get { return (ConfigurationDataRecaptchaLogin)base.configData; }
set { base.configData = value; }
}
public DigitalHive(IIndexerManagerService i, Logger l, IWebClient w, IProtectionService ps)
: base(name: "DigitalHive",
description: "DigitalHive is one of the oldest general trackers",
link: "https://www.digitalhive.org/",
caps: new TorznabCapabilities(),
manager: i,
client: w,
logger: l,
p: ps,
configData: new ConfigurationDataRecaptchaLogin())
{
AddCategoryMapping(0, TorznabCatType.Other);
AddCategoryMapping(48, TorznabCatType.Other); // 0Day
AddCategoryMapping(56, TorznabCatType.XXXImageset); // 0Day-Imagesets
AddCategoryMapping(6, TorznabCatType.Audio); // 0Day-Music
AddCategoryMapping(51, TorznabCatType.XXX); // 0Day-XXX
AddCategoryMapping(2, TorznabCatType.TVAnime); // Anime
AddCategoryMapping(59, TorznabCatType.MoviesBluRay); // BluRay
AddCategoryMapping(40, TorznabCatType.TVDocumentary); // Documentary
AddCategoryMapping(20, TorznabCatType.MoviesDVD); // DVD-R
AddCategoryMapping(25, TorznabCatType.BooksEbook); // Ebooks
AddCategoryMapping(38, TorznabCatType.PCPhoneIOS); // HandHeld
AddCategoryMapping(38, TorznabCatType.PCPhoneAndroid); // HandHeld
AddCategoryMapping(38, TorznabCatType.PCPhoneOther); // HandHeld
AddCategoryMapping(37, TorznabCatType.Other); // Kids Stuff
AddCategoryMapping(23, TorznabCatType.PC); // Linux
AddCategoryMapping(24, TorznabCatType.PCMac); // Mac
AddCategoryMapping(22, TorznabCatType.OtherMisc); // Misc
AddCategoryMapping(35, TorznabCatType.MoviesOther); // Movie Pack
AddCategoryMapping(36, TorznabCatType.MoviesHD); // Movie-HD
AddCategoryMapping(19, TorznabCatType.MoviesSD); // Movie-SD
AddCategoryMapping(50, TorznabCatType.Audio); // Music
AddCategoryMapping(53, TorznabCatType.AudioLossless); // Music-FLAC
AddCategoryMapping(49, TorznabCatType.AudioVideo); // MVID
AddCategoryMapping(1, TorznabCatType.PC); // PC Apps
AddCategoryMapping(4, TorznabCatType.PCGames); // PC Games
AddCategoryMapping(17, TorznabCatType.ConsolePS3); // Playstation
AddCategoryMapping(17, TorznabCatType.ConsolePS4); // Playstation
AddCategoryMapping(17, TorznabCatType.ConsolePSVita); // Playstation
AddCategoryMapping(17, TorznabCatType.ConsolePSP); // Playstation
AddCategoryMapping(28, TorznabCatType.ConsolePSP); // PSP
AddCategoryMapping(34, TorznabCatType.TVOTHER); // TV Pack
AddCategoryMapping(32, TorznabCatType.TVHD); // TV-HD
AddCategoryMapping(55, TorznabCatType.TVOTHER); // TV-HDRip
AddCategoryMapping(7, TorznabCatType.TVSD); // TV-SD
AddCategoryMapping(57, TorznabCatType.TVOTHER); // TV-SDRip
AddCategoryMapping(33, TorznabCatType.ConsoleWii); // WII
AddCategoryMapping(33, TorznabCatType.ConsoleWiiU); // WII
AddCategoryMapping(45, TorznabCatType.ConsoleXbox); // XBox
AddCategoryMapping(45, TorznabCatType.ConsoleXbox360); // XBox
AddCategoryMapping(45, TorznabCatType.ConsoleXBOX360DLC); // XBox
AddCategoryMapping(45, TorznabCatType.ConsoleXboxOne); // XBox
AddCategoryMapping(9, TorznabCatType.XXX); // XXX
AddCategoryMapping(52, TorznabCatType.XXXOther); // XXX-ISO
}
public override async Task<ConfigurationData> GetConfigurationForSetup()
{
var loginPage = await RequestStringWithCookies(LoginUrl, configData.CookieHeader.Value);
CQ cq = loginPage.Content;
string recaptchaSiteKey = cq.Find(".g-recaptcha").Attr("data-sitekey");
var result = new ConfigurationDataRecaptchaLogin();
result.CookieHeader.Value = loginPage.Cookies;
result.Captcha.SiteKey = recaptchaSiteKey;
return result;
}
public async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
{
configData.LoadValuesFromJson(configJson);
var pairs = new Dictionary<string, string> {
{ "returnto" , "/" },
{ "username", configData.Username.Value },
{ "password", configData.Password.Value },
{ "g-recaptcha-response", configData.Captcha.Value }
};
if (!string.IsNullOrWhiteSpace(configData.Captcha.Cookie))
{
// Cookie was manually supplied
CookieHeader = configData.Captcha.Cookie;
try
{
var results = await PerformQuery(new TorznabQuery());
if (!results.Any())
{
throw new Exception("Your cookie did not work");
}
SaveConfig();
IsConfigured = true;
return IndexerConfigurationStatus.Completed;
}
catch (Exception e)
{
IsConfigured = false;
throw new Exception("Your cookie did not work: " + e.Message);
}
}
var result = await RequestLoginAndFollowRedirect(AjaxLoginUrl, pairs, configData.CookieHeader.Value, true, SiteLink, LoginUrl);
await ConfigureIfOK(result.Cookies, result.Content.Contains("logout.php"), () =>
{
var errorMessage = result.Content;
throw new ExceptionWithConfigData(errorMessage, configData);
});
return IndexerConfigurationStatus.RequiresTesting;
}
public async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
{
List<ReleaseInfo> releases = new List<ReleaseInfo>();
var queryCollection = new NameValueCollection();
var searchString = query.GetQueryString();
var searchUrl = SearchUrl;
foreach (var cat in MapTorznabCapsToTrackers(query))
{
queryCollection.Add("c" + cat, "1");
}
if (!string.IsNullOrWhiteSpace(searchString))
{
queryCollection.Add("search", searchString);
}
queryCollection.Add("blah", "0");
var results = await RequestStringWithCookiesAndRetry(searchUrl + "?" + queryCollection.GetQueryString());
try
{
releases.AddRange(contentToReleaseInfos(results.Content));
}
catch (Exception ex)
{
OnParseError(results.Content, ex);
}
return releases;
}
private IEnumerable<ReleaseInfo> contentToReleaseInfos(CQ dom)
{
List<ReleaseInfo> releases = new List<ReleaseInfo>();
// Doesn't handle pagination yet...
var rows = dom["div.panel-body > table.table > tbody > tr"];
foreach (var row in rows)
{
var release = new ReleaseInfo();
release.MinimumRatio = 1;
release.MinimumSeedTime = 259200;
var qRow = row.Cq();
release.Title = qRow.Find("td:nth-child(2) > a").First().Text().Trim();
release.Description = release.Title;
release.Guid = new Uri(SiteLink + qRow.Find("td:nth-child(2) > a").First().Attr("href"));
release.Comments = release.Guid;
release.Link = new Uri(SiteLink + qRow.Find("td:nth-child(3) > a").First().Attr("href"));
var pubDate = qRow.Find("td:nth-child(2) > span").First().Text().Trim().Replace("Added: ", "");
release.PublishDate = DateTime.Parse(pubDate).ToLocalTime();
release.Category = MapTrackerCatToNewznab(qRow.Find("td:nth-child(1) > a").First().Attr("href").Split('=')[1]);
release.Size = ReleaseInfo.GetBytes(qRow.Find("td:nth-child(7)").First().Text());
release.Seeders = ParseUtil.CoerceInt(qRow.Find("td:nth-child(9)").First().Text());
release.Peers = ParseUtil.CoerceInt(qRow.Find("td:nth-child(10)").First().Text()) + release.Seeders;
releases.Add(release);
}
return releases;
}
}
}

4
src/Jackett/Jackett.csproj
View file

@ -50,8 +50,8 @@
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="CloudFlareUtilities, Version=0.3.1.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\CloudFlareUtilities.0.3.1-alpha\lib\portable45-net45+win8+wpa81\CloudFlareUtilities.dll</HintPath>
<Reference Include="CloudFlareUtilities, Version=0.3.2.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\CloudFlareUtilities.0.3.2-alpha\lib\portable45-net45+win8+wpa81\CloudFlareUtilities.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System" />

48
src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs
View file

@ -11,39 +11,17 @@ using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
using System.Reflection;
using CloudFlareUtilities;
namespace Jackett.Utils.Clients
{
public class UnixLibCurlWebClient : IWebClient
{
private Logger logger;
private static Boolean CloudFlareUtilitiesInit = false;
private static Assembly CloudFlareUtilities = null;
private static MethodInfo ChallengeSolverSolveMethod = null;
private static MethodInfo ChallengeSolutionGetClearanceQueryMethod = null;
public UnixLibCurlWebClient(Logger l)
{
logger = l;
// try loading CloudFlareUtilities
if (!CloudFlareUtilitiesInit)
{
try
{
// use reflections to get the internal CloudFlareUtilities methods we need
CloudFlareUtilities = Assembly.LoadFile(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "/CloudFlareUtilities.dll");
ChallengeSolverSolveMethod = CloudFlareUtilities.GetType("CloudFlareUtilities.ChallengeSolver").GetMethod("Solve");
ChallengeSolutionGetClearanceQueryMethod = CloudFlareUtilities.GetType("CloudFlareUtilities.ChallengeSolution").GetMethod("get_ClearanceQuery");
}
catch (Exception e)
{
Engine.Logger.Error(e.ToString());
Engine.Logger.Error(string.Format("UnixLibCurlWebClient: Error while loading CloudFlareUtilities.dll from {0}", Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)));
}
}
CloudFlareUtilitiesInit = true;
}
public async Task<WebClientByteResult> GetBytes(WebRequest request)
@ -64,21 +42,20 @@ namespace Jackett.Utils.Clients
private string CloudFlareChallengeSolverSolve(string challengePageContent, Uri uri)
{
var solution = ChallengeSolverSolveMethod.Invoke(null, new object[] { challengePageContent, uri.Host });
string clearanceQuery = (string)ChallengeSolutionGetClearanceQueryMethod.Invoke(solution, new object[] { });
string clearanceUri = uri.Scheme + Uri.SchemeDelimiter + uri.Host + ":" + uri.Port + clearanceQuery;
var solution = ChallengeSolver.Solve(challengePageContent, uri.Host);
string clearanceUri = uri.Scheme + Uri.SchemeDelimiter + uri.Host + ":" + uri.Port + solution.ClearanceQuery;
return clearanceUri;
}
}
public void Init()
{
try
{
Engine.Logger.Info("LibCurl init " + Curl.GlobalInit(CurlInitFlag.All).ToString());
CurlHelper.OnErrorMessage += (msg) =>
{
Engine.Logger.Error(msg);
};
CurlHelper.OnErrorMessage += (msg) =>
{
Engine.Logger.Error(msg);
};
}
catch (Exception e)
{
@ -107,11 +84,6 @@ namespace Jackett.Utils.Clients
if (result.Status == HttpStatusCode.ServiceUnavailable && ((request.Cookies != null && request.Cookies.Contains("__cfduid")) || result.Cookies.Contains("__cfduid")))
{
logger.Info("UnixLibCurlWebClient: Received a new CloudFlare challenge");
if(ChallengeSolutionGetClearanceQueryMethod == null)
{
logger.Error("UnixLibCurlWebClient: CloudFlareUtilities not available, can't solve challenge");
return result;
}
// solve the challenge
string pageContent = Encoding.UTF8.GetString(result.Content);
@ -156,7 +128,7 @@ namespace Jackett.Utils.Clients
logger.Debug("UnixLibCurlWebClient: Posting " + StringUtil.PostDataFromDict(request.PostData));
}
response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer, request.RawBody);
response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer, request.RawBody);
}
var result = new WebClientByteResult()

2
src/Jackett/packages.config
View file

@ -7,7 +7,7 @@
<package id="Autofac.WebApi2" version="3.4.0" targetFramework="net45" />
<package id="Autofac.WebApi2.Owin" version="3.3.0" targetFramework="net45" />
<package id="AutoMapper" version="4.1.1" targetFramework="net45" />
<package id="CloudFlareUtilities" version="0.3.1-alpha" targetFramework="net45" />
<package id="CloudFlareUtilities" version="0.3.2-alpha" targetFramework="net45" />
<package id="CsQuery" version="1.3.4" targetFramework="net45" />
<package id="Microsoft.AspNet.Identity.Core" version="2.2.1" targetFramework="net45" />
<package id="Microsoft.AspNet.WebApi.Client" version="5.2.3" targetFramework="net45" />