mirror
/
Jackett
mirror of https://github.com/Jackett/Jackett
1
0
Fork 0
Code Issues Releases Wiki Activity

Post refactor fixes

This commit is contained in:
KZ 2015-07-28 20:22:23 +01:00
parent 51042e91fc
commit 499b53e9ed
18 changed files with 166 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/Jackett.Console/Program.cs
View File

@ -173,7 +173,6 @@ namespace JackettConsole
Engine.Server.Initalize();
Engine.Server.Start();
Engine.Logger.Info("Running in console mode!");
Engine.RunTime.Spin();
Engine.Logger.Info("Server thread exit");
}

4
src/Jackett/Controllers/APIController.cs
View File

@ -95,10 +95,10 @@ namespace Jackett.Controllers
// add Jackett proxy to download links...
foreach (var release in releases)
{
if (release.Link == null || release.Link.Scheme == "magnet")
if (release.Link == null || (release.Link.IsAbsoluteUri && release.Link.Scheme == "magnet"))
continue;
var originalLink = release.Link;
var encodedLink = HttpServerUtility.UrlTokenEncode(Encoding.UTF8.GetBytes(originalLink.ToString())) + "/download.torrent";
var encodedLink = HttpServerUtility.UrlTokenEncode(Encoding.UTF8.GetBytes(originalLink.ToString())) + "/t.torrent";
var proxyLink = string.Format("{0}api/{1}/download/{2}", severUrl, indexer.ID, encodedLink);
release.Link = new Uri(proxyLink);
}

2
src/Jackett/Controllers/DownloadController.cs
View File

@ -39,7 +39,7 @@ namespace Jackett.Controllers
}
var remoteFile = Encoding.UTF8.GetString(HttpServerUtility.UrlTokenDecode(path));
var downloadBytes = await indexer.Download(new Uri(remoteFile));
var downloadBytes = await indexer.Download(new Uri(remoteFile, UriKind.RelativeOrAbsolute));
var result = new HttpResponseMessage(HttpStatusCode.OK);
result.Content = new ByteArrayContent(downloadBytes);

28
src/Jackett/Indexers/AnimeBytes.cs
View File

@ -30,7 +30,7 @@ namespace Jackett.Indexers
public AnimeBytes(IIndexerManagerService i, IWebClient client, Logger l)
: base(name: "AnimeBytes",
link: "https://animebytes.tv/",
description: "The web's best Chinese cartoons",
description: "Powered by Tentacles",
manager: i,
client: client,
caps: new TorznabCapabilities(TorznabCategory.Anime),
@ -48,6 +48,11 @@ namespace Jackett.Indexers
var config = new ConfigurationDataBasicLoginAnimeBytes();
config.LoadValuesFromJson(configJson);
lock (cache)
{
cache.Clear();
}
// Get the login form as we need the CSRF Token
var loginPage = await webclient.GetString(new Utils.Clients.WebRequest()
{
@ -79,7 +84,7 @@ namespace Jackett.Indexers
var response = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, null);
// Follow the redirect
await FollowIfRedirect(request, response, SearchUrl);
await FollowIfRedirect(response, request.Url, SearchUrl);
if (!(response.Content != null && response.Content.Contains("/user/logout")))
{
@ -252,9 +257,9 @@ namespace Jackett.Indexers
release.PublishDate = release.PublishDate.AddDays(Math.Min(DateTime.Now.DayOfYear, 365) - 1);
var infoLink = links.Get(1);
release.Comments = new Uri(SiteLink + "/" + infoLink.Attributes.GetAttribute("href"));
release.Guid = new Uri(SiteLink + "/" + infoLink.Attributes.GetAttribute("href") + "&nh=" + StringUtil.Hash(title)); // Sonarr should dedupe on this url - allow a url per name.
release.Link = new Uri(SiteLink + "/" + downloadLink.Attributes.GetAttribute("href"));
release.Comments = new Uri(SiteLink + infoLink.Attributes.GetAttribute("href"));
release.Guid = new Uri(SiteLink + infoLink.Attributes.GetAttribute("href") + "&nh=" + StringUtil.Hash(title)); // Sonarr should dedupe on this url - allow a url per name.
release.Link = new Uri(downloadLink.Attributes.GetAttribute("href"), UriKind.Relative);
// We dont actually have a release name >.> so try to create one
var releaseTags = infoLink.InnerText.Split("|".ToCharArray(), StringSplitOptions.RemoveEmptyEntries).ToList();
@ -324,5 +329,18 @@ namespace Jackett.Indexers
return releases.Select(s => (ReleaseInfo)s.Clone()).ToArray();
}
public async override Task<byte[]> Download(Uri link)
{
// The urls for this tracker are quite long so append the domain after the incoming request.
var response = await webclient.GetBytes(new Utils.Clients.WebRequest()
{
Url = SiteLink + link.ToString(),
Cookies = cookieHeader
});
return response.Content;
}
}
}

62
src/Jackett/Indexers/BaseIndexer.cs
View File

@ -79,41 +79,42 @@ namespace Jackett.Indexers
}
}
protected async Task FollowIfRedirect(WebRequest request, WebClientStringResult incomingResponse, string overrideRedirectUrl = null, string overrideCookies = null)
protected async Task FollowIfRedirect(WebClientStringResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null)
{
if (incomingResponse.Status == System.Net.HttpStatusCode.Redirect ||
incomingResponse.Status == System.Net.HttpStatusCode.RedirectKeepVerb ||
incomingResponse.Status == System.Net.HttpStatusCode.RedirectMethod ||
incomingResponse.Status == System.Net.HttpStatusCode.Found)
var byteResult = new WebClientByteResult();
// Map to byte
Mapper.Map(response, byteResult);
await FollowIfRedirect(byteResult, referrer, overrideRedirectUrl, overrideCookies);
// Map to string
Mapper.Map(byteResult, response);
}
protected async Task FollowIfRedirect(WebClientByteResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null)
{
// Follow up to 5 redirects
for (int i = 0; i < 5; i++)
{
if (!response.IsRedirect)
break;
await DoFollowIfRedirect(response, referrer, overrideRedirectUrl, overrideCookies);
}
}
private async Task DoFollowIfRedirect(WebClientByteResult incomingResponse, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null)
{
if (incomingResponse.IsRedirect)
{
// Do redirect
var redirectedResponse = await webclient.GetString(new WebRequest()
var redirectedResponse = await webclient.GetBytes(new WebRequest()
{
Url = overrideRedirectUrl??incomingResponse.RedirectingTo,
Referer = request.Url,
Cookies = overrideCookies??cookieHeader
Url = overrideRedirectUrl ?? incomingResponse.RedirectingTo,
Referer = referrer,
Cookies = overrideCookies ?? cookieHeader
});
Mapper.Map(redirectedResponse, incomingResponse);
}
}
protected async void FollowIfRedirect(WebRequest request, WebClientByteResult incomingResponse, string overrideRedirectUrl)
{
if (incomingResponse.Status == System.Net.HttpStatusCode.Redirect ||
incomingResponse.Status == System.Net.HttpStatusCode.RedirectKeepVerb ||
incomingResponse.Status == System.Net.HttpStatusCode.RedirectMethod ||
incomingResponse.Status == System.Net.HttpStatusCode.Found)
{
// Do redirect
var redirectedResponse = await webclient.GetBytes(new WebRequest()
{
Url = overrideRedirectUrl??incomingResponse.RedirectingTo,
Referer = request.Url,
Cookies = cookieHeader
});
Mapper.Map(redirectedResponse, incomingResponse);
}
}
protected void LoadCookieHeaderAndConfigure(JToken jsonConfig)
{
@ -210,7 +211,14 @@ namespace Jackett.Indexers
};
var response = await webclient.GetString(request);
var firstCallCookies = response.Cookies;
await FollowIfRedirect(request, response, SiteLink, response.Cookies);
// Follow up to 5 redirects
for(int i = 0; i < 5; i++)
{
if (!response.IsRedirect)
break;
await FollowIfRedirect(response, request.Url, null, response.Cookies);
}
if (returnCookiesFromFirstCall)
{

3
src/Jackett/Indexers/BeyondHD.cs
View File

@ -49,7 +49,7 @@ namespace Jackett.Indexers
Cookies = cookieHeader
});
ConfigureIfOK(response.Cookies, response.Content.Contains("logout.php"), () =>
ConfigureIfOK(cookieHeader, response.Content.Contains("logout.php"), () =>
{
CQ dom = response.Content;
throw new ExceptionWithConfigData("Invalid cookie header", (ConfigurationData)config);
@ -63,6 +63,7 @@ namespace Jackett.Indexers
var searchString = query.SanitizedSearchTerm + " " + query.GetEpisodeSearchString();
var episodeSearchUrl = string.Format(SearchUrl, HttpUtility.UrlEncode(searchString));
var results = await RequestStringWithCookies(episodeSearchUrl);
await FollowIfRedirect(results);
try
{
CQ dom = results.Content;

2
src/Jackett/Indexers/Freshon.cs
View File

@ -53,7 +53,7 @@ namespace Jackett.Indexers
// Get inital cookies
cookieHeader = string.Empty;
var loginPage = await RequestStringWithCookies(LoginUrl);
var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty);
var response = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, null, LoginUrl);
ConfigureIfOK(response.Cookies, response.Content != null && response.Content.Contains("/logout.php"), () =>

14
src/Jackett/Indexers/HDSpace.cs
View File

@ -42,7 +42,7 @@ namespace Jackett.Indexers
var config = new ConfigurationDataBasicLogin();
config.LoadValuesFromJson(configJson);
var loginPage = await RequestStringWithCookies(LoginUrl, null);
var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty);
var pairs = new Dictionary<string, string> {
{ "uid", config.Username.Value },
@ -50,23 +50,15 @@ namespace Jackett.Indexers
};
// Send Post
var loginPost = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, null, LoginUrl);
var response = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, null, LoginUrl);
if (loginPost.Status == System.Net.HttpStatusCode.OK)
ConfigureIfOK(response.Cookies, response.Content != null && response.Content.Contains("logout.php"), () =>
{
var errorStr = "You have {0} remaining login attempts";
var remainingAttemptSpan = new Regex(string.Format(errorStr, "(.*?)")).Match(loginPage.Content).Groups[1].ToString();
var attempts = Regex.Replace(remainingAttemptSpan, "<.*?>", String.Empty);
var errorMessage = string.Format(errorStr, attempts);
throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)config);
}
// Get result from redirect
var loginResult = await RequestStringWithCookies(SiteLink + loginPost.RedirectingTo, loginPost.Cookies);
ConfigureIfOK(loginPost.Cookies, loginResult.Content.Contains("logout.php"), () =>
{
throw new ExceptionWithConfigData("Login failed", (ConfigurationData)config);
});
}

131
src/Jackett/Indexers/HDTorrents.cs
View File

@ -19,7 +19,7 @@ namespace Jackett.Indexers
{
public class HDTorrents : BaseIndexer, IIndexer
{
private string SearchUrl { get { return SiteLink + "torrents.php?search={0}&active=1&options=0&category%5B%5D=59&category%5B%5D=60&category%5B%5D=30&category%5B%5D=38&page={1}"; } }
private string SearchUrl { get { return SiteLink + "torrents.php?search={0}&active=1&options=0&category%5B%5D=59&category%5B%5D=60&category%5B%5D=30&category%5B%5D=38&page=0"; } }
private string LoginUrl { get { return SiteLink + "login.php"; } }
private const int MAXPAGES = 3;
@ -43,7 +43,7 @@ namespace Jackett.Indexers
{
var incomingConfig = new ConfigurationDataBasicLogin();
incomingConfig.LoadValuesFromJson(configJson);
var loginPage = await RequestStringWithCookies(LoginUrl, null);
var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty);
var pairs = new Dictionary<string, string> {
{ "uid", incomingConfig.Username.Value },
@ -52,7 +52,7 @@ namespace Jackett.Indexers
var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, null, LoginUrl);
ConfigureIfOK(result.Content, result.Content != null && result.Content.Contains("If your browser doesn't have javascript enabled"), () =>
ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("If your browser doesn't have javascript enabled"), () =>
{
var errorMessage = "Couldn't login";
throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)incomingConfig);
@ -65,78 +65,71 @@ namespace Jackett.Indexers
var searchurls = new List<string>();
var searchString = query.SanitizedSearchTerm + " " + query.GetEpisodeSearchString();
for (int page = 0; page < MAXPAGES; page++)
var searchUrl = string.Format(SearchUrl, HttpUtility.UrlEncode(searchString.Trim()));
var results = await RequestStringWithCookies(searchUrl);
try
{
searchurls.Add(string.Format(SearchUrl, HttpUtility.UrlEncode(searchString.Trim()), page));
}
CQ dom = results.Content;
ReleaseInfo release;
foreach (string searchUrl in searchurls)
{
var results = await RequestStringWithCookies(searchUrl);
try
int rowCount = 0;
var rows = dom[".mainblockcontenttt > tbody > tr"];
foreach (var row in rows)
{
CQ dom = results.Content;
ReleaseInfo release;
int rowCount = 0;
var rows = dom[".mainblockcontenttt > tbody > tr"];
foreach (var row in rows)
CQ qRow = row.Cq();
if (rowCount < 2 || qRow.Children().Count() != 12) //skip 2 rows because there's an empty row & a title/sort row
{
CQ qRow = row.Cq();
if (rowCount < 2 || qRow.Children().Count() != 12) //skip 2 rows because there's an empty row & a title/sort row
{
rowCount++;
continue;
}
release = new ReleaseInfo();
release.Title = qRow.Find("td.mainblockcontent b a").Text();
release.Description = release.Title;
if (0 != qRow.Find("td.mainblockcontent u").Length)
{
var imdbStr = qRow.Find("td.mainblockcontent u").Parent().First().Attr("href").Replace("http://www.imdb.com/title/tt", "").Replace("/", "");
long imdb;
if (ParseUtil.TryCoerceLong(imdbStr, out imdb))
{
release.Imdb = imdb;
}
}
release.MinimumRatio = 1;
release.MinimumSeedTime = 172800;
int seeders, peers;
if (ParseUtil.TryCoerceInt(qRow.Find("td").Get(9).FirstChild.FirstChild.InnerText, out seeders))
{
release.Seeders = seeders;
if (ParseUtil.TryCoerceInt(qRow.Find("td").Get(10).FirstChild.FirstChild.InnerText, out peers))
{
release.Peers = peers + release.Seeders;
}
}
string fullSize = qRow.Find("td.mainblockcontent").Get(6).InnerText;
release.Size = ReleaseInfo.GetBytes(fullSize);
release.Guid = new Uri(SiteLink + "/" + qRow.Find("td.mainblockcontent b a").Attr("href"));
release.Link = new Uri(SiteLink + "/" + qRow.Find("td.mainblockcontent").Get(3).FirstChild.GetAttribute("href"));
release.Comments = new Uri(SiteLink + "/" + qRow.Find("td.mainblockcontent b a").Attr("href") + "#comments");
string[] dateSplit = qRow.Find("td.mainblockcontent").Get(5).InnerHTML.Split(',');
string dateString = dateSplit[1].Substring(0, dateSplit[1].IndexOf('>'));
release.PublishDate = DateTime.Parse(dateString, CultureInfo.InvariantCulture);
releases.Add(release);
rowCount++;
continue;
}
release = new ReleaseInfo();
release.Title = qRow.Find("td.mainblockcontent b a").Text();
release.Description = release.Title;
if (0 != qRow.Find("td.mainblockcontent u").Length)
{
var imdbStr = qRow.Find("td.mainblockcontent u").Parent().First().Attr("href").Replace("http://www.imdb.com/title/tt", "").Replace("/", "");
long imdb;
if (ParseUtil.TryCoerceLong(imdbStr, out imdb))
{
release.Imdb = imdb;
}
}
release.MinimumRatio = 1;
release.MinimumSeedTime = 172800;
int seeders, peers;
if (ParseUtil.TryCoerceInt(qRow.Find("td").Get(9).FirstChild.FirstChild.InnerText, out seeders))
{
release.Seeders = seeders;
if (ParseUtil.TryCoerceInt(qRow.Find("td").Get(10).FirstChild.FirstChild.InnerText, out peers))
{
release.Peers = peers + release.Seeders;
}
}
string fullSize = qRow.Find("td.mainblockcontent").Get(6).InnerText;
release.Size = ReleaseInfo.GetBytes(fullSize);
release.Guid = new Uri(SiteLink + "/" + qRow.Find("td.mainblockcontent b a").Attr("href"));
release.Link = new Uri(SiteLink + "/" + qRow.Find("td.mainblockcontent").Get(3).FirstChild.GetAttribute("href"));
release.Comments = new Uri(SiteLink + "/" + qRow.Find("td.mainblockcontent b a").Attr("href") + "#comments");
string[] dateSplit = qRow.Find("td.mainblockcontent").Get(5).InnerHTML.Split(',');
string dateString = dateSplit[1].Substring(0, dateSplit[1].IndexOf('>'));
release.PublishDate = DateTime.Parse(dateString, CultureInfo.InvariantCulture);
releases.Add(release);
}
catch (Exception ex)
{
OnParseError(results.Content, ex);
}
}
catch (Exception ex)
{
OnParseError(results.Content, ex);
}
return releases.ToArray();

6
src/Jackett/Indexers/IPTorrents.cs
View File

@ -55,10 +55,8 @@ namespace Jackett.Indexers
};
var response = await webclient.GetString(request);
var firstCallCookies = response.Cookies;
// Redirect to ?
await FollowIfRedirect(request, response,null, firstCallCookies);
// Redirect to /t
await FollowIfRedirect(request, response, null, firstCallCookies);
// Redirect to ? then to /t
await FollowIfRedirect(response, request.Url, null, firstCallCookies);
ConfigureIfOK(firstCallCookies, response.Content.Contains("/my.php"), () =>
{

17
src/Jackett/Indexers/Pretome.cs
View File

@ -42,11 +42,7 @@ namespace Jackett.Indexers
var config = new PretomeConfiguration();
config.LoadValuesFromJson(configJson);
var loginPage = await webclient.GetString(new WebRequest()
{
Url = LoginUrl,
Type = RequestType.GET
});
var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty);
var pairs = new Dictionary<string, string> {
{ "returnto", "%2F" },
@ -57,17 +53,18 @@ namespace Jackett.Indexers
};
// Send Post
var loginPost = await PostDataWithCookies(LoginUrl, pairs, loginPage.Cookies);
if (loginPost.RedirectingTo == null)
var result = await PostDataWithCookies(LoginUrl, pairs, loginPage.Cookies);
if (result.RedirectingTo == null)
{
throw new ExceptionWithConfigData("Login failed. Did you use the PIN number that pretome emailed you?", (ConfigurationData)config);
}
var loginCookies = result.Cookies;
// Get result from redirect
var loginResult = await RequestStringWithCookies(loginPost.RedirectingTo);
await FollowIfRedirect(result,LoginUrl,null, loginCookies);
ConfigureIfOK(loginPost.Cookies, loginResult.Content != null && loginResult.Content.Contains("logout.php"), () =>
ConfigureIfOK(loginCookies, result.Content != null && result.Content.Contains("logout.php"), () =>
{
cookieHeader = string.Empty;
throw new ExceptionWithConfigData("Failed", (ConfigurationData)config);
});
}

4
src/Jackett/Indexers/PrivateHD.cs
View File

@ -37,7 +37,7 @@ namespace Jackett.Indexers
{
var incomingConfig = new ConfigurationDataBasicLogin();
incomingConfig.LoadValuesFromJson(configJson);
var loginPage = await RequestStringWithCookies(LoginUrl);
var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty);
var token = new Regex("Avz.CSRF_TOKEN = '(.*?)';").Match(loginPage.Content).Groups[1].ToString();
var pairs = new Dictionary<string, string> {
{ "_token", token },
@ -46,7 +46,7 @@ namespace Jackett.Indexers
{ "remember", "on" }
};
var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, null, true, null, LoginUrl);
var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, null, LoginUrl);
ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("auth/logout"), () =>
{
CQ dom = result.Content;

2
src/Jackett/Indexers/Torrentz.cs
View File

@ -176,7 +176,7 @@ namespace Jackett.Indexers
switch (counter)
{
case 0:
this.Size = ReleaseInfo.BytesFromMB(ParseUtil.CoerceLong(val.Substring(0, val.IndexOf(" ") - 1)));
this.Size = ReleaseInfo.GetBytes(val);
break;
case 1:
this.Seeders = ParseUtil.CoerceInt(val.Contains(",") ? val.Remove(val.IndexOf(","), 1) : val);

10
src/Jackett/JackettModule.cs
View File

@ -43,11 +43,19 @@ namespace Jackett
builder.RegisterType(indexer).Named<IIndexer>(BaseIndexer.GetIndexerID(indexer));
}
Mapper.CreateMap<WebClientByteResult, WebClientStringResult>().AfterMap((be, str) =>
Mapper.CreateMap<WebClientByteResult, WebClientStringResult>().ForMember(x => x.Content, opt => opt.Ignore()).AfterMap((be, str) =>
{
str.Content = Encoding.UTF8.GetString(be.Content);
});
Mapper.CreateMap<WebClientStringResult, WebClientByteResult>().ForMember(x => x.Content, opt => opt.Ignore()).AfterMap((str, be) =>
{
if (!string.IsNullOrEmpty(str.Content))
{
be.Content = Encoding.UTF8.GetBytes(str.Content);
}
});
Mapper.CreateMap<WebClientStringResult, WebClientStringResult>();
Mapper.CreateMap<WebClientByteResult, WebClientByteResult>();
}

3
src/Jackett/Services/IndexerManagerService.cs
View File

@ -2,6 +2,7 @@
using Jackett.Indexers;
using Jackett.Models;
using Jackett.Utils;
using Jackett.Utils.Clients;
using Newtonsoft.Json.Linq;
using NLog;
using System;
@ -39,6 +40,8 @@ namespace Jackett.Services
public void InitIndexers()
{
logger.Info("Using HTTP Client: " + container.Resolve<IWebClient>().GetType().Name);
foreach (var idx in container.Resolve<IEnumerable<IIndexer>>().OrderBy(_ => _.DisplayName))
{
indexers.Add(idx.ID, idx);

1
src/Jackett/Services/ServerService.cs
View File

@ -111,6 +111,7 @@ namespace Jackett.Services
// Load indexers
indexerService.InitIndexers();
}
public void Start()

2
src/Jackett/Startup.cs
View File

@ -81,7 +81,7 @@ namespace Jackett
config.Routes.MapHttpRoute(
name: "download",
routeTemplate: "api/{indexerID}/download/{path}/download.torrent",
routeTemplate: "api/{indexerID}/download/{path}/t.torrent",
defaults: new { controller = "Download", action = "Download" }
);

11
src/Jackett/Utils/Clients/BaseWebResult.cs
View File

@ -12,5 +12,16 @@ namespace Jackett.Utils.Clients
public HttpStatusCode Status { get; set; }
public string Cookies { get; set; }
public string RedirectingTo { get; set; }
public bool IsRedirect
{
get
{
return Status == System.Net.HttpStatusCode.Redirect ||
Status == System.Net.HttpStatusCode.RedirectKeepVerb ||
Status == System.Net.HttpStatusCode.RedirectMethod ||
Status == System.Net.HttpStatusCode.Found;
}
}
}
}