mirror
/
Jackett
mirror of https://github.com/Jackett/Jackett
1
0
Fork 0
Code Issues Releases Wiki Activity

PirateBay querying & parsing working

This commit is contained in:
zone117x 2015-04-16 00:04:28 -06:00
parent de111c4202
commit 0608d4ee17
12 changed files with 220 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/Jackett/IndexerInterface.cs
View File

@ -31,5 +31,7 @@ namespace Jackett
// Called on startup when initializing indexers from saved configuration
void LoadFromSavedConfiguration(JToken jsonConfig);
Task<ReleaseInfo[]> PerformQuery(TorznabQuery query);
}
}

9
src/Jackett/Indexers/BitMeTV.cs
View File

@ -136,5 +136,14 @@ namespace Jackett
cookies.FillFromJson(new Uri(BaseUrl), (JArray)jsonConfig["cookies"]);
IsConfigured = true;
}
public Task<ReleaseInfo[]> PerformQuery(TorznabQuery query)
{
return Task<ReleaseInfo[]>.Run(async () =>
{
List<ReleaseInfo> releases = new List<ReleaseInfo>();
return releases.ToArray();
});
}
}
}

9
src/Jackett/Indexers/Freshon.cs
View File

@ -126,5 +126,14 @@ namespace Jackett
cookies.FillFromJson(new Uri(BaseUrl), (JArray)jsonConfig["cookies"]);
IsConfigured = true;
}
public Task<ReleaseInfo[]> PerformQuery(TorznabQuery query)
{
return Task<ReleaseInfo[]>.Run(async () =>
{
List<ReleaseInfo> releases = new List<ReleaseInfo>();
return releases.ToArray();
});
}
}
}

9
src/Jackett/Indexers/IPTorrents.cs
View File

@ -122,5 +122,14 @@ namespace Jackett.Indexers
cookies.FillFromJson(new Uri(BaseUrl), (JArray)jsonConfig["cookies"]);
IsConfigured = true;
}
public Task<ReleaseInfo[]> PerformQuery(TorznabQuery query)
{
return Task<ReleaseInfo[]>.Run(async () =>
{
List<ReleaseInfo> releases = new List<ReleaseInfo>();
return releases.ToArray();
});
}
}
}

59
src/Jackett/Indexers/ThePirateBay.cs
View File

@ -1,16 +1,20 @@
using Newtonsoft.Json.Linq;
using CsQuery;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Web;
namespace Jackett.Indexers
{
public class ThePirateBay : IndexerInterface
{
class ThePirateBayConfig : ConfigurationData
{
public StringItem Url { get; private set; }
@ -36,7 +40,7 @@ namespace Jackett.Indexers
public bool IsConfigured { get; private set; }
static string SearchUrl = "s/?q=test";
static string SearchUrl = "s/?q=\"{0}\"&category=205&page=0&orderby=99";
static string BrowserUrl = "browse";
string BaseUrl;
@ -113,5 +117,56 @@ namespace Jackett.Indexers
BaseUrl = (string)jsonConfig["base_url"];
IsConfigured = true;
}
public Task<ReleaseInfo[]> PerformQuery(TorznabQuery query)
{
return Task<ReleaseInfo[]>.Run(async () =>
{
List<ReleaseInfo> releases = new List<ReleaseInfo>();
var search = BaseUrl + string.Format(SearchUrl, HttpUtility.UrlEncode("game of thrones s03e09"));
var results = await client.GetStringAsync(search);
CQ dom = results;
var descRegex = new Regex("Uploaded (?<month>.*?)-(?<day>.*?) (?<year>.*?), Size (?<size>.*?) (?<unit>.*?), ULed by");
var rows = dom["#searchResult > tbody > tr"];
foreach (var row in rows)
{
var release = new ReleaseInfo();
CQ qRow = row.Cq();
CQ qLink = qRow[".detLink"].First();
CQ qPeerCols = qRow["td[align=\"right\"]"];
//Uploaded 08-02 2007, Size 47.15 MiB, ULed
var description = qRow[".detDesc"][0].ChildNodes[0].NodeValue.Trim();
var descGroups = descRegex.Match(description).Groups;
release.PublishDate = new DateTime(
int.Parse(descGroups["year"].Value),
int.Parse(descGroups["month"].Value),
int.Parse(descGroups["day"].Value)
);
var size = float.Parse(descGroups["size"].Value);
switch (descGroups["unit"].Value)
{
case "GiB": release.Size = ReleaseInfo.BytesFromGB(size); break;
case "MiB": release.Size = ReleaseInfo.BytesFromMB(size); break;
case "KiB": release.Size = ReleaseInfo.BytesFromKB(size); break;
}
release.Comments = new Uri(BaseUrl + qLink.Attr("href").TrimStart('/'));
release.Guid = release.Comments;
release.Title = qLink.Text().Trim();
release.Description = release.Title;
release.MagnetUrl = new Uri(qRow["td > a"].First().Attr("href"));
release.InfoHash = release.MagnetUrl.ToString().Split(':')[3].Split('&')[0];
release.Seeders = int.Parse(qPeerCols.ElementAt(0).InnerText);
release.Peers = int.Parse(qPeerCols.ElementAt(1).InnerText) + release.Seeders;
release.MinimumRatio = 1;
release.MinimumSeedTime = 172800;
releases.Add(release);
}
return releases.ToArray();
});
}
}
}

4
src/Jackett/Jackett.csproj
View File

@ -110,7 +110,9 @@
</EmbeddedResource>
</ItemGroup>
<ItemGroup>
<Content Include="WebContent\animate.css" />
<Content Include="WebContent\animate.css">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="WebContent\bootstrap-notify.js">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>

13
src/Jackett/Program.cs
View File

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
@ -16,6 +17,18 @@ namespace Jackett
static void Main(string[] args)
{
//var descRegex = new Regex("Uploaded (?<month>.*?)-(?<day>.*?) (?<year>.*?), Size (?<size>.*?) (?<unit>.*?), ULed");
var descRegex = new Regex("Uploaded (?<month>.*?)-(?<day>.*?) (?<year>.*?), Size (?<size>.*?) (?<unit>.*?), ULed by");
var m = descRegex.Match(("Uploaded 06-03 2013, Size 329.84 MiB, ULed by"));
List<string> matches = new List<string>();
var date = m.Groups["month"].Value;
for (var i = 0; i < m.Groups.Count; i++)
{
var group = m.Groups[i];
matches.Add(group.Value); ;
}
//Uploaded 08-02&nbsp;2007, Size 47.15&nbsp;MiB, ULed
//Uploaded (<date>.*?)&nbsp;2007, Size 47.15&nbsp;MiB, ULed
var resultPage = new ResultPage(new ChannelInfo
{

16
src/Jackett/ReleaseInfo.cs
View File

@ -24,9 +24,25 @@ namespace Jackett
public Uri ConverUrl { get; set; }
public Uri BannerUrl { get; set; }
public string InfoHash { get; set; }
public Uri MagnetUrl { get; set; }
public double? MinimumRatio { get; set; }
public long? MinimumSeedTime { get; set; }
public static long BytesFromGB(float gb)
{
return BytesFromMB(gb * 1024f);
}
public static long BytesFromMB(float mb)
{
return BytesFromKB(mb * 1024f);
}
public static long BytesFromKB(float kb)
{
return (long)(kb * 1024f);
}
}
}

50
src/Jackett/ResultPage.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
@ -36,7 +37,7 @@ namespace Jackett
public string ToXml(Uri selfAtom)
{
var doc = new XDocument(
var xdoc = new XDocument(
new XDeclaration("1.0", "UTF-8", null),
new XElement("rss",
new XAttribute("version", "1.0"),
@ -58,31 +59,38 @@ namespace Jackett
new XElement("title", ChannelInfo.ImageTitle),
new XElement("link", ChannelInfo.ImageLink.ToString()),
new XElement("description", ChannelInfo.ImageDescription)
)
),
),
from r in Releases
select new XElement("item",
new XElement("title", r.Title),
new XElement("guid", r.Guid),
new XElement("link", r.Link),
new XElement("comments", r.Comments.ToString()),
new XElement("pubDate", xmlDateFormat(r.PublishDate)),
new XElement("category", r.Category),
new XElement("size", r.Size),
new XElement("description", r.Description),
new XElement("enclosure", new XAttribute("url", r.Link), new XAttribute("length", r.Size), new XAttribute("type", "application/x-bittorrent")),
getTorznabElement("rageid", r.RageID),
getTorznabElement("seeders", r.Seeders),
getTorznabElement("peers", r.Peers),
getTorznabElement("infohash", r.InfoHash),
getTorznabElement("minimumratio", r.MinimumRatio),
getTorznabElement("minimumseedtime", r.MinimumSeedTime)
from r in Releases
select new XElement("item",
new XElement("title", r.Title),
new XElement("guid", r.Guid),
new XElement("comments", r.Comments.ToString()),
new XElement("pubDate", xmlDateFormat(r.PublishDate)),
new XElement("size", r.Size),
new XElement("description", r.Description),
r.Link == null ? null : new XElement("link", r.Link),
r.Category == null ? null : new XElement("category", r.Category),
new XElement(
"enclosure",
new XAttribute("url", r.Link ?? r.MagnetUrl),
new XAttribute("length", r.Size),
new XAttribute("type", r.Link == null ? "application/x-bittorrent;x-scheme-handler/magnet" : "application/x-bittorrent")
),
getTorznabElement("magneturl", r.MagnetUrl),
getTorznabElement("rageid", r.RageID),
getTorznabElement("seeders", r.Seeders),
getTorznabElement("peers", r.Peers),
getTorznabElement("infohash", r.InfoHash),
getTorznabElement("minimumratio", r.MinimumRatio),
getTorznabElement("minimumseedtime", r.MinimumSeedTime)
)
)
)
);
return doc.ToString();
return xdoc.Declaration.ToString() + Environment.NewLine + xdoc.ToString();
}
}
}

70
src/Jackett/Server.cs
View File

@ -1,6 +1,7 @@
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.IO;
using System.Linq;
using System.Net;
@ -62,20 +63,67 @@ namespace Jackett
return;
}
var query = HttpUtility.ParseQueryString(context.Request.Url.Query);
if (context.Request.Url.AbsolutePath.StartsWith("/api/"))
{
ProcessTorznab(context);
return;
}
var inputStream = context.Request.InputStream;
var reader = new StreamReader(inputStream, context.Request.ContentEncoding);
var bytes = await reader.ReadToEndAsync();
var indexer = context.Request.Url.AbsolutePath.TrimStart('/').Replace("/api", "").ToLower();
var responseBytes = Encoding.UTF8.GetBytes(Properties.Resources.validator_reply);
context.Response.ContentEncoding = Encoding.UTF8;
context.Response.ContentLength64 = responseBytes.LongLength;
context.Response.ContentType = "application/rss+xml";
var responseBytes = Encoding.UTF8.GetBytes("Invalid request");
await context.Response.OutputStream.WriteAsync(responseBytes, 0, responseBytes.Length);
context.Response.Close();
}
async void ProcessTorznab(HttpListenerContext context)
{
Exception exception;
try
{
var query = HttpUtility.ParseQueryString(context.Request.Url.Query);
var inputStream = context.Request.InputStream;
var reader = new StreamReader(inputStream, context.Request.ContentEncoding);
var bytes = await reader.ReadToEndAsync();
var indexerId = context.Request.Url.AbsolutePath.Replace("/api", "").TrimStart('/').ToLower();
var indexer = indexerManager.GetIndexer(indexerId);
var torznabQuery = TorznabQuery.FromHttpQuery(query);
var severUrl = string.Format("{0}://{1}:{2}/", context.Request.Url.Scheme, context.Request.Url.Host, context.Request.Url.Port);
var resultPage = new ResultPage(new ChannelInfo
{
Title = indexer.DisplayName,
Description = indexer.DisplayDescription,
Link = indexer.SiteLink,
ImageUrl = new Uri(severUrl + "logos/" + indexerId + ".png"),
ImageTitle = indexer.DisplayName,
ImageLink = indexer.SiteLink,
ImageDescription = indexer.DisplayName
});
var releases = await indexer.PerformQuery(torznabQuery);
resultPage.Releases.AddRange(releases);
var xml = resultPage.ToXml(new Uri(severUrl));
var responseBytes = Encoding.UTF8.GetBytes(xml);
context.Response.ContentEncoding = Encoding.UTF8;
context.Response.ContentLength64 = responseBytes.LongLength;
context.Response.ContentType = "application/rss+xml";
await context.Response.OutputStream.WriteAsync(responseBytes, 0, responseBytes.Length);
context.Response.Close();
return;
}
catch (Exception ex)
{
exception = ex;
}
var errorBytes = Encoding.UTF8.GetBytes(exception.Message);
await context.Response.OutputStream.WriteAsync(errorBytes, 0, errorBytes.Length);
context.Response.Close();
}
}
}

13
src/Jackett/TorznabQuery.cs
View File

@ -21,6 +21,7 @@ namespace Jackett
public static TorznabQuery FromHttpQuery(NameValueCollection query)
{
//{t=tvsearch&cat=5030%2c5040&extended=1&apikey=test&offset=0&limit=100&rid=24493&season=5&ep=1}
var q = new TorznabQuery();
q.QueryType = query["t"];
@ -29,9 +30,15 @@ namespace Jackett
q.ApiKey = query["apikey"];
q.Limit = int.Parse(query["limit"]);
q.Offset = int.Parse(query["offset"]);
q.RageID = int.Parse(query["rid"]);
q.Season = int.Parse(query["season"]);
q.Episode = int.Parse(query["ep"]);
int temp;
if (int.TryParse(query["rid"], out temp))
q.RageID = temp;
if (int.TryParse(query["season"], out temp))
q.Season = temp;
if (int.TryParse(query["ep"], out temp))
q.Episode = int.Parse(query["ep"]);
return q;
}
}

6
src/Jackett/WebContent/index.html
View File

@ -3,11 +3,13 @@
<html lang="en" xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta charset="utf-8" />
<script src="jquery-2.1.3.min.js"></script>
<script src="handlebars-v3.0.1.js"></script>
<script src="bootstrap/bootstrap.min.js"></script>
<link href="bootstrap/bootstrap.min.css" rel="stylesheet">
<script src="bootstrap-notify.js"></script>
<link href="bootstrap/bootstrap.min.css" rel="stylesheet">
<link href="animate.css" rel="stylesheet">
<title>Jackett</title>
@ -298,7 +300,7 @@
var unconfiguredIndexerTemplate = Handlebars.compile($("#templates > .unconfigured-indexer")[0].outerHTML);
for (var i = 0; i < items.length; i++) {
var item = items[i];
item.torznab_host = resolveUrl("/torznab/" + item.id);
item.torznab_host = resolveUrl("/api/" + item.id);
if (item.configured)
$('#indexers').prepend(indexerTemplate(item));
else