mirror
/
Jackett
mirror of https://github.com/Jackett/Jackett
1
0
Fork 0
Code Issues Releases Wiki Activity

darmowetorrenty: migrate indexer to C# resolves #9782 (#9808)

This commit is contained in:
Ertugrul K 2020-10-12 22:51:29 +02:00 committed by GitHub
parent 5b785ffcc5
commit fed43f9f50
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 233 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

174
src/Jackett.Common/Definitions/darmowetorenty.yml
View File

@ -1,174 +0,0 @@
---
id: darmowetorenty
name: Darmowe torrenty
description: "Darmowe torrenty is a POLISH Semi-Private Torrent Tracker for MOVIES / TV / GENERAL"
language: pl-pl
type: semi-private
encoding: iso-8859-2
links:
- https://darmowe-torenty.pl/
caps:
categorymappings:
- {id: 14, cat: Movies, desc: "Filmy"}
- {id: 27, cat: Movies/DVD, desc: "Filmy DVD-R"}
- {id: 28, cat: Movies/SD, desc: "Filmy VCD/SVCD"}
- {id: 29, cat: Movies/BluRay, desc: "Filmy BluRay/x264"}
- {id: 30, cat: Movies/SD, desc: "Filmy DivX/XviD LEKTOR/NAPISY PL"}
- {id: 72, cat: Movies/SD, desc: "Filmy DivX/XviD ENG/..."}
- {id: 31, cat: Movies, desc: "Filmy RMVB"}
- {id: 74, cat: Movies/HD, desc: "Filmy HD"}
- {id: 75, cat: Movies/3D, desc: "Filmy 3D"}
- {id: 16, cat: TV, desc: "Seriale"}
- {id: 25, cat: TV, desc: "Seriale Polskie"}
- {id: 26, cat: TV, desc: "Seriale Zagraniczne"}
- {id: 17, cat: Movies, desc: "Dla Dzieci"}
- {id: 32, cat: Movies, desc: "Bajki Pl/Eng"}
- {id: 18, cat: PC/Games, desc: "Gry"}
- {id: 34, cat: PC/Games, desc: "Gry PC"}
- {id: 35, cat: Console/PSP, desc: "Gry PS2/PS3/PSP"}
- {id: 36, cat: Console/Xbox, desc: "Gry Xbox"}
- {id: 37, cat: Console, desc: "Gry Inne Konsole"}
- {id: 19, cat: Audio, desc: "Muzyka"}
- {id: 38, cat: Audio, desc: "Muzyka Polska/Zagraniczna"}
- {id: 39, cat: Audio, desc: "Muzyka Soundtracki"}
- {id: 40, cat: Audio, desc: "Muzyka Teledyski/Koncerty"}
- {id: 20, cat: PC/Phone-Other, desc: "GSM/PDA"}
- {id: 42, cat: PC/Phone-Other, desc: " Tapety GSM/PDA"}
- {id: 43, cat: PC/Phone-Other, desc: " Programy GSM/PDA"}
- {id: 44, cat: PC/Phone-Other, desc: " Filmy GSM/PDA"}
- {id: 45, cat: PC/Phone-Other, desc: " Dzwonki GSM/PDA"}
- {id: 46, cat: PC/Phone-Other, desc: " Gry GSM/PDA"}
- {id: 21, cat: Books, desc: "Książki/Czasopisma"}
- {id: 47, cat: Books/Ebook, desc: "Książki/Czasopisma E-Booki"}
- {id: 48, cat: Audio/Audiobook, desc: "Książki/Czasopisma Audio-Booki"}
- {id: 49, cat: Books/Magazines, desc: "Książki/Czasopisma Czasopisma"}
- {id: 50, cat: Books/Comics, desc: "Książki/Czasopisma Komiksy"}
- {id: 22, cat: PC, desc: "Programy"}
- {id: 51, cat: PC/0day, desc: "Programy Windows"}
- {id: 52, cat: PC, desc: "Programy Linux"}
- {id: 53, cat: PC/Mac, desc: "Programy Macintosh"}
- {id: 23, cat: Other, desc: "Inne"}
- {id: 55, cat: Other, desc: "Inne Tapety"}
- {id: 54, cat: Other, desc: "Inne Śmieszne"}
- {id: 56, cat: TV/Sport, desc: "Inne Sport"}
- {id: 57, cat: Other, desc: "Inne Pozostałe"}
- {id: 24, cat: XXX, desc: "Erotyka"}
- {id: 58, cat: XXX, desc: "Erotyka Czasopisma"}
- {id: 59, cat: XXX, desc: "Erotyka Zdjęcia "}
- {id: 60, cat: XXX, desc: "Erotyka Filmy"}
- {id: 61, cat: XXX, desc: "Erotyka Gry "}
- {id: 63, cat: XXX, desc: "Erotyka Hentai+18"}
- {id: 68, cat: TV/Anime, desc: "Anime"}
- {id: 69, cat: TV/Anime, desc: "Anime Pl"}
- {id: 70, cat: TV/Anime, desc: "Anime"}
- {id: 76, cat: Other, desc: "Archiwum"}
modes:
search: [q]
tv-search: [q, season, ep]
movie-search: [q]
music-search: [q]
settings:
- name: username
type: text
label: Username
- name: password
type: password
label: Password
login:
path: login.php
method: form
form: form[action="login.php"]
inputs:
uid: "{{ .Config.username }}"
pwd: "{{ .Config.password }}"
returnto: /
error:
- selector: div:contains("Podany login jest")
test:
path: index.php
selector: a[href^="logout.php?check_hash="]
download:
selector: table[align="center"][width="100%"] tbody tr:nth-child(2) td:nth-child(2)
filters:
- name: toupper
- name: trim
- name: prepend
args: "magnet:?xt=urn:btih:"
- name: append
args: "&tr=udp://tracker.opentrackr.org:1337&tr=udp://tracker.coppersurfer.tk:6969&tr=udp://tracker.internetwarriors.net:1337&tr=udp://tracker.leechers-paradise.org:6969"
search:
paths:
# https://darmowe-torenty.pl/torrenty.php?search=&category=0
- path: torrenty.php
inputs:
# does not support multi cat search
category: 0
search: "{{ .Keywords }}"
erotyka: 1
strona: 0
# does not support imdbid searching and does not display imdb links in results
rows:
selector: table.header > tbody > tr:has(td)
after: 1
fields:
category:
selector: a[href^="/torrenty.php?category="]
attribute: href
filters:
- name: querystring
args: category
title:
selector: a[href^="details.php?id="]:has(span)
details:
selector: a[href^="details.php?id="]:has(span)
attribute: href
download:
selector: a[href^="details.php?id="]:has(span)
attribute: href
banner:
selector: img[src^="./imgtorrent/"]
attribute: src
date:
selector: p
remove: br
filters:
- name: regexp
args: "Dodano: (\\d{2}\\/\\d{2}\\/\\d{4})"
- name: append
args: " +01:00" # CET
- name: dateparse
args: "02/01/2006 -07:00"
size:
selector: p
filters:
- name: regexp
args: "Rozmiar: (\\d{1,4}\\.\\d{2}\\s[K|M|G][B])"
seeders:
text: 0
seeders:
selector: p
optional: true
filters:
- name: regexp
args: "Seedów: (\\d+)"
leechers:
text: 0
leechers:
selector: p
optional: true
filters:
- name: regexp
args: "Leecherów: (\\d+)"
downloadvolumefactor:
text: 0
uploadvolumefactor:
text: 1
# engine n/a

232
src/Jackett.Common/Indexers/DarmoweTorrenty.cs Normal file
View File

@ -0,0 +1,232 @@
using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Web;
using AngleSharp.Dom;
using AngleSharp.Html.Parser;
using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json.Linq;
using NLog;
namespace Jackett.Common.Indexers
{
[ExcludeFromCodeCoverage]
public class DarmoweTorenty : BaseWebIndexer
{
private string LoginUrl => SiteLink + "login.php";
private string BrowseUrl => SiteLink + "torrenty.php";
private static readonly Regex _SizeRegex = new Regex("Rozmiar: (\\d{1,4}\\.\\d{2}\\s[K|M|G][B])", RegexOptions.Compiled);
private static readonly Regex _DateRegex = new Regex("Dodano: (\\d{2}\\/\\d{2}\\/\\d{4})", RegexOptions.Compiled);
private static readonly Regex _SeedsRegex = new Regex("Seedów: (\\d+)", RegexOptions.Compiled);
private static readonly Regex _LeechersRegex = new Regex("Leecherów: (\\d+)", RegexOptions.Compiled);
private new ConfigurationDataBasicLoginWithRSSAndDisplay configData
{
get => (ConfigurationDataBasicLoginWithRSSAndDisplay)base.configData;
set => base.configData = value;
}
private void MapCategories()
{
AddCategoryMapping(14, TorznabCatType.Movies, "Filmy");
AddCategoryMapping(27, TorznabCatType.MoviesDVD, "Filmy DVD-R");
AddCategoryMapping(28, TorznabCatType.MoviesSD, "Filmy VCD/SVCD");
AddCategoryMapping(29, TorznabCatType.MoviesBluRay, "Filmy BluRay/x264");
AddCategoryMapping(30, TorznabCatType.MoviesSD, "Filmy DivX/XviD LEKTOR/NAPISY PL");
AddCategoryMapping(72, TorznabCatType.MoviesSD, "Filmy DivX/XviD ENG/...");
AddCategoryMapping(31, TorznabCatType.Movies, "Filmy RMVB");
AddCategoryMapping(74, TorznabCatType.MoviesHD, "Filmy HD");
AddCategoryMapping(75, TorznabCatType.Movies3D, "Filmy 3D");
AddCategoryMapping(16, TorznabCatType.TV, "Seriale");
AddCategoryMapping(25, TorznabCatType.TV, "Seriale Polskie");
AddCategoryMapping(26, TorznabCatType.TV, "Seriale Zagraniczne");
AddCategoryMapping(17, TorznabCatType.Movies, "Dla Dzieci");
AddCategoryMapping(32, TorznabCatType.Movies, "Bajki Pl/Eng");
AddCategoryMapping(18, TorznabCatType.PCGames, "Gry");
AddCategoryMapping(34, TorznabCatType.PCGames, "Gry PC");
AddCategoryMapping(35, TorznabCatType.ConsolePSP, "Gry PS2/PS3/PSP");
AddCategoryMapping(36, TorznabCatType.ConsoleXbox, "Gry Xbox");
AddCategoryMapping(37, TorznabCatType.Console, "Gry Inne Konsole");
AddCategoryMapping(19, TorznabCatType.Audio, "Muzyka");
AddCategoryMapping(38, TorznabCatType.Audio, "Muzyka Polska/Zagraniczna");
AddCategoryMapping(39, TorznabCatType.Audio, "Muzyka Soundtracki");
AddCategoryMapping(40, TorznabCatType.Audio, "Muzyka Teledyski/Koncerty");
AddCategoryMapping(20, TorznabCatType.PCPhoneOther, "GSM/PDA");
AddCategoryMapping(42, TorznabCatType.PCPhoneOther, " Tapety GSM/PDA");
AddCategoryMapping(43, TorznabCatType.PCPhoneOther, " Programy GSM/PDA");
AddCategoryMapping(44, TorznabCatType.PCPhoneOther, " Filmy GSM/PDA");
AddCategoryMapping(45, TorznabCatType.PCPhoneOther, " Dzwonki GSM/PDA");
AddCategoryMapping(46, TorznabCatType.PCPhoneOther, " Gry GSM/PDA");
AddCategoryMapping(21, TorznabCatType.Books, "Książki/Czasopisma");
AddCategoryMapping(47, TorznabCatType.BooksEbook, "Książki/Czasopisma E-Booki");
AddCategoryMapping(48, TorznabCatType.AudioAudiobook, "Książki/Czasopisma Audio-Booki");
AddCategoryMapping(49, TorznabCatType.BooksMagazines, "Książki/Czasopisma Czasopisma");
AddCategoryMapping(50, TorznabCatType.BooksComics, "Książki/Czasopisma Komiksy");
AddCategoryMapping(22, TorznabCatType.PC, "Programy");
AddCategoryMapping(51, TorznabCatType.PC0day, "Programy Windows");
AddCategoryMapping(52, TorznabCatType.PC, "Programy Linux");
AddCategoryMapping(53, TorznabCatType.PCMac, "Programy Macintosh");
AddCategoryMapping(23, TorznabCatType.Other, "Inne");
AddCategoryMapping(55, TorznabCatType.Other, "Inne Tapety");
AddCategoryMapping(54, TorznabCatType.Other, "Inne Śmieszne");
AddCategoryMapping(56, TorznabCatType.TVSport, "Inne Sport");
AddCategoryMapping(57, TorznabCatType.Other, "Inne Pozostałe");
AddCategoryMapping(24, TorznabCatType.XXX, "Erotyka");
AddCategoryMapping(58, TorznabCatType.XXX, "Erotyka Czasopisma");
AddCategoryMapping(59, TorznabCatType.XXX, "Erotyka Zdjęcia");
AddCategoryMapping(60, TorznabCatType.XXX, "Erotyka Filmy");
AddCategoryMapping(61, TorznabCatType.XXX, "Erotyka Gry");
AddCategoryMapping(63, TorznabCatType.XXX, "Erotyka Hentai+18");
AddCategoryMapping(68, TorznabCatType.TVAnime, "Anime");
AddCategoryMapping(69, TorznabCatType.TVAnime, "Anime Pl");
AddCategoryMapping(70, TorznabCatType.TVAnime, "Anime");
AddCategoryMapping(76, TorznabCatType.Other, "Archiwum");
}
public DarmoweTorenty(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps)
: base(id: "darmowetorenty",
name: "Darmowe torenty",
description: "Darmowe torenty is a POLISH Semi-Private Torrent Tracker for MOVIES / TV / GENERAL",
link: "https://darmowe-torenty.pl/",
caps: TorznabUtil.CreateDefaultTorznabTVCaps(),
configService: configService,
client: wc,
logger: l,
p: ps,
configData: new ConfigurationDataBasicLoginWithRSSAndDisplay())
{
Encoding = Encoding.GetEncoding("iso-8859-2");
Language = "pl-pl";
Type = "semi-private";
MapCategories();
}
public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
{
LoadValuesFromJson(configJson);
var pairs = new Dictionary<string, string>
{
{ "uid", configData.Username.Value },
{ "pwd", configData.Password.Value }
};
var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, null, true, null, LoginUrl, true);
await ConfigureIfOK(result.Cookies, result.ContentString != null && result.Cookies.Contains("pass=") && !result.Cookies.Contains("deleted"), () =>
{
var parser = new HtmlParser();
var dom = parser.ParseDocument(result.ContentString);
var invalidPasswordDiv = dom.QuerySelector("div:contains(\"Podane hasło jest\")");
var invalidLoginDiv = dom.QuerySelector("div:contains(\"Podany login jest\")");
var bannedUserElement = dom.QuerySelector("b:contains(\"has been blocked - fill captcha that\")");
var errorMessage = invalidLoginDiv?.TextContent ??
invalidPasswordDiv?.TextContent ??
bannedUserElement?.TextContent;
throw new ExceptionWithConfigData(errorMessage, configData);
});
return IndexerConfigurationStatus.RequiresTesting;
}
private ReleaseInfo ParseRow(IParentNode titleRow, IElement detailsRow)
{
var categoryAttribute = detailsRow.QuerySelector("a[href^=\"/torrenty.php?category=\"]").GetAttribute("href");
var categoryUrl = new Uri(SiteLink + Uri.UnescapeDataString(categoryAttribute));
var trackerCategory = HttpUtility.ParseQueryString(categoryUrl.Query)["category"];
var categories = MapTrackerCatToNewznab(trackerCategory);
var seedsMatch = _SeedsRegex.Match(detailsRow.TextContent);
var leechersMatch = _LeechersRegex.Match(detailsRow.TextContent);
var dateMatch = _DateRegex.Match(detailsRow.TextContent);
var sizeMatch = _SizeRegex.Match(detailsRow.TextContent);
var date = DateTime.MinValue; // In case of parsing failure
if (dateMatch.Success)
{
date = DateTime.ParseExact(
$"{dateMatch.Groups[1].Value} +01:00", "dd/MM/yyyy zzz", CultureInfo.InvariantCulture);
}
var details = titleRow.QuerySelector("a[href^=\"details.php?id=\"]:has(span)");
var detailsLink = new Uri(SiteLink + details.GetAttribute("href"));
var encodedDownloadLink = detailsRow.QuerySelector("a[id^=\"download_\"]").GetAttribute("data-href");
var siteDownloadLink = new Uri(SiteLink + Uri.UnescapeDataString(StringUtil.FromBase64(encodedDownloadLink)));
var hash = HttpUtility.ParseQueryString(siteDownloadLink.Query)["id"];
var magnet =
$"magnet:?xt=urn:btih:{hash}&tr=udp://tracker.opentrackr.org:1337&tr=udp://tracker.coppersurfer.tk:6969&tr=udp://tracker.internetwarriors.net:1337&tr=udp://tracker.leechers-paradise.org:6969";
var downloadLink = new Uri(magnet);
var bannerLink = detailsRow.QuerySelector("img[src^=\"./imgtorrent/\"]")?.GetAttribute("src");
var seeders = seedsMatch.Success ? int.Parse(seedsMatch.Groups[1].Value) : 0;
var leechers = leechersMatch.Success ? int.Parse(leechersMatch.Groups[1].Value) : 0;
var peers = seeders + leechers;
var release = new ReleaseInfo
{
Title = details.TextContent,
Category = categories,
Seeders = seeders,
BannerUrl = !string.IsNullOrEmpty(bannerLink) ? new Uri(SiteLink + bannerLink) : null,
Peers = peers,
PublishDate = date,
DownloadVolumeFactor = 0,
UploadVolumeFactor = 1,
Link = downloadLink,
Guid = detailsLink,
Comments = detailsLink,
Size = sizeMatch.Success ? ReleaseInfo.GetBytes(sizeMatch.Groups[1].Value) : 0
};
return release;
}
protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
{
var releases = new List<ReleaseInfo>();
var searchString = query.GetQueryString();
var searchUrl = BrowseUrl;
var queryCollection = new NameValueCollection
{
{"search", searchString},
{"category", "0"}, // multi category search not supported
{"erotyka", "1"}
};
searchUrl += "?" + queryCollection.GetQueryString(Encoding);
var response = await RequestWithCookiesAsync(searchUrl);
if (response.IsRedirect || response.Cookies != null && response.Cookies.Contains("pass=deleted;"))
{
// re-login
await ApplyConfiguration(null);
response = await RequestWithCookiesAsync(searchUrl);
}
var results = response.ContentString;
try
{
var parser = new HtmlParser();
var dom = parser.ParseDocument(results);
var rows = dom.QuerySelectorAll("table.header > tbody > tr:has(td)");
if (rows[0].TextContent.Contains("Nie ma torrentów")) // issue #9782
{
return releases;
}
for (var i = 0; i < rows.Length; i+=2)
{
// First row contains table, the second row contains the rest of the details
var releaseInfo = ParseRow(rows[i], rows[i+1]);
releases.Add(releaseInfo);
}
}
catch (Exception ex)
{
OnParseError(results, ex);
}
return releases;
}
}
}

1
src/Jackett.Updater/Program.cs
View File

@ -290,6 +290,7 @@ namespace Jackett.Updater
"Definitions/crazyscorner.yml",
"Definitions/czteam.yml",
"Definitions/cztorrent.yml",
"Definitions/darmowetorenty.yml", // migrated to C#
"Definitions/demonsite.yml",
"Definitions/digbt.yml",
"Definitions/downloadville.yml",