retroflix: migrate the indexer to c# and use the new api. resolves #10838 (#10847)

This commit is contained in:
Diego Heras 2021-01-17 21:12:54 +01:00 committed by GitHub
parent 878f3a92aa
commit 3f4e88bcbe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 254 additions and 336 deletions

View File

@ -1,181 +0,0 @@
---
id: retroflix
name: RetroFlix
description: "Private Torrent Tracker for Classic Movies / TV / General Releases."
language: en-us
type: private
encoding: UTF-8
links:
- https://retroflix.club/
legacylinks:
- https://retroflix.net/
caps:
categorymappings:
- {id: 401, cat: Movies, desc: "Movies"}
- {id: 402, cat: TV, desc: "TV Series"}
- {id: 406, cat: Audio/Video, desc: "Music Videos"}
- {id: 407, cat: TV/Sport, desc: "Sports"}
- {id: 409, cat: Books, desc: "Books"}
- {id: 408, cat: Audio, desc: "HQ Audio"}
modes:
search: [q]
tv-search: [q, season, ep, imdbid]
movie-search: [q, imdbid]
music-search: [q]
book-search: [q]
settings:
- name: cookie
type: text
label: Cookie
- name: info
type: info
label: How to get the Cookie
default: "<ol><li>Login to this tracker with your browser<li>Open the <b>DevTools</b> panel by pressing <b>F12</b><li>Select the <b>Network</b> tab<li>Click on the <b>Doc</b> button (Chrome Browser) or <b>HTML</b> button (FireFox)<li>Refresh the page by pressing <b>F5</b><li>Click on the first row entry<li>Select the <b>Headers</b> tab on the Right panel<li>Find <b>'cookie:'</b> in the <b>Request Headers</b> section<li><b>Select</b> and <b>Copy</b> the whole cookie string <i>(everything after 'cookie: ')</i> and <b>Paste</b> here.</ol>"
- name: freeleech
type: checkbox
label: Search freeleech only
default: false
- name: sort
type: select
label: Sort requested from site
default: 4
options:
4: created
7: seeders
5: size
1: title
- name: type
type: select
label: Order requested from site
default: desc
options:
desc: desc
asc: asc
- name: info_tpp
type: info
label: Results Per Page
default: For best results, change the <b>Torrents per page:</b> setting to <b>100</b> on your account profile.
login:
method: cookie
inputs:
cookie: "{{ .Config.cookie }}"
test:
path: torrents.php
selector: a[href*="/logout?"]
search:
# https://retroflix.club/torrents.php?incldead=0&spstate=0&inclbookmarked=0&search=tt0055254&search_area=4&search_mode=0
paths:
- path: torrents.php
inputs:
$raw: "{{ range .Categories }}cat{{.}}=1&{{end}}"
search: "{{ if .Query.IMDBID }}{{ .Query.IMDBID }}{{ else }}{{ .Keywords }}{{ end }}"
# 0 incldead, 1 active, 2 dead
incldead: 0
# 0 all, 1 normal, 2 free, 3 2x, 4 2xfree, 5 50%, 6 2x50%, 7 30%
spstate: "{{ if .Config.freeleech }}2{{ else }}0{{ end }}"
# 0 title, 1 descr, 3 uploader, 4 imdburl
search_area: "{{ if .Query.IMDBID }}4{{ else }}0{{ end }}"
# 0 AND, 1 OR, 2 Exact
search_mode: 0
sort: "{{ .Config.sort }}"
type: "{{ .Config.type }}"
rows:
selector: table.torrents > tbody > tr:has(table.torrentname)
fields:
category:
selector: a[href^="?cat="]
attribute: href
filters:
- name: querystring
args: cat
release_year:
selector: a[href^="/torrents.php?processing="]
optional: true
quality:
selector: a[href^="/torrents.php?standard="]
optional: true
title:
selector: a[href^="details.php?id="]
filters:
- name: append
args: " {{ .Result.release_year }}"
- name: append
args: " {{ .Result.quality }}"
title:
selector: a[title][href^="details.php?id="]
attribute: title
optional: true
filters:
- name: append
args: " {{ .Result.release_year }}"
- name: append
args: " {{ .Result.quality }}"
details:
selector: a[href^="details.php?id="]
attribute: href
download:
selector: a[href^="download.php?id="]
attribute: href
poster:
selector: tr[onmouseover]
attribute: onmouseover
filters:
- name: regexp
args: "src=(.+?) "
imdb:
selector: a[href*="imdb.com/title/tt"]
attribute: href
date:
# time type: time elapsed (default)
selector: td:nth-child(4) > span[title]
attribute: title
optional: true
filters:
- name: append
args: " +00:00" # auto adjusted by site account profile
- name: dateparse
args: "02-01-2006 15:04:05 -07:00"
date:
# time added
selector: td:nth-child(4):not(:has(span))
optional: true
filters:
- name: append
args: " +00:00" # auto adjusted by site account profile
- name: dateparse
args: "02-01-200615:04:05 -07:00"
size:
selector: td:nth-child(5)
seeders:
selector: td:nth-child(6)
leechers:
selector: td:nth-child(7)
grabs:
selector: td:nth-child(8)
downloadvolumefactor:
case:
img.pro_free: 0
img.pro_free2up: 0
img.pro_50pctdown: 0.5
img.pro_50pctdown2up: 0.5
img.pro_30pctdown: 0.3
"*": 1
uploadvolumefactor:
case:
img.pro_50pctdown2up: 2
img.pro_free2up: 2
img.pro_2up: 2
"*": 1
minimumratio:
text: 1.0
minimumseedtime:
# 3 days (as seconds = 3 x 24 x 60 x 60)
text: 259200
# NexusPHP

View File

@ -0,0 +1,184 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
using WebClient = Jackett.Common.Utils.Clients.WebClient;
namespace Jackett.Common.Indexers.Abstract
{
[ExcludeFromCodeCoverage]
public abstract class SpeedAppTracker : BaseWebIndexer
{
private readonly Dictionary<string, string> _apiHeaders = new Dictionary<string, string>
{
{"Accept", "application/json"},
{"Content-Type", "application/json"}
};
// API DOC: https://speedapp.io/api/doc
private string LoginUrl => SiteLink + "api/login";
private string SearchUrl => SiteLink + "api/torrent";
private string _token;
private new ConfigurationDataBasicLoginWithEmail configData => (ConfigurationDataBasicLoginWithEmail)base.configData;
protected SpeedAppTracker(string link, string id, string name, string description,
IIndexerConfigurationService configService, WebClient client, Logger logger,
IProtectionService p, ICacheService cs, TorznabCapabilities caps)
: base(id: id,
name: name,
description: description,
link: link,
caps: caps,
configService: configService,
client: client,
logger: logger,
p: p,
cacheService: cs,
configData: new ConfigurationDataBasicLoginWithEmail())
{
}
public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
{
LoadValuesFromJson(configJson);
await RenewalTokenAsync();
var releases = await PerformQuery(new TorznabQuery());
await ConfigureIfOK(string.Empty, releases.Any(),
() => throw new Exception("Could not find releases."));
return IndexerConfigurationStatus.Completed;
}
private async Task RenewalTokenAsync()
{
if (configData.Email.Value == null || configData.Password.Value == null)
throw new Exception("Please, check the indexer configuration.");
var body = new Dictionary<string, string>
{
{ "username", configData.Email.Value.Trim() },
{ "password", configData.Password.Value.Trim() }
};
var jsonData = JsonConvert.SerializeObject(body);
var result = await RequestWithCookiesAsync(
LoginUrl, method: RequestType.POST, headers: _apiHeaders, rawbody: jsonData);
var json = JObject.Parse(result.ContentString);
_token = json.Value<string>("token");
if (_token == null)
throw new Exception(json.Value<string>("message"));
}
protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
{
var releases = new List<ReleaseInfo>();
//var categoryMapping = MapTorznabCapsToTrackers(query).Distinct().ToList();
var qc = new List<KeyValuePair<string, string>> // NameValueCollection don't support cat[]=19&cat[]=6
{
{"itemsPerPage", "100"},
{"sort", "torrent.createdAt"},
{"direction", "desc"}
};
foreach (var cat in MapTorznabCapsToTrackers(query))
qc.Add("categories[]", cat);
if (query.IsImdbQuery)
qc.Add("imdbId", query.ImdbID);
else
qc.Add("search", query.GetQueryString());
if (string.IsNullOrWhiteSpace(_token)) // fist time login
await RenewalTokenAsync();
var searchUrl = SearchUrl + "?" + qc.GetQueryString();
var response = await RequestWithCookiesAsync(searchUrl, headers: GetSearchHeaders());
if (response.Status == HttpStatusCode.Unauthorized)
{
await RenewalTokenAsync(); // re-login
response = await RequestWithCookiesAsync(searchUrl, headers: GetSearchHeaders());
}
else if (response.Status != HttpStatusCode.OK)
throw new Exception($"Unknown error in search: {response.ContentString}");
try
{
var rows = JArray.Parse(response.ContentString);
foreach (var row in rows)
{
var id = row.Value<string>("id");
var details = new Uri($"{SiteLink}browse/{id}");
var link = new Uri($"{SiteLink}api/torrent/{id}/download");
var publishDate = DateTime.Parse(row.Value<string>("created_at"), CultureInfo.InvariantCulture);
var cat = row.Value<JToken>("category").Value<string>("id");
// "description" field in API has too much HTML code
var description = row.Value<string>("short_description");
var posterStr = row.Value<string>("poster");
var poster = Uri.TryCreate(posterStr, UriKind.Absolute, out var posterUri) ? posterUri : null;
var dlVolumeFactor = row.Value<double>("download_volume_factor");
var ulVolumeFactor = row.Value<double>("upload_volume_factor");
var release = new ReleaseInfo
{
Title = row.Value<string>("name"),
Link = link,
Details = details,
Guid = details,
Category = MapTrackerCatToNewznab(cat),
PublishDate = publishDate,
Description = description,
Poster = poster,
Size = row.Value<long>("size"),
Grabs = row.Value<long>("times_completed"),
Seeders = row.Value<int>("seeders"),
Peers = row.Value<int>("leechers") + row.Value<int>("seeders"),
DownloadVolumeFactor = dlVolumeFactor,
UploadVolumeFactor = ulVolumeFactor,
MinimumRatio = 1,
MinimumSeedTime = 172800 // 48 hours
};
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(response.ContentString, ex);
}
return releases;
}
public override async Task<byte[]> Download(Uri link)
{
var response = await RequestWithCookiesAsync(link.ToString(), headers: GetSearchHeaders());
if (response.Status == HttpStatusCode.Unauthorized)
{
await RenewalTokenAsync();
response = await RequestWithCookiesAsync(link.ToString(), headers: GetSearchHeaders());
}
else if (response.Status != HttpStatusCode.OK)
throw new Exception($"Unknown error in download: {response.ContentBytes}");
return response.ContentBytes;
}
private Dictionary<string, string> GetSearchHeaders() => new Dictionary<string, string>
{
{"Authorization", $"Bearer {_token}"}
};
}
}

View File

@ -0,0 +1,66 @@
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Text;
using Jackett.Common.Indexers.Abstract;
using Jackett.Common.Models;
using Jackett.Common.Services.Interfaces;
using NLog;
using WebClient = Jackett.Common.Utils.Clients.WebClient;
namespace Jackett.Common.Indexers
{
[ExcludeFromCodeCoverage]
public class RetroFlix : SpeedAppTracker
{
public override string[] LegacySiteLinks { get; protected set; } = {
"https://retroflix.net/"
};
public RetroFlix(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps,
ICacheService cs)
: base(
id: "retroflix",
name: "RetroFlix",
description: "Private Torrent Tracker for Classic Movies / TV / General Releases",
link: "https://retroflix.club/",
caps: new TorznabCapabilities
{
TvSearchParams = new List<TvSearchParam>
{
TvSearchParam.Q, TvSearchParam.Season, TvSearchParam.Ep, TvSearchParam.ImdbId
},
MovieSearchParams = new List<MovieSearchParam>
{
MovieSearchParam.Q, MovieSearchParam.ImdbId
},
MusicSearchParams = new List<MusicSearchParam>
{
MusicSearchParam.Q
},
BookSearchParams = new List<BookSearchParam>
{
BookSearchParam.Q
}
},
configService: configService,
client: wc,
logger: l,
p: ps,
cs: cs)
{
Encoding = Encoding.UTF8;
Language = "en-us";
Type = "private";
// requestDelay for API Limit (1 request per 2 seconds)
webclient.requestDelay = 2.1;
AddCategoryMapping(401, TorznabCatType.Movies, "Movies");
AddCategoryMapping(402, TorznabCatType.TV, "TV Series");
AddCategoryMapping(406, TorznabCatType.AudioVideo, "Music Videos");
AddCategoryMapping(407, TorznabCatType.TVSport, "Sports");
AddCategoryMapping(409, TorznabCatType.Books, "Books");
AddCategoryMapping(408, TorznabCatType.Audio, "HQ Audio");
}
}
}

View File

@ -1,38 +1,17 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using Jackett.Common.Indexers.Abstract;
using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
using WebClient = Jackett.Common.Utils.Clients.WebClient;
namespace Jackett.Common.Indexers
{
[ExcludeFromCodeCoverage]
public class SpeedApp : BaseWebIndexer
public class SpeedApp : SpeedAppTracker
{
private readonly Dictionary<string, string> _apiHeaders = new Dictionary<string, string>
{
{"Accept", "application/json"},
{"Content-Type", "application/json"}
};
// API DOC: https://speedapp.io/api/doc
private string LoginUrl => SiteLink + "api/login";
private string SearchUrl => SiteLink + "api/torrent";
private string _token;
private new ConfigurationDataBasicLoginWithEmail configData => (ConfigurationDataBasicLoginWithEmail)base.configData;
public override string[] LegacySiteLinks { get; protected set; } = {
"https://www.icetorrent.org/",
"https://icetorrent.org/",
@ -74,8 +53,7 @@ namespace Jackett.Common.Indexers
client: wc,
logger: l,
p: ps,
cacheService: cs,
configData: new ConfigurationDataBasicLoginWithEmail())
cs: cs)
{
Encoding = Encoding.UTF8;
Language = "ro-ro";
@ -128,135 +106,5 @@ namespace Jackett.Common.Indexers
AddCategoryMapping(50, TorznabCatType.XXX, "XXX Packs");
AddCategoryMapping(51, TorznabCatType.XXX, "XXX SD");
}
public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
{
LoadValuesFromJson(configJson);
await RenewalTokenAsync();
var releases = await PerformQuery(new TorznabQuery());
await ConfigureIfOK(string.Empty, releases.Any(),
() => throw new Exception("Could not find releases."));
return IndexerConfigurationStatus.Completed;
}
private async Task RenewalTokenAsync()
{
var body = new Dictionary<string, string>
{
{ "username", configData.Email.Value.Trim() },
{ "password", configData.Password.Value.Trim() }
};
var jsonData = JsonConvert.SerializeObject(body);
var result = await RequestWithCookiesAsync(
LoginUrl, method: RequestType.POST, headers: _apiHeaders, rawbody: jsonData);
var json = JObject.Parse(result.ContentString);
_token = json.Value<string>("token");
if (_token == null)
throw new Exception(json.Value<string>("message"));
}
protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
{
var releases = new List<ReleaseInfo>();
//var categoryMapping = MapTorznabCapsToTrackers(query).Distinct().ToList();
var qc = new List<KeyValuePair<string, string>> // NameValueCollection don't support cat[]=19&cat[]=6
{
{"itemsPerPage", "100"},
{"sort", "torrent.createdAt"},
{"direction", "desc"}
};
foreach (var cat in MapTorznabCapsToTrackers(query))
qc.Add("categories[]", cat);
if (query.IsImdbQuery)
qc.Add("imdbId", query.ImdbID);
else
qc.Add("search", query.GetQueryString());
if (string.IsNullOrWhiteSpace(_token)) // fist time login
await RenewalTokenAsync();
var searchUrl = SearchUrl + "?" + qc.GetQueryString();
var response = await RequestWithCookiesAsync(searchUrl, headers: GetSearchHeaders());
if (response.Status == HttpStatusCode.Unauthorized)
{
await RenewalTokenAsync(); // re-login
response = await RequestWithCookiesAsync(searchUrl, headers: GetSearchHeaders());
}
else if (response.Status != HttpStatusCode.OK)
throw new Exception($"Unknown error in search: {response.ContentString}");
try
{
var rows = JArray.Parse(response.ContentString);
foreach (var row in rows)
{
var id = row.Value<string>("id");
var details = new Uri($"{SiteLink}browse/{id}");
var link = new Uri($"{SiteLink}api/torrent/{id}/download");
var publishDate = DateTime.Parse(row.Value<string>("created_at"), CultureInfo.InvariantCulture);
var cat = row.Value<JToken>("category").Value<string>("id");
// "description" field in API has too much HTML code
var description = row.Value<string>("short_description");
var posterStr = row.Value<string>("poster");
var poster = Uri.TryCreate(posterStr, UriKind.Absolute, out var posterUri) ? posterUri : null;
var dlVolumeFactor = row.Value<double>("download_volume_factor");
var ulVolumeFactor = row.Value<double>("upload_volume_factor");
var release = new ReleaseInfo
{
Title = row.Value<string>("name"),
Link = link,
Details = details,
Guid = details,
Category = MapTrackerCatToNewznab(cat),
PublishDate = publishDate,
Description = description,
Poster = poster,
Size = row.Value<long>("size"),
Grabs = row.Value<long>("times_completed"),
Seeders = row.Value<int>("seeders"),
Peers = row.Value<int>("leechers") + row.Value<int>("seeders"),
DownloadVolumeFactor = dlVolumeFactor,
UploadVolumeFactor = ulVolumeFactor,
MinimumRatio = 1,
MinimumSeedTime = 172800 // 48 hours
};
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(response.ContentString, ex);
}
return releases;
}
public override async Task<byte[]> Download(Uri link)
{
var response = await RequestWithCookiesAsync(link.ToString(), headers: GetSearchHeaders());
if (response.Status == HttpStatusCode.Unauthorized)
{
await RenewalTokenAsync();
response = await RequestWithCookiesAsync(link.ToString(), headers: GetSearchHeaders());
}
else if (response.Status != HttpStatusCode.OK)
throw new Exception($"Unknown error in download: {response.ContentBytes}");
return response.ContentBytes;
}
private Dictionary<string, string> GetSearchHeaders() => new Dictionary<string, string>
{
{"Authorization", $"Bearer {_token}"}
};
}
}

View File

@ -368,6 +368,7 @@ namespace Jackett.Updater
"Definitions/rapidetracker.yml",
"Definitions/rarbg.yml", // migrated to C#
"Definitions/redtopia.yml",
"Definitions/retroflix.yml", // migrated to C#
"Definitions/rgu.yml",
"Definitions/rns.yml", // site merged with audiobooktorrents
"Definitions/rockethd.yml",