pretome: add imdb search and code clean up (#7916)

This commit is contained in:
Diego Heras 2020-03-29 13:45:30 +02:00 committed by GitHub
parent 96d1d4eee7
commit e808df2e53
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 126 additions and 197 deletions

View File

@ -1,6 +1,5 @@
using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
@ -19,19 +18,11 @@ namespace Jackett.Common.Indexers
public class Pretome : BaseWebIndexer
{
private string LoginUrl => SiteLink + "takelogin.php";
private string LoginReferer => SiteLink + "index.php?cat=1";
private string SearchUrl => SiteLink + "browse.php";
private readonly List<CategoryMapping> resultMapping = new List<CategoryMapping>();
private new ConfigurationDataPinNumber configData
{
get => (ConfigurationDataPinNumber)base.configData;
set => base.configData = value;
}
private new ConfigurationDataPinNumber configData => (ConfigurationDataPinNumber)base.configData;
public Pretome(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps)
: base(name: "PreToMe",
: base("PreToMe",
description: "BitTorrent site for High Quality, High Definition (HD) movies and TV Shows",
link: "https://pretome.info/",
caps: TorznabUtil.CreateDefaultTorznabTVCaps(),
@ -45,130 +36,84 @@ namespace Jackett.Common.Indexers
Language = "en-us";
Type = "private";
AddCategoryMapping("cat[]=22&tags=Windows", TorznabCatType.PC0day);
AddCategoryMapping("cat[]=22&tags=MAC", TorznabCatType.PCMac);
AddCategoryMapping("cat[]=22&tags=Linux", TorznabCatType.PC);
TorznabCaps.SupportsImdbMovieSearch = true;
TorznabCaps.SupportsImdbTVSearch = true;
AddCategoryMapping("cat[]=27", TorznabCatType.BooksEbook);
// Unfortunately most of them are tags not categories and they return the parent category
// we have to re-add the tags with the parent category so the results are not removed with the filtering
AddCategoryMapping("cat[]=4&tags=PC", TorznabCatType.PCGames);
AddCategoryMapping("cat[]=4&tags=RIP", TorznabCatType.PCGames);
AddCategoryMapping("cat[]=4&tags=ISO", TorznabCatType.PCGames);
AddCategoryMapping("cat[]=4&tags=XBOX360", TorznabCatType.ConsoleXbox360);
AddCategoryMapping("cat[]=4&tags=PS3", TorznabCatType.ConsolePS3);
AddCategoryMapping("cat[]=4&tags=Wii", TorznabCatType.ConsoleWii);
AddCategoryMapping("cat[]=4&tags=PSP", TorznabCatType.ConsolePSP);
AddCategoryMapping("cat[]=4&tags=NSD", TorznabCatType.ConsoleNDS);
AddCategoryMapping("cat[]=4&tags=XBox", TorznabCatType.ConsoleXbox);
AddCategoryMapping("cat[]=4&tags=PS2", TorznabCatType.ConsoleOther);
// Applications
AddCategoryMappingAndParent("cat[]=22", TorznabCatType.PC);
AddCategoryMappingAndParent("cat[]=22&tags=Windows", TorznabCatType.PC0day, TorznabCatType.PC);
AddCategoryMappingAndParent("cat[]=22&tags=MAC", TorznabCatType.PCMac, TorznabCatType.PC);
AddCategoryMappingAndParent("cat[]=22&tags=Linux", TorznabCatType.PC, TorznabCatType.PC);
AddCategoryMapping("cat[]=31&tags=Ebook", TorznabCatType.BooksEbook);
AddCategoryMapping("cat[]=31&tags=RARFiX", TorznabCatType.Other);
// Ebooks
AddCategoryMappingAndParent("cat[]=27", TorznabCatType.BooksEbook);
AddCategoryMapping("cat[]=19&tags=x264", TorznabCatType.Movies);
AddCategoryMapping("cat[]=19&tags=720p", TorznabCatType.MoviesHD);
AddCategoryMapping("cat[]=19&tags=XviD", TorznabCatType.MoviesSD);
AddCategoryMapping("cat[]=19&tags=BluRay", TorznabCatType.MoviesHD);
AddCategoryMapping("cat[]=19&tags=DVDRiP", TorznabCatType.MoviesSD);
AddCategoryMapping("cat[]=19&tags=1080p", TorznabCatType.MoviesHD);
AddCategoryMapping("cat[]=19&tags=DVD", TorznabCatType.MoviesSD);
AddCategoryMapping("cat[]=19&tags=DVDR", TorznabCatType.MoviesSD);
AddCategoryMapping("cat[]=19&tags=WMV", TorznabCatType.Movies);
AddCategoryMapping("cat[]=19&tags=CAM", TorznabCatType.Movies);
// Games
AddCategoryMappingAndParent("cat[]=4", TorznabCatType.Console);
AddCategoryMappingAndParent("cat[]=4&tags=PC", TorznabCatType.PCGames, TorznabCatType.Console);
AddCategoryMappingAndParent("cat[]=4&tags=RIP", TorznabCatType.PCGames, TorznabCatType.Console);
AddCategoryMappingAndParent("cat[]=4&tags=ISO", TorznabCatType.PCGames, TorznabCatType.Console);
AddCategoryMappingAndParent("cat[]=4&tags=XBOX360", TorznabCatType.ConsoleXbox360, TorznabCatType.Console);
AddCategoryMappingAndParent("cat[]=4&tags=PS3", TorznabCatType.ConsolePS3, TorznabCatType.Console);
AddCategoryMappingAndParent("cat[]=4&tags=Wii", TorznabCatType.ConsoleWii, TorznabCatType.Console);
AddCategoryMappingAndParent("cat[]=4&tags=PSP", TorznabCatType.ConsolePSP, TorznabCatType.Console);
AddCategoryMappingAndParent("cat[]=4&tags=NSD", TorznabCatType.ConsoleNDS, TorznabCatType.Console);
AddCategoryMappingAndParent("cat[]=4&tags=XBox", TorznabCatType.ConsoleXbox, TorznabCatType.Console);
AddCategoryMappingAndParent("cat[]=4&tags=PS2", TorznabCatType.ConsoleOther, TorznabCatType.Console);
AddCategoryMapping("cat[]=6&tags=MP3", TorznabCatType.AudioMP3);
AddCategoryMapping("cat[]=6&tags=V2", TorznabCatType.AudioMP3);
AddCategoryMapping("cat[]=6&tags=FLAC", TorznabCatType.AudioLossless);
AddCategoryMapping("cat[]=6&tags=320kbps", TorznabCatType.AudioMP3);
// Miscellaneous
AddCategoryMappingAndParent("cat[]=31", TorznabCatType.Other);
AddCategoryMappingAndParent("cat[]=31&tags=Ebook", TorznabCatType.BooksEbook, TorznabCatType.Other);
AddCategoryMappingAndParent("cat[]=31&tags=RARFiX", TorznabCatType.Other, TorznabCatType.Other);
AddCategoryMapping("cat[]=7&tags=x264", TorznabCatType.TVHD);
AddCategoryMapping("cat[]=7&tags=720p", TorznabCatType.TVHD);
AddCategoryMapping("cat[]=7&tags=HDTV", TorznabCatType.TVHD);
AddCategoryMapping("cat[]=7&tags=XviD", TorznabCatType.TVSD);
AddCategoryMapping("cat[]=7&BluRay", TorznabCatType.TVHD);
AddCategoryMapping("cat[]=7&tags=DVDRip", TorznabCatType.TVSD);
AddCategoryMapping("cat[]=7&tags=DVD", TorznabCatType.TVSD);
AddCategoryMapping("cat[]=7&tags=Documentary", TorznabCatType.TVDocumentary);
AddCategoryMapping("cat[]=7&tags=PDTV", TorznabCatType.TVSD);
AddCategoryMapping("cat[]=7&tags=HD-DVD", TorznabCatType.TVSD);
// Movies
AddCategoryMappingAndParent("cat[]=19", TorznabCatType.Movies);
AddCategoryMappingAndParent("cat[]=19&tags=x264", TorznabCatType.Movies, TorznabCatType.Movies);
AddCategoryMappingAndParent("cat[]=19&tags=720p", TorznabCatType.MoviesHD, TorznabCatType.Movies);
AddCategoryMappingAndParent("cat[]=19&tags=XviD", TorznabCatType.MoviesSD, TorznabCatType.Movies);
AddCategoryMappingAndParent("cat[]=19&tags=BluRay", TorznabCatType.MoviesHD, TorznabCatType.Movies);
AddCategoryMappingAndParent("cat[]=19&tags=DVDRiP", TorznabCatType.MoviesSD, TorznabCatType.Movies);
AddCategoryMappingAndParent("cat[]=19&tags=1080p", TorznabCatType.MoviesHD, TorznabCatType.Movies);
AddCategoryMappingAndParent("cat[]=19&tags=DVD", TorznabCatType.MoviesSD, TorznabCatType.Movies);
AddCategoryMappingAndParent("cat[]=19&tags=DVDR", TorznabCatType.MoviesSD, TorznabCatType.Movies);
AddCategoryMappingAndParent("cat[]=19&tags=WMV", TorznabCatType.Movies, TorznabCatType.Movies);
AddCategoryMappingAndParent("cat[]=19&tags=CAM", TorznabCatType.Movies, TorznabCatType.Movies);
AddCategoryMapping("cat[]=51&tags=XviD", TorznabCatType.XXXXviD);
AddCategoryMapping("cat[]=51&tags=DVDRiP", TorznabCatType.XXXDVD);
// Music
AddCategoryMappingAndParent("cat[]=6", TorznabCatType.Audio);
AddCategoryMappingAndParent("cat[]=6&tags=MP3", TorznabCatType.AudioMP3, TorznabCatType.Audio);
AddCategoryMappingAndParent("cat[]=6&tags=V2", TorznabCatType.AudioMP3, TorznabCatType.Audio);
AddCategoryMappingAndParent("cat[]=6&tags=FLAC", TorznabCatType.AudioLossless, TorznabCatType.Audio);
AddCategoryMappingAndParent("cat[]=6&tags=320kbps", TorznabCatType.AudioMP3, TorznabCatType.Audio);
// Unfortunately they are tags not categories so return the results
// as the parent category so do not get results removed with the filtering.
// TV
AddCategoryMappingAndParent("cat[]=7", TorznabCatType.TV);
AddCategoryMappingAndParent("cat[]=7&tags=x264", TorznabCatType.TVHD, TorznabCatType.TV);
AddCategoryMappingAndParent("cat[]=7&tags=720p", TorznabCatType.TVHD, TorznabCatType.TV);
AddCategoryMappingAndParent("cat[]=7&tags=HDTV", TorznabCatType.TVHD, TorznabCatType.TV);
AddCategoryMappingAndParent("cat[]=7&tags=XviD", TorznabCatType.TVSD, TorznabCatType.TV);
AddCategoryMappingAndParent("cat[]=7&BluRay", TorznabCatType.TVHD, TorznabCatType.TV);
AddCategoryMappingAndParent("cat[]=7&tags=DVDRip", TorznabCatType.TVSD, TorznabCatType.TV);
AddCategoryMappingAndParent("cat[]=7&tags=DVD", TorznabCatType.TVSD, TorznabCatType.TV);
AddCategoryMappingAndParent("cat[]=7&tags=Documentary", TorznabCatType.TVDocumentary, TorznabCatType.TV);
AddCategoryMappingAndParent("cat[]=7&tags=PDTV", TorznabCatType.TVSD, TorznabCatType.TV);
AddCategoryMappingAndParent("cat[]=7&tags=HD-DVD", TorznabCatType.TVSD, TorznabCatType.TV);
AddResultCategoryMapping("cat[]=22&tags=Windows", TorznabCatType.PC);
AddResultCategoryMapping("cat[]=22&tags=MAC", TorznabCatType.PC);
AddResultCategoryMapping("cat[]=22&tags=Linux", TorznabCatType.PC);
AddResultCategoryMapping("cat[]=22&tags=All", TorznabCatType.PC);
AddResultCategoryMapping("cat[]=22", TorznabCatType.PC);
AddResultCategoryMapping("cat[]=27&tags=All", TorznabCatType.Books);
AddResultCategoryMapping("cat[]=27", TorznabCatType.Books);
AddResultCategoryMapping("cat[]=4&tags=PC", TorznabCatType.PC);
AddResultCategoryMapping("cat[]=4&tags=RIP", TorznabCatType.PC);
AddResultCategoryMapping("cat[]=4&tags=ISO", TorznabCatType.PC);
AddResultCategoryMapping("cat[]=4&tags=XBOX360", TorznabCatType.Console);
AddResultCategoryMapping("cat[]=4&tags=PS3", TorznabCatType.Console);
AddResultCategoryMapping("cat[]=4&tags=Wii", TorznabCatType.Console);
AddResultCategoryMapping("cat[]=4&tags=PSP", TorznabCatType.Console);
AddResultCategoryMapping("cat[]=4&tags=NSD", TorznabCatType.Console);
AddResultCategoryMapping("cat[]=4&tags=XBox", TorznabCatType.Console);
AddResultCategoryMapping("cat[]=4&tags=PS2", TorznabCatType.Console);
AddResultCategoryMapping("cat[]=4&tags=All", TorznabCatType.Console);
AddResultCategoryMapping("cat[]=4", TorznabCatType.Console);
AddResultCategoryMapping("cat[]=31&tags=Ebook", TorznabCatType.Books);
AddResultCategoryMapping("cat[]=31&tags=RARFiX", TorznabCatType.Other);
AddResultCategoryMapping("cat[]=31&tags=All", TorznabCatType.Other);
AddResultCategoryMapping("cat[]=31", TorznabCatType.Other);
AddResultCategoryMapping("cat[]=19&tags=x264", TorznabCatType.Movies);
AddResultCategoryMapping("cat[]=19&tags=720p", TorznabCatType.Movies);
AddResultCategoryMapping("cat[]=19&tags=XviD", TorznabCatType.Movies);
AddResultCategoryMapping("cat[]=19&tags=BluRay", TorznabCatType.Movies);
AddResultCategoryMapping("cat[]=19&tags=DVDRiP", TorznabCatType.Movies);
AddResultCategoryMapping("cat[]=19&tags=1080p", TorznabCatType.Movies);
AddResultCategoryMapping("cat[]=19&tags=DVD", TorznabCatType.Movies);
AddResultCategoryMapping("cat[]=19&tags=DVDR", TorznabCatType.Movies);
AddResultCategoryMapping("cat[]=19&tags=WMV", TorznabCatType.Movies);
AddResultCategoryMapping("cat[]=19&tags=CAM", TorznabCatType.Movies);
AddResultCategoryMapping("cat[]=19&tags=All", TorznabCatType.Movies);
AddResultCategoryMapping("cat[]=19", TorznabCatType.Movies);
AddResultCategoryMapping("cat[]=6&tags=MP3", TorznabCatType.Audio);
AddResultCategoryMapping("cat[]=6&tags=V2", TorznabCatType.Audio);
AddResultCategoryMapping("cat[]=6&tags=FLAC", TorznabCatType.Audio);
AddResultCategoryMapping("cat[]=6&tags=320kbps", TorznabCatType.Audio);
AddResultCategoryMapping("cat[]=6&tags=All", TorznabCatType.Audio);
AddResultCategoryMapping("cat[]=6", TorznabCatType.Audio);
AddResultCategoryMapping("cat[]=7&tags=x264", TorznabCatType.TV);
AddResultCategoryMapping("cat[]=7&tags=720p", TorznabCatType.TV);
AddResultCategoryMapping("cat[]=7&tags=HDTV", TorznabCatType.TV);
AddResultCategoryMapping("cat[]=7&tags=XviD", TorznabCatType.TV);
AddResultCategoryMapping("cat[]=7&BluRay", TorznabCatType.TV);
AddResultCategoryMapping("cat[]=7&tags=DVDRip", TorznabCatType.TV);
AddResultCategoryMapping("cat[]=7&tags=DVD", TorznabCatType.TV);
AddResultCategoryMapping("cat[]=7&tags=Documentary", TorznabCatType.TV);
AddResultCategoryMapping("cat[]=7&tags=PDTV", TorznabCatType.TV);
AddResultCategoryMapping("cat[]=7&tags=HD-DVD", TorznabCatType.TV);
AddResultCategoryMapping("cat[]=7&tags=All", TorznabCatType.TV);
AddResultCategoryMapping("cat[]=7", TorznabCatType.TV);
AddResultCategoryMapping("cat[]=51&tags=XviD", TorznabCatType.XXX);
AddResultCategoryMapping("cat[]=51&tags=DVDRiP", TorznabCatType.XXX);
AddResultCategoryMapping("cat[]=51&tags=All", TorznabCatType.XXX);
AddResultCategoryMapping("cat[]=51", TorznabCatType.XXX);
// XXX
AddCategoryMappingAndParent("cat[]=51", TorznabCatType.XXX);
AddCategoryMappingAndParent("cat[]=51&tags=XviD", TorznabCatType.XXXXviD, TorznabCatType.XXX);
AddCategoryMappingAndParent("cat[]=51&tags=DVDRiP", TorznabCatType.XXXDVD, TorznabCatType.XXX);
}
protected void AddResultCategoryMapping(string trackerCategory, TorznabCategory newznabCategory)
private void AddCategoryMappingAndParent(string trackerCategory, TorznabCategory newznabCategory,
TorznabCategory parentCategory = null)
{
resultMapping.Add(new CategoryMapping(trackerCategory.ToString(), null, newznabCategory.ID));
if (!TorznabCaps.Categories.Contains(newznabCategory))
TorznabCaps.Categories.Add(newznabCategory);
AddCategoryMapping(trackerCategory, newznabCategory);
if (parentCategory != null && parentCategory.ID != newznabCategory.ID)
AddCategoryMapping(trackerCategory, parentCategory);
}
public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
@ -188,18 +133,14 @@ namespace Jackett.Common.Indexers
// Send Post
var result = await PostDataWithCookies(LoginUrl, pairs, loginPage.Cookies);
if (result.RedirectingTo == null)
{
throw new ExceptionWithConfigData("Login failed. Did you use the PIN number that pretome emailed you?", configData);
}
var loginCookies = result.Cookies;
// Get result from redirect
var loginCookies = result.Cookies;
await FollowIfRedirect(result, LoginUrl, null, loginCookies);
await ConfigureIfOK(loginCookies, result.Content != null && result.Content.Contains("logout.php"), () =>
{
CookieHeader = string.Empty;
throw new ExceptionWithConfigData("Failed", configData);
});
await ConfigureIfOK(loginCookies, result.Content?.Contains("logout.php") == true,
() => throw new ExceptionWithConfigData("Failed", configData));
return IndexerConfigurationStatus.RequiresTesting;
}
@ -207,74 +148,61 @@ namespace Jackett.Common.Indexers
protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
{
var releases = new List<ReleaseInfo>();
var queryUrl = SearchUrl;
var queryCollection = new NameValueCollection();
var qc = new List<KeyValuePair<string, string>>(); // NameValueCollection don't support cat[]=19&cat[]=6
if (query.IsImdbQuery)
{
qc.Add("search", query.ImdbID);
qc.Add("st", "1");
qc.Add("sd", "1");
}
else if (!string.IsNullOrWhiteSpace(query.GetQueryString()))
{
qc.Add("search", query.GetQueryString());
qc.Add("st", "1");
}
// parse categories and tags
var catGroups = new HashSet<string>(); // HashSet instead of List to avoid duplicates
var tagGroups = new HashSet<string>();
var cats = MapTorznabCapsToTrackers(query);
var tags = string.Empty;
var catGroups = new List<string>();
foreach (var cat in cats)
{
// "cat[]=7&tags=x264"
var cSplit = cat.Split('&');
if (cSplit.Length > 0)
{
var gsplit = cSplit[0].Split('=');
if (gsplit.Length > 1)
{
catGroups.Add(gsplit[1]);
}
}
var gSplit = cSplit[0].Split('=');
if (gSplit.Length > 1)
catGroups.Add(gSplit[1]); // category = 7
if (cSplit.Length > 1)
{
var gsplit = cSplit[1].Split('=');
if (gsplit.Length > 1)
{
if (tags != string.Empty)
tags += ",";
tags += gsplit[1];
}
var tSplit = cSplit[1].Split('=');
if (tSplit.Length > 1)
tagGroups.Add(tSplit[1]); // tag = x264
}
}
if (catGroups.Distinct().Count() == 1)
{
queryCollection.Add("cat[]", catGroups.First());
}
// add categories
foreach (var cat in catGroups)
qc.Add("cat[]", cat);
if (!string.IsNullOrWhiteSpace(query.GetQueryString()))
{
queryCollection.Add("st", "1");
queryCollection.Add("search", query.GetQueryString());
}
// Do not include too many tags as it'll mess with their servers.
if (tags.Split(',').Length < 7)
{
queryCollection.Add("tags", tags);
if (!string.IsNullOrWhiteSpace(tags))
// do not include too many tags as it'll mess with their servers
if (tagGroups.Count < 7)
{
qc.Add("tags", string.Join(",", tagGroups));
// if tags are specified match any
queryCollection.Add("tf", "any");
}
else
{
// if no tags are specified match all, with any we get random results
queryCollection.Add("tf", "all");
}
qc.Add("tf", tagGroups.Any() ? "any" : "all");
}
if (queryCollection.Count > 0)
{
queryUrl += "?" + queryCollection.GetQueryString();
}
var searchUrl = SearchUrl + "?" + qc.GetQueryString();
var response = await RequestStringWithCookiesAndRetry(searchUrl);
var response = await RequestStringWithCookiesAndRetry(queryUrl);
if (response.IsRedirect)
if (response.IsRedirect) // re-login
{
await ApplyConfiguration(null);
response = await RequestStringWithCookiesAndRetry(queryUrl);
response = await RequestStringWithCookiesAndRetry(searchUrl);
}
try
@ -287,40 +215,41 @@ namespace Jackett.Common.Indexers
var qLink = row.Children[1].QuerySelector("a");
var title = qLink.GetAttribute("title");
if (qLink.QuerySelectorAll("span").Length == 1 && title.StartsWith("NEW! |"))
{
title = title.Substring(6).Trim();
}
if (!query.MatchQueryStringAND(title))
continue; // we have to skip bad titles due to tags + any word search
var comments = new Uri(SiteLink + qLink.GetAttribute("href"));
var qDownload = row.Children[2].QuerySelector("a");
var link = new Uri(SiteLink + row.Children[2].QuerySelector("a").GetAttribute("href"));
var dateStr = Regex.Replace(row.Children[5].InnerHtml, @"\<br[\s]{0,1}[\/]{0,1}\>", " ");
var sizeStr = row.Children[7].TextContent;
var seeders = ParseUtil.CoerceInt(row.Children[9].TextContent);
var files = ParseUtil.CoerceInt(row.QuerySelector("td:nth-child(4)").TextContent);
var cat = row.FirstElementChild.FirstElementChild.GetAttribute("href").Replace("browse.php?", string.Empty);
var link = new Uri(SiteLink + qDownload.GetAttribute("href"));
var publishDate = DateTimeUtil.FromTimeAgo(dateStr);
var size = ReleaseInfo.GetBytes(sizeStr);
var files = ParseUtil.CoerceInt(row.Children[3].TextContent);
var size = ReleaseInfo.GetBytes(row.Children[7].TextContent);
var grabs = ParseUtil.CoerceInt(row.Children[8].TextContent);
var seeders = ParseUtil.CoerceInt(row.Children[9].TextContent);
var leechers = ParseUtil.CoerceInt(row.Children[10].TextContent);
var grabs = ParseUtil.CoerceInt(row.QuerySelector("td:nth-child(9)").TextContent);
var cat = row.FirstElementChild.FirstElementChild.GetAttribute("href").Replace("browse.php?", string.Empty);
var release = new ReleaseInfo
{
MinimumRatio = 1,
MinimumSeedTime = 172800, // 48 hours
Title = title,
Comments = comments,
Guid = comments,
Link = link,
PublishDate = publishDate,
Size = size,
Seeders = seeders,
Peers = leechers + seeders,
Category = MapTrackerCatToNewznab(cat),
Files = files,
Grabs = grabs,
Seeders = seeders,
Peers = leechers + seeders,
MinimumRatio = 1,
MinimumSeedTime = 172800, // 48 hours
DownloadVolumeFactor = 0, // ratioless
UploadVolumeFactor = 1
};
releases.Add(release);
}
}