mirror
/
Jackett
mirror of https://github.com/Jackett/Jackett
1
0
Fork 0
Code Issues Releases Wiki Activity

MoreThanTV: Rewrite & Header Support for Login (#11237) resolves #6783

This commit is contained in:
Ryan McDonald 2021-03-14 16:19:48 -05:00 committed by GitHub
parent 752c5856ea
commit b24ca38860
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 240 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/Jackett.Common/Indexers/BaseIndexer.cs
View File

@ -495,7 +495,7 @@ namespace Jackett.Common.Indexers
return result;
}
protected async Task<WebResult> RequestLoginAndFollowRedirect(string url, IEnumerable<KeyValuePair<string, string>> data, string cookies, bool returnCookiesFromFirstCall, string redirectUrlOverride = null, string referer = null, bool accumulateCookies = false)
protected async Task<WebResult> RequestLoginAndFollowRedirect(string url, IEnumerable<KeyValuePair<string, string>> data, string cookies, bool returnCookiesFromFirstCall, string redirectUrlOverride = null, string referer = null, bool accumulateCookies = false, Dictionary<string, string> headers = null)
{
var request = new WebRequest
{
@ -504,7 +504,8 @@ namespace Jackett.Common.Indexers
Cookies = cookies,
Referer = referer,
PostData = data,
Encoding = Encoding
Encoding = Encoding,
Headers = headers,
};
var response = await webclient.GetResultAsync(request);
CheckSiteDown(response);

394
src/Jackett.Common/Indexers/MoreThanTV.cs
View File

@ -8,6 +8,7 @@ using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using AngleSharp.Dom;
using AngleSharp.Html.Dom;
using AngleSharp.Html.Parser;
using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
@ -15,10 +16,13 @@ using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json.Linq;
using System.Web;
using NLog;
using static Jackett.Common.Models.IndexerConfig.ConfigurationData;
namespace Jackett.Common.Indexers
{
// This tracker uses a hybrid Luminance (based on GazelleTracker)
[ExcludeFromCodeCoverage]
public class MoreThanTV : BaseWebIndexer
{
@ -26,13 +30,17 @@ namespace Jackett.Common.Indexers
"https://www.morethan.tv/"
};
private string LoginUrl => SiteLink + "login.php";
private string SearchUrl => SiteLink + "ajax.php?action=browse&searchstr=";
private string DownloadUrl => SiteLink + "torrents.php?action=download&id=";
private string DetailsUrl => SiteLink + "torrents.php?torrentid=";
private string LoginUrl => SiteLink + "login";
private string BrowseUrl => SiteLink + "torrents.php";
private string DetailsUrl => SiteLink + "details.php";
private string _sort;
private string _order;
private ConfigurationDataBasicLogin ConfigData => (ConfigurationDataBasicLogin)configData;
private readonly Dictionary<string, string> _emulatedBrowserHeaders = new Dictionary<string, string>();
public MoreThanTV(IIndexerConfigurationService configService, WebClient c, Logger l, IProtectionService ps,
ICacheService cs)
: base(id: "morethantv",
@ -43,11 +51,11 @@ namespace Jackett.Common.Indexers
{
TvSearchParams = new List<TvSearchParam>
{
TvSearchParam.Q, TvSearchParam.Season, TvSearchParam.Ep
TvSearchParam.Q
},
MovieSearchParams = new List<MovieSearchParam>
{
MovieSearchParam.Q, MovieSearchParam.ImdbId
MovieSearchParam.Q
}
},
configService: configService,
@ -61,35 +69,112 @@ namespace Jackett.Common.Indexers
Language = "en-us";
Type = "private";
var sort = new SelectItem(new Dictionary<string, string>
{
{"time", "time"},
{"size", "size"},
{"snatched", "snatched"},
{"seeders", "seeders"},
{"leechers", "leechers"},
})
{ Name = "Sort requested from site", Value = "time" };
configData.AddDynamic("sort", sort);
var order = new SelectItem(new Dictionary<string, string>
{
{"desc", "desc"},
{"asc", "asc"}
})
{ Name = "Order requested from site", Value = "desc" };
configData.AddDynamic("order", order);
AddCategoryMapping(1, TorznabCatType.Movies);
AddCategoryMapping(2, TorznabCatType.TV);
AddCategoryMapping(3, TorznabCatType.Other);
}
/// <summary>
/// Parse and Return CSRF token
/// </summary>
/// <param name="content"></param>
/// <returns></returns>
private string GetToken(string content)
{
try
{
var parser = new HtmlParser();
var dom = parser.ParseDocument(content.Trim());
return dom.QuerySelector<IHtmlInputElement>("input[name=\"token\"]").Value;
}
catch (Exception e)
{
throw new Exception("Token Could not be parsed from Response, Error?", e);
}
}
/// <summary>
/// Emulate browser headers -- REQUIRED
/// </summary>
private void SetRequestHeaders()
{
_emulatedBrowserHeaders.Clear();
_emulatedBrowserHeaders.Add("referer", SiteLink);
_emulatedBrowserHeaders.Add("Upgrade-Insecure-Requests", "1");
_emulatedBrowserHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.72 Safari/537.36");
}
public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
{
LoadValuesFromJson(configJson);
var pairs = new Dictionary<string, string> {
{ "username", ConfigData.Username.Value },
{ "password", ConfigData.Password.Value },
{ "login", "Log in" },
{ "keeplogged", "1" }
{ "submit", "login" },
{ "keeplogged", "1" },
{ "cinfo", "3440|1440|24|360" }
};
var preRequest = await RequestWithCookiesAndRetryAsync(LoginUrl, string.Empty);
var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, preRequest.Cookies, true, SearchUrl, SiteLink);
await ConfigureIfOK(result.Cookies, result.ContentString != null && result.ContentString.Contains("status\":\"success\""), () =>
SetRequestHeaders();
// Fetch CSRF token
var preRequest = await RequestWithCookiesAndRetryAsync(LoginUrl, referer: SiteLink, headers: _emulatedBrowserHeaders);
// Check if user is logged in. /login redirects to / if so)
if (preRequest.IsRedirect)
{
if (result.ContentString.Contains("Your IP address has been banned."))
throw new ExceptionWithConfigData("Your IP address has been banned.", ConfigData);
await FollowIfRedirect(preRequest, SiteLink, null, preRequest.Cookies, true);
}
// sid was not set after redirect, attempt to log in again
if (!preRequest.Cookies.Contains("sid="))
{
string token = null;
try
{
token = GetToken(preRequest.ContentString);
}
catch (Exception)
{
string errorMessage = ParseErrorMessage(preRequest);
throw new ExceptionWithConfigData(errorMessage, configData);
}
// Add CSRF Token to payload
pairs.Add("token", token);
var response = await RequestLoginAndFollowRedirect(LoginUrl, pairs, preRequest.Cookies, true, null, SiteLink, headers: _emulatedBrowserHeaders);
await ConfigureIfOK(response.Cookies, response.Cookies.Contains("sid="), () =>
{
// Couldn't find "sid" cookie, so check for error
var parser = new HtmlParser();
var dom = parser.ParseDocument(result.ContentString);
foreach (var element in dom.QuerySelectorAll("#loginform > table"))
element.Remove();
var errorMessage = dom.QuerySelector("#loginform").TextContent.Trim().Replace("\n\t", " ");
throw new ExceptionWithConfigData(errorMessage, ConfigData);
});
var dom = parser.ParseDocument(response.ContentString);
var errorMessage = dom.QuerySelector(".flash.error").TextContent.Trim();
throw new ExceptionWithConfigData(errorMessage, configData);
});
}
return IndexerConfigurationStatus.RequiresTesting;
}
@ -98,44 +183,45 @@ namespace Jackett.Common.Indexers
{
var releases = new List<ReleaseInfo>();
if (!string.IsNullOrWhiteSpace(query.ImdbID))
await GetReleases(releases, query, query.GetQueryString());
else
{
var searchQuery = query.GetQueryString();
searchQuery = searchQuery.Replace("Marvels", "Marvel"); // strip 's for better results
var newSearchQuery = Regex.Replace(searchQuery, @"(S\d{2})$", "$1*"); // If we're just seaching for a season (no episode) append an * to include all episodes of that season.
await GetReleases(releases, query, newSearchQuery);
var searchQuery = query.GetQueryString();
searchQuery = searchQuery.Replace("Marvels", "Marvel"); // strip 's for better results
var newSearchQuery = Regex.Replace(searchQuery, @"(S\d{2})$", "$1*"); // If we're just seaching for a season (no episode) append an * to include all episodes of that season.
await GetReleasesAsync(releases, query, newSearchQuery);
// Always search for torrent groups (complete seasons) too
var seasonMatch = new Regex(@".*\s[Ss]{1}\d{2}([Ee]{1}\d{2,3})?$").Match(searchQuery);
if (seasonMatch.Success)
{
newSearchQuery = Regex.Replace(searchQuery, @"[Ss]{1}\d{2}([Ee]{1}\d{2,3})?", $"Season {query.Season}");
await GetReleases(releases, query, newSearchQuery);
}
// Always search for torrent groups (complete seasons) too
var seasonMatch = new Regex(@".*\s[Ss]{1}\d{2}([Ee]{1}\d{2,3})?$").Match(searchQuery);
if (seasonMatch.Success)
{
newSearchQuery = Regex.Replace(searchQuery, @"[Ss]{1}\d{2}([Ee]{1}\d{2,3})?", $"Season {query.Season}");
await GetReleasesAsync(releases, query, newSearchQuery);
}
return releases;
}
public override void LoadValuesFromJson(JToken jsonConfig, bool useProtectionService = false)
{
base.LoadValuesFromJson(jsonConfig, useProtectionService);
var sort = (SelectItem)configData.GetDynamic("sort");
_sort = sort != null ? sort.Value : "time";
var order = (SelectItem)configData.GetDynamic("order");
_order = order != null && order.Value.Equals("asc") ? order.Value : "desc";
}
private string GetTorrentSearchUrl(TorznabQuery query, string searchQuery)
{
var qc = new NameValueCollection
{
{ "tags_type", "1" },
{ "order_by", "time" },
{ "order_way", "desc" },
{ "group_results", "1" },
{ "action", "basic" },
{ "searchsubmit", "1" }
{ "order_by", _sort },
{ "order_way", _order },
{ "action", "advanced" },
{ "sizetype", "gb" },
{ "sizerange", "0.01" },
{ "title", searchQuery }
};
if (!string.IsNullOrWhiteSpace(query.ImdbID))
qc.Add("description", query.ImdbID);
else
qc.Add("searchstr", searchQuery);
if (query.Categories.Contains(TorznabCatType.Movies.ID))
qc.Add("filter_cat[1]", "1");
if (query.Categories.Contains(TorznabCatType.TV.ID))
@ -143,16 +229,16 @@ namespace Jackett.Common.Indexers
if (query.Categories.Contains(TorznabCatType.Other.ID))
qc.Add("filter_cat[3]", "1");
return SiteLink + "torrents.php?" + qc.GetQueryString();
return BrowseUrl + "?" + qc.GetQueryString();
}
private async Task GetReleases(ICollection<ReleaseInfo> releases, TorznabQuery query, string searchQuery)
private async Task GetReleasesAsync(ICollection<ReleaseInfo> releases, TorznabQuery query, string searchQuery)
{
var searchUrl = GetTorrentSearchUrl(query, searchQuery);
var response = await RequestWithCookiesAndRetryAsync(searchUrl);
if (response.IsRedirect)
{
// re login
// re-login
await ApplyConfiguration(null);
response = await RequestWithCookiesAndRetryAsync(searchUrl);
}
@ -161,99 +247,31 @@ namespace Jackett.Common.Indexers
{
var parser = new HtmlParser();
var document = parser.ParseDocument(response.ContentString);
var groups = document.QuerySelectorAll(".torrent_table > tbody > tr.group");
var torrents = document.QuerySelectorAll(".torrent_table > tbody > tr.torrent");
var torrents = document.QuerySelectorAll("#torrent_table > tbody > tr.torrent");
var movies = new[] { "movie" };
var tv = new[] { "season", "episode" };
// Loop through all torrent (season) groups
foreach (var group in groups)
{
var showName = group.QuerySelector(".tp-showname a").InnerHtml.Replace("(", "").Replace(")", "").Replace(' ', '.');
var season = group.QuerySelector(".big_info a").InnerHtml;
var seasonNumber = SeasonToNumber(season);
if (seasonNumber != null && query.Season > 0 && seasonNumber != query.Season) // filter unwanted seasons
continue;
var seasonTag = SeasonNumberToShortSeason(seasonNumber) ?? season;
// Loop through all group items
var previousElement = group;
var qualityEdition = string.Empty;
while (true)
{
var groupItem = previousElement.NextElementSibling;
if (groupItem == null)
break;
if (!groupItem.ClassList[0].Equals("group_torrent") ||
!groupItem.ClassList[1].StartsWith("groupid_"))
break;
// Found a new edition
if (groupItem.ClassList[2].Equals("edition"))
qualityEdition = groupItem.QuerySelector(".edition_info strong").TextContent.Split('/')[1].Trim();
else if (groupItem.ClassList[2].StartsWith("edition_"))
{
if (qualityEdition.Equals(string.Empty))
break;
// Parse required data
var downloadAnchor = groupItem.QuerySelectorAll("td a").Last();
var qualityData = downloadAnchor.InnerHtml.Split('/');
switch (qualityData.Length)
{
case 0:
Array.Resize(ref qualityData, 2);
qualityData[0] = " ";
qualityData[1] = " ";
break;
case 1:
Array.Resize(ref qualityData, 2);
qualityData[1] = " ";
break;
}
// Replace 4K quality tag with 2160p, so Sonarr etc. can properly parse it
qualityData[1] = qualityData[1].Replace("4K", "2160p");
// Build title
var title = string.Join(".", new List<string>
{
showName,
seasonTag,
qualityData[1].Trim(),
qualityEdition, // Audio quality should be after this one. Unobtainable at the moment.
$"{qualityData[0].Trim()}-MTV"
});
releases.Add(GetReleaseInfo(groupItem, downloadAnchor, title, TorznabCatType.TV.ID));
}
else
break;
previousElement = groupItem;
}
}
// Loop through all torrents
// Loop through all torrents checking for groups
foreach (var torrent in torrents)
{
// Parse required data
var downloadAnchor = torrent.QuerySelector(".big_info > .group_info > a");
var title = downloadAnchor.TextContent;
var torrentGroup = torrent.QuerySelectorAll("table a[href^=\"/torrents.php?action=download\"]");
foreach (var downloadAnchor in torrentGroup)
{
var title = downloadAnchor.ParentElement.ParentElement.ParentElement.TextContent.Trim();
title = CleanUpTitle(title);
int category;
var categories = torrent.QuerySelector(".cats_col div").ClassList;
if (categories.Contains("cats_tv"))
category = TorznabCatType.TV.ID;
else if (categories.Contains("cats_movies"))
category = TorznabCatType.Movies.ID;
else if (categories.Contains("cats_other"))
category = TorznabCatType.Other.ID;
else
throw new Exception("Couldn't find category.");
var category = torrent.QuerySelector(".cats_col div").GetAttribute("title");
// default to Other
int categoryId = TorznabCatType.Other.ID;
releases.Add(GetReleaseInfo(torrent, downloadAnchor, title, category));
if (movies.Any(category.Contains))
categoryId = TorznabCatType.Movies.ID;
else if (tv.Any(category.Contains))
categoryId = TorznabCatType.TV.ID;
releases.Add(GetReleaseInfo(torrent, downloadAnchor, title, categoryId));
}
}
}
catch (Exception ex)
@ -262,47 +280,73 @@ namespace Jackett.Common.Indexers
}
}
/// <summary>
/// Gather Release info from torrent table. Target using css
/// </summary>
/// <param name="row"></param>
/// <param name="downloadAnchor"></param>
/// <param name="title"></param>
/// <param name="category"></param>
/// <returns></returns>
private ReleaseInfo GetReleaseInfo(IElement row, IElement downloadAnchor, string title, int category)
{
var downloadAnchorHref = downloadAnchor.Attributes["href"].Value;
var torrentId = downloadAnchorHref.Substring(downloadAnchorHref.LastIndexOf('=') + 1);
if (torrentId.Contains('#'))
torrentId = torrentId.Split('#')[0];
var qFiles = row.QuerySelector("td:nth-last-child(6)");
var files = ParseUtil.CoerceLong(qFiles.TextContent);
var qPublishDate = row.QuerySelector(".time.tooltip").Attributes["title"].Value;
// count from bottom
const int FILES_COL = 8;
/*const int COMMENTS_COL = 7;*/
const int DATE_COL = 6;
const int FILESIZE_COL = 5;
const int SNATCHED_COL = 4;
const int SEEDS_COL = 3;
const int LEECHERS_COL = 2;
/*const int USER_COL = 1;*/
string downloadAnchorHref = (downloadAnchor as IHtmlAnchorElement).Href;
var queryParams = HttpUtility.ParseQueryString(downloadAnchorHref, Encoding.UTF8);
string torrentId = queryParams["id"];
var qFiles = row.QuerySelector("td:nth-last-child(" + FILES_COL + ")").TextContent;
var fileCount = ParseUtil.CoerceLong(qFiles);
var qPublishDate = row.QuerySelector("td:nth-last-child(" + DATE_COL + ") .time").Attributes["title"].Value;
var publishDate = DateTime.ParseExact(qPublishDate, "MMM dd yyyy, HH:mm", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal).ToLocalTime();
var qPoster = row.QuerySelector("div.tp-banner img")?.GetAttribute("src");
var poster = (qPoster != null && !qPoster.Contains("/static/styles/")) ? new Uri(qPoster) : null;
var poster = (qPoster != null && !qPoster.Contains("caticons")) ? new Uri(qPoster) : null;
var description = row.QuerySelector("div.tags")?.TextContent.Trim();
var fileSize = row.QuerySelector("td:nth-last-child(" + FILESIZE_COL + ")").TextContent.Trim();
var snatched = row.QuerySelector("td:nth-last-child(" + SNATCHED_COL + ")").TextContent.Trim();
var seeds = row.QuerySelector("td:nth-last-child(" + SEEDS_COL + ")").TextContent.Trim();
var leechs = row.QuerySelector("td:nth-last-child(" + LEECHERS_COL + ")").TextContent.Trim();
var torrentData = row.QuerySelectorAll(".number_column");
if (torrentData.Length != 4) // Size (xx.xx GB[ (Max)]) Snatches (xx) Seeders (xx) Leechers (xx)
throw new Exception($"We expected 4 torrent datas, instead we have {torrentData.Length}.");
if (fileSize.Length <= 0 || snatched.Length <= 0 || seeds.Length <= 0 || leechs.Length <= 0)
{
// Size (xx.xx GB[ (Max)]) Snatches (xx) Seeders (xx) Leechers (xx)
throw new Exception($"We expected 4 torrent datas.");
}
var size = ReleaseInfo.GetBytes(torrentData[0].TextContent);
var grabs = int.Parse(torrentData[1].TextContent, NumberStyles.AllowThousands, CultureInfo.InvariantCulture);
var seeders = int.Parse(torrentData[2].TextContent, NumberStyles.AllowThousands, CultureInfo.InvariantCulture);
var leechers = int.Parse(torrentData[3].TextContent, NumberStyles.AllowThousands, CultureInfo.InvariantCulture);
var details = new Uri(DetailsUrl + torrentId);
var link = new Uri(DownloadUrl + torrentId);
long size = ReleaseInfo.GetBytes(fileSize);
int grabs = int.Parse(snatched, NumberStyles.AllowThousands, CultureInfo.InvariantCulture);
int seeders = int.Parse(seeds, NumberStyles.AllowThousands, CultureInfo.InvariantCulture);
int leechers = int.Parse(leechs, NumberStyles.AllowThousands, CultureInfo.InvariantCulture);
var detailsUri = new Uri(DetailsUrl + "?torrentid=" + torrentId);
var downloadLink = new Uri(BrowseUrl + "?action=download&id=" + torrentId);
return new ReleaseInfo
{
Title = title,
Category = new List<int> { category }, // Who seasons movies right
Link = link,
Category = new List<int> { category },
Link = downloadLink,
PublishDate = publishDate,
Poster = poster,
Description = description,
Seeders = seeders,
Peers = seeders + leechers,
Files = files,
Files = fileCount,
Size = size,
Grabs = grabs,
Guid = details,
Details = details,
Guid = downloadLink,
Details = detailsUri,
DownloadVolumeFactor = 0, // ratioless tracker
UploadVolumeFactor = 1
};
@ -325,5 +369,41 @@ namespace Jackett.Common.Indexers
return null;
return $"S{season:00}";
}
/// <summary>
/// Parse Error Messages from using CSS classes
/// </summary>
/// <param name="response"></param>
/// <returns></returns>
private string ParseErrorMessage(WebResult response)
{
var parser = new HtmlParser();
var dom = parser.ParseDocument(response.ContentString);
var errorMessage = "Unknown Error";
switch (response.Status)
{
case System.Net.HttpStatusCode.Forbidden:
errorMessage = dom.QuerySelector(".time").Parent.TextContent.Trim();
break;
default:
errorMessage = dom.QuerySelector(".flash.error").TextContent.Trim();
break;
}
return errorMessage;
}
/// <summary>
/// Clean Up any title stuff
/// </summary>
/// <param name="title"></param>
/// <returns></returns>
private string CleanUpTitle(string title)
{
return title
.Replace(".", " ")
.Replace("4K", "2160p"); // sonarr cleanup
}
}
}