mirror of
https://github.com/Jackett/Jackett
synced 2025-02-25 07:32:38 +00:00
Rewrote MoreThanTV indexer (#477)
* MTV: Rewrote searching for torrents. Modified title output so sonarr can actually download seasons. * MTV: Add individual torrent parsing and cleanup. * MTV: Remove TODO as it's already done.
This commit is contained in:
Mike
flightlevel
parent
79418d3b1f
commit
aa1e31a110
1 changed files with 191 additions and 77 deletions
|
|
@ -1,36 +1,30 @@
|
|||
using CsQuery;
|
||||
using Jackett.Models;
|
||||
using Jackett.Models;
|
||||
using Jackett.Services;
|
||||
using Jackett.Utils;
|
||||
using Jackett.Utils.Clients;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using NLog;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
using System.Web;
|
||||
using AngleSharp.Dom;
|
||||
using AngleSharp.Parser.Html;
|
||||
using CsQuery;
|
||||
using Jackett.Models.IndexerConfig;
|
||||
|
||||
namespace Jackett.Indexers
|
||||
{
|
||||
public class MoreThanTV : BaseIndexer, IIndexer
|
||||
{
|
||||
private string LoginUrl { get { return SiteLink + "login.php"; } }
|
||||
private string SearchUrl { get { return SiteLink + "ajax.php?action=browse&searchstr="; } }
|
||||
private string DownloadUrl { get { return SiteLink + "torrents.php?action=download&id="; } }
|
||||
private string GuidUrl { get { return SiteLink + "torrents.php?torrentid="; } }
|
||||
private string LoginUrl => SiteLink + "login.php";
|
||||
private string SearchUrl => SiteLink + "ajax.php?action=browse&searchstr=";
|
||||
private string DownloadUrl => SiteLink + "torrents.php?action=download&id=";
|
||||
private string GuidUrl => SiteLink + "torrents.php?torrentid=";
|
||||
|
||||
new ConfigurationDataBasicLogin configData
|
||||
{
|
||||
get { return (ConfigurationDataBasicLogin)base.configData; }
|
||||
set { base.configData = value; }
|
||||
}
|
||||
private ConfigurationDataBasicLogin ConfigData => (ConfigurationDataBasicLogin) configData;
|
||||
|
||||
public MoreThanTV(IIndexerManagerService i, IWebClient c, Logger l, IProtectionService ps)
|
||||
: base(name: "MoreThanTV",
|
||||
|
|
@ -48,111 +42,231 @@ namespace Jackett.Indexers
|
|||
|
||||
public async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
|
||||
{
|
||||
configData.LoadValuesFromJson(configJson);
|
||||
ConfigData.LoadValuesFromJson(configJson);
|
||||
var pairs = new Dictionary<string, string> {
|
||||
{ "username", configData.Username.Value },
|
||||
{ "password", configData.Password.Value },
|
||||
{ "username", ConfigData.Username.Value },
|
||||
{ "password", ConfigData.Password.Value },
|
||||
{ "login", "Log in" },
|
||||
{ "keeplogged", "1" }
|
||||
};
|
||||
|
||||
var preRequest = await RequestStringWithCookiesAndRetry(LoginUrl, string.Empty);
|
||||
|
||||
var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, preRequest.Cookies, true, SearchUrl, SiteLink);
|
||||
|
||||
await ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("status\":\"success\""), () =>
|
||||
{
|
||||
CQ dom = result.Content;
|
||||
dom["#loginform > table"].Remove();
|
||||
var errorMessage = dom["#loginform"].Text().Trim().Replace("\n\t", " ");
|
||||
throw new ExceptionWithConfigData(errorMessage, configData);
|
||||
throw new ExceptionWithConfigData(errorMessage, ConfigData);
|
||||
});
|
||||
|
||||
return IndexerConfigurationStatus.RequiresTesting;
|
||||
}
|
||||
|
||||
private void FillReleaseInfoFromJson(ReleaseInfo release, JObject r)
|
||||
{
|
||||
var id = r["torrentId"];
|
||||
release.Size = (long)r["size"];
|
||||
release.Seeders = (int)r["seeders"];
|
||||
release.Peers = (int)r["leechers"] + release.Seeders;
|
||||
release.Guid = new Uri(GuidUrl + id);
|
||||
release.Comments = release.Guid;
|
||||
|
||||
if ((string)r["category"] == "TV")
|
||||
{
|
||||
release.Category = TorznabCatType.TV.ID;
|
||||
}
|
||||
else if ((string)r["category"] == "Movies")
|
||||
{
|
||||
release.Category = TorznabCatType.Movies.ID;
|
||||
}
|
||||
|
||||
release.Link = new Uri(DownloadUrl + id);
|
||||
}
|
||||
|
||||
public async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
|
||||
{
|
||||
var isTv = Array.IndexOf(query.Categories, TorznabCatType.TV.ID) > -1;
|
||||
var releases = new List<ReleaseInfo>();
|
||||
string qryString = query.GetQueryString();
|
||||
var searchQuery = query.GetQueryString();
|
||||
|
||||
Match matchQry = new Regex(@".*\s[Ss]{1}\d{2}$").Match(qryString);
|
||||
if (matchQry.Success)
|
||||
await GetReleases(releases, query, searchQuery);
|
||||
|
||||
// Search for torrent groups
|
||||
if (isTv)
|
||||
{
|
||||
//If search string ends in S## eg. S03 (season search) add an asterix to search term
|
||||
qryString += "*";
|
||||
var seasonMatch = new Regex(@".*\s[Ss]{1}\d{2}").Match(query.GetQueryString());
|
||||
if (seasonMatch.Success)
|
||||
{
|
||||
var newSearchQuery = Regex.Replace(searchQuery, @"[Ss]{1}\d{2}", $"Season {query.Season}");
|
||||
|
||||
await GetReleases(releases, query, newSearchQuery);
|
||||
}
|
||||
}
|
||||
|
||||
var episodeSearchUrl = SearchUrl + HttpUtility.UrlEncode(qryString);
|
||||
WebClientStringResult response = await RequestStringWithCookiesAndRetry(episodeSearchUrl);
|
||||
return releases;
|
||||
}
|
||||
|
||||
private string GetTorrentSearchUrl(int[] categories, string searchQuery)
|
||||
{
|
||||
var extra = "";
|
||||
|
||||
if (Array.IndexOf(categories, TorznabCatType.Movies.ID) > -1)
|
||||
extra += "&filter_cat%5B1%5D=1";
|
||||
|
||||
if (Array.IndexOf(categories, TorznabCatType.TV.ID) > -1)
|
||||
extra += "&filter_cat%5B2%5D=1";
|
||||
|
||||
return SiteLink + $"torrents.php?searchstr={HttpUtility.UrlEncode(searchQuery)}&tags_type=1&order_by=time&order_way=desc&group_results=1{extra}&action=basic&searchsubmit=1";
|
||||
}
|
||||
|
||||
private async Task GetReleases(ICollection<ReleaseInfo> releases, TorznabQuery query, string searchQuery)
|
||||
{
|
||||
var searchUrl = GetTorrentSearchUrl(query.Categories, searchQuery);
|
||||
var response = await RequestStringWithCookiesAndRetry(searchUrl);
|
||||
|
||||
try
|
||||
{
|
||||
string decodedResponse = WebUtility.HtmlDecode(response.Content);
|
||||
var json = JObject.Parse(decodedResponse);
|
||||
foreach (JObject r in json["response"]["results"])
|
||||
var parser = new HtmlParser();
|
||||
var document = parser.Parse(response.Content);
|
||||
var groups = document.QuerySelectorAll(".torrent_table > tbody > tr.group");
|
||||
var torrents = document.QuerySelectorAll(".torrent_table > tbody > tr.torrent");
|
||||
|
||||
// Loop through all torrent (season) groups
|
||||
foreach (var group in groups)
|
||||
{
|
||||
DateTime pubDate = DateTime.MinValue;
|
||||
double dateNum;
|
||||
if (double.TryParse((string)r["groupTime"], out dateNum))
|
||||
var showName = group.QuerySelector(".tp-showname a").InnerHtml.Replace("(", "").Replace(")", "").Replace(' ', '.');
|
||||
var season = group.QuerySelector(".big_info a").InnerHtml;
|
||||
|
||||
// Loop through all group items
|
||||
var previousElement = group;
|
||||
var qualityEdition = string.Empty;
|
||||
while (true)
|
||||
{
|
||||
pubDate = DateTimeUtil.UnixTimestampToDateTime(dateNum);
|
||||
pubDate = DateTime.SpecifyKind(pubDate, DateTimeKind.Utc).ToLocalTime();
|
||||
var groupItem = previousElement.NextElementSibling;
|
||||
|
||||
if (groupItem == null) break;
|
||||
|
||||
if (!groupItem.ClassList[0].Equals("group_torrent") ||
|
||||
!groupItem.ClassList[1].StartsWith("groupid_")) break;
|
||||
|
||||
// Found a new edition
|
||||
if (groupItem.ClassList[2].Equals("edition"))
|
||||
{
|
||||
qualityEdition = groupItem.QuerySelector(".edition_info strong").TextContent.Split('/')[1].Trim();
|
||||
}
|
||||
|
||||
string groupName = (string)r["groupName"];
|
||||
|
||||
if (r["torrents"] is JArray)
|
||||
else if (groupItem.ClassList[2].StartsWith("edition_"))
|
||||
{
|
||||
string showName = (string) r["artist"];
|
||||
if (qualityEdition.Equals(string.Empty)) break;
|
||||
|
||||
foreach (JObject t in r["torrents"])
|
||||
// Parse required data
|
||||
var downloadAnchor = groupItem.QuerySelectorAll("td a").Last();
|
||||
var qualityData = downloadAnchor.InnerHtml.Split('/');
|
||||
|
||||
if (qualityData.Length < 2)
|
||||
throw new Exception($"We expected 2 or more quality datas, instead we have {qualityData.Length}.");
|
||||
|
||||
// Build title
|
||||
var title = string.Join(".", new List<string>
|
||||
{
|
||||
var release = new ReleaseInfo();
|
||||
release.PublishDate = pubDate;
|
||||
release.Title = $"{showName} {groupName}";
|
||||
release.Description = $"{showName} {groupName}";
|
||||
FillReleaseInfoFromJson(release, t);
|
||||
releases.Add(release);
|
||||
}
|
||||
showName,
|
||||
SeasonToShortSeason(season),
|
||||
qualityData[1].Trim(),
|
||||
qualityEdition, // Audio quality should be after this one. Unobtainable at the moment.
|
||||
$"{qualityData[0].Trim()}-MTV"
|
||||
});
|
||||
|
||||
releases.Add(GetReleaseInfo(groupItem, downloadAnchor, title, TorznabCatType.TV.ID));
|
||||
}
|
||||
else
|
||||
{
|
||||
var release = new ReleaseInfo();
|
||||
release.PublishDate = pubDate;
|
||||
release.Title = groupName;
|
||||
release.Description = groupName;
|
||||
FillReleaseInfoFromJson(release, r);
|
||||
releases.Add(release);
|
||||
break;
|
||||
}
|
||||
|
||||
previousElement = groupItem;
|
||||
}
|
||||
}
|
||||
|
||||
// Loop through all torrents
|
||||
foreach (var torrent in torrents)
|
||||
{
|
||||
// Parse required data
|
||||
var downloadAnchor = torrent.QuerySelector(".big_info > .group_info > a");
|
||||
var title = downloadAnchor.TextContent;
|
||||
|
||||
int category;
|
||||
var categories = torrent.QuerySelector(".cats_col div").ClassList;
|
||||
if (categories.Contains("cats_tv"))
|
||||
{
|
||||
category = TorznabCatType.TV.ID;
|
||||
}
|
||||
else if (categories.Contains("cats_movies"))
|
||||
{
|
||||
category = TorznabCatType.Movies.ID;
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new Exception("Couldn't find category.");
|
||||
}
|
||||
|
||||
releases.Add(GetReleaseInfo(torrent, downloadAnchor, title, category));
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
OnParseError(response.Content, ex);
|
||||
}
|
||||
}
|
||||
|
||||
private ReleaseInfo GetReleaseInfo(IElement row, IElement downloadAnchor, string title, int category)
|
||||
{
|
||||
// Parse required data
|
||||
var downloadAnchorHref = downloadAnchor.Attributes["href"].Value;
|
||||
var torrentId = downloadAnchorHref.Substring(downloadAnchorHref.LastIndexOf('=') + 1);
|
||||
var publishDate = DateTime.ParseExact(row.QuerySelector(".time.tooltip").Attributes["title"].Value, "MMM dd yyyy, HH:mm", CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal);
|
||||
var torrentData = row.QuerySelectorAll(".number_column"); // Size (xx.xx GB[ (Max)]) Snatches (xx) Seeders (xx) Leechers (xx)
|
||||
|
||||
if (torrentData.Length != 4)
|
||||
throw new Exception($"We expected 4 torrent datas, instead we have {torrentData.Length}.");
|
||||
|
||||
if (torrentId.Contains('#'))
|
||||
torrentId = torrentId.Split('#')[0];
|
||||
|
||||
var size = ParseSizeToBytes(torrentData[0].TextContent);
|
||||
var seeders = int.Parse(torrentData[2].TextContent);
|
||||
var guid = new Uri(GuidUrl + torrentId);
|
||||
|
||||
// Build releaseinfo
|
||||
return new ReleaseInfo
|
||||
{
|
||||
Title = title,
|
||||
Description = title,
|
||||
Category = category, // Who seasons movies right
|
||||
Link = new Uri(DownloadUrl + torrentId),
|
||||
PublishDate = publishDate,
|
||||
Seeders = seeders,
|
||||
Peers = seeders,
|
||||
Size = size,
|
||||
Guid = guid,
|
||||
Comments = guid
|
||||
};
|
||||
}
|
||||
|
||||
// Changes "Season 1" to "S01"
|
||||
private static string SeasonToShortSeason(string season)
|
||||
{
|
||||
var seasonMatch = new Regex(@"Season (?<seasonNumber>\d{1,2})").Match(season);
|
||||
if (seasonMatch.Success)
|
||||
{
|
||||
season = $"S{int.Parse(seasonMatch.Groups["seasonNumber"].Value):00}";
|
||||
}
|
||||
|
||||
return season;
|
||||
}
|
||||
|
||||
// Changes "xx.xx GB/MB" to bytes
|
||||
private static long ParseSizeToBytes(string strSize)
|
||||
{
|
||||
var sizeParts = strSize.Split(' ');
|
||||
if (sizeParts.Length != 2)
|
||||
throw new Exception($"We expected 2 size parts, instead we have {sizeParts.Length}.");
|
||||
|
||||
var size = double.Parse(sizeParts[0]);
|
||||
|
||||
switch (sizeParts[1].Trim())
|
||||
{
|
||||
case "GB":
|
||||
size = size*1000*1000*1000;
|
||||
break;
|
||||
case "MB":
|
||||
size = size*1000*1000;
|
||||
break;
|
||||
default:
|
||||
throw new Exception($"Unknown size type {sizeParts[1].Trim()}.");
|
||||
}
|
||||
|
||||
return (long) Math.Ceiling(size);
|
||||
}
|
||||
|
||||
return releases;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue