From 6b1dba2dc3a5bc3032143ec6f9a942da3a235b26 Mon Sep 17 00:00:00 2001 From: some-guy-23 Date: Mon, 21 Dec 2015 20:03:50 -0500 Subject: [PATCH] Added RSS feed to RevolutionTT --- src/Jackett/Indexers/RevolutionTT.cs | 230 ++++++++++++++++++++------- 1 file changed, 169 insertions(+), 61 deletions(-) diff --git a/src/Jackett/Indexers/RevolutionTT.cs b/src/Jackett/Indexers/RevolutionTT.cs index c39baa02d..923219d84 100644 --- a/src/Jackett/Indexers/RevolutionTT.cs +++ b/src/Jackett/Indexers/RevolutionTT.cs @@ -15,6 +15,8 @@ using System.Text; using System.Threading.Tasks; using System.Web; using Jackett.Models.IndexerConfig; +using System.Text.RegularExpressions; +using System.Xml.Linq; namespace Jackett.Indexers { @@ -22,11 +24,15 @@ namespace Jackett.Indexers { private string LandingPageURL { get { return SiteLink + "login.php"; } } private string LoginUrl { get { return SiteLink + "takelogin.php"; } } - private string SearchUrl { get { return SiteLink + "browse.php"; } } + private string GetRSSKeyUrl { get { return SiteLink + "getrss.php"; } } + private string RSSUrl { get { return SiteLink + "rss.php?feed=dl&passkey="; } } + private string SearchUrl { get { return SiteLink + "browse.php"; } } + private string DetailsURL { get { return SiteLink + "details.php?id={0}&hit=1"; } } - new ConfigurationDataBasicLogin configData + + new ConfigurationDataBasicLoginWithRSS configData { - get { return (ConfigurationDataBasicLogin)base.configData; } + get { return (ConfigurationDataBasicLoginWithRSS)base.configData; } set { base.configData = value; } } @@ -40,7 +46,7 @@ namespace Jackett.Indexers logger: l, p: ps, downloadBase: "https://revolutiontt.me/download.php/", - configData: new ConfigurationDataBasicLogin()) + configData: new ConfigurationDataBasicLoginWithRSS()) { /* Original RevolutionTT Categories - @@ -146,6 +152,35 @@ namespace Jackett.Indexers //AddCategoryMapping("cat_id", TorznabCatType.BooksTechnical); //AddCategoryMapping("cat_id", TorznabCatType.BooksOther); //AddCategoryMapping("cat_id", TorznabCatType.BooksForeign); + + // RSS Textual categories + AddCategoryMapping("Anime", TorznabCatType.TVAnime); + AddCategoryMapping("Appz/Misc", TorznabCatType.PC0day); + AddCategoryMapping("Appz/PC-ISO", TorznabCatType.Books); + AddCategoryMapping("E-Book", TorznabCatType.BooksEbook); + AddCategoryMapping("Games/PC-ISO", TorznabCatType.PCGames); + AddCategoryMapping("Games/PC-Rips", TorznabCatType.PCGames); + AddCategoryMapping("Games/PS3", TorznabCatType.ConsolePS3); + AddCategoryMapping("Games/Wii", TorznabCatType.ConsoleWii); + AddCategoryMapping("Games/XBOX360", TorznabCatType.ConsoleXbox360); + AddCategoryMapping("Handheld/NDS", TorznabCatType.ConsoleNDS); + AddCategoryMapping("Handheld/PSP", TorznabCatType.ConsolePSP); + AddCategoryMapping("Mac", TorznabCatType.PCMac); + AddCategoryMapping("Movies/BluRay", TorznabCatType.MoviesBluRay); + AddCategoryMapping("Movies/DVDR", TorznabCatType.MoviesDVD); + AddCategoryMapping("Movies/HDx264", TorznabCatType.MoviesHD); + AddCategoryMapping("Movies/Packs", TorznabCatType.Movies); + AddCategoryMapping("Movies/SDx264", TorznabCatType.MoviesSD); + AddCategoryMapping("Movies/XviD", TorznabCatType.MoviesSD); + AddCategoryMapping("Music", TorznabCatType.Audio); + AddCategoryMapping("Music/FLAC", TorznabCatType.AudioLossless); + AddCategoryMapping("Music/Packs", TorznabCatType.AudioOther); + AddCategoryMapping("MusicVideos", TorznabCatType.AudioVideo); + AddCategoryMapping("TV/DVDR", TorznabCatType.TV); + AddCategoryMapping("TV/HDx264", TorznabCatType.TVHD); + AddCategoryMapping("TV/Packs", TorznabCatType.TV); + AddCategoryMapping("TV/SDx264", TorznabCatType.TVSD); + AddCategoryMapping("TV/XViD", TorznabCatType.TVSD); } public async Task ApplyConfiguration(JToken configJson) @@ -157,22 +192,38 @@ namespace Jackett.Indexers { "password", configData.Password.Value } }; - //--need to do an initial request to get PHP session cookie (any better way to do this?) + // need to do an initial request to get PHP session cookie (any better way to do this?) var homePageLoad = await RequestLoginAndFollowRedirect(LandingPageURL, new Dictionary { }, null, true, null, SiteLink); - var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, homePageLoad.Cookies, true, null, LoginUrl); + var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, homePageLoad.Cookies, true, null, LandingPageURL); await ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("/logout.php"), () => { CQ dom = result.Content; var messageEl = dom[".error"]; var errorMessage = messageEl.Text().Trim(); - //--CloudFlare error? - if (errorMessage == "") - { - errorMessage = result.Content; - } throw new ExceptionWithConfigData(errorMessage, configData); }); + + // Store RSS key from feed generator page + try + { + var rssParams = new Dictionary { + { "feed", "dl" } + }; + var rssPage = await PostDataWithCookies(GetRSSKeyUrl, rssParams, result.Cookies); + var match = Regex.Match(rssPage.Content, "(?<=passkey\\=)([a-zA-z0-9]*)"); + configData.RSSKey.Value = match.Success ? match.Value : string.Empty; + if (string.IsNullOrWhiteSpace(configData.RSSKey.Value)) + throw new Exception("Failed to get RSS Key"); + SaveConfig(); + } + catch (Exception e) + { + IsConfigured = false; + throw e; + } + + return IndexerConfigurationStatus.RequiresTesting; } @@ -182,69 +233,126 @@ namespace Jackett.Indexers var searchString = query.GetQueryString(); var searchUrl = SearchUrl; - if (!string.IsNullOrWhiteSpace(searchString)) + // If query is empty, use the RSS Feed + if (string.IsNullOrWhiteSpace(searchString)) { - searchUrl += "?titleonly=1&search=" + HttpUtility.UrlEncode(searchString); - } - string.Format(SearchUrl, HttpUtility.UrlEncode(searchString)); + var rssPage = await RequestStringWithCookiesAndRetry(RSSUrl + configData.RSSKey.Value); + var rssDoc = XDocument.Parse(rssPage.Content); - var cats = MapTorznabCapsToTrackers(query); - if (cats.Count > 0) - { - foreach (var cat in cats) + foreach (var item in rssDoc.Descendants("item")) { - searchUrl += "&c" + cat + "=1"; - } - } + var title = item.Descendants("title").First().Value; + var description = item.Descendants("description").First().Value; + var link = item.Descendants("link").First().Value; + var date = item.Descendants("pubDate").First().Value; - var results = await RequestStringWithCookiesAndRetry(searchUrl); - try - { - CQ dom = results.Content; + var torrentIdMatch = Regex.Match(link, "(?<=download\\.php/)([a-zA-z0-9]*)"); + var torrentId = torrentIdMatch.Success ? torrentIdMatch.Value : string.Empty; + if (string.IsNullOrWhiteSpace(torrentId)) + throw new Exception("Missing torrent id"); - //--table header is the first in table body, get all rows except this - CQ qRows = dom["#torrents-table > tbody > tr:not(:first-child)"]; + var infoMatch = Regex.Match(description, @"Category:\W(?.*)\W\n\WSize:\W(?.*)\n\WStatus:\W(?.*)\Wseeder(.*)\Wand\W(?.*)\Wleecher(.*)\n\WAdded:\W(?.*)\n\WDescription:"); + if (!infoMatch.Success) + throw new Exception("Unable to find info"); - foreach (var row in qRows) - { - var release = new ReleaseInfo(); + var imdbMatch = Regex.Match(description, "(?<=http://www.imdb.com/title/tt)([0-9]*)"); + long? imdbID = null; + if(imdbMatch.Success) + { + long l; + if(long.TryParse(imdbMatch.Value, out l)) + { + imdbID = l; + } + } - var qRow = row.Cq(); + var release = new ReleaseInfo() + { + Title = title, + Description = title, + Guid = new Uri(string.Format(DetailsURL, torrentId)), + Comments = new Uri(string.Format(DetailsURL, torrentId) + "&tocomm=1"), + PublishDate = DateTime.ParseExact(infoMatch.Groups["added"].Value, "yyyy-MM-dd H:mm:ss", CultureInfo.InvariantCulture), //2015-08-08 21:20:31 + Link = new Uri(link), + Seeders = ParseUtil.CoerceInt(infoMatch.Groups["seeders"].Value == "no" ? "0" : infoMatch.Groups["seeders"].Value), + Peers = ParseUtil.CoerceInt(infoMatch.Groups["leechers"].Value == "no" ? "0" : infoMatch.Groups["leechers"].Value), + Size = ReleaseInfo.GetBytes(infoMatch.Groups["size"].Value), + Category = MapTrackerCatToNewznab(infoMatch.Groups["cat"].Value), + Imdb = imdbID + }; - var debug = qRow.Html(); - - release.MinimumRatio = 1; - release.MinimumSeedTime = 172800; - - CQ qLink = qRow.Find(".br_right > a").First(); - release.Guid = new Uri(SiteLink + qLink.Attr("href")); - release.Comments = new Uri(SiteLink + qLink.Attr("href") + "&tocomm=1"); - release.Title = qLink.Find("b").Text(); - release.Description = release.Title; - - release.Link = new Uri(SiteLink + qRow.Find("td:nth-child(4) > a").Attr("href")); - - var dateString = qRow.Find("td:nth-child(6) nobr")[0].InnerText.Trim(); - //"2015-04-25 23:38:12" - //"yyyy-MMM-dd hh:mm:ss" - release.PublishDate = DateTime.ParseExact(dateString, "yyyy-MM-ddHH:mm:ss", CultureInfo.InvariantCulture); - - var sizeStr = qRow.Children().ElementAt(6).InnerText.Trim(); - release.Size = ReleaseInfo.GetBytes(sizeStr); - - release.Seeders = ParseUtil.CoerceInt(qRow.Find("td:nth-child(9)").Text()); - release.Peers = release.Seeders + ParseUtil.CoerceInt(qRow.Find("td:nth-child(10)").Text()); - - var category = qRow.Find(".br_type > a").Attr("href").Replace("browse.php?cat=", string.Empty); - release.Category = MapTrackerCatToNewznab(category); + // if unknown category, set to "other" + if (release.Category == 0) + release.Category = 7000; + release.Peers += release.Seeders; releases.Add(release); } - } - catch (Exception ex) + } + else { - OnParseError(results.Content, ex); + + searchUrl += "?titleonly=1&search=" + HttpUtility.UrlEncode(searchString); + string.Format(SearchUrl, HttpUtility.UrlEncode(searchString)); + + var cats = MapTorznabCapsToTrackers(query); + if (cats.Count > 0) + { + foreach (var cat in cats) + { + searchUrl += "&c" + cat + "=1"; + } + } + + var results = await RequestStringWithCookiesAndRetry(searchUrl); + try + { + CQ dom = results.Content; + + // table header is the first in table body, get all rows except this + CQ qRows = dom["#torrents-table > tbody > tr:not(:first-child)"]; + + foreach (var row in qRows) + { + var release = new ReleaseInfo(); + + var qRow = row.Cq(); + + var debug = qRow.Html(); + + release.MinimumRatio = 1; + release.MinimumSeedTime = 172800; + + CQ qLink = qRow.Find(".br_right > a").First(); + release.Guid = new Uri(SiteLink + qLink.Attr("href")); + release.Comments = new Uri(SiteLink + qLink.Attr("href") + "&tocomm=1"); + release.Title = qLink.Find("b").Text(); + release.Description = release.Title; + + release.Link = new Uri(SiteLink + qRow.Find("td:nth-child(4) > a").Attr("href")); + + var dateString = qRow.Find("td:nth-child(6) nobr")[0].InnerText.Trim(); + //"2015-04-25 23:38:12" + //"yyyy-MMM-dd hh:mm:ss" + release.PublishDate = DateTime.ParseExact(dateString, "yyyy-MM-ddHH:mm:ss", CultureInfo.InvariantCulture); + + var sizeStr = qRow.Children().ElementAt(6).InnerText.Trim(); + release.Size = ReleaseInfo.GetBytes(sizeStr); + + release.Seeders = ParseUtil.CoerceInt(qRow.Find("td:nth-child(9)").Text()); + release.Peers = release.Seeders + ParseUtil.CoerceInt(qRow.Find("td:nth-child(10)").Text()); + + var category = qRow.Find(".br_type > a").Attr("href").Replace("browse.php?cat=", string.Empty); + release.Category = MapTrackerCatToNewznab(category); + + releases.Add(release); + } + } + catch (Exception ex) + { + OnParseError(results.Content, ex); + } } return releases;