Rewrote MoreThanTV indexer (#477)

* MTV: Rewrote searching for torrents. Modified title output so sonarr can actually download seasons.

* MTV: Add individual torrent parsing and cleanup.

* MTV: Remove TODO as it's already done.

src/Jackett/Indexers/MoreThanTV.cs

@@ -1,36 +1,30 @@
-using CsQuery;
+using Jackett.Models;
-using Jackett.Models;
 using Jackett.Services;
-using Jackett.Utils;
 using Jackett.Utils.Clients;
 using Newtonsoft.Json.Linq;
 using NLog;
 using System;
 using System.Collections.Generic;
-using System.IO;
+using System.Globalization;
 using System.Linq;
-using System.Net;
-using System.Net.Http;
-using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using System.Web;
+using AngleSharp.Dom;
+using AngleSharp.Parser.Html;
+using CsQuery;
 using Jackett.Models.IndexerConfig;
 
 namespace Jackett.Indexers
 {
     public class MoreThanTV : BaseIndexer, IIndexer
     {
-        private string LoginUrl { get { return SiteLink + "login.php"; } }
+        private string LoginUrl => SiteLink + "login.php";
-        private string SearchUrl { get { return SiteLink + "ajax.php?action=browse&searchstr="; } }
+        private string SearchUrl => SiteLink + "ajax.php?action=browse&searchstr=";
-        private string DownloadUrl { get { return SiteLink + "torrents.php?action=download&id="; } }
+        private string DownloadUrl => SiteLink + "torrents.php?action=download&id=";
-        private string GuidUrl { get { return SiteLink + "torrents.php?torrentid="; } }
+        private string GuidUrl => SiteLink + "torrents.php?torrentid=";
 
-        new ConfigurationDataBasicLogin configData
+        private ConfigurationDataBasicLogin ConfigData => (ConfigurationDataBasicLogin) configData;
-        {
-            get { return (ConfigurationDataBasicLogin)base.configData; }
-            set { base.configData = value; }
-        }
 
         public MoreThanTV(IIndexerManagerService i, IWebClient c, Logger l, IProtectionService ps)
             : base(name: "MoreThanTV",
@@ -48,111 +42,231 @@ namespace Jackett.Indexers
 
         public async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
         {
-            configData.LoadValuesFromJson(configJson);
+            ConfigData.LoadValuesFromJson(configJson);
             var pairs = new Dictionary<string, string> {
-                { "username", configData.Username.Value },
+                { "username", ConfigData.Username.Value },
-                { "password", configData.Password.Value },
+                { "password", ConfigData.Password.Value },
                 { "login", "Log in" },
                 { "keeplogged", "1" }
             };
 
             var preRequest = await RequestStringWithCookiesAndRetry(LoginUrl, string.Empty);
-
             var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, preRequest.Cookies, true, SearchUrl, SiteLink);
+
             await ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("status\":\"success\""), () =>
             {
                 CQ dom = result.Content;
                 dom["#loginform > table"].Remove();
                 var errorMessage = dom["#loginform"].Text().Trim().Replace("\n\t", " ");
-                throw new ExceptionWithConfigData(errorMessage, configData);
+                throw new ExceptionWithConfigData(errorMessage, ConfigData);
             });
 
             return IndexerConfigurationStatus.RequiresTesting;
         }
 
-        private void FillReleaseInfoFromJson(ReleaseInfo release, JObject r)
-        {
-            var id = r["torrentId"];
-            release.Size = (long)r["size"];
-            release.Seeders = (int)r["seeders"];
-            release.Peers = (int)r["leechers"] + release.Seeders;
-            release.Guid = new Uri(GuidUrl + id);
-            release.Comments = release.Guid;
-
-            if ((string)r["category"] == "TV")
-            {
-                release.Category = TorznabCatType.TV.ID;
-            }
-            else if ((string)r["category"] == "Movies")
-            {
-                release.Category = TorznabCatType.Movies.ID;
-            }
-
-            release.Link = new Uri(DownloadUrl + id);
-        }
-
         public async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
         {
+            var isTv = Array.IndexOf(query.Categories, TorznabCatType.TV.ID) > -1;
             var releases = new List<ReleaseInfo>();
-            string qryString = query.GetQueryString();
+            var searchQuery = query.GetQueryString();
 
-            Match matchQry = new Regex(@".*\s[Ss]{1}\d{2}$").Match(qryString);
+            await GetReleases(releases, query, searchQuery);
-            if (matchQry.Success)
+
+            // Search for torrent groups
+            if (isTv)
             {
-                //If search string ends in S## eg. S03 (season search) add an asterix to search term
+                var seasonMatch = new Regex(@".*\s[Ss]{1}\d{2}").Match(query.GetQueryString());
-                qryString += "*";
+                if (seasonMatch.Success)
+                {
+                    var newSearchQuery = Regex.Replace(searchQuery, @"[Ss]{1}\d{2}", $"Season {query.Season}");
+
+                    await GetReleases(releases, query, newSearchQuery);
+                }
             }
 
-            var episodeSearchUrl = SearchUrl + HttpUtility.UrlEncode(qryString);
+            return releases;
-            WebClientStringResult response = await RequestStringWithCookiesAndRetry(episodeSearchUrl);
+        }
+
+        private string GetTorrentSearchUrl(int[] categories, string searchQuery)
+        {
+            var extra = "";
+
+            if (Array.IndexOf(categories, TorznabCatType.Movies.ID) > -1)
+                extra += "&filter_cat%5B1%5D=1";
+
+            if (Array.IndexOf(categories, TorznabCatType.TV.ID) > -1)
+                extra += "&filter_cat%5B2%5D=1";
+
+            return SiteLink + $"torrents.php?searchstr={HttpUtility.UrlEncode(searchQuery)}&tags_type=1&order_by=time&order_way=desc&group_results=1{extra}&action=basic&searchsubmit=1";
+        }
+
+        private async Task GetReleases(ICollection<ReleaseInfo> releases, TorznabQuery query, string searchQuery)
+        {
+            var searchUrl = GetTorrentSearchUrl(query.Categories, searchQuery);
+            var response = await RequestStringWithCookiesAndRetry(searchUrl);
 
             try
             {
-                string decodedResponse = WebUtility.HtmlDecode(response.Content);
+                var parser = new HtmlParser();
-                var json = JObject.Parse(decodedResponse);
+                var document = parser.Parse(response.Content);
-                foreach (JObject r in json["response"]["results"])
+                var groups = document.QuerySelectorAll(".torrent_table > tbody > tr.group");
+                var torrents = document.QuerySelectorAll(".torrent_table > tbody > tr.torrent");
+
+                // Loop through all torrent (season) groups
+                foreach (var group in groups)
                 {
-                    DateTime pubDate = DateTime.MinValue;
+                    var showName = group.QuerySelector(".tp-showname a").InnerHtml.Replace("(", "").Replace(")", "").Replace(' ', '.');
-                    double dateNum;
+                    var season = group.QuerySelector(".big_info a").InnerHtml;
-                    if (double.TryParse((string)r["groupTime"], out dateNum))
+
+                    // Loop through all group items
+                    var previousElement = group;
+                    var qualityEdition = string.Empty;
+                    while (true)
                     {
-                        pubDate = DateTimeUtil.UnixTimestampToDateTime(dateNum);
+                        var groupItem = previousElement.NextElementSibling;
-                        pubDate = DateTime.SpecifyKind(pubDate, DateTimeKind.Utc).ToLocalTime();
+
+                        if (groupItem == null) break;
+
+                        if (!groupItem.ClassList[0].Equals("group_torrent") ||
+                            !groupItem.ClassList[1].StartsWith("groupid_")) break;
+
+                        // Found a new edition
+                        if (groupItem.ClassList[2].Equals("edition"))
+                        {
+                            qualityEdition = groupItem.QuerySelector(".edition_info strong").TextContent.Split('/')[1].Trim();
                         }
-
+                        else if (groupItem.ClassList[2].StartsWith("edition_"))
-                    string groupName = (string)r["groupName"];
-
-                    if (r["torrents"] is JArray)
                         {
-                        string showName = (string) r["artist"];
+                            if (qualityEdition.Equals(string.Empty)) break;
 
-                        foreach (JObject t in r["torrents"])
+                            // Parse required data
+                            var downloadAnchor = groupItem.QuerySelectorAll("td a").Last();
+                            var qualityData = downloadAnchor.InnerHtml.Split('/');
+
+                            if (qualityData.Length < 2)
+                                throw new Exception($"We expected 2 or more quality datas, instead we have {qualityData.Length}.");
+
+                            // Build title
+                            var title = string.Join(".", new List<string>
                             {
-                            var release = new ReleaseInfo();
+                                showName,
-                            release.PublishDate = pubDate;
+                                SeasonToShortSeason(season),
-                            release.Title = $"{showName} {groupName}";
+                                qualityData[1].Trim(),
-                            release.Description = $"{showName} {groupName}";
+                                qualityEdition, // Audio quality should be after this one. Unobtainable at the moment.
-                            FillReleaseInfoFromJson(release, t);
+                                $"{qualityData[0].Trim()}-MTV"
-                            releases.Add(release);
+                            });
-                        }
+                            
+                            releases.Add(GetReleaseInfo(groupItem, downloadAnchor, title, TorznabCatType.TV.ID));
                         }
                         else
                         {
-                        var release = new ReleaseInfo();
+                            break;
-                        release.PublishDate = pubDate;
-                        release.Title = groupName;
-                        release.Description = groupName;
-                        FillReleaseInfoFromJson(release, r);
-                        releases.Add(release);
                         }
+
+                        previousElement = groupItem;
+                    }
+                }
+
+                // Loop through all torrents
+                foreach (var torrent in torrents)
+                {
+                    // Parse required data
+                    var downloadAnchor = torrent.QuerySelector(".big_info > .group_info > a");
+                    var title = downloadAnchor.TextContent;
+
+                    int category;
+                    var categories = torrent.QuerySelector(".cats_col div").ClassList;
+                    if (categories.Contains("cats_tv"))
+                    {
+                        category = TorznabCatType.TV.ID;
+                    }
+                    else if (categories.Contains("cats_movies"))
+                    {
+                        category = TorznabCatType.Movies.ID;
+                    }
+                    else
+                    {
+                        throw new Exception("Couldn't find category.");
+                    }
+
+                    releases.Add(GetReleaseInfo(torrent, downloadAnchor, title, category));
                 }
             }
             catch (Exception ex)
             {
                 OnParseError(response.Content, ex);
             }
-
-            return releases;
         }
+
+        private ReleaseInfo GetReleaseInfo(IElement row, IElement downloadAnchor, string title, int category)
+        {
+            // Parse required data
+            var downloadAnchorHref = downloadAnchor.Attributes["href"].Value;
+            var torrentId = downloadAnchorHref.Substring(downloadAnchorHref.LastIndexOf('=') + 1);
+            var publishDate = DateTime.ParseExact(row.QuerySelector(".time.tooltip").Attributes["title"].Value, "MMM dd yyyy, HH:mm", CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal);
+            var torrentData = row.QuerySelectorAll(".number_column"); // Size (xx.xx GB[ (Max)]) Snatches (xx) Seeders (xx) Leechers (xx)
+
+            if (torrentData.Length != 4)
+                throw new Exception($"We expected 4 torrent datas, instead we have {torrentData.Length}.");
+
+            if (torrentId.Contains('#'))
+                torrentId = torrentId.Split('#')[0];
+
+            var size = ParseSizeToBytes(torrentData[0].TextContent);
+            var seeders = int.Parse(torrentData[2].TextContent);
+            var guid = new Uri(GuidUrl + torrentId);
+
+            // Build releaseinfo
+            return new ReleaseInfo
+            {
+                Title = title,
+                Description = title,
+                Category = category, // Who seasons movies right
+                Link = new Uri(DownloadUrl + torrentId),
+                PublishDate = publishDate,
+                Seeders = seeders,
+                Peers = seeders,
+                Size = size,
+                Guid = guid,
+                Comments = guid
+            };
+        }
+
+        // Changes "Season 1" to "S01"
+        private static string SeasonToShortSeason(string season)
+        {
+            var seasonMatch = new Regex(@"Season (?<seasonNumber>\d{1,2})").Match(season);
+            if (seasonMatch.Success)
+            {
+                season = $"S{int.Parse(seasonMatch.Groups["seasonNumber"].Value):00}";
+            }
+
+            return season;
+        }
+
+        // Changes "xx.xx GB/MB" to bytes
+        private static long ParseSizeToBytes(string strSize)
+        {
+            var sizeParts = strSize.Split(' ');
+            if (sizeParts.Length != 2)
+                throw new Exception($"We expected 2 size parts, instead we have {sizeParts.Length}.");
+
+            var size = double.Parse(sizeParts[0]);
+
+            switch (sizeParts[1].Trim())
+            {
+                case "GB":
+                    size = size*1000*1000*1000;
+                    break;
+                case "MB":
+                    size = size*1000*1000;
+                    break;
+                default:
+                    throw new Exception($"Unknown size type {sizeParts[1].Trim()}.");
+            }
+
+            return (long) Math.Ceiling(size);
+        }
+
     }
 }