From 4d07daaebe8f4aaf02813637d4a47d443df61a26 Mon Sep 17 00:00:00 2001 From: IIIspaceIII Date: Sat, 18 Apr 2020 05:18:09 +0300 Subject: [PATCH] FileList: Add API functionality. Resolves #7004 Resolves #5190 (#7987) --- src/Jackett.Common/Indexers/FileList.cs | 261 +++++++++--------- .../Bespoke/ConfigurationDataFileList.cs | 8 +- 2 files changed, 128 insertions(+), 141 deletions(-) diff --git a/src/Jackett.Common/Indexers/FileList.cs b/src/Jackett.Common/Indexers/FileList.cs index 6d18d2272..18171ac55 100644 --- a/src/Jackett.Common/Indexers/FileList.cs +++ b/src/Jackett.Common/Indexers/FileList.cs @@ -1,12 +1,8 @@ using System; using System.Collections.Generic; using System.Collections.Specialized; -using System.Globalization; -using System.Linq; using System.Text; -using System.Text.RegularExpressions; using System.Threading.Tasks; -using AngleSharp.Html.Parser; using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig.Bespoke; using Jackett.Common.Services.Interfaces; @@ -19,27 +15,27 @@ namespace Jackett.Common.Indexers { public class FileList : BaseWebIndexer { - public override string[] LegacySiteLinks { get; protected set; } = { + public override string[] LegacySiteLinks { get; protected set; } = + { "http://filelist.ro/", "https://filelist.ro/", "https://flro.org/", "http://flro.org/", }; - private string LoginUrl => SiteLink + "takelogin.php"; - private string BrowseUrl => SiteLink + "browse.php"; + private string ApiUrl => SiteLink + "api.php"; + private string DetailsUrl => SiteLink + "details.php"; - private new ConfigurationDataFileList configData - { - get => (ConfigurationDataFileList)base.configData; - set => base.configData = value; - } + private new ConfigurationDataFileList configData => (ConfigurationDataFileList)base.configData; public FileList(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps) - : base(name: "FileList", + : base("FileList", description: "The best Romanian site.", link: "https://filelist.io/", - caps: TorznabUtil.CreateDefaultTorznabTVCaps(), + caps: new TorznabCapabilities + { + SupportsImdbMovieSearch = true + }, configService: configService, client: wc, logger: l, @@ -50,161 +46,152 @@ namespace Jackett.Common.Indexers Language = "ro-ro"; Type = "private"; - TorznabCaps.SupportsImdbMovieSearch = true; - - AddCategoryMapping(24, TorznabCatType.TVAnime, "Anime"); - AddCategoryMapping(11, TorznabCatType.Audio, "Audio"); - AddCategoryMapping(15, TorznabCatType.TV, "Desene"); - AddCategoryMapping(18, TorznabCatType.Other, "Diverse"); - AddCategoryMapping(16, TorznabCatType.Books, "Docs"); - AddCategoryMapping(25, TorznabCatType.Movies3D, "Filme 3D"); - AddCategoryMapping(6, TorznabCatType.MoviesHD, "Filme 4K"); - AddCategoryMapping(26, TorznabCatType.MoviesBluRay, "Filme 4K Blu-Ray"); - AddCategoryMapping(20, TorznabCatType.MoviesBluRay, "Filme Blu-Ray"); + AddCategoryMapping(1, TorznabCatType.MoviesSD, "Filme SD"); AddCategoryMapping(2, TorznabCatType.MoviesDVD, "Filme DVD"); AddCategoryMapping(3, TorznabCatType.MoviesForeign, "Filme DVD-RO"); AddCategoryMapping(4, TorznabCatType.MoviesHD, "Filme HD"); - AddCategoryMapping(19, TorznabCatType.MoviesForeign, "Filme HD-RO"); - AddCategoryMapping(1, TorznabCatType.MoviesSD, "Filme SD"); AddCategoryMapping(5, TorznabCatType.AudioLossless, "FLAC"); - AddCategoryMapping(10, TorznabCatType.Console, "Jocuri Console"); - AddCategoryMapping(9, TorznabCatType.PCGames, "Jocuri PC"); - AddCategoryMapping(17, TorznabCatType.PC, "Linux"); - AddCategoryMapping(22, TorznabCatType.PCPhoneOther, "Mobile"); - AddCategoryMapping(8, TorznabCatType.PC, "Programe"); - AddCategoryMapping(27, TorznabCatType.TVHD, "Seriale 4K"); - AddCategoryMapping(21, TorznabCatType.TVHD, "Seriale HD"); - AddCategoryMapping(23, TorznabCatType.TVSD, "Seriale SD"); - AddCategoryMapping(13, TorznabCatType.TVSport, "Sport"); - AddCategoryMapping(12, TorznabCatType.AudioVideo, "Videoclip"); + AddCategoryMapping(6, TorznabCatType.MoviesUHD, "Filme 4K"); AddCategoryMapping(7, TorznabCatType.XXX, "XXX"); + AddCategoryMapping(8, TorznabCatType.PC, "Programe"); + AddCategoryMapping(9, TorznabCatType.PCGames, "Jocuri PC"); + AddCategoryMapping(10, TorznabCatType.Console, "Jocuri Console"); + AddCategoryMapping(11, TorznabCatType.Audio, "Audio"); + AddCategoryMapping(12, TorznabCatType.AudioVideo, "Videoclip"); + AddCategoryMapping(13, TorznabCatType.TVSport, "Sport"); + AddCategoryMapping(13, TorznabCatType.TVSport, "Sport"); + AddCategoryMapping(15, TorznabCatType.TV, "Desene"); + AddCategoryMapping(16, TorznabCatType.Books, "Docs"); + AddCategoryMapping(17, TorznabCatType.PC, "Linux"); + AddCategoryMapping(18, TorznabCatType.Other, "Diverse"); + AddCategoryMapping(19, TorznabCatType.MoviesForeign, "Filme HD-RO"); + AddCategoryMapping(20, TorznabCatType.MoviesBluRay, "Filme Blu-Ray"); + AddCategoryMapping(21, TorznabCatType.TVHD, "Seriale HD"); + AddCategoryMapping(22, TorznabCatType.PCPhoneOther, "Mobile"); + AddCategoryMapping(23, TorznabCatType.TVSD, "Seriale SD"); + AddCategoryMapping(24, TorznabCatType.TVAnime, "Anime"); + AddCategoryMapping(25, TorznabCatType.Movies3D, "Filme 3D"); + AddCategoryMapping(26, TorznabCatType.MoviesBluRay, "Filme 4K Blu-Ray"); + AddCategoryMapping(27, TorznabCatType.TVUHD, "Seriale 4K"); } public override async Task ApplyConfiguration(JToken configJson) { LoadValuesFromJson(configJson); - var responseFirstPage = await RequestStringWithCookiesAndRetry(SiteLink + "login.php?returnto=%2F", ""); - var parser = new HtmlParser(); - var domFirstPage = parser.ParseDocument(responseFirstPage.Content); - var validator = domFirstPage.QuerySelector("input[name =\"validator\"]").GetAttribute("value"); - var pairs = new Dictionary { - { "validator", validator}, - { "username", configData.Username.Value }, - { "password", configData.Password.Value } - }; + logger.Info("Testing provider filelist..."); + var pingResponse = await CallProviderAsync(new TorznabQuery()); - var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, responseFirstPage.Cookies, true, null, LoginUrl); - await ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("logout.php"), () => + try { - var dom = parser.ParseDocument(result.Content); - var errorMessage = dom.QuerySelector(".main").TextContent.Trim(); - throw new ExceptionWithConfigData(errorMessage, configData); - }); + var json = JArray.Parse(pingResponse); + if (json.Count > 0) + { + IsConfigured = true; + SaveConfig(); + return IndexerConfigurationStatus.Completed; + } + } + catch (Exception ex) + { + throw new ExceptionWithConfigData(ex.Message, configData); + } + return IndexerConfigurationStatus.RequiresTesting; } protected override async Task> PerformQuery(TorznabQuery query) { var releases = new List(); - var searchUrl = BrowseUrl; - var searchString = query.GetQueryString(); - var cat = MapTorznabCapsToTrackers(query).FirstIfSingleOrDefault("0"); + var response = await CallProviderAsync(query); - var queryCollection = new NameValueCollection(); + if (response.StartsWith("{\"error\"")) + throw new ExceptionWithConfigData(response, configData); - if (query.ImdbID != null) - queryCollection.Add("search", query.ImdbID); - else if (!string.IsNullOrWhiteSpace(searchString)) - queryCollection.Add("search", searchString); - - queryCollection.Add("cat", cat); - queryCollection.Add("searchin", "1"); - queryCollection.Add("sort", "2"); - - searchUrl += "?" + queryCollection.GetQueryString(); - - var response = await RequestStringWithCookiesAndRetry(searchUrl, null, BrowseUrl); - - // Occasionally the cookies become invalid, login again if that happens - if (response.IsRedirect) - { - await ApplyConfiguration(null); - response = await RequestStringWithCookiesAndRetry(searchUrl, null, BrowseUrl); - } - - var results = response.Content; try { - var parser = new HtmlParser(); - var dom = parser.ParseDocument(results); - var globalFreeLeech = dom.QuerySelectorAll("div.globalFreeLeech").Any(); - var rows = dom.QuerySelectorAll(".torrentrow"); - foreach (var row in rows) + var json = JArray.Parse(response); + foreach (var row in json) { - var release = new ReleaseInfo(); - - var qTitleLink = row.QuerySelector(".torrenttable:nth-of-type(2) a"); - release.Title = row.QuerySelector(".torrenttable:nth-of-type(2) b").TextContent; - var longtitle = row.QuerySelector(".torrenttable:nth-of-type(2) a[title]").GetAttribute("title"); - if (!string.IsNullOrEmpty(longtitle) && !longtitle.Contains("<")) // releases with cover image have no full title - release.Title = longtitle; - - if (query.ImdbID == null && !query.MatchQueryStringAND(release.Title)) - continue; - - release.Description = row.QuerySelector(".torrenttable:nth-of-type(2) > span > font.small")?.TextContent; - - var tooltip = qTitleLink.GetAttribute("title"); - if (!string.IsNullOrEmpty(tooltip)) + var detailsUri = new Uri(DetailsUrl + "?id=" + (string)row["id"]); + var seeders = (int)row["seeders"]; + var peers = seeders + (int)row["leechers"]; + var publishDate = DateTimeUtil.FromFuzzyTime((string)row["upload_date"] + " +0200"); + var downloadVolumeFactor = (int)row["freeleech"] == 1 ? 0 : 1; + var imdbId = ((JObject)row).ContainsKey("imdb") ? ParseUtil.GetImdbID((string)row["imdb"]) : null; + var link = new Uri((string)row["download_link"]); + var release = new ReleaseInfo { - var imgRegexp = new Regex("src='(.*?)'"); - var imgRegexpMatch = imgRegexp.Match(tooltip); - if (imgRegexpMatch.Success) - release.BannerUrl = new Uri(imgRegexpMatch.Groups[1].Value); - } - - release.Guid = new Uri(SiteLink + qTitleLink.GetAttribute("href")); - release.Comments = release.Guid; - - //22:05:3716/02/2013 - var dateStr = row.QuerySelector(".torrenttable:nth-of-type(6)").TextContent.Trim() + " +0200"; - release.PublishDate = DateTime.ParseExact(dateStr, "H:mm:ssdd/MM/yyyy zzz", CultureInfo.InvariantCulture); - - var qLink = row.QuerySelector("a[href^=\"download.php?id=\"]"); - release.Link = new Uri(SiteLink + qLink.GetAttribute("href").Replace("&usetoken=1", "")); - - var sizeStr = row.QuerySelector(".torrenttable:nth-of-type(7)").TextContent.Trim(); - release.Size = ReleaseInfo.GetBytes(sizeStr); - - var grabs = row.QuerySelector(".torrenttable:nth-of-type(8)").TextContent.Replace("times", "").Trim(); - release.Grabs = ParseUtil.CoerceLong(grabs); - - release.Seeders = ParseUtil.CoerceInt(row.QuerySelector(".torrenttable:nth-of-type(9)").TextContent.Trim()); - release.Peers = ParseUtil.CoerceInt(row.QuerySelector(".torrenttable:nth-of-type(10)").TextContent.Trim()) + release.Seeders; - - var catId = row.QuerySelector(".torrenttable:nth-of-type(1) a").GetAttribute("href").Substring(15); - release.Category = MapTrackerCatToNewznab(catId); - - if (globalFreeLeech || row.QuerySelectorAll("img[alt=\"FreeLeech\"]").Any()) - release.DownloadVolumeFactor = 0; - else - release.DownloadVolumeFactor = 1; - - release.UploadVolumeFactor = 1; - - // Skip Romanian releases - if (release.Category.Contains(TorznabCatType.MoviesForeign.ID) && !configData.IncludeRomanianReleases.Value) - continue; - + Title = (string)row["name"], + Comments = detailsUri, + Link = link, + Category = MapTrackerCatDescToNewznab((string)row["category"]), + Size = (long)row["size"], + Files = (long)row["files"], + Grabs = (long)row["times_completed"], + Seeders = seeders, + Peers = peers, + MinimumRatio = 1, + MinimumSeedTime = 172800, //48 hours + PublishDate = publishDate, + UploadVolumeFactor = 1, + DownloadVolumeFactor = downloadVolumeFactor, + Guid = detailsUri, + Imdb = imdbId + }; releases.Add(release); } + + return releases; } catch (Exception ex) { - OnParseError(results, ex); + OnParseError(response, ex); } return releases; } + + private async Task CallProviderAsync(TorznabQuery query) + { + var searchUrl = ApiUrl; + var searchString = query.GetQueryString(); + var cat = string.Join(",", MapTorznabCapsToTrackers(query)); + var queryCollection = new NameValueCollection + { + {"category", cat} + }; + + if (query.IsImdbQuery) + { + queryCollection.Add("type", "imdb"); + queryCollection.Add("query", query.ImdbID); + queryCollection.Add("action", "search-torrents"); + } + else if (!string.IsNullOrWhiteSpace(searchString)) + { + queryCollection.Add("type", "name"); + queryCollection.Add("query", searchString); + queryCollection.Add("action", "search-torrents"); + } + else + queryCollection.Add("action", "latest-torrents"); + + searchUrl += "?" + queryCollection.GetQueryString(); + try + { + var auth = Convert.ToBase64String(Encoding.UTF8.GetBytes(configData.Username.Value + ":" + configData.Passkey.Value)); + var headers = new Dictionary + { + {"Authorization", "Basic " + auth} + }; + var response = await RequestStringWithCookies(searchUrl, headers: headers); + return response.Content; + } + catch (Exception inner) + { + throw new Exception("Error calling provider filelist", inner); + } + } } } + diff --git a/src/Jackett.Common/Models/IndexerConfig/Bespoke/ConfigurationDataFileList.cs b/src/Jackett.Common/Models/IndexerConfig/Bespoke/ConfigurationDataFileList.cs index fa14fa99b..9f735e51a 100644 --- a/src/Jackett.Common/Models/IndexerConfig/Bespoke/ConfigurationDataFileList.cs +++ b/src/Jackett.Common/Models/IndexerConfig/Bespoke/ConfigurationDataFileList.cs @@ -1,15 +1,15 @@ namespace Jackett.Common.Models.IndexerConfig.Bespoke { - internal class ConfigurationDataFileList : ConfigurationDataBasicLogin + internal class ConfigurationDataFileList : ConfigurationDataUserPasskey { public BoolItem IncludeRomanianReleases { get; private set; } public DisplayItem CatWarning { get; private set; } public ConfigurationDataFileList() - : base() + : base("Go into your filelist profile and copy the passkey.") { - IncludeRomanianReleases = new BoolItem() { Name = "IncludeRomanianReleases", Value = false }; - CatWarning = new DisplayItem("When mapping TV ensure you add category 5000 in addition to 5030,5040.") { Name = "CatWarning" }; + IncludeRomanianReleases = new BoolItem {Name = "IncludeRomanianReleases", Value = false}; + CatWarning = new DisplayItem("When mapping TV ensure you add category 5000 in addition to 5030, 5040.") {Name = "CatWarning"}; } } }