mirror/Jackett
1
0
Fork 0
mirror of https://github.com/Jackett/Jackett synced 2025-01-01 04:38:20 +00:00
Code Issues Releases Wiki Activity

FileList: Add API functionality. Resolves #7004 Resolves #5190 (#7987)

Browse source
This commit is contained in:
IIIspaceIII 2020-04-18 05:18:09 +03:00 committed by GitHub
parent dcc5527dd6
commit 4d07daaebe
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 128 additions and 141 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

261
src/Jackett.Common/Indexers/FileList.cs
View file

@ -1,12 +1,8 @@
using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Globalization;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using AngleSharp.Html.Parser;
using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig.Bespoke;
using Jackett.Common.Services.Interfaces;
@ -19,27 +15,27 @@ namespace Jackett.Common.Indexers
{
public class FileList : BaseWebIndexer
{
public override string[] LegacySiteLinks { get; protected set; } = {
public override string[] LegacySiteLinks { get; protected set; } =
{
"http://filelist.ro/",
"https://filelist.ro/",
"https://flro.org/",
"http://flro.org/",
};
private string LoginUrl => SiteLink + "takelogin.php";
private string BrowseUrl => SiteLink + "browse.php";
private string ApiUrl => SiteLink + "api.php";
private string DetailsUrl => SiteLink + "details.php";
private new ConfigurationDataFileList configData
{
get => (ConfigurationDataFileList)base.configData;
set => base.configData = value;
}
private new ConfigurationDataFileList configData => (ConfigurationDataFileList)base.configData;
public FileList(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps)
: base(name: "FileList",
: base("FileList",
description: "The best Romanian site.",
link: "https://filelist.io/",
caps: TorznabUtil.CreateDefaultTorznabTVCaps(),
caps: new TorznabCapabilities
{
SupportsImdbMovieSearch = true
},
configService: configService,
client: wc,
logger: l,
@ -50,161 +46,152 @@ namespace Jackett.Common.Indexers
Language = "ro-ro";
Type = "private";
TorznabCaps.SupportsImdbMovieSearch = true;
AddCategoryMapping(24, TorznabCatType.TVAnime, "Anime");
AddCategoryMapping(11, TorznabCatType.Audio, "Audio");
AddCategoryMapping(15, TorznabCatType.TV, "Desene");
AddCategoryMapping(18, TorznabCatType.Other, "Diverse");
AddCategoryMapping(16, TorznabCatType.Books, "Docs");
AddCategoryMapping(25, TorznabCatType.Movies3D, "Filme 3D");
AddCategoryMapping(6, TorznabCatType.MoviesHD, "Filme 4K");
AddCategoryMapping(26, TorznabCatType.MoviesBluRay, "Filme 4K Blu-Ray");
AddCategoryMapping(20, TorznabCatType.MoviesBluRay, "Filme Blu-Ray");
AddCategoryMapping(1, TorznabCatType.MoviesSD, "Filme SD");
AddCategoryMapping(2, TorznabCatType.MoviesDVD, "Filme DVD");
AddCategoryMapping(3, TorznabCatType.MoviesForeign, "Filme DVD-RO");
AddCategoryMapping(4, TorznabCatType.MoviesHD, "Filme HD");
AddCategoryMapping(19, TorznabCatType.MoviesForeign, "Filme HD-RO");
AddCategoryMapping(1, TorznabCatType.MoviesSD, "Filme SD");
AddCategoryMapping(5, TorznabCatType.AudioLossless, "FLAC");
AddCategoryMapping(10, TorznabCatType.Console, "Jocuri Console");
AddCategoryMapping(9, TorznabCatType.PCGames, "Jocuri PC");
AddCategoryMapping(17, TorznabCatType.PC, "Linux");
AddCategoryMapping(22, TorznabCatType.PCPhoneOther, "Mobile");
AddCategoryMapping(8, TorznabCatType.PC, "Programe");
AddCategoryMapping(27, TorznabCatType.TVHD, "Seriale 4K");
AddCategoryMapping(21, TorznabCatType.TVHD, "Seriale HD");
AddCategoryMapping(23, TorznabCatType.TVSD, "Seriale SD");
AddCategoryMapping(13, TorznabCatType.TVSport, "Sport");
AddCategoryMapping(12, TorznabCatType.AudioVideo, "Videoclip");
AddCategoryMapping(6, TorznabCatType.MoviesUHD, "Filme 4K");
AddCategoryMapping(7, TorznabCatType.XXX, "XXX");
AddCategoryMapping(8, TorznabCatType.PC, "Programe");
AddCategoryMapping(9, TorznabCatType.PCGames, "Jocuri PC");
AddCategoryMapping(10, TorznabCatType.Console, "Jocuri Console");
AddCategoryMapping(11, TorznabCatType.Audio, "Audio");
AddCategoryMapping(12, TorznabCatType.AudioVideo, "Videoclip");
AddCategoryMapping(13, TorznabCatType.TVSport, "Sport");
AddCategoryMapping(13, TorznabCatType.TVSport, "Sport");
AddCategoryMapping(15, TorznabCatType.TV, "Desene");
AddCategoryMapping(16, TorznabCatType.Books, "Docs");
AddCategoryMapping(17, TorznabCatType.PC, "Linux");
AddCategoryMapping(18, TorznabCatType.Other, "Diverse");
AddCategoryMapping(19, TorznabCatType.MoviesForeign, "Filme HD-RO");
AddCategoryMapping(20, TorznabCatType.MoviesBluRay, "Filme Blu-Ray");
AddCategoryMapping(21, TorznabCatType.TVHD, "Seriale HD");
AddCategoryMapping(22, TorznabCatType.PCPhoneOther, "Mobile");
AddCategoryMapping(23, TorznabCatType.TVSD, "Seriale SD");
AddCategoryMapping(24, TorznabCatType.TVAnime, "Anime");
AddCategoryMapping(25, TorznabCatType.Movies3D, "Filme 3D");
AddCategoryMapping(26, TorznabCatType.MoviesBluRay, "Filme 4K Blu-Ray");
AddCategoryMapping(27, TorznabCatType.TVUHD, "Seriale 4K");
}
public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
{
LoadValuesFromJson(configJson);
var responseFirstPage = await RequestStringWithCookiesAndRetry(SiteLink + "login.php?returnto=%2F", "");
var parser = new HtmlParser();
var domFirstPage = parser.ParseDocument(responseFirstPage.Content);
var validator = domFirstPage.QuerySelector("input[name =\"validator\"]").GetAttribute("value");
var pairs = new Dictionary<string, string> {
{ "validator", validator},
{ "username", configData.Username.Value },
{ "password", configData.Password.Value }
};
logger.Info("Testing provider filelist...");
var pingResponse = await CallProviderAsync(new TorznabQuery());
var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, responseFirstPage.Cookies, true, null, LoginUrl);
await ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("logout.php"), () =>
try
{
var dom = parser.ParseDocument(result.Content);
var errorMessage = dom.QuerySelector(".main").TextContent.Trim();
throw new ExceptionWithConfigData(errorMessage, configData);
});
var json = JArray.Parse(pingResponse);
if (json.Count > 0)
{
IsConfigured = true;
SaveConfig();
return IndexerConfigurationStatus.Completed;
}
}
catch (Exception ex)
{
throw new ExceptionWithConfigData(ex.Message, configData);
}
return IndexerConfigurationStatus.RequiresTesting;
}
protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
{
var releases = new List<ReleaseInfo>();
var searchUrl = BrowseUrl;
var searchString = query.GetQueryString();
var cat = MapTorznabCapsToTrackers(query).FirstIfSingleOrDefault("0");
var response = await CallProviderAsync(query);
var queryCollection = new NameValueCollection();
if (response.StartsWith("{\"error\""))
throw new ExceptionWithConfigData(response, configData);
if (query.ImdbID != null)
queryCollection.Add("search", query.ImdbID);
else if (!string.IsNullOrWhiteSpace(searchString))
queryCollection.Add("search", searchString);
queryCollection.Add("cat", cat);
queryCollection.Add("searchin", "1");
queryCollection.Add("sort", "2");
searchUrl += "?" + queryCollection.GetQueryString();
var response = await RequestStringWithCookiesAndRetry(searchUrl, null, BrowseUrl);
// Occasionally the cookies become invalid, login again if that happens
if (response.IsRedirect)
{
await ApplyConfiguration(null);
response = await RequestStringWithCookiesAndRetry(searchUrl, null, BrowseUrl);
}
var results = response.Content;
try
{
var parser = new HtmlParser();
var dom = parser.ParseDocument(results);
var globalFreeLeech = dom.QuerySelectorAll("div.globalFreeLeech").Any();
var rows = dom.QuerySelectorAll(".torrentrow");
foreach (var row in rows)
var json = JArray.Parse(response);
foreach (var row in json)
{
var release = new ReleaseInfo();
var qTitleLink = row.QuerySelector(".torrenttable:nth-of-type(2) a");
release.Title = row.QuerySelector(".torrenttable:nth-of-type(2) b").TextContent;
var longtitle = row.QuerySelector(".torrenttable:nth-of-type(2) a[title]").GetAttribute("title");
if (!string.IsNullOrEmpty(longtitle) && !longtitle.Contains("<")) // releases with cover image have no full title
release.Title = longtitle;
if (query.ImdbID == null && !query.MatchQueryStringAND(release.Title))
continue;
release.Description = row.QuerySelector(".torrenttable:nth-of-type(2) > span > font.small")?.TextContent;
var tooltip = qTitleLink.GetAttribute("title");
if (!string.IsNullOrEmpty(tooltip))
var detailsUri = new Uri(DetailsUrl + "?id=" + (string)row["id"]);
var seeders = (int)row["seeders"];
var peers = seeders + (int)row["leechers"];
var publishDate = DateTimeUtil.FromFuzzyTime((string)row["upload_date"] + " +0200");
var downloadVolumeFactor = (int)row["freeleech"] == 1 ? 0 : 1;
var imdbId = ((JObject)row).ContainsKey("imdb") ? ParseUtil.GetImdbID((string)row["imdb"]) : null;
var link = new Uri((string)row["download_link"]);
var release = new ReleaseInfo
{
var imgRegexp = new Regex("src='(.*?)'");
var imgRegexpMatch = imgRegexp.Match(tooltip);
if (imgRegexpMatch.Success)
release.BannerUrl = new Uri(imgRegexpMatch.Groups[1].Value);
}
release.Guid = new Uri(SiteLink + qTitleLink.GetAttribute("href"));
release.Comments = release.Guid;
//22:05:3716/02/2013
var dateStr = row.QuerySelector(".torrenttable:nth-of-type(6)").TextContent.Trim() + " +0200";
release.PublishDate = DateTime.ParseExact(dateStr, "H:mm:ssdd/MM/yyyy zzz", CultureInfo.InvariantCulture);
var qLink = row.QuerySelector("a[href^=\"download.php?id=\"]");
release.Link = new Uri(SiteLink + qLink.GetAttribute("href").Replace("&usetoken=1", ""));
var sizeStr = row.QuerySelector(".torrenttable:nth-of-type(7)").TextContent.Trim();
release.Size = ReleaseInfo.GetBytes(sizeStr);
var grabs = row.QuerySelector(".torrenttable:nth-of-type(8)").TextContent.Replace("times", "").Trim();
release.Grabs = ParseUtil.CoerceLong(grabs);
release.Seeders = ParseUtil.CoerceInt(row.QuerySelector(".torrenttable:nth-of-type(9)").TextContent.Trim());
release.Peers = ParseUtil.CoerceInt(row.QuerySelector(".torrenttable:nth-of-type(10)").TextContent.Trim()) + release.Seeders;
var catId = row.QuerySelector(".torrenttable:nth-of-type(1) a").GetAttribute("href").Substring(15);
release.Category = MapTrackerCatToNewznab(catId);
if (globalFreeLeech || row.QuerySelectorAll("img[alt=\"FreeLeech\"]").Any())
release.DownloadVolumeFactor = 0;
else
release.DownloadVolumeFactor = 1;
release.UploadVolumeFactor = 1;
// Skip Romanian releases
if (release.Category.Contains(TorznabCatType.MoviesForeign.ID) && !configData.IncludeRomanianReleases.Value)
continue;
Title = (string)row["name"],
Comments = detailsUri,
Link = link,
Category = MapTrackerCatDescToNewznab((string)row["category"]),
Size = (long)row["size"],
Files = (long)row["files"],
Grabs = (long)row["times_completed"],
Seeders = seeders,
Peers = peers,
MinimumRatio = 1,
MinimumSeedTime = 172800, //48 hours
PublishDate = publishDate,
UploadVolumeFactor = 1,
DownloadVolumeFactor = downloadVolumeFactor,
Guid = detailsUri,
Imdb = imdbId
};
releases.Add(release);
}
return releases;
}
catch (Exception ex)
{
OnParseError(results, ex);
OnParseError(response, ex);
}
return releases;
}
private async Task<string> CallProviderAsync(TorznabQuery query)
{
var searchUrl = ApiUrl;
var searchString = query.GetQueryString();
var cat = string.Join(",", MapTorznabCapsToTrackers(query));
var queryCollection = new NameValueCollection
{
{"category", cat}
};
if (query.IsImdbQuery)
{
queryCollection.Add("type", "imdb");
queryCollection.Add("query", query.ImdbID);
queryCollection.Add("action", "search-torrents");
}
else if (!string.IsNullOrWhiteSpace(searchString))
{
queryCollection.Add("type", "name");
queryCollection.Add("query", searchString);
queryCollection.Add("action", "search-torrents");
}
else
queryCollection.Add("action", "latest-torrents");
searchUrl += "?" + queryCollection.GetQueryString();
try
{
var auth = Convert.ToBase64String(Encoding.UTF8.GetBytes(configData.Username.Value + ":" + configData.Passkey.Value));
var headers = new Dictionary<string, string>
{
{"Authorization", "Basic " + auth}
};
var response = await RequestStringWithCookies(searchUrl, headers: headers);
return response.Content;
}
catch (Exception inner)
{
throw new Exception("Error calling provider filelist", inner);
}
}
}
}

8
src/Jackett.Common/Models/IndexerConfig/Bespoke/ConfigurationDataFileList.cs
View file

@ -1,15 +1,15 @@
namespace Jackett.Common.Models.IndexerConfig.Bespoke
{
internal class ConfigurationDataFileList : ConfigurationDataBasicLogin
internal class ConfigurationDataFileList : ConfigurationDataUserPasskey
{
public BoolItem IncludeRomanianReleases { get; private set; }
public DisplayItem CatWarning { get; private set; }
public ConfigurationDataFileList()
: base()
: base("Go into your filelist profile and copy the passkey.")
{
IncludeRomanianReleases = new BoolItem() { Name = "IncludeRomanianReleases", Value = false };
CatWarning = new DisplayItem("When mapping TV ensure you add category 5000 in addition to 5030,5040.") { Name = "CatWarning" };
IncludeRomanianReleases = new BoolItem {Name = "IncludeRomanianReleases", Value = false};
CatWarning = new DisplayItem("When mapping TV ensure you add category 5000 in addition to 5030, 5040.") {Name = "CatWarning"};
}
}
}