From d703488ccc2278ad49dc77fa513eae7faa3c1cb9 Mon Sep 17 00:00:00 2001 From: kaso17 Date: Tue, 31 Jan 2017 19:37:28 +0100 Subject: [PATCH] RARBG: change to torrentapi.org --- src/Jackett.Updater/Program.cs | 1 + src/Jackett/Definitions/rarbg.yml | 91 ------------ src/Jackett/Indexers/Rarbg.cs | 230 ++++++++++++++++++++++++++++++ src/Jackett/Jackett.csproj | 4 +- 4 files changed, 232 insertions(+), 94 deletions(-) delete mode 100644 src/Jackett/Definitions/rarbg.yml create mode 100644 src/Jackett/Indexers/Rarbg.cs diff --git a/src/Jackett.Updater/Program.cs b/src/Jackett.Updater/Program.cs index 71394f1bb..a152ce167 100644 --- a/src/Jackett.Updater/Program.cs +++ b/src/Jackett.Updater/Program.cs @@ -182,6 +182,7 @@ namespace Jackett.Updater "Content/css/jquery.dataTables_themeroller.css", "Definitions/tspate.yml", "Definitions/freakstrackingsystem.yml", + "Definitions/rarbg.yml", }; foreach (var oldFIle in oldFiles) diff --git a/src/Jackett/Definitions/rarbg.yml b/src/Jackett/Definitions/rarbg.yml deleted file mode 100644 index 47a7f70e8..000000000 --- a/src/Jackett/Definitions/rarbg.yml +++ /dev/null @@ -1,91 +0,0 @@ ---- - site: rarbg - name: RARBG - language: en-us - type: public - encoding: UTF-8 - links: - - https://rarbg.to/ - - caps: - categories: - 4: XXX # XXX (18+) - 14: Movies/SD # Movies/XVID - 48: Movies/HD # Movies/XVID/720 - 17: Movies/SD # Movies/x264 - 44: Movies/HD # Movies/x264/1080 - 45: Movies/HD # Movies/x264/720 - 47: Movies/3D # Movies/x264/3D - 42: Movies/BluRay # Movies/Full BD - 46: Movies/HD # Movies/BD Remux - 18: TV/SD # TV Episodes - 41: TV/HD # TV HD Episodes - 23: Audio/MP3 # Music/MP3 - 25: Audio/Lossless # Music/FLAC - 27: PC/Games # Games/PC ISO - 28: PC/Games # Games/PC RIP - 40: Console/PS3 # Games/PS3 - 32: Console/Xbox # Games/XBOX-360 - 33: PC/0day # Software/PC ISO - 35: Books # e-Books - - modes: - search: [q] - tv-search: [q, season, ep] - movie-search: [q, imdbid] - - settings: [] - - search: - path: /torrents.php - inputs: - $raw: "{{range .Categories}}category[]={{.}}&{{end}}" - search: "{{if .Query.IMDBID}}{{ .Query.IMDBID }}{{else}}{{ .Query.Keywords }}{{end}}" - - rows: - selector: table.lista2t > tbody > tr.lista2 - fields: - download: - selector: a[href^="/torrent/"] - attribute: href - filters: - - name: replace - args: ["/torrent/", "/download.php?id="] - - name: append - args: "&f=dummy.torrent" - title: - selector: a[href^="/torrent/"] - details: - selector: a[href^="/torrent/"] - attribute: href - banner: - selector: a[href^="/torrent/"] - attribute: onmouseover - filters: - - name: regexp - args: src=\\'(.+?)\\' - imdb|optional: - selector: a[href^="/torrents.php?imdb="] - downloadvolumefactor: - text: "0" - uploadvolumefactor: - text: "1" - category: - selector: a[href^="/torrents.php?category="] - attribute: href - filters: - - name: querystring - args: category - size: - selector: td:nth-child(4) - seeders: - selector: td:nth-child(5) - leechers: - selector: td:nth-child(6) - date: - selector: td:nth-child(3) - filters: - - name: append - args: " +01:00" - - name: dateparse - args: "2006-01-02 15:04:05 -07:00" diff --git a/src/Jackett/Indexers/Rarbg.cs b/src/Jackett/Indexers/Rarbg.cs new file mode 100644 index 000000000..4393c10d8 --- /dev/null +++ b/src/Jackett/Indexers/Rarbg.cs @@ -0,0 +1,230 @@ +using Jackett.Models; +using Jackett.Models.IndexerConfig; +using Jackett.Services; +using Jackett.Utils; +using Jackett.Utils.Clients; +using Newtonsoft.Json.Linq; +using NLog; +using System; +using System.Collections.Generic; +using System.Collections.Specialized; +using System.Globalization; +using System.Linq; +using System.Threading.Tasks; +using System.Text; +using System.Web; + +namespace Jackett.Indexers +{ + public class Rarbg : BaseIndexer, IIndexer + { + readonly static string defaultSiteLink = "https://torrentapi.org/"; + + private Uri BaseUri + { + get { return new Uri(configData.Url.Value); } + set { configData.Url.Value = value.ToString(); } + } + + private string ApiEndpoint { get { return BaseUri + "pubapi_v2.php"; } } + + new ConfigurationDataUrl configData + { + get { return (ConfigurationDataUrl)base.configData; } + set { base.configData = value; } + } + + private DateTime lastTokenFetch; + private string token; + + readonly TimeSpan TOKEN_DURATION = TimeSpan.FromMinutes(10); + + private bool HasValidToken { get { return !string.IsNullOrEmpty(token) && lastTokenFetch > DateTime.Now - TOKEN_DURATION; } } + + public Rarbg(IIndexerManagerService i, IWebClient wc, Logger l, IProtectionService ps) + : base(name: "RARBG", + description: null, + link: "https://rarbg.to/", + caps: new TorznabCapabilities(), + manager: i, + client: wc, + logger: l, + p: ps, + configData: new ConfigurationDataUrl(defaultSiteLink)) + { + Encoding = Encoding.GetEncoding("windows-1252"); + Language = "en-us"; + Type = "public"; + + TorznabCaps.SupportsImdbSearch = true; + + AddCategoryMapping(4, TorznabCatType.XXX, "XXX (18+)"); + AddCategoryMapping(14, TorznabCatType.MoviesSD, "Movies/XVID"); + AddCategoryMapping(48, TorznabCatType.MoviesHD, "Movies/XVID/720"); + AddCategoryMapping(17, TorznabCatType.MoviesSD, "Movies/x264"); + AddCategoryMapping(44, TorznabCatType.MoviesHD, "Movies/x264/1080"); + AddCategoryMapping(45, TorznabCatType.MoviesHD, "Movies/x264/720"); + AddCategoryMapping(47, TorznabCatType.Movies3D, "Movies/x264/3D"); + AddCategoryMapping(42, TorznabCatType.MoviesBluRay, "Movies/Full BD"); + AddCategoryMapping(46, TorznabCatType.MoviesBluRay, "Movies/BD Remux"); + AddCategoryMapping(18, TorznabCatType.TVSD, "TV Episodes"); + AddCategoryMapping(41, TorznabCatType.TVHD, "TV HD Episodes"); + AddCategoryMapping(23, TorznabCatType.AudioMP3, "Music/MP3"); + AddCategoryMapping(25, TorznabCatType.AudioLossless, "Music/FLAC"); + AddCategoryMapping(27, TorznabCatType.PCGames, "Games/PC ISO"); + AddCategoryMapping(28, TorznabCatType.PCGames, "Games/PC RIP"); + AddCategoryMapping(40, TorznabCatType.ConsolePS3, "Games/PS3"); + AddCategoryMapping(32, TorznabCatType.ConsoleXbox360, "Games/XBOX-360"); + AddCategoryMapping(33, TorznabCatType.PCISO, "Software/PC ISO"); + AddCategoryMapping(35, TorznabCatType.BooksEbook, "e-Books"); + } + + + async Task CheckToken() + { + if (!HasValidToken) + { + var queryCollection = new NameValueCollection(); + queryCollection.Add("get_token", "get_token"); + + var tokenUrl = ApiEndpoint + "?" + queryCollection.GetQueryString(); + + var result = await RequestStringWithCookiesAndRetry(tokenUrl); + var json = JObject.Parse(result.Content); + token = json.Value("token"); + lastTokenFetch = DateTime.Now; + } + } + + public async Task ApplyConfiguration(JToken configJson) + { + configData.LoadValuesFromJson(configJson); + var releases = await PerformQuery(new TorznabQuery()); + + await ConfigureIfOK(string.Empty, releases.Count() > 0, () => + { + throw new Exception("Could not find releases from this URL"); + }); + + return IndexerConfigurationStatus.Completed; + } + + public Task> PerformQuery(TorznabQuery query) + { + return PerformQuery(query, 0); + } + + public async Task> PerformQuery(TorznabQuery query, int attempts = 0) + { + await CheckToken(); + var releases = new List(); + var searchString = query.GetQueryString(); + + var queryCollection = new NameValueCollection(); + queryCollection.Add("token", token); + queryCollection.Add("format", "json_extended"); + queryCollection.Add("app_id", "jackett_v" + Engine.ConfigService.GetVersion()); + queryCollection.Add("limit", "100"); + queryCollection.Add("ranked", "0"); + + if (query.ImdbID != null) + { + queryCollection.Add("mode", "search"); + queryCollection.Add("search_imdb", query.ImdbID); + } + else if (query.RageID != null) + { + queryCollection.Add("mode", "search"); + queryCollection.Add("search_tvrage", query.RageID.ToString()); + } + /*else if (query.TvdbID != null) + { + queryCollection.Add("mode", "search"); + queryCollection.Add("search_tvdb", query.TvdbID); + }*/ + else if (!string.IsNullOrWhiteSpace(searchString)) + { + queryCollection.Add("mode", "search"); + queryCollection.Add("search_string", searchString); + } + else + { + queryCollection.Add("mode", "list"); + } + + var cats = string.Join(";", MapTorznabCapsToTrackers(query)); + if (!string.IsNullOrEmpty(cats)) + { + queryCollection.Add("category", cats); + } + + var searchUrl = ApiEndpoint + "?" + queryCollection.GetQueryString(); + var response = await RequestStringWithCookiesAndRetry(searchUrl, string.Empty); + + try + { + var jsonContent = JObject.Parse(response.Content); + + int errorCode = jsonContent.Value("error_code"); + if (errorCode == 20) // no results found + { + return releases.ToArray(); + } + + if (errorCode > 0) // too many requests per second + { + if (attempts < 3) + { + await Task.Delay(TimeSpan.FromSeconds(2)); + return await PerformQuery(query, ++attempts); + } + else + { + throw new Exception(jsonContent.Value("error")); + } + } + + foreach (var item in jsonContent.Value("torrent_results")) + { + var release = new ReleaseInfo(); + release.Title = item.Value("title"); + release.Category = MapTrackerCatDescToNewznab(item.Value("category")); + + release.MagnetUri = new Uri(item.Value("download")); + release.InfoHash = release.MagnetUri.ToString().Split(':')[3].Split('&')[0]; + + release.Comments = new Uri(item.Value("info_page")); + release.Guid = release.Comments; + + var episode_info = item.Value("episode_info"); + + var imdb = episode_info.Value("imdb"); + release.Imdb = ParseUtil.GetImdbID(imdb); + release.TVDBId = episode_info.Value("tvdb"); + release.RageID = episode_info.Value("tvrage"); + release.TMDb = episode_info.Value("themoviedb"); + + // ex: 2015-08-16 21:25:08 +0000 + var dateStr = item.Value("pubdate").Replace(" +0000", ""); + var dateTime = DateTime.ParseExact(dateStr, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture); + release.PublishDate = DateTime.SpecifyKind(dateTime, DateTimeKind.Utc).ToLocalTime(); + + release.Seeders = item.Value("seeders"); + release.Peers = item.Value("leechers") + release.Seeders; + release.Size = item.Value("size"); + release.DownloadVolumeFactor = 0; + release.UploadVolumeFactor = 1; + + releases.Add(release); + } + } + catch (Exception ex) + { + OnParseError(response.Content, ex); + } + + return releases; + } + + } +} \ No newline at end of file diff --git a/src/Jackett/Jackett.csproj b/src/Jackett/Jackett.csproj index 3fd8bbcfa..e392e9e98 100644 --- a/src/Jackett/Jackett.csproj +++ b/src/Jackett/Jackett.csproj @@ -180,6 +180,7 @@ + @@ -604,9 +605,6 @@ PreserveNewest - - PreserveNewest - PreserveNewest