From 8bc5d813b75bcc30a2d883c11c923ce5a2337c00 Mon Sep 17 00:00:00 2001 From: flightlevel Date: Fri, 22 Jun 2018 22:49:17 +1000 Subject: [PATCH 1/8] Tray Icon fix log message --- src/Jackett.Tray/Main.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Jackett.Tray/Main.cs b/src/Jackett.Tray/Main.cs index b2dcb07cb..68283890e 100644 --- a/src/Jackett.Tray/Main.cs +++ b/src/Jackett.Tray/Main.cs @@ -265,10 +265,9 @@ namespace Jackett.Tray private void ProcessExited(object sender, EventArgs e) { - logger.Info("Tray icon not responsible for process exit"); - if (!closeApplicationInitiated) { + logger.Info("Tray icon not responsible for process exit"); CloseTrayApplication(); } } From 11c7015c17b69b490ed36bb9d4349511fe56e199 Mon Sep 17 00:00:00 2001 From: morpheus133 Date: Fri, 22 Jun 2018 16:12:22 +0200 Subject: [PATCH 2/8] NCore: fix title generation (#3270) --- src/Jackett.Common/Indexers/NCore.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Jackett.Common/Indexers/NCore.cs b/src/Jackett.Common/Indexers/NCore.cs index aed1f7edb..0fda6890e 100644 --- a/src/Jackett.Common/Indexers/NCore.cs +++ b/src/Jackett.Common/Indexers/NCore.cs @@ -227,8 +227,7 @@ namespace Jackett.Common.Indexers description = release.Description.Split('['); description[1] = "[" + description[1]; } - else - + release.Title = (description[0].Trim() + "." + seasonep.Trim() + "." + releasedata.Trim('.')).Replace(' ', '.'); // if search is done for S0X than we dont want to put . between S0X and E0X From 8fb92ca05cd53bf1162cca6e01f8b43db27be325 Mon Sep 17 00:00:00 2001 From: kaso17 Date: Fri, 22 Jun 2018 16:16:26 +0200 Subject: [PATCH 3/8] Cinemageddon: attempt to fix download --- src/Jackett.Common/Definitions/cinemageddon.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Jackett.Common/Definitions/cinemageddon.yml b/src/Jackett.Common/Definitions/cinemageddon.yml index 7021c874b..5f11ad3c9 100644 --- a/src/Jackett.Common/Definitions/cinemageddon.yml +++ b/src/Jackett.Common/Definitions/cinemageddon.yml @@ -45,10 +45,7 @@ - selector: table:contains("Login failed!") test: path: index.php - - download: - selector: a[href^="download.php?id="] - + search: paths: - path: browse.php From d5a698739025dd3d05f9733a4e71db6fa2c31144 Mon Sep 17 00:00:00 2001 From: Ivan de la Beldad Fernandez Date: Fri, 22 Jun 2018 16:48:33 +0200 Subject: [PATCH 4/8] add mejortorrent indexer (#3268) * feat: start mejortorrent indexer, create basic configuration * feat: add basic tv show scraper for series search * fix: add guid so more than one episode is returned * feat: add date to episodes * feat: add support for multi-episodes * feat: add proper filtering * fix: change filter when there is no season or no episode * feat: implement tv show rss scraper with no links by the moment * feat: finish tv-shows rss * fix: prevent sdtv to be marked as hdtv * docs: add mejortorrent indexer to readme * add url encode * fix encoding Content seems to be UTF8 * add missing include * fix my fixes * fix encoding --- README.md | 1 + src/Jackett.Common/Indexers/MejorTorrent.cs | 701 ++++++++++++++++++++ 2 files changed, 702 insertions(+) create mode 100644 src/Jackett.Common/Indexers/MejorTorrent.cs diff --git a/README.md b/README.md index 82d6dae9a..b60089cbe 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ Developer note: The software implements the [Torznab](https://github.com/Sonarr/ * KickAssTorrent (thekat.se clone) * LimeTorrents * MagnetDL + * MejorTorrent * NextTorrent * Newpct (aka: tvsinpagar, descargas2020, torrentlocura, torrentrapid, etc) * Nyaa.si diff --git a/src/Jackett.Common/Indexers/MejorTorrent.cs b/src/Jackett.Common/Indexers/MejorTorrent.cs new file mode 100644 index 000000000..b721f9b19 --- /dev/null +++ b/src/Jackett.Common/Indexers/MejorTorrent.cs @@ -0,0 +1,701 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using AngleSharp.Dom.Html; +using AngleSharp.Parser.Html; +using Jackett.Common.Models; +using Jackett.Common.Models.IndexerConfig; +using Jackett.Common.Services.Interfaces; +using Jackett.Common.Utils.Clients; +using Jackett.Common.Helpers; +using Newtonsoft.Json.Linq; +using NLog; + +namespace Jackett.Common.Indexers +{ + class MejorTorrent : BaseWebIndexer + { + public static Uri WebUri = new Uri("http://www.mejortorrent.com/"); + public static Uri DownloadUri = new Uri(WebUri, "secciones.php?sec=descargas&ap=contar_varios"); + private static Uri SearchUriBase = new Uri(WebUri, "secciones.php"); + public static Uri NewTorrentsUri = new Uri(WebUri, "secciones.php?sec=ultimos_torrents"); + public static Encoding MEEncoding = Encoding.GetEncoding("windows-1252"); + + public MejorTorrent(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps) + : base(name: "MejorTorrent", + description: "MejorTorrent - Hay veces que un torrent viene mejor! :)", + link: WebUri.AbsoluteUri, + caps: new TorznabCapabilities(TorznabCatType.TV, + TorznabCatType.TVSD, + TorznabCatType.TVHD), + configService: configService, + client: wc, + logger: l, + p: ps, + configData: new ConfigurationData()) + { + Encoding = MEEncoding; + Language = "es-es"; + Type = "public"; + } + + public override async Task ApplyConfiguration(JToken configJson) + { + configData.LoadValuesFromJson(configJson); + var releases = await PerformQuery(new TorznabQuery()); + + await ConfigureIfOK(string.Empty, releases.Count() > 0, () => + { + throw new Exception("Could not find releases from this URL"); + }); + + return IndexerConfigurationStatus.Completed; + } + + protected override async Task> PerformQuery(TorznabQuery query) + { + return await PerformQuery(query, 0); + } + + public async Task> PerformQuery(TorznabQuery query, int attempts) + { + var requester = new MejorTorrentRequester(this); + var tvShowScraper = new TvShowScraper(); + var seasonScraper = new SeasonScraper(); + var downloadScraper = new DownloadScraper(); + var rssScraper = new RssScraper(); + var downloadGenerator = new DownloadGenerator(requester, downloadScraper); + var tvShowPerformer = new TvShowPerformer(requester, tvShowScraper, seasonScraper, downloadGenerator); + var rssPerformer = new RssPerformer(requester, rssScraper, seasonScraper, downloadGenerator); + + if (string.IsNullOrEmpty(query.SanitizedSearchTerm)) + { + return await rssPerformer.PerformQuery(query); + } + return await tvShowPerformer.PerformQuery(query); + } + + public static Uri CreateSearchUri(string search) + { + var finalUri = SearchUriBase.AbsoluteUri; + finalUri += "?sec=buscador&valor=" + WebUtilityHelpers.UrlEncode(search, MEEncoding); + return new Uri(finalUri); + } + + interface IScraper + { + T Extract(IHtmlDocument html); + } + + class RssScraper : IScraper>> + { + private readonly string LinkQuerySelector = "a[href*=\"/serie\"]"; + + public IEnumerable> Extract(IHtmlDocument html) + { + var episodes = GetNewEpisodesScratch(html); + var links = GetLinks(html); + var results = new List>(); + for (var i = 0; i < episodes.Count(); i++) + { + results.Add(new KeyValuePair(episodes.ElementAt(i), links.ElementAt(i))); + } + return results; + } + + private List GetNewEpisodesScratch(IHtmlDocument html) + { + var tvShowsElements = html.QuerySelectorAll(LinkQuerySelector); + var seasonLinks = tvShowsElements.Select(e => e.Attributes["href"].Value); + var dates = GetDates(html); + var titles = GetTitles(html); + var qualities = GetQualities(html); + var seasonsFirstEpisodesAndLast = GetSeasonsFirstEpisodesAndLast(html); + + var episodes = new List(); + for(var i = 0; i < tvShowsElements.Count(); i++) + { + var e = new MejorTorrentReleaseInfo(); + e.TitleOriginal = titles.ElementAt(i); + e.PublishDate = dates.ElementAt(i); + e.CategoryText = qualities.ElementAt(i); + var sfeal = seasonsFirstEpisodesAndLast.ElementAt(i); + e.Season = sfeal.Key; + e.EpisodeNumber = sfeal.Value.Key; + if (sfeal.Value.Value != null && sfeal.Value.Value > sfeal.Value.Key) + { + e.Files = sfeal.Value.Value - sfeal.Value.Key + 1; + } + else + { + e.Files = 1; + } + episodes.Add(e); + } + return episodes; + } + + private List GetLinks(IHtmlDocument html) + { + return html.QuerySelectorAll(LinkQuerySelector) + .Select(e => e.Attributes["href"].Value) + .Select(relativeLink => new Uri(WebUri, relativeLink)) + .ToList(); + } + + private List GetDates(IHtmlDocument html) + { + return html.QuerySelectorAll(LinkQuerySelector) + .Select(e => e.PreviousElementSibling.TextContent) + .Select(dateString => dateString.Split('-')) + .Select(parts => new int[] { Int32.Parse(parts[0]), Int32.Parse(parts[1]), Int32.Parse(parts[2]) }) + .Select(intParts => new DateTime(intParts[0], intParts[1], intParts[2])) + .ToList(); + } + + private List GetTitles(IHtmlDocument html) + { + var texts = LinkTexts(html); + var completeTitles = texts.Select(text => text.Substring(0, text.IndexOf('-') - 1)); + var regex = new Regex(@".+\((.+)\)"); + var finalTitles = completeTitles.Select(title => + { + var match = regex.Match(title); + if (!match.Success) return title; + return match.Groups[1].Value; + }); + return finalTitles.ToList(); + } + + private List GetQualities(IHtmlDocument html) + { + var texts = LinkTexts(html); + var regex = new Regex(@".+\[(.*)\].+"); + var qualities = texts.Select(text => + { + var match = regex.Match(text); + if (!match.Success) return "HDTV"; + var quality = match.Groups[1].Value; + switch(quality) + { + case "720p": + return "HDTV-720p"; + case "1080p": + return "HDTV-1080p"; + default: + return "HDTV"; + } + }); + return qualities.ToList(); + } + + private List>> GetSeasonsFirstEpisodesAndLast(IHtmlDocument html) + { + var texts = LinkTexts(html); + // SEASON | START EPISODE | [END EPISODE] + var regex = new Regex(@"(\d{1,2})x(\d{1,2})(?:.*\d{1,2}x(\d{1,2})?)?", RegexOptions.IgnoreCase); + var seasonsFirstEpisodesAndLast = texts.Select(text => + { + var match = regex.Match(text); + int season = 0; + int episode = 0; + int? finalEpisode = null; + if (!match.Success) return new KeyValuePair>(season, new KeyValuePair(episode, finalEpisode)); + season = Int32.Parse(match.Groups[1].Value); + episode = Int32.Parse(match.Groups[2].Value); + if (match.Groups[3].Success) + { + finalEpisode = Int32.Parse(match.Groups[3].Value); + } + return new KeyValuePair>(season, new KeyValuePair(episode, finalEpisode)); + }); + return seasonsFirstEpisodesAndLast.ToList(); + } + + private List LinkTexts(IHtmlDocument html) + { + return html.QuerySelectorAll(LinkQuerySelector) + .Select(e => e.TextContent).ToList(); + } + + } + + class TvShowScraper : IScraper> + { + public IEnumerable Extract(IHtmlDocument html) + { + var tvSelector = "a[href*=\"/serie-\"]"; + var seasonsElements = html.QuerySelectorAll(tvSelector).Select(e => e.ParentElement); + + var newTvShows = new List(); + + // EXAMPLES: + // Stranger Things - 1ª Temporada (HDTV) + // Stranger Things - 1ª Temporada [720p] (HDTV-720p) + var regex = new Regex(@"(.+) - ([0-9]+).*\((.*)\)"); + foreach (var seasonElement in seasonsElements) + { + var link = seasonElement.QuerySelector("a[href*=\"/serie-\"]").Attributes["href"].Value; + var info = seasonElement.TextContent; // Stranger Things - 1 ... + var searchMatch = regex.Match(info); + if (!searchMatch.Success) + { + continue; + } + int seasonNumber; + if (!Int32.TryParse(searchMatch.Groups[2].Value, out seasonNumber)) + { + seasonNumber = 0; + } + var season = new Season + { + Title = searchMatch.Groups[1].Value, + Number = seasonNumber, + Type = searchMatch.Groups[3].Value, + Link = new Uri(WebUri, link) + }; + + // EXAMPLE: El cuento de la criada (Handmaids Tale) + var originalTitleRegex = new Regex(@".+\((.+)\)"); + var originalTitleMath = originalTitleRegex.Match(season.Title); + if (originalTitleMath.Success) + { + season.Title = originalTitleMath.Groups[1].Value; + } + newTvShows.Add(season); + } + return newTvShows; + } + } + + class SeasonScraper : IScraper> + { + public IEnumerable Extract(IHtmlDocument html) + { + var episodesLinksHtml = html.QuerySelectorAll("a[href*=\"/serie-episodio-descargar-torrent\"]"); + var episodesTexts = episodesLinksHtml.Select(l => l.TextContent).ToList(); + var episodesLinks = episodesLinksHtml.Select(e => e.Attributes["href"].Value).ToList(); + var dates = episodesLinksHtml + .Select(e => e.ParentElement.ParentElement.QuerySelector("div").TextContent) + .Select(stringDate => stringDate.Replace("Fecha: ", "")) + .Select(stringDate => stringDate.Split('-')) + .Select(stringParts => new int[]{ Int32.Parse(stringParts[0]), Int32.Parse(stringParts[1]), Int32.Parse(stringParts[2]) }) + .Select(intParts => new DateTime(intParts[0], intParts[1], intParts[2])); + + var episodes = episodesLinks.Select(e => new MejorTorrentReleaseInfo()).ToList(); + + for (var i = 0; i < episodes.Count(); i++) + { + GuessEpisodes(episodes.ElementAt(i), episodesTexts.ElementAt(i)); + ExtractLinkInfo(episodes.ElementAt(i), episodesLinks.ElementAt(i)); + episodes.ElementAt(i).PublishDate = dates.ElementAt(i); + } + + return episodes; + } + + private void GuessEpisodes(MejorTorrentReleaseInfo release, string episodeText) + { + var seasonEpisodeRegex = new Regex(@"(\d{1,2}).*?(\d{1,2})", RegexOptions.IgnoreCase); + var matchSeasonEpisode = seasonEpisodeRegex.Match(episodeText); + if (!matchSeasonEpisode.Success) return; + release.Season = Int32.Parse(matchSeasonEpisode.Groups[1].Value); + release.EpisodeNumber = Int32.Parse(matchSeasonEpisode.Groups[2].Value); + + char[] textArray = episodeText.ToCharArray(); + Array.Reverse(textArray); + var reversedText = new string(textArray); + var finalEpisodeRegex = new Regex(@"(\d{1,2})"); + var matchFinalEpisode = finalEpisodeRegex.Match(reversedText); + if (!matchFinalEpisode.Success) return; + var finalEpisodeArray = matchFinalEpisode.Groups[1].Value.ToCharArray(); + Array.Reverse(finalEpisodeArray); + var finalEpisode = Int32.Parse(new string(finalEpisodeArray)); + if (finalEpisode > release.EpisodeNumber) + { + release.Files = (finalEpisode + 1) - release.EpisodeNumber; + release.Size = release.Size * release.Files; + } + } + + private void ExtractLinkInfo(MejorTorrentReleaseInfo release, String link) + { + // LINK FORMAT: /serie-episodio-descargar-torrent-${ID}-${TITLE}-${SEASON_NUMBER}x${EPISODE_NUMBER}[range].html + var regex = new Regex(@"\/serie-episodio-descargar-torrent-(\d+)-(.*)-(\d{1,2}).*(\d{1,2}).*\.html", RegexOptions.IgnoreCase); + var linkMatch = regex.Match(link); + + if (!linkMatch.Success) + { + return; + } + release.MejorTorrentID = linkMatch.Groups[1].Value; + release.Title = linkMatch.Groups[2].Value; + } + } + + class DownloadScraper : IScraper> + { + public IEnumerable Extract(IHtmlDocument html) + { + return html.QuerySelectorAll("a[href*=\".torrent\"]") + .Select(e => e.Attributes["href"].Value) + .Select(link => new Uri(WebUri, link)); + } + } + + class Season + { + public String Title; + public int Number; + public Uri Link; + public TorznabCategory Category; // HDTV or HDTV-720 + private string _type; + public string Type + { + get { return _type; } + set + { + switch(value) + { + case "HDTV": + Category = TorznabCatType.TVSD; + _type = "SDTV"; + break; + case "HDTV-720p": + Category = TorznabCatType.TVHD; + _type = "HDTV-720p"; + break; + case "HDTV-1080p": + Category = TorznabCatType.TVHD; + _type = "HDTV-1080p"; + break; + default: + Category = TorznabCatType.TV; + _type = "HDTV-720p"; + break; + } + } + } + } + + class MejorTorrentReleaseInfo : ReleaseInfo + { + public string MejorTorrentID; + public int _season; + public int _episodeNumber; + private string _categoryText; + private string _originalTitle; + + public MejorTorrentReleaseInfo() + { + this.Category = new List(); + this.Grabs = 5; + this.Files = 1; + this.PublishDate = new DateTime(); + this.Peers = 10; + this.Seeders = 10; + this.Size = ReleaseInfo.BytesFromGB(1); + this._originalTitle = ""; + } + + public int Season { get { return _season; } set { _season = value; TitleOriginal = _originalTitle; } } + + public int EpisodeNumber { get { return _episodeNumber; } set { _episodeNumber = value; TitleOriginal = _originalTitle; } } + + public string CategoryText { + get { return _categoryText; } + set + { + switch (value) + { + case "SDTV": + Category.Add(TorznabCatType.TVSD.ID); + _categoryText = "SDTV"; + break; + case "HDTV": + Category.Add(TorznabCatType.TVSD.ID); + _categoryText = "SDTV"; + break; + case "HDTV-720p": + Category.Add(TorznabCatType.TVHD.ID); + _categoryText = "HDTV-720p"; + break; + case "HDTV-1080p": + Category.Add(TorznabCatType.TVHD.ID); + _categoryText = "HDTV-1080p"; + break; + default: + Category.Add(TorznabCatType.TV.ID); + _categoryText = "HDTV-720p"; + break; + } + TitleOriginal = _originalTitle; + } + } + + public int FinalEpisodeNumber { get { return (int)(EpisodeNumber + Files - 1); } } + + public string TitleOriginal + { + get { return _originalTitle; } + set + { + _originalTitle = value; + if (_originalTitle != "") + { + Title = _originalTitle.Replace(' ', '.'); + Title = char.ToUpper(Title[0]) + Title.Substring(1); + } + var seasonAndEpisode = "S" + Season.ToString("00") + "E" + EpisodeNumber.ToString("00"); + if (Files > 1) + { + seasonAndEpisode += "-" + FinalEpisodeNumber.ToString("00"); + } + Title = String.Join(".", new List() { Title, seasonAndEpisode, CategoryText, "Spanish" }); + } + } + } + + interface IRequester + { + Task MakeRequest( + Uri uri, + RequestType method = RequestType.GET, + IEnumerable> data = null, + Dictionary headers = null); + } + + class MejorTorrentRequester : IRequester + { + private MejorTorrent mt; + + public MejorTorrentRequester(MejorTorrent mt) + { + this.mt = mt; + } + + public async Task MakeRequest( + Uri uri, + RequestType method = RequestType.GET, + IEnumerable> data = null, + Dictionary headers = null) + { + var result = await mt.RequestBytesWithCookies(uri.AbsoluteUri, null, method, null, data, headers); + var SearchResultParser = new HtmlParser(); + var doc = SearchResultParser.Parse(mt.Encoding.GetString(result.Content)); + return doc; + } + } + + class MejorTorrentDownloadRequesterDecorator + { + private IRequester r; + + public MejorTorrentDownloadRequesterDecorator(IRequester r) + { + this.r = r; + } + + public async Task MakeRequest(IEnumerable ids) + { + var downloadHtmlTasks = new List>(); + var formData = new List>(); + int index = 1; + ids.ToList().ForEach(id => + { + var episodeID = new KeyValuePair("episodios[" + index + "]", id); + formData.Add(episodeID); + index++; + }); + formData.Add(new KeyValuePair("total_capis", index.ToString())); + formData.Add(new KeyValuePair("tabla", "series")); + return await r.MakeRequest(DownloadUri, RequestType.POST, formData); + } + } + + interface IPerformer + { + Task> PerformQuery(TorznabQuery query); + } + + class RssPerformer : IPerformer + { + private IRequester requester; + private IScraper>> rssScraper; + private IScraper> seasonScraper; + private IDownloadGenerator downloadGenerator; + + public RssPerformer( + IRequester requester, + IScraper>> rssScraper, + IScraper> seasonScraper, + IDownloadGenerator downloadGenerator) + { + this.requester = requester; + this.rssScraper = rssScraper; + this.seasonScraper = seasonScraper; + this.downloadGenerator = downloadGenerator; + } + + public async Task> PerformQuery(TorznabQuery query) + { + var html = await requester.MakeRequest(NewTorrentsUri); + var episodesAndSeasonsUri = rssScraper.Extract(html); + + Task.WaitAll(episodesAndSeasonsUri.ToList().Select(async epAndSeasonUri => + { + var episode = epAndSeasonUri.Key; + var seasonUri = epAndSeasonUri.Value; + await AddMejorTorrentIDs(episode, seasonUri); + }).ToArray()); + + var episodes = episodesAndSeasonsUri.Select(epAndSeason => epAndSeason.Key).ToList(); + await downloadGenerator.AddDownloadLinks(episodes); + return episodes; + } + + private async Task AddMejorTorrentIDs(MejorTorrentReleaseInfo episode, Uri seasonUri) + { + var html = await requester.MakeRequest(seasonUri); + var newEpisodes = seasonScraper.Extract(html); + // GET BY EPISODE NUMBER + newEpisodes = newEpisodes.Where(e => e.EpisodeNumber == episode.EpisodeNumber); + if (newEpisodes.Count() == 0) + { + throw new Exception("Imposible to detect episode ID in RSS"); + } + episode.MejorTorrentID = newEpisodes.First().MejorTorrentID; + } + + } + + class TvShowPerformer : IPerformer + { + private IRequester requester; + private IScraper> tvShowScraper; + private IScraper> seasonScraper; + private IDownloadGenerator downloadGenerator; + + public TvShowPerformer( + IRequester requester, + IScraper> tvShowScraper, + IScraper> seasonScraper, + IDownloadGenerator downloadGenerator) + { + this.requester = requester; + this.tvShowScraper = tvShowScraper; + this.seasonScraper = seasonScraper; + this.downloadGenerator = downloadGenerator; + } + + public async Task> PerformQuery(TorznabQuery query) + { + query = FixQuery(query); + var seasons = await GetSeasons(query); + var episodes = await GetEpisodes(query, seasons); + await downloadGenerator.AddDownloadLinks(episodes); + if (seasons.Count() > 0) + { + episodes.ForEach(e => e.TitleOriginal = seasons.First().Title); + } + return episodes; + } + + private TorznabQuery FixQuery(TorznabQuery query) + { + var seasonRegex = new Regex(@".*?(s\d{1,2})", RegexOptions.IgnoreCase); + var episodeRegex = new Regex(@".*?(e\d{1,2})", RegexOptions.IgnoreCase); + var seasonMatch = seasonRegex.Match(query.SearchTerm); + var episodeMatch = episodeRegex.Match(query.SearchTerm); + if (seasonMatch.Success) + { + query.Season = Int32.Parse(seasonMatch.Groups[1].Value.Substring(1)); + query.SearchTerm = query.SearchTerm.Replace(seasonMatch.Groups[1].Value, ""); + } + if (episodeMatch.Success) + { + query.Episode = episodeMatch.Groups[1].Value.Substring(1); + query.SearchTerm = query.SearchTerm.Replace(episodeMatch.Groups[1].Value, ""); + } + query.SearchTerm = query.SearchTerm.Trim(); + return query; + } + + private async Task> GetSeasons(TorznabQuery query) + { + var seasonHtml = await requester.MakeRequest(CreateSearchUri(query.SanitizedSearchTerm)); + var seasons = tvShowScraper.Extract(seasonHtml); + if (query.Season != 0) + { + seasons = seasons.Where(s => s.Number == query.Season); + } + if (query.Categories.Count() != 0) + { + seasons = seasons.Where(s => new List(query.Categories).Contains(s.Category.ID)); + } + return seasons.ToList(); + } + + private async Task> GetEpisodes(TorznabQuery query, IEnumerable seasons) + { + var episodesHtmlTasks = new Dictionary>(); + seasons.ToList().ForEach(season => + { + episodesHtmlTasks.Add(season, requester.MakeRequest(new Uri(WebUri, season.Link))); + }); + var episodesHtml = await Task.WhenAll(episodesHtmlTasks.Values); + var episodes = episodesHtmlTasks.SelectMany(seasonAndHtml => + { + var season = seasonAndHtml.Key; + var html = seasonAndHtml.Value.Result; + var eps = seasonScraper.Extract(html); + return eps.ToList().Select(e => + { + e.CategoryText = season.Type; + return e; + }); + }); + if (!string.IsNullOrEmpty(query.Episode)) + { + var episodeNumber = Int32.Parse(query.Episode); + episodes = episodes.Where(e => e.EpisodeNumber <= episodeNumber && episodeNumber <= e.FinalEpisodeNumber); + } + return episodes.ToList(); + } + } + + interface IDownloadGenerator + { + Task AddDownloadLinks(IEnumerable episodes); + } + + class DownloadGenerator : IDownloadGenerator + { + private IRequester requester; + private IScraper> downloadScraper; + + public DownloadGenerator(IRequester requester, IScraper> downloadScraper) + { + this.requester = requester; + this.downloadScraper = downloadScraper; + } + + public async Task AddDownloadLinks(IEnumerable episodes) + { + var downloadRequester = new MejorTorrentDownloadRequesterDecorator(requester); + var downloadHtml = await downloadRequester.MakeRequest(episodes.Select(e => e.MejorTorrentID)); + var downloads = downloadScraper.Extract(downloadHtml).ToList(); + + for (var i = 0; i < downloads.Count; i++) + { + var e = episodes.ElementAt(i); + episodes.ElementAt(i).Link = downloads.ElementAt(i); + episodes.ElementAt(i).Guid = downloads.ElementAt(i); + } + } + } + } +} From ef8653f7d214d3e4071e819f006441e9265db344 Mon Sep 17 00:00:00 2001 From: kaso17 Date: Fri, 22 Jun 2018 17:32:44 +0200 Subject: [PATCH 5/8] AST4u: add support for alternative layout --- src/Jackett.Common/Definitions/ast4u.yml | 36 ++++++++++++++++++------ 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/src/Jackett.Common/Definitions/ast4u.yml b/src/Jackett.Common/Definitions/ast4u.yml index 1081e5805..8d3c0a6ab 100644 --- a/src/Jackett.Common/Definitions/ast4u.yml +++ b/src/Jackett.Common/Definitions/ast4u.yml @@ -40,6 +40,18 @@ search: [q] tv-search: [q, season, ep] movie-search: [q] + + settings: + - name: username + type: text + label: Username + - name: password + type: password + label: Password + - name: info + type: info + label: Results Per Page + default: For best results, change the 'Torrentliste' setting to "Platzsparendes Layout mit PopUp für zusätzliche Informationen" in your profile. login: path: takelogin.php @@ -63,9 +75,13 @@ rows: selector: table.tableinborder > tbody > tr:has(a[href^="details.php"]) - fields: + fields: # note: two alternative layouts available title: selector: a[href^="details.php"] + title: + optional: true + selector: a[href^="details.php"][title] + attribute: title category: selector: a[href^="browse.php?cat="] attribute: href @@ -79,29 +95,31 @@ selector: a[href^=" /gettorrent/ssl/"] attribute: href files: - selector: td:nth-child(2) > table > tbody > tr:nth-child(2) > td:nth-child(1) > b:nth-child(2) + selector: td:nth-child(2) > table > tbody > tr:nth-child(2) > td:nth-child(1) > b:nth-child(2), a[href*="&filelist=1"] grabs: - selector: td:nth-child(2) > table > tbody > tr:nth-child(2) > td:nth-child(3) > b:nth-child(1) + selector: td:nth-child(2) > table > tbody > tr:nth-child(2) > td:nth-child(3) > b:nth-child(1), a[href*="&tosnatchers=1"] size: - selector: td:nth-child(2) > table > tbody > tr:nth-child(2) > td:nth-child(1) > b:nth-child(1) + selector: td:nth-child(2) > table > tbody > tr:nth-child(2) > td:nth-child(1) > b:nth-child(1), td:nth-child(7):has(br) filters: - name: replace args: [".", ""] - name: replace args: [",", "."] seeders: - selector: td:nth-child(2) > table > tbody > tr:nth-child(2) > td:nth-child(2) > b:nth-child(1) + selector: td:nth-child(2) > table > tbody > tr:nth-child(2) > td:nth-child(2) > b:nth-child(1), a[href*="&toseeders=1"] leechers: - selector: td:nth-child(2) > table > tbody > tr:nth-child(2) > td:nth-child(2) > b:nth-child(3) + selector: td:nth-child(2) > table > tbody > tr:nth-child(2) > td:nth-child(2) > b:nth-child(3), a[href*="&todlers=1"] date: - selector: td:nth-child(2) > table > tbody > tr:nth-child(2) > td:nth-child(5) + selector: td:nth-child(2) > table > tbody > tr:nth-child(2) > td:nth-child(5), td:nth-child(5):has(br) filters: + - name: replace + args: [" ", ""] - name: append args: " +2:00" - name: replace - args: ["\xA0", " "] + args: ["\xA0", ""] - name: dateparse - args: "02.01.2006 15:04:05 -07:00" + args: "02.01.200615:04:05 -07:00" downloadvolumefactor: case: img[src="/pic/free.gif"]: "0" From b56552e0f4c08be72b50f82ac6d36bea8369a271 Mon Sep 17 00:00:00 2001 From: flightlevel Date: Sat, 23 Jun 2018 11:36:29 +1000 Subject: [PATCH 6/8] Updater: Remove unused reference --- src/Jackett.Updater/Jackett.Updater.csproj | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Jackett.Updater/Jackett.Updater.csproj b/src/Jackett.Updater/Jackett.Updater.csproj index 983a21abc..221b75bcd 100644 --- a/src/Jackett.Updater/Jackett.Updater.csproj +++ b/src/Jackett.Updater/Jackett.Updater.csproj @@ -69,10 +69,6 @@ {6B854A1B-9A90-49C0-BC37-9A35C75BCA73} Jackett.Common - - {bf611f7b-4658-4cb8-aa9e-0736fadaa3ba} - Jackett.Service - {e636d5f8-68b4-4903-b4ed-ccfd9c9e899f} Jackett From 06758964c055e2e336ce1f3e0e8be7bb40ed5773 Mon Sep 17 00:00:00 2001 From: flightlevel Date: Sat, 23 Jun 2018 11:37:49 +1000 Subject: [PATCH 7/8] Jackett.Service: Make compatible with both legacy & .NET Core --- src/Jackett.Service/App.config | 12 --- src/Jackett.Service/Jackett.Service.csproj | 12 +-- src/Jackett.Service/Service.cs | 87 ++++++++++++++++++++-- 3 files changed, 80 insertions(+), 31 deletions(-) delete mode 100644 src/Jackett.Service/App.config diff --git a/src/Jackett.Service/App.config b/src/Jackett.Service/App.config deleted file mode 100644 index 2532b3ad8..000000000 --- a/src/Jackett.Service/App.config +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - - - diff --git a/src/Jackett.Service/Jackett.Service.csproj b/src/Jackett.Service/Jackett.Service.csproj index ed3dce7ec..38bbce463 100644 --- a/src/Jackett.Service/Jackett.Service.csproj +++ b/src/Jackett.Service/Jackett.Service.csproj @@ -14,6 +14,7 @@ true PackageReference win + AnyCPU @@ -65,25 +66,14 @@ - - - - - {74420a79-cc16-442c-8b1e-7c1b913844f0} - CurlSharp - {6B854A1B-9A90-49C0-BC37-9A35C75BCA73} Jackett.Common - - {e636d5f8-68b4-4903-b4ed-ccfd9c9e899f} - Jackett -