From 09d00eba32450e8b8ca74a3093282ffc25008228 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Simonsen?= Date: Mon, 26 Oct 2015 20:48:04 +0100 Subject: [PATCH] Pagination on NxtGn I've added support for offset and limit parameters on the NextGen tracker --- src/Jackett/Indexers/NxtGn.cs | 182 ++++++++++++++++------------------ src/Jackett/Jackett.csproj | 4 +- 2 files changed, 89 insertions(+), 97 deletions(-) diff --git a/src/Jackett/Indexers/NxtGn.cs b/src/Jackett/Indexers/NxtGn.cs index 27e70546e..53e259ca5 100644 --- a/src/Jackett/Indexers/NxtGn.cs +++ b/src/Jackett/Indexers/NxtGn.cs @@ -136,109 +136,99 @@ namespace Jackett.Indexers public async Task> PerformQuery(TorznabQuery query) { + var releasesPerPage = 100; var releases = new List(); - var breakWhile = false; - var page = 0; - while (page < 3) + var page = (query.Offset / releasesPerPage) + 1; + string episodeSearchUrl; + if (string.IsNullOrEmpty(query.GetQueryString())) { - string episodeSearchUrl; - if (string.IsNullOrEmpty(query.GetQueryString())) - { - episodeSearchUrl = SearchUrl + "?page=" + page; - breakWhile = true; - } - else - { - var cats = MapTorznabCapsToTrackers(query); - var catsUrlPart = string.Join("&", cats.Select(c => $"c{c}=1")); - episodeSearchUrl = string.Format("{0}?search={1}&cat=0&incldead=0&{2}&page={3}", SearchUrl, HttpUtility.UrlEncode(query.GetQueryString()), catsUrlPart, page); - } - page++; - var results = await RequestStringWithCookiesAndRetry(episodeSearchUrl); - try - { - CQ dom = results.Content; + episodeSearchUrl = SearchUrl + "?page=" + page; + } + else + { + var cats = MapTorznabCapsToTrackers(query); + var catsUrlPart = string.Join("&", cats.Select(c => $"c{c}=1")); + episodeSearchUrl = string.Format("{0}?search={1}&cat=0&incldead=0&{2}&page={3}", SearchUrl, HttpUtility.UrlEncode(query.GetQueryString()), catsUrlPart, page); + } + var results = await RequestStringWithCookiesAndRetry(episodeSearchUrl); + try + { + CQ dom = results.Content; - var rows = dom["#torrent-table-wrapper > div"]; + var rows = dom["#torrent-table-wrapper > div"]; - foreach (var row in rows.Skip(1)) + foreach (var row in rows.Skip(1)) + { + var release = new ReleaseInfo(); + + var qRow = row.Cq(); + var qLink = qRow.Find("#torrent-udgivelse2-users > a").First(); + var qDesc = qRow.Find("#torrent-udgivelse2-users > p").FirstOrDefault(); + + + var moviesCatsDanish = new[] { 38, 23, 22, 33, 17 }; + var moviesCatsIntl = new[] { 47, 5, 9 }; + var moviesCats = configData.OnlyDanishCategories.Value + ? moviesCatsDanish + : moviesCatsDanish.Concat(moviesCatsIntl); + var seriesCatsDanish = new[] { 26, 43, 46 }; + var seriesCatsIntl = new[] { 4, 21, 24, 31, 45 }; + var seriesCats = configData.OnlyDanishCategories.Value + ? seriesCatsDanish + : seriesCatsDanish.Concat(seriesCatsIntl); + var catUrl = qRow.Find(".torrent-icon > a").Attr("href"); + var cat = catUrl.Substring(catUrl.LastIndexOf('=') + 1); + var catNo = int.Parse(cat); + if (moviesCats.Contains(catNo)) + release.Category = TorznabCatType.Movies.ID; + else if (seriesCats.Contains(catNo)) + release.Category = TorznabCatType.TV.ID; + else + continue; + + releases.Add(release); + + var torrentUrl = qLink.Attr("href"); + var torrentId = torrentUrl.Substring(torrentUrl.LastIndexOf('=') + 1); + + release.MinimumRatio = 1; + release.MinimumSeedTime = 172800; + release.Title = qLink.Attr("title"); + release.Description = qDesc != null ? qDesc.InnerText : release.Title; + release.Guid = new Uri(SiteLink + torrentUrl); + release.Comments = new Uri(release.Guid + "#startcomments"); + + var downloadUrl = $"{SiteLink}download.php?id={torrentId}&rss&passkey={configData.RSSKey.Value}"; + release.Link = new Uri(downloadUrl); + + var qAdded = qRow.Find("#torrent-added").First(); + var addedStr = qAdded.Text().Trim(); + release.PublishDate = DateTime.ParseExact(addedStr, "dd-MM-yyyyHH:mm:ss", CultureInfo.InvariantCulture); + + release.Seeders = ParseUtil.CoerceInt(qRow.Find("#torrent-seeders").Text().Trim()); + release.Peers = ParseUtil.CoerceInt(qRow.Find("#torrent-leechers").Text().Trim()) + release.Seeders; + + var sizeStr = qRow.Find("#torrent-size").First().Text(); + release.Size = ReleaseInfo.GetBytes(sizeStr); + + var infoLink = qRow.Find("#infolink"); + var linkContainer = infoLink.Children().First().Children().First(); + var url = linkContainer.Attr("href"); + var img = linkContainer.Children().First(); + var imgUrl = img.Attr("src"); + if (imgUrl == "/pic/imdb.png") { - var release = new ReleaseInfo(); - - var qRow = row.Cq(); - var qLink = qRow.Find("#torrent-udgivelse2-users > a").First(); - var qDesc = qRow.Find("#torrent-udgivelse2-users > p").FirstOrDefault(); - - - var moviesCatsDanish = new[] { 38, 23, 22, 33, 17 }; - var moviesCatsIntl = new[] { 47, 5, 9 }; - var moviesCats = configData.OnlyDanishCategories.Value - ? moviesCatsDanish - : moviesCatsDanish.Concat(moviesCatsIntl); - var seriesCatsDanish = new[] {26, 43, 46}; - var seriesCatsIntl = new[] { 4, 21, 24, 31, 45 }; - var seriesCats = configData.OnlyDanishCategories.Value - ? seriesCatsDanish - : seriesCatsDanish.Concat(seriesCatsIntl); - var catUrl = qRow.Find(".torrent-icon > a").Attr("href"); - var cat = catUrl.Substring(catUrl.LastIndexOf('=') + 1); - var catNo = int.Parse(cat); - if (moviesCats.Contains(catNo)) - release.Category = TorznabCatType.Movies.ID; - else if (seriesCats.Contains(catNo)) - release.Category = TorznabCatType.TV.ID; - else - continue; - - releases.Add(release); - - var torrentUrl = qLink.Attr("href"); - var torrentId = torrentUrl.Substring(torrentUrl.LastIndexOf('=') + 1); - - release.MinimumRatio = 1; - release.MinimumSeedTime = 172800; - release.Title = qLink.Attr("title"); - release.Description = qDesc != null ? qDesc.InnerText : release.Title; - release.Guid = new Uri(SiteLink + torrentUrl); - release.Comments = new Uri(release.Guid + "#startcomments"); - - var downloadUrl = $"{SiteLink}download.php?id={torrentId}&rss&passkey={configData.RSSKey.Value}"; - release.Link = new Uri(downloadUrl); - - var qAdded = qRow.Find("#torrent-added").First(); - var addedStr = qAdded.Text().Trim(); - release.PublishDate = DateTime.ParseExact(addedStr, "dd-MM-yyyyHH:mm:ss", CultureInfo.InvariantCulture); - - release.Seeders = ParseUtil.CoerceInt(qRow.Find("#torrent-seeders").Text().Trim()); - release.Peers = ParseUtil.CoerceInt(qRow.Find("#torrent-leechers").Text().Trim()) + release.Seeders; - - var sizeStr = qRow.Find("#torrent-size").First().Text(); - release.Size = ReleaseInfo.GetBytes(sizeStr); - - var infoLink = qRow.Find("#infolink"); - var linkContainer = infoLink.Children().First().Children().First(); - var url = linkContainer.Attr("href"); - var img = linkContainer.Children().First(); - var imgUrl = img.Attr("src"); - if (imgUrl == "/pic/imdb.png") - { - release.Imdb = long.Parse(url.Substring(url.LastIndexOf('t') + 1)); - } - else if (imgUrl == "/pic/TV.png") - { - release.TVDBId = long.Parse(url.Substring(url.LastIndexOf('=') + 1)); - } + release.Imdb = long.Parse(url.Substring(url.LastIndexOf('t') + 1)); + } + else if (imgUrl == "/pic/TV.png") + { + release.TVDBId = long.Parse(url.Substring(url.LastIndexOf('=') + 1)); } - var nextPage = dom["#torrent-table-wrapper + p[align=center]"].Children().Last(); - if (!nextPage.Is("a")) - breakWhile = true; } - catch (Exception ex) - { - OnParseError(results.Content, ex); - } - if (breakWhile) - break; + } + catch (Exception ex) + { + OnParseError(results.Content, ex); } return releases; } diff --git a/src/Jackett/Jackett.csproj b/src/Jackett/Jackett.csproj index bc5a8c01c..b9dd78a40 100644 --- a/src/Jackett/Jackett.csproj +++ b/src/Jackett/Jackett.csproj @@ -442,7 +442,9 @@ PreserveNewest - + + PreserveNewest + PreserveNewest