using System; using System.Collections.Generic; using System.Collections.Specialized; using System.Linq; using System.Net; using System.Text; using System.Threading.Tasks; using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; using Jackett.Common.Utils.Clients; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NLog; using static Jackett.Common.Models.IndexerConfig.ConfigurationData; using WebClient = Jackett.Common.Utils.Clients.WebClient; namespace Jackett.Common.Indexers { // ReSharper disable once UnusedType.Global public class InternetArchive : BaseWebIndexer { private string SearchUrl => SiteLink + "advancedsearch.php"; private string CommentsUrl => SiteLink + "details/"; private string LinkUrl => SiteLink + "download/"; private readonly NameValueCollection _trackers = new NameValueCollection { {"tr", "udp://tracker.coppersurfer.tk:6969/announce"}, {"tr", "udp://tracker.leechers-paradise.org:6969/announce"}, {"tr", "udp://tracker.opentrackr.org:1337/announce"}, {"tr", "udp://tracker.internetwarriors.net:1337/announce"}, {"tr", "udp://open.demonii.si:1337/announce"} }; private string _sort; private string _order; private bool _titleOnly; private ConfigurationData ConfigData => configData; public InternetArchive(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps) : base(name: "Internet Archive", description: "Internet Archive is a non-profit digital library offering free universal access to books, movies & music, as well as 406 billion archived web pages", link: "https://archive.org/", caps: new TorznabCapabilities(), configService: configService, client: wc, logger: l, p: ps, configData: new ConfigurationData()) { Encoding = Encoding.UTF8; Language = "en-us"; Type = "public"; var sort = new SelectItem(new Dictionary { {"publicdate", "created"}, {"downloads", "downloads"}, {"item_size", "size"} }) { Name = "Sort requested from site", Value = "publicdate" }; configData.AddDynamic("sort", sort); var order = new SelectItem(new Dictionary { {"desc", "desc"}, {"asc", "asc"}, }) { Name = "Order requested from site", Value = "desc" }; configData.AddDynamic("order", order); var titleOnly = new BoolItem() { Name = "Search only in title", Value = true }; configData.AddDynamic("titleOnly", titleOnly); AddCategoryMapping("audio", TorznabCatType.Audio); AddCategoryMapping("etree", TorznabCatType.Audio); AddCategoryMapping("movies", TorznabCatType.Movies); AddCategoryMapping("image", TorznabCatType.OtherMisc); AddCategoryMapping("texts", TorznabCatType.Books); AddCategoryMapping("software", TorznabCatType.PC); AddCategoryMapping("web", TorznabCatType.Other); AddCategoryMapping("collection", TorznabCatType.Other); AddCategoryMapping("account", TorznabCatType.Other); AddCategoryMapping("data", TorznabCatType.Other); AddCategoryMapping("other", TorznabCatType.Other); } public override async Task ApplyConfiguration(JToken configJson) { LoadValuesFromJson(configJson); var releases = await PerformQuery(new TorznabQuery()); await ConfigureIfOK(string.Empty, releases.Any(), () => throw new Exception("Could not find release from this URL.")); return IndexerConfigurationStatus.Completed; } public override void LoadValuesFromJson(JToken jsonConfig, bool useProtectionService = false) { base.LoadValuesFromJson(jsonConfig, useProtectionService); var sort = (SelectItem)configData.GetDynamic("sort"); _sort = sort != null ? sort.Value : "publicdate"; var order = (SelectItem)configData.GetDynamic("order"); _order = order != null && order.Value.Equals("asc") ? "" : "-"; var titleOnly = (BoolItem)configData.GetDynamic("titleOnly"); _titleOnly = titleOnly != null && titleOnly.Value; } protected override async Task> PerformQuery(TorznabQuery query) { var releases = new List(); var searchTerm = "format:(\"Archive BitTorrent\")"; var sort = "-publicdate"; if (!string.IsNullOrEmpty(query.SearchTerm)) { if (_titleOnly) searchTerm = "title:(" + query.SearchTerm + ") AND " + searchTerm; else searchTerm = query.SearchTerm + " AND " + searchTerm; sort = _order + _sort; } var querycats = MapTorznabCapsToTrackers(query); if (querycats.Any()) searchTerm += " AND mediatype:(" + string.Join(" OR ", querycats) + ")"; var qc = new NameValueCollection { {"q", searchTerm}, {"fl[]", "identifier,title,mediatype,item_size,downloads,btih,publicdate"}, {"sort", sort}, {"rows", "100"}, {"output", "json"} }; var fullSearchUrl = SearchUrl + "?" + qc.GetQueryString(); var result = await RequestStringWithCookiesAndRetry(fullSearchUrl); foreach (var torrent in ParseResponse(result)) releases.Add(MakeRelease(torrent)); return releases; } private JArray ParseResponse(WebClientStringResult result) { try { if (result.Status != HttpStatusCode.OK) throw new Exception("Response code error. HTTP code: " + result.Status); var json = JsonConvert.DeserializeObject(result.Content); if (!(json is JObject) || !(json["response"] is JObject) || !(json["response"]["docs"] is JArray)) throw new Exception("Response format error"); return (JArray)json["response"]["docs"]; } catch (Exception e) { logger.Error("ParseResponse Error: ", e.Message); throw new ExceptionWithConfigData(result.Content, ConfigData); } } private ReleaseInfo MakeRelease(JToken torrent) { var release = new ReleaseInfo(); var title = GetFieldAs("title", torrent); release.Title = title; var id = GetFieldAs("identifier", torrent); release.Comments = new Uri(CommentsUrl + id); release.Guid = release.Comments; release.PublishDate = GetFieldAs("publicdate", torrent); release.Category = MapTrackerCatToNewznab(GetFieldAs("mediatype", torrent)); release.Size = GetFieldAs("item_size", torrent); release.Seeders = 1; release.Peers = 2; release.Grabs = GetFieldAs("downloads", torrent); var btih = GetFieldAs("btih", torrent); release.Link = new Uri(LinkUrl + id + "/" + id + "_archive.torrent"); release.MagnetUri = GenerateMagnetLink(btih, title); release.InfoHash = btih; release.MinimumRatio = 1; release.MinimumSeedTime = 172800; // 48 hours release.DownloadVolumeFactor = 0; release.UploadVolumeFactor = 1; return release; } private Uri GenerateMagnetLink(string btih, string title) { _trackers.Set("dn", title); return new Uri("magnet:?xt=urn:btih:" + btih + "&" + _trackers.GetQueryString()); } private static T GetFieldAs(string field, JToken torrent) => torrent[field] is JArray array ? array.First.ToObject() : torrent.Value(field); } }