mirror of https://github.com/Jackett/Jackett
internetarchive: C# -> yaml
This commit is contained in:
Garfield69
parent
4400ddc97e
commit
8583438b4b
|
|
@ -0,0 +1,106 @@
|
|||
---
|
||||
id: internetarchive
|
||||
name: Internet Archive
|
||||
description: "Internet Archive is a non-profit digital library offering free universal access to books, movies & music, as well as 406 billion archived web pages"
|
||||
language: en-US
|
||||
type: public
|
||||
encoding: UTF-8
|
||||
links:
|
||||
- https://archive.org/
|
||||
|
||||
caps:
|
||||
categorymappings:
|
||||
- {id: audio, cat: Audio, desc: audio}
|
||||
- {id: etree, cat: Audio, desc: etree}
|
||||
- {id: movies, cat: Movies, desc: movies}
|
||||
- {id: movies, cat: TV, desc: movies}
|
||||
- {id: image, cat: Other/Misc, desc: image}
|
||||
- {id: texts, cat: Books, desc: texts}
|
||||
- {id: software, cat: PC, desc: software}
|
||||
- {id: web, cat: Other, desc: web}
|
||||
- {id: collection, cat: Other, desc: collection}
|
||||
- {id: account, cat: Other, desc: account}
|
||||
- {id: data, cat: Other, desc: data}
|
||||
- {id: other, cat: Other, desc: other}
|
||||
|
||||
modes:
|
||||
search: [q]
|
||||
tv-search: [q]
|
||||
movie-search: [q]
|
||||
music-search: [q]
|
||||
book-search: [q]
|
||||
|
||||
settings:
|
||||
- name: titleOnly
|
||||
type: checkbox
|
||||
label: Search only in title
|
||||
default: true
|
||||
- name: noMagnet
|
||||
type: checkbox
|
||||
label: Download using .torrent only. No Magnets.
|
||||
default: false
|
||||
- name: sort
|
||||
type: select
|
||||
label: Sort requested from site
|
||||
default: publicdate
|
||||
options:
|
||||
publicdate: created
|
||||
downloads: downloads
|
||||
item_size: size
|
||||
- name: type
|
||||
type: select
|
||||
label: Order requested from site
|
||||
default: "-"
|
||||
options:
|
||||
"-": desc
|
||||
"_": asc
|
||||
|
||||
search:
|
||||
paths:
|
||||
- path: advancedsearch.php
|
||||
response:
|
||||
type: json
|
||||
inputs:
|
||||
q: "{{ if and .Config.titleOnly .Keywords }}title:({{ else }}{{ end }}{{ if .Keywords }}{{ .Keywords }}{{ else }}{{ end }}{{ if and .Config.titleOnly .Keywords }}){{ else }}{{ end }}{{ if .Keywords }} AND {{ else }}{{ end }}format:(\"Archive BitTorrent\"){{ if .Categories }} AND mediatype:({{ join .Categories \" OR \" }}){{ else }}{{ end }}"
|
||||
fl[]: "identifier,title,mediatype,item_size,downloads,btih,publicdate"
|
||||
sort: "{{ if .Keywords }}{{ re_replace .Config.type \"_\" \"\" }}{{ .Config.sort }}{{ else }}-publicdate{{ end }}"
|
||||
rows: 100
|
||||
output: json
|
||||
|
||||
rows:
|
||||
selector: response.docs
|
||||
count:
|
||||
selector: response.numFound
|
||||
|
||||
fields:
|
||||
category:
|
||||
selector: mediatype
|
||||
title:
|
||||
selector: title
|
||||
id:
|
||||
selector: identifier
|
||||
details:
|
||||
text: "details/{{ .Result.id }}"
|
||||
download:
|
||||
text: "download/{{ .Result.id }}/{{ .Result.id }}_archive.torrent"
|
||||
btih:
|
||||
selector: btih
|
||||
optional: true
|
||||
infohash:
|
||||
text: "{{ if .Config.noMagnet }}{{ else }}{{ .Result.btih }}{{ end }}"
|
||||
date:
|
||||
# 2021-10-25T16:44:43Z
|
||||
selector: publicdate
|
||||
size:
|
||||
selector: item_size
|
||||
grabs:
|
||||
selector: downloads
|
||||
seeders:
|
||||
text: 1
|
||||
leechers:
|
||||
text: 1
|
||||
downloadvolumefactor:
|
||||
text: 0
|
||||
uploadvolumefactor:
|
||||
text: 1
|
||||
# json Elasticsearch
|
||||
|
|
@ -1,221 +0,0 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Specialized;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Jackett.Common.Models;
|
||||
using Jackett.Common.Models.IndexerConfig;
|
||||
using Jackett.Common.Services.Interfaces;
|
||||
using Jackett.Common.Utils;
|
||||
using Jackett.Common.Utils.Clients;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using NLog;
|
||||
using static Jackett.Common.Models.IndexerConfig.ConfigurationData;
|
||||
using WebClient = Jackett.Common.Utils.Clients.WebClient;
|
||||
|
||||
namespace Jackett.Common.Indexers
|
||||
{
|
||||
[ExcludeFromCodeCoverage]
|
||||
public class InternetArchive : BaseWebIndexer
|
||||
{
|
||||
private string SearchUrl => SiteLink + "advancedsearch.php";
|
||||
private string DetailsUrl => SiteLink + "details/";
|
||||
private string LinkUrl => SiteLink + "download/";
|
||||
|
||||
private string _sort;
|
||||
private string _order;
|
||||
private bool _titleOnly;
|
||||
private bool _noMagnet;
|
||||
|
||||
private ConfigurationData ConfigData => configData;
|
||||
|
||||
public InternetArchive(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps,
|
||||
ICacheService cs)
|
||||
: base(id: "internetarchive",
|
||||
name: "Internet Archive",
|
||||
description: "Internet Archive is a non-profit digital library offering free universal access to books, movies & music, as well as 406 billion archived web pages",
|
||||
link: "https://archive.org/",
|
||||
caps: new TorznabCapabilities
|
||||
{
|
||||
TvSearchParams = new List<TvSearchParam>
|
||||
{
|
||||
TvSearchParam.Q
|
||||
},
|
||||
MovieSearchParams = new List<MovieSearchParam>
|
||||
{
|
||||
MovieSearchParam.Q
|
||||
},
|
||||
MusicSearchParams = new List<MusicSearchParam>
|
||||
{
|
||||
MusicSearchParam.Q
|
||||
},
|
||||
BookSearchParams = new List<BookSearchParam>
|
||||
{
|
||||
BookSearchParam.Q
|
||||
}
|
||||
},
|
||||
configService: configService,
|
||||
client: wc,
|
||||
logger: l,
|
||||
p: ps,
|
||||
cacheService: cs,
|
||||
configData: new ConfigurationData())
|
||||
{
|
||||
Encoding = Encoding.UTF8;
|
||||
Language = "en-US";
|
||||
Type = "public";
|
||||
|
||||
var sort = new SingleSelectConfigurationItem("Sort requested from site", new Dictionary<string, string>
|
||||
{
|
||||
{"publicdate", "created"},
|
||||
{"downloads", "downloads"},
|
||||
{"item_size", "size"}
|
||||
})
|
||||
{ Value = "publicdate" };
|
||||
configData.AddDynamic("sort", sort);
|
||||
|
||||
var order = new SingleSelectConfigurationItem("Order requested from site", new Dictionary<string, string>
|
||||
{
|
||||
{"desc", "desc"},
|
||||
{"asc", "asc"}
|
||||
})
|
||||
{ Value = "desc" };
|
||||
configData.AddDynamic("order", order);
|
||||
|
||||
var titleOnly = new BoolConfigurationItem("Search only in title") { Value = true };
|
||||
configData.AddDynamic("titleOnly", titleOnly);
|
||||
|
||||
var noMagnet = new BoolConfigurationItem("Download using .torrent only. No Magnets.") { Value = false };
|
||||
configData.AddDynamic("noMagnet", noMagnet);
|
||||
|
||||
AddCategoryMapping("audio", TorznabCatType.Audio);
|
||||
AddCategoryMapping("etree", TorznabCatType.Audio);
|
||||
AddCategoryMapping("movies", TorznabCatType.Movies);
|
||||
AddCategoryMapping("movies", TorznabCatType.TV);
|
||||
AddCategoryMapping("image", TorznabCatType.OtherMisc);
|
||||
AddCategoryMapping("texts", TorznabCatType.Books);
|
||||
AddCategoryMapping("software", TorznabCatType.PC);
|
||||
AddCategoryMapping("web", TorznabCatType.Other);
|
||||
AddCategoryMapping("collection", TorznabCatType.Other);
|
||||
AddCategoryMapping("account", TorznabCatType.Other);
|
||||
AddCategoryMapping("data", TorznabCatType.Other);
|
||||
AddCategoryMapping("other", TorznabCatType.Other);
|
||||
}
|
||||
|
||||
public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
|
||||
{
|
||||
LoadValuesFromJson(configJson);
|
||||
var releases = await PerformQuery(new TorznabQuery());
|
||||
|
||||
await ConfigureIfOK(string.Empty, releases.Any(), () =>
|
||||
throw new Exception("Could not find release from this URL."));
|
||||
|
||||
return IndexerConfigurationStatus.Completed;
|
||||
}
|
||||
|
||||
public override void LoadValuesFromJson(JToken jsonConfig, bool useProtectionService = false)
|
||||
{
|
||||
base.LoadValuesFromJson(jsonConfig, useProtectionService);
|
||||
|
||||
var sort = (SingleSelectConfigurationItem)configData.GetDynamic("sort");
|
||||
_sort = sort != null ? sort.Value : "publicdate";
|
||||
|
||||
var order = (SingleSelectConfigurationItem)configData.GetDynamic("order");
|
||||
_order = order != null && order.Value.Equals("asc") ? "" : "-";
|
||||
|
||||
var titleOnly = (BoolConfigurationItem)configData.GetDynamic("titleOnly");
|
||||
_titleOnly = titleOnly != null && titleOnly.Value;
|
||||
|
||||
var noMagnet = (BoolConfigurationItem)configData.GetDynamic("noMagnet");
|
||||
_noMagnet = noMagnet != null && noMagnet.Value;
|
||||
}
|
||||
|
||||
protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
|
||||
{
|
||||
var releases = new List<ReleaseInfo>();
|
||||
|
||||
var searchTerm = "format:(\"Archive BitTorrent\")";
|
||||
var sort = "-publicdate";
|
||||
if (!string.IsNullOrEmpty(query.SearchTerm))
|
||||
{
|
||||
if (_titleOnly)
|
||||
searchTerm = "title:(" + query.SearchTerm + ") AND " + searchTerm;
|
||||
else
|
||||
searchTerm = query.SearchTerm + " AND " + searchTerm;
|
||||
sort = _order + _sort;
|
||||
}
|
||||
|
||||
var querycats = MapTorznabCapsToTrackers(query);
|
||||
if (querycats.Any())
|
||||
searchTerm += " AND mediatype:(" + string.Join(" OR ", querycats) + ")";
|
||||
|
||||
var qc = new NameValueCollection
|
||||
{
|
||||
{"q", searchTerm},
|
||||
{"fl[]", "identifier,title,mediatype,item_size,downloads,btih,publicdate"},
|
||||
{"sort", sort},
|
||||
{"rows", "100"},
|
||||
{"output", "json"}
|
||||
};
|
||||
var fullSearchUrl = SearchUrl + "?" + qc.GetQueryString();
|
||||
var result = await RequestWithCookiesAndRetryAsync(fullSearchUrl);
|
||||
foreach (var torrent in ParseResponse(result))
|
||||
releases.Add(MakeRelease(torrent));
|
||||
|
||||
return releases;
|
||||
}
|
||||
|
||||
private JArray ParseResponse(WebResult result)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (result.Status != HttpStatusCode.OK)
|
||||
throw new Exception("Response code error. HTTP code: " + result.Status);
|
||||
var json = JsonConvert.DeserializeObject<dynamic>(result.ContentString);
|
||||
if (!(json is JObject) || !(json["response"] is JObject) || !(json["response"]["docs"] is JArray))
|
||||
throw new Exception("Response format error");
|
||||
return (JArray)json["response"]["docs"];
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
logger.Error("ParseResponse Error: ", e.Message);
|
||||
throw new ExceptionWithConfigData(result.ContentString, ConfigData);
|
||||
}
|
||||
}
|
||||
|
||||
private ReleaseInfo MakeRelease(JToken torrent)
|
||||
{
|
||||
var id = GetFieldAs<string>("identifier", torrent);
|
||||
var title = GetFieldAs<string>("title", torrent) ?? id;
|
||||
var details = new Uri(DetailsUrl + id);
|
||||
var btih = GetFieldAs<string>("btih", torrent);
|
||||
var link = new Uri(LinkUrl + id + "/" + id + "_archive.torrent");
|
||||
|
||||
var release = new ReleaseInfo
|
||||
{
|
||||
Title = title,
|
||||
Details = details,
|
||||
Guid = details,
|
||||
PublishDate = GetFieldAs<DateTime>("publicdate", torrent),
|
||||
Category = MapTrackerCatToNewznab(GetFieldAs<string>("mediatype", torrent)),
|
||||
Size = GetFieldAs<long>("item_size", torrent),
|
||||
Seeders = 1,
|
||||
Peers = 2,
|
||||
Grabs = GetFieldAs<long>("downloads", torrent),
|
||||
Link = link,
|
||||
InfoHash = _noMagnet ? null : btih, // magnet link is auto generated from infohash
|
||||
DownloadVolumeFactor = 0,
|
||||
UploadVolumeFactor = 1
|
||||
};
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
private static T GetFieldAs<T>(string field, JToken torrent) =>
|
||||
torrent[field] is JArray array ? array.First.ToObject<T>() : torrent.Value<T>(field);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue