solidtorrents: rewrite in c#. resolves #6681 (#6817)

This commit is contained in:
Diego Heras 2020-01-04 20:11:13 +01:00 committed by garfield69
parent e7a48c4234
commit 22f5b59393
3 changed files with 180 additions and 86 deletions

View File

@ -1,86 +0,0 @@
---
site: solidtorrents
name: SolidTorrents
description: "SolidTorrents is a Public torrent meta-search engine"
language: en-us
type: public
encoding: UTF-8
links:
- https://solidtorrents.net/
caps:
categories:
"Audio": Audio
"Video": Movies
"Image": Other/Misc
"Document": Books/Comics
"eBook": Books/Ebook
"Program": PC/0day
"Android": PC/Phone-Android
"Archive": Other
"Diskimage": PC/ISO
"Sourcecode": Movies/Other
"Database": Movies/DVD
"Unknown": Other
modes:
search: [q]
tv-search: [q, season, ep]
movie-search: [q]
settings:
- name: sort
type: select
label: Sort requested from site
default: "date"
options:
"date": "created"
"seeders": "seeders"
"size": "size"
search:
paths:
# https://solidtorrents.net/search?q=gotham%20s05e02&sort=date&category=all
- path: "search?q={{ .Keywords }}&sort={{ .Config.sort }}&category=all"
rows:
selector: div[role="listitem"]:has(a[href^="magnet:?xt="])
fields:
title:
selector: div[class$="__title"] a[title]
attribute: title
details:
selector: div[class$="__title"] a
attribute: href
download:
selector: a[href^="magnet:?xt="]
attribute: href
category:
selector: div.v-list__tile__content div:nth-of-type(2)
filters:
- name: regexp
args: "(.+?) \\|"
size:
selector: div.v-list__tile__sub-title:last-of-type strong
date:
selector: div.v-list__tile__content div:nth-of-type(2)
filters:
- name: replace
args: ["a few", "1"]
- name: replace
args: ["an ", "1 "]
- name: replace
args: ["a ", "1 "]
- name: regexp
args: "(\\d+ \\w+ \\w+)"
grabs:
selector: div.v-list__tile__sub-title span:last-of-type
seeders:
selector: div.v-list__tile__sub-title span.green--text
leechers:
selector: div.v-list__tile__sub-title span.red--text
downloadvolumefactor:
text: 0
uploadvolumefactor:
text: 1

View File

@ -0,0 +1,179 @@
using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
namespace Jackett.Common.Indexers
{
public class SolidTorrents : BaseWebIndexer
{
private string SearchUrl => SiteLink + "api/v1/search";
private readonly Dictionary<string, string> APIHeaders = new Dictionary<string, string>()
{
{"Accept", "application/json, text/plain, */*"},
};
private readonly int MAX_RESULTS_PER_PAGE = 20;
private readonly int MAX_SEARCH_PAGE_LIMIT = 3; // 20 items per page, 60
private ConfigurationData ConfigData
{
get => (ConfigurationData)base.configData;
set => base.configData = value;
}
public SolidTorrents(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps)
: base(name: "Solid Torrents",
description: "SolidTorrents is a Public torrent meta-search engine",
link: "https://solidtorrents.net/",
caps: new TorznabCapabilities(),
configService: configService,
client: wc,
logger: l,
p: ps,
configData: new ConfigurationData())
{
Encoding = Encoding.UTF8;
Language = "en-us";
Type = "public";
AddCategoryMapping("Audio", TorznabCatType.Audio);
AddCategoryMapping("Video", TorznabCatType.Movies);
AddCategoryMapping("Image", TorznabCatType.OtherMisc);
AddCategoryMapping("Document", TorznabCatType.BooksComics);
AddCategoryMapping("eBook", TorznabCatType.BooksEbook);
AddCategoryMapping("Program", TorznabCatType.PC0day);
AddCategoryMapping("Android", TorznabCatType.PCPhoneAndroid);
AddCategoryMapping("Archive", TorznabCatType.Other);
AddCategoryMapping("Diskimage", TorznabCatType.PCISO);
AddCategoryMapping("Sourcecode", TorznabCatType.MoviesOther);
AddCategoryMapping("Database", TorznabCatType.MoviesDVD);
AddCategoryMapping("Unknown", TorznabCatType.Other);
}
public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
{
base.LoadValuesFromJson(configJson);
var releases = await PerformQuery(new TorznabQuery());
await ConfigureIfOK(string.Empty, releases.Any(), () =>
{
throw new Exception("Could not find release from this URL.");
});
return IndexerConfigurationStatus.Completed;
}
private JArray CheckResponse(WebClientStringResult result)
{
try
{
var json = JsonConvert.DeserializeObject<dynamic>(result.Content);
if (!(json is JObject) || !(json["results"] is JArray) || json["results"] == null)
throw new Exception("Server error");
return (JArray)json["results"];
}
catch (Exception e)
{
logger.Error("CheckResponse() Error: ", e.Message);
throw new ExceptionWithConfigData(result.Content, ConfigData);
}
}
private async Task<JArray> SendSearchRequest(string searchString, string category, int page)
{
var queryCollection = new NameValueCollection();
queryCollection.Add("q", searchString);
queryCollection.Add("category", category);
queryCollection.Add("skip", (page * MAX_RESULTS_PER_PAGE).ToString());
queryCollection.Add("sort", "date");
queryCollection.Add("fuv", "no");
var fullSearchUrl = SearchUrl + "?" + queryCollection.GetQueryString();
var result = await RequestStringWithCookiesAndRetry(fullSearchUrl, null, null, APIHeaders);
return CheckResponse(result);
}
protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
{
var releases = new List<ReleaseInfo>();
var searchString = query.GetQueryString();
var page = 0;
var cats = MapTorznabCapsToTrackers(query);
var category = cats.Count > 0 ? string.Join(",", cats) : "all";
var isLatestSearch = string.IsNullOrWhiteSpace(searchString);
var isLastPage = false;
do
{
var result = await SendSearchRequest(searchString, category, page);
try
{
foreach (var torrent in result)
{
releases.Add(MakeRelease(torrent));
}
}
catch (Exception ex)
{
OnParseError(result.ToString(), ex);
}
isLastPage = result.Count < MAX_RESULTS_PER_PAGE;
page++; // update page number
} while (!isLatestSearch && !isLastPage && page < MAX_SEARCH_PAGE_LIMIT);
return releases;
}
private ReleaseInfo MakeRelease(JToken torrent)
{
var release = new ReleaseInfo();
release.Title = (string)torrent["title"];
// https://solidtorrents.net/view/5e10885d651df640a70ee826
release.Comments = new Uri(SiteLink + "view/" + (string)torrent["_id"]);
release.Guid = release.Comments;
release.PublishDate = DateTime.Now;
if (torrent["imported"] != null)
release.PublishDate = DateTime.Parse((string)torrent["imported"]);
release.Category = MapTrackerCatToNewznab((string)torrent["category"]);
release.Size = (long)torrent["size"];
var swarm = torrent["swarm"];
release.Seeders = (int)swarm["seeders"];
release.Peers = release.Seeders + (int)swarm["leechers"];
release.Grabs = (long)swarm["downloads"];
release.InfoHash = (string)torrent["infohash"];
release.MagnetUri = new Uri((string)torrent["magnet"]);
/*
release.MinimumRatio = 0;
release.MinimumSeedTime = 0;
*/
release.DownloadVolumeFactor = 0;
release.UploadVolumeFactor = 1;
return release;
}
}
}

View File

@ -382,6 +382,7 @@ namespace Jackett.Updater
"Definitions/rgu.yml",
"Definitions/elittracker.yml",
"Definitions/hon3yhd-net.yml",
"Definitions/solidtorrents.yml",
};
foreach (var oldFile in oldFiles)