From 79d26b39d1f18c84e5682b239b7a68dbb9af7d76 Mon Sep 17 00:00:00 2001 From: Bogdan Date: Wed, 4 Oct 2023 06:49:00 +0300 Subject: [PATCH] filelist: parse response with STJson (#14740) --- src/Jackett.Common/Indexers/FileList.cs | 146 ++++++++++++------ src/Jackett.Common/Jackett.Common.csproj | 1 + .../Serializer/BooleanConverter.cs | 31 ++++ src/Jackett.Common/Serializer/STJson.cs | 36 +++++ 4 files changed, 167 insertions(+), 47 deletions(-) create mode 100644 src/Jackett.Common/Serializer/BooleanConverter.cs create mode 100644 src/Jackett.Common/Serializer/STJson.cs diff --git a/src/Jackett.Common/Indexers/FileList.cs b/src/Jackett.Common/Indexers/FileList.cs index f73ca9f7b..d6f640597 100644 --- a/src/Jackett.Common/Indexers/FileList.cs +++ b/src/Jackett.Common/Indexers/FileList.cs @@ -3,16 +3,22 @@ using System.Collections.Generic; using System.Collections.Specialized; using System.Diagnostics.CodeAnalysis; using System.Globalization; +using System.Linq; +using System.Net; using System.Text; +using System.Text.Json.Serialization; using System.Threading.Tasks; +using Jackett.Common.Exceptions; using Jackett.Common.Extensions; using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig.Bespoke; +using Jackett.Common.Serializer; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; +using WebClient = Jackett.Common.Utils.Clients.WebClient; namespace Jackett.Common.Indexers { @@ -112,78 +118,72 @@ namespace Jackett.Common.Indexers public override async Task ApplyConfiguration(JToken configJson) { LoadValuesFromJson(configJson); - var pingResponse = await CallProviderAsync(new TorznabQuery()); - if (pingResponse.StartsWith("{\"error\"")) - { - throw new ExceptionWithConfigData(pingResponse, configData); - } + var releases = await PerformQuery(new TorznabQuery()); + await ConfigureIfOK(string.Empty, releases.Any(), + () => throw new Exception("Could not find releases.")); - try - { - var json = JArray.Parse(pingResponse); - if (json.Count > 0) - { - IsConfigured = true; - SaveConfig(); - return IndexerConfigurationStatus.Completed; - } - } - catch (Exception ex) - { - throw new ExceptionWithConfigData(ex.Message, configData); - } - - return IndexerConfigurationStatus.RequiresTesting; + return IndexerConfigurationStatus.Completed; } protected override async Task> PerformQuery(TorznabQuery query) { var releases = new List(); - var response = await CallProviderAsync(query); + + var indexerResponse = await CallProviderAsync(query); + var response = indexerResponse.ContentString; + + if ((int)indexerResponse.Status == 429) + { + throw new TooManyRequestsException("Rate limited", indexerResponse); + } if (response.StartsWith("{\"error\"")) { - throw new ExceptionWithConfigData(response, configData); + var error = STJson.Deserialize(response).Error; + + throw new ExceptionWithConfigData(error, configData); + } + + if (indexerResponse.Status != HttpStatusCode.OK) + { + throw new Exception($"Unknown status code: {(int)indexerResponse.Status} ({indexerResponse.Status})"); } try { - var json = JArray.Parse(response); + var results = STJson.Deserialize>(response); - foreach (var row in json) + foreach (var row in results) { - var isFreeleech = row.Value("freeleech"); + var isFreeleech = row.FreeLeech; // skip non-freeleech results when freeleech only is set if (configData.Freeleech.Value && !isFreeleech) + { continue; + } - var detailsUri = new Uri(DetailsUrl + "?id=" + row.Value("id")); - var seeders = row.Value("seeders"); - var peers = seeders + row.Value("leechers"); - var publishDate = DateTime.Parse(row.Value("upload_date") + " +0300", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal); - var downloadVolumeFactor = isFreeleech ? 0 : 1; - var uploadVolumeFactor = row.Value("doubleup") ? 2 : 1; - var imdbId = ((JObject)row).ContainsKey("imdb") ? ParseUtil.GetImdbId(row.Value("imdb")) : null; - var link = new Uri(row.Value("download_link")); + var detailsUri = new Uri($"{DetailsUrl}?id={row.Id}"); + var link = new Uri(row.DownloadLink); + var imdbId = row.ImdbId.IsNotNullOrWhiteSpace() ? ParseUtil.GetImdbId(row.ImdbId) : null; var release = new ReleaseInfo { Guid = detailsUri, Details = detailsUri, Link = link, - Title = row.Value("name").Trim(), - Category = MapTrackerCatDescToNewznab(row.Value("category")), - Size = row.Value("size"), - Files = row.Value("files"), - Grabs = row.Value("times_completed"), - Seeders = seeders, - Peers = peers, + Title = row.Name.Trim(), + Category = MapTrackerCatDescToNewznab(row.Category), + Size = row.Size, + Files = row.Files, + Grabs = row.TimesCompleted, + Seeders = row.Seeders, + Peers = row.Seeders + row.Leechers, Imdb = imdbId, - PublishDate = publishDate, - DownloadVolumeFactor = downloadVolumeFactor, - UploadVolumeFactor = uploadVolumeFactor, + PublishDate = DateTime.Parse(row.UploadDate + " +0300", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal), + DownloadVolumeFactor = isFreeleech ? 0 : 1, + UploadVolumeFactor = row.DoubleUp ? 2 : 1, MinimumRatio = 1, MinimumSeedTime = 172800 // 48 hours }; @@ -201,7 +201,7 @@ namespace Jackett.Common.Indexers return releases; } - private async Task CallProviderAsync(TorznabQuery query) + private async Task CallProviderAsync(TorznabQuery query) { var searchUrl = ApiUrl; var searchString = query.SanitizedSearchTerm.Trim(); @@ -212,7 +212,9 @@ namespace Jackett.Common.Indexers }; if (configData.Freeleech.Value) + { queryCollection.Set("freeleech", "1"); + } if (query.IsImdbQuery || searchString.IsNotNullOrWhiteSpace()) { @@ -230,13 +232,19 @@ namespace Jackett.Common.Indexers } if (query.Season > 0) + { queryCollection.Set("season", query.Season.ToString()); + } if (query.Episode.IsNotNullOrWhiteSpace()) + { queryCollection.Set("episode", query.Episode); + } } else + { queryCollection.Set("action", "latest-torrents"); + } searchUrl += "?" + queryCollection.GetQueryString(); @@ -247,9 +255,8 @@ namespace Jackett.Common.Indexers { {"Authorization", "Basic " + auth} }; - var response = await RequestWithCookiesAsync(searchUrl, headers: headers); - return response.ContentString; + return await RequestWithCookiesAsync(searchUrl, headers: headers); } catch (Exception inner) { @@ -257,4 +264,49 @@ namespace Jackett.Common.Indexers } } } + + public class FileListTorrent + { + public uint Id { get; set; } + + public string Name { get; set; } + + [JsonPropertyName("download_link")] + public string DownloadLink { get; set; } + + public long Size { get; set; } + + public int Leechers { get; set; } + + public int Seeders { get; set; } + + [JsonPropertyName("times_completed")] + public uint TimesCompleted { get; set; } + + public uint Files { get; set; } + + [JsonPropertyName("imdb")] + public string ImdbId { get; set; } + + public bool Internal { get; set; } + + [JsonPropertyName("freeleech")] + public bool FreeLeech { get; set; } + + [JsonPropertyName("doubleup")] + public bool DoubleUp { get; set; } + + [JsonPropertyName("upload_date")] + public string UploadDate { get; set; } + + public string Category { get; set; } + + [JsonPropertyName("small_description")] + public string SmallDescription { get; set; } + } + + public class FileListErrorResponse + { + public string Error { get; set; } + } } diff --git a/src/Jackett.Common/Jackett.Common.csproj b/src/Jackett.Common/Jackett.Common.csproj index 83a20c528..88fd8415d 100644 --- a/src/Jackett.Common/Jackett.Common.csproj +++ b/src/Jackett.Common/Jackett.Common.csproj @@ -28,6 +28,7 @@ + diff --git a/src/Jackett.Common/Serializer/BooleanConverter.cs b/src/Jackett.Common/Serializer/BooleanConverter.cs new file mode 100644 index 000000000..aa3add1d2 --- /dev/null +++ b/src/Jackett.Common/Serializer/BooleanConverter.cs @@ -0,0 +1,31 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Jackett.Common.Serializer +{ + public class BooleanConverter : JsonConverter + { + public override bool Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + return reader.TokenType switch + { + JsonTokenType.True => true, + JsonTokenType.False => false, + JsonTokenType.Number => reader.GetInt64() switch + { + 1 => true, + 0 => false, + _ => throw new JsonException() + }, + _ => throw new JsonException() + }; + } + + public override void Write(Utf8JsonWriter writer, bool value, JsonSerializerOptions options) + { + writer.WriteBooleanValue(value); + } + } + +} diff --git a/src/Jackett.Common/Serializer/STJson.cs b/src/Jackett.Common/Serializer/STJson.cs new file mode 100644 index 000000000..59f89d76b --- /dev/null +++ b/src/Jackett.Common/Serializer/STJson.cs @@ -0,0 +1,36 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Jackett.Common.Serializer +{ + public static class STJson + { + private static readonly JsonSerializerOptions _SerializerSettings = GetSerializerSettings(); + + public static JsonSerializerOptions GetSerializerSettings() + { + var settings = new JsonSerializerOptions(); + ApplySerializerSettings(settings); + return settings; + } + + public static void ApplySerializerSettings(JsonSerializerOptions serializerSettings) + { + serializerSettings.AllowTrailingCommas = true; + serializerSettings.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull; + serializerSettings.PropertyNameCaseInsensitive = true; + serializerSettings.DictionaryKeyPolicy = JsonNamingPolicy.CamelCase; + serializerSettings.PropertyNamingPolicy = JsonNamingPolicy.CamelCase; + serializerSettings.WriteIndented = true; + + serializerSettings.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase, true)); + serializerSettings.Converters.Add(new BooleanConverter()); + } + + public static T Deserialize(string json) + where T : new() + { + return JsonSerializer.Deserialize(json, _SerializerSettings); + } + } +}