filelist: parse response with STJson (#14740)

src/Jackett.Common/Indexers/FileList.cs

@@ -3,16 +3,22 @@ using System.Collections.Generic;
 using System.Collections.Specialized;
 using System.Diagnostics.CodeAnalysis;
 using System.Globalization;
+using System.Linq;
+using System.Net;
 using System.Text;
+using System.Text.Json.Serialization;
 using System.Threading.Tasks;
+using Jackett.Common.Exceptions;
 using Jackett.Common.Extensions;
 using Jackett.Common.Models;
 using Jackett.Common.Models.IndexerConfig.Bespoke;
+using Jackett.Common.Serializer;
 using Jackett.Common.Services.Interfaces;
 using Jackett.Common.Utils;
 using Jackett.Common.Utils.Clients;
 using Newtonsoft.Json.Linq;
 using NLog;
+using WebClient = Jackett.Common.Utils.Clients.WebClient;
 
 namespace Jackett.Common.Indexers
 {
@@ -112,78 +118,72 @@ namespace Jackett.Common.Indexers
         public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
         {
             LoadValuesFromJson(configJson);
-            var pingResponse = await CallProviderAsync(new TorznabQuery());
 
-            if (pingResponse.StartsWith("{\"error\""))
+            var releases = await PerformQuery(new TorznabQuery());
-            {
+            await ConfigureIfOK(string.Empty, releases.Any(),
-                throw new ExceptionWithConfigData(pingResponse, configData);
+                                () => throw new Exception("Could not find releases."));
-            }
 
-            try
+            return IndexerConfigurationStatus.Completed;
-            {
-                var json = JArray.Parse(pingResponse);
-                if (json.Count > 0)
-                {
-                    IsConfigured = true;
-                    SaveConfig();
-                    return IndexerConfigurationStatus.Completed;
-                }
-            }
-            catch (Exception ex)
-            {
-                throw new ExceptionWithConfigData(ex.Message, configData);
-            }
-
-            return IndexerConfigurationStatus.RequiresTesting;
         }
 
         protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
         {
             var releases = new List<ReleaseInfo>();
-            var response = await CallProviderAsync(query);
+
+            var indexerResponse = await CallProviderAsync(query);
+            var response = indexerResponse.ContentString;
+
+            if ((int)indexerResponse.Status == 429)
+            {
+                throw new TooManyRequestsException("Rate limited", indexerResponse);
+            }
 
             if (response.StartsWith("{\"error\""))
             {
-                throw new ExceptionWithConfigData(response, configData);
+                var error = STJson.Deserialize<FileListErrorResponse>(response).Error;
+
+                throw new ExceptionWithConfigData(error, configData);
+            }
+
+            if (indexerResponse.Status != HttpStatusCode.OK)
+            {
+                throw new Exception($"Unknown status code: {(int)indexerResponse.Status} ({indexerResponse.Status})");
             }
 
             try
             {
-                var json = JArray.Parse(response);
+                var results = STJson.Deserialize<List<FileListTorrent>>(response);
 
-                foreach (var row in json)
+                foreach (var row in results)
                 {
-                    var isFreeleech = row.Value<bool>("freeleech");
+                    var isFreeleech = row.FreeLeech;
 
                     // skip non-freeleech results when freeleech only is set
                     if (configData.Freeleech.Value && !isFreeleech)
+                    {
                         continue;
+                    }
 
-                    var detailsUri = new Uri(DetailsUrl + "?id=" + row.Value<string>("id"));
+                    var detailsUri = new Uri($"{DetailsUrl}?id={row.Id}");
-                    var seeders = row.Value<int>("seeders");
+                    var link = new Uri(row.DownloadLink);
-                    var peers = seeders + row.Value<int>("leechers");
+                    var imdbId = row.ImdbId.IsNotNullOrWhiteSpace() ? ParseUtil.GetImdbId(row.ImdbId) : null;
-                    var publishDate = DateTime.Parse(row.Value<string>("upload_date") + " +0300", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal);
-                    var downloadVolumeFactor = isFreeleech ? 0 : 1;
-                    var uploadVolumeFactor = row.Value<bool>("doubleup") ? 2 : 1;
-                    var imdbId = ((JObject)row).ContainsKey("imdb") ? ParseUtil.GetImdbId(row.Value<string>("imdb")) : null;
-                    var link = new Uri(row.Value<string>("download_link"));
 
                     var release = new ReleaseInfo
                     {
                         Guid = detailsUri,
                         Details = detailsUri,
                         Link = link,
-                        Title = row.Value<string>("name").Trim(),
+                        Title = row.Name.Trim(),
-                        Category = MapTrackerCatDescToNewznab(row.Value<string>("category")),
+                        Category = MapTrackerCatDescToNewznab(row.Category),
-                        Size = row.Value<long>("size"),
+                        Size = row.Size,
-                        Files = row.Value<long>("files"),
+                        Files = row.Files,
-                        Grabs = row.Value<long>("times_completed"),
+                        Grabs = row.TimesCompleted,
-                        Seeders = seeders,
+                        Seeders = row.Seeders,
-                        Peers = peers,
+                        Peers = row.Seeders + row.Leechers,
                         Imdb = imdbId,
-                        PublishDate = publishDate,
+                        PublishDate = DateTime.Parse(row.UploadDate + " +0300", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal),
-                        DownloadVolumeFactor = downloadVolumeFactor,
+                        DownloadVolumeFactor = isFreeleech ? 0 : 1,
-                        UploadVolumeFactor = uploadVolumeFactor,
+                        UploadVolumeFactor = row.DoubleUp ? 2 : 1,
                         MinimumRatio = 1,
                         MinimumSeedTime = 172800 // 48 hours
                     };
@@ -201,7 +201,7 @@ namespace Jackett.Common.Indexers
             return releases;
         }
 
-        private async Task<string> CallProviderAsync(TorznabQuery query)
+        private async Task<WebResult> CallProviderAsync(TorznabQuery query)
         {
             var searchUrl = ApiUrl;
             var searchString = query.SanitizedSearchTerm.Trim();
@@ -212,7 +212,9 @@ namespace Jackett.Common.Indexers
             };
 
             if (configData.Freeleech.Value)
+            {
                 queryCollection.Set("freeleech", "1");
+            }
 
             if (query.IsImdbQuery || searchString.IsNotNullOrWhiteSpace())
             {
@@ -230,13 +232,19 @@ namespace Jackett.Common.Indexers
                 }
 
                 if (query.Season > 0)
+                {
                     queryCollection.Set("season", query.Season.ToString());
+                }
 
                 if (query.Episode.IsNotNullOrWhiteSpace())
+                {
                     queryCollection.Set("episode", query.Episode);
+                }
             }
             else
+            {
                 queryCollection.Set("action", "latest-torrents");
+            }
 
             searchUrl += "?" + queryCollection.GetQueryString();
 
@@ -247,9 +255,8 @@ namespace Jackett.Common.Indexers
                 {
                     {"Authorization", "Basic " + auth}
                 };
-                var response = await RequestWithCookiesAsync(searchUrl, headers: headers);
 
-                return response.ContentString;
+                return await RequestWithCookiesAsync(searchUrl, headers: headers);
             }
             catch (Exception inner)
             {
@@ -257,4 +264,49 @@ namespace Jackett.Common.Indexers
             }
         }
     }
+
+    public class FileListTorrent
+    {
+        public uint Id { get; set; }
+
+        public string Name { get; set; }
+
+        [JsonPropertyName("download_link")]
+        public string DownloadLink { get; set; }
+
+        public long Size { get; set; }
+
+        public int Leechers { get; set; }
+
+        public int Seeders { get; set; }
+
+        [JsonPropertyName("times_completed")]
+        public uint TimesCompleted { get; set; }
+
+        public uint Files { get; set; }
+
+        [JsonPropertyName("imdb")]
+        public string ImdbId { get; set; }
+
+        public bool Internal { get; set; }
+
+        [JsonPropertyName("freeleech")]
+        public bool FreeLeech { get; set; }
+
+        [JsonPropertyName("doubleup")]
+        public bool DoubleUp { get; set; }
+
+        [JsonPropertyName("upload_date")]
+        public string UploadDate { get; set; }
+
+        public string Category { get; set; }
+
+        [JsonPropertyName("small_description")]
+        public string SmallDescription { get; set; }
+    }
+
+    public class FileListErrorResponse
+    {
+        public string Error { get; set; }
+    }
 }

src/Jackett.Common/Jackett.Common.csproj

@@ -28,6 +28,7 @@
     <PackageReference Include="SharpZipLib" Version="1.4.2" />
     <PackageReference Include="System.IO.FileSystem.AccessControl" Version="5.0.0" />
     <PackageReference Include="System.ServiceProcess.ServiceController" Version="6.0.0" />
+    <PackageReference Include="System.Text.Json" Version="6.0.8" />
     <PackageReference Include="YamlDotNet" Version="13.0.1" />
   </ItemGroup>
 

src/Jackett.Common/Serializer/BooleanConverter.cs

@@ -0,0 +1,31 @@
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Jackett.Common.Serializer
+{
+    public class BooleanConverter : JsonConverter<bool>
+    {
+        public override bool Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+        {
+            return reader.TokenType switch
+            {
+                JsonTokenType.True => true,
+                JsonTokenType.False => false,
+                JsonTokenType.Number => reader.GetInt64() switch
+                {
+                    1 => true,
+                    0 => false,
+                    _ => throw new JsonException()
+                },
+                _ => throw new JsonException()
+            };
+        }
+
+        public override void Write(Utf8JsonWriter writer, bool value, JsonSerializerOptions options)
+        {
+            writer.WriteBooleanValue(value);
+        }
+    }
+
+}

src/Jackett.Common/Serializer/STJson.cs

@@ -0,0 +1,36 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Jackett.Common.Serializer
+{
+    public static class STJson
+    {
+        private static readonly JsonSerializerOptions _SerializerSettings = GetSerializerSettings();
+
+        public static JsonSerializerOptions GetSerializerSettings()
+        {
+            var settings = new JsonSerializerOptions();
+            ApplySerializerSettings(settings);
+            return settings;
+        }
+
+        public static void ApplySerializerSettings(JsonSerializerOptions serializerSettings)
+        {
+            serializerSettings.AllowTrailingCommas = true;
+            serializerSettings.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull;
+            serializerSettings.PropertyNameCaseInsensitive = true;
+            serializerSettings.DictionaryKeyPolicy = JsonNamingPolicy.CamelCase;
+            serializerSettings.PropertyNamingPolicy = JsonNamingPolicy.CamelCase;
+            serializerSettings.WriteIndented = true;
+
+            serializerSettings.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase, true));
+            serializerSettings.Converters.Add(new BooleanConverter());
+        }
+
+        public static T Deserialize<T>(string json)
+            where T : new()
+        {
+            return JsonSerializer.Deserialize<T>(json, _SerializerSettings);
+        }
+    }
+}