mirror
/
Jackett
mirror of https://github.com/Jackett/Jackett
1
0
Fork 0
Code Issues Releases Wiki Activity

filelist: parse response with STJson (#14740)

This commit is contained in:
Bogdan 2023-10-04 06:49:00 +03:00 committed by GitHub
parent 41227f2c07
commit 79d26b39d1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 167 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

146
src/Jackett.Common/Indexers/FileList.cs
View File

@ -3,16 +3,22 @@ using System.Collections.Generic;
using System.Collections.Specialized;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
using Jackett.Common.Exceptions;
using Jackett.Common.Extensions;
using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig.Bespoke;
using Jackett.Common.Serializer;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json.Linq;
using NLog;
using WebClient = Jackett.Common.Utils.Clients.WebClient;
namespace Jackett.Common.Indexers
{
@ -112,78 +118,72 @@ namespace Jackett.Common.Indexers
public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
{
LoadValuesFromJson(configJson);
var pingResponse = await CallProviderAsync(new TorznabQuery());
if (pingResponse.StartsWith("{\"error\""))
{
throw new ExceptionWithConfigData(pingResponse, configData);
}
var releases = await PerformQuery(new TorznabQuery());
await ConfigureIfOK(string.Empty, releases.Any(),
() => throw new Exception("Could not find releases."));
try
{
var json = JArray.Parse(pingResponse);
if (json.Count > 0)
{
IsConfigured = true;
SaveConfig();
return IndexerConfigurationStatus.Completed;
}
}
catch (Exception ex)
{
throw new ExceptionWithConfigData(ex.Message, configData);
}
return IndexerConfigurationStatus.RequiresTesting;
return IndexerConfigurationStatus.Completed;
}
protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
{
var releases = new List<ReleaseInfo>();
var response = await CallProviderAsync(query);
var indexerResponse = await CallProviderAsync(query);
var response = indexerResponse.ContentString;
if ((int)indexerResponse.Status == 429)
{
throw new TooManyRequestsException("Rate limited", indexerResponse);
}
if (response.StartsWith("{\"error\""))
{
throw new ExceptionWithConfigData(response, configData);
var error = STJson.Deserialize<FileListErrorResponse>(response).Error;
throw new ExceptionWithConfigData(error, configData);
}
if (indexerResponse.Status != HttpStatusCode.OK)
{
throw new Exception($"Unknown status code: {(int)indexerResponse.Status} ({indexerResponse.Status})");
}
try
{
var json = JArray.Parse(response);
var results = STJson.Deserialize<List<FileListTorrent>>(response);
foreach (var row in json)
foreach (var row in results)
{
var isFreeleech = row.Value<bool>("freeleech");
var isFreeleech = row.FreeLeech;
// skip non-freeleech results when freeleech only is set
if (configData.Freeleech.Value && !isFreeleech)
{
continue;
}
var detailsUri = new Uri(DetailsUrl + "?id=" + row.Value<string>("id"));
var seeders = row.Value<int>("seeders");
var peers = seeders + row.Value<int>("leechers");
var publishDate = DateTime.Parse(row.Value<string>("upload_date") + " +0300", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal);
var downloadVolumeFactor = isFreeleech ? 0 : 1;
var uploadVolumeFactor = row.Value<bool>("doubleup") ? 2 : 1;
var imdbId = ((JObject)row).ContainsKey("imdb") ? ParseUtil.GetImdbId(row.Value<string>("imdb")) : null;
var link = new Uri(row.Value<string>("download_link"));
var detailsUri = new Uri($"{DetailsUrl}?id={row.Id}");
var link = new Uri(row.DownloadLink);
var imdbId = row.ImdbId.IsNotNullOrWhiteSpace() ? ParseUtil.GetImdbId(row.ImdbId) : null;
var release = new ReleaseInfo
{
Guid = detailsUri,
Details = detailsUri,
Link = link,
Title = row.Value<string>("name").Trim(),
Category = MapTrackerCatDescToNewznab(row.Value<string>("category")),
Size = row.Value<long>("size"),
Files = row.Value<long>("files"),
Grabs = row.Value<long>("times_completed"),
Seeders = seeders,
Peers = peers,
Title = row.Name.Trim(),
Category = MapTrackerCatDescToNewznab(row.Category),
Size = row.Size,
Files = row.Files,
Grabs = row.TimesCompleted,
Seeders = row.Seeders,
Peers = row.Seeders + row.Leechers,
Imdb = imdbId,
PublishDate = publishDate,
DownloadVolumeFactor = downloadVolumeFactor,
UploadVolumeFactor = uploadVolumeFactor,
PublishDate = DateTime.Parse(row.UploadDate + " +0300", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal),
DownloadVolumeFactor = isFreeleech ? 0 : 1,
UploadVolumeFactor = row.DoubleUp ? 2 : 1,
MinimumRatio = 1,
MinimumSeedTime = 172800 // 48 hours
};
@ -201,7 +201,7 @@ namespace Jackett.Common.Indexers
return releases;
}
private async Task<string> CallProviderAsync(TorznabQuery query)
private async Task<WebResult> CallProviderAsync(TorznabQuery query)
{
var searchUrl = ApiUrl;
var searchString = query.SanitizedSearchTerm.Trim();
@ -212,7 +212,9 @@ namespace Jackett.Common.Indexers
};
if (configData.Freeleech.Value)
{
queryCollection.Set("freeleech", "1");
}
if (query.IsImdbQuery || searchString.IsNotNullOrWhiteSpace())
{
@ -230,13 +232,19 @@ namespace Jackett.Common.Indexers
}
if (query.Season > 0)
{
queryCollection.Set("season", query.Season.ToString());
}
if (query.Episode.IsNotNullOrWhiteSpace())
{
queryCollection.Set("episode", query.Episode);
}
}
else
{
queryCollection.Set("action", "latest-torrents");
}
searchUrl += "?" + queryCollection.GetQueryString();
@ -247,9 +255,8 @@ namespace Jackett.Common.Indexers
{
{"Authorization", "Basic " + auth}
};
var response = await RequestWithCookiesAsync(searchUrl, headers: headers);
return response.ContentString;
return await RequestWithCookiesAsync(searchUrl, headers: headers);
}
catch (Exception inner)
{
@ -257,4 +264,49 @@ namespace Jackett.Common.Indexers
}
}
}
public class FileListTorrent
{
public uint Id { get; set; }
public string Name { get; set; }
[JsonPropertyName("download_link")]
public string DownloadLink { get; set; }
public long Size { get; set; }
public int Leechers { get; set; }
public int Seeders { get; set; }
[JsonPropertyName("times_completed")]
public uint TimesCompleted { get; set; }
public uint Files { get; set; }
[JsonPropertyName("imdb")]
public string ImdbId { get; set; }
public bool Internal { get; set; }
[JsonPropertyName("freeleech")]
public bool FreeLeech { get; set; }
[JsonPropertyName("doubleup")]
public bool DoubleUp { get; set; }
[JsonPropertyName("upload_date")]
public string UploadDate { get; set; }
public string Category { get; set; }
[JsonPropertyName("small_description")]
public string SmallDescription { get; set; }
}
public class FileListErrorResponse
{
public string Error { get; set; }
}
}

1
src/Jackett.Common/Jackett.Common.csproj
View File

@ -28,6 +28,7 @@
<PackageReference Include="SharpZipLib" Version="1.4.2" />
<PackageReference Include="System.IO.FileSystem.AccessControl" Version="5.0.0" />
<PackageReference Include="System.ServiceProcess.ServiceController" Version="6.0.0" />
<PackageReference Include="System.Text.Json" Version="6.0.8" />
<PackageReference Include="YamlDotNet" Version="13.0.1" />
</ItemGroup>

31
src/Jackett.Common/Serializer/BooleanConverter.cs Normal file
View File

@ -0,0 +1,31 @@
using System;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace Jackett.Common.Serializer
{
public class BooleanConverter : JsonConverter<bool>
{
public override bool Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
return reader.TokenType switch
{
JsonTokenType.True => true,
JsonTokenType.False => false,
JsonTokenType.Number => reader.GetInt64() switch
{
1 => true,
0 => false,
_ => throw new JsonException()
},
_ => throw new JsonException()
};
}
public override void Write(Utf8JsonWriter writer, bool value, JsonSerializerOptions options)
{
writer.WriteBooleanValue(value);
}
}
}

36
src/Jackett.Common/Serializer/STJson.cs Normal file
View File

@ -0,0 +1,36 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace Jackett.Common.Serializer
{
public static class STJson
{
private static readonly JsonSerializerOptions _SerializerSettings = GetSerializerSettings();
public static JsonSerializerOptions GetSerializerSettings()
{
var settings = new JsonSerializerOptions();
ApplySerializerSettings(settings);
return settings;
}
public static void ApplySerializerSettings(JsonSerializerOptions serializerSettings)
{
serializerSettings.AllowTrailingCommas = true;
serializerSettings.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull;
serializerSettings.PropertyNameCaseInsensitive = true;
serializerSettings.DictionaryKeyPolicy = JsonNamingPolicy.CamelCase;
serializerSettings.PropertyNamingPolicy = JsonNamingPolicy.CamelCase;
serializerSettings.WriteIndented = true;
serializerSettings.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase, true));
serializerSettings.Converters.Add(new BooleanConverter());
}
public static T Deserialize<T>(string json)
where T : new()
{
return JsonSerializer.Deserialize<T>(json, _SerializerSettings);
}
}
}