mirror
/
Jackett
mirror of https://github.com/Jackett/Jackett
1
0
Fork 0
Code Issues Releases Wiki Activity

indexers: filter invalid releases without categories or size (#15213) 

* indexers: filter invalid releases without categories or size

* Update BaseIndexer.cs

---------

Co-authored-by: garfield69 <garfield69@outlook.com>
This commit is contained in:
Bogdan 2024-04-01 03:46:50 +03:00 committed by GitHub
parent 00540e76a9
commit 10b3d84482
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 87 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/Jackett.Common/Indexers/BaseIndexer.cs
View File

@ -241,7 +241,21 @@ namespace Jackett.Common.Indexers
{
if (release.Title.IsNullOrWhiteSpace())
{
logger.Error("Invalid Release: '{0}' from indexer: {1}. No title provided.", release.Details, Name);
logger.Error("[{0}] Invalid Release: '{1}'. No title provided.", Id, release.Details);
return false;
}
if (release.Size == null)
{
logger.Warn("[{0}] Invalid Release: '{1}'. No size provided.", Id, release.Details);
return false;
}
if (release.Category == null || !release.Category.Any())
{
logger.Warn("[{0}] Invalid Release: '{1}'. No categories provided.", Id, release.Details);
return false;
}

42
src/Jackett.Common/Indexers/CardigannIndexer.cs
View File

@ -2193,34 +2193,52 @@ namespace Jackett.Common.Indexers
case "category":
if (FieldModifiers.Contains("noappend"))
{
logger.Warn($"CardigannIndexer ({Id}): The \"noappend\" modifier is deprecated. Please switch to \"default\". See the Definition Format in the Wiki for more information.");
logger.Warn("CardigannIndexer ({0}): The \"noappend\" modifier is deprecated. Please switch to \"default\". See the Definition Format in the Wiki for more information.", Id);
}
var cats = MapTrackerCatToNewznab(value);
if (cats.Any())
{
if (release.Category == null || FieldModifiers.Contains("noappend"))
release.Category = cats;
else
release.Category = release.Category.Union(cats).ToList();
release.Category = release.Category == null || FieldModifiers.Contains("noappend")
? cats
: release.Category.Union(cats).ToList();
}
value = release.Category.ToString();
if (value.IsNotNullOrWhiteSpace() && !release.Category.Any())
{
logger.Warn("[{0}] Invalid category for value: '{1}'", Id, value);
}
else
{
value = release.Category.ToString();
}
break;
case "categorydesc":
if (FieldModifiers.Contains("noappend"))
{
logger.Warn($"CardigannIndexer ({Id}): The \"noappend\" modifier is deprecated. Please switch to \"default\". See the Definition Format in the Wiki for more information.");
logger.Warn("CardigannIndexer ({0}): The \"noappend\" modifier is deprecated. Please switch to \"default\". See the Definition Format in the Wiki for more information.", Id);
}
var catsDesc = MapTrackerCatDescToNewznab(value);
if (catsDesc.Any())
{
if (release.Category == null || FieldModifiers.Contains("noappend"))
release.Category = catsDesc;
else
release.Category = release.Category.Union(catsDesc).ToList();
release.Category = release.Category == null || FieldModifiers.Contains("noappend")
? catsDesc
: release.Category.Union(catsDesc).ToList();
}
value = release.Category.ToString();
if (value.IsNotNullOrWhiteSpace() && !release.Category.Any())
{
logger.Warn("[{0}] Invalid category for value: '{1}'", Id, value);
}
else
{
value = release.Category.ToString();
}
break;
case "size":
release.Size = ParseUtil.GetBytes(value);

1
src/Jackett.Common/Models/ReleaseInfo.cs
View File

@ -54,6 +54,7 @@ namespace Jackett.Common.Models
public ReleaseInfo()
{
Category = new List<int>();
Languages = new List<string>();
Subs = new List<string>();
}

57
src/Jackett.Test/Common/Indexers/BaseWebIndexerTests.cs
View File

@ -1,8 +1,10 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Jackett.Common.Helpers;
using Jackett.Common.Models;
using Jackett.Test.TestHelpers;
using NLog;
using NUnit.Framework;
using Assert = NUnit.Framework.Assert;
@ -11,10 +13,12 @@ namespace Jackett.Test.Common.Indexers
[TestFixture]
public class BaseWebIndexerTests
{
private readonly Logger _logger = LogManager.GetCurrentClassLogger();
[Test]
public void TestConstructor()
{
var indexer = new TestWebIndexer();
var indexer = new TestWebIndexer(_logger);
var caps = indexer.TorznabCaps;
Assert.True(caps.SearchAvailable);
@ -45,7 +49,7 @@ namespace Jackett.Test.Common.Indexers
[Test]
public void TestFilterResultsCategories()
{
var indexer = new TestWebIndexer();
var indexer = new TestWebIndexer(_logger);
indexer.AddTestCategories();
var results = new List<ReleaseInfo>
@ -53,76 +57,81 @@ namespace Jackett.Test.Common.Indexers
new ReleaseInfo
{
Title = "Release 1",
Category = new List<int> { TorznabCatType.Movies.ID }
Category = new List<int> { TorznabCatType.Movies.ID },
Size = 2.Gigabytes()
},
new ReleaseInfo
{
Title = "Release 2",
Category = new List<int> { TorznabCatType.MoviesSD.ID }
Category = new List<int> { TorznabCatType.MoviesSD.ID },
Size = 2.Gigabytes()
},
new ReleaseInfo
{
Title = "Release 3",
Category = new List<int> { TorznabCatType.BooksEBook.ID, 100004 } // torznab (mandatory) + custom cat
Category = new List<int> { TorznabCatType.BooksEBook.ID, 100004 }, // torznab (mandatory) + custom cat
Size = 2.Gigabytes()
},
new ReleaseInfo
{
Title = "Release 4",
Category = new List<int> { TorznabCatType.AudioLossless.ID } // unsupported category in this indexer
Category = new List<int> { TorznabCatType.AudioLossless.ID }, // unsupported category in this indexer
Size = 2.Gigabytes()
},
new ReleaseInfo
{
Title = "Release 5",
Title = "Release 5"
}
};
var query = new TorznabQuery(); // without categories
var filteredResults = indexer._FilterResults(query, results).ToList();
Assert.AreEqual(5, filteredResults.Count);
Assert.AreEqual(4, filteredResults.Count);
query = new TorznabQuery // with child category
{
Categories = new[] { TorznabCatType.MoviesSD.ID }
};
filteredResults = indexer._FilterResults(query, results).ToList();
Assert.AreEqual(2, filteredResults.Count);
Assert.AreEqual(1, filteredResults.Count);
Assert.AreEqual(TorznabCatType.MoviesSD.ID, filteredResults[0].Category.First());
Assert.AreEqual(null, filteredResults[1].Category);
query = new TorznabQuery // with parent category
{
Categories = new[] { TorznabCatType.Movies.ID }
};
filteredResults = indexer._FilterResults(query, results).ToList();
Assert.AreEqual(3, filteredResults.Count);
Assert.AreEqual(2, filteredResults.Count);
Assert.AreEqual(TorznabCatType.Movies.ID, filteredResults[0].Category.First());
Assert.AreEqual(TorznabCatType.MoviesSD.ID, filteredResults[1].Category.First());
Assert.AreEqual(null, filteredResults[2].Category);
query = new TorznabQuery // with custom category
{
Categories = new[] { 100004 }
};
filteredResults = indexer._FilterResults(query, results).ToList();
Assert.AreEqual(2, filteredResults.Count);
Assert.AreEqual(1, filteredResults.Count);
Assert.AreEqual(TorznabCatType.BooksEBook.ID, filteredResults[0].Category.First());
Assert.AreEqual(null, filteredResults[1].Category);
}
[Test]
public void TestFilterResultsLimit()
{
var indexer = new TestWebIndexer();
var indexer = new TestWebIndexer(_logger);
var results = new List<ReleaseInfo>
{
new ReleaseInfo
{
Title = "Release 1"
Title = "Release 1",
Category = new List<int> { TorznabCatType.Movies.ID },
Size = 2.Gigabytes()
},
new ReleaseInfo
{
Title = "Release 2"
Title = "Release 2",
Category = new List<int> { TorznabCatType.Movies.ID },
Size = 2.Gigabytes()
}
};
@ -141,7 +150,7 @@ namespace Jackett.Test.Common.Indexers
[Test]
public void TestFixResultsOriginPublishDate()
{
var indexer = new TestWebIndexer();
var indexer = new TestWebIndexer(_logger);
var query = new TorznabQuery();
var results = new List<ReleaseInfo>
{
@ -163,7 +172,7 @@ namespace Jackett.Test.Common.Indexers
[Test]
public void TestFixResultsMagnet()
{
var indexer = new TestWebIndexer();
var indexer = new TestWebIndexer(_logger);
var query = new TorznabQuery();
// get info_hash from magnet
@ -202,7 +211,7 @@ namespace Jackett.Test.Common.Indexers
[Test]
public void TestAddCategoryMapping()
{
var indexer = new TestWebIndexer();
var indexer = new TestWebIndexer(_logger);
// you can find more complex tests in TorznabCapabilitiesCategoriesTests.cs
indexer._AddCategoryMapping("11", TorznabCatType.MoviesSD, "MoviesSD");
@ -222,7 +231,7 @@ namespace Jackett.Test.Common.Indexers
[Test]
public void TestAddMultiCategoryMapping()
{
var indexer = new TestWebIndexer();
var indexer = new TestWebIndexer(_logger);
indexer._AddMultiCategoryMapping(TorznabCatType.MoviesHD, 19, 18);
Assert.AreEqual(1, indexer.TorznabCaps.Categories.GetTorznabCategoryTree().Count);
@ -231,7 +240,7 @@ namespace Jackett.Test.Common.Indexers
[Test]
public void TestMapTorznabCapsToTrackers()
{
var indexer = new TestWebIndexer();
var indexer = new TestWebIndexer(_logger);
indexer.AddTestCategories();
// you can find more complex tests in TorznabCapabilitiesCategoriesTests.cs
@ -248,7 +257,7 @@ namespace Jackett.Test.Common.Indexers
[Test]
public void TestMapTrackerCatToNewznab()
{
var indexer = new TestWebIndexer();
var indexer = new TestWebIndexer(_logger);
indexer.AddTestCategories();
// you can find more complex tests in TorznabCapabilitiesCategoriesTests.cs
@ -260,7 +269,7 @@ namespace Jackett.Test.Common.Indexers
[Test]
public void TestMapTrackerCatDescToNewznab()
{
var indexer = new TestWebIndexer();
var indexer = new TestWebIndexer(_logger);
indexer.AddTestCategories();
// you can find more complex tests in TorznabCapabilitiesCategoriesTests.cs

7
src/Jackett.Test/Common/Models/DTO/IndexerTests.cs
View File

@ -1,6 +1,7 @@
using System.Linq;
using Jackett.Common.Models.DTO;
using Jackett.Test.TestHelpers;
using NLog;
using NUnit.Framework;
using Assert = NUnit.Framework.Assert;
@ -9,10 +10,12 @@ namespace Jackett.Test.Common.Models.DTO
[TestFixture]
public class IndexerTests
{
private readonly Logger _logger = LogManager.GetCurrentClassLogger();
[Test]
public void TestConstructor()
{
var indexer = new TestWebIndexer();
var indexer = new TestWebIndexer(_logger);
var dto = new Indexer(indexer);
Assert.AreEqual("test_id", dto.id);
@ -31,7 +34,7 @@ namespace Jackett.Test.Common.Models.DTO
[Test]
public void TestConstructorWithCategories()
{
var indexer = new TestWebIndexer();
var indexer = new TestWebIndexer(_logger);
indexer.AddTestCategories();
// test Jackett UI categories (internal JSON)

5
src/Jackett.Test/TestHelpers/TestWebIndexer.cs
View File

@ -6,6 +6,7 @@ using Jackett.Common.Indexers;
using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Newtonsoft.Json.Linq;
using NLog;
namespace Jackett.Test.TestHelpers
{
@ -30,8 +31,8 @@ namespace Jackett.Test.TestHelpers
public override TorznabCapabilities TorznabCaps { get; protected set; } = new TorznabCapabilities();
public TestWebIndexer()
: base(client: null, configService: null, logger: null, configData: new ConfigurationData(), p: null, cacheService: null)
public TestWebIndexer(Logger logger)
: base(client: null, configService: null, logger: logger, configData: new ConfigurationData(), p: null, cacheService: null)
{
}