mirror
/
Radarr
mirror of https://github.com/Radarr/Radarr
1
0
Fork 0
Code Issues Releases Wiki Activity

New: Use languages from Torznab/Newznab attributes if given

This commit is contained in:
Qstick 2023-04-23 18:11:43 -05:00
parent 9bef430635
commit efd4abfa3e
7 changed files with 141 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/NzbDrone.Core.Test/Download/Aggregation/Aggregators/AggregateLanguagesFixture.cs
View File

@ -32,6 +32,7 @@ namespace NzbDrone.Core.Test.Download.Aggregation.Aggregators
_remoteMovie = Builder<RemoteMovie>.CreateNew()
.With(l => l.ParsedMovieInfo = null)
.With(l => l.Movie = _movie)
.With(l => l.Release = new ReleaseInfo())
.Build();
}

41
src/NzbDrone.Core/Download/Aggregation/Aggregators/AggregateLanguages.cs
View File

@ -20,6 +20,7 @@ namespace NzbDrone.Core.Download.Aggregation.Aggregators
public RemoteMovie Aggregate(RemoteMovie remoteMovie)
{
var parsedMovieInfo = remoteMovie.ParsedMovieInfo;
var releaseInfo = remoteMovie.Release;
var languages = parsedMovieInfo.Languages;
var movie = remoteMovie.Movie;
var releaseTokens = parsedMovieInfo.SimpleReleaseTitle ?? parsedMovieInfo.ReleaseTitle;
@ -30,30 +31,40 @@ namespace NzbDrone.Core.Download.Aggregation.Aggregators
{
_logger.Debug("Unable to aggregate languages, using parsed values: {0}", string.Join(", ", languages.ToList()));
remoteMovie.Languages = languages;
remoteMovie.Languages = releaseInfo.Languages.Any() ? releaseInfo.Languages : languages;
return remoteMovie;
}
var movieTitleLanguage = LanguageParser.ParseLanguages(movie.Title);
if (!movieTitleLanguage.Contains(Language.Unknown))
if (releaseInfo != null && releaseInfo.Languages.Any())
{
var normalizedEpisodeTitle = Parser.Parser.NormalizeEpisodeTitle(movie.Title);
var movieTitleIndex = normalizedReleaseTokens.IndexOf(normalizedEpisodeTitle, StringComparison.CurrentCultureIgnoreCase);
_logger.Debug("Languages provided by indexer, using release values: {0}", string.Join(", ", releaseInfo.Languages));
if (movieTitleIndex >= 0)
{
releaseTokens = releaseTokens.Remove(movieTitleIndex, normalizedEpisodeTitle.Length);
languagesToRemove.AddRange(movieTitleLanguage);
}
// Use languages from release (given by indexer or user) if available
languages = releaseInfo.Languages;
}
else
{
var movieTitleLanguage = LanguageParser.ParseLanguages(movie.Title);
// Remove any languages still in the title that would normally be removed
languagesToRemove = languagesToRemove.Except(LanguageParser.ParseLanguages(releaseTokens)).ToList();
if (!movieTitleLanguage.Contains(Language.Unknown))
{
var normalizedEpisodeTitle = Parser.Parser.NormalizeEpisodeTitle(movie.Title);
var movieTitleIndex = normalizedReleaseTokens.IndexOf(normalizedEpisodeTitle, StringComparison.CurrentCultureIgnoreCase);
// Remove all languages that aren't part of the updated releaseTokens
languages = languages.Except(languagesToRemove).ToList();
if (movieTitleIndex >= 0)
{
releaseTokens = releaseTokens.Remove(movieTitleIndex, normalizedEpisodeTitle.Length);
languagesToRemove.AddRange(movieTitleLanguage);
}
}
// Remove any languages still in the title that would normally be removed
languagesToRemove = languagesToRemove.Except(LanguageParser.ParseLanguages(releaseTokens)).ToList();
// Remove all languages that aren't part of the updated releaseTokens
languages = languages.Except(languagesToRemove).ToList();
}
// Use movie language as fallback if we couldn't parse a language
if (languages.Count == 0 || (languages.Count == 1 && languages.First() == Language.Unknown))

12
src/NzbDrone.Core/Indexers/IndexerBase.cs
View File

@ -1,11 +1,14 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.IndexerSearch.Definitions;
using NzbDrone.Core.Languages;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.ThingiProvider;
@ -15,6 +18,8 @@ namespace NzbDrone.Core.Indexers
public abstract class IndexerBase<TSettings> : IIndexer
where TSettings : IIndexerSettings, new()
{
private static readonly Regex MultiRegex = new (@"\b(?<multi>multi)\b", RegexOptions.Compiled | RegexOptions.IgnoreCase);
protected readonly IIndexerStatusService _indexerStatusService;
protected readonly IConfigService _configService;
protected readonly IParsingService _parsingService;
@ -73,9 +78,16 @@ namespace NzbDrone.Core.Indexers
protected virtual IList<ReleaseInfo> CleanupReleases(IEnumerable<ReleaseInfo> releases)
{
var result = releases.DistinctBy(v => v.Guid).ToList();
var settings = Definition.Settings as IIndexerSettings;
result.ForEach(c =>
{
// Use multi languages from setting if ReleaseInfo languages is empty
if (c.Languages.Empty() && MultiRegex.IsMatch(c.Title) && settings.MultiLanguages.Any())
{
c.Languages = settings.MultiLanguages.Select(i => (Language)i).ToList();
}
c.IndexerId = Definition.Id;
c.Indexer = Definition.Name;
c.DownloadProtocol = Protocol;

43
src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs
View File

@ -5,6 +5,8 @@ using System.Linq;
using System.Xml.Linq;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Indexers.Exceptions;
using NzbDrone.Core.Languages;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers.Newznab
@ -97,6 +99,30 @@ namespace NzbDrone.Core.Indexers.Newznab
return ParseUrl(item.TryGetValue("comments"));
}
protected override List<Language> GetLanguages(XElement item)
{
var languges = TryGetMultipleNewznabAttributes(item, "language");
var results = new List<Language>();
// Try to find <language> elements for some indexers that suck at following the rules.
if (languges.Count == 0)
{
languges = item.Elements("language").Select(e => e.Value).ToList();
}
foreach (var language in languges)
{
var mappedLanguage = IsoLanguages.FindByName(language)?.Language ?? null;
if (mappedLanguage != null)
{
results.Add(mappedLanguage);
}
}
return results;
}
protected override long GetSize(XElement item)
{
long size;
@ -188,5 +214,22 @@ namespace NzbDrone.Core.Indexers.Newznab
return defaultValue;
}
protected List<string> TryGetMultipleNewznabAttributes(XElement item, string key)
{
var attrElements = item.Elements(ns + "attr").Where(e => e.Attribute("name").Value.Equals(key, StringComparison.OrdinalIgnoreCase));
var results = new List<string>();
foreach (var element in attrElements)
{
var attrValue = element.Attribute("value");
if (attrValue != null)
{
results.Add(attrValue.Value);
}
}
return results;
}
}
}

9
src/NzbDrone.Core/Indexers/RssParser.cs
View File

@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
@ -12,6 +12,7 @@ using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Common.Instrumentation;
using NzbDrone.Core.Indexers.Exceptions;
using NzbDrone.Core.Languages;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers
@ -160,6 +161,7 @@ namespace NzbDrone.Core.Indexers
releaseInfo.DownloadUrl = GetDownloadUrl(item);
releaseInfo.InfoUrl = GetInfoUrl(item);
releaseInfo.CommentUrl = GetCommentUrl(item);
releaseInfo.Languages = GetLanguages(item);
try
{
@ -226,6 +228,11 @@ namespace NzbDrone.Core.Indexers
return ParseUrl((string)item.Element("comments"));
}
protected virtual List<Language> GetLanguages(XElement item)
{
return new List<Language>();
}
protected virtual long GetSize(XElement item)
{
if (UseEnclosureLength)

43
src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs
View File

@ -4,6 +4,8 @@ using System.Linq;
using System.Xml.Linq;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Indexers.Exceptions;
using NzbDrone.Core.Languages;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers.Torznab
@ -93,6 +95,30 @@ namespace NzbDrone.Core.Indexers.Torznab
return ParseUrl(item.TryGetValue("comments"));
}
protected override List<Language> GetLanguages(XElement item)
{
var languges = TryGetMultipleTorznabAttributes(item, "language");
var results = new List<Language>();
// Try to find <language> elements for some indexers that suck at following the rules.
if (languges.Count == 0)
{
languges = item.Elements("language").Select(e => e.Value).ToList();
}
foreach (var language in languges)
{
var mappedLanguage = IsoLanguages.FindByName(language)?.Language ?? null;
if (mappedLanguage != null)
{
results.Add(mappedLanguage);
}
}
return results;
}
protected override long GetSize(XElement item)
{
long size;
@ -224,5 +250,22 @@ namespace NzbDrone.Core.Indexers.Torznab
return defaultValue;
}
protected List<string> TryGetMultipleTorznabAttributes(XElement item, string key)
{
var attrElements = item.Elements(ns + "attr").Where(e => e.Attribute("name").Value.Equals(key, StringComparison.OrdinalIgnoreCase));
var results = new List<string>();
foreach (var element in attrElements)
{
var attrValue = element.Attribute("value");
if (attrValue != null)
{
results.Add(attrValue.Value);
}
}
return results;
}
}
}

8
src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs
View File

@ -1,11 +1,18 @@
using System;
using System.Collections.Generic;
using System.Text;
using NzbDrone.Core.Indexers;
using NzbDrone.Core.Languages;
namespace NzbDrone.Core.Parser.Model
{
public class ReleaseInfo
{
public ReleaseInfo()
{
Languages = new List<Language>();
}
public string Guid { get; set; }
public string Title { get; set; }
public long Size { get; set; }
@ -27,6 +34,7 @@ namespace NzbDrone.Core.Parser.Model
public string Resolution { get; set; }
public IndexerFlags IndexerFlags { get; set; }
public List<Language> Languages { get; set; }
public int Age
{