New: Use Artist/Album aliases for indexer search and parsing

This commit is contained in:
ta264 2019-07-30 21:47:48 +01:00
parent 4538a5669b
commit 3830b349d4
14 changed files with 149 additions and 22 deletions

View File

@ -0,0 +1,14 @@
using FluentMigrator;
using NzbDrone.Core.Datastore.Migration.Framework;
namespace NzbDrone.Core.Datastore.Migration
{
[Migration(35)]
public class release_group_alias : NzbDroneMigrationBase
{
protected override void MainDbUpgrade()
{
Alter.Table("Albums").AddColumn("Aliases").AsString().WithDefaultValue("[]");
}
}
}

View File

@ -30,16 +30,16 @@ namespace NzbDrone.Core.DecisionEngine.Specifications.Search
{
return Decision.Accept();
}
if (Parser.Parser.CleanArtistName(singleAlbumSpec.AlbumTitle) != Parser.Parser.CleanArtistName(remoteAlbum.ParsedAlbumInfo.AlbumTitle))
if (!remoteAlbum.Albums.Any(x => x.Title == singleAlbumSpec.AlbumTitle))
{
_logger.Debug("Album does not match searched album title, skipping.");
return Decision.Reject("Wrong album");
}
if (!remoteAlbum.ParsedAlbumInfo.AlbumTitle.Any())
if (remoteAlbum.Albums.Count > 1)
{
_logger.Debug("Full discography result during single album search, skipping.");
_logger.Debug("Discography result during single album search, skipping.");
return Decision.Reject("Full artist pack");
}

View File

@ -1,3 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using NzbDrone.Common.Extensions;
namespace NzbDrone.Core.IndexerSearch.Definitions
@ -5,10 +8,21 @@ namespace NzbDrone.Core.IndexerSearch.Definitions
public class AlbumSearchCriteria : SearchCriteriaBase
{
public string AlbumTitle { get; set; }
public List<string> AlbumAliases { get; set; }
public int AlbumYear { get; set; }
public string Disambiguation { get; set; }
public string AlbumQuery => GetQueryTitle($"{AlbumTitle}{(Disambiguation.IsNullOrWhiteSpace() ? string.Empty : $"+{Disambiguation}")}");
public string AlbumQuery => GetQueryTitle(AddDisambiguation(AlbumTitle));
public List<string> AlbumQueries => OrderQueries(AlbumTitle, AlbumAliases)
.Select(x => GetQueryTitle(AddDisambiguation(x)))
.Distinct()
.ToList();
private string AddDisambiguation(string term)
{
return Disambiguation.IsNullOrWhiteSpace() ? term : $"{term}+{Disambiguation}";
}
public override string ToString()
{

View File

@ -12,6 +12,7 @@ namespace NzbDrone.Core.IndexerSearch.Definitions
private static readonly Regex SpecialCharacter = new Regex(@"[`'.]", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex NonWord = new Regex(@"[\W]", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex BeginningThe = new Regex(@"^the\s", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex IsAllWord = new Regex(@"^[\sA-Za-z0-9_`'.&:-]*$", RegexOptions.IgnoreCase | RegexOptions.Compiled);
public virtual bool MonitoredEpisodesOnly { get; set; }
public virtual bool UserInvokedSearch { get; set; }
@ -22,6 +23,39 @@ namespace NzbDrone.Core.IndexerSearch.Definitions
public List<Track> Tracks { get; set; }
public string ArtistQuery => GetQueryTitle(Artist.Name);
public List<string> ArtistQueries => OrderQueries(Artist.Metadata.Value.Name, Artist.Metadata.Value.Aliases);
protected List<string> OrderQueries(string title, List<string> aliases)
{
var result = new List<string>();
// find the primary search term. This will be title if there are no special characters in the title,
// otherwise the first alias with no special characters
if (IsAllWord.IsMatch(title))
{
result.Add(title);
}
else
{
result.Add(aliases.FirstOrDefault(x => IsAllWord.IsMatch(x)) ?? title);
result.Add(title);
}
// insert remaining aliases
result.AddRange(aliases.Except(result));
return result;
}
protected List<List<string>> GetQueryTiers(List<string> titles)
{
var result = new List<List<string>>();
var queries = titles.Select(GetQueryTitle).Distinct();
result.Add(queries.Take(1).ToList());
result.Add(queries.Skip(1).ToList());
return result;
}
public static string GetQueryTitle(string title)
{

View File

@ -73,6 +73,7 @@ namespace NzbDrone.Core.IndexerSearch
var searchSpec = Get<AlbumSearchCriteria>(artist, new List<Album> { album }, userInvokedSearch, interactiveSearch);
searchSpec.AlbumTitle = album.Title;
searchSpec.AlbumAliases = album.Aliases;
if (album.ReleaseDate.HasValue)
{
searchSpec.AlbumYear = album.ReleaseDate.Value.Year;

View File

@ -1,3 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using NzbDrone.Common.Extensions;
@ -69,42 +70,83 @@ namespace NzbDrone.Core.Indexers.Newznab
if (SupportsAudioSearch)
{
AddAudioPageableRequests(pageableRequests, searchCriteria,
NewsnabifyTitle($"&artist={searchCriteria.ArtistQuery}&album={searchCriteria.AlbumQuery}"));
AddAlbumRequests(pageableRequests, searchCriteria, "&artist={0}&album={1}", AddAudioPageableRequests);
}
if (SupportsSearch)
{
pageableRequests.AddTier();
pageableRequests.Add(GetPagedRequests(MaxPages, Settings.Categories, "search",
NewsnabifyTitle($"&q={searchCriteria.ArtistQuery}+{searchCriteria.AlbumQuery}")));
AddAlbumRequests(pageableRequests, searchCriteria, "&q={0}+{1}", AddSearchPageableRequests);
}
return pageableRequests;
}
private void AddAlbumRequests(IndexerPageableRequestChain pageableRequests, AlbumSearchCriteria searchCriteria, string paramFormat, Action<IndexerPageableRequestChain, SearchCriteriaBase, string> AddRequests)
{
var albumQuery = searchCriteria.AlbumQueries[0];
var artistQuery = searchCriteria.ArtistQueries[0];
// search using standard name
pageableRequests.AddTier();
AddRequests(pageableRequests, searchCriteria, NewsnabifyTitle(string.Format(paramFormat, artistQuery, albumQuery)));
// using artist alias
pageableRequests.AddTier();
foreach (var artistAlt in searchCriteria.ArtistQueries.Skip(1))
{
AddRequests(pageableRequests, searchCriteria, NewsnabifyTitle(string.Format(paramFormat, artistAlt, albumQuery)));
}
// using album alias
pageableRequests.AddTier();
foreach (var albumAlt in searchCriteria.AlbumQueries.Skip(1))
{
AddRequests(pageableRequests, searchCriteria, NewsnabifyTitle(string.Format(paramFormat, artistQuery, albumAlt)));
}
// using aliases for both
foreach (var artistAlt in searchCriteria.ArtistQueries.Skip(1))
{
foreach (var albumAlt in searchCriteria.AlbumQueries.Skip(1))
{
AddRequests(pageableRequests, searchCriteria, NewsnabifyTitle(string.Format(paramFormat, artistAlt, albumAlt)));
}
}
}
public virtual IndexerPageableRequestChain GetSearchRequests(ArtistSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();
if (SupportsAudioSearch)
{
AddAudioPageableRequests(pageableRequests, searchCriteria,
NewsnabifyTitle($"&artist={searchCriteria.ArtistQuery}"));
AddArtistRequests(pageableRequests, searchCriteria, "&artist={0}", AddAudioPageableRequests);
}
if (SupportsSearch)
{
pageableRequests.AddTier();
pageableRequests.Add(GetPagedRequests(MaxPages, Settings.Categories, "search",
NewsnabifyTitle($"&q={searchCriteria.ArtistQuery}")));
AddArtistRequests(pageableRequests, searchCriteria, "&q={0}", AddSearchPageableRequests);
}
return pageableRequests;
}
private void AddArtistRequests(IndexerPageableRequestChain pageableRequests, SearchCriteriaBase searchCriteria, string paramFormat, Action<IndexerPageableRequestChain, SearchCriteriaBase, string> AddRequests)
{
var artistQuery = searchCriteria.ArtistQueries[0];
// search using standard name
pageableRequests.AddTier();
AddRequests(pageableRequests, searchCriteria, NewsnabifyTitle(string.Format(paramFormat, artistQuery)));
// using artist alias
pageableRequests.AddTier();
foreach (var artistAlt in searchCriteria.ArtistQueries.Skip(1))
{
AddRequests(pageableRequests, searchCriteria, NewsnabifyTitle(string.Format(paramFormat, artistAlt)));
}
}
private void AddAudioPageableRequests(IndexerPageableRequestChain chain, SearchCriteriaBase searchCriteria, string parameters)
{
chain.AddTier();
@ -112,6 +154,11 @@ namespace NzbDrone.Core.Indexers.Newznab
chain.Add(GetPagedRequests(MaxPages, Settings.Categories, "music", $"&q={parameters}"));
}
private void AddSearchPageableRequests(IndexerPageableRequestChain chain, SearchCriteriaBase searchCriteria, string parameters)
{
chain.Add(GetPagedRequests(MaxPages, Settings.Categories, "search", parameters));
}
private IEnumerable<IndexerRequest> GetPagedRequests(int maxPages, IEnumerable<int> categories, string searchType, string parameters)
{
if (categories.Empty())

View File

@ -5,6 +5,12 @@ namespace NzbDrone.Core.MetadataSource.SkyHook.Resource
{
public class AlbumResource
{
public AlbumResource()
{
Aliases = new List<string>();
}
public List<string> Aliases { get; set; }
public string ArtistId { get; set; }
public List<ArtistResource> Artists { get; set; }
public string Disambiguation { get; set; }

View File

@ -3,7 +3,6 @@ using System.Collections.Generic;
using System.Linq;
using System.Net;
using NLog;
using NzbDrone.Common.Cloud;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Exceptions;
@ -300,6 +299,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
album.ForeignAlbumId = resource.Id;
album.OldForeignAlbumIds = resource.OldIds;
album.Title = resource.Title;
album.Aliases = resource.Aliases;
album.Overview = resource.Overview;
album.Disambiguation = resource.Disambiguation;
album.ReleaseDate = resource.ReleaseDate;

View File

@ -12,6 +12,7 @@ namespace NzbDrone.Core.Music
{
public Album()
{
Aliases = new List<string>();
Genres = new List<string>();
Images = new List<MediaCover.MediaCover>();
Links = new List<Links>();
@ -28,6 +29,7 @@ namespace NzbDrone.Core.Music
public string ForeignAlbumId { get; set; }
public List<string> OldForeignAlbumIds { get; set; }
public string Title { get; set; }
public List<string> Aliases { get; set; }
public string Overview { get; set; }
public string Disambiguation { get; set; }
public DateTime? ReleaseDate { get; set; }
@ -80,6 +82,7 @@ namespace NzbDrone.Core.Music
ForeignAlbumId == other.ForeignAlbumId &&
(OldForeignAlbumIds?.SequenceEqual(other.OldForeignAlbumIds) ?? true) &&
Title == other.Title &&
(Aliases?.SequenceEqual(other.Aliases) ?? true) &&
Overview == other.Overview &&
Disambiguation == other.Disambiguation &&
ReleaseDate == other.ReleaseDate &&
@ -123,6 +126,7 @@ namespace NzbDrone.Core.Music
hash = hash * 23 + ForeignAlbumId.GetHashCode();
hash = hash * 23 + OldForeignAlbumIds?.GetHashCode() ?? 0;
hash = hash * 23 + Title?.GetHashCode() ?? 0;
hash = hash * 23 + Aliases?.GetHashCode() ?? 0;
hash = hash * 23 + Overview?.GetHashCode() ?? 0;
hash = hash * 23 + Disambiguation?.GetHashCode() ?? 0;
hash = hash * 23 + ReleaseDate?.GetHashCode() ?? 0;

View File

@ -87,6 +87,7 @@ namespace NzbDrone.Core.Music
var scoringFunctions = new List<Tuple<Func<Album, string, double>, string>> {
tc((a, t) => a.CleanTitle.FuzzyMatch(t), cleanTitle),
tc((a, t) => a.Title.FuzzyMatch(t), title),
tc((a, t) => a.Aliases.Any() ? a.Aliases.Select(x => x.CleanArtistName().FuzzyMatch(t)).Max() : 0, cleanTitle),
tc((a, t) => a.CleanTitle.FuzzyMatch(t), title.RemoveBracketsAndContents().CleanArtistName()),
tc((a, t) => a.CleanTitle.FuzzyMatch(t), title.RemoveAfterDash().CleanArtistName()),
tc((a, t) => a.CleanTitle.FuzzyMatch(t), title.RemoveBracketsAndContents().RemoveAfterDash().CleanArtistName()),

View File

@ -104,7 +104,8 @@ namespace NzbDrone.Core.Music
Func< Func<Artist, string, double>, string, Tuple<Func<Artist, string, double>, string>> tc = Tuple.Create;
var scoringFunctions = new List<Tuple<Func<Artist, string, double>, string>> {
tc((a, t) => a.CleanName.FuzzyMatch(t), cleanTitle),
tc((a, t) => a.Name.FuzzyMatch(t), title),
tc((a, t) => a.Metadata.Value.Name.FuzzyMatch(t), title),
tc((a, t) => a.Metadata.Value.Aliases.Any() ? a.Metadata.Value.Aliases.Select(x => x.CleanArtistName().FuzzyMatch(t)).Max() : 0, cleanTitle)
};
if (title.StartsWith("The ", StringComparison.CurrentCultureIgnoreCase))

View File

@ -163,6 +163,7 @@ namespace NzbDrone.Core.Music
local.LastInfoSync = DateTime.UtcNow;
local.CleanTitle = remote.CleanTitle;
local.Title = remote.Title ?? "Unknown";
local.Aliases = remote.Aliases;
local.Overview = remote.Overview.IsNullOrWhiteSpace() ? local.Overview : remote.Overview;
local.Disambiguation = remote.Disambiguation;
local.AlbumType = remote.AlbumType;

View File

@ -174,6 +174,7 @@
<Compile Include="Datastore\Migration\031_add_artistmetadataid_constraint.cs" />
<Compile Include="Datastore\Migration\032_old_ids_and_artist_alias.cs" />
<Compile Include="Datastore\Migration\033_download_propers_config.cs" />
<Compile Include="Datastore\Migration\035_release_group_alias.cs" />
<Compile Include="Datastore\Migration\Framework\MigrationContext.cs" />
<Compile Include="Datastore\Migration\Framework\MigrationController.cs" />
<Compile Include="Datastore\Migration\Framework\MigrationDbFactory.cs" />
@ -1355,4 +1356,4 @@
<Target Name="AfterBuild">
</Target>
-->
</Project>
</Project>

View File

@ -322,8 +322,11 @@ namespace NzbDrone.Core.Parser
simpleTitle = CleanTorrentSuffixRegex.Replace(simpleTitle, string.Empty);
var escapedArtist = Regex.Escape(artist.Name.RemoveAccent()).Replace(@"\ ", @"[\W_]");
var escapedAlbums = string.Join("|", album.Select(s => Regex.Escape(s.Title.RemoveAccent())).ToList()).Replace(@"\ ", @"[\W_]");
var artistAliases = new [] { artist.Name }.Concat(artist.Metadata.Value.Aliases);
var escapedArtist = string.Join("|", artistAliases.Select(x => Regex.Escape(x.RemoveAccent())).ToList()).Replace(@"\ ", @"[\W_]");
var albumAliases = album.Select(x => x.Title).Concat(album.SelectMany(x => x.Aliases));
var escapedAlbums = string.Join("|", albumAliases.Select(s => Regex.Escape(s.RemoveAccent())).ToList()).Replace(@"\ ", @"[\W_]");
var releaseRegex = new Regex(@"^(\W*|\b)(?<artist>" + escapedArtist + @")(\W*|\b).*(\W*|\b)(?<album>" + escapedAlbums + @")(\W*|\b)", RegexOptions.IgnoreCase);