From 76db27e8c2482d7de8dd02b3ad9422a5e564c0c9 Mon Sep 17 00:00:00 2001 From: ta264 Date: Tue, 9 Jul 2019 21:13:52 +0100 Subject: [PATCH] New: Add old ids, artist aliases and genres --- .../Identification/GetCandidatesFixture.cs | 4 ++-- .../Migration/032_old_ids_and_artist_alias.cs | 20 +++++++++++++++++++ .../Identification/IdentificationService.cs | 10 ++++++---- .../SkyHook/Resource/AlbumResource.cs | 4 +++- .../SkyHook/Resource/ArtistResource.cs | 2 ++ .../SkyHook/Resource/ReleaseResource.cs | 1 + .../SkyHook/Resource/TrackResource.cs | 4 ++++ .../MetadataSource/SkyHook/SkyHookProxy.cs | 7 +++++++ src/NzbDrone.Core/Music/Album.cs | 3 ++- src/NzbDrone.Core/Music/ArtistMetadata.cs | 10 ++++++++++ .../Music/RefreshAlbumService.cs | 1 + src/NzbDrone.Core/Music/Release.cs | 8 ++++++++ src/NzbDrone.Core/Music/ReleaseRepository.cs | 14 ++++++++++--- src/NzbDrone.Core/Music/ReleaseService.cs | 6 +++--- src/NzbDrone.Core/Music/Track.cs | 10 ++++++++++ src/NzbDrone.Core/NzbDrone.Core.csproj | 1 + 16 files changed, 91 insertions(+), 14 deletions(-) create mode 100644 src/NzbDrone.Core/Datastore/Migration/032_old_ids_and_artist_alias.cs diff --git a/src/NzbDrone.Core.Test/MediaFiles/TrackImport/Identification/GetCandidatesFixture.cs b/src/NzbDrone.Core.Test/MediaFiles/TrackImport/Identification/GetCandidatesFixture.cs index 25f57e787..c73c7ad50 100644 --- a/src/NzbDrone.Core.Test/MediaFiles/TrackImport/Identification/GetCandidatesFixture.cs +++ b/src/NzbDrone.Core.Test/MediaFiles/TrackImport/Identification/GetCandidatesFixture.cs @@ -148,8 +148,8 @@ namespace NzbDrone.Core.Test.MediaFiles.TrackImport.Identification var localAlbumRelease = new LocalAlbumRelease(localTracks); Mocker.GetMock() - .Setup(x => x.GetReleaseByForeignReleaseId("xxx")) - .Returns(release); + .Setup(x => x.GetReleaseByForeignReleaseId("xxx", true)) + .Returns(release); Subject.GetCandidatesFromTags(localAlbumRelease, null, null, null, false).ShouldBeEquivalentTo( new List { new CandidateAlbumRelease(release) } diff --git a/src/NzbDrone.Core/Datastore/Migration/032_old_ids_and_artist_alias.cs b/src/NzbDrone.Core/Datastore/Migration/032_old_ids_and_artist_alias.cs new file mode 100644 index 000000000..f2df39fdc --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/032_old_ids_and_artist_alias.cs @@ -0,0 +1,20 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(32)] + public class old_ids_and_artist_alias : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Alter.Table("ArtistMetadata").AddColumn("Aliases").AsString().WithDefaultValue("[]"); + + Alter.Table("ArtistMetadata").AddColumn("OldForeignArtistIds").AsString().WithDefaultValue("[]"); + Alter.Table("Albums").AddColumn("OldForeignAlbumIds").AsString().WithDefaultValue("[]"); + Alter.Table("AlbumReleases").AddColumn("OldForeignReleaseIds").AsString().WithDefaultValue("[]"); + Alter.Table("Tracks").AddColumn("OldForeignRecordingIds").AsString().WithDefaultValue("[]"); + Alter.Table("Tracks").AddColumn("OldForeignTrackIds").AsString().WithDefaultValue("[]"); + } + } +} diff --git a/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs b/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs index e1225d352..06a183291 100644 --- a/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs +++ b/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs @@ -275,7 +275,8 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Identification if (releaseIds.Count == 1 && releaseIds[0].IsNotNullOrWhiteSpace()) { _logger.Debug("Selecting release from consensus ForeignReleaseId [{0}]", releaseIds[0]); - tagMbidRelease = _releaseService.GetReleaseByForeignReleaseId(releaseIds[0]); + tagMbidRelease = _releaseService.GetReleaseByForeignReleaseId(releaseIds[0], true); + if (tagMbidRelease != null) { tagCandidate = GetCandidatesByRelease(new List { tagMbidRelease }, includeExisting); @@ -555,7 +556,8 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Identification var recordingId = localTrack.FileTrackInfo.RecordingMBId; if (recordingId.IsNotNullOrWhiteSpace()) { - dist.AddBool("recording_id", localTrack.FileTrackInfo.RecordingMBId != mbTrack.ForeignRecordingId); + dist.AddBool("recording_id", localTrack.FileTrackInfo.RecordingMBId != mbTrack.ForeignRecordingId && + !mbTrack.OldForeignRecordingIds.Contains(localTrack.FileTrackInfo.RecordingMBId)); } // for fingerprinted files @@ -657,8 +659,8 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Identification var mbAlbumId = MostCommon(localTracks.Select(x => x.FileTrackInfo.ReleaseMBId)); if (mbAlbumId.IsNotNullOrWhiteSpace()) { - dist.AddEquality("album_id", mbAlbumId, new List { release.ForeignReleaseId }); - _logger.Trace("album_id: {0} vs {1}; {2}", mbAlbumId, release.ForeignReleaseId, dist.NormalizedDistance()); + dist.AddBool("album_id", mbAlbumId != release.ForeignReleaseId && !release.OldForeignReleaseIds.Contains(mbAlbumId)); + _logger.Trace("album_id: {0} vs {1} or {2}; {3}", mbAlbumId, release.ForeignReleaseId, string.Join(", ", release.OldForeignReleaseIds), dist.NormalizedDistance()); } // tracks diff --git a/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/AlbumResource.cs b/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/AlbumResource.cs index 9d7b2031c..16de7ee09 100644 --- a/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/AlbumResource.cs +++ b/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/AlbumResource.cs @@ -9,9 +9,11 @@ namespace NzbDrone.Core.MetadataSource.SkyHook.Resource public List Artists { get; set; } public string Disambiguation { get; set; } public string Overview { get; set; } - public string Id { get; set; } + public string Id { get; set; } + public List OldIds { get; set; } public List Images { get; set; } public List Links { get; set; } + public List Genres { get; set; } public RatingResource Rating { get; set; } public DateTime? ReleaseDate { get; set; } public List Releases { get; set; } diff --git a/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/ArtistResource.cs b/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/ArtistResource.cs index 85fc14ea8..52a0dc852 100644 --- a/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/ArtistResource.cs +++ b/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/ArtistResource.cs @@ -17,9 +17,11 @@ namespace NzbDrone.Core.MetadataSource.SkyHook.Resource public string Type { get; set; } public string Disambiguation { get; set; } public string Id { get; set; } + public List OldIds { get; set; } public List Images { get; set; } public List Links { get; set; } public string ArtistName { get; set; } + public List ArtistAliases { get; set; } public List Albums { get; set; } public string Status { get; set; } public RatingResource Rating { get; set; } diff --git a/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/ReleaseResource.cs b/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/ReleaseResource.cs index d23eff10b..d4f675feb 100644 --- a/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/ReleaseResource.cs +++ b/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/ReleaseResource.cs @@ -9,6 +9,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook.Resource public List Country { get; set; } public DateTime? ReleaseDate { get; set; } public string Id { get; set; } + public List OldIds { get; set; } public List Label { get; set; } public List Media { get; set; } public string Title { get; set; } diff --git a/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/TrackResource.cs b/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/TrackResource.cs index e79c637b9..b9296d4ad 100644 --- a/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/TrackResource.cs +++ b/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/TrackResource.cs @@ -1,3 +1,5 @@ +using System.Collections.Generic; + namespace NzbDrone.Core.MetadataSource.SkyHook.Resource { public class TrackResource @@ -10,7 +12,9 @@ namespace NzbDrone.Core.MetadataSource.SkyHook.Resource public string ArtistId { get; set; } public int DurationMs { get; set; } public string Id { get; set; } + public List OldIds { get; set; } public string RecordingId { get; set; } + public List OldRecordingIds { get; set; } public string TrackName { get; set; } public string TrackNumber { get; set; } public int TrackPosition { get; set; } diff --git a/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs b/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs index a440f8cb4..362c361e7 100644 --- a/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs +++ b/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs @@ -301,6 +301,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook { Album album = new Album(); album.ForeignAlbumId = resource.Id; + album.OldForeignAlbumIds = resource.OldIds; album.Title = resource.Title; album.Overview = resource.Overview; album.Disambiguation = resource.Disambiguation; @@ -315,6 +316,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook album.SecondaryTypes = resource.SecondaryTypes.Select(MapSecondaryTypes).ToList(); album.Ratings = MapRatings(resource.Rating); album.Links = resource.Links?.Select(MapLink).ToList(); + album.Genres = resource.Genres; album.CleanTitle = Parser.Parser.CleanArtistName(album.Title); if (resource.Releases != null) @@ -331,6 +333,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook { AlbumRelease release = new AlbumRelease(); release.ForeignReleaseId = resource.Id; + release.OldForeignReleaseIds = resource.OldIds; release.Title = resource.Title; release.Status = resource.Status; release.Label = resource.Label; @@ -384,7 +387,9 @@ namespace NzbDrone.Core.MetadataSource.SkyHook ArtistMetadata = artistDict[resource.ArtistId], Title = resource.TrackName, ForeignTrackId = resource.Id, + OldForeignTrackIds = resource.OldIds, ForeignRecordingId = resource.RecordingId, + OldForeignRecordingIds = resource.OldRecordingIds, TrackNumber = resource.TrackNumber, AbsoluteTrackNumber = resource.TrackPosition, Duration = resource.DurationMs, @@ -400,7 +405,9 @@ namespace NzbDrone.Core.MetadataSource.SkyHook ArtistMetadata artist = new ArtistMetadata(); artist.Name = resource.ArtistName; + artist.Aliases = resource.ArtistAliases; artist.ForeignArtistId = resource.Id; + artist.OldForeignArtistIds = resource.OldIds; artist.Genres = resource.Genres; artist.Overview = resource.Overview; artist.Disambiguation = resource.Disambiguation; diff --git a/src/NzbDrone.Core/Music/Album.cs b/src/NzbDrone.Core/Music/Album.cs index 9379804f5..2ecc69564 100644 --- a/src/NzbDrone.Core/Music/Album.cs +++ b/src/NzbDrone.Core/Music/Album.cs @@ -1,7 +1,6 @@ using NzbDrone.Common.Extensions; using NzbDrone.Core.Datastore; using System; -using System.Linq; using System.Collections.Generic; using Marr.Data; @@ -16,6 +15,7 @@ namespace NzbDrone.Core.Music Links = new List(); Ratings = new Ratings(); Artist = new Artist(); + OldForeignAlbumIds = new List(); } public const string RELEASE_DATE_FORMAT = "yyyy-MM-dd"; @@ -24,6 +24,7 @@ namespace NzbDrone.Core.Music // These are metadata entries public int ArtistMetadataId { get; set; } public string ForeignAlbumId { get; set; } + public List OldForeignAlbumIds { get; set; } public string Title { get; set; } public string Overview { get; set; } public string Disambiguation { get; set; } diff --git a/src/NzbDrone.Core/Music/ArtistMetadata.cs b/src/NzbDrone.Core/Music/ArtistMetadata.cs index 87946a695..45732c7c4 100644 --- a/src/NzbDrone.Core/Music/ArtistMetadata.cs +++ b/src/NzbDrone.Core/Music/ArtistMetadata.cs @@ -15,10 +15,14 @@ namespace NzbDrone.Core.Music Genres = new List(); Members = new List(); Links = new List(); + OldForeignArtistIds = new List(); + Aliases = new List(); } public string ForeignArtistId { get; set; } + public List OldForeignArtistIds { get; set; } public string Name { get; set; } + public List Aliases { get; set; } public string Overview { get; set; } public string Disambiguation { get; set; } public string Type { get; set; } @@ -37,7 +41,9 @@ namespace NzbDrone.Core.Music public void ApplyChanges(ArtistMetadata otherArtist) { ForeignArtistId = otherArtist.ForeignArtistId; + OldForeignArtistIds = otherArtist.OldForeignArtistIds; Name = otherArtist.Name; + Aliases = otherArtist.Aliases; Overview = otherArtist.Overview.IsNullOrWhiteSpace() ? Overview : otherArtist.Overview; Disambiguation = otherArtist.Disambiguation; Type = otherArtist.Type; @@ -58,7 +64,9 @@ namespace NzbDrone.Core.Music if (Id == other.Id && ForeignArtistId == other.ForeignArtistId && + (OldForeignArtistIds?.SequenceEqual(other.OldForeignArtistIds) ?? true) && Name == other.Name && + (Aliases?.SequenceEqual(other.Aliases) ?? true) && Overview == other.Overview && Disambiguation == other.Disambiguation && Type == other.Type && @@ -100,7 +108,9 @@ namespace NzbDrone.Core.Music int hash = 17; hash = hash * 23 + Id; hash = hash * 23 + ForeignArtistId.GetHashCode(); + hash = hash * 23 + OldForeignArtistIds.GetHashCode(); hash = hash * 23 + Name?.GetHashCode() ?? 0; + hash = hash * 23 + Aliases?.GetHashCode() ?? 0; hash = hash * 23 + Overview?.GetHashCode() ?? 0; hash = hash * 23 + Disambiguation?.GetHashCode() ?? 0; hash = hash * 23 + Type?.GetHashCode() ?? 0; diff --git a/src/NzbDrone.Core/Music/RefreshAlbumService.cs b/src/NzbDrone.Core/Music/RefreshAlbumService.cs index 2ca351cdd..e11cc809a 100644 --- a/src/NzbDrone.Core/Music/RefreshAlbumService.cs +++ b/src/NzbDrone.Core/Music/RefreshAlbumService.cs @@ -143,6 +143,7 @@ namespace NzbDrone.Core.Music forceUpdateFileTags |= album.Title != (albumInfo.Title ?? "Unknown"); updated |= forceUpdateFileTags; + album.OldForeignAlbumIds = albumInfo.OldForeignAlbumIds; album.LastInfoSync = DateTime.UtcNow; album.CleanTitle = albumInfo.CleanTitle; album.Title = albumInfo.Title ?? "Unknown"; diff --git a/src/NzbDrone.Core/Music/Release.cs b/src/NzbDrone.Core/Music/Release.cs index bafd17b91..54ece47f6 100644 --- a/src/NzbDrone.Core/Music/Release.cs +++ b/src/NzbDrone.Core/Music/Release.cs @@ -10,9 +10,15 @@ namespace NzbDrone.Core.Music { public class AlbumRelease : ModelBase, IEquatable { + public AlbumRelease() + { + OldForeignReleaseIds = new List(); + } + // These correspond to columns in the AlbumReleases table public int AlbumId { get; set; } public string ForeignReleaseId { get; set; } + public List OldForeignReleaseIds { get; set; } public string Title { get; set; } public string Status { get; set; } public int Duration { get; set; } @@ -43,6 +49,7 @@ namespace NzbDrone.Core.Music if (Id == other.Id && AlbumId == other.AlbumId && ForeignReleaseId == other.ForeignReleaseId && + (OldForeignReleaseIds?.SequenceEqual(other.OldForeignReleaseIds) ?? true) && Title == other.Title && Status == other.Status && Duration == other.Duration && @@ -86,6 +93,7 @@ namespace NzbDrone.Core.Music hash = hash * 23 + Id; hash = hash * 23 + AlbumId; hash = hash * 23 + ForeignReleaseId.GetHashCode(); + hash = hash * 23 + OldForeignReleaseIds?.GetHashCode() ?? 0; hash = hash * 23 + Title?.GetHashCode() ?? 0; hash = hash * 23 + Status?.GetHashCode() ?? 0; hash = hash * 23 + Duration; diff --git a/src/NzbDrone.Core/Music/ReleaseRepository.cs b/src/NzbDrone.Core/Music/ReleaseRepository.cs index bc418aeee..5688aceaf 100644 --- a/src/NzbDrone.Core/Music/ReleaseRepository.cs +++ b/src/NzbDrone.Core/Music/ReleaseRepository.cs @@ -9,7 +9,7 @@ namespace NzbDrone.Core.Music { public interface IReleaseRepository : IBasicRepository { - AlbumRelease FindByForeignReleaseId(string foreignReleaseId); + AlbumRelease FindByForeignReleaseId(string foreignReleaseId, bool checkRedirect = false); List FindByAlbum(int id); List FindByRecordingId(List recordingIds); List GetReleasesForRefresh(int albumId, IEnumerable foreignReleaseIds); @@ -23,11 +23,19 @@ namespace NzbDrone.Core.Music { } - public AlbumRelease FindByForeignReleaseId(string foreignReleaseId) + public AlbumRelease FindByForeignReleaseId(string foreignReleaseId, bool checkRedirect = false) { - return Query + var release = Query .Where(x => x.ForeignReleaseId == foreignReleaseId) .SingleOrDefault(); + + if (release == null && checkRedirect) + { + release = Query.Where(x => x.OldForeignReleaseIds.Contains(foreignReleaseId)) + .SingleOrDefault(); + } + + return release; } public List GetReleasesForRefresh(int albumId, IEnumerable foreignReleaseIds) diff --git a/src/NzbDrone.Core/Music/ReleaseService.cs b/src/NzbDrone.Core/Music/ReleaseService.cs index ced97d025..a8bb90057 100644 --- a/src/NzbDrone.Core/Music/ReleaseService.cs +++ b/src/NzbDrone.Core/Music/ReleaseService.cs @@ -7,7 +7,7 @@ namespace NzbDrone.Core.Music public interface IReleaseService { AlbumRelease GetRelease(int id); - AlbumRelease GetReleaseByForeignReleaseId(string foreignReleaseId); + AlbumRelease GetReleaseByForeignReleaseId(string foreignReleaseId, bool checkRedirect = false); void InsertMany(List releases); void UpdateMany(List releases); void DeleteMany(List releases); @@ -35,9 +35,9 @@ namespace NzbDrone.Core.Music return _releaseRepository.Get(id); } - public AlbumRelease GetReleaseByForeignReleaseId(string foreignReleaseId) + public AlbumRelease GetReleaseByForeignReleaseId(string foreignReleaseId, bool checkRedirect = false) { - return _releaseRepository.FindByForeignReleaseId(foreignReleaseId); + return _releaseRepository.FindByForeignReleaseId(foreignReleaseId, checkRedirect); } public void InsertMany(List releases) diff --git a/src/NzbDrone.Core/Music/Track.cs b/src/NzbDrone.Core/Music/Track.cs index 373505441..dad9cb7dc 100644 --- a/src/NzbDrone.Core/Music/Track.cs +++ b/src/NzbDrone.Core/Music/Track.cs @@ -4,6 +4,8 @@ using Marr.Data; using NzbDrone.Common.Extensions; using System; using NzbDrone.Common.Serializer; +using System.Collections.Generic; +using System.Linq; namespace NzbDrone.Core.Music { @@ -11,11 +13,15 @@ namespace NzbDrone.Core.Music { public Track() { + OldForeignTrackIds = new List(); + OldForeignRecordingIds = new List(); } // These are model fields public string ForeignTrackId { get; set; } + public List OldForeignTrackIds { get; set; } public string ForeignRecordingId { get; set; } + public List OldForeignRecordingIds { get; set; } public int AlbumReleaseId { get; set; } public int ArtistMetadataId { get; set; } public string TrackNumber { get; set; } @@ -53,7 +59,9 @@ namespace NzbDrone.Core.Music if (Id == other.Id && ForeignTrackId == other.ForeignTrackId && + (OldForeignTrackIds?.SequenceEqual(other.OldForeignTrackIds) ?? true) && ForeignRecordingId == other.ForeignRecordingId && + (OldForeignRecordingIds?.SequenceEqual(other.OldForeignRecordingIds) ?? true) && AlbumReleaseId == other.AlbumReleaseId && ArtistMetadataId == other.ArtistMetadataId && TrackNumber == other.TrackNumber && @@ -96,7 +104,9 @@ namespace NzbDrone.Core.Music int hash = 17; hash = hash * 23 + Id; hash = hash * 23 + ForeignTrackId.GetHashCode(); + hash = hash * 23 + OldForeignTrackIds?.GetHashCode() ?? 0; hash = hash * 23 + ForeignRecordingId.GetHashCode(); + hash = hash * 23 + OldForeignRecordingIds?.GetHashCode() ?? 0; hash = hash * 23 + AlbumReleaseId; hash = hash * 23 + ArtistMetadataId; hash = hash * 23 + TrackNumber?.GetHashCode() ?? 0; diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 4913d1eb5..c6b7edf98 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -172,6 +172,7 @@ +