From 2d7942d69cb286d32cee428ae9af5116e4eceb3e Mon Sep 17 00:00:00 2001 From: ta264 Date: Wed, 7 Oct 2020 21:45:04 +0100 Subject: [PATCH] Fixed: Speed up RSS sync --- .../ParsingServiceTests/GetMovieFixture.cs | 2 +- .../ParsingServiceTests/MapFixture.cs | 2 +- .../185_add_alternative_title_indices.cs | 15 ++++ src/NzbDrone.Core/Movies/MovieRepository.cs | 68 +++++++++++++++++-- src/NzbDrone.Core/Movies/MovieService.cs | 58 +++++++++------- src/NzbDrone.Core/Parser/ParsingService.cs | 16 ++--- 6 files changed, 124 insertions(+), 37 deletions(-) create mode 100644 src/NzbDrone.Core/Datastore/Migration/185_add_alternative_title_indices.cs diff --git a/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/GetMovieFixture.cs b/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/GetMovieFixture.cs index 6ccbd53b5..6abc4bef3 100644 --- a/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/GetMovieFixture.cs +++ b/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/GetMovieFixture.cs @@ -28,7 +28,7 @@ public void should_use_parsed_series_title() Subject.GetMovie(title); Mocker.GetMock() - .Verify(s => s.FindByTitle(Parser.Parser.ParseMovieTitle(title, false).MovieTitle), Times.Once()); + .Verify(s => s.FindByTitle(Parser.Parser.ParseMovieTitle(title, false).MovieTitle, It.IsAny(), null, null, null), Times.Once()); } /*[Test] diff --git a/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/MapFixture.cs b/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/MapFixture.cs index 9756600a8..78115c5d0 100644 --- a/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/MapFixture.cs +++ b/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/MapFixture.cs @@ -118,7 +118,7 @@ public void should_lookup_Movie_by_name() Subject.Map(_parsedMovieInfo, "", null); Mocker.GetMock() - .Verify(v => v.FindByTitle(It.IsAny(), It.IsAny()), Times.Once()); + .Verify(v => v.FindByTitle(It.IsAny(), It.IsAny(), null, null, null), Times.Once()); } [Test] diff --git a/src/NzbDrone.Core/Datastore/Migration/185_add_alternative_title_indices.cs b/src/NzbDrone.Core/Datastore/Migration/185_add_alternative_title_indices.cs new file mode 100644 index 000000000..4a34a5920 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/185_add_alternative_title_indices.cs @@ -0,0 +1,15 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(185)] + public class add_alternative_title_indices : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Create.Index().OnTable("AlternativeTitles").OnColumn("CleanTitle"); + Create.Index().OnTable("MovieTranslations").OnColumn("CleanTitle"); + } + } +} diff --git a/src/NzbDrone.Core/Movies/MovieRepository.cs b/src/NzbDrone.Core/Movies/MovieRepository.cs index c930e68ef..b107ec5ca 100644 --- a/src/NzbDrone.Core/Movies/MovieRepository.cs +++ b/src/NzbDrone.Core/Movies/MovieRepository.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using Dapper; +using NzbDrone.Common.Extensions; using NzbDrone.Core.Datastore; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.Messaging.Events; @@ -109,13 +110,28 @@ public bool MoviePathExists(string path) public List FindByTitles(List titles) { var distinct = titles.Distinct().ToList(); + + var results = new List(); + + results.AddRange(FindByMovieTitles(distinct)); + results.AddRange(FindByAltTitles(distinct)); + results.AddRange(FindByTransTitles(distinct)); + + return results.DistinctBy(x => x.Id).ToList(); + } + + // This is a bit of a hack, but if you try to combine / rationalise these then + // SQLite makes a mess of the query plan and ends up doing a table scan + private List FindByMovieTitles(List titles) + { var movieDictionary = new Dictionary(); - var builder = Builder() + var builder = new SqlBuilder() + .LeftJoin((m, t) => m.Id == t.MovieId) + .LeftJoin((m, f) => m.Id == f.MovieId) .LeftJoin((m, tr) => m.Id == tr.MovieId) - .OrWhere(x => distinct.Contains(x.CleanTitle)) - .OrWhere(x => distinct.Contains(x.CleanTitle)) - .OrWhere(x => distinct.Contains(x.CleanTitle)); + .Join((m, p) => m.ProfileId == p.Id) + .Where(x => titles.Contains(x.CleanTitle)); _ = _database.QueryJoined( builder, @@ -124,6 +140,50 @@ public List FindByTitles(List titles) return movieDictionary.Values.ToList(); } + private List FindByAltTitles(List titles) + { + var movieDictionary = new Dictionary(); + + var builder = new SqlBuilder() + .LeftJoin((t, m) => t.MovieId == m.Id) + .LeftJoin((m, f) => m.Id == f.MovieId) + .LeftJoin((m, tr) => m.Id == tr.MovieId) + .Join((m, p) => m.ProfileId == p.Id) + .Where(x => titles.Contains(x.CleanTitle)); + + _ = _database.QueryJoined( + builder, + (altTitle, profile, movie, file, trans) => + { + _ = Map(movieDictionary, movie, profile, altTitle, file, trans); + return null; + }); + + return movieDictionary.Values.ToList(); + } + + private List FindByTransTitles(List titles) + { + var movieDictionary = new Dictionary(); + + var builder = new SqlBuilder() + .LeftJoin((tr, m) => tr.MovieId == m.Id) + .LeftJoin((m, t) => m.Id == t.MovieId) + .LeftJoin((m, f) => m.Id == f.MovieId) + .Join((m, p) => m.ProfileId == p.Id) + .Where(x => titles.Contains(x.CleanTitle)); + + _ = _database.QueryJoined( + builder, + (trans, profile, movie, file, altTitle) => + { + _ = Map(movieDictionary, movie, profile, altTitle, file, trans); + return null; + }); + + return movieDictionary.Values.ToList(); + } + public Movie FindByImdbId(string imdbid) { var imdbIdWithPrefix = Parser.Parser.NormalizeImdbId(imdbid); diff --git a/src/NzbDrone.Core/Movies/MovieService.cs b/src/NzbDrone.Core/Movies/MovieService.cs index 2ab02821e..3cec78c6b 100644 --- a/src/NzbDrone.Core/Movies/MovieService.cs +++ b/src/NzbDrone.Core/Movies/MovieService.cs @@ -27,6 +27,8 @@ public interface IMovieService List FindByTmdbId(List tmdbids); Movie FindByTitle(string title); Movie FindByTitle(string title, int year); + Movie FindByTitle(string title, int? year, string arabicTitle, string romanTitle, List candidates); + List FindByTitleCandidates(string title, out string roman, out string arabic); Movie FindByTitleSlug(string slug); Movie FindByPath(string path); List AllMoviePaths(); @@ -103,45 +105,35 @@ public List AddMovies(List newMovies) public Movie FindByTitle(string title) { - return FindByTitle(title.CleanMovieTitle(), null); + var candidates = FindByTitleCandidates(title, out var arabicTitle, out var romanTitle); + + return FindByTitle(title, null, arabicTitle, romanTitle, candidates); } public Movie FindByTitle(string title, int year) { - return FindByTitle(title.CleanMovieTitle(), year as int?); + var candidates = FindByTitleCandidates(title, out var arabicTitle, out var romanTitle); + + return FindByTitle(title, year, arabicTitle, romanTitle, candidates); } - private Movie FindByTitle(string cleanTitle, int? year) + public Movie FindByTitle(string cleanTitle, int? year, string arabicTitle, string romanTitle, List candidates) { - cleanTitle = cleanTitle.ToLowerInvariant(); - var cleanTitleWithRomanNumbers = cleanTitle; - var cleanTitleWithArabicNumbers = cleanTitle; - - foreach (var arabicRomanNumeral in RomanNumeralParser.GetArabicRomanNumeralsMapping()) - { - var arabicNumber = arabicRomanNumeral.ArabicNumeralAsString; - var romanNumber = arabicRomanNumeral.RomanNumeral; - cleanTitleWithRomanNumbers = cleanTitleWithRomanNumbers.Replace(arabicNumber, romanNumber); - cleanTitleWithArabicNumbers = cleanTitleWithArabicNumbers.Replace(romanNumber, arabicNumber); - } - - var candidates = _movieRepository.FindByTitles(new List { cleanTitle, cleanTitleWithArabicNumbers, cleanTitleWithRomanNumbers }); - var result = candidates.Where(x => x.CleanTitle == cleanTitle).FirstWithYear(year); if (result == null) { result = - candidates.Where(movie => movie.CleanTitle == cleanTitleWithArabicNumbers).FirstWithYear(year) ?? - candidates.Where(movie => movie.CleanTitle == cleanTitleWithRomanNumbers).FirstWithYear(year); + candidates.Where(movie => movie.CleanTitle == arabicTitle).FirstWithYear(year) ?? + candidates.Where(movie => movie.CleanTitle == romanTitle).FirstWithYear(year); } if (result == null) { result = candidates .Where(m => m.AlternativeTitles.Any(t => t.CleanTitle == cleanTitle || - t.CleanTitle == cleanTitleWithArabicNumbers || - t.CleanTitle == cleanTitleWithRomanNumbers)) + t.CleanTitle == arabicTitle || + t.CleanTitle == romanTitle)) .FirstWithYear(year); } @@ -149,14 +141,34 @@ private Movie FindByTitle(string cleanTitle, int? year) { result = candidates .Where(m => m.Translations.Any(t => t.CleanTitle == cleanTitle || - t.CleanTitle == cleanTitleWithArabicNumbers || - t.CleanTitle == cleanTitleWithRomanNumbers)) + t.CleanTitle == arabicTitle || + t.CleanTitle == romanTitle)) .FirstWithYear(year); } return result; } + public List FindByTitleCandidates(string title, out string arabicTitle, out string romanTitle) + { + var cleanTitle = title.CleanMovieTitle().ToLowerInvariant(); + romanTitle = cleanTitle; + arabicTitle = cleanTitle; + + foreach (var arabicRomanNumeral in RomanNumeralParser.GetArabicRomanNumeralsMapping()) + { + var arabicNumber = arabicRomanNumeral.ArabicNumeralAsString; + var romanNumber = arabicRomanNumeral.RomanNumeral; + + romanTitle = romanTitle.Replace(arabicNumber, romanNumber); + arabicTitle = arabicTitle.Replace(romanNumber, arabicNumber); + } + + romanTitle = romanTitle.ToLowerInvariant(); + + return _movieRepository.FindByTitles(new List { cleanTitle, arabicTitle, romanTitle }); + } + public Movie FindByImdbId(string imdbid) { return _movieRepository.FindByImdbId(imdbid); diff --git a/src/NzbDrone.Core/Parser/ParsingService.cs b/src/NzbDrone.Core/Parser/ParsingService.cs index edc30e29b..8ecd58d53 100644 --- a/src/NzbDrone.Core/Parser/ParsingService.cs +++ b/src/NzbDrone.Core/Parser/ParsingService.cs @@ -201,20 +201,20 @@ private bool TryGetMovieByImDbId(ParsedMovieInfo parsedMovieInfo, string imdbId, private bool TryGetMovieByTitleAndOrYear(ParsedMovieInfo parsedMovieInfo, out MappingResult result) { - Func isNotNull = movie => movie != null; - Movie movieByTitleAndOrYear; + var candidates = _movieService.FindByTitleCandidates(parsedMovieInfo.MovieTitle, out var arabicTitle, out var romanTitle); + Movie movieByTitleAndOrYear; if (parsedMovieInfo.Year > 1800) { - movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle, parsedMovieInfo.Year); - if (isNotNull(movieByTitleAndOrYear)) + movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle, parsedMovieInfo.Year, arabicTitle, romanTitle, candidates); + if (movieByTitleAndOrYear != null) { result = new MappingResult { Movie = movieByTitleAndOrYear }; return true; } - movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle); - if (isNotNull(movieByTitleAndOrYear)) + movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle, null, arabicTitle, romanTitle, candidates); + if (movieByTitleAndOrYear != null) { result = new MappingResult { Movie = movieByTitleAndOrYear, MappingResultType = MappingResultType.WrongYear }; return false; @@ -224,8 +224,8 @@ private bool TryGetMovieByTitleAndOrYear(ParsedMovieInfo parsedMovieInfo, out Ma return false; } - movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle); - if (isNotNull(movieByTitleAndOrYear)) + movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle, null, arabicTitle, romanTitle, candidates); + if (movieByTitleAndOrYear != null) { result = new MappingResult { Movie = movieByTitleAndOrYear }; return true;