mirror/Radarr
1
0
Fork 0
mirror of https://github.com/Radarr/Radarr synced 2024-12-27 02:09:59 +00:00
Code Issues Releases Wiki Activity

Fixed: Speed up RSS sync

Browse source
This commit is contained in:
ta264 2020-10-07 21:45:04 +01:00 committed by Qstick
parent 0a8dd85856
commit 2d7942d69c
6 changed files with 124 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/GetMovieFixture.cs
View file

@ -28,7 +28,7 @@ public void should_use_parsed_series_title()
Subject.GetMovie(title);
Mocker.GetMock<IMovieService>()
.Verify(s => s.FindByTitle(Parser.Parser.ParseMovieTitle(title, false).MovieTitle), Times.Once());
.Verify(s => s.FindByTitle(Parser.Parser.ParseMovieTitle(title, false).MovieTitle, It.IsAny<int>(), null, null, null), Times.Once());
}
/*[Test]

2
src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/MapFixture.cs
View file

@ -118,7 +118,7 @@ public void should_lookup_Movie_by_name()
Subject.Map(_parsedMovieInfo, "", null);
Mocker.GetMock<IMovieService>()
.Verify(v => v.FindByTitle(It.IsAny<string>(), It.IsAny<int>()), Times.Once());
.Verify(v => v.FindByTitle(It.IsAny<string>(), It.IsAny<int>(), null, null, null), Times.Once());
}
[Test]

15
src/NzbDrone.Core/Datastore/Migration/185_add_alternative_title_indices.cs Normal file
View file

@ -0,0 +1,15 @@
using FluentMigrator;
using NzbDrone.Core.Datastore.Migration.Framework;
namespace NzbDrone.Core.Datastore.Migration
{
[Migration(185)]
public class add_alternative_title_indices : NzbDroneMigrationBase
{
protected override void MainDbUpgrade()
{
Create.Index().OnTable("AlternativeTitles").OnColumn("CleanTitle");
Create.Index().OnTable("MovieTranslations").OnColumn("CleanTitle");
}
}
}

68
src/NzbDrone.Core/Movies/MovieRepository.cs
View file

@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.Linq;
using Dapper;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Datastore;
using NzbDrone.Core.MediaFiles;
using NzbDrone.Core.Messaging.Events;
@ -109,13 +110,28 @@ public bool MoviePathExists(string path)
public List<Movie> FindByTitles(List<string> titles)
{
var distinct = titles.Distinct().ToList();
var results = new List<Movie>();
results.AddRange(FindByMovieTitles(distinct));
results.AddRange(FindByAltTitles(distinct));
results.AddRange(FindByTransTitles(distinct));
return results.DistinctBy(x => x.Id).ToList();
}
// This is a bit of a hack, but if you try to combine / rationalise these then
// SQLite makes a mess of the query plan and ends up doing a table scan
private List<Movie> FindByMovieTitles(List<string> titles)
{
var movieDictionary = new Dictionary<int, Movie>();
var builder = Builder()
var builder = new SqlBuilder()
.LeftJoin<Movie, AlternativeTitle>((m, t) => m.Id == t.MovieId)
.LeftJoin<Movie, MovieFile>((m, f) => m.Id == f.MovieId)
.LeftJoin<Movie, MovieTranslation>((m, tr) => m.Id == tr.MovieId)
.OrWhere<Movie>(x => distinct.Contains(x.CleanTitle))
.OrWhere<AlternativeTitle>(x => distinct.Contains(x.CleanTitle))
.OrWhere<MovieTranslation>(x => distinct.Contains(x.CleanTitle));
.Join<Movie, Profile>((m, p) => m.ProfileId == p.Id)
.Where<Movie>(x => titles.Contains(x.CleanTitle));
_ = _database.QueryJoined<Movie, Profile, AlternativeTitle, MovieFile, MovieTranslation>(
builder,
@ -124,6 +140,50 @@ public List<Movie> FindByTitles(List<string> titles)
return movieDictionary.Values.ToList();
}
private List<Movie> FindByAltTitles(List<string> titles)
{
var movieDictionary = new Dictionary<int, Movie>();
var builder = new SqlBuilder()
.LeftJoin<AlternativeTitle, Movie>((t, m) => t.MovieId == m.Id)
.LeftJoin<Movie, MovieFile>((m, f) => m.Id == f.MovieId)
.LeftJoin<Movie, MovieTranslation>((m, tr) => m.Id == tr.MovieId)
.Join<Movie, Profile>((m, p) => m.ProfileId == p.Id)
.Where<AlternativeTitle>(x => titles.Contains(x.CleanTitle));
_ = _database.QueryJoined<AlternativeTitle, Profile, Movie, MovieFile, MovieTranslation>(
builder,
(altTitle, profile, movie, file, trans) =>
{
_ = Map(movieDictionary, movie, profile, altTitle, file, trans);
return null;
});
return movieDictionary.Values.ToList();
}
private List<Movie> FindByTransTitles(List<string> titles)
{
var movieDictionary = new Dictionary<int, Movie>();
var builder = new SqlBuilder()
.LeftJoin<MovieTranslation, Movie>((tr, m) => tr.MovieId == m.Id)
.LeftJoin<Movie, AlternativeTitle>((m, t) => m.Id == t.MovieId)
.LeftJoin<Movie, MovieFile>((m, f) => m.Id == f.MovieId)
.Join<Movie, Profile>((m, p) => m.ProfileId == p.Id)
.Where<MovieTranslation>(x => titles.Contains(x.CleanTitle));
_ = _database.QueryJoined<MovieTranslation, Profile, Movie, MovieFile, AlternativeTitle>(
builder,
(trans, profile, movie, file, altTitle) =>
{
_ = Map(movieDictionary, movie, profile, altTitle, file, trans);
return null;
});
return movieDictionary.Values.ToList();
}
public Movie FindByImdbId(string imdbid)
{
var imdbIdWithPrefix = Parser.Parser.NormalizeImdbId(imdbid);

58
src/NzbDrone.Core/Movies/MovieService.cs
View file

@ -27,6 +27,8 @@ public interface IMovieService
List<Movie> FindByTmdbId(List<int> tmdbids);
Movie FindByTitle(string title);
Movie FindByTitle(string title, int year);
Movie FindByTitle(string title, int? year, string arabicTitle, string romanTitle, List<Movie> candidates);
List<Movie> FindByTitleCandidates(string title, out string roman, out string arabic);
Movie FindByTitleSlug(string slug);
Movie FindByPath(string path);
List<string> AllMoviePaths();
@ -103,45 +105,35 @@ public List<Movie> AddMovies(List<Movie> newMovies)
public Movie FindByTitle(string title)
{
return FindByTitle(title.CleanMovieTitle(), null);
var candidates = FindByTitleCandidates(title, out var arabicTitle, out var romanTitle);
return FindByTitle(title, null, arabicTitle, romanTitle, candidates);
}
public Movie FindByTitle(string title, int year)
{
return FindByTitle(title.CleanMovieTitle(), year as int?);
var candidates = FindByTitleCandidates(title, out var arabicTitle, out var romanTitle);
return FindByTitle(title, year, arabicTitle, romanTitle, candidates);
}
private Movie FindByTitle(string cleanTitle, int? year)
public Movie FindByTitle(string cleanTitle, int? year, string arabicTitle, string romanTitle, List<Movie> candidates)
{
cleanTitle = cleanTitle.ToLowerInvariant();
var cleanTitleWithRomanNumbers = cleanTitle;
var cleanTitleWithArabicNumbers = cleanTitle;
foreach (var arabicRomanNumeral in RomanNumeralParser.GetArabicRomanNumeralsMapping())
{
var arabicNumber = arabicRomanNumeral.ArabicNumeralAsString;
var romanNumber = arabicRomanNumeral.RomanNumeral;
cleanTitleWithRomanNumbers = cleanTitleWithRomanNumbers.Replace(arabicNumber, romanNumber);
cleanTitleWithArabicNumbers = cleanTitleWithArabicNumbers.Replace(romanNumber, arabicNumber);
}
var candidates = _movieRepository.FindByTitles(new List<string> { cleanTitle, cleanTitleWithArabicNumbers, cleanTitleWithRomanNumbers });
var result = candidates.Where(x => x.CleanTitle == cleanTitle).FirstWithYear(year);
if (result == null)
{
result =
candidates.Where(movie => movie.CleanTitle == cleanTitleWithArabicNumbers).FirstWithYear(year) ??
candidates.Where(movie => movie.CleanTitle == cleanTitleWithRomanNumbers).FirstWithYear(year);
candidates.Where(movie => movie.CleanTitle == arabicTitle).FirstWithYear(year) ??
candidates.Where(movie => movie.CleanTitle == romanTitle).FirstWithYear(year);
}
if (result == null)
{
result = candidates
.Where(m => m.AlternativeTitles.Any(t => t.CleanTitle == cleanTitle ||
t.CleanTitle == cleanTitleWithArabicNumbers ||
t.CleanTitle == cleanTitleWithRomanNumbers))
t.CleanTitle == arabicTitle ||
t.CleanTitle == romanTitle))
.FirstWithYear(year);
}
@ -149,14 +141,34 @@ private Movie FindByTitle(string cleanTitle, int? year)
{
result = candidates
.Where(m => m.Translations.Any(t => t.CleanTitle == cleanTitle ||
t.CleanTitle == cleanTitleWithArabicNumbers ||
t.CleanTitle == cleanTitleWithRomanNumbers))
t.CleanTitle == arabicTitle ||
t.CleanTitle == romanTitle))
.FirstWithYear(year);
}
return result;
}
public List<Movie> FindByTitleCandidates(string title, out string arabicTitle, out string romanTitle)
{
var cleanTitle = title.CleanMovieTitle().ToLowerInvariant();
romanTitle = cleanTitle;
arabicTitle = cleanTitle;
foreach (var arabicRomanNumeral in RomanNumeralParser.GetArabicRomanNumeralsMapping())
{
var arabicNumber = arabicRomanNumeral.ArabicNumeralAsString;
var romanNumber = arabicRomanNumeral.RomanNumeral;
romanTitle = romanTitle.Replace(arabicNumber, romanNumber);
arabicTitle = arabicTitle.Replace(romanNumber, arabicNumber);
}
romanTitle = romanTitle.ToLowerInvariant();
return _movieRepository.FindByTitles(new List<string> { cleanTitle, arabicTitle, romanTitle });
}
public Movie FindByImdbId(string imdbid)
{
return _movieRepository.FindByImdbId(imdbid);

16
src/NzbDrone.Core/Parser/ParsingService.cs
View file

@ -201,20 +201,20 @@ private bool TryGetMovieByImDbId(ParsedMovieInfo parsedMovieInfo, string imdbId,
private bool TryGetMovieByTitleAndOrYear(ParsedMovieInfo parsedMovieInfo, out MappingResult result)
{
Func<Movie, bool> isNotNull = movie => movie != null;
Movie movieByTitleAndOrYear;
var candidates = _movieService.FindByTitleCandidates(parsedMovieInfo.MovieTitle, out var arabicTitle, out var romanTitle);
Movie movieByTitleAndOrYear;
if (parsedMovieInfo.Year > 1800)
{
movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle, parsedMovieInfo.Year);
if (isNotNull(movieByTitleAndOrYear))
movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle, parsedMovieInfo.Year, arabicTitle, romanTitle, candidates);
if (movieByTitleAndOrYear != null)
{
result = new MappingResult { Movie = movieByTitleAndOrYear };
return true;
}
movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle);
if (isNotNull(movieByTitleAndOrYear))
movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle, null, arabicTitle, romanTitle, candidates);
if (movieByTitleAndOrYear != null)
{
result = new MappingResult { Movie = movieByTitleAndOrYear, MappingResultType = MappingResultType.WrongYear };
return false;
@ -224,8 +224,8 @@ private bool TryGetMovieByTitleAndOrYear(ParsedMovieInfo parsedMovieInfo, out Ma
return false;
}
movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle);
if (isNotNull(movieByTitleAndOrYear))
movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle, null, arabicTitle, romanTitle, candidates);
if (movieByTitleAndOrYear != null)
{
result = new MappingResult { Movie = movieByTitleAndOrYear };
return true;