From d0c93759c624e800636e5107bb1d0b4a54658795 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Fri, 13 Jan 2017 15:49:57 -0500 Subject: [PATCH 1/2] Fix Movie Title Slugs #233 --- .../Migration/117_fix_movie_slugs.cs | 72 +++++++++++++++++++ .../SkyHook/Resource/TMDBResources.cs | 1 + .../MetadataSource/SkyHook/SkyHookProxy.cs | 72 +++++++++++++------ src/NzbDrone.Core/NzbDrone.Core.csproj | 1 + 4 files changed, 126 insertions(+), 20 deletions(-) create mode 100644 src/NzbDrone.Core/Datastore/Migration/117_fix_movie_slugs.cs diff --git a/src/NzbDrone.Core/Datastore/Migration/117_fix_movie_slugs.cs b/src/NzbDrone.Core/Datastore/Migration/117_fix_movie_slugs.cs new file mode 100644 index 000000000..1f3591b23 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/117_fix_movie_slugs.cs @@ -0,0 +1,72 @@ +using System.Data; +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; +using System.Text; +using System.Text.RegularExpressions; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(117)] + public class fix_movie_slugs : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Execute.WithConnection(SetTitleSlug); + } + + private void SetTitleSlug(IDbConnection conn, IDbTransaction tran) + { + using (IDbCommand getSeriesCmd = conn.CreateCommand()) + { + getSeriesCmd.Transaction = tran; + getSeriesCmd.CommandText = @"SELECT Id, Title, Year FROM Movies"; + using (IDataReader seriesReader = getSeriesCmd.ExecuteReader()) + { + while (seriesReader.Read()) + { + var id = seriesReader.GetInt32(0); + var title = seriesReader.GetString(1); + var year = seriesReader.GetString(2); + + var titleSlug = ToUrlSlug(title + "-" + year); + + using (IDbCommand updateCmd = conn.CreateCommand()) + { + updateCmd.Transaction = tran; + updateCmd.CommandText = "UPDATE Movies SET TitleSlug = ? WHERE Id = ?"; + updateCmd.AddParameter(titleSlug); + updateCmd.AddParameter(id); + + updateCmd.ExecuteNonQuery(); + } + } + } + } + } + + public static string ToUrlSlug(string value) + { + + //First to lower case + value = value.ToLowerInvariant(); + + //Remove all accents + var bytes = Encoding.GetEncoding("Cyrillic").GetBytes(value); + value = Encoding.ASCII.GetString(bytes); + + //Replace spaces + value = Regex.Replace(value, @"\s", "-", RegexOptions.Compiled); + + //Remove invalid chars + value = Regex.Replace(value, @"[^a-z0-9\s-_]", "", RegexOptions.Compiled); + + //Trim dashes from end + value = value.Trim('-', '_'); + + //Replace double occurences of - or _ + value = Regex.Replace(value, @"([-_]){2,}", "$1", RegexOptions.Compiled); + + return value; + } + } +} diff --git a/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/TMDBResources.cs b/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/TMDBResources.cs index c0987e04f..399aec27d 100644 --- a/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/TMDBResources.cs +++ b/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/TMDBResources.cs @@ -34,6 +34,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook.Resource public int vote_count { get; set; } public bool video { get; set; } public float vote_average { get; set; } + public string Slug { get; set; } } diff --git a/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs b/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs index 318dccbd4..f0a114d20 100644 --- a/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs +++ b/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs @@ -13,6 +13,7 @@ using NzbDrone.Core.MetadataSource; using NzbDrone.Core.Tv; using Newtonsoft.Json; using System.Text.RegularExpressions; +using System.Text; namespace NzbDrone.Core.MetadataSource.SkyHook { @@ -29,7 +30,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook public SkyHookProxy(IHttpClient httpClient, ISonarrCloudRequestBuilder requestBuilder, ITmdbConfigService configService, IMovieService movieService, Logger logger) { _httpClient = httpClient; - _requestBuilder = requestBuilder.SkyHookTvdb; + _requestBuilder = requestBuilder.SkyHookTvdb; _movieBuilder = requestBuilder.TMDB; _configService = configService; _movieService = movieService; @@ -88,7 +89,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook movie.TmdbId = TmdbId; movie.ImdbId = resource.imdb_id; movie.Title = resource.title; - movie.TitleSlug = movie.Title.ToLower().Replace(" ", "-"); + movie.TitleSlug = ToUrlSlug(movie.Title); movie.CleanTitle = Parser.Parser.CleanSeriesTitle(movie.Title); movie.SortTitle = Parser.Parser.NormalizeTitle(movie.Title); movie.Overview = resource.overview; @@ -104,20 +105,21 @@ namespace NzbDrone.Core.MetadataSource.SkyHook { _logger.Debug("Movie with this title slug already exists. Adding year..."); } + //movie.TitleSlug += "-" + movie.Year.ToString(); movie.TitleSlug += "-" + movie.Year.ToString(); movie.Images.Add(_configService.GetCoverForURL(resource.poster_path, MediaCoverTypes.Poster));//TODO: Update to load image specs from tmdb page! movie.Images.Add(_configService.GetCoverForURL(resource.backdrop_path, MediaCoverTypes.Banner)); movie.Runtime = resource.runtime; - foreach(Title title in resource.alternative_titles.titles) + foreach (Title title in resource.alternative_titles.titles) { movie.AlternativeTitles.Add(title.title); } - foreach(ReleaseDates releaseDates in resource.release_dates.results) + foreach (ReleaseDates releaseDates in resource.release_dates.results) { - foreach(ReleaseDate releaseDate in releaseDates.release_dates) + foreach (ReleaseDate releaseDate in releaseDates.release_dates) { if (releaseDate.type == 5 || releaseDate.type == 4) { @@ -140,7 +142,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook movie.Ratings.Votes = resource.vote_count; movie.Ratings.Value = (decimal)resource.vote_average; - foreach(Genre genre in resource.genres) + foreach (Genre genre in resource.genres) { movie.Genres.Add(genre.name); } @@ -176,10 +178,11 @@ namespace NzbDrone.Core.MetadataSource.SkyHook private string StripTrailingTheFromTitle(string title) { - if(title.EndsWith(",the")) + if (title.EndsWith(",the")) { title = title.Substring(0, title.Length - 4); - } else if(title.EndsWith(", the")) + } + else if (title.EndsWith(", the")) { title = title.Substring(0, title.Length - 5); } @@ -202,7 +205,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook { yearTerm = parserResult.Year.ToString(); } - + if (parserResult.ImdbId.IsNotNullOrWhiteSpace()) { return new List { GetMovieInfo(parserResult.ImdbId) }; @@ -298,14 +301,14 @@ namespace NzbDrone.Core.MetadataSource.SkyHook } } - + var httpRequest = _requestBuilder.Create() .SetSegment("route", "search") .AddQueryParam("term", title.ToLower().Trim()) .Build(); - + var httpResponse = _httpClient.Get>(httpRequest); @@ -330,8 +333,8 @@ namespace NzbDrone.Core.MetadataSource.SkyHook { imdbMovie.SortTitle = Parser.Parser.NormalizeTitle(result.title); imdbMovie.Title = result.title; - string titleSlug = result.title; - imdbMovie.TitleSlug = titleSlug.ToLower().Replace(" ", "-"); + string titleSlug = ToUrlSlug(result.title); + // imdbMovie.TitleSlug = titleSlug.ToLower().Replace(" ", "-"); if (result.release_date.IsNotNullOrWhiteSpace()) { @@ -340,12 +343,16 @@ namespace NzbDrone.Core.MetadataSource.SkyHook - var slugResult = _movieService.FindByTitleSlug(imdbMovie.TitleSlug); - if (slugResult != null) - { - _logger.Debug("Movie with this title slug already exists. Adding year..."); - } - imdbMovie.TitleSlug += "-" + imdbMovie.Year.ToString(); + //var slugResult = _movieService.FindByTitleSlug(titleSlug); + //if (slugResult != null) + //{ + // _logger.Debug("Movie with this title slug already exists. Adding year..."); + //} + //imdbMovie.TitleSlug += "-" + imdbMovie.Year.ToString(); + + titleSlug += "-" + imdbMovie.Year.ToString(); ; + + imdbMovie.TitleSlug = titleSlug; imdbMovie.Images = new List(); imdbMovie.Overview = result.overview; @@ -419,7 +426,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook { series.Certification = show.ContentRating.ToUpper(); } - + series.Actors = show.Actors.Select(MapActors).ToList(); series.Seasons = show.Seasons.Select(MapSeason).ToList(); series.Images = show.Images.Select(MapImage).ToList(); @@ -525,5 +532,30 @@ namespace NzbDrone.Core.MetadataSource.SkyHook return MediaCoverTypes.Unknown; } } + + public static string ToUrlSlug(string value) + { + + //First to lower case + value = value.ToLowerInvariant(); + + //Remove all accents + var bytes = Encoding.GetEncoding("Cyrillic").GetBytes(value); + value = Encoding.ASCII.GetString(bytes); + + //Replace spaces + value = Regex.Replace(value, @"\s", "-", RegexOptions.Compiled); + + //Remove invalid chars + value = Regex.Replace(value, @"[^a-z0-9\s-_]", "", RegexOptions.Compiled); + + //Trim dashes from end + value = value.Trim('-', '_'); + + //Replace double occurences of - or _ + value = Regex.Replace(value, @"([-_]){2,}", "$1", RegexOptions.Compiled); + + return value; + } } } diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index a0221be03..c8f72f327 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -294,6 +294,7 @@ + From e9b6c250f76e8794fa2a2a3a40f520fb79e49347 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Fri, 13 Jan 2017 17:56:34 -0500 Subject: [PATCH 2/2] Revert TMDBResources --- .../MetadataSource/SkyHook/Resource/TMDBResources.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/TMDBResources.cs b/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/TMDBResources.cs index 399aec27d..c0987e04f 100644 --- a/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/TMDBResources.cs +++ b/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/TMDBResources.cs @@ -34,7 +34,6 @@ namespace NzbDrone.Core.MetadataSource.SkyHook.Resource public int vote_count { get; set; } public bool video { get; set; } public float vote_average { get; set; } - public string Slug { get; set; } }