From c538424229eadaed0ec8ce8eace95bd516676841 Mon Sep 17 00:00:00 2001 From: Qstick Date: Sun, 28 Nov 2021 15:42:44 -0600 Subject: [PATCH] New: Reanalyze media files if file size changes Fixes #6757 Fixes #6765 Fixes #4482 Co-Authored-By: Mark McDowall --- .../DiskScanServiceTests/ScanFixture.cs | 34 ++++++++------- .../MediaFiles/DiskScanService.cs | 43 ++++++++++++++++++- .../MediaFiles/MediaFileService.cs | 12 ++++++ .../MediaInfo/UpdateMediaInfoService.cs | 26 ++++++----- .../MovieImport/ImportDecisionMaker.cs | 6 +++ 5 files changed, 94 insertions(+), 27 deletions(-) diff --git a/src/NzbDrone.Core.Test/MediaFiles/DiskScanServiceTests/ScanFixture.cs b/src/NzbDrone.Core.Test/MediaFiles/DiskScanServiceTests/ScanFixture.cs index 8728ae950..e24cbbe0b 100644 --- a/src/NzbDrone.Core.Test/MediaFiles/DiskScanServiceTests/ScanFixture.cs +++ b/src/NzbDrone.Core.Test/MediaFiles/DiskScanServiceTests/ScanFixture.cs @@ -45,6 +45,10 @@ namespace NzbDrone.Core.Test.MediaFiles.DiskScanServiceTests Mocker.GetMock() .Setup(s => s.GetBestRootFolderPath(It.IsAny())) .Returns(_rootFolder); + + Mocker.GetMock() + .Setup(s => s.GetFilesByMovie(It.IsAny())) + .Returns(new List()); } private void GivenRootFolder(params string[] subfolders) @@ -117,7 +121,7 @@ namespace NzbDrone.Core.Test.MediaFiles.DiskScanServiceTests .Verify(v => v.Clean(It.IsAny(), It.IsAny>()), Times.Never()); Mocker.GetMock() - .Verify(v => v.GetImportDecisions(It.IsAny>(), _movie), Times.Never()); + .Verify(v => v.GetImportDecisions(It.IsAny>(), _movie, false), Times.Never()); } [Test] @@ -152,7 +156,7 @@ namespace NzbDrone.Core.Test.MediaFiles.DiskScanServiceTests Subject.Scan(_movie); Mocker.GetMock() - .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 1), _movie), Times.Once()); + .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 1), _movie, false), Times.Once()); } [Test] @@ -177,7 +181,7 @@ namespace NzbDrone.Core.Test.MediaFiles.DiskScanServiceTests Subject.Scan(_movie); Mocker.GetMock() - .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 1), _movie), Times.Once()); + .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 1), _movie, false), Times.Once()); } [Test] @@ -197,7 +201,7 @@ namespace NzbDrone.Core.Test.MediaFiles.DiskScanServiceTests Subject.Scan(_movie); Mocker.GetMock() - .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 1), _movie), Times.Once()); + .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 1), _movie, false), Times.Once()); } [Test] @@ -229,7 +233,7 @@ namespace NzbDrone.Core.Test.MediaFiles.DiskScanServiceTests .Verify(v => v.Clean(It.IsAny(), It.IsAny>()), Times.Once()); Mocker.GetMock() - .Verify(v => v.GetImportDecisions(It.IsAny>(), _movie), Times.Never()); + .Verify(v => v.GetImportDecisions(It.IsAny>(), _movie, false), Times.Never()); } [Test] @@ -247,7 +251,7 @@ namespace NzbDrone.Core.Test.MediaFiles.DiskScanServiceTests Subject.Scan(_movie); Mocker.GetMock() - .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 1), _movie), Times.Once()); + .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 1), _movie, false), Times.Once()); } [Test] @@ -270,7 +274,7 @@ namespace NzbDrone.Core.Test.MediaFiles.DiskScanServiceTests Subject.Scan(_movie); Mocker.GetMock() - .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 4), _movie), Times.Once()); + .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 4), _movie, false), Times.Once()); } [Test] @@ -289,7 +293,7 @@ namespace NzbDrone.Core.Test.MediaFiles.DiskScanServiceTests Subject.Scan(_movie); Mocker.GetMock() - .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 1), _movie), Times.Once()); + .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 1), _movie, false), Times.Once()); } [Test] @@ -309,7 +313,7 @@ namespace NzbDrone.Core.Test.MediaFiles.DiskScanServiceTests Subject.Scan(_movie); Mocker.GetMock() - .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 1), _movie), Times.Once()); + .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 1), _movie, false), Times.Once()); } [Test] @@ -326,7 +330,7 @@ namespace NzbDrone.Core.Test.MediaFiles.DiskScanServiceTests Subject.Scan(_movie); Mocker.GetMock() - .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 1), _movie), Times.Once()); + .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 1), _movie, false), Times.Once()); } [Test] @@ -343,7 +347,7 @@ namespace NzbDrone.Core.Test.MediaFiles.DiskScanServiceTests Subject.Scan(_movie); Mocker.GetMock() - .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 1), _movie), Times.Once()); + .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 1), _movie, false), Times.Once()); } [Test] @@ -362,7 +366,7 @@ namespace NzbDrone.Core.Test.MediaFiles.DiskScanServiceTests Subject.Scan(_movie); Mocker.GetMock() - .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 2), _movie), Times.Once()); + .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 2), _movie, false), Times.Once()); } [Test] @@ -379,7 +383,7 @@ namespace NzbDrone.Core.Test.MediaFiles.DiskScanServiceTests Subject.Scan(_movie); Mocker.GetMock() - .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 2), _movie), Times.Once()); + .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 2), _movie, false), Times.Once()); } [Test] @@ -397,7 +401,7 @@ namespace NzbDrone.Core.Test.MediaFiles.DiskScanServiceTests Subject.Scan(_movie); Mocker.GetMock() - .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 1), _movie), Times.Once()); + .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 1), _movie, false), Times.Once()); } [Test] @@ -414,7 +418,7 @@ namespace NzbDrone.Core.Test.MediaFiles.DiskScanServiceTests Subject.Scan(_movie); Mocker.GetMock() - .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 1), _movie), Times.Once()); + .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 1), _movie, false), Times.Once()); } } } diff --git a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs index a828a862e..263066d3a 100644 --- a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs +++ b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs @@ -11,6 +11,7 @@ using NzbDrone.Common.Instrumentation.Extensions; using NzbDrone.Core.Configuration; using NzbDrone.Core.MediaFiles.Commands; using NzbDrone.Core.MediaFiles.Events; +using NzbDrone.Core.MediaFiles.MediaInfo; using NzbDrone.Core.MediaFiles.MovieImport; using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.Messaging.Events; @@ -36,8 +37,10 @@ namespace NzbDrone.Core.MediaFiles private readonly IImportApprovedMovie _importApprovedMovies; private readonly IConfigService _configService; private readonly IMovieService _movieService; + private readonly IMediaFileService _mediaFileService; private readonly IMediaFileTableCleanupService _mediaFileTableCleanupService; private readonly IRootFolderService _rootFolderService; + private readonly IUpdateMediaInfo _updateMediaInfoService; private readonly IEventAggregator _eventAggregator; private readonly Logger _logger; @@ -46,8 +49,10 @@ namespace NzbDrone.Core.MediaFiles IImportApprovedMovie importApprovedMovies, IConfigService configService, IMovieService movieService, + IMediaFileService mediaFileService, IMediaFileTableCleanupService mediaFileTableCleanupService, IRootFolderService rootFolderService, + IUpdateMediaInfo updateMediaInfoService, IEventAggregator eventAggregator, Logger logger) { @@ -56,8 +61,10 @@ namespace NzbDrone.Core.MediaFiles _importApprovedMovies = importApprovedMovies; _configService = configService; _movieService = movieService; + _mediaFileService = mediaFileService; _mediaFileTableCleanupService = mediaFileTableCleanupService; _rootFolderService = rootFolderService; + _updateMediaInfoService = updateMediaInfoService; _eventAggregator = eventAggregator; _logger = logger; } @@ -126,12 +133,46 @@ namespace NzbDrone.Core.MediaFiles CleanMediaFiles(movie, mediaFileList); + var movieFiles = _mediaFileService.GetFilesByMovie(movie.Id); + var unmappedFiles = MediaFileService.FilterExistingFiles(mediaFileList, movieFiles, movie); + var decisionsStopwatch = Stopwatch.StartNew(); - var decisions = _importDecisionMaker.GetImportDecisions(mediaFileList, movie); + var decisions = _importDecisionMaker.GetImportDecisions(unmappedFiles, movie, false); decisionsStopwatch.Stop(); _logger.Trace("Import decisions complete for: {0} [{1}]", movie, decisionsStopwatch.Elapsed); _importApprovedMovies.Import(decisions, false); + // Update existing files that have a different file size + var fileInfoStopwatch = Stopwatch.StartNew(); + var filesToUpdate = new List(); + + foreach (var file in movieFiles) + { + var path = Path.Combine(movie.Path, file.RelativePath); + var fileSize = _diskProvider.GetFileSize(path); + + if (file.Size == fileSize) + { + continue; + } + + file.Size = fileSize; + + if (!_updateMediaInfoService.Update(file, movie)) + { + filesToUpdate.Add(file); + } + } + + // Update any files that had a file size change, but didn't get media info updated. + if (filesToUpdate.Any()) + { + _mediaFileService.Update(filesToUpdate); + } + + fileInfoStopwatch.Stop(); + _logger.Trace("Reprocessing existing files complete for: {0} [{1}]", movie, decisionsStopwatch.Elapsed); + RemoveEmptyMovieFolder(movie.Path); CompletedScanning(movie); } diff --git a/src/NzbDrone.Core/MediaFiles/MediaFileService.cs b/src/NzbDrone.Core/MediaFiles/MediaFileService.cs index c44775bca..fb5981dc7 100644 --- a/src/NzbDrone.Core/MediaFiles/MediaFileService.cs +++ b/src/NzbDrone.Core/MediaFiles/MediaFileService.cs @@ -110,5 +110,17 @@ namespace NzbDrone.Core.MediaFiles { _mediaFileRepository.DeleteForMovies(message.Movies.Select(m => m.Id).ToList()); } + + public static List FilterExistingFiles(List files, List movieFiles, Movie movie) + { + var seriesFilePaths = movieFiles.Select(f => Path.Combine(movie.Path, f.RelativePath)).ToList(); + + if (!seriesFilePaths.Any()) + { + return files; + } + + return files.Except(seriesFilePaths, PathEqualityComparer.Instance).ToList(); + } } } diff --git a/src/NzbDrone.Core/MediaFiles/MediaInfo/UpdateMediaInfoService.cs b/src/NzbDrone.Core/MediaFiles/MediaInfo/UpdateMediaInfoService.cs index 50036e259..052026679 100644 --- a/src/NzbDrone.Core/MediaFiles/MediaInfo/UpdateMediaInfoService.cs +++ b/src/NzbDrone.Core/MediaFiles/MediaInfo/UpdateMediaInfoService.cs @@ -11,10 +11,10 @@ namespace NzbDrone.Core.MediaFiles.MediaInfo { public interface IUpdateMediaInfo { - void Update(MovieFile movieFile, Movie movie); + bool Update(MovieFile movieFile, Movie movie); } - public class UpdateMediaInfoService : IHandle, IUpdateMediaInfo + public class UpdateMediaInfoService : IUpdateMediaInfo, IHandle { private readonly IDiskProvider _diskProvider; private readonly IMediaFileService _mediaFileService; @@ -54,35 +54,39 @@ namespace NzbDrone.Core.MediaFiles.MediaInfo } } - public void Update(MovieFile movieFile, Movie movie) + public bool Update(MovieFile movieFile, Movie movie) { if (!_configService.EnableMediaInfo) { _logger.Debug("MediaInfo is disabled"); - return; + return false; } - UpdateMediaInfo(movieFile, movie); + return UpdateMediaInfo(movieFile, movie); } - private void UpdateMediaInfo(MovieFile movieFile, Movie movie) + private bool UpdateMediaInfo(MovieFile movieFile, Movie movie) { var path = Path.Combine(movie.Path, movieFile.RelativePath); if (!_diskProvider.FileExists(path)) { _logger.Debug("Can't update MediaInfo because '{0}' does not exist", path); - return; + return false; } var updatedMediaInfo = _videoFileInfoReader.GetMediaInfo(path); - if (updatedMediaInfo != null) + if (updatedMediaInfo == null) { - movieFile.MediaInfo = updatedMediaInfo; - _mediaFileService.Update(movieFile); - _logger.Debug("Updated MediaInfo for '{0}'", path); + return false; } + + movieFile.MediaInfo = updatedMediaInfo; + _mediaFileService.Update(movieFile); + _logger.Debug("Updated MediaInfo for '{0}'", path); + + return true; } } } diff --git a/src/NzbDrone.Core/MediaFiles/MovieImport/ImportDecisionMaker.cs b/src/NzbDrone.Core/MediaFiles/MovieImport/ImportDecisionMaker.cs index ed707c1da..bd256ab10 100644 --- a/src/NzbDrone.Core/MediaFiles/MovieImport/ImportDecisionMaker.cs +++ b/src/NzbDrone.Core/MediaFiles/MovieImport/ImportDecisionMaker.cs @@ -16,6 +16,7 @@ namespace NzbDrone.Core.MediaFiles.MovieImport public interface IMakeImportDecision { List GetImportDecisions(List videoFiles, Movie movie); + List GetImportDecisions(List videoFiles, Movie movie, bool filterExistingFiles); List GetImportDecisions(List videoFiles, Movie movie, DownloadClientItem downloadClientItem, ParsedMovieInfo folderInfo, bool sceneSource); List GetImportDecisions(List videoFiles, Movie movie, DownloadClientItem downloadClientItem, ParsedMovieInfo folderInfo, bool sceneSource, bool filterExistingFiles); ImportDecision GetDecision(LocalMovie localMovie, DownloadClientItem downloadClientItem); @@ -53,6 +54,11 @@ namespace NzbDrone.Core.MediaFiles.MovieImport return GetImportDecisions(videoFiles, movie, null, null, false); } + public List GetImportDecisions(List videoFiles, Movie movie, bool filterExistingFiles) + { + return GetImportDecisions(videoFiles, movie, null, null, false, filterExistingFiles); + } + public List GetImportDecisions(List videoFiles, Movie movie, DownloadClientItem downloadClientItem, ParsedMovieInfo folderInfo, bool sceneSource) { return GetImportDecisions(videoFiles, movie, downloadClientItem, folderInfo, sceneSource, true);