From b4ac495983d61819d9ab84f49c880957ba57418b Mon Sep 17 00:00:00 2001 From: Jendrik Weise Date: Sun, 19 Nov 2023 19:52:37 +0100 Subject: [PATCH] New: Custom import scripts can communicate information back --- .../DiskScanServiceTests/ScanFixture.cs | 2 +- .../UpdateMediaInfoServiceFixture.cs | 13 ++- .../Extras/ExistingExtraFileService.cs | 39 +++---- src/NzbDrone.Core/Extras/ExtraService.cs | 11 ++ .../MediaFiles/DiskScanService.cs | 11 +- .../MediaFiles/EpisodeFileMovingService.cs | 1 + .../EpisodeImport/ImportApprovedEpisodes.cs | 18 ++- .../MediaFiles/Events/SeriesScannedEvent.cs | 7 +- .../MediaFiles/ScriptImportDecider.cs | 107 ++++++++++++++++-- .../MediaFiles/ScriptImportInfo.cs | 20 ++++ .../Parser/Model/LocalEpisode.cs | 4 + 11 files changed, 186 insertions(+), 47 deletions(-) create mode 100644 src/NzbDrone.Core/MediaFiles/ScriptImportInfo.cs diff --git a/src/NzbDrone.Core.Test/MediaFiles/DiskScanServiceTests/ScanFixture.cs b/src/NzbDrone.Core.Test/MediaFiles/DiskScanServiceTests/ScanFixture.cs index cd9da2e05..d3b8c000f 100644 --- a/src/NzbDrone.Core.Test/MediaFiles/DiskScanServiceTests/ScanFixture.cs +++ b/src/NzbDrone.Core.Test/MediaFiles/DiskScanServiceTests/ScanFixture.cs @@ -267,7 +267,7 @@ namespace NzbDrone.Core.Test.MediaFiles.DiskScanServiceTests Subject.Scan(_series); Mocker.GetMock() - .Verify(v => v.GetFiles(It.IsAny(), It.IsAny()), Times.Once()); + .Verify(v => v.GetFiles(It.IsAny(), It.IsAny()), Times.Exactly(2)); Mocker.GetMock() .Verify(v => v.GetImportDecisions(It.Is>(l => l.Count == 1), _series, false), Times.Once()); diff --git a/src/NzbDrone.Core.Test/MediaFiles/MediaInfo/UpdateMediaInfoServiceFixture.cs b/src/NzbDrone.Core.Test/MediaFiles/MediaInfo/UpdateMediaInfoServiceFixture.cs index de3621fcd..72af3a7ca 100644 --- a/src/NzbDrone.Core.Test/MediaFiles/MediaInfo/UpdateMediaInfoServiceFixture.cs +++ b/src/NzbDrone.Core.Test/MediaFiles/MediaInfo/UpdateMediaInfoServiceFixture.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.IO; using FizzWare.NBuilder; using FluentAssertions; @@ -71,7 +72,7 @@ namespace NzbDrone.Core.Test.MediaFiles.MediaInfo GivenFileExists(); GivenSuccessfulScan(); - Subject.Handle(new SeriesScannedEvent(_series)); + Subject.Handle(new SeriesScannedEvent(_series, new List())); Mocker.GetMock() .Verify(v => v.GetMediaInfo(Path.Combine(_series.Path, "media.mkv")), Times.Exactly(2)); @@ -97,7 +98,7 @@ namespace NzbDrone.Core.Test.MediaFiles.MediaInfo GivenFileExists(); GivenSuccessfulScan(); - Subject.Handle(new SeriesScannedEvent(_series)); + Subject.Handle(new SeriesScannedEvent(_series, new List())); Mocker.GetMock() .Verify(v => v.GetMediaInfo(Path.Combine(_series.Path, "media.mkv")), Times.Exactly(2)); @@ -123,7 +124,7 @@ namespace NzbDrone.Core.Test.MediaFiles.MediaInfo GivenFileExists(); GivenSuccessfulScan(); - Subject.Handle(new SeriesScannedEvent(_series)); + Subject.Handle(new SeriesScannedEvent(_series, new List())); Mocker.GetMock() .Verify(v => v.GetMediaInfo(Path.Combine(_series.Path, "media.mkv")), Times.Exactly(3)); @@ -146,7 +147,7 @@ namespace NzbDrone.Core.Test.MediaFiles.MediaInfo GivenSuccessfulScan(); - Subject.Handle(new SeriesScannedEvent(_series)); + Subject.Handle(new SeriesScannedEvent(_series, new List())); Mocker.GetMock() .Verify(v => v.GetMediaInfo("media.mkv"), Times.Never()); @@ -173,7 +174,7 @@ namespace NzbDrone.Core.Test.MediaFiles.MediaInfo GivenSuccessfulScan(); GivenFailedScan(Path.Combine(_series.Path, "media2.mkv")); - Subject.Handle(new SeriesScannedEvent(_series)); + Subject.Handle(new SeriesScannedEvent(_series, new List())); Mocker.GetMock() .Verify(v => v.GetMediaInfo(Path.Combine(_series.Path, "media.mkv")), Times.Exactly(1)); @@ -203,7 +204,7 @@ namespace NzbDrone.Core.Test.MediaFiles.MediaInfo GivenFileExists(); GivenSuccessfulScan(); - Subject.Handle(new SeriesScannedEvent(_series)); + Subject.Handle(new SeriesScannedEvent(_series, new List())); Mocker.GetMock() .Verify(v => v.GetMediaInfo(It.IsAny()), Times.Never()); diff --git a/src/NzbDrone.Core/Extras/ExistingExtraFileService.cs b/src/NzbDrone.Core/Extras/ExistingExtraFileService.cs index 5cfe4944f..ae17b44c6 100644 --- a/src/NzbDrone.Core/Extras/ExistingExtraFileService.cs +++ b/src/NzbDrone.Core/Extras/ExistingExtraFileService.cs @@ -2,45 +2,33 @@ using System.Collections.Generic; using System.IO; using System.Linq; using NLog; -using NzbDrone.Common.Disk; -using NzbDrone.Core.MediaFiles; using NzbDrone.Core.MediaFiles.Events; using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.Tv; namespace NzbDrone.Core.Extras { - public class ExistingExtraFileService : IHandle + public interface IExistingExtraFiles + { + List ImportExtraFiles(Series series, List possibleExtraFiles); + } + + public class ExistingExtraFileService : IExistingExtraFiles, IHandle { - private readonly IDiskProvider _diskProvider; - private readonly IDiskScanService _diskScanService; private readonly List _existingExtraFileImporters; private readonly Logger _logger; - public ExistingExtraFileService(IDiskProvider diskProvider, - IDiskScanService diskScanService, - IEnumerable existingExtraFileImporters, + public ExistingExtraFileService(IEnumerable existingExtraFileImporters, Logger logger) { - _diskProvider = diskProvider; - _diskScanService = diskScanService; _existingExtraFileImporters = existingExtraFileImporters.OrderBy(e => e.Order).ToList(); _logger = logger; } - public void Handle(SeriesScannedEvent message) + public List ImportExtraFiles(Series series, List possibleExtraFiles) { - var series = message.Series; - - if (!_diskProvider.FolderExists(series.Path)) - { - return; - } - _logger.Debug("Looking for existing extra files in {0}", series.Path); - var filesOnDisk = _diskScanService.GetNonVideoFiles(series.Path); - var possibleExtraFiles = _diskScanService.FilterPaths(series.Path, filesOnDisk); - var importedFiles = new List(); foreach (var existingExtraFileImporter in _existingExtraFileImporters) @@ -50,6 +38,15 @@ namespace NzbDrone.Core.Extras importedFiles.AddRange(imported.Select(f => Path.Combine(series.Path, f.RelativePath))); } + return importedFiles; + } + + public void Handle(SeriesScannedEvent message) + { + var series = message.Series; + var possibleExtraFiles = message.PossibleExtraFiles; + var importedFiles = ImportExtraFiles(series, possibleExtraFiles); + _logger.Info("Found {0} possible extra files, imported {1} files.", possibleExtraFiles.Count, importedFiles.Count); } } diff --git a/src/NzbDrone.Core/Extras/ExtraService.cs b/src/NzbDrone.Core/Extras/ExtraService.cs index eca970898..b29fa6d17 100644 --- a/src/NzbDrone.Core/Extras/ExtraService.cs +++ b/src/NzbDrone.Core/Extras/ExtraService.cs @@ -17,6 +17,7 @@ namespace NzbDrone.Core.Extras { public interface IExtraService { + void MoveFilesAfterRename(Series series, EpisodeFile episodeFile); void ImportEpisode(LocalEpisode localEpisode, EpisodeFile episodeFile, bool isReadOnly); } @@ -139,6 +140,16 @@ namespace NzbDrone.Core.Extras } } + public void MoveFilesAfterRename(Series series, EpisodeFile episodeFile) + { + var episodeFiles = new List { episodeFile }; + + foreach (var extraFileManager in _extraFileManagers) + { + extraFileManager.MoveFilesAfterRename(series, episodeFiles); + } + } + public void Handle(SeriesRenamedEvent message) { var series = message.Series; diff --git a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs index 589a5c1c4..4c92b0d93 100644 --- a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs +++ b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs @@ -121,7 +121,7 @@ namespace NzbDrone.Core.MediaFiles } CleanMediaFiles(series, new List()); - CompletedScanning(series); + CompletedScanning(series, new List()); return; } @@ -174,8 +174,11 @@ namespace NzbDrone.Core.MediaFiles fileInfoStopwatch.Stop(); _logger.Trace("Reprocessing existing files complete for: {0} [{1}]", series, decisionsStopwatch.Elapsed); + var filesOnDisk = GetNonVideoFiles(series.Path); + var possibleExtraFiles = FilterPaths(series.Path, filesOnDisk); + RemoveEmptySeriesFolder(series.Path); - CompletedScanning(series); + CompletedScanning(series, possibleExtraFiles); } private void CleanMediaFiles(Series series, List mediaFileList) @@ -184,10 +187,10 @@ namespace NzbDrone.Core.MediaFiles _mediaFileTableCleanupService.Clean(series, mediaFileList); } - private void CompletedScanning(Series series) + private void CompletedScanning(Series series, List possibleExtraFiles) { _logger.Info("Completed scanning disk for {0}", series.Title); - _eventAggregator.PublishEvent(new SeriesScannedEvent(series)); + _eventAggregator.PublishEvent(new SeriesScannedEvent(series, possibleExtraFiles)); } public string[] GetVideoFiles(string path, bool allDirectories = true) diff --git a/src/NzbDrone.Core/MediaFiles/EpisodeFileMovingService.cs b/src/NzbDrone.Core/MediaFiles/EpisodeFileMovingService.cs index 3ea8a9fc1..eac485671 100644 --- a/src/NzbDrone.Core/MediaFiles/EpisodeFileMovingService.cs +++ b/src/NzbDrone.Core/MediaFiles/EpisodeFileMovingService.cs @@ -130,6 +130,7 @@ namespace NzbDrone.Core.MediaFiles try { MoveEpisodeFile(episodeFile, series, episodeFile.Episodes); + localEpisode.FileRenamedAfterScriptImport = true; } catch (SameFilenameException) { diff --git a/src/NzbDrone.Core/MediaFiles/EpisodeImport/ImportApprovedEpisodes.cs b/src/NzbDrone.Core/MediaFiles/EpisodeImport/ImportApprovedEpisodes.cs index a77cc8957..5d4bb77f6 100644 --- a/src/NzbDrone.Core/MediaFiles/EpisodeImport/ImportApprovedEpisodes.cs +++ b/src/NzbDrone.Core/MediaFiles/EpisodeImport/ImportApprovedEpisodes.cs @@ -26,6 +26,7 @@ namespace NzbDrone.Core.MediaFiles.EpisodeImport private readonly IUpgradeMediaFiles _episodeFileUpgrader; private readonly IMediaFileService _mediaFileService; private readonly IExtraService _extraService; + private readonly IExistingExtraFiles _existingExtraFiles; private readonly IDiskProvider _diskProvider; private readonly IEventAggregator _eventAggregator; private readonly IManageCommandQueue _commandQueueManager; @@ -34,6 +35,7 @@ namespace NzbDrone.Core.MediaFiles.EpisodeImport public ImportApprovedEpisodes(IUpgradeMediaFiles episodeFileUpgrader, IMediaFileService mediaFileService, IExtraService extraService, + IExistingExtraFiles existingExtraFiles, IDiskProvider diskProvider, IEventAggregator eventAggregator, IManageCommandQueue commandQueueManager, @@ -42,6 +44,7 @@ namespace NzbDrone.Core.MediaFiles.EpisodeImport _episodeFileUpgrader = episodeFileUpgrader; _mediaFileService = mediaFileService; _extraService = extraService; + _existingExtraFiles = existingExtraFiles; _diskProvider = diskProvider; _eventAggregator = eventAggregator; _commandQueueManager = commandQueueManager; @@ -130,7 +133,20 @@ namespace NzbDrone.Core.MediaFiles.EpisodeImport if (newDownload) { - _extraService.ImportEpisode(localEpisode, episodeFile, copyOnly); + if (localEpisode.ScriptImported) + { + _existingExtraFiles.ImportExtraFiles(localEpisode.Series, localEpisode.PossibleExtraFiles); + + if (localEpisode.FileRenamedAfterScriptImport) + { + _extraService.MoveFilesAfterRename(localEpisode.Series, episodeFile); + } + } + + if (!localEpisode.ScriptImported || localEpisode.ShouldImportExtras) + { + _extraService.ImportEpisode(localEpisode, episodeFile, copyOnly); + } } _eventAggregator.PublishEvent(new EpisodeImportedEvent(localEpisode, episodeFile, oldFiles, newDownload, downloadClientItem)); diff --git a/src/NzbDrone.Core/MediaFiles/Events/SeriesScannedEvent.cs b/src/NzbDrone.Core/MediaFiles/Events/SeriesScannedEvent.cs index e07bbd75f..a76348b39 100644 --- a/src/NzbDrone.Core/MediaFiles/Events/SeriesScannedEvent.cs +++ b/src/NzbDrone.Core/MediaFiles/Events/SeriesScannedEvent.cs @@ -1,4 +1,5 @@ -using NzbDrone.Common.Messaging; +using System.Collections.Generic; +using NzbDrone.Common.Messaging; using NzbDrone.Core.Tv; namespace NzbDrone.Core.MediaFiles.Events @@ -6,10 +7,12 @@ namespace NzbDrone.Core.MediaFiles.Events public class SeriesScannedEvent : IEvent { public Series Series { get; private set; } + public List PossibleExtraFiles { get; set; } - public SeriesScannedEvent(Series series) + public SeriesScannedEvent(Series series, List possibleExtraFiles) { Series = series; + PossibleExtraFiles = possibleExtraFiles; } } } diff --git a/src/NzbDrone.Core/MediaFiles/ScriptImportDecider.cs b/src/NzbDrone.Core/MediaFiles/ScriptImportDecider.cs index c626d5b44..45ab383d7 100644 --- a/src/NzbDrone.Core/MediaFiles/ScriptImportDecider.cs +++ b/src/NzbDrone.Core/MediaFiles/ScriptImportDecider.cs @@ -1,6 +1,8 @@ +using System.Collections.Generic; using System.Collections.Specialized; using System.IO; using System.Linq; +using System.Text.RegularExpressions; using NLog; using NzbDrone.Common.Disk; using NzbDrone.Common.Extensions; @@ -25,6 +27,7 @@ namespace NzbDrone.Core.MediaFiles private readonly IProcessProvider _processProvider; private readonly IConfigService _configService; private readonly ITagRepository _tagRepository; + private readonly IDiskProvider _diskProvider; private readonly Logger _logger; public ImportScriptService(IProcessProvider processProvider, @@ -32,6 +35,7 @@ namespace NzbDrone.Core.MediaFiles IConfigService configService, IConfigFileProvider configFileProvider, ITagRepository tagRepository, + IDiskProvider diskProvider, Logger logger) { _processProvider = processProvider; @@ -39,9 +43,73 @@ namespace NzbDrone.Core.MediaFiles _configService = configService; _configFileProvider = configFileProvider; _tagRepository = tagRepository; + _diskProvider = diskProvider; _logger = logger; } + private static readonly Regex OutputRegex = new Regex(@"^(?:\[(?:(?MediaFile)|(?ExtraFile))\]\s?(?.+)|(?\[PreventExtraImport\])|\[MoveStatus\]\s?(?:(?DeferMove)|(?MoveComplete)|(?RenameRequested)))$", RegexOptions.Compiled); + + private ScriptImportInfo ProcessOutput(List processOutputLines) + { + var possibleExtraFiles = new List(); + string mediaFile = null; + var decision = ScriptImportDecision.MoveComplete; + var importExtraFiles = true; + + foreach (var line in processOutputLines) + { + var match = OutputRegex.Match(line.Content); + + if (match.Groups["mediaFile"].Success) + { + if (mediaFile is not null) + { + throw new ScriptImportException("Script output contains multiple media files. Only one media file can be returned."); + } + + mediaFile = match.Groups["fileName"].Value; + + if (!MediaFileExtensions.Extensions.Contains(Path.GetExtension(mediaFile))) + { + throw new ScriptImportException("Script output contains invalid media file: {0}", mediaFile); + } + else if (!_diskProvider.FileExists(mediaFile)) + { + throw new ScriptImportException("Script output contains non-existent media file: {0}", mediaFile); + } + } + else if (match.Groups["extraFile"].Success) + { + var fileName = match.Groups["fileName"].Value; + + if (!_diskProvider.FileExists(fileName)) + { + _logger.Warn("Script output contains non-existent possible extra file: {0}", fileName); + } + + possibleExtraFiles.Add(fileName); + } + else if (match.Groups["moveComplete"].Success) + { + decision = ScriptImportDecision.MoveComplete; + } + else if (match.Groups["renameRequested"].Success) + { + decision = ScriptImportDecision.RenameRequested; + } + else if (match.Groups["deferMove"].Success) + { + decision = ScriptImportDecision.DeferMove; + } + else if (match.Groups["preventExtraImport"].Success) + { + importExtraFiles = false; + } + } + + return new ScriptImportInfo(possibleExtraFiles, mediaFile, decision, importExtraFiles); + } + public ScriptImportDecision TryImport(string sourcePath, string destinationFilePath, LocalEpisode localEpisode, EpisodeFile episodeFile, TransferMode mode) { var series = localEpisode.Series; @@ -115,22 +183,37 @@ namespace NzbDrone.Core.MediaFiles var processOutput = _processProvider.StartAndCapture(_configService.ScriptImportPath, $"\"{sourcePath}\" \"{destinationFilePath}\"", environmentVariables); - _logger.Debug("Executed external script: {0} - Status: {1}", _configService.ScriptImportPath, processOutput.ExitCode); _logger.Debug("Script Output: \r\n{0}", string.Join("\r\n", processOutput.Lines)); - switch (processOutput.ExitCode) + if (processOutput.ExitCode != 0) { - case 0: // Copy complete - return ScriptImportDecision.MoveComplete; - case 2: // Copy complete, file potentially changed, should try renaming again - episodeFile.MediaInfo = _videoFileInfoReader.GetMediaInfo(destinationFilePath); - episodeFile.Path = null; - return ScriptImportDecision.RenameRequested; - case 3: // Let Sonarr handle it - return ScriptImportDecision.DeferMove; - default: // Error, fail to import - throw new ScriptImportException("Moving with script failed! Exit code {0}", processOutput.ExitCode); + throw new ScriptImportException("Script exited with non-zero exit code: {0}", processOutput.ExitCode); } + + var scriptImportInfo = ProcessOutput(processOutput.Lines); + + var mediaFile = scriptImportInfo.MediaFile ?? destinationFilePath; + localEpisode.PossibleExtraFiles = scriptImportInfo.PossibleExtraFiles; + + episodeFile.RelativePath = series.Path.GetRelativePath(mediaFile); + episodeFile.Path = mediaFile; + + var exitCode = processOutput.ExitCode; + + localEpisode.ShouldImportExtras = scriptImportInfo.ImportExtraFiles; + + if (scriptImportInfo.Decision != ScriptImportDecision.DeferMove) + { + localEpisode.ScriptImported = true; + } + + if (scriptImportInfo.Decision == ScriptImportDecision.RenameRequested) + { + episodeFile.MediaInfo = _videoFileInfoReader.GetMediaInfo(mediaFile); + episodeFile.Path = null; + } + + return scriptImportInfo.Decision; } } } diff --git a/src/NzbDrone.Core/MediaFiles/ScriptImportInfo.cs b/src/NzbDrone.Core/MediaFiles/ScriptImportInfo.cs new file mode 100644 index 000000000..6c861fa8d --- /dev/null +++ b/src/NzbDrone.Core/MediaFiles/ScriptImportInfo.cs @@ -0,0 +1,20 @@ +using System.Collections.Generic; + +namespace NzbDrone.Core.MediaFiles +{ + public struct ScriptImportInfo + { + public List PossibleExtraFiles { get; set; } + public string MediaFile { get; set; } + public ScriptImportDecision Decision { get; set; } + public bool ImportExtraFiles { get; set; } + + public ScriptImportInfo(List possibleExtraFiles, string mediaFile, ScriptImportDecision decision, bool importExtraFiles) + { + PossibleExtraFiles = possibleExtraFiles; + MediaFile = mediaFile; + Decision = decision; + ImportExtraFiles = importExtraFiles; + } + } +} diff --git a/src/NzbDrone.Core/Parser/Model/LocalEpisode.cs b/src/NzbDrone.Core/Parser/Model/LocalEpisode.cs index bcb9a5132..470310b7c 100644 --- a/src/NzbDrone.Core/Parser/Model/LocalEpisode.cs +++ b/src/NzbDrone.Core/Parser/Model/LocalEpisode.cs @@ -40,6 +40,10 @@ namespace NzbDrone.Core.Parser.Model public List CustomFormats { get; set; } public int CustomFormatScore { get; set; } public GrabbedReleaseInfo Release { get; set; } + public bool ScriptImported { get; set; } + public bool FileRenamedAfterScriptImport { get; set; } + public bool ShouldImportExtras { get; set; } + public List PossibleExtraFiles { get; set; } public int SeasonNumber {