From b8f7ca0749ec89742891efdf30da9cf8ddf9d638 Mon Sep 17 00:00:00 2001 From: Qstick Date: Tue, 9 Jul 2019 23:14:53 -0400 Subject: [PATCH] Fixed: DiskScanService Updates --- .../Blackhole/ScanWatchFolderFixture.cs | 15 ++- .../Blackhole/TorrentBlackholeFixture.cs | 5 + .../Blackhole/UsenetBlackholeFixture.cs | 8 ++ .../HadoukenTests/HadoukenFixture.cs | 3 + .../NzbVortexTests/NzbVortexFixture.cs | 17 +++- .../SabnzbdTests/SabnzbdFixture.cs | 15 +++ .../VuzeTests/VuzeFixture.cs | 17 ++-- .../DownloadedMoviesImportServiceFixture.cs | 9 +- .../GetVideoFilesFixture.cs | 7 +- .../DownloadDecisionPriorizationService.cs | 8 +- .../DiskSpace/DiskSpaceService.cs | 2 +- .../Clients/Blackhole/ScanWatchFolder.cs | 17 ++-- .../Extras/ExistingExtraFileService.cs | 2 +- .../MediaFiles/DiskScanService.cs | 94 ++++++++++--------- .../DownloadedMovieImportService.cs | 2 +- .../MovieImport/Manual/ManualImportService.cs | 2 +- .../MetadataSource/SkyHook/SkyHookProxy.cs | 23 +++-- 17 files changed, 155 insertions(+), 91 deletions(-) diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/ScanWatchFolderFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/ScanWatchFolderFixture.cs index 49fc1863c..bb085409d 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/ScanWatchFolderFixture.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/ScanWatchFolderFixture.cs @@ -1,15 +1,17 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; using FluentAssertions; using Moq; using NUnit.Framework; using NzbDrone.Common.Disk; using NzbDrone.Core.Download; -using NzbDrone.Test.Common; -using System.Threading; -using NzbDrone.Core.Test.Framework; using NzbDrone.Core.Download.Clients.Blackhole; +using NzbDrone.Core.MediaFiles; +using NzbDrone.Core.Test.Framework; +using NzbDrone.Test.Common; namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole { @@ -18,7 +20,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole { protected readonly string _title = "Droned.1998.1080p.WEB-DL-DRONE"; protected string _completedDownloadFolder = @"c:\blackhole\completed".AsOsAgnostic(); - + protected void GivenCompletedItem() { var targetDir = Path.Combine(_completedDownloadFolder, _title); @@ -33,6 +35,9 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole Mocker.GetMock() .Setup(c => c.GetFileSize(It.IsAny())) .Returns(1000000); + + Mocker.GetMock().Setup(c => c.FilterFiles(It.IsAny(), It.IsAny>())) + .Returns>((b, s) => s.ToList()); } protected void GivenChangedItem() @@ -43,7 +48,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole .Setup(c => c.GetFileSize(It.IsAny())) .Returns(currentSize + 1); } - + private void VerifySingleItem(DownloadItemStatus status) { var items = Subject.GetItems(_completedDownloadFolder, TimeSpan.FromMilliseconds(50)).ToList(); diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/TorrentBlackholeFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/TorrentBlackholeFixture.cs index 554f09e85..ac7aee2af 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/TorrentBlackholeFixture.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/TorrentBlackholeFixture.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; @@ -10,6 +11,7 @@ using NzbDrone.Common.Http; using NzbDrone.Core.Download; using NzbDrone.Core.Download.Clients.Blackhole; using NzbDrone.Core.Exceptions; +using NzbDrone.Core.MediaFiles; using NzbDrone.Core.MediaFiles.TorrentInfo; using NzbDrone.Core.Parser.Model; using NzbDrone.Test.Common; @@ -48,6 +50,9 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole Mocker.GetMock() .Setup(c => c.GetHashFromTorrentFile(It.IsAny())) .Returns("myhash"); + + Mocker.GetMock().Setup(c => c.FilterFiles(It.IsAny(), It.IsAny>())) + .Returns>((b, s) => s.ToList()); } protected void GivenFailedDownload() diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/UsenetBlackholeFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/UsenetBlackholeFixture.cs index e730f71e3..071765976 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/UsenetBlackholeFixture.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/UsenetBlackholeFixture.cs @@ -1,5 +1,6 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; @@ -10,6 +11,7 @@ using NzbDrone.Common.Disk; using NzbDrone.Common.Http; using NzbDrone.Core.Download; using NzbDrone.Core.Download.Clients.Blackhole; +using NzbDrone.Core.MediaFiles; using NzbDrone.Test.Common; namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole @@ -41,6 +43,9 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole Mocker.GetMock() .Setup(c => c.OpenWriteStream(It.IsAny())) .Returns(() => new FileStream(GetTempFilePath(), FileMode.Create)); + + Mocker.GetMock().Setup(c => c.FilterFiles(It.IsAny(), It.IsAny>())) + .Returns>((b, s) => s.ToList()); } protected void GivenFailedDownload() @@ -77,6 +82,9 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole var result = Subject.GetItems().Single(); VerifyCompleted(result); + + result.CanBeRemoved.Should().BeTrue(); + result.CanMoveFiles.Should().BeTrue(); } [Test] diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientTests/HadoukenTests/HadoukenFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadClientTests/HadoukenTests/HadoukenFixture.cs index 93bd165a1..d9f87a880 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadClientTests/HadoukenTests/HadoukenFixture.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadClientTests/HadoukenTests/HadoukenFixture.cs @@ -190,6 +190,9 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.HadoukenTests PrepareClientToReturnCompletedItem(); var item = Subject.GetItems().Single(); VerifyCompleted(item); + + item.CanBeRemoved.Should().BeTrue(); + item.CanMoveFiles.Should().BeTrue(); } [Test] diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientTests/NzbVortexTests/NzbVortexFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadClientTests/NzbVortexTests/NzbVortexFixture.cs index 3673f34b6..964c5978e 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadClientTests/NzbVortexTests/NzbVortexFixture.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadClientTests/NzbVortexTests/NzbVortexFixture.cs @@ -103,10 +103,13 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.NzbVortexTests public void queued_item_should_have_required_properties() { GivenQueue(_queued); - + var result = Subject.GetItems().Single(); VerifyQueued(result); + + result.CanBeRemoved.Should().BeTrue(); + result.CanMoveFiles.Should().BeTrue(); } [Test] @@ -118,6 +121,9 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.NzbVortexTests var result = Subject.GetItems().Single(); VerifyPaused(result); + + result.CanBeRemoved.Should().BeTrue(); + result.CanMoveFiles.Should().BeTrue(); } [Test] @@ -129,6 +135,9 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.NzbVortexTests var result = Subject.GetItems().Single(); VerifyDownloading(result); + + result.CanBeRemoved.Should().BeTrue(); + result.CanMoveFiles.Should().BeTrue(); } [Test] @@ -139,6 +148,9 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.NzbVortexTests var result = Subject.GetItems().Single(); VerifyCompleted(result); + + result.CanBeRemoved.Should().BeTrue(); + result.CanMoveFiles.Should().BeTrue(); } [Test] @@ -149,6 +161,9 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.NzbVortexTests var result = Subject.GetItems().Single(); VerifyFailed(result); + + result.CanBeRemoved.Should().BeTrue(); + result.CanMoveFiles.Should().BeTrue(); } [Test] diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientTests/SabnzbdTests/SabnzbdFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadClientTests/SabnzbdTests/SabnzbdFixture.cs index 1b72fb9f7..e3e1a19ec 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadClientTests/SabnzbdTests/SabnzbdFixture.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadClientTests/SabnzbdTests/SabnzbdFixture.cs @@ -191,7 +191,10 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.SabnzbdTests var result = Subject.GetItems().Single(); VerifyQueued(result); + result.RemainingTime.Should().NotBe(TimeSpan.Zero); + result.CanBeRemoved.Should().BeTrue(); + result.CanMoveFiles.Should().BeTrue(); } [TestCase(SabnzbdDownloadStatus.Paused)] @@ -205,6 +208,9 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.SabnzbdTests var result = Subject.GetItems().Single(); VerifyPaused(result); + + result.CanBeRemoved.Should().BeTrue(); + result.CanMoveFiles.Should().BeTrue(); } [TestCase(SabnzbdDownloadStatus.Checking)] @@ -227,7 +233,10 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.SabnzbdTests var result = Subject.GetItems().Single(); VerifyDownloading(result); + result.RemainingTime.Should().NotBe(TimeSpan.Zero); + result.CanBeRemoved.Should().BeTrue(); + result.CanMoveFiles.Should().BeTrue(); } [Test] @@ -239,6 +248,9 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.SabnzbdTests var result = Subject.GetItems().Single(); VerifyCompleted(result); + + result.CanBeRemoved.Should().BeTrue(); + result.CanMoveFiles.Should().BeTrue(); } [Test] @@ -252,6 +264,9 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.SabnzbdTests var result = Subject.GetItems().Single(); VerifyFailed(result); + + result.CanBeRemoved.Should().BeTrue(); + result.CanMoveFiles.Should().BeTrue(); } [Test] diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientTests/VuzeTests/VuzeFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadClientTests/VuzeTests/VuzeFixture.cs index 1abf92193..88359757c 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadClientTests/VuzeTests/VuzeFixture.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadClientTests/VuzeTests/VuzeFixture.cs @@ -47,9 +47,12 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.VuzeTests [Test] public void completed_download_should_have_required_properties() { - PrepareClientToReturnCompletedItem(); + PrepareClientToReturnCompletedItem(true, ratioLimit: 0.5); var item = Subject.GetItems().Single(); VerifyCompleted(item); + + item.CanBeRemoved.Should().BeTrue(); + item.CanMoveFiles.Should().BeTrue(); } [Test] @@ -72,7 +75,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.VuzeTests } [Test] - public void Download_with_TvDirectory_should_force_directory() + public void Download_with_MovieDirectory_should_force_directory() { GivenMovieDirectory(); GivenSuccessfulDownload(); @@ -122,7 +125,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.VuzeTests } [Test] - public void Download_without_TvDirectory_and_Category_should_use_default() + public void Download_without_MovieDirectory_and_Category_should_use_default() { GivenSuccessfulDownload(); @@ -181,13 +184,13 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.VuzeTests item.Status.Should().Be(expectedItemStatus); } - [TestCase(TransmissionTorrentStatus.Stopped, DownloadItemStatus.Completed, true)] + [TestCase(TransmissionTorrentStatus.Stopped, DownloadItemStatus.Completed, false)] [TestCase(TransmissionTorrentStatus.CheckWait, DownloadItemStatus.Downloading, false)] [TestCase(TransmissionTorrentStatus.Check, DownloadItemStatus.Downloading, false)] [TestCase(TransmissionTorrentStatus.Queued, DownloadItemStatus.Queued, false)] [TestCase(TransmissionTorrentStatus.SeedingWait, DownloadItemStatus.Completed, false)] [TestCase(TransmissionTorrentStatus.Seeding, DownloadItemStatus.Completed, false)] - public void GetItems_should_return_completed_item_as_downloadItemStatus(TransmissionTorrentStatus apiStatus, DownloadItemStatus expectedItemStatus, bool expectedReadOnly) + public void GetItems_should_return_completed_item_as_downloadItemStatus(TransmissionTorrentStatus apiStatus, DownloadItemStatus expectedItemStatus, bool expectedValue) { _completed.Status = apiStatus; @@ -196,8 +199,8 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.VuzeTests var item = Subject.GetItems().Single(); item.Status.Should().Be(expectedItemStatus); - item.CanBeRemoved.Should().Be(expectedReadOnly); - item.CanMoveFiles.Should().Be(expectedReadOnly); + item.CanBeRemoved.Should().Be(expectedValue); + item.CanMoveFiles.Should().Be(expectedValue); } [Test] diff --git a/src/NzbDrone.Core.Test/MediaFiles/DownloadedMoviesImportServiceFixture.cs b/src/NzbDrone.Core.Test/MediaFiles/DownloadedMoviesImportServiceFixture.cs index c93545bae..01652b721 100644 --- a/src/NzbDrone.Core.Test/MediaFiles/DownloadedMoviesImportServiceFixture.cs +++ b/src/NzbDrone.Core.Test/MediaFiles/DownloadedMoviesImportServiceFixture.cs @@ -2,9 +2,12 @@ using System.Collections.Generic; using System.IO; using System.Linq; using FizzWare.NBuilder; +using FluentAssertions; using Moq; using NUnit.Framework; using NzbDrone.Common.Disk; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Download; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.MediaFiles.MovieImport; using NzbDrone.Core.Parser; @@ -13,9 +16,6 @@ using NzbDrone.Core.Qualities; using NzbDrone.Core.Test.Framework; using NzbDrone.Core.Movies; using NzbDrone.Test.Common; -using FluentAssertions; -using NzbDrone.Core.Configuration; -using NzbDrone.Core.Download; namespace NzbDrone.Core.Test.MediaFiles { @@ -35,6 +35,9 @@ namespace NzbDrone.Core.Test.MediaFiles Mocker.GetMock().Setup(c => c.GetVideoFiles(It.IsAny(), It.IsAny())) .Returns(_videoFiles); + Mocker.GetMock().Setup(c => c.FilterFiles(It.IsAny(), It.IsAny>())) + .Returns>((b, s) => s.ToList()); + Mocker.GetMock().Setup(c => c.GetDirectories(It.IsAny())) .Returns(_subFolders); diff --git a/src/NzbDrone.Core.Test/ProviderTests/DiskScanProviderTests/GetVideoFilesFixture.cs b/src/NzbDrone.Core.Test/ProviderTests/DiskScanProviderTests/GetVideoFilesFixture.cs index 28d34c001..9358ca63d 100644 --- a/src/NzbDrone.Core.Test/ProviderTests/DiskScanProviderTests/GetVideoFilesFixture.cs +++ b/src/NzbDrone.Core.Test/ProviderTests/DiskScanProviderTests/GetVideoFilesFixture.cs @@ -92,6 +92,7 @@ namespace NzbDrone.Core.Test.ProviderTests.DiskScanProviderTests [TestCase("Plex Versions")] [TestCase(".secret")] [TestCase(".hidden")] + [TestCase(".unwanted")] public void should_filter_certain_sub_folders(string subFolder) { var path = @"C:\Test\"; @@ -99,11 +100,7 @@ namespace NzbDrone.Core.Test.ProviderTests.DiskScanProviderTests var specialFiles = GetFiles(path, subFolder).ToList(); var allFiles = files.Concat(specialFiles); - var series = Builder.CreateNew() - .With(s => s.Path = path) - .Build(); - - var filteredFiles = Subject.FilterFiles(series, allFiles); + var filteredFiles = Subject.FilterFiles(path, allFiles); filteredFiles.Should().NotContain(specialFiles); filteredFiles.Count.Should().BeGreaterThan(0); } diff --git a/src/NzbDrone.Core/DecisionEngine/DownloadDecisionPriorizationService.cs b/src/NzbDrone.Core/DecisionEngine/DownloadDecisionPriorizationService.cs index f77b1bff5..3b3acbb72 100644 --- a/src/NzbDrone.Core/DecisionEngine/DownloadDecisionPriorizationService.cs +++ b/src/NzbDrone.Core/DecisionEngine/DownloadDecisionPriorizationService.cs @@ -1,7 +1,7 @@ using System.Linq; using System.Collections.Generic; -using NzbDrone.Core.Profiles.Delay; using NzbDrone.Core.Configuration; +using NzbDrone.Core.Profiles.Delay; using NzbDrone.Core.Parser; namespace NzbDrone.Core.DecisionEngine @@ -13,13 +13,13 @@ namespace NzbDrone.Core.DecisionEngine public class DownloadDecisionPriorizationService : IPrioritizeDownloadDecision { - private readonly IDelayProfileService _delayProfileService; private readonly IConfigService _configService; + private readonly IDelayProfileService _delayProfileService; - public DownloadDecisionPriorizationService(IDelayProfileService delayProfileService, IConfigService configService) + public DownloadDecisionPriorizationService(IConfigService configService, IDelayProfileService delayProfileService) { - _delayProfileService = delayProfileService; _configService = configService; + _delayProfileService = delayProfileService; } public List PrioritizeDecisionsForMovies(List decisions) diff --git a/src/NzbDrone.Core/DiskSpace/DiskSpaceService.cs b/src/NzbDrone.Core/DiskSpace/DiskSpaceService.cs index 09e83749c..cb340619c 100644 --- a/src/NzbDrone.Core/DiskSpace/DiskSpaceService.cs +++ b/src/NzbDrone.Core/DiskSpace/DiskSpaceService.cs @@ -21,7 +21,7 @@ namespace NzbDrone.Core.DiskSpace private readonly IDiskProvider _diskProvider; private readonly Logger _logger; - private static readonly Regex _regexSpecialDrive = new Regex("^/var/lib/(docker|rancher|kubelet)(/|$)|^/(boot|etc|snap)(/|$)|/docker(/var)?/aufs(/|$)", RegexOptions.Compiled); + private static readonly Regex _regexSpecialDrive = new Regex("^/var/lib/(docker|rancher|kubelet)(/|$)|^/(boot|etc)(/|$)|/docker(/var)?/aufs(/|$)", RegexOptions.Compiled); public DiskSpaceService(IMovieService movieService, IDiskProvider diskProvider, Logger logger) { diff --git a/src/NzbDrone.Core/Download/Clients/Blackhole/ScanWatchFolder.cs b/src/NzbDrone.Core/Download/Clients/Blackhole/ScanWatchFolder.cs index 105e0e837..fe69586d2 100644 --- a/src/NzbDrone.Core/Download/Clients/Blackhole/ScanWatchFolder.cs +++ b/src/NzbDrone.Core/Download/Clients/Blackhole/ScanWatchFolder.cs @@ -1,3 +1,8 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; using NLog; using NzbDrone.Common.Cache; using NzbDrone.Common.Crypto; @@ -5,11 +10,6 @@ using NzbDrone.Common.Disk; using NzbDrone.Common.Extensions; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.Organizer; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; namespace NzbDrone.Core.Download.Clients.Blackhole { @@ -39,7 +39,7 @@ namespace NzbDrone.Core.Download.Clients.Blackhole { var newWatchItems = new Dictionary(); var lastWatchItems = _watchFolderItemCache.Get(watchFolder, () => newWatchItems); - + foreach (var newWatchItem in GetDownloadItems(watchFolder, lastWatchItems, waitPeriod)) { newWatchItems[newWatchItem.DownloadId] = newWatchItem; @@ -52,8 +52,7 @@ namespace NzbDrone.Core.Download.Clients.Blackhole private IEnumerable GetDownloadItems(string watchFolder, Dictionary lastWatchItems, TimeSpan waitPeriod) { - // get a fresh naming config each time, in case the user has made changes - foreach (var folder in _diskProvider.GetDirectories(watchFolder)) + foreach (var folder in _diskScanService.FilterFiles(watchFolder, _diskProvider.GetDirectories(watchFolder))) { var title = FileNameBuilder.CleanFileName(Path.GetFileName(folder)); @@ -89,7 +88,7 @@ namespace NzbDrone.Core.Download.Clients.Blackhole yield return newWatchItem; } - foreach (var videoFile in _diskScanService.GetVideoFiles(watchFolder, false)) + foreach (var videoFile in _diskScanService.FilterFiles(watchFolder, _diskScanService.GetVideoFiles(watchFolder, false))) { var title = FileNameBuilder.CleanFileName(Path.GetFileName(videoFile)); diff --git a/src/NzbDrone.Core/Extras/ExistingExtraFileService.cs b/src/NzbDrone.Core/Extras/ExistingExtraFileService.cs index 4bfeea61e..3f6d4a003 100644 --- a/src/NzbDrone.Core/Extras/ExistingExtraFileService.cs +++ b/src/NzbDrone.Core/Extras/ExistingExtraFileService.cs @@ -41,7 +41,7 @@ namespace NzbDrone.Core.Extras _logger.Debug("Looking for existing extra files in {0}", movie.Path); var filesOnDisk = _diskScanService.GetNonVideoFiles(movie.Path); - var possibleExtraFiles = _diskScanService.FilterFiles(movie, filesOnDisk); + var possibleExtraFiles = _diskScanService.FilterFiles(movie.Path, filesOnDisk); var filteredFiles = possibleExtraFiles; var importedFiles = new List(); diff --git a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs index 4e04c4987..4ae7a5481 100644 --- a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs +++ b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs @@ -16,6 +16,7 @@ using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Movies; using NzbDrone.Core.Movies.Events; +using NzbDrone.Core.RootFolders; namespace NzbDrone.Core.MediaFiles { @@ -24,7 +25,7 @@ namespace NzbDrone.Core.MediaFiles void Scan(Movie movie); string[] GetVideoFiles(string path, bool allDirectories = true); string[] GetNonVideoFiles(string path, bool allDirectories = true); - List FilterFiles(Movie movie, IEnumerable files); + List FilterFiles(string basePath, IEnumerable files); } public class DiskScanService : @@ -35,9 +36,10 @@ namespace NzbDrone.Core.MediaFiles private readonly IMakeImportDecision _importDecisionMaker; private readonly IImportApprovedMovie _importApprovedMovies; private readonly IConfigService _configService; - private readonly IMediaFileTableCleanupService _mediaFileTableCleanupService; - private readonly IEventAggregator _eventAggregator; private readonly IMovieService _movieService; + private readonly IMediaFileTableCleanupService _mediaFileTableCleanupService; + private readonly IRootFolderService _rootFolderService; + private readonly IEventAggregator _eventAggregator; private readonly IMediaFileService _movieFileRepository; private readonly IRenameMovieFileService _renameMovieFiles; private readonly Logger _logger; @@ -46,9 +48,10 @@ namespace NzbDrone.Core.MediaFiles IMakeImportDecision importDecisionMaker, IImportApprovedMovie importApprovedMovies, IConfigService configService, - IMediaFileTableCleanupService mediaFileTableCleanupService, - IEventAggregator eventAggregator, IMovieService movieService, + IMediaFileTableCleanupService mediaFileTableCleanupService, + IRootFolderService rootFolderService, + IEventAggregator eventAggregator, IMediaFileService movieFileRepository, IRenameMovieFileService renameMovieFiles, Logger logger) @@ -57,23 +60,21 @@ namespace NzbDrone.Core.MediaFiles _importDecisionMaker = importDecisionMaker; _importApprovedMovies = importApprovedMovies; _configService = configService; - _mediaFileTableCleanupService = mediaFileTableCleanupService; - _eventAggregator = eventAggregator; _movieService = movieService; + _mediaFileTableCleanupService = mediaFileTableCleanupService; + _rootFolderService = rootFolderService; + _eventAggregator = eventAggregator; _movieFileRepository = movieFileRepository; - _renameMovieFiles = renameMovieFiles; + _renameMovieFiles = renameMovieFiles; _logger = logger; } - private static readonly Regex ExcludedSubFoldersRegex = new Regex(@"(?:\\|\/|^)(extras|@eadir|extrafanart|plex\sversions|\..+)(?:\\|\/)", RegexOptions.Compiled | RegexOptions.IgnoreCase); - private static readonly Regex ExcludedFilesRegex = new Regex(@"^\._|Thumbs\.db", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex ExcludedSubFoldersRegex = new Regex(@"(?:\\|\/|^)(?:extras|@eadir|extrafanart|plex versions|\.[^\\/]+)(?:\\|\/)", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex ExcludedFilesRegex = new Regex(@"^\._|^Thumbs\.db$", RegexOptions.Compiled | RegexOptions.IgnoreCase); public void Scan(Movie movie) { - //Try renaming the movie path in case anything changed such as year, title or something else. - _renameMovieFiles.RenameMoviePath(movie, true); - - var rootFolder = _diskProvider.GetParentFolder(movie.Path); + var rootFolder = _rootFolderService.GetBestRootFolderPath(movie.Path); if (!_diskProvider.FolderExists(rootFolder)) { @@ -93,17 +94,9 @@ namespace NzbDrone.Core.MediaFiles if (!_diskProvider.FolderExists(movie.Path)) { - if (movie.MovieFileId != 0) + if (_configService.CreateEmptyMovieFolders) { - //Since there is no folder, there can't be any files right? - _mediaFileTableCleanupService.Clean(movie, new List()); - - _logger.Debug("Movies folder doesn't exist: {0}", movie.Path); - } - else if (_configService.CreateEmptyMovieFolders && - _diskProvider.FolderExists(rootFolder)) - { - _logger.Debug("Creating missing movies folder: {0}", movie.Path); + _logger.Debug("Creating missing movie folder: {0}", movie.Path); _diskProvider.CreateFolder(movie.Path); SetPermissions(movie.Path); } @@ -112,28 +105,37 @@ namespace NzbDrone.Core.MediaFiles _logger.Debug("Movies folder doesn't exist: {0}", movie.Path); } - _eventAggregator.PublishEvent(new MovieScanSkippedEvent(movie, MovieScanSkippedReason.MovieFolderDoesNotExist)); + CleanMediaFiles(movie, new List()); + CompletedScanning(movie); + return; } var videoFilesStopwatch = Stopwatch.StartNew(); - var mediaFileList = FilterFiles(movie, GetVideoFiles(movie.Path)).ToList(); - + var mediaFileList = FilterFiles(movie.Path, GetVideoFiles(movie.Path)).ToList(); videoFilesStopwatch.Stop(); _logger.Trace("Finished getting movie files for: {0} [{1}]", movie, videoFilesStopwatch.Elapsed); - _logger.Debug("{0} Cleaning up media files in DB", movie); - _mediaFileTableCleanupService.Clean(movie, mediaFileList); + CleanMediaFiles(movie, mediaFileList); var decisionsStopwatch = Stopwatch.StartNew(); var decisions = _importDecisionMaker.GetImportDecisions(mediaFileList, movie, true); decisionsStopwatch.Stop(); _logger.Trace("Import decisions complete for: {0} [{1}]", movie, decisionsStopwatch.Elapsed); - _importApprovedMovies.Import(decisions, false); RemoveEmptyMovieFolder(movie.Path); + CompletedScanning(movie); + } + private void CleanMediaFiles(Movie movie, List mediaFileList) + { + _logger.Debug("{0} Cleaning up media files in DB", movie); + _mediaFileTableCleanupService.Clean(movie, mediaFileList); + } + + private void CompletedScanning(Movie movie) + { _logger.Info("Completed scanning disk for {0}", movie.Title); _eventAggregator.PublishEvent(new MovieScannedEvent(movie)); } @@ -143,12 +145,14 @@ namespace NzbDrone.Core.MediaFiles _logger.Debug("Scanning '{0}' for video files", path); var searchOption = allDirectories ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly; - var filesOnDisk = _diskProvider.GetFiles(path, searchOption); + var filesOnDisk = _diskProvider.GetFiles(path, searchOption).ToList(); var mediaFileList = filesOnDisk.Where(file => MediaFileExtensions.Extensions.Contains(Path.GetExtension(file))) .ToList(); + _logger.Trace("{0} files were found in {1}", filesOnDisk.Count, path); _logger.Debug("{0} video files were found in {1}", mediaFileList.Count, path); + return mediaFileList.ToArray(); } @@ -157,18 +161,20 @@ namespace NzbDrone.Core.MediaFiles _logger.Debug("Scanning '{0}' for non-video files", path); var searchOption = allDirectories ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly; - var filesOnDisk = _diskProvider.GetFiles(path, searchOption); + var filesOnDisk = _diskProvider.GetFiles(path, searchOption).ToList(); var mediaFileList = filesOnDisk.Where(file => !MediaFileExtensions.Extensions.Contains(Path.GetExtension(file))) .ToList(); + _logger.Trace("{0} files were found in {1}", filesOnDisk.Count, path); _logger.Debug("{0} non-video files were found in {1}", mediaFileList.Count, path); + return mediaFileList.ToArray(); } - public List FilterFiles(Movie movie, IEnumerable files) + public List FilterFiles(string basePath, IEnumerable files) { - return files.Where(file => !ExcludedSubFoldersRegex.IsMatch(movie.Path.GetRelativePath(file))) + return files.Where(file => !ExcludedSubFoldersRegex.IsMatch(basePath.GetRelativePath(file))) .Where(file => !ExcludedFilesRegex.IsMatch(Path.GetFileName(file))) .ToList(); } @@ -196,20 +202,17 @@ namespace NzbDrone.Core.MediaFiles private void RemoveEmptyMovieFolder(string path) { - if (_diskProvider.GetFiles(path, SearchOption.AllDirectories).Empty() && - !_configService.CreateEmptyMovieFolders) - if (_configService.DeleteEmptyFolders) + if (_configService.DeleteEmptyFolders) + { + if (_diskProvider.GetFiles(path, SearchOption.AllDirectories).Empty()) { _diskProvider.DeleteFolder(path, true); - if (_diskProvider.GetFiles(path, SearchOption.AllDirectories).Empty()) - { - _diskProvider.DeleteFolder(path, true); - } - else - { - _diskProvider.RemoveEmptySubfolders(path); - } } + else + { + _diskProvider.RemoveEmptySubfolders(path); + } + } } public void Execute(RescanMovieCommand message) @@ -219,6 +222,7 @@ namespace NzbDrone.Core.MediaFiles var movie = _movieService.GetMovie(message.MovieId.Value); Scan(movie); } + else { var allMovies = _movieService.GetAllMovies(); diff --git a/src/NzbDrone.Core/MediaFiles/DownloadedMovieImportService.cs b/src/NzbDrone.Core/MediaFiles/DownloadedMovieImportService.cs index 3283a727a..54a72254f 100644 --- a/src/NzbDrone.Core/MediaFiles/DownloadedMovieImportService.cs +++ b/src/NzbDrone.Core/MediaFiles/DownloadedMovieImportService.cs @@ -182,7 +182,7 @@ namespace NzbDrone.Core.MediaFiles _logger.Debug("{0} folder quality: {1}", cleanedUpName, folderInfo.Quality); } - var videoFiles = _diskScanService.GetVideoFiles(directoryInfo.FullName); + var videoFiles = _diskScanService.FilterFiles(directoryInfo.FullName, _diskScanService.GetVideoFiles(directoryInfo.FullName)); if (downloadClientItem == null) { diff --git a/src/NzbDrone.Core/MediaFiles/MovieImport/Manual/ManualImportService.cs b/src/NzbDrone.Core/MediaFiles/MovieImport/Manual/ManualImportService.cs index c1253c151..d5ff026f2 100644 --- a/src/NzbDrone.Core/MediaFiles/MovieImport/Manual/ManualImportService.cs +++ b/src/NzbDrone.Core/MediaFiles/MovieImport/Manual/ManualImportService.cs @@ -116,7 +116,7 @@ namespace NzbDrone.Core.MediaFiles.MovieImport.Manual if (movie == null) { - var files = _diskScanService.GetVideoFiles(folder); + var files = _diskScanService.FilterFiles(folder, _diskScanService.GetVideoFiles(folder)); return files.Select(file => ProcessFile(file, downloadId, folder)).Where(i => i != null).ToList(); } diff --git a/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs b/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs index f0aa00536..b2004a72d 100644 --- a/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs +++ b/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs @@ -84,11 +84,15 @@ namespace NzbDrone.Core.MetadataSource.SkyHook // The dude abides, so should us, Lets be nice to TMDb // var allowed = int.Parse(response.Headers.GetValues("X-RateLimit-Limit").First()); // get allowed // var reset = long.Parse(response.Headers.GetValues("X-RateLimit-Reset").First()); // get time when it resets - var remaining = int.Parse(response.Headers.GetValues("X-RateLimit-Remaining").First()); - if (remaining <= 5) + + if (response.Headers.ContainsKey("X-RateLimit-Remaining")) { - _logger.Trace("Waiting 5 seconds to get information for the next 35 movies"); - Thread.Sleep(5000); + var remaining = int.Parse(response.Headers.GetValues("X-RateLimit-Remaining").First()); + if (remaining <= 5) + { + _logger.Trace("Waiting 5 seconds to get information for the next 35 movies"); + Thread.Sleep(5000); + } } var resource = response.Resource; @@ -295,11 +299,14 @@ namespace NzbDrone.Core.MetadataSource.SkyHook // The dude abides, so should us, Lets be nice to TMDb // var allowed = int.Parse(response.Headers.GetValues("X-RateLimit-Limit").First()); // get allowed // var reset = long.Parse(response.Headers.GetValues("X-RateLimit-Reset").First()); // get time when it resets - var remaining = int.Parse(response.Headers.GetValues("X-RateLimit-Remaining").First()); - if (remaining <= 5) + if (response.Headers.ContainsKey("X-RateLimit-Remaining")) { - _logger.Trace("Waiting 5 seconds to get information for the next 35 movies"); - Thread.Sleep(5000); + var remaining = int.Parse(response.Headers.GetValues("X-RateLimit-Remaining").First()); + if (remaining <= 5) + { + _logger.Trace("Waiting 5 seconds to get information for the next 35 movies"); + Thread.Sleep(5000); + } } var resources = response.Resource;