From 1c1549820b3cce3d30db1f94962fc5905fd95518 Mon Sep 17 00:00:00 2001 From: Qstick Date: Sat, 28 Oct 2017 23:36:09 -0400 Subject: [PATCH] Fixed: Ignore '.unwanted' directory when importing. --- .../Blackhole/ScanWatchFolderFixture.cs | 13 +++++--- .../Blackhole/TorrentBlackholeFixture.cs | 5 ++++ .../Blackhole/UsenetBlackholeFixture.cs | 5 ++++ .../DownloadedTracksImportServiceFixture.cs | 6 +++- .../GetAudioFilesFixture.cs | 11 +++---- .../Clients/Blackhole/ScanWatchFolder.cs | 30 +++++++++---------- .../Extras/ExistingExtraFileService.cs | 2 +- .../MediaFiles/DiskScanService.cs | 13 ++++---- .../DownloadedTracksImportService.cs | 2 +- .../TrackImport/Manual/ManualImportService.cs | 2 +- 10 files changed, 52 insertions(+), 37 deletions(-) diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/ScanWatchFolderFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/ScanWatchFolderFixture.cs index 40aada01a..a63bfba6a 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/ScanWatchFolderFixture.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/ScanWatchFolderFixture.cs @@ -1,15 +1,17 @@ -using System; +using System; +using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; using FluentAssertions; using Moq; using NUnit.Framework; using NzbDrone.Common.Disk; using NzbDrone.Core.Download; -using NzbDrone.Test.Common; -using System.Threading; -using NzbDrone.Core.Test.Framework; using NzbDrone.Core.Download.Clients.Blackhole; +using NzbDrone.Core.MediaFiles; +using NzbDrone.Core.Test.Framework; +using NzbDrone.Test.Common; namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole { @@ -33,6 +35,9 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole Mocker.GetMock() .Setup(c => c.GetFileSize(It.IsAny())) .Returns(1000000); + + Mocker.GetMock().Setup(c => c.FilterFiles(It.IsAny(), It.IsAny>())) + .Returns>((b, s) => s.ToList()); } protected void GivenChangedItem() diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/TorrentBlackholeFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/TorrentBlackholeFixture.cs index f9a8f908f..d43459362 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/TorrentBlackholeFixture.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/TorrentBlackholeFixture.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; @@ -10,6 +11,7 @@ using NzbDrone.Common.Http; using NzbDrone.Core.Download; using NzbDrone.Core.Download.Clients.Blackhole; using NzbDrone.Core.Exceptions; +using NzbDrone.Core.MediaFiles; using NzbDrone.Core.MediaFiles.TorrentInfo; using NzbDrone.Core.Parser.Model; using NzbDrone.Test.Common; @@ -48,6 +50,9 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole Mocker.GetMock() .Setup(c => c.GetHashFromTorrentFile(It.IsAny())) .Returns("myhash"); + + Mocker.GetMock().Setup(c => c.FilterFiles(It.IsAny(), It.IsAny>())) + .Returns>((b, s) => s.ToList()); } protected void GivenFailedDownload() diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/UsenetBlackholeFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/UsenetBlackholeFixture.cs index c490027f2..938643ad7 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/UsenetBlackholeFixture.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/UsenetBlackholeFixture.cs @@ -1,5 +1,6 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; @@ -10,6 +11,7 @@ using NzbDrone.Common.Disk; using NzbDrone.Common.Http; using NzbDrone.Core.Download; using NzbDrone.Core.Download.Clients.Blackhole; +using NzbDrone.Core.MediaFiles; using NzbDrone.Test.Common; namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole @@ -41,6 +43,9 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole Mocker.GetMock() .Setup(c => c.OpenWriteStream(It.IsAny())) .Returns(() => new FileStream(GetTempFilePath(), FileMode.Create)); + + Mocker.GetMock().Setup(c => c.FilterFiles(It.IsAny(), It.IsAny>())) + .Returns>((b, s) => s.ToList()); } protected void GivenFailedDownload() diff --git a/src/NzbDrone.Core.Test/MediaFiles/DownloadedTracksImportServiceFixture.cs b/src/NzbDrone.Core.Test/MediaFiles/DownloadedTracksImportServiceFixture.cs index 5bfe5230e..876bf9b2a 100644 --- a/src/NzbDrone.Core.Test/MediaFiles/DownloadedTracksImportServiceFixture.cs +++ b/src/NzbDrone.Core.Test/MediaFiles/DownloadedTracksImportServiceFixture.cs @@ -2,9 +2,11 @@ using System.Collections.Generic; using System.IO; using System.Linq; using FizzWare.NBuilder; +using FluentAssertions; using Moq; using NUnit.Framework; using NzbDrone.Common.Disk; +using NzbDrone.Core.Download; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.MediaFiles.TrackImport; using NzbDrone.Core.Parser; @@ -13,7 +15,6 @@ using NzbDrone.Core.Qualities; using NzbDrone.Core.Test.Framework; using NzbDrone.Core.Music; using NzbDrone.Test.Common; -using FluentAssertions; namespace NzbDrone.Core.Test.MediaFiles { @@ -30,6 +31,9 @@ namespace NzbDrone.Core.Test.MediaFiles Mocker.GetMock().Setup(c => c.GetAudioFiles(It.IsAny(), It.IsAny())) .Returns(_videoFiles); + Mocker.GetMock().Setup(c => c.FilterFiles(It.IsAny(), It.IsAny>())) + .Returns>((b, s) => s.ToList()); + Mocker.GetMock().Setup(c => c.GetDirectories(It.IsAny())) .Returns(_subFolders); diff --git a/src/NzbDrone.Core.Test/ProviderTests/DiskScanProviderTests/GetAudioFilesFixture.cs b/src/NzbDrone.Core.Test/ProviderTests/DiskScanProviderTests/GetAudioFilesFixture.cs index 6a1aa73c1..4247b5b4d 100644 --- a/src/NzbDrone.Core.Test/ProviderTests/DiskScanProviderTests/GetAudioFilesFixture.cs +++ b/src/NzbDrone.Core.Test/ProviderTests/DiskScanProviderTests/GetAudioFilesFixture.cs @@ -1,4 +1,4 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.IO; using System.Linq; using FizzWare.NBuilder; @@ -93,6 +93,7 @@ namespace NzbDrone.Core.Test.ProviderTests.DiskScanProviderTests [TestCase("Plex Versions")] [TestCase(".secret")] [TestCase(".hidden")] + [TestCase(".unwanted")] public void should_filter_certain_sub_folders(string subFolder) { var path = @"C:\Test\"; @@ -100,13 +101,9 @@ namespace NzbDrone.Core.Test.ProviderTests.DiskScanProviderTests var specialFiles = GetFiles(path, subFolder).ToList(); var allFiles = files.Concat(specialFiles); - var artist = Builder.CreateNew() - .With(s => s.Path = path) - .Build(); - - var filteredFiles = Subject.FilterFiles(artist, allFiles); + var filteredFiles = Subject.FilterFiles(path, allFiles); filteredFiles.Should().NotContain(specialFiles); filteredFiles.Count.Should().BeGreaterThan(0); } } -} \ No newline at end of file +} diff --git a/src/NzbDrone.Core/Download/Clients/Blackhole/ScanWatchFolder.cs b/src/NzbDrone.Core/Download/Clients/Blackhole/ScanWatchFolder.cs index be868d51c..924436600 100644 --- a/src/NzbDrone.Core/Download/Clients/Blackhole/ScanWatchFolder.cs +++ b/src/NzbDrone.Core/Download/Clients/Blackhole/ScanWatchFolder.cs @@ -1,15 +1,15 @@ -using NLog; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using NLog; using NzbDrone.Common.Cache; using NzbDrone.Common.Crypto; using NzbDrone.Common.Disk; using NzbDrone.Common.Extensions; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.Organizer; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; namespace NzbDrone.Core.Download.Clients.Blackhole { @@ -50,7 +50,7 @@ namespace NzbDrone.Core.Download.Clients.Blackhole private IEnumerable GetDownloadItems(string watchFolder, Dictionary lastWatchItems, TimeSpan waitPeriod) { - foreach (var folder in _diskProvider.GetDirectories(watchFolder)) + foreach (var folder in _diskScanService.FilterFiles(watchFolder, _diskProvider.GetDirectories(watchFolder))) { var title = FileNameBuilder.CleanFileName(Path.GetFileName(folder)); @@ -86,16 +86,16 @@ namespace NzbDrone.Core.Download.Clients.Blackhole yield return newWatchItem; } - foreach (var videoFile in _diskScanService.GetAudioFiles(watchFolder, false)) + foreach (var audioFile in _diskScanService.FilterFiles(watchFolder, _diskScanService.GetAudioFiles(watchFolder, false))) { - var title = FileNameBuilder.CleanFileName(Path.GetFileName(videoFile)); + var title = FileNameBuilder.CleanFileName(Path.GetFileName(audioFile)); var newWatchItem = new WatchFolderItem { - DownloadId = Path.GetFileName(videoFile) + "_" + _diskProvider.FileGetLastWrite(videoFile).Ticks, + DownloadId = Path.GetFileName(audioFile) + "_" + _diskProvider.FileGetLastWrite(audioFile).Ticks, Title = title, - OutputPath = new OsPath(videoFile), + OutputPath = new OsPath(audioFile), Status = DownloadItemStatus.Completed, RemainingTime = TimeSpan.Zero @@ -105,10 +105,10 @@ namespace NzbDrone.Core.Download.Clients.Blackhole if (PreCheckWatchItemExpiry(newWatchItem, oldWatchItem)) { - newWatchItem.TotalSize = _diskProvider.GetFileSize(videoFile); - newWatchItem.Hash = GetHash(videoFile); + newWatchItem.TotalSize = _diskProvider.GetFileSize(audioFile); + newWatchItem.Hash = GetHash(audioFile); - if (_diskProvider.IsFileLocked(videoFile)) + if (_diskProvider.IsFileLocked(audioFile)) { newWatchItem.Status = DownloadItemStatus.Downloading; } @@ -193,4 +193,4 @@ namespace NzbDrone.Core.Download.Clients.Blackhole return HashConverter.GetHash(data.ToString()).ToHexString(); } } -} \ No newline at end of file +} diff --git a/src/NzbDrone.Core/Extras/ExistingExtraFileService.cs b/src/NzbDrone.Core/Extras/ExistingExtraFileService.cs index b2fb06980..14dfb622a 100644 --- a/src/NzbDrone.Core/Extras/ExistingExtraFileService.cs +++ b/src/NzbDrone.Core/Extras/ExistingExtraFileService.cs @@ -41,7 +41,7 @@ namespace NzbDrone.Core.Extras _logger.Debug("Looking for existing extra files in {0}", artist.Path); var filesOnDisk = _diskScanService.GetNonAudioFiles(artist.Path); - var possibleExtraFiles = _diskScanService.FilterFiles(artist, filesOnDisk); + var possibleExtraFiles = _diskScanService.FilterFiles(artist.Path, filesOnDisk); var filteredFiles = possibleExtraFiles; var importedFiles = new List(); diff --git a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs index bb7d40143..fdb01eb2d 100644 --- a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs +++ b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs @@ -24,7 +24,7 @@ namespace NzbDrone.Core.MediaFiles void Scan(Artist artist); string[] GetAudioFiles(string path, bool allDirectories = true); string[] GetNonAudioFiles(string path, bool allDirectories = true); - List FilterFiles(Artist artist, IEnumerable files); + List FilterFiles(string basePath, IEnumerable files); } public class DiskScanService : @@ -59,9 +59,8 @@ namespace NzbDrone.Core.MediaFiles _eventAggregator = eventAggregator; _logger = logger; } - - private static readonly Regex ExcludedSubFoldersRegex = new Regex(@"(?:\\|\/|^)(extras|@eadir|extrafanart|plex\sversions|\..+)(?:\\|\/)", RegexOptions.Compiled | RegexOptions.IgnoreCase); - private static readonly Regex ExcludedFilesRegex = new Regex(@"^\._|Thumbs\.db", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex ExcludedSubFoldersRegex = new Regex(@"(?:\\|\/|^)(?:extras|@eadir|extrafanart|plex versions|\.[^\\/]+)(?:\\|\/|$)", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex ExcludedFilesRegex = new Regex(@"^\._|^Thumbs\.db$", RegexOptions.Compiled | RegexOptions.IgnoreCase); public void Scan(Artist artist) { @@ -101,7 +100,7 @@ namespace NzbDrone.Core.MediaFiles } var musicFilesStopwatch = Stopwatch.StartNew(); - var mediaFileList = FilterFiles(artist, GetAudioFiles(artist.Path)).ToList(); + var mediaFileList = FilterFiles(artist.Path, GetAudioFiles(artist.Path)).ToList(); musicFilesStopwatch.Stop(); _logger.Trace("Finished getting track files for: {0} [{1}]", artist, musicFilesStopwatch.Elapsed); @@ -158,9 +157,9 @@ namespace NzbDrone.Core.MediaFiles return mediaFileList.ToArray(); } - public List FilterFiles(Artist artist, IEnumerable files) + public List FilterFiles(string basePath, IEnumerable files) { - return files.Where(file => !ExcludedSubFoldersRegex.IsMatch(artist.Path.GetRelativePath(file))) + return files.Where(file => !ExcludedSubFoldersRegex.IsMatch(basePath.GetRelativePath(file))) .Where(file => !ExcludedFilesRegex.IsMatch(Path.GetFileName(file))) .ToList(); } diff --git a/src/NzbDrone.Core/MediaFiles/DownloadedTracksImportService.cs b/src/NzbDrone.Core/MediaFiles/DownloadedTracksImportService.cs index dea14b31c..326019885 100644 --- a/src/NzbDrone.Core/MediaFiles/DownloadedTracksImportService.cs +++ b/src/NzbDrone.Core/MediaFiles/DownloadedTracksImportService.cs @@ -176,7 +176,7 @@ namespace NzbDrone.Core.MediaFiles }; } - var audioFiles = _diskScanService.GetAudioFiles(directoryInfo.FullName); + var audioFiles = _diskScanService.FilterFiles(directoryInfo.FullName, _diskScanService.GetAudioFiles(directoryInfo.FullName)); if (downloadClientItem == null) { diff --git a/src/NzbDrone.Core/MediaFiles/TrackImport/Manual/ManualImportService.cs b/src/NzbDrone.Core/MediaFiles/TrackImport/Manual/ManualImportService.cs index 87a8ba449..6a32984b9 100644 --- a/src/NzbDrone.Core/MediaFiles/TrackImport/Manual/ManualImportService.cs +++ b/src/NzbDrone.Core/MediaFiles/TrackImport/Manual/ManualImportService.cs @@ -108,7 +108,7 @@ namespace NzbDrone.Core.MediaFiles.TrackImport.Manual if (artist == null) { - var files = _diskScanService.GetAudioFiles(folder); + var files = _diskScanService.FilterFiles(folder, _diskScanService.GetAudioFiles(folder)); return files.Select(file => ProcessFile(file, downloadId, folder)).Where(i => i != null).ToList(); }