1
0
Fork 0
mirror of https://github.com/lidarr/Lidarr synced 2025-02-25 15:22:42 +00:00

New: Watch filesystem for changes to library

This commit is contained in:
ta264 2020-02-27 21:27:03 +00:00 committed by Qstick
parent 87d29ec978
commit a2ba8e76bb
12 changed files with 406 additions and 17 deletions

View file

@ -256,6 +256,22 @@ class MediaManagement extends Component {
/>
</FormGroup>
<FormGroup
advancedSettings={advancedSettings}
isAdvanced={true}
size={sizes.MEDIUM}
>
<FormLabel>Watch Root Folders for file changes</FormLabel>
<FormInputGroup
type={inputTypes.CHECK}
name="watchLibraryForChanges"
helpText="Rescan automatically when files change in a root folder"
onChange={onInputChange}
{...settings.watchLibraryForChanges}
/>
</FormGroup>
<FormGroup
advancedSettings={advancedSettings}
isAdvanced={true}

View file

@ -14,6 +14,7 @@ public class MediaManagementConfigResource : RestResource
public bool CreateEmptyArtistFolders { get; set; }
public bool DeleteEmptyFolders { get; set; }
public FileDateType FileDate { get; set; }
public bool WatchLibraryForChanges { get; set; }
public RescanAfterRefreshType RescanAfterRefresh { get; set; }
public AllowFingerprinting AllowFingerprinting { get; set; }
@ -43,6 +44,7 @@ public static MediaManagementConfigResource ToResource(IConfigService model)
CreateEmptyArtistFolders = model.CreateEmptyArtistFolders,
DeleteEmptyFolders = model.DeleteEmptyFolders,
FileDate = model.FileDate,
WatchLibraryForChanges = model.WatchLibraryForChanges,
RescanAfterRefresh = model.RescanAfterRefresh,
AllowFingerprinting = model.AllowFingerprinting,

View file

@ -38,6 +38,28 @@ public void should_hold_the_call_for_debounce_duration()
counter.Count.Should().Be(1);
}
[Test]
[Retry(3)]
public void should_wait_for_last_call_if_execute_resets_timer()
{
var counter = new Counter();
var debounceFunction = new Debouncer(counter.Hit, TimeSpan.FromMilliseconds(200), true);
debounceFunction.Execute();
Thread.Sleep(100);
debounceFunction.Execute();
Thread.Sleep(150);
counter.Count.Should().Be(0);
Thread.Sleep(100);
counter.Count.Should().Be(1);
}
[Test]
[Retry(3)]
public void should_throttle_calls()

View file

@ -32,19 +32,23 @@ public static string CleanFilePath(this string path)
Ensure.That(path, () => path).IsValidPath();
var info = new FileInfo(path.Trim());
return info.FullName.CleanFilePathBasic();
}
public static string CleanFilePathBasic(this string path)
{
//UNC
if (OsInfo.IsWindows && info.FullName.StartsWith(@"\\"))
if (OsInfo.IsWindows && path.StartsWith(@"\\"))
{
return info.FullName.TrimEnd('/', '\\', ' ');
return path.TrimEnd('/', '\\', ' ');
}
if (OsInfo.IsNotWindows && info.FullName.TrimEnd('/').Length == 0)
if (OsInfo.IsNotWindows && path.TrimEnd('/').Length == 0)
{
return "/";
}
return info.FullName.TrimEnd('/').Trim('\\', ' ');
return path.TrimEnd('/').Trim('\\', ' ');
}
public static bool PathNotEquals(this string firstPath, string secondPath, StringComparison? comparison = null)

View file

@ -6,15 +6,17 @@ public class Debouncer
{
private readonly Action _action;
private readonly System.Timers.Timer _timer;
private readonly bool _executeRestartsTimer;
private volatile int _paused;
private volatile bool _triggered;
public Debouncer(Action action, TimeSpan debounceDuration)
public Debouncer(Action action, TimeSpan debounceDuration, bool executeRestartsTimer = false)
{
_action = action;
_timer = new System.Timers.Timer(debounceDuration.TotalMilliseconds);
_timer.Elapsed += timer_Elapsed;
_executeRestartsTimer = executeRestartsTimer;
}
private void timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
@ -32,6 +34,11 @@ public void Execute()
lock (_timer)
{
_triggered = true;
if (_executeRestartsTimer)
{
_timer.Stop();
}
if (_paused == 0)
{
_timer.Start();

View file

@ -227,6 +227,13 @@ public string ExtraFileExtensions
set { SetValue("ExtraFileExtensions", value); }
}
public bool WatchLibraryForChanges
{
get { return GetValueBoolean("WatchLibraryForChanges", true); }
set { SetValue("WatchLibraryForChanges", value); }
}
public RescanAfterRefreshType RescanAfterRefresh
{
get { return GetValueEnum("RescanAfterRefresh", RescanAfterRefreshType.Always); }

View file

@ -36,6 +36,7 @@ public interface IConfigService
bool CopyUsingHardlinks { get; set; }
bool ImportExtraFiles { get; set; }
string ExtraFileExtensions { get; set; }
bool WatchLibraryForChanges { get; set; }
RescanAfterRefreshType RescanAfterRefresh { get; set; }
AllowFingerprinting AllowFingerprinting { get; set; }

View file

@ -39,6 +39,7 @@ public class AudioTagService : IAudioTagService,
private readonly IConfigService _configService;
private readonly IMediaFileService _mediaFileService;
private readonly IDiskProvider _diskProvider;
private readonly IRootFolderWatchingService _rootFolderWatchingService;
private readonly IArtistService _artistService;
private readonly IMapCoversToLocal _mediaCoverService;
private readonly IEventAggregator _eventAggregator;
@ -47,6 +48,7 @@ public class AudioTagService : IAudioTagService,
public AudioTagService(IConfigService configService,
IMediaFileService mediaFileService,
IDiskProvider diskProvider,
IRootFolderWatchingService rootFolderWatchingService,
IArtistService artistService,
IMapCoversToLocal mediaCoverService,
IEventAggregator eventAggregator,
@ -55,6 +57,7 @@ public AudioTagService(IConfigService configService,
_configService = configService;
_mediaFileService = mediaFileService;
_diskProvider = diskProvider;
_rootFolderWatchingService = rootFolderWatchingService;
_artistService = artistService;
_mediaCoverService = mediaCoverService;
_eventAggregator = eventAggregator;
@ -186,6 +189,7 @@ public void RemoveMusicBrainzTags(string path)
tags.MusicBrainzAlbumComment = null;
tags.MusicBrainzReleaseTrackId = null;
_rootFolderWatchingService.ReportFileSystemChangeBeginning(path);
tags.Write(path);
}
@ -211,6 +215,8 @@ public void WriteTags(TrackFile trackfile, bool newDownload, bool force = false)
var diff = ReadAudioTag(path).Diff(newTags);
_rootFolderWatchingService.ReportFileSystemChangeBeginning(path);
if (_configService.ScrubAudioTags)
{
_logger.Debug($"Scrubbing tags for {trackfile}");
@ -218,6 +224,7 @@ public void WriteTags(TrackFile trackfile, bool newDownload, bool force = false)
}
_logger.Debug($"Writing tags for {trackfile}");
newTags.Write(path);
UpdateTrackfileSizeAndModified(trackfile, path);

View file

@ -34,6 +34,9 @@ public class DiskScanService :
IDiskScanService,
IExecute<RescanFoldersCommand>
{
public static readonly Regex ExcludedSubFoldersRegex = new Regex(@"(?:\\|\/|^)(?:extras|@eadir|extrafanart|plex versions|\.[^\\/]+)(?:\\|\/)", RegexOptions.Compiled | RegexOptions.IgnoreCase);
public static readonly Regex ExcludedFilesRegex = new Regex(@"^\._|^Thumbs\.db$|^\.DS_store$|\.partial~$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
private readonly IDiskProvider _diskProvider;
private readonly IMediaFileService _mediaFileService;
private readonly IMakeImportDecision _importDecisionMaker;
@ -65,9 +68,6 @@ public DiskScanService(IDiskProvider diskProvider,
_logger = logger;
}
private static readonly Regex ExcludedSubFoldersRegex = new Regex(@"(?:\\|\/|^)(?:extras|@eadir|extrafanart|plex versions|\.[^\\/]+)(?:\\|\/)", RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly Regex ExcludedFilesRegex = new Regex(@"^\._|^Thumbs\.db$|^\.DS_store$|\.partial~$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
public void Scan(List<string> folders = null, FilterFilesType filter = FilterFilesType.Known, List<int> artistIds = null)
{
if (folders == null)

View file

@ -0,0 +1,311 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using NLog;
using NzbDrone.Common.Disk;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.TPL;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Configuration.Events;
using NzbDrone.Core.Datastore.Events;
using NzbDrone.Core.Lifecycle;
using NzbDrone.Core.MediaFiles.Commands;
using NzbDrone.Core.Messaging.Commands;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.RootFolders;
namespace NzbDrone.Core.MediaFiles
{
public interface IRootFolderWatchingService
{
void ReportFileSystemChangeBeginning(params string[] paths);
}
public sealed class RootFolderWatchingService : IRootFolderWatchingService,
IDisposable,
IHandle<ModelEvent<RootFolder>>,
IHandle<ApplicationStartedEvent>,
IHandle<ConfigSavedEvent>
{
private const int DEBOUNCE_TIMEOUT_SECONDS = 30;
private readonly ConcurrentDictionary<string, FileSystemWatcher> _fileSystemWatchers = new ConcurrentDictionary<string, FileSystemWatcher>();
private readonly ConcurrentDictionary<string, int> _tempIgnoredPaths = new ConcurrentDictionary<string, int>();
private readonly ConcurrentDictionary<string, string> _changedPaths = new ConcurrentDictionary<string, string>();
private readonly IRootFolderService _rootFolderService;
private readonly IManageCommandQueue _commandQueueManager;
private readonly IConfigService _configService;
private readonly Logger _logger;
private readonly Debouncer _scanDebouncer;
private bool _watchForChanges;
public RootFolderWatchingService(IRootFolderService rootFolderService,
IManageCommandQueue commandQueueManager,
IConfigService configService,
Logger logger)
{
_rootFolderService = rootFolderService;
_commandQueueManager = commandQueueManager;
_configService = configService;
_logger = logger;
_scanDebouncer = new Debouncer(ScanPending, TimeSpan.FromSeconds(DEBOUNCE_TIMEOUT_SECONDS), true);
}
public void Dispose()
{
foreach (var watcher in _fileSystemWatchers.Values)
{
DisposeWatcher(watcher, false);
}
}
public void ReportFileSystemChangeBeginning(params string[] paths)
{
foreach (var path in paths.Where(x => x.IsNotNullOrWhiteSpace()))
{
_logger.Trace($"reporting start of change to {path}");
_tempIgnoredPaths.AddOrUpdate(path.CleanFilePathBasic(), 1, (key, value) => value + 1);
}
}
public void Handle(ApplicationStartedEvent message)
{
_watchForChanges = _configService.WatchLibraryForChanges;
if (_watchForChanges)
{
_rootFolderService.All().ForEach(x => StartWatchingPath(x.Path));
}
}
public void Handle(ConfigSavedEvent message)
{
var oldWatch = _watchForChanges;
_watchForChanges = _configService.WatchLibraryForChanges;
if (_watchForChanges != oldWatch)
{
if (_watchForChanges)
{
_rootFolderService.All().ForEach(x => StartWatchingPath(x.Path));
}
else
{
_rootFolderService.All().ForEach(x => StopWatchingPath(x.Path));
}
}
}
public void Handle(ModelEvent<RootFolder> message)
{
if (message.Action == ModelAction.Created && _watchForChanges)
{
StartWatchingPath(message.Model.Path);
}
else if (message.Action == ModelAction.Deleted)
{
StopWatchingPath(message.Model.Path);
}
}
private void StartWatchingPath(string path)
{
// Already being watched
if (_fileSystemWatchers.ContainsKey(path))
{
return;
}
// Creating a FileSystemWatcher over the LAN can take hundreds of milliseconds, so wrap it in a Task to do them all in parallel
Task.Run(() =>
{
try
{
var newWatcher = new FileSystemWatcher(path, "*")
{
IncludeSubdirectories = true,
InternalBufferSize = 65536,
NotifyFilter = NotifyFilters.DirectoryName | NotifyFilters.FileName | NotifyFilters.LastWrite
};
newWatcher.Created += Watcher_Changed;
newWatcher.Deleted += Watcher_Changed;
newWatcher.Renamed += Watcher_Changed;
newWatcher.Changed += Watcher_Changed;
newWatcher.Error += Watcher_Error;
if (_fileSystemWatchers.TryAdd(path, newWatcher))
{
newWatcher.EnableRaisingEvents = true;
_logger.Info("Watching directory {0}", path);
}
else
{
DisposeWatcher(newWatcher, false);
}
}
catch (Exception ex)
{
_logger.Error(ex, "Error watching path: {0}", path);
}
});
}
private void StopWatchingPath(string path)
{
if (_fileSystemWatchers.TryGetValue(path, out var watcher))
{
DisposeWatcher(watcher, true);
}
}
private void Watcher_Error(object sender, ErrorEventArgs e)
{
var ex = e.GetException();
var dw = (FileSystemWatcher)sender;
if (ex.GetType() == typeof(InternalBufferOverflowException))
{
_logger.Warn(ex, "The file system watcher experienced an internal buffer overflow for: {0}", dw.Path);
_changedPaths.TryAdd(dw.Path, dw.Path);
_scanDebouncer.Execute();
}
else
{
_logger.Error(ex, "Error in Directory watcher for: {0}" + dw.Path);
DisposeWatcher(dw, true);
}
}
private void Watcher_Changed(object sender, FileSystemEventArgs e)
{
try
{
var rootFolder = ((FileSystemWatcher)sender).Path;
var path = e.FullPath;
if (path.IsNullOrWhiteSpace())
{
throw new ArgumentNullException("path");
}
_changedPaths.TryAdd(path, rootFolder);
_scanDebouncer.Execute();
}
catch (Exception ex)
{
_logger.Error(ex, "Exception in ReportFileSystemChanged. Path: {0}", e.FullPath);
}
}
private void ScanPending()
{
var pairs = _changedPaths.ToArray();
_changedPaths.Clear();
var ignored = _tempIgnoredPaths.Keys.ToArray();
_tempIgnoredPaths.Clear();
var toScan = new HashSet<string>();
foreach (var item in pairs)
{
var path = item.Key.CleanFilePathBasic();
var rootFolder = item.Value;
if (!ShouldIgnoreChange(path, ignored))
{
_logger.Trace("Actioning change to {0}", path);
toScan.Add(rootFolder);
}
else
{
_logger.Trace("Ignoring change to {0}", path);
}
}
if (toScan.Any())
{
_commandQueueManager.Push(new RescanFoldersCommand(toScan.ToList(), FilterFilesType.Known, true, null));
}
}
private bool ShouldIgnoreChange(string cleanPath, string[] ignoredPaths)
{
var cleaned = cleanPath.CleanFilePathBasic();
// Skip partial/backup
if (cleanPath.EndsWith(".partial~") ||
cleanPath.EndsWith(".backup~"))
{
return true;
}
// only proceed for directories and files with music extensions
var extension = Path.GetExtension(cleaned);
if (extension.IsNullOrWhiteSpace() && !Directory.Exists(cleaned))
{
return true;
}
if (extension.IsNotNullOrWhiteSpace() && !MediaFileExtensions.Extensions.Contains(extension))
{
return true;
}
// If the parent of an ignored path has a change event, ignore that too
// Note that we can't afford to use the PathEquals or IsParentPath functions because
// these rely on disk access which is too slow when trying to handle many update events
return ignoredPaths.Any(i => i.Equals(cleaned, DiskProviderBase.PathStringComparison) ||
i.StartsWith(cleaned + Path.DirectorySeparatorChar, DiskProviderBase.PathStringComparison) ||
Path.GetDirectoryName(i).Equals(cleaned, DiskProviderBase.PathStringComparison));
}
private void DisposeWatcher(FileSystemWatcher watcher, bool removeFromList)
{
try
{
using (watcher)
{
_logger.Info("Stopping directory watching for path {0}", watcher.Path);
watcher.Created -= Watcher_Changed;
watcher.Deleted -= Watcher_Changed;
watcher.Renamed -= Watcher_Changed;
watcher.Changed -= Watcher_Changed;
watcher.Error -= Watcher_Error;
try
{
watcher.EnableRaisingEvents = false;
}
catch (InvalidOperationException)
{
// Seeing this under mono on linux sometimes
// Collection was modified; enumeration operation may not execute.
}
}
}
catch
{
// we don't care about exceptions disposing
}
finally
{
if (removeFromList)
{
_fileSystemWatchers.TryRemove(watcher.Path, out _);
}
}
}
}
}

View file

@ -30,21 +30,23 @@ public class TrackFileMovingService : IMoveTrackFiles
private readonly IBuildFileNames _buildFileNames;
private readonly IDiskTransferService _diskTransferService;
private readonly IDiskProvider _diskProvider;
private readonly IRootFolderWatchingService _rootFolderWatchingService;
private readonly IMediaFileAttributeService _mediaFileAttributeService;
private readonly IEventAggregator _eventAggregator;
private readonly IConfigService _configService;
private readonly Logger _logger;
public TrackFileMovingService(ITrackService trackService,
IAlbumService albumService,
IUpdateTrackFileService updateTrackFileService,
IBuildFileNames buildFileNames,
IDiskTransferService diskTransferService,
IDiskProvider diskProvider,
IMediaFileAttributeService mediaFileAttributeService,
IEventAggregator eventAggregator,
IConfigService configService,
Logger logger)
IAlbumService albumService,
IUpdateTrackFileService updateTrackFileService,
IBuildFileNames buildFileNames,
IDiskTransferService diskTransferService,
IDiskProvider diskProvider,
IRootFolderWatchingService rootFolderWatchingService,
IMediaFileAttributeService mediaFileAttributeService,
IEventAggregator eventAggregator,
IConfigService configService,
Logger logger)
{
_trackService = trackService;
_albumService = albumService;
@ -52,6 +54,7 @@ public TrackFileMovingService(ITrackService trackService,
_buildFileNames = buildFileNames;
_diskTransferService = diskTransferService;
_diskProvider = diskProvider;
_rootFolderWatchingService = rootFolderWatchingService;
_mediaFileAttributeService = mediaFileAttributeService;
_eventAggregator = eventAggregator;
_configService = configService;
@ -119,6 +122,7 @@ private TrackFile TransferFile(TrackFile trackFile, Artist artist, List<Track> t
throw new SameFilenameException("File not moved, source and destination are the same", trackFilePath);
}
_rootFolderWatchingService.ReportFileSystemChangeBeginning(trackFilePath, destinationFilePath);
_diskTransferService.TransferFile(trackFilePath, destinationFilePath, mode);
trackFile.Path = destinationFilePath;
@ -166,6 +170,8 @@ private void EnsureTrackFolder(TrackFile trackFile, Artist artist, Album album,
var changed = false;
var newEvent = new TrackFolderCreatedEvent(artist, trackFile);
_rootFolderWatchingService.ReportFileSystemChangeBeginning(artistFolder, albumFolder, trackFolder);
if (!_diskProvider.FolderExists(artistFolder))
{
CreateFolder(artistFolder);

View file

@ -2,6 +2,7 @@
using NLog;
using NzbDrone.Common.Disk;
using NzbDrone.Common.Instrumentation.Extensions;
using NzbDrone.Core.MediaFiles;
using NzbDrone.Core.Messaging.Commands;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.Music.Commands;
@ -15,6 +16,7 @@ public class MoveArtistService : IExecute<MoveArtistCommand>, IExecute<BulkMoveA
private readonly IArtistService _artistService;
private readonly IBuildFileNames _filenameBuilder;
private readonly IDiskProvider _diskProvider;
private readonly IRootFolderWatchingService _rootFolderWatchingService;
private readonly IDiskTransferService _diskTransferService;
private readonly IEventAggregator _eventAggregator;
private readonly Logger _logger;
@ -22,6 +24,7 @@ public class MoveArtistService : IExecute<MoveArtistCommand>, IExecute<BulkMoveA
public MoveArtistService(IArtistService artistService,
IBuildFileNames filenameBuilder,
IDiskProvider diskProvider,
IRootFolderWatchingService rootFolderWatchingService,
IDiskTransferService diskTransferService,
IEventAggregator eventAggregator,
Logger logger)
@ -29,6 +32,7 @@ public MoveArtistService(IArtistService artistService,
_artistService = artistService;
_filenameBuilder = filenameBuilder;
_diskProvider = diskProvider;
_rootFolderWatchingService = rootFolderWatchingService;
_diskTransferService = diskTransferService;
_eventAggregator = eventAggregator;
_logger = logger;
@ -55,6 +59,8 @@ private void MoveSingleArtist(Artist artist, string sourcePath, string destinati
{
if (moveFiles)
{
_rootFolderWatchingService.ReportFileSystemChangeBeginning(sourcePath, destinationPath);
_diskTransferService.TransferFolder(sourcePath, destinationPath, TransferMode.Move);
_logger.ProgressInfo("{0} moved successfully to {1}", artist.Name, artist.Path);