New: Fast copy using reflink on btrfs volumes

This commit is contained in:
Taloth Saldono 2020-06-10 20:48:54 +02:00
parent 0e7b404121
commit 2c286f7b60
7 changed files with 231 additions and 1 deletions

View File

@ -199,6 +199,24 @@ namespace NzbDrone.Common.Disk
File.Delete(path);
}
public void CloneFile(string source, string destination, bool overwrite = false)
{
Ensure.That(source, () => source).IsValidPath();
Ensure.That(destination, () => destination).IsValidPath();
if (source.PathEquals(destination))
{
throw new IOException(string.Format("Source and destination can't be the same {0}", source));
}
CloneFileInternal(source, destination, overwrite);
}
protected virtual void CloneFileInternal(string source, string destination, bool overwrite = false)
{
CopyFileInternal(source, destination, overwrite);
}
public void CopyFile(string source, string destination, bool overwrite = false)
{
Ensure.That(source, () => source).IsValidPath();
@ -249,8 +267,18 @@ namespace NzbDrone.Common.Disk
File.Move(source, destination);
}
public virtual bool TryRenameFile(string source, string destination)
{
return false;
}
public abstract bool TryCreateHardLink(string source, string destination);
public virtual bool TryCreateRefLink(string source, string destination)
{
return false;
}
public void DeleteFolder(string path, bool recursive)
{
Ensure.That(path, () => path).IsValidPath();

View File

@ -265,18 +265,45 @@ namespace NzbDrone.Common.Disk
var targetDriveFormat = targetMount?.DriveFormat ?? string.Empty;
var isCifs = targetDriveFormat == "cifs";
var isBtrfs = sourceDriveFormat == "btrfs" && targetDriveFormat == "btrfs";
if (mode.HasFlag(TransferMode.Copy))
{
if (isBtrfs)
{
if (_diskProvider.TryCreateRefLink(sourcePath, targetPath))
{
return TransferMode.Copy;
}
}
TryCopyFileVerified(sourcePath, targetPath, originalSize);
return TransferMode.Copy;
}
if (mode.HasFlag(TransferMode.Move))
{
if (isBtrfs)
{
if (isSameMount && _diskProvider.TryRenameFile(sourcePath, targetPath))
{
_logger.Trace("Renamed [{0}] to [{1}].", sourcePath, targetPath);
return TransferMode.Move;
}
if (_diskProvider.TryCreateRefLink(sourcePath, targetPath))
{
_logger.Trace("Reflink successful, deleting source [{0}].", sourcePath);
_diskProvider.DeleteFile(sourcePath);
return TransferMode.Move;
}
}
if (isCifs && !isSameMount)
{
_logger.Trace("On cifs mount. Starting verified copy [{0}] to [{1}].", sourcePath, targetPath);
TryCopyFileVerified(sourcePath, targetPath, originalSize);
_logger.Trace("Copy successful, deleting source [{0}].", sourcePath);
_diskProvider.DeleteFile(sourcePath);
return TransferMode.Move;
}

View File

@ -29,10 +29,13 @@ namespace NzbDrone.Common.Disk
long GetFileSize(string path);
void CreateFolder(string path);
void DeleteFile(string path);
void CloneFile(string source, string destination, bool overwrite = false);
void CopyFile(string source, string destination, bool overwrite = false);
void MoveFile(string source, string destination, bool overwrite = false);
void MoveFolder(string source, string destination);
bool TryRenameFile(string source, string destination);
bool TryCreateHardLink(string source, string destination);
bool TryCreateRefLink(string source, string destination);
void DeleteFolder(string path, bool recursive);
string ReadAllText(string filePath);
void WriteAllText(string filename, string contents);

View File

@ -20,15 +20,17 @@ namespace NzbDrone.Mono.Disk
private readonly Logger _logger;
private readonly IProcMountProvider _procMountProvider;
private readonly ISymbolicLinkResolver _symLinkResolver;
private readonly ICreateRefLink _createRefLink;
// Mono supports sending -1 for a uint to indicate that the owner or group should not be set
// `unchecked((uint)-1)` and `uint.MaxValue` are the same thing.
private const uint UNCHANGED_ID = uint.MaxValue;
public DiskProvider(IProcMountProvider procMountProvider, ISymbolicLinkResolver symLinkResolver, Logger logger)
public DiskProvider(IProcMountProvider procMountProvider, ISymbolicLinkResolver symLinkResolver, ICreateRefLink createRefLink, Logger logger)
{
_procMountProvider = procMountProvider;
_symLinkResolver = symLinkResolver;
_createRefLink = createRefLink;
_logger = logger;
}
@ -177,6 +179,19 @@ namespace NzbDrone.Mono.Disk
return mount?.TotalSize;
}
protected override void CloneFileInternal(string source, string destination, bool overwrite)
{
if (!File.Exists(destination) && !UnixFileSystemInfo.GetFileSystemEntry(source).IsSymbolicLink)
{
if (_createRefLink.TryCreateRefLink(source, destination))
{
return;
}
}
CopyFileInternal(source, destination, overwrite);
}
protected override void CopyFileInternal(string source, string destination, bool overwrite)
{
var sourceInfo = UnixFileSystemInfo.GetFileSystemEntry(source);
@ -379,6 +394,11 @@ namespace NzbDrone.Mono.Disk
}
}
public override bool TryRenameFile(string source, string destination)
{
return Syscall.rename(source, destination) == 0;
}
public override bool TryCreateHardLink(string source, string destination)
{
try
@ -397,6 +417,11 @@ namespace NzbDrone.Mono.Disk
}
}
public override bool TryCreateRefLink(string source, string destination)
{
return _createRefLink.TryCreateRefLink(source, destination);
}
private uint GetUserId(string user)
{
if (user.IsNullOrWhiteSpace())

View File

@ -0,0 +1,79 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using NLog;
using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Mono.Interop;
using Mono.Unix.Native;
using Mono.Unix;
namespace NzbDrone.Mono.Disk
{
public interface ICreateRefLink
{
bool TryCreateRefLink(string srcPath, string linkPath);
}
public class RefLinkCreator : ICreateRefLink
{
private readonly Logger _logger;
private readonly bool _supported;
public RefLinkCreator(Logger logger)
{
_logger = logger;
// Only support x86_64 because we know the FICLONE value is valid for it
_supported = OsInfo.IsLinux && (Syscall.uname(out var results) == 0 && results.machine == "x86_64");
}
public bool TryCreateRefLink(string srcPath, string linkPath)
{
if (!_supported)
{
return false;
}
try
{
using (var srcHandle = NativeMethods.open(srcPath, OpenFlags.O_RDONLY))
{
if (srcHandle.IsInvalid)
{
_logger.Trace("Failed to create reflink at '{0}' to '{1}': Couldn't open source file", linkPath, srcPath);
return false;
}
using (var linkHandle = NativeMethods.open(linkPath, OpenFlags.O_WRONLY | OpenFlags.O_CREAT | OpenFlags.O_TRUNC))
{
if (linkHandle.IsInvalid)
{
_logger.Trace("Failed to create reflink at '{0}' to '{1}': Couldn't create new link file", linkPath, srcPath);
return false;
}
if (NativeMethods.clone_file(linkHandle, srcHandle) == -1)
{
var error = new UnixIOException();
linkHandle.Dispose();
Syscall.unlink(linkPath);
_logger.Trace("Failed to create reflink at '{0}' to '{1}': {2}", linkPath, srcPath, error.Message);
return false;
}
_logger.Trace("Created reflink at '{0}' to '{1}'", linkPath, srcPath);
return true;
}
}
}
catch (Exception ex)
{
Syscall.unlink(linkPath);
_logger.Trace(ex, "Failed to create reflink at '{0}' to '{1}'", linkPath, srcPath);
return false;
}
}
}
}

View File

@ -0,0 +1,31 @@
using System;
using System.Runtime.ConstrainedExecution;
using System.Runtime.InteropServices;
using System.Text;
using Mono.Unix.Native;
namespace NzbDrone.Mono.Interop
{
internal enum IoctlRequest : uint
{
// Hardcoded ioctl for FICLONE on a typical linux system
// #define FICLONE _IOW(0x94, 9, int)
FICLONE = 0x40049409
}
internal static class NativeMethods
{
[DllImport("libc", EntryPoint = "ioctl", SetLastError = true)]
private static extern int Ioctl(SafeUnixHandle dst_fd, IoctlRequest request, SafeUnixHandle src_fd);
public static SafeUnixHandle open(string pathname, OpenFlags flags)
{
return new SafeUnixHandle(Syscall.open(pathname, flags));
}
internal static int clone_file(SafeUnixHandle link_fd, SafeUnixHandle src_fd)
{
return Ioctl(link_fd, IoctlRequest.FICLONE, src_fd);
}
}
}

View File

@ -0,0 +1,37 @@
using System;
using System.Runtime.ConstrainedExecution;
using System.Runtime.InteropServices;
using System.Security.Permissions;
using Mono.Unix.Native;
namespace NzbDrone.Mono.Interop
{
[SecurityPermission(SecurityAction.InheritanceDemand, UnmanagedCode = true)]
[SecurityPermission(SecurityAction.Demand, UnmanagedCode = true)]
internal sealed class SafeUnixHandle : SafeHandle
{
[ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)]
private SafeUnixHandle()
: base(new IntPtr(-1), true)
{
}
[ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)]
public SafeUnixHandle(int fd)
: base(new IntPtr(-1), true)
{
handle = new IntPtr(fd);
}
public override bool IsInvalid
{
get { return this.handle == new IntPtr(-1); }
}
[ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)]
protected override bool ReleaseHandle()
{
return Syscall.close(this.handle.ToInt32()) != -1;
}
}
}