2017-10-06 21:10:24 +00:00
using System ;
2014-11-15 00:42:22 +00:00
using System.IO ;
using System.Linq ;
using System.Threading ;
using NLog ;
using NzbDrone.Common.EnsureThat ;
using NzbDrone.Common.Extensions ;
namespace NzbDrone.Common.Disk
{
public interface IDiskTransferService
{
2020-06-11 21:54:28 +00:00
TransferMode TransferFolder ( string sourcePath , string targetPath , TransferMode mode ) ;
TransferMode TransferFile ( string sourcePath , string targetPath , TransferMode mode , bool overwrite = false ) ;
2016-09-08 23:05:27 +00:00
int MirrorFolder ( string sourcePath , string targetPath ) ;
2014-11-15 00:42:22 +00:00
}
2020-06-11 21:54:28 +00:00
2014-11-15 00:42:22 +00:00
public class DiskTransferService : IDiskTransferService
{
private readonly IDiskProvider _diskProvider ;
private readonly Logger _logger ;
public DiskTransferService ( IDiskProvider diskProvider , Logger logger )
{
_diskProvider = diskProvider ;
_logger = logger ;
}
2020-09-24 09:53:20 +00:00
private string ResolveRealParentPath ( string path )
{
var parentPath = path . GetParentPath ( ) ;
2020-10-11 17:26:33 +00:00
if ( ! _diskProvider . FolderExists ( parentPath ) )
2020-09-24 09:53:20 +00:00
{
return path ;
}
2020-10-11 17:26:33 +00:00
var realParentPath = parentPath . GetActualCasing ( ) ;
var partialChildPath = path . Substring ( parentPath . Length ) ;
return realParentPath + partialChildPath ;
2020-09-24 09:53:20 +00:00
}
2020-06-11 21:54:28 +00:00
public TransferMode TransferFolder ( string sourcePath , string targetPath , TransferMode mode )
2014-11-15 00:42:22 +00:00
{
Ensure . That ( sourcePath , ( ) = > sourcePath ) . IsValidPath ( ) ;
Ensure . That ( targetPath , ( ) = > targetPath ) . IsValidPath ( ) ;
2020-09-24 09:53:20 +00:00
sourcePath = ResolveRealParentPath ( sourcePath ) ;
targetPath = ResolveRealParentPath ( targetPath ) ;
_logger . Debug ( "{0} Directory [{1}] > [{2}]" , mode , sourcePath , targetPath ) ;
if ( sourcePath = = targetPath )
{
throw new IOException ( string . Format ( "Source and destination can't be the same {0}" , sourcePath ) ) ;
}
if ( mode = = TransferMode . Move & & sourcePath . PathEquals ( targetPath , StringComparison . InvariantCultureIgnoreCase ) & & _diskProvider . FolderExists ( targetPath ) )
{
// Move folder out of the way to allow case-insensitive renames
var tempPath = sourcePath + ".backup~" ;
_logger . Trace ( "Rename Intermediate Directory [{0}] > [{1}]" , sourcePath , tempPath ) ;
_diskProvider . MoveFolder ( sourcePath , tempPath ) ;
if ( ! _diskProvider . FolderExists ( targetPath ) )
{
_logger . Trace ( "Rename Intermediate Directory [{0}] > [{1}]" , tempPath , targetPath ) ;
_logger . Debug ( "Rename Directory [{0}] > [{1}]" , sourcePath , targetPath ) ;
_diskProvider . MoveFolder ( tempPath , targetPath ) ;
return mode ;
}
// There were two separate folders, revert the intermediate rename and let the recursion deal with it
_logger . Trace ( "Rename Intermediate Directory [{0}] > [{1}]" , tempPath , sourcePath ) ;
_diskProvider . MoveFolder ( tempPath , sourcePath ) ;
}
2019-02-07 11:55:15 +00:00
if ( mode = = TransferMode . Move & & ! _diskProvider . FolderExists ( targetPath ) )
2019-01-21 21:18:37 +00:00
{
2020-06-11 21:54:28 +00:00
var sourceMount = _diskProvider . GetMount ( sourcePath ) ;
var targetMount = _diskProvider . GetMount ( targetPath ) ;
2019-01-21 21:18:37 +00:00
2020-06-11 21:54:28 +00:00
// If we're on the same mount, do a simple folder move.
if ( sourceMount ! = null & & targetMount ! = null & & sourceMount . RootDirectory = = targetMount . RootDirectory )
{
2020-09-24 09:53:20 +00:00
_logger . Debug ( "Rename Directory [{0}] > [{1}]" , sourcePath , targetPath ) ;
2020-06-11 21:54:28 +00:00
_diskProvider . MoveFolder ( sourcePath , targetPath ) ;
return mode ;
2019-01-21 21:18:37 +00:00
}
}
2014-11-15 00:42:22 +00:00
if ( ! _diskProvider . FolderExists ( targetPath ) )
{
_diskProvider . CreateFolder ( targetPath ) ;
2019-09-07 10:12:22 +00:00
_diskProvider . CopyPermissions ( sourcePath , targetPath ) ;
2014-11-15 00:42:22 +00:00
}
var result = mode ;
foreach ( var subDir in _diskProvider . GetDirectoryInfos ( sourcePath ) )
{
2017-02-09 18:32:13 +00:00
if ( ShouldIgnore ( subDir ) ) continue ;
2020-06-11 21:54:28 +00:00
result & = TransferFolder ( subDir . FullName , Path . Combine ( targetPath , subDir . Name ) , mode ) ;
2014-11-15 00:42:22 +00:00
}
foreach ( var sourceFile in _diskProvider . GetFileInfos ( sourcePath ) )
{
2017-02-09 18:32:13 +00:00
if ( ShouldIgnore ( sourceFile ) ) continue ;
2014-11-15 00:42:22 +00:00
var destFile = Path . Combine ( targetPath , sourceFile . Name ) ;
2020-06-11 21:54:28 +00:00
result & = TransferFile ( sourceFile . FullName , destFile , mode , true ) ;
2014-11-15 00:42:22 +00:00
}
if ( mode . HasFlag ( TransferMode . Move ) )
{
2020-09-24 09:53:20 +00:00
var totalSize = _diskProvider . GetFileInfos ( sourcePath ) . Sum ( v = > v . Length ) ;
if ( totalSize > ( 100 * 1024L * 1024L ) )
{
throw new IOException ( $"Large files still exist in {sourcePath} after folder move, not deleting source folder" ) ;
}
2014-11-15 00:42:22 +00:00
_diskProvider . DeleteFolder ( sourcePath , true ) ;
}
return result ;
}
2016-09-08 23:05:27 +00:00
public int MirrorFolder ( string sourcePath , string targetPath )
{
var filesCopied = 0 ;
Ensure . That ( sourcePath , ( ) = > sourcePath ) . IsValidPath ( ) ;
Ensure . That ( targetPath , ( ) = > targetPath ) . IsValidPath ( ) ;
2020-09-24 09:53:20 +00:00
sourcePath = ResolveRealParentPath ( sourcePath ) ;
targetPath = ResolveRealParentPath ( targetPath ) ;
_logger . Debug ( "Mirror Folder [{0}] > [{1}]" , sourcePath , targetPath ) ;
2016-09-08 23:05:27 +00:00
if ( ! _diskProvider . FolderExists ( targetPath ) )
{
_diskProvider . CreateFolder ( targetPath ) ;
}
var sourceFolders = _diskProvider . GetDirectoryInfos ( sourcePath ) ;
var targetFolders = _diskProvider . GetDirectoryInfos ( targetPath ) ;
foreach ( var subDir in targetFolders . Where ( v = > ! sourceFolders . Any ( d = > d . Name = = v . Name ) ) )
{
2017-02-09 18:32:13 +00:00
if ( ShouldIgnore ( subDir ) ) continue ;
2016-09-08 23:05:27 +00:00
_diskProvider . DeleteFolder ( subDir . FullName , true ) ;
}
foreach ( var subDir in sourceFolders )
{
2017-02-09 18:32:13 +00:00
if ( ShouldIgnore ( subDir ) ) continue ;
2016-09-08 23:05:27 +00:00
filesCopied + = MirrorFolder ( subDir . FullName , Path . Combine ( targetPath , subDir . Name ) ) ;
}
var sourceFiles = _diskProvider . GetFileInfos ( sourcePath ) ;
var targetFiles = _diskProvider . GetFileInfos ( targetPath ) ;
foreach ( var targetFile in targetFiles . Where ( v = > ! sourceFiles . Any ( d = > d . Name = = v . Name ) ) )
{
2017-02-09 18:32:13 +00:00
if ( ShouldIgnore ( targetFile ) ) continue ;
2016-09-08 23:05:27 +00:00
_diskProvider . DeleteFile ( targetFile . FullName ) ;
}
foreach ( var sourceFile in sourceFiles )
{
2017-02-09 18:32:13 +00:00
if ( ShouldIgnore ( sourceFile ) ) continue ;
2016-09-08 23:05:27 +00:00
var targetFile = Path . Combine ( targetPath , sourceFile . Name ) ;
if ( CompareFiles ( sourceFile . FullName , targetFile ) )
{
continue ;
}
2020-06-11 21:54:28 +00:00
TransferFile ( sourceFile . FullName , targetFile , TransferMode . Copy , true ) ;
2016-09-08 23:05:27 +00:00
filesCopied + + ;
}
return filesCopied ;
}
private bool CompareFiles ( string sourceFile , string targetFile )
{
if ( ! _diskProvider . FileExists ( sourceFile ) | | ! _diskProvider . FileExists ( targetFile ) )
{
return false ;
}
if ( _diskProvider . GetFileSize ( sourceFile ) ! = _diskProvider . GetFileSize ( targetFile ) )
{
return false ;
}
var sourceBuffer = new byte [ 64 * 1024 ] ;
var targetBuffer = new byte [ 64 * 1024 ] ;
using ( var sourceStream = _diskProvider . OpenReadStream ( sourceFile ) )
using ( var targetStream = _diskProvider . OpenReadStream ( targetFile ) )
{
while ( true )
{
var sourceLength = sourceStream . Read ( sourceBuffer , 0 , sourceBuffer . Length ) ;
var targetLength = targetStream . Read ( targetBuffer , 0 , targetBuffer . Length ) ;
if ( sourceLength ! = targetLength )
{
return false ;
}
if ( sourceLength = = 0 )
{
return true ;
}
for ( var i = 0 ; i < sourceLength ; i + + )
{
if ( sourceBuffer [ i ] ! = targetBuffer [ i ] )
{
return false ;
}
}
}
}
}
2020-06-11 21:54:28 +00:00
public TransferMode TransferFile ( string sourcePath , string targetPath , TransferMode mode , bool overwrite = false )
2014-11-15 00:42:22 +00:00
{
Ensure . That ( sourcePath , ( ) = > sourcePath ) . IsValidPath ( ) ;
Ensure . That ( targetPath , ( ) = > targetPath ) . IsValidPath ( ) ;
2020-09-24 09:53:20 +00:00
sourcePath = ResolveRealParentPath ( sourcePath ) ;
targetPath = ResolveRealParentPath ( targetPath ) ;
2014-11-15 00:42:22 +00:00
_logger . Debug ( "{0} [{1}] > [{2}]" , mode , sourcePath , targetPath ) ;
2016-09-08 23:05:27 +00:00
2015-10-21 21:28:20 +00:00
var originalSize = _diskProvider . GetFileSize ( sourcePath ) ;
2014-11-15 00:42:22 +00:00
2015-06-26 23:01:33 +00:00
if ( sourcePath = = targetPath )
2014-11-15 00:42:22 +00:00
{
throw new IOException ( string . Format ( "Source and destination can't be the same {0}" , sourcePath ) ) ;
}
2015-06-26 23:01:33 +00:00
if ( sourcePath . PathEquals ( targetPath , StringComparison . InvariantCultureIgnoreCase ) )
{
if ( mode . HasFlag ( TransferMode . HardLink ) | | mode . HasFlag ( TransferMode . Copy ) )
{
throw new IOException ( string . Format ( "Source and destination can't be the same {0}" , sourcePath ) ) ;
}
if ( mode . HasFlag ( TransferMode . Move ) )
{
var tempPath = sourcePath + ".backup~" ;
2015-07-16 15:10:39 +00:00
_diskProvider . MoveFile ( sourcePath , tempPath , true ) ;
try
2015-06-26 23:01:33 +00:00
{
2017-07-29 13:34:23 +00:00
ClearTargetPath ( sourcePath , targetPath , overwrite ) ;
2015-06-26 23:01:33 +00:00
2015-07-16 15:10:39 +00:00
_diskProvider . MoveFile ( tempPath , targetPath ) ;
return TransferMode . Move ;
}
catch
{
RollbackMove ( sourcePath , tempPath ) ;
throw ;
}
2015-06-26 23:01:33 +00:00
}
return TransferMode . None ;
}
2015-10-21 21:28:20 +00:00
if ( sourcePath . GetParentPath ( ) = = targetPath . GetParentPath ( ) )
{
if ( mode . HasFlag ( TransferMode . Move ) )
{
TryMoveFileVerified ( sourcePath , targetPath , originalSize ) ;
return TransferMode . Move ;
}
}
2014-11-15 00:42:22 +00:00
if ( sourcePath . IsParentPath ( targetPath ) )
{
throw new IOException ( string . Format ( "Destination cannot be a child of the source [{0}] => [{1}]" , sourcePath , targetPath ) ) ;
}
2017-07-29 13:34:23 +00:00
ClearTargetPath ( sourcePath , targetPath , overwrite ) ;
2014-11-15 00:42:22 +00:00
if ( mode . HasFlag ( TransferMode . HardLink ) )
{
var createdHardlink = _diskProvider . TryCreateHardLink ( sourcePath , targetPath ) ;
if ( createdHardlink )
{
return TransferMode . HardLink ;
}
if ( ! mode . HasFlag ( TransferMode . Copy ) )
{
throw new IOException ( "Hardlinking from '" + sourcePath + "' to '" + targetPath + "' failed." ) ;
}
}
2019-11-19 16:34:52 +00:00
// Adjust the transfer mode depending on the filesystems
2020-06-11 21:54:28 +00:00
var sourceMount = _diskProvider . GetMount ( sourcePath ) ;
var targetMount = _diskProvider . GetMount ( targetPath ) ;
2015-12-30 15:22:41 +00:00
2020-06-11 21:54:28 +00:00
var isSameMount = ( sourceMount ! = null & & targetMount ! = null & & sourceMount . RootDirectory = = targetMount . RootDirectory ) ;
2019-11-19 16:34:52 +00:00
2020-06-11 21:54:28 +00:00
var sourceDriveFormat = sourceMount ? . DriveFormat ? ? string . Empty ;
var targetDriveFormat = targetMount ? . DriveFormat ? ? string . Empty ;
2014-11-15 00:42:22 +00:00
2020-06-11 21:54:28 +00:00
var isCifs = targetDriveFormat = = "cifs" ;
2020-06-10 18:48:54 +00:00
var isBtrfs = sourceDriveFormat = = "btrfs" & & targetDriveFormat = = "btrfs" ;
2020-06-11 21:54:28 +00:00
2015-12-30 15:22:41 +00:00
if ( mode . HasFlag ( TransferMode . Copy ) )
{
2020-06-10 18:48:54 +00:00
if ( isBtrfs )
{
if ( _diskProvider . TryCreateRefLink ( sourcePath , targetPath ) )
{
return TransferMode . Copy ;
}
}
2020-06-11 21:54:28 +00:00
TryCopyFileVerified ( sourcePath , targetPath , originalSize ) ;
return TransferMode . Copy ;
2015-07-16 15:10:39 +00:00
}
2015-12-30 15:22:41 +00:00
if ( mode . HasFlag ( TransferMode . Move ) )
2014-11-15 00:42:22 +00:00
{
2020-06-10 18:48:54 +00:00
if ( isBtrfs )
{
if ( isSameMount & & _diskProvider . TryRenameFile ( sourcePath , targetPath ) )
{
_logger . Trace ( "Renamed [{0}] to [{1}]." , sourcePath , targetPath ) ;
return TransferMode . Move ;
}
if ( _diskProvider . TryCreateRefLink ( sourcePath , targetPath ) )
{
_logger . Trace ( "Reflink successful, deleting source [{0}]." , sourcePath ) ;
_diskProvider . DeleteFile ( sourcePath ) ;
return TransferMode . Move ;
}
}
2020-06-11 21:54:28 +00:00
if ( isCifs & & ! isSameMount )
2014-11-15 00:42:22 +00:00
{
2020-06-10 18:48:54 +00:00
_logger . Trace ( "On cifs mount. Starting verified copy [{0}] to [{1}]." , sourcePath , targetPath ) ;
2020-06-11 21:54:28 +00:00
TryCopyFileVerified ( sourcePath , targetPath , originalSize ) ;
2020-06-10 18:48:54 +00:00
_logger . Trace ( "Copy successful, deleting source [{0}]." , sourcePath ) ;
2020-06-11 21:54:28 +00:00
_diskProvider . DeleteFile ( sourcePath ) ;
2014-11-15 00:42:22 +00:00
return TransferMode . Move ;
}
2020-06-11 21:54:28 +00:00
TryMoveFileVerified ( sourcePath , targetPath , originalSize ) ;
return TransferMode . Move ;
2014-11-15 00:42:22 +00:00
}
return TransferMode . None ;
}
2017-07-29 13:34:23 +00:00
private void ClearTargetPath ( string sourcePath , string targetPath , bool overwrite )
2015-07-16 15:10:39 +00:00
{
if ( _diskProvider . FileExists ( targetPath ) )
{
if ( overwrite )
{
_diskProvider . DeleteFile ( targetPath ) ;
}
else
{
2017-10-06 21:10:24 +00:00
throw new DestinationAlreadyExistsException ( $"Destination {targetPath} already exists." ) ;
2015-07-16 15:10:39 +00:00
}
}
}
private void RollbackPartialMove ( string sourcePath , string targetPath )
{
try
{
_logger . Debug ( "Rolling back incomplete file move [{0}] to [{1}]." , sourcePath , targetPath ) ;
WaitForIO ( ) ;
if ( _diskProvider . FileExists ( sourcePath ) )
{
_diskProvider . DeleteFile ( targetPath ) ;
}
else
{
_logger . Error ( "Failed to properly rollback the file move [{0}] to [{1}], incomplete file may be left in target path." , sourcePath , targetPath ) ;
}
}
catch ( Exception ex )
{
2017-01-05 23:32:17 +00:00
_logger . Error ( ex , "Failed to properly rollback the file move [{0}] to [{1}], incomplete file may be left in target path." , sourcePath , targetPath ) ;
2015-07-16 15:10:39 +00:00
}
}
private void RollbackMove ( string sourcePath , string targetPath )
{
try
{
_logger . Debug ( "Rolling back file move [{0}] to [{1}]." , sourcePath , targetPath ) ;
WaitForIO ( ) ;
_diskProvider . MoveFile ( targetPath , sourcePath ) ;
}
catch ( Exception ex )
{
2017-01-05 23:32:17 +00:00
_logger . Error ( ex , "Failed to properly rollback the file move [{0}] to [{1}], file may be left in target path." , sourcePath , targetPath ) ;
2015-07-16 15:10:39 +00:00
}
}
private void RollbackCopy ( string sourcePath , string targetPath )
{
try
{
_logger . Debug ( "Rolling back file copy [{0}] to [{1}]." , sourcePath , targetPath ) ;
WaitForIO ( ) ;
if ( _diskProvider . FileExists ( targetPath ) )
{
_diskProvider . DeleteFile ( targetPath ) ;
}
}
catch ( Exception ex )
{
2017-01-05 23:32:17 +00:00
_logger . Error ( ex , "Failed to properly rollback the file copy [{0}] to [{1}], file may be left in target path." , sourcePath , targetPath ) ;
2015-07-16 15:10:39 +00:00
}
}
private void WaitForIO ( )
{
// This delay is intended to give the IO stack a bit of time to recover, this is especially required if remote NAS devices are involved.
Thread . Sleep ( 3000 ) ;
}
2015-10-21 21:28:20 +00:00
private void TryCopyFileVerified ( string sourcePath , string targetPath , long originalSize )
{
try
{
_diskProvider . CopyFile ( sourcePath , targetPath ) ;
var targetSize = _diskProvider . GetFileSize ( targetPath ) ;
if ( targetSize ! = originalSize )
{
throw new IOException ( string . Format ( "File copy incomplete. [{0}] was {1} bytes long instead of {2} bytes." , targetPath , targetSize , originalSize ) ) ;
}
}
catch
{
RollbackCopy ( sourcePath , targetPath ) ;
throw ;
}
}
private void TryMoveFileVerified ( string sourcePath , string targetPath , long originalSize )
{
try
{
_diskProvider . MoveFile ( sourcePath , targetPath ) ;
var targetSize = _diskProvider . GetFileSize ( targetPath ) ;
if ( targetSize ! = originalSize )
{
throw new IOException ( string . Format ( "File move incomplete, data loss may have occurred. [{0}] was {1} bytes long instead of the expected {2}." , targetPath , targetSize , originalSize ) ) ;
}
}
catch
{
RollbackPartialMove ( sourcePath , targetPath ) ;
throw ;
}
}
2017-02-09 18:32:13 +00:00
private bool ShouldIgnore ( DirectoryInfo folder )
{
if ( folder . Name . StartsWith ( ".nfs" ) )
{
_logger . Trace ( "Ignoring folder {0}" , folder . FullName ) ;
return true ;
}
return false ;
}
private bool ShouldIgnore ( FileInfo file )
{
2017-04-07 20:07:42 +00:00
if ( file . Name . StartsWith ( ".nfs" ) | | file . Name = = "debug.log" | | file . Name . EndsWith ( ".socket" ) )
2017-02-09 18:32:13 +00:00
{
_logger . Trace ( "Ignoring file {0}" , file . FullName ) ;
return true ;
}
return false ;
}
2014-11-15 00:42:22 +00:00
}
}