mirror of https://github.com/Radarr/Radarr
Fixed: Added verified file transfer mode that doesn't revert to copy.
This commit is contained in:
parent
04de0049fe
commit
cc72699b8a
|
@ -34,7 +34,7 @@ namespace NzbDrone.Common.Test.DiskTests
|
||||||
{
|
{
|
||||||
MonoOnly();
|
MonoOnly();
|
||||||
|
|
||||||
Subject.VerificationMode.Should().Be(DiskTransferVerificationMode.Transactional);
|
Subject.VerificationMode.Should().Be(DiskTransferVerificationMode.TryTransactional);
|
||||||
}
|
}
|
||||||
|
|
||||||
[Test]
|
[Test]
|
||||||
|
@ -199,9 +199,6 @@ namespace NzbDrone.Common.Test.DiskTests
|
||||||
|
|
||||||
Mocker.GetMock<IDiskProvider>()
|
Mocker.GetMock<IDiskProvider>()
|
||||||
.Verify(v => v.CopyFile(_sourcePath, _targetPath, false), Times.Once());
|
.Verify(v => v.CopyFile(_sourcePath, _targetPath, false), Times.Once());
|
||||||
|
|
||||||
Mocker.GetMock<IDiskProvider>()
|
|
||||||
.Verify(v => v.GetFileSize(It.IsAny<string>()), Times.Never());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
[Test]
|
[Test]
|
||||||
|
@ -213,9 +210,6 @@ namespace NzbDrone.Common.Test.DiskTests
|
||||||
|
|
||||||
Mocker.GetMock<IDiskProvider>()
|
Mocker.GetMock<IDiskProvider>()
|
||||||
.Verify(v => v.MoveFile(_sourcePath, _targetPath, false), Times.Once());
|
.Verify(v => v.MoveFile(_sourcePath, _targetPath, false), Times.Once());
|
||||||
|
|
||||||
Mocker.GetMock<IDiskProvider>()
|
|
||||||
.Verify(v => v.GetFileSize(It.IsAny<string>()), Times.Never());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
[Test]
|
[Test]
|
||||||
|
@ -377,6 +371,52 @@ namespace NzbDrone.Common.Test.DiskTests
|
||||||
ExceptionVerification.ExpectedErrors(1);
|
ExceptionVerification.ExpectedErrors(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void mode_transactional_should_move_and_verify_if_same_folder()
|
||||||
|
{
|
||||||
|
Subject.VerificationMode = DiskTransferVerificationMode.Transactional;
|
||||||
|
|
||||||
|
var targetPath = _sourcePath + ".test";
|
||||||
|
|
||||||
|
var result = Subject.TransferFile(_sourcePath, targetPath, TransferMode.Move);
|
||||||
|
|
||||||
|
Mocker.GetMock<IDiskProvider>()
|
||||||
|
.Verify(v => v.TryCreateHardLink(_sourcePath, _backupPath), Times.Never());
|
||||||
|
|
||||||
|
Mocker.GetMock<IDiskProvider>()
|
||||||
|
.Verify(v => v.CopyFile(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>()), Times.Never());
|
||||||
|
|
||||||
|
Mocker.GetMock<IDiskProvider>()
|
||||||
|
.Verify(v => v.MoveFile(_sourcePath, targetPath, false), Times.Once());
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void mode_trytransactional_should_revert_to_verifyonly_if_hardlink_fails()
|
||||||
|
{
|
||||||
|
Subject.VerificationMode = DiskTransferVerificationMode.TryTransactional;
|
||||||
|
|
||||||
|
WithFailedHardlink();
|
||||||
|
|
||||||
|
Mocker.GetMock<IDiskProvider>()
|
||||||
|
.Setup(v => v.MoveFile(_sourcePath, _targetPath, false))
|
||||||
|
.Callback(() =>
|
||||||
|
{
|
||||||
|
WithExistingFile(_sourcePath, false);
|
||||||
|
WithExistingFile(_targetPath, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
var result = Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Move);
|
||||||
|
|
||||||
|
Mocker.GetMock<IDiskProvider>()
|
||||||
|
.Verify(v => v.TryCreateHardLink(_sourcePath, _backupPath), Times.Once());
|
||||||
|
|
||||||
|
Mocker.GetMock<IDiskProvider>()
|
||||||
|
.Verify(v => v.CopyFile(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>()), Times.Never());
|
||||||
|
|
||||||
|
Mocker.GetMock<IDiskProvider>()
|
||||||
|
.Verify(v => v.MoveFile(_sourcePath, _targetPath, false), Times.Once());
|
||||||
|
}
|
||||||
|
|
||||||
[Test]
|
[Test]
|
||||||
public void mode_transactional_should_delete_old_backup_on_move()
|
public void mode_transactional_should_delete_old_backup_on_move()
|
||||||
{
|
{
|
||||||
|
|
|
@ -21,6 +21,7 @@ namespace NzbDrone.Common.Disk
|
||||||
{
|
{
|
||||||
None,
|
None,
|
||||||
VerifyOnly,
|
VerifyOnly,
|
||||||
|
TryTransactional,
|
||||||
Transactional
|
Transactional
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -40,7 +41,7 @@ namespace NzbDrone.Common.Disk
|
||||||
|
|
||||||
// TODO: Atm we haven't seen partial transfers on windows so we disable verified transfer.
|
// TODO: Atm we haven't seen partial transfers on windows so we disable verified transfer.
|
||||||
// (If enabled in the future, be sure to check specifically for ReFS, which doesn't support hardlinks.)
|
// (If enabled in the future, be sure to check specifically for ReFS, which doesn't support hardlinks.)
|
||||||
VerificationMode = OsInfo.IsWindows ? DiskTransferVerificationMode.VerifyOnly : DiskTransferVerificationMode.Transactional;
|
VerificationMode = OsInfo.IsWindows ? DiskTransferVerificationMode.VerifyOnly : DiskTransferVerificationMode.TryTransactional;
|
||||||
}
|
}
|
||||||
|
|
||||||
public TransferMode TransferFolder(string sourcePath, string targetPath, TransferMode mode, bool verified = true)
|
public TransferMode TransferFolder(string sourcePath, string targetPath, TransferMode mode, bool verified = true)
|
||||||
|
@ -48,11 +49,6 @@ namespace NzbDrone.Common.Disk
|
||||||
Ensure.That(sourcePath, () => sourcePath).IsValidPath();
|
Ensure.That(sourcePath, () => sourcePath).IsValidPath();
|
||||||
Ensure.That(targetPath, () => targetPath).IsValidPath();
|
Ensure.That(targetPath, () => targetPath).IsValidPath();
|
||||||
|
|
||||||
if (VerificationMode != DiskTransferVerificationMode.Transactional)
|
|
||||||
{
|
|
||||||
verified = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!_diskProvider.FolderExists(targetPath))
|
if (!_diskProvider.FolderExists(targetPath))
|
||||||
{
|
{
|
||||||
_diskProvider.CreateFolder(targetPath);
|
_diskProvider.CreateFolder(targetPath);
|
||||||
|
@ -85,13 +81,15 @@ namespace NzbDrone.Common.Disk
|
||||||
Ensure.That(sourcePath, () => sourcePath).IsValidPath();
|
Ensure.That(sourcePath, () => sourcePath).IsValidPath();
|
||||||
Ensure.That(targetPath, () => targetPath).IsValidPath();
|
Ensure.That(targetPath, () => targetPath).IsValidPath();
|
||||||
|
|
||||||
if (VerificationMode != DiskTransferVerificationMode.Transactional)
|
if (VerificationMode != DiskTransferVerificationMode.Transactional && VerificationMode != DiskTransferVerificationMode.TryTransactional)
|
||||||
{
|
{
|
||||||
verified = false;
|
verified = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
_logger.Debug("{0} [{1}] > [{2}]", mode, sourcePath, targetPath);
|
_logger.Debug("{0} [{1}] > [{2}]", mode, sourcePath, targetPath);
|
||||||
|
|
||||||
|
var originalSize = _diskProvider.GetFileSize(sourcePath);
|
||||||
|
|
||||||
if (sourcePath == targetPath)
|
if (sourcePath == targetPath)
|
||||||
{
|
{
|
||||||
throw new IOException(string.Format("Source and destination can't be the same {0}", sourcePath));
|
throw new IOException(string.Format("Source and destination can't be the same {0}", sourcePath));
|
||||||
|
@ -127,6 +125,15 @@ namespace NzbDrone.Common.Disk
|
||||||
return TransferMode.None;
|
return TransferMode.None;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (sourcePath.GetParentPath() == targetPath.GetParentPath())
|
||||||
|
{
|
||||||
|
if (mode.HasFlag(TransferMode.Move))
|
||||||
|
{
|
||||||
|
TryMoveFileVerified(sourcePath, targetPath, originalSize);
|
||||||
|
return TransferMode.Move;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (sourcePath.IsParentPath(targetPath))
|
if (sourcePath.IsParentPath(targetPath))
|
||||||
{
|
{
|
||||||
throw new IOException(string.Format("Destination cannot be a child of the source [{0}] => [{1}]", sourcePath, targetPath));
|
throw new IOException(string.Format("Destination cannot be a child of the source [{0}] => [{1}]", sourcePath, targetPath));
|
||||||
|
@ -151,7 +158,7 @@ namespace NzbDrone.Common.Disk
|
||||||
{
|
{
|
||||||
if (mode.HasFlag(TransferMode.Copy))
|
if (mode.HasFlag(TransferMode.Copy))
|
||||||
{
|
{
|
||||||
if (TryCopyFile(sourcePath, targetPath))
|
if (TryCopyFileTransactional(sourcePath, targetPath, originalSize))
|
||||||
{
|
{
|
||||||
return TransferMode.Copy;
|
return TransferMode.Copy;
|
||||||
}
|
}
|
||||||
|
@ -159,7 +166,7 @@ namespace NzbDrone.Common.Disk
|
||||||
|
|
||||||
if (mode.HasFlag(TransferMode.Move))
|
if (mode.HasFlag(TransferMode.Move))
|
||||||
{
|
{
|
||||||
if (TryMoveFile(sourcePath, targetPath))
|
if (TryMoveFileTransactional(sourcePath, targetPath, originalSize))
|
||||||
{
|
{
|
||||||
return TransferMode.Move;
|
return TransferMode.Move;
|
||||||
}
|
}
|
||||||
|
@ -167,51 +174,19 @@ namespace NzbDrone.Common.Disk
|
||||||
|
|
||||||
throw new IOException(string.Format("Failed to completely transfer [{0}] to [{1}], aborting.", sourcePath, targetPath));
|
throw new IOException(string.Format("Failed to completely transfer [{0}] to [{1}], aborting.", sourcePath, targetPath));
|
||||||
}
|
}
|
||||||
else if (VerificationMode == DiskTransferVerificationMode.VerifyOnly)
|
else if (VerificationMode != DiskTransferVerificationMode.None)
|
||||||
{
|
{
|
||||||
var originalSize = _diskProvider.GetFileSize(sourcePath);
|
|
||||||
|
|
||||||
if (mode.HasFlag(TransferMode.Copy))
|
if (mode.HasFlag(TransferMode.Copy))
|
||||||
{
|
{
|
||||||
try
|
TryCopyFileVerified(sourcePath, targetPath, originalSize);
|
||||||
{
|
|
||||||
_diskProvider.CopyFile(sourcePath, targetPath);
|
|
||||||
|
|
||||||
var targetSize = _diskProvider.GetFileSize(targetPath);
|
|
||||||
if (targetSize != originalSize)
|
|
||||||
{
|
|
||||||
throw new IOException(string.Format("File copy incomplete. [{0}] was {1} bytes long instead of {2} bytes.", targetPath, targetSize, originalSize));
|
|
||||||
}
|
|
||||||
|
|
||||||
return TransferMode.Copy;
|
return TransferMode.Copy;
|
||||||
}
|
}
|
||||||
catch
|
|
||||||
{
|
|
||||||
RollbackCopy(sourcePath, targetPath);
|
|
||||||
throw;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (mode.HasFlag(TransferMode.Move))
|
if (mode.HasFlag(TransferMode.Move))
|
||||||
{
|
{
|
||||||
try
|
TryMoveFileVerified(sourcePath, targetPath, originalSize);
|
||||||
{
|
|
||||||
_diskProvider.MoveFile(sourcePath, targetPath);
|
|
||||||
|
|
||||||
var targetSize = _diskProvider.GetFileSize(targetPath);
|
|
||||||
if (targetSize != originalSize)
|
|
||||||
{
|
|
||||||
throw new IOException(string.Format("File copy incomplete, data loss may have occured. [{0}] was {1} bytes long instead of the expected {2}.", targetPath, targetSize, originalSize));
|
|
||||||
}
|
|
||||||
|
|
||||||
return TransferMode.Move;
|
return TransferMode.Move;
|
||||||
}
|
}
|
||||||
catch
|
|
||||||
{
|
|
||||||
RollbackPartialMove(sourcePath, targetPath);
|
|
||||||
throw;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
@ -310,10 +285,8 @@ namespace NzbDrone.Common.Disk
|
||||||
Thread.Sleep(3000);
|
Thread.Sleep(3000);
|
||||||
}
|
}
|
||||||
|
|
||||||
private bool TryCopyFile(string sourcePath, string targetPath)
|
private bool TryCopyFileTransactional(string sourcePath, string targetPath, long originalSize)
|
||||||
{
|
{
|
||||||
var originalSize = _diskProvider.GetFileSize(sourcePath);
|
|
||||||
|
|
||||||
var tempTargetPath = targetPath + ".partial~";
|
var tempTargetPath = targetPath + ".partial~";
|
||||||
|
|
||||||
if (_diskProvider.FileExists(tempTargetPath))
|
if (_diskProvider.FileExists(tempTargetPath))
|
||||||
|
@ -367,10 +340,8 @@ namespace NzbDrone.Common.Disk
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
private bool TryMoveFile(string sourcePath, string targetPath)
|
private bool TryMoveFileTransactional(string sourcePath, string targetPath, long originalSize)
|
||||||
{
|
{
|
||||||
var originalSize = _diskProvider.GetFileSize(sourcePath);
|
|
||||||
|
|
||||||
var backupPath = sourcePath + ".backup~";
|
var backupPath = sourcePath + ".backup~";
|
||||||
var tempTargetPath = targetPath + ".partial~";
|
var tempTargetPath = targetPath + ".partial~";
|
||||||
|
|
||||||
|
@ -423,16 +394,63 @@ namespace NzbDrone.Common.Disk
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (VerificationMode == DiskTransferVerificationMode.Transactional)
|
||||||
|
{
|
||||||
_logger.Trace("Hardlink move failed, reverting to copy.");
|
_logger.Trace("Hardlink move failed, reverting to copy.");
|
||||||
if (TryCopyFile(sourcePath, targetPath))
|
if (TryCopyFileTransactional(sourcePath, targetPath, originalSize))
|
||||||
{
|
{
|
||||||
_logger.Trace("Copy succeeded, deleting source.");
|
_logger.Trace("Copy succeeded, deleting source.");
|
||||||
_diskProvider.DeleteFile(sourcePath);
|
_diskProvider.DeleteFile(sourcePath);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_logger.Trace("Hardlink move failed, reverting to move.");
|
||||||
|
TryMoveFileVerified(sourcePath, targetPath, originalSize);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
_logger.Trace("Copy failed.");
|
_logger.Trace("Move failed.");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void TryCopyFileVerified(string sourcePath, string targetPath, long originalSize)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_diskProvider.CopyFile(sourcePath, targetPath);
|
||||||
|
|
||||||
|
var targetSize = _diskProvider.GetFileSize(targetPath);
|
||||||
|
if (targetSize != originalSize)
|
||||||
|
{
|
||||||
|
throw new IOException(string.Format("File copy incomplete. [{0}] was {1} bytes long instead of {2} bytes.", targetPath, targetSize, originalSize));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
RollbackCopy(sourcePath, targetPath);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void TryMoveFileVerified(string sourcePath, string targetPath, long originalSize)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_diskProvider.MoveFile(sourcePath, targetPath);
|
||||||
|
|
||||||
|
var targetSize = _diskProvider.GetFileSize(targetPath);
|
||||||
|
if (targetSize != originalSize)
|
||||||
|
{
|
||||||
|
throw new IOException(string.Format("File move incomplete, data loss may have occurred. [{0}] was {1} bytes long instead of the expected {2}.", targetPath, targetSize, originalSize));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
RollbackPartialMove(sourcePath, targetPath);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue