Fixed: Added verified file transfer mode that doesn't revert to copy.

This commit is contained in:
Taloth Saldono 2015-10-21 23:28:20 +02:00
parent 04de0049fe
commit cc72699b8a
2 changed files with 122 additions and 64 deletions

View File

@ -34,7 +34,7 @@ namespace NzbDrone.Common.Test.DiskTests
{ {
MonoOnly(); MonoOnly();
Subject.VerificationMode.Should().Be(DiskTransferVerificationMode.Transactional); Subject.VerificationMode.Should().Be(DiskTransferVerificationMode.TryTransactional);
} }
[Test] [Test]
@ -199,9 +199,6 @@ namespace NzbDrone.Common.Test.DiskTests
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
.Verify(v => v.CopyFile(_sourcePath, _targetPath, false), Times.Once()); .Verify(v => v.CopyFile(_sourcePath, _targetPath, false), Times.Once());
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.GetFileSize(It.IsAny<string>()), Times.Never());
} }
[Test] [Test]
@ -213,9 +210,6 @@ namespace NzbDrone.Common.Test.DiskTests
Mocker.GetMock<IDiskProvider>() Mocker.GetMock<IDiskProvider>()
.Verify(v => v.MoveFile(_sourcePath, _targetPath, false), Times.Once()); .Verify(v => v.MoveFile(_sourcePath, _targetPath, false), Times.Once());
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.GetFileSize(It.IsAny<string>()), Times.Never());
} }
[Test] [Test]
@ -377,6 +371,52 @@ namespace NzbDrone.Common.Test.DiskTests
ExceptionVerification.ExpectedErrors(1); ExceptionVerification.ExpectedErrors(1);
} }
[Test]
public void mode_transactional_should_move_and_verify_if_same_folder()
{
Subject.VerificationMode = DiskTransferVerificationMode.Transactional;
var targetPath = _sourcePath + ".test";
var result = Subject.TransferFile(_sourcePath, targetPath, TransferMode.Move);
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.TryCreateHardLink(_sourcePath, _backupPath), Times.Never());
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.CopyFile(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>()), Times.Never());
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.MoveFile(_sourcePath, targetPath, false), Times.Once());
}
[Test]
public void mode_trytransactional_should_revert_to_verifyonly_if_hardlink_fails()
{
Subject.VerificationMode = DiskTransferVerificationMode.TryTransactional;
WithFailedHardlink();
Mocker.GetMock<IDiskProvider>()
.Setup(v => v.MoveFile(_sourcePath, _targetPath, false))
.Callback(() =>
{
WithExistingFile(_sourcePath, false);
WithExistingFile(_targetPath, true);
});
var result = Subject.TransferFile(_sourcePath, _targetPath, TransferMode.Move);
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.TryCreateHardLink(_sourcePath, _backupPath), Times.Once());
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.CopyFile(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>()), Times.Never());
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.MoveFile(_sourcePath, _targetPath, false), Times.Once());
}
[Test] [Test]
public void mode_transactional_should_delete_old_backup_on_move() public void mode_transactional_should_delete_old_backup_on_move()
{ {

View File

@ -21,6 +21,7 @@ namespace NzbDrone.Common.Disk
{ {
None, None,
VerifyOnly, VerifyOnly,
TryTransactional,
Transactional Transactional
} }
@ -40,7 +41,7 @@ namespace NzbDrone.Common.Disk
// TODO: Atm we haven't seen partial transfers on windows so we disable verified transfer. // TODO: Atm we haven't seen partial transfers on windows so we disable verified transfer.
// (If enabled in the future, be sure to check specifically for ReFS, which doesn't support hardlinks.) // (If enabled in the future, be sure to check specifically for ReFS, which doesn't support hardlinks.)
VerificationMode = OsInfo.IsWindows ? DiskTransferVerificationMode.VerifyOnly : DiskTransferVerificationMode.Transactional; VerificationMode = OsInfo.IsWindows ? DiskTransferVerificationMode.VerifyOnly : DiskTransferVerificationMode.TryTransactional;
} }
public TransferMode TransferFolder(string sourcePath, string targetPath, TransferMode mode, bool verified = true) public TransferMode TransferFolder(string sourcePath, string targetPath, TransferMode mode, bool verified = true)
@ -48,11 +49,6 @@ namespace NzbDrone.Common.Disk
Ensure.That(sourcePath, () => sourcePath).IsValidPath(); Ensure.That(sourcePath, () => sourcePath).IsValidPath();
Ensure.That(targetPath, () => targetPath).IsValidPath(); Ensure.That(targetPath, () => targetPath).IsValidPath();
if (VerificationMode != DiskTransferVerificationMode.Transactional)
{
verified = false;
}
if (!_diskProvider.FolderExists(targetPath)) if (!_diskProvider.FolderExists(targetPath))
{ {
_diskProvider.CreateFolder(targetPath); _diskProvider.CreateFolder(targetPath);
@ -85,13 +81,15 @@ namespace NzbDrone.Common.Disk
Ensure.That(sourcePath, () => sourcePath).IsValidPath(); Ensure.That(sourcePath, () => sourcePath).IsValidPath();
Ensure.That(targetPath, () => targetPath).IsValidPath(); Ensure.That(targetPath, () => targetPath).IsValidPath();
if (VerificationMode != DiskTransferVerificationMode.Transactional) if (VerificationMode != DiskTransferVerificationMode.Transactional && VerificationMode != DiskTransferVerificationMode.TryTransactional)
{ {
verified = false; verified = false;
} }
_logger.Debug("{0} [{1}] > [{2}]", mode, sourcePath, targetPath); _logger.Debug("{0} [{1}] > [{2}]", mode, sourcePath, targetPath);
var originalSize = _diskProvider.GetFileSize(sourcePath);
if (sourcePath == targetPath) if (sourcePath == targetPath)
{ {
throw new IOException(string.Format("Source and destination can't be the same {0}", sourcePath)); throw new IOException(string.Format("Source and destination can't be the same {0}", sourcePath));
@ -127,6 +125,15 @@ namespace NzbDrone.Common.Disk
return TransferMode.None; return TransferMode.None;
} }
if (sourcePath.GetParentPath() == targetPath.GetParentPath())
{
if (mode.HasFlag(TransferMode.Move))
{
TryMoveFileVerified(sourcePath, targetPath, originalSize);
return TransferMode.Move;
}
}
if (sourcePath.IsParentPath(targetPath)) if (sourcePath.IsParentPath(targetPath))
{ {
throw new IOException(string.Format("Destination cannot be a child of the source [{0}] => [{1}]", sourcePath, targetPath)); throw new IOException(string.Format("Destination cannot be a child of the source [{0}] => [{1}]", sourcePath, targetPath));
@ -151,7 +158,7 @@ namespace NzbDrone.Common.Disk
{ {
if (mode.HasFlag(TransferMode.Copy)) if (mode.HasFlag(TransferMode.Copy))
{ {
if (TryCopyFile(sourcePath, targetPath)) if (TryCopyFileTransactional(sourcePath, targetPath, originalSize))
{ {
return TransferMode.Copy; return TransferMode.Copy;
} }
@ -159,7 +166,7 @@ namespace NzbDrone.Common.Disk
if (mode.HasFlag(TransferMode.Move)) if (mode.HasFlag(TransferMode.Move))
{ {
if (TryMoveFile(sourcePath, targetPath)) if (TryMoveFileTransactional(sourcePath, targetPath, originalSize))
{ {
return TransferMode.Move; return TransferMode.Move;
} }
@ -167,50 +174,18 @@ namespace NzbDrone.Common.Disk
throw new IOException(string.Format("Failed to completely transfer [{0}] to [{1}], aborting.", sourcePath, targetPath)); throw new IOException(string.Format("Failed to completely transfer [{0}] to [{1}], aborting.", sourcePath, targetPath));
} }
else if (VerificationMode == DiskTransferVerificationMode.VerifyOnly) else if (VerificationMode != DiskTransferVerificationMode.None)
{ {
var originalSize = _diskProvider.GetFileSize(sourcePath);
if (mode.HasFlag(TransferMode.Copy)) if (mode.HasFlag(TransferMode.Copy))
{ {
try TryCopyFileVerified(sourcePath, targetPath, originalSize);
{ return TransferMode.Copy;
_diskProvider.CopyFile(sourcePath, targetPath);
var targetSize = _diskProvider.GetFileSize(targetPath);
if (targetSize != originalSize)
{
throw new IOException(string.Format("File copy incomplete. [{0}] was {1} bytes long instead of {2} bytes.", targetPath, targetSize, originalSize));
}
return TransferMode.Copy;
}
catch
{
RollbackCopy(sourcePath, targetPath);
throw;
}
} }
if (mode.HasFlag(TransferMode.Move)) if (mode.HasFlag(TransferMode.Move))
{ {
try TryMoveFileVerified(sourcePath, targetPath, originalSize);
{ return TransferMode.Move;
_diskProvider.MoveFile(sourcePath, targetPath);
var targetSize = _diskProvider.GetFileSize(targetPath);
if (targetSize != originalSize)
{
throw new IOException(string.Format("File copy incomplete, data loss may have occured. [{0}] was {1} bytes long instead of the expected {2}.", targetPath, targetSize, originalSize));
}
return TransferMode.Move;
}
catch
{
RollbackPartialMove(sourcePath, targetPath);
throw;
}
} }
} }
else else
@ -310,10 +285,8 @@ namespace NzbDrone.Common.Disk
Thread.Sleep(3000); Thread.Sleep(3000);
} }
private bool TryCopyFile(string sourcePath, string targetPath) private bool TryCopyFileTransactional(string sourcePath, string targetPath, long originalSize)
{ {
var originalSize = _diskProvider.GetFileSize(sourcePath);
var tempTargetPath = targetPath + ".partial~"; var tempTargetPath = targetPath + ".partial~";
if (_diskProvider.FileExists(tempTargetPath)) if (_diskProvider.FileExists(tempTargetPath))
@ -367,10 +340,8 @@ namespace NzbDrone.Common.Disk
return false; return false;
} }
private bool TryMoveFile(string sourcePath, string targetPath) private bool TryMoveFileTransactional(string sourcePath, string targetPath, long originalSize)
{ {
var originalSize = _diskProvider.GetFileSize(sourcePath);
var backupPath = sourcePath + ".backup~"; var backupPath = sourcePath + ".backup~";
var tempTargetPath = targetPath + ".partial~"; var tempTargetPath = targetPath + ".partial~";
@ -423,16 +394,63 @@ namespace NzbDrone.Common.Disk
} }
} }
_logger.Trace("Hardlink move failed, reverting to copy."); if (VerificationMode == DiskTransferVerificationMode.Transactional)
if (TryCopyFile(sourcePath, targetPath))
{ {
_logger.Trace("Copy succeeded, deleting source."); _logger.Trace("Hardlink move failed, reverting to copy.");
_diskProvider.DeleteFile(sourcePath); if (TryCopyFileTransactional(sourcePath, targetPath, originalSize))
{
_logger.Trace("Copy succeeded, deleting source.");
_diskProvider.DeleteFile(sourcePath);
return true;
}
}
else
{
_logger.Trace("Hardlink move failed, reverting to move.");
TryMoveFileVerified(sourcePath, targetPath, originalSize);
return true; return true;
} }
_logger.Trace("Copy failed."); _logger.Trace("Move failed.");
return false; return false;
} }
private void TryCopyFileVerified(string sourcePath, string targetPath, long originalSize)
{
try
{
_diskProvider.CopyFile(sourcePath, targetPath);
var targetSize = _diskProvider.GetFileSize(targetPath);
if (targetSize != originalSize)
{
throw new IOException(string.Format("File copy incomplete. [{0}] was {1} bytes long instead of {2} bytes.", targetPath, targetSize, originalSize));
}
}
catch
{
RollbackCopy(sourcePath, targetPath);
throw;
}
}
private void TryMoveFileVerified(string sourcePath, string targetPath, long originalSize)
{
try
{
_diskProvider.MoveFile(sourcePath, targetPath);
var targetSize = _diskProvider.GetFileSize(targetPath);
if (targetSize != originalSize)
{
throw new IOException(string.Format("File move incomplete, data loss may have occurred. [{0}] was {1} bytes long instead of the expected {2}.", targetPath, targetSize, originalSize));
}
}
catch
{
RollbackPartialMove(sourcePath, targetPath);
throw;
}
}
} }
} }