mirror/transmission
1
0
Fork 0
mirror of https://github.com/transmission/transmission synced 2025-01-30 19:03:04 +00:00
Code Issues Releases Wiki Activity

Add in-kernel file copying for several platforms. (#1092)

Browse source 
* Add in-kernel copying support for Linux (sendfile64(2), copy_file_range(2)), FreeBSD 13 (copy_file_range(2)), MacOS (copyfile(2)), and Windows (CopyFileExA).

* Fix macro name USE_COPY_FILE_RANGE.

* Minor bugfixes for userspace fallback.

* Fix linux sendfile64 bugs.

* Remove some overzealous asserts.

* Allow transmission-test-copy to take an optional argument for an external reference file.

* Fix return value error of tr_sys_path_copy.

* Use COPYFILE_ALL for Macs without COPYFILE_CLONE.

* Add in-kernel file copying for several platforms.

Numerous operating systems now have support for copying files directly in the
kernel, which is generally more efficient than copying in a userspace read(2)/
write(2) loop. (This becomes particularly relevant for 4th gen PCI-E storage,
which approaches the latency of DRAM.) For Linux I use sendfile64(2), and, for
later kernels, copy_file_range(2). FreeBSD 13 will also support
copy_file_range(2). MacOS has copyfile(2), and Windows has CopyFileExA.
Operating systems lacking such a syscall continue to use the existing
read(2)/write(2) loop.

* Appease uncrustify.

* Appease uncrustify.

* copy-test: generate random content at run time.

* copy-test: Stylistic changes and more check()s.

* copy-test: files_are_identical should follow test idioms

* tr_sys_path_copy: numerous tweaks as requested by review.

* s/old file/source file; s/new file/destination file.

* tr_sys_path_copy: handle win32 wide characters in paths.

* Uncrustify.

* test-copy: Use non-string create_file_with_contents.

* tr_sys_path_copy: numerous fixes.

Per review: generate test file content at runtime; tidy use of check();
fix style; re-measure file sizes in the copy; define a macro when the
system does not provide it; use Unicode APIs on Windows; and fix
documentation.

* Updated as per comments.

* Rebase kernel-copy changes onto 3.0 with gtest.

* Undo irrelevant comment change.

* Fix syntax error.

* Use tr_malloc() instead of tr_valloc().

* Use EXPECT instead of TR_ASSERT in gtest.

* Add error handling.

* Acceptable coding style has changed again.

Now it's camelCase. Also use nullptr instead of NULL, etc.

* Fix east/west const.

Co-authored-by: Mike Gelfand <mikedld@users.noreply.github.com>
This commit is contained in:
RobCrowston 2021-08-07 10:04:03 +01:00 committed by GitHub
parent af3a4d4557
commit 0155252823
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 327 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
CMakeLists.txt
View file

@ -533,6 +533,8 @@ endforeach()
set(NEEDED_FUNCTIONS
_configthreadlocale
canonicalize_file_name
copy_file_range
copyfile
daemon
fallocate64
flock
@ -549,6 +551,7 @@ set(NEEDED_FUNCTIONS
posix_fallocate
pread
pwrite
sendfile64
statvfs
strcasestr
strlcpy

141
libtransmission/file-posix.c
View file

@ -14,6 +14,7 @@
#include <fcntl.h> /* O_LARGEFILE, posix_fadvise(), [posix_]fallocate(), fcntl() */
#include <libgen.h> /* basename(), dirname() */
#include <limits.h> /* PATH_MAX */
#include <stdint.h> /* SIZE_MAX */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@ -27,6 +28,27 @@
#include <xfs/xfs.h>
#endif
/* OS-specific file copy (copy_file_range, sendfile64, or copyfile). */
#if defined(__linux__)
# include <linux/version.h>
/* Linux's copy_file_range(2) is buggy prior to 5.3. */
# if defined(HAVE_COPY_FILE_RANGE) && LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0)
# define USE_COPY_FILE_RANGE
# elif defined(HAVE_SENDFILE64)
# include <sys/sendfile.h>
# define USE_SENDFILE64
# endif
#elif defined(__APPLE__) && defined(HAVE_COPYFILE)
# include <copyfile.h>
# ifndef COPYFILE_CLONE /* macos < 10.12 */
# define COPYFILE_CLONE 0
# endif
# define USE_COPYFILE
#elif defined(HAVE_COPY_FILE_RANGE)
/* Presently this is only FreeBSD 13+. */
# define USE_COPY_FILE_RANGE
#endif /* __linux__ */
#include "transmission.h"
#include "error.h"
#include "file.h"
@ -415,6 +437,125 @@ bool tr_sys_path_rename(char const* src_path, char const* dst_path, tr_error** e
return ret;
}
/* We try to do a fast (in-kernel) copy using a variety of non-portable system
* calls. If the current implementation does not support in-kernel copying, we
* use a user-space fallback instead. */
bool tr_sys_path_copy(char const* src_path, char const* dst_path, tr_error** error)
{
TR_ASSERT(src_path != NULL);
TR_ASSERT(dst_path != NULL);
#if defined(USE_COPYFILE)
if (copyfile(src_path, dst_path, NULL, COPYFILE_CLONE | COPYFILE_ALL) < 0)
{
set_system_error(error, errno);
return false;
}
return true;
#else /* USE_COPYFILE */
/* Other OSes require us to copy between file descriptors, so open them. */
tr_sys_file_t in = tr_sys_file_open(src_path, TR_SYS_FILE_READ | TR_SYS_FILE_SEQUENTIAL, 0, error);
if (in == TR_BAD_SYS_FILE)
{
tr_error_prefix(error, "Unable to open source file: ");
return false;
}
tr_sys_path_info info;
if (!tr_sys_file_get_info(in, &info, error))
{
tr_error_prefix(error, "Unable to get information on source file: ");
tr_sys_file_close(in, NULL);
return false;
}
tr_sys_file_t out = tr_sys_file_open(dst_path, TR_SYS_FILE_WRITE | TR_SYS_FILE_CREATE | TR_SYS_FILE_TRUNCATE, 0666, error);
if (out == TR_BAD_SYS_FILE)
{
tr_error_prefix(error, "Unable to open destination file: ");
tr_sys_file_close(in, NULL);
return false;
}
uint64_t file_size = info.size;
#if defined(USE_COPY_FILE_RANGE) || defined(USE_SENDFILE64)
while (file_size > 0)
{
size_t const chunk_size = MIN(file_size, SSIZE_MAX);
ssize_t const copied =
#ifdef USE_COPY_FILE_RANGE
copy_file_range(in, NULL, out, NULL, chunk_size, 0);
#elif defined(USE_SENDFILE64)
sendfile64(out, in, NULL, chunk_size);
#else
#error File copy mechanism not implemented.
#endif
TR_ASSERT(copied == -1 || copied >= 0); /* -1 for error; some non-negative value otherwise. */
if (copied == -1)
{
set_system_error(error, errno);
break;
}
TR_ASSERT(copied >= 0 && ((uint64_t)copied) <= file_size);
TR_ASSERT(copied >= 0 && ((uint64_t)copied) <= chunk_size);
file_size -= copied;
}
#else /* USE_COPY_FILE_RANGE || USE_SENDFILE64 */
/* Fallback to user-space copy. */
size_t const buflen = 1024 * 1024; /* 1024 KiB buffer */
char* buf = tr_malloc(buflen);
while (file_size > 0)
{
uint64_t const chunk_size = MIN(file_size, buflen);
uint64_t bytes_read;
uint64_t bytes_written;
if (!tr_sys_file_read(in, buf, chunk_size, &bytes_read, error))
{
break;
}
if (!tr_sys_file_write(out, buf, bytes_read, &bytes_written, error))
{
break;
}
TR_ASSERT(bytes_read == bytes_written);
TR_ASSERT(bytes_written <= file_size);
file_size -= bytes_written;
}
/* cleanup */
tr_free(buf);
#endif /* USE_COPY_FILE_RANGE || USE_SENDFILE64 */
/* cleanup */
tr_sys_file_close(out, NULL);
tr_sys_file_close(in, NULL);
if (file_size != 0)
{
tr_error_prefix(error, "Unable to read/write: ");
return false;
}
return true;
#endif /* USE_COPYFILE */
}
bool tr_sys_path_remove(char const* path, tr_error** error)
{
TR_ASSERT(path != NULL);

35
libtransmission/file-win32.c
View file

@ -733,6 +733,41 @@ bool tr_sys_path_rename(char const* src_path, char const* dst_path, tr_error** e
return ret;
}
bool tr_sys_path_copy(char const* src_path, char const* dst_path, tr_error** error)
{
TR_ASSERT(src_path != NULL);
TR_ASSERT(dst_path != NULL);
bool ret = false;
wchar_t* wide_src_path = path_to_native_path(src_path);
wchar_t* wide_dst_path = path_to_native_path(dst_path);
if (wide_src_path == NULL || wide_dst_path == NULL)
{
set_system_error(error, ERROR_INVALID_PARAMETER);
goto out;
}
LPBOOL cancel = FALSE;
DWORD const flags = COPY_FILE_ALLOW_DECRYPTED_DESTINATION | COPY_FILE_FAIL_IF_EXISTS;
if (CopyFileExW(wide_src_path, wide_dst_path, NULL, NULL, &cancel, flags) == 0)
{
set_system_error(error, GetLastError());
goto out;
}
else
{
ret = true;
}
out:
tr_free(wide_src_path);
tr_free(wide_dst_path);
return ret;
}
bool tr_sys_path_remove(char const* path, tr_error** error)
{
TR_ASSERT(path != NULL);

13
libtransmission/file.h
View file

@ -144,6 +144,19 @@ tr_sys_path_info;
/* Path-related wrappers */
/**
* @brief Portability wrapper for various in-kernel file copy functions, with a
* fallback to a userspace read/write loop.
*
* @param[in] src_path Path to source file.
* @param[in] dst_path Path to destination file.
* @param[out] error Pointer to error object. Optional, pass `NULL` if you
* are not interested in error details.
*
* @return `True` on success, `false` otherwise (with `error` set accordingly).
*/
bool tr_sys_path_copy(char const* src_path, char const* dst_path, struct tr_error** error);
/**
* @brief Portability wrapper for `stat()`.
*

58
libtransmission/utils.c
View file

@ -15,6 +15,7 @@
#include <float.h> /* DBL_DIG */
#include <locale.h> /* localeconv() */
#include <math.h> /* fabs(), floor() */
#include <stdint.h> /* SIZE_MAX */
#include <stdio.h>
#include <stdlib.h> /* getenv() */
#include <string.h> /* strerror(), memset(), memmem() */
@ -1719,12 +1720,7 @@ char* tr_strratio(char* buf, size_t buflen, double ratio, char const* infinity)
bool tr_moveFile(char const* oldpath, char const* newpath, tr_error** error)
{
tr_sys_file_t in;
tr_sys_file_t out;
char* buf = NULL;
tr_sys_path_info info;
uint64_t bytesLeft;
size_t const buflen = 1024 * 1024; // 1024 KiB buffer
/* make sure the old file exists */
if (!tr_sys_path_get_info(oldpath, 0, &info, error))
@ -1758,56 +1754,10 @@ bool tr_moveFile(char const* oldpath, char const* newpath, tr_error** error)
return true;
}
/* copy the file */
in = tr_sys_file_open(oldpath, TR_SYS_FILE_READ | TR_SYS_FILE_SEQUENTIAL, 0, error);
if (in == TR_BAD_SYS_FILE)
/* Otherwise, copy the file. */
if (!tr_sys_path_copy(oldpath, newpath, error))
{
tr_error_prefix(error, "Unable to open old file: ");
return false;
}
out = tr_sys_file_open(newpath, TR_SYS_FILE_WRITE | TR_SYS_FILE_CREATE | TR_SYS_FILE_TRUNCATE, 0666, error);
if (out == TR_BAD_SYS_FILE)
{
tr_error_prefix(error, "Unable to open new file: ");
tr_sys_file_close(in, NULL);
return false;
}
buf = tr_malloc(buflen);
bytesLeft = info.size;
while (bytesLeft > 0)
{
uint64_t const bytesThisPass = MIN(bytesLeft, buflen);
uint64_t numRead;
uint64_t bytesWritten;
if (!tr_sys_file_read(in, buf, bytesThisPass, &numRead, error))
{
break;
}
if (!tr_sys_file_write(out, buf, numRead, &bytesWritten, error))
{
break;
}
TR_ASSERT(numRead == bytesWritten);
TR_ASSERT(bytesWritten <= bytesLeft);
bytesLeft -= bytesWritten;
}
/* cleanup */
tr_free(buf);
tr_sys_file_close(out, NULL);
tr_sys_file_close(in, NULL);
if (bytesLeft != 0)
{
tr_error_prefix(error, "Unable to read/write: ");
tr_error_prefix(error, "Unable to copy: ");
return false;
}

2
tests/libtransmission/CMakeLists.txt
View file

@ -28,7 +28,7 @@ include_directories(
set_property(DIRECTORY PROPERTY FOLDER "UnitTests")
set_property(DIRECTORY PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreaded")
foreach(T bitfield blocklist clients crypto error file getopt
foreach(T bitfield blocklist clients copy crypto error file getopt
history json magnet makemeta metainfo move peer-msgs
quark rename rpc session subprocess utils variant watchdir)
set(TP libtransmission-test-${T})

130
tests/libtransmission/copy-test.cc Normal file
View file

@ -0,0 +1,130 @@
/*
* This file copyright Transmission authors and contributors
*
* It may be used under the GNU GPL versions 2 or 3
* or any future license endorsed by Mnemosyne LLC.
*
*/
#include "transmission.h"
#include "error.h"
#include "file.h"
#include "test-fixtures.h"
namespace libtransmission
{
namespace test
{
class CopyTest : public SandboxedTest
{
protected:
void testImpl(char const* filename1, char const* filename2, size_t const file_length)
{
auto const path1 = tr_buildPath(sandboxDir().data(), filename1, nullptr);
/* Create a file. */
char* file_content = static_cast<char*>(tr_malloc(file_length));
tr_rand_buffer(file_content, file_length);
createFileWithContents(path1, file_content, file_length);
tr_free(file_content);
auto const path2 = tr_buildPath(sandboxDir().data(), filename2, nullptr);
tr_error* err = nullptr;
/* Copy it. */
EXPECT_TRUE(tr_sys_path_copy(path1, path2, &err));
EXPECT_EQ(nullptr, err);
tr_error_clear(&err);
EXPECT_TRUE(filesAreIdentical(path1, path2));
/* Dispose of those files that we created. */
tr_sys_path_remove(path1, nullptr);
tr_free(path1);
tr_sys_path_remove(path2, nullptr);
tr_free(path2);
}
private:
uint64_t fillBufferFromFd(tr_sys_file_t fd, uint64_t bytes_remaining, char* buf, size_t buf_len)
{
memset(buf, 0, buf_len);
size_t buf_pos = 0;
while (buf_pos < buf_len && bytes_remaining > 0)
{
uint64_t const chunk_size = MIN(buf_len - buf_pos, bytes_remaining);
uint64_t bytes_read = 0;
tr_sys_file_read(fd, buf + buf_pos, chunk_size, &bytes_read, nullptr);
EXPECT_LE(buf_pos + bytes_read, buf_len);
EXPECT_LE(bytes_read, bytes_remaining);
buf_pos += bytes_read;
bytes_remaining -= bytes_read;
}
return bytes_remaining;
}
bool filesAreIdentical(char const* fn1, char const* fn2)
{
tr_sys_file_t fd1 = tr_sys_file_open(fn1, TR_SYS_FILE_READ | TR_SYS_FILE_SEQUENTIAL, 0, nullptr);
tr_sys_file_t fd2 = tr_sys_file_open(fn2, TR_SYS_FILE_READ | TR_SYS_FILE_SEQUENTIAL, 0, nullptr);
EXPECT_NE(fd1, TR_BAD_SYS_FILE);
EXPECT_NE(fd2, TR_BAD_SYS_FILE);
tr_sys_path_info info1;
tr_sys_path_info info2;
tr_sys_file_get_info(fd1, &info1, nullptr);
tr_sys_file_get_info(fd2, &info2, nullptr);
EXPECT_EQ(info1.size, info2.size);
uint64_t bytes_left1 = info1.size;
uint64_t bytes_left2 = info2.size;
size_t const buflen = 2 * 1024 * 1024; /* 2 MiB buffer */
char* readbuf1 = static_cast<char*>(tr_malloc(buflen));
char* readbuf2 = static_cast<char*>(tr_malloc(buflen));
while (bytes_left1 > 0 || bytes_left2 > 0)
{
bytes_left1 = fillBufferFromFd(fd1, bytes_left1, readbuf1, buflen);
bytes_left2 = fillBufferFromFd(fd2, bytes_left2, readbuf2, buflen);
if (bytes_left1 != bytes_left2)
{
return false;
}
if (memcmp(readbuf1, readbuf2, buflen) != 0)
{
return false;
}
}
tr_free(readbuf1);
tr_free(readbuf2);
tr_sys_file_close(fd1, nullptr);
tr_sys_file_close(fd2, nullptr);
return true;
}
};
TEST_F(CopyTest, copy)
{
char const* filename1 = "orig-blob.txt";
char const* filename2 = "copy-blob.txt";
auto const random_file_length = 1024 * 1024 * 10;
testImpl(filename1, filename2, random_file_length);
}
} // namespace test
} // namespace libtransmission