Add in-kernel file copying for several platforms. (#1092)

* Add in-kernel copying support for Linux (sendfile64(2), copy_file_range(2)), FreeBSD 13 (copy_file_range(2)), MacOS (copyfile(2)), and Windows (CopyFileExA).

* Fix macro name USE_COPY_FILE_RANGE.

* Minor bugfixes for userspace fallback.

* Fix linux sendfile64 bugs.

* Remove some overzealous asserts.

* Allow transmission-test-copy to take an optional argument for an external reference file.

* Fix return value error of tr_sys_path_copy.

* Use COPYFILE_ALL for Macs without COPYFILE_CLONE.

* Add in-kernel file copying for several platforms.

Numerous operating systems now have support for copying files directly in the
kernel, which is generally more efficient than copying in a userspace read(2)/
write(2) loop. (This becomes particularly relevant for 4th gen PCI-E storage,
which approaches the latency of DRAM.) For Linux I use sendfile64(2), and, for
later kernels, copy_file_range(2). FreeBSD 13 will also support
copy_file_range(2). MacOS has copyfile(2), and Windows has CopyFileExA.
Operating systems lacking such a syscall continue to use the existing
read(2)/write(2) loop.

* Appease uncrustify.

* Appease uncrustify.

* copy-test: generate random content at run time.

* copy-test: Stylistic changes and more check()s.

* copy-test: files_are_identical should follow test idioms

* tr_sys_path_copy: numerous tweaks as requested by review.

* s/old file/source file; s/new file/destination file.

* tr_sys_path_copy: handle win32 wide characters in paths.

* Uncrustify.

* test-copy: Use non-string create_file_with_contents.

* tr_sys_path_copy: numerous fixes.

Per review: generate test file content at runtime; tidy use of check();
fix style; re-measure file sizes in the copy; define a macro when the
system does not provide it; use Unicode APIs on Windows; and fix
documentation.

* Updated as per comments.

* Rebase kernel-copy changes onto 3.0 with gtest.

* Undo irrelevant comment change.

* Fix syntax error.

* Use tr_malloc() instead of tr_valloc().

* Use EXPECT instead of TR_ASSERT in gtest.

* Add error handling.

* Acceptable coding style has changed again.

Now it's camelCase. Also use nullptr instead of NULL, etc.

* Fix east/west const.

Co-authored-by: Mike Gelfand <mikedld@users.noreply.github.com>

CMakeLists.txt

@@ -533,6 +533,8 @@ endforeach()
 set(NEEDED_FUNCTIONS
     _configthreadlocale
     canonicalize_file_name
+    copy_file_range
+    copyfile
     daemon
     fallocate64
     flock
@@ -549,6 +551,7 @@ set(NEEDED_FUNCTIONS
     posix_fallocate
     pread
     pwrite
+    sendfile64
     statvfs
     strcasestr
     strlcpy

libtransmission/file-posix.c

@@ -14,6 +14,7 @@
 #include <fcntl.h> /* O_LARGEFILE, posix_fadvise(), [posix_]fallocate(), fcntl() */
 #include <libgen.h> /* basename(), dirname() */
 #include <limits.h> /* PATH_MAX */
+#include <stdint.h> /* SIZE_MAX */
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -27,6 +28,27 @@
 #include <xfs/xfs.h>
 #endif
 
+/* OS-specific file copy (copy_file_range, sendfile64, or copyfile). */
+#if defined(__linux__)
+#   include <linux/version.h>
+/* Linux's copy_file_range(2) is buggy prior to 5.3. */
+#   if defined(HAVE_COPY_FILE_RANGE) && LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0)
+#       define USE_COPY_FILE_RANGE
+#   elif defined(HAVE_SENDFILE64)
+#       include <sys/sendfile.h>
+#       define USE_SENDFILE64
+#   endif
+#elif defined(__APPLE__) && defined(HAVE_COPYFILE)
+#   include <copyfile.h>
+#   ifndef COPYFILE_CLONE /* macos < 10.12 */
+#       define COPYFILE_CLONE 0
+#   endif
+#   define USE_COPYFILE
+#elif defined(HAVE_COPY_FILE_RANGE)
+/* Presently this is only FreeBSD 13+. */
+#   define USE_COPY_FILE_RANGE
+#endif /* __linux__ */
+
 #include "transmission.h"
 #include "error.h"
 #include "file.h"
@@ -415,6 +437,125 @@ bool tr_sys_path_rename(char const* src_path, char const* dst_path, tr_error** e
     return ret;
 }
 
+/* We try to do a fast (in-kernel) copy using a variety of non-portable system
+ * calls. If the current implementation does not support in-kernel copying, we
+ * use a user-space fallback instead. */
+bool tr_sys_path_copy(char const* src_path, char const* dst_path, tr_error** error)
+{
+    TR_ASSERT(src_path != NULL);
+    TR_ASSERT(dst_path != NULL);
+
+#if defined(USE_COPYFILE)
+    if (copyfile(src_path, dst_path, NULL, COPYFILE_CLONE | COPYFILE_ALL) < 0)
+    {
+        set_system_error(error, errno);
+        return false;
+    }
+
+    return true;
+
+#else /* USE_COPYFILE */
+
+    /* Other OSes require us to copy between file descriptors, so open them. */
+    tr_sys_file_t in = tr_sys_file_open(src_path, TR_SYS_FILE_READ | TR_SYS_FILE_SEQUENTIAL, 0, error);
+    if (in == TR_BAD_SYS_FILE)
+    {
+        tr_error_prefix(error, "Unable to open source file: ");
+        return false;
+    }
+
+    tr_sys_path_info info;
+    if (!tr_sys_file_get_info(in, &info, error))
+    {
+        tr_error_prefix(error, "Unable to get information on source file: ");
+        tr_sys_file_close(in, NULL);
+        return false;
+    }
+
+    tr_sys_file_t out = tr_sys_file_open(dst_path, TR_SYS_FILE_WRITE | TR_SYS_FILE_CREATE | TR_SYS_FILE_TRUNCATE, 0666, error);
+    if (out == TR_BAD_SYS_FILE)
+    {
+        tr_error_prefix(error, "Unable to open destination file: ");
+        tr_sys_file_close(in, NULL);
+        return false;
+    }
+
+    uint64_t file_size = info.size;
+
+#if defined(USE_COPY_FILE_RANGE) || defined(USE_SENDFILE64)
+
+    while (file_size > 0)
+    {
+        size_t const chunk_size = MIN(file_size, SSIZE_MAX);
+        ssize_t const copied =
+#ifdef USE_COPY_FILE_RANGE
+            copy_file_range(in, NULL, out, NULL, chunk_size, 0);
+#elif defined(USE_SENDFILE64)
+            sendfile64(out, in, NULL, chunk_size);
+#else
+#error File copy mechanism not implemented.
+#endif
+        TR_ASSERT(copied == -1 || copied >= 0); /* -1 for error; some non-negative value otherwise. */
+
+        if (copied == -1)
+        {
+            set_system_error(error, errno);
+            break;
+        }
+
+        TR_ASSERT(copied >= 0 && ((uint64_t)copied) <= file_size);
+        TR_ASSERT(copied >= 0 && ((uint64_t)copied) <= chunk_size);
+        file_size -= copied;
+    }
+
+#else /* USE_COPY_FILE_RANGE || USE_SENDFILE64 */
+
+    /* Fallback to user-space copy. */
+
+    size_t const buflen = 1024 * 1024; /* 1024 KiB buffer */
+    char* buf = tr_malloc(buflen);
+
+    while (file_size > 0)
+    {
+        uint64_t const chunk_size = MIN(file_size, buflen);
+        uint64_t bytes_read;
+        uint64_t bytes_written;
+
+        if (!tr_sys_file_read(in, buf, chunk_size, &bytes_read, error))
+        {
+            break;
+        }
+
+        if (!tr_sys_file_write(out, buf, bytes_read, &bytes_written, error))
+        {
+            break;
+        }
+
+        TR_ASSERT(bytes_read == bytes_written);
+        TR_ASSERT(bytes_written <= file_size);
+        file_size -= bytes_written;
+    }
+
+    /* cleanup */
+    tr_free(buf);
+
+#endif /* USE_COPY_FILE_RANGE || USE_SENDFILE64 */
+
+    /* cleanup */
+    tr_sys_file_close(out, NULL);
+    tr_sys_file_close(in, NULL);
+
+    if (file_size != 0)
+    {
+        tr_error_prefix(error, "Unable to read/write: ");
+        return false;
+    }
+
+    return true;
+
+#endif /* USE_COPYFILE */
+}
+
 bool tr_sys_path_remove(char const* path, tr_error** error)
 {
     TR_ASSERT(path != NULL);

libtransmission/file-win32.c

@@ -733,6 +733,41 @@ bool tr_sys_path_rename(char const* src_path, char const* dst_path, tr_error** e
     return ret;
 }
 
+bool tr_sys_path_copy(char const* src_path, char const* dst_path, tr_error** error)
+{
+    TR_ASSERT(src_path != NULL);
+    TR_ASSERT(dst_path != NULL);
+
+    bool ret = false;
+
+    wchar_t* wide_src_path = path_to_native_path(src_path);
+    wchar_t* wide_dst_path = path_to_native_path(dst_path);
+
+    if (wide_src_path == NULL || wide_dst_path == NULL)
+    {
+        set_system_error(error, ERROR_INVALID_PARAMETER);
+        goto out;
+    }
+
+    LPBOOL cancel = FALSE;
+    DWORD const flags = COPY_FILE_ALLOW_DECRYPTED_DESTINATION | COPY_FILE_FAIL_IF_EXISTS;
+    if (CopyFileExW(wide_src_path, wide_dst_path, NULL, NULL, &cancel, flags) == 0)
+    {
+        set_system_error(error, GetLastError());
+        goto out;
+    }
+    else
+    {
+        ret = true;
+    }
+
+out:
+    tr_free(wide_src_path);
+    tr_free(wide_dst_path);
+
+    return ret;
+}
+
 bool tr_sys_path_remove(char const* path, tr_error** error)
 {
     TR_ASSERT(path != NULL);

libtransmission/file.h

@@ -144,6 +144,19 @@ tr_sys_path_info;
 
 /* Path-related wrappers */
 
+/**
+ * @brief Portability wrapper for various in-kernel file copy functions, with a
+ *        fallback to a userspace read/write loop.
+ *
+ * @param[in]  src_path  Path to source file.
+ * @param[in]  dst_path  Path to destination file.
+ * @param[out] error     Pointer to error object. Optional, pass `NULL` if you
+ *                       are not interested in error details.
+ *
+ * @return `True` on success, `false` otherwise (with `error` set accordingly).
+ */
+bool tr_sys_path_copy(char const* src_path, char const* dst_path, struct tr_error** error);
+
 /**
  * @brief Portability wrapper for `stat()`.
  *

libtransmission/utils.c

@@ -15,6 +15,7 @@
 #include <float.h> /* DBL_DIG */
 #include <locale.h> /* localeconv() */
 #include <math.h> /* fabs(), floor() */
+#include <stdint.h> /* SIZE_MAX */
 #include <stdio.h>
 #include <stdlib.h> /* getenv() */
 #include <string.h> /* strerror(), memset(), memmem() */
@@ -1719,12 +1720,7 @@ char* tr_strratio(char* buf, size_t buflen, double ratio, char const* infinity)
 
 bool tr_moveFile(char const* oldpath, char const* newpath, tr_error** error)
 {
-    tr_sys_file_t in;
-    tr_sys_file_t out;
-    char* buf = NULL;
     tr_sys_path_info info;
-    uint64_t bytesLeft;
-    size_t const buflen = 1024 * 1024; // 1024 KiB buffer
 
     /* make sure the old file exists */
     if (!tr_sys_path_get_info(oldpath, 0, &info, error))
@@ -1758,56 +1754,10 @@ bool tr_moveFile(char const* oldpath, char const* newpath, tr_error** error)
         return true;
     }
 
-    /* copy the file */
+    /* Otherwise, copy the file. */
-    in = tr_sys_file_open(oldpath, TR_SYS_FILE_READ | TR_SYS_FILE_SEQUENTIAL, 0, error);
+    if (!tr_sys_path_copy(oldpath, newpath, error))
-
-    if (in == TR_BAD_SYS_FILE)
     {
-        tr_error_prefix(error, "Unable to open old file: ");
+        tr_error_prefix(error, "Unable to copy: ");
-        return false;
-    }
-
-    out = tr_sys_file_open(newpath, TR_SYS_FILE_WRITE | TR_SYS_FILE_CREATE | TR_SYS_FILE_TRUNCATE, 0666, error);
-
-    if (out == TR_BAD_SYS_FILE)
-    {
-        tr_error_prefix(error, "Unable to open new file: ");
-        tr_sys_file_close(in, NULL);
-        return false;
-    }
-
-    buf = tr_malloc(buflen);
-    bytesLeft = info.size;
-
-    while (bytesLeft > 0)
-    {
-        uint64_t const bytesThisPass = MIN(bytesLeft, buflen);
-        uint64_t numRead;
-        uint64_t bytesWritten;
-
-        if (!tr_sys_file_read(in, buf, bytesThisPass, &numRead, error))
-        {
-            break;
-        }
-
-        if (!tr_sys_file_write(out, buf, numRead, &bytesWritten, error))
-        {
-            break;
-        }
-
-        TR_ASSERT(numRead == bytesWritten);
-        TR_ASSERT(bytesWritten <= bytesLeft);
-        bytesLeft -= bytesWritten;
-    }
-
-    /* cleanup */
-    tr_free(buf);
-    tr_sys_file_close(out, NULL);
-    tr_sys_file_close(in, NULL);
-
-    if (bytesLeft != 0)
-    {
-        tr_error_prefix(error, "Unable to read/write: ");
         return false;
     }
 

tests/libtransmission/CMakeLists.txt

@@ -28,7 +28,7 @@ include_directories(
 set_property(DIRECTORY PROPERTY FOLDER "UnitTests")
 set_property(DIRECTORY PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreaded")
 
-foreach(T bitfield blocklist clients crypto error file getopt
+foreach(T bitfield blocklist clients copy crypto error file getopt
           history json magnet makemeta metainfo move peer-msgs
           quark rename rpc session subprocess utils variant watchdir)
   set(TP libtransmission-test-${T})

tests/libtransmission/copy-test.cc

@@ -0,0 +1,130 @@
+/*
+ * This file copyright Transmission authors and contributors
+ *
+ * It may be used under the GNU GPL versions 2 or 3
+ * or any future license endorsed by Mnemosyne LLC.
+ *
+ */
+
+#include "transmission.h"
+#include "error.h"
+#include "file.h"
+
+#include "test-fixtures.h"
+
+namespace libtransmission
+{
+
+namespace test
+{
+
+class CopyTest : public SandboxedTest
+{
+protected:
+    void testImpl(char const* filename1, char const* filename2, size_t const file_length)
+    {
+        auto const path1 = tr_buildPath(sandboxDir().data(), filename1, nullptr);
+
+        /* Create a file. */
+        char* file_content = static_cast<char*>(tr_malloc(file_length));
+        tr_rand_buffer(file_content, file_length);
+        createFileWithContents(path1, file_content, file_length);
+        tr_free(file_content);
+
+        auto const path2 = tr_buildPath(sandboxDir().data(), filename2, nullptr);
+
+        tr_error* err = nullptr;
+        /* Copy it. */
+        EXPECT_TRUE(tr_sys_path_copy(path1, path2, &err));
+        EXPECT_EQ(nullptr, err);
+        tr_error_clear(&err);
+
+        EXPECT_TRUE(filesAreIdentical(path1, path2));
+
+        /* Dispose of those files that we created. */
+        tr_sys_path_remove(path1, nullptr);
+        tr_free(path1);
+
+        tr_sys_path_remove(path2, nullptr);
+        tr_free(path2);
+    }
+
+private:
+    uint64_t fillBufferFromFd(tr_sys_file_t fd, uint64_t bytes_remaining, char* buf, size_t buf_len)
+    {
+        memset(buf, 0, buf_len);
+
+        size_t buf_pos = 0;
+        while (buf_pos < buf_len && bytes_remaining > 0)
+        {
+            uint64_t const chunk_size = MIN(buf_len - buf_pos, bytes_remaining);
+            uint64_t bytes_read = 0;
+
+            tr_sys_file_read(fd, buf + buf_pos, chunk_size, &bytes_read, nullptr);
+
+            EXPECT_LE(buf_pos + bytes_read, buf_len);
+            EXPECT_LE(bytes_read, bytes_remaining);
+            buf_pos += bytes_read;
+            bytes_remaining -= bytes_read;
+        }
+
+        return bytes_remaining;
+    }
+
+    bool filesAreIdentical(char const* fn1, char const* fn2)
+    {
+        tr_sys_file_t fd1 = tr_sys_file_open(fn1, TR_SYS_FILE_READ | TR_SYS_FILE_SEQUENTIAL, 0, nullptr);
+        tr_sys_file_t fd2 = tr_sys_file_open(fn2, TR_SYS_FILE_READ | TR_SYS_FILE_SEQUENTIAL, 0, nullptr);
+        EXPECT_NE(fd1, TR_BAD_SYS_FILE);
+        EXPECT_NE(fd2, TR_BAD_SYS_FILE);
+
+        tr_sys_path_info info1;
+        tr_sys_path_info info2;
+        tr_sys_file_get_info(fd1, &info1, nullptr);
+        tr_sys_file_get_info(fd2, &info2, nullptr);
+        EXPECT_EQ(info1.size, info2.size);
+
+        uint64_t bytes_left1 = info1.size;
+        uint64_t bytes_left2 = info2.size;
+
+        size_t const buflen = 2 * 1024 * 1024; /* 2 MiB buffer */
+        char* readbuf1 = static_cast<char*>(tr_malloc(buflen));
+        char* readbuf2 = static_cast<char*>(tr_malloc(buflen));
+
+        while (bytes_left1 > 0 || bytes_left2 > 0)
+        {
+            bytes_left1 = fillBufferFromFd(fd1, bytes_left1, readbuf1, buflen);
+            bytes_left2 = fillBufferFromFd(fd2, bytes_left2, readbuf2, buflen);
+
+            if (bytes_left1 != bytes_left2)
+            {
+                return false;
+            }
+
+            if (memcmp(readbuf1, readbuf2, buflen) != 0)
+            {
+                return false;
+            }
+        }
+
+        tr_free(readbuf1);
+        tr_free(readbuf2);
+        tr_sys_file_close(fd1, nullptr);
+        tr_sys_file_close(fd2, nullptr);
+
+        return true;
+    }
+};
+
+TEST_F(CopyTest, copy)
+{
+    char const* filename1 = "orig-blob.txt";
+    char const* filename2 = "copy-blob.txt";
+    auto const random_file_length = 1024 * 1024 * 10;
+
+    testImpl(filename1, filename2, random_file_length);
+}
+
+} // namespace test
+
+} // namespace libtransmission