mirror
/
transmission
mirror of https://github.com/transmission/transmission
1
0
Fork 0
Code Issues Releases Wiki Activity

refactor: replace tr_fdlimit with tr_open_files (#3016)

This commit is contained in:
Charles Kerr 2022-04-28 21:35:47 -05:00 committed by GitHub
parent 2f16e4a143
commit 5dbe1f4669
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 736 additions and 535 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
Transmission.xcodeproj/project.pbxproj
View File

@ -314,8 +314,8 @@
BEFC1E4D0C07861A00B0BB3C /* session.h in Headers */ = {isa = PBXBuildFile; fileRef = BEFC1E140C07861A00B0BB3C /* session.h */; };
BEFC1E4E0C07861A00B0BB3C /* inout.h in Headers */ = {isa = PBXBuildFile; fileRef = BEFC1E150C07861A00B0BB3C /* inout.h */; };
BEFC1E4F0C07861A00B0BB3C /* inout.cc in Sources */ = {isa = PBXBuildFile; fileRef = BEFC1E160C07861A00B0BB3C /* inout.cc */; };
BEFC1E520C07861A00B0BB3C /* fdlimit.h in Headers */ = {isa = PBXBuildFile; fileRef = BEFC1E190C07861A00B0BB3C /* fdlimit.h */; };
BEFC1E530C07861A00B0BB3C /* fdlimit.cc in Sources */ = {isa = PBXBuildFile; fileRef = BEFC1E1A0C07861A00B0BB3C /* fdlimit.cc */; };
BEFC1E520C07861A00B0BB3C /* open-files.h in Headers */ = {isa = PBXBuildFile; fileRef = BEFC1E190C07861A00B0BB3C /* open-files.h */; };
BEFC1E530C07861A00B0BB3C /* open-files.cc in Sources */ = {isa = PBXBuildFile; fileRef = BEFC1E1A0C07861A00B0BB3C /* open-files.cc */; };
BEFC1E550C07861A00B0BB3C /* completion.h in Headers */ = {isa = PBXBuildFile; fileRef = BEFC1E1C0C07861A00B0BB3C /* completion.h */; };
BEFC1E560C07861A00B0BB3C /* completion.cc in Sources */ = {isa = PBXBuildFile; fileRef = BEFC1E1D0C07861A00B0BB3C /* completion.cc */; };
BEFC1E570C07861A00B0BB3C /* clients.h in Headers */ = {isa = PBXBuildFile; fileRef = BEFC1E1E0C07861A00B0BB3C /* clients.h */; };
@ -1089,8 +1089,8 @@
BEFC1E140C07861A00B0BB3C /* session.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = session.h; sourceTree = "<group>"; };
BEFC1E150C07861A00B0BB3C /* inout.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = inout.h; sourceTree = "<group>"; };
BEFC1E160C07861A00B0BB3C /* inout.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = inout.cc; sourceTree = "<group>"; };
BEFC1E190C07861A00B0BB3C /* fdlimit.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = fdlimit.h; sourceTree = "<group>"; };
BEFC1E1A0C07861A00B0BB3C /* fdlimit.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = fdlimit.cc; sourceTree = "<group>"; };
BEFC1E190C07861A00B0BB3C /* open-files.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = open-files.h; sourceTree = "<group>"; };
BEFC1E1A0C07861A00B0BB3C /* open-files.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = open-files.cc; sourceTree = "<group>"; };
BEFC1E1C0C07861A00B0BB3C /* completion.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = completion.h; sourceTree = "<group>"; };
BEFC1E1D0C07861A00B0BB3C /* completion.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = completion.cc; sourceTree = "<group>"; };
BEFC1E1E0C07861A00B0BB3C /* clients.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = clients.h; sourceTree = "<group>"; };
@ -1716,8 +1716,8 @@
BEFC1E0F0C07861A00B0BB3C /* natpmp.cc */,
BEFC1E150C07861A00B0BB3C /* inout.h */,
BEFC1E160C07861A00B0BB3C /* inout.cc */,
BEFC1E190C07861A00B0BB3C /* fdlimit.h */,
BEFC1E1A0C07861A00B0BB3C /* fdlimit.cc */,
BEFC1E190C07861A00B0BB3C /* open-files.h */,
BEFC1E1A0C07861A00B0BB3C /* open-files.cc */,
BEFC1E1C0C07861A00B0BB3C /* completion.h */,
BEFC1E1D0C07861A00B0BB3C /* completion.cc */,
BEFC1E1E0C07861A00B0BB3C /* clients.h */,
@ -2180,7 +2180,7 @@
BEFC1E4D0C07861A00B0BB3C /* session.h in Headers */,
C1FEE5771C3223CC00D62832 /* watchdir-common.h in Headers */,
BEFC1E4E0C07861A00B0BB3C /* inout.h in Headers */,
BEFC1E520C07861A00B0BB3C /* fdlimit.h in Headers */,
BEFC1E520C07861A00B0BB3C /* open-files.h in Headers */,
ED8A163F2735A8AA000D61F9 /* peer-mgr-active-requests.h in Headers */,
BEFC1E550C07861A00B0BB3C /* completion.h in Headers */,
BEFC1E570C07861A00B0BB3C /* clients.h in Headers */,
@ -2939,7 +2939,7 @@
BEFC1E480C07861A00B0BB3C /* natpmp.cc in Sources */,
C1077A4E183EB29600634C22 /* error.cc in Sources */,
BEFC1E4F0C07861A00B0BB3C /* inout.cc in Sources */,
BEFC1E530C07861A00B0BB3C /* fdlimit.cc in Sources */,
BEFC1E530C07861A00B0BB3C /* open-files.cc in Sources */,
C1FEE5781C3223CC00D62832 /* watchdir-generic.cc in Sources */,
BEFC1E560C07861A00B0BB3C /* completion.cc in Sources */,
BEFC1E580C07861A00B0BB3C /* clients.cc in Sources */,

5
libtransmission/CMakeLists.txt
View File

@ -25,7 +25,6 @@ set(PROJECT_FILES
crypto-utils.cc
crypto.cc
error.cc
fdlimit.cc
file-info.cc
file-piece-map.cc
file-posix.cc
@ -38,6 +37,7 @@ set(PROJECT_FILES
makemeta.cc
natpmp.cc
net.cc
open-files.cc
peer-io.cc
peer-mgr-active-requests.cc
peer-mgr-wishlist.cc
@ -168,16 +168,17 @@ set(${PROJECT_NAME}_PRIVATE_HEADERS
completion.h
crypto-utils.h
crypto.h
fdlimit.h
file-info.h
file-piece-map.h
handshake.h
history.h
inout.h
lru-cache.h
magnet-metainfo.h
mime-types.h
natpmp_local.h
net.h
open-files.h
peer-common.h
peer-io.h
peer-mgr-active-requests.h

502
libtransmission/fdlimit.cc
View File

@ -1,502 +0,0 @@
// This file Copyright © 2005-2022 Mnemosyne LLC.
// It may be used under GPLv2 (SPDX: GPL-2.0-only), GPLv3 (SPDX: GPL-3.0-only),
// or any future license endorsed by Mnemosyne LLC.
// License text can be found in the licenses/ folder.
#include <algorithm>
#include <array>
#include <cerrno>
#include <cstdint> // uint8_t, uint64_t
#include <ctime>
#include <fmt/core.h>
#include "transmission.h"
#include "error-types.h"
#include "error.h"
#include "fdlimit.h"
#include "file.h"
#include "log.h"
#include "session.h"
#include "torrent.h" /* tr_isTorrent() */
#include "tr-assert.h"
#include "utils.h" // tr_time()
/***
****
**** Local Files
****
***/
static bool preallocate_file_sparse(tr_sys_file_t fd, uint64_t length, tr_error** error)
{
tr_error* my_error = nullptr;
if (length == 0)
{
return true;
}
if (tr_sys_file_preallocate(fd, length, TR_SYS_FILE_PREALLOC_SPARSE, &my_error))
{
return true;
}
tr_logAddDebug(fmt::format("Fast preallocation failed: {} ({})", my_error->message, my_error->code));
if (!TR_ERROR_IS_ENOSPC(my_error->code))
{
char const zero = '\0';
tr_error_clear(&my_error);
/* fallback: the old-style seek-and-write */
if (tr_sys_file_write_at(fd, &zero, 1, length - 1, nullptr, &my_error) && tr_sys_file_truncate(fd, length, &my_error))
{
return true;
}
tr_logAddDebug(fmt::format("Fast prellocation fallback failed: {} ({})", my_error->message, my_error->code));
}
tr_error_propagate(error, &my_error);
return false;
}
static bool preallocate_file_full(tr_sys_file_t fd, uint64_t length, tr_error** error)
{
tr_error* my_error = nullptr;
if (length == 0)
{
return true;
}
if (tr_sys_file_preallocate(fd, length, 0, &my_error))
{
return true;
}
tr_logAddDebug(fmt::format("Full preallocation failed: {} ({})", my_error->message, my_error->code));
if (!TR_ERROR_IS_ENOSPC(my_error->code))
{
auto buf = std::array<uint8_t, 4096>{};
bool success = true;
tr_error_clear(&my_error);
/* fallback: the old-fashioned way */
while (success && length > 0)
{
uint64_t const thisPass = std::min(length, uint64_t{ std::size(buf) });
uint64_t bytes_written = 0;
success = tr_sys_file_write(fd, std::data(buf), thisPass, &bytes_written, &my_error);
length -= bytes_written;
}
if (success)
{
return true;
}
tr_logAddDebug(fmt::format("Full preallocation fallback failed: {} ({})", my_error->message, my_error->code));
}
tr_error_propagate(error, &my_error);
return false;
}
/*****
******
******
******
*****/
struct tr_cached_file
{
bool is_writable;
tr_sys_file_t fd;
int torrent_id;
tr_file_index_t file_index;
time_t used_at;
};
static constexpr bool cached_file_is_open(struct tr_cached_file const* o)
{
TR_ASSERT(o != nullptr);
return (o != nullptr) && (o->fd != TR_BAD_SYS_FILE);
}
static void cached_file_close(struct tr_cached_file* o)
{
TR_ASSERT(cached_file_is_open(o));
if (o != nullptr)
{
tr_sys_file_close(o->fd);
o->fd = TR_BAD_SYS_FILE;
}
}
/**
* returns 0 on success, or an errno value on failure.
* errno values include ENOENT if the parent folder doesn't exist,
* plus the errno values set by tr_sys_dir_create () and tr_sys_file_open ().
*/
// TODO: remove goto
static int cached_file_open(
struct tr_cached_file* o,
char const* filename,
bool writable,
tr_preallocation_mode allocation,
uint64_t file_size)
{
int flags = 0;
tr_sys_path_info info = {};
bool already_existed = false;
bool resize_needed = false;
tr_sys_file_t fd = TR_BAD_SYS_FILE;
tr_error* error = nullptr;
/* create subfolders, if any */
if (writable)
{
auto const dir = tr_sys_path_dirname(filename, &error);
if (std::empty(dir))
{
tr_logAddError(fmt::format(
_("Couldn't create '{path}': {error} ({error_code})"),
fmt::arg("path", filename),
fmt::arg("error", error->message),
fmt::arg("error_code", error->code)));
goto FAIL;
}
if (!tr_sys_dir_create(dir.c_str(), TR_SYS_DIR_CREATE_PARENTS, 0777, &error))
{
tr_logAddError(fmt::format(
_("Couldn't create '{path}': {error} ({error_code})"),
fmt::arg("path", dir),
fmt::arg("error", error->message),
fmt::arg("error_code", error->code)));
goto FAIL;
}
}
already_existed = tr_sys_path_get_info(filename, 0, &info) && info.type == TR_SYS_PATH_IS_FILE;
/* we can't resize the file w/o write permissions */
resize_needed = already_existed && (file_size < info.size);
writable |= resize_needed;
/* open the file */
flags = writable ? (TR_SYS_FILE_WRITE | TR_SYS_FILE_CREATE) : 0;
flags |= TR_SYS_FILE_READ | TR_SYS_FILE_SEQUENTIAL;
fd = tr_sys_file_open(filename, flags, 0666, &error);
if (fd == TR_BAD_SYS_FILE)
{
tr_logAddError(fmt::format(
_("Couldn't open '{path}': {error} ({error_code})"),
fmt::arg("path", filename),
fmt::arg("error", error->message),
fmt::arg("error_code", error->code)));
goto FAIL;
}
if (writable && !already_existed && allocation != TR_PREALLOCATE_NONE)
{
bool success = false;
char const* type = nullptr;
if (allocation == TR_PREALLOCATE_FULL)
{
success = preallocate_file_full(fd, file_size, &error);
type = "full";
}
else if (allocation == TR_PREALLOCATE_SPARSE)
{
success = preallocate_file_sparse(fd, file_size, &error);
type = "sparse";
}
TR_ASSERT(type != nullptr);
if (!success)
{
tr_logAddError(fmt::format(
_("Couldn't preallocate '{path}': {error} ({error_code})"),
fmt::arg("path", filename),
fmt::arg("error", error->message),
fmt::arg("error_code", error->code)));
goto FAIL;
}
tr_logAddDebug(fmt::format("Preallocated file '{}' ({}, size: {})", filename, type, file_size));
}
/* If the file already exists and it's too large, truncate it.
* This is a fringe case that happens if a torrent's been updated
* and one of the updated torrent's files is smaller.
* https://trac.transmissionbt.com/ticket/2228
* https://bugs.launchpad.net/ubuntu/+source/transmission/+bug/318249
*/
if (resize_needed && !tr_sys_file_truncate(fd, file_size, &error))
{
tr_logAddWarn(fmt::format(
_("Couldn't truncate '{path}': {error} ({error_code})"),
fmt::arg("path", filename),
fmt::arg("error", error->message),
fmt::arg("error_code", error->code)));
goto FAIL;
}
o->fd = fd;
return 0;
FAIL:
int const err = error->code;
tr_error_free(error);
if (fd != TR_BAD_SYS_FILE)
{
tr_sys_file_close(fd);
}
return err;
}
/***
****
***/
struct tr_fileset
{
struct tr_cached_file* begin;
struct tr_cached_file const* end;
};
static void fileset_construct(struct tr_fileset* set, int n)
{
set->begin = tr_new(struct tr_cached_file, n);
set->end = set->begin + n;
for (struct tr_cached_file* o = set->begin; o != set->end; ++o)
{
*o = { false, TR_BAD_SYS_FILE, 0, 0, 0 };
}
}
static void fileset_close_all(struct tr_fileset* set)
{
if (set != nullptr)
{
for (struct tr_cached_file* o = set->begin; o != set->end; ++o)
{
if (cached_file_is_open(o))
{
cached_file_close(o);
}
}
}
}
static void fileset_destruct(struct tr_fileset* set)
{
fileset_close_all(set);
tr_free(set->begin);
set->end = set->begin = nullptr;
}
static void fileset_close_torrent(struct tr_fileset* set, int torrent_id)
{
if (set != nullptr)
{
for (struct tr_cached_file* o = set->begin; o != set->end; ++o)
{
if (o->torrent_id == torrent_id && cached_file_is_open(o))
{
cached_file_close(o);
}
}
}
}
static struct tr_cached_file* fileset_lookup(struct tr_fileset* set, int torrent_id, tr_file_index_t i)
{
if (set != nullptr)
{
for (struct tr_cached_file* o = set->begin; o != set->end; ++o)
{
if (torrent_id == o->torrent_id && i == o->file_index && cached_file_is_open(o))
{
return o;
}
}
}
return nullptr;
}
static struct tr_cached_file* fileset_get_empty_slot(struct tr_fileset* set)
{
struct tr_cached_file* cull = nullptr;
if (set != nullptr && set->begin != nullptr)
{
/* try to find an unused slot */
for (struct tr_cached_file* o = set->begin; o != set->end; ++o)
{
if (!cached_file_is_open(o))
{
return o;
}
}
/* all slots are full... recycle the least recently used */
for (struct tr_cached_file* o = set->begin; o != set->end; ++o)
{
if (cull == nullptr || o->used_at < cull->used_at)
{
cull = o;
}
}
cached_file_close(cull);
}
return cull;
}
/***
****
**** Startup / Shutdown
****
***/
struct tr_fdInfo
{
struct tr_fileset fileset;
};
static void ensureSessionFdInfoExists(tr_session* session)
{
TR_ASSERT(tr_isSession(session));
if (session->fdInfo == nullptr)
{
int const FILE_CACHE_SIZE = 32;
/* Create the local file cache */
auto* const i = tr_new0(struct tr_fdInfo, 1);
fileset_construct(&i->fileset, FILE_CACHE_SIZE);
session->fdInfo = i;
}
}
void tr_fdClose(tr_session* session)
{
if (session != nullptr && session->fdInfo != nullptr)
{
struct tr_fdInfo* i = session->fdInfo;
fileset_destruct(&i->fileset);
tr_free(i);
session->fdInfo = nullptr;
}
}
/***
****
***/
static struct tr_fileset* get_fileset(tr_session* session)
{
if (session == nullptr)
{
return nullptr;
}
ensureSessionFdInfoExists(session);
return &session->fdInfo->fileset;
}
void tr_fdFileClose(tr_session* s, tr_torrent const* tor, tr_file_index_t i)
{
tr_cached_file* const o = fileset_lookup(get_fileset(s), tr_torrentId(tor), i);
if (o != nullptr)
{
/* flush writable files so that their mtimes will be
* up-to-date when this function returns to the caller... */
if (o->is_writable)
{
tr_sys_file_flush(o->fd);
}
cached_file_close(o);
}
}
tr_sys_file_t tr_fdFileGetCached(tr_session* s, int torrent_id, tr_file_index_t i, bool writable)
{
struct tr_cached_file* o = fileset_lookup(get_fileset(s), torrent_id, i);
if (o == nullptr || (writable && !o->is_writable))
{
return TR_BAD_SYS_FILE;
}
o->used_at = tr_time();
return o->fd;
}
void tr_fdTorrentClose(tr_session* session, int torrent_id)
{
auto const lock = session->unique_lock();
fileset_close_torrent(get_fileset(session), torrent_id);
}
/* returns an fd on success, or a TR_BAD_SYS_FILE on failure and sets errno */
tr_sys_file_t tr_fdFileCheckout(
tr_session* session,
int torrent_id,
tr_file_index_t i,
char const* filename,
bool writable,
tr_preallocation_mode allocation,
uint64_t file_size)
{
struct tr_fileset* set = get_fileset(session);
struct tr_cached_file* o = fileset_lookup(set, torrent_id, i);
if (o != nullptr && writable && !o->is_writable)
{
cached_file_close(o); /* close it so we can reopen in rw mode */
}
else if (o == nullptr)
{
o = fileset_get_empty_slot(set);
}
if (!cached_file_is_open(o))
{
if (int const err = cached_file_open(o, filename, writable, allocation, file_size); err != 0)
{
errno = err;
return TR_BAD_SYS_FILE;
}
tr_logAddTrace(fmt::format("opened '{}' writable {}", filename, writable ? 'y' : 'n'));
o->is_writable = writable;
}
tr_logAddTrace(fmt::format("checking out '{}'", filename));
o->torrent_id = torrent_id;
o->file_index = i;
o->used_at = tr_time();
return o->fd;
}

16
libtransmission/inout.cc
View File

@ -15,7 +15,6 @@
#include "cache.h" /* tr_cacheReadBlock() */
#include "crypto-utils.h"
#include "error.h"
#include "fdlimit.h"
#include "file.h"
#include "inout.h"
#include "log.h"
@ -101,9 +100,8 @@ int readOrWriteBytes(
**** Find the fd
***/
auto fd = tr_fdFileGetCached(session, tr_torrentId(tor), file_index, do_write);
if (fd == TR_BAD_SYS_FILE) // it's not cached, so open/create it now
auto fd = session->openFiles().get(tor->uniqueId, file_index, do_write);
if (!fd) // it's not cached, so open/create it now
{
auto found = tor->findFile(file_index); // see if the file exists...
if (!found)
@ -126,8 +124,8 @@ int readOrWriteBytes(
auto const prealloc = (!do_write || !tor->fileIsWanted(file_index)) ? TR_PREALLOCATE_NONE :
tor->session->preallocationMode;
fd = tr_fdFileCheckout(session, tor->uniqueId, file_index, found->filename(), do_write, prealloc, file_size);
if (fd == TR_BAD_SYS_FILE)
fd = session->openFiles().get(tor->uniqueId, file_index, do_write, found->filename(), prealloc, file_size);
if (!fd)
{
err = errno;
tr_logAddErrorTor(
@ -160,7 +158,7 @@ int readOrWriteBytes(
switch (io_mode)
{
case IoMode::Read:
if (!readEntireBuf(fd, file_offset, buf, buflen, &error))
if (!readEntireBuf(*fd, file_offset, buf, buflen, &error))
{
err = error->code;
tr_logAddErrorTor(
@ -175,7 +173,7 @@ int readOrWriteBytes(
break;
case IoMode::Write:
if (!writeEntireBuf(fd, file_offset, buf, buflen, &error))
if (!writeEntireBuf(*fd, file_offset, buf, buflen, &error))
{
err = error->code;
tr_logAddErrorTor(
@ -190,7 +188,7 @@ int readOrWriteBytes(
break;
case IoMode::Prefetch:
tr_sys_file_advise(fd, file_offset, buflen, TR_SYS_FILE_ADVICE_WILL_NEED);
tr_sys_file_advise(*fd, file_offset, buflen, TR_SYS_FILE_ADVICE_WILL_NEED);
break;
}

141
libtransmission/lru-cache.h Normal file
View File

@ -0,0 +1,141 @@
// This file Copyright © 2022 Mnemosyne LLC.
// It may be used under GPLv2 (SPDX: GPL-2.0), GPLv3 (SPDX: GPL-3.0),
// or any future license endorsed by Mnemosyne LLC.
// License text can be found in the licenses/ folder.
#pragma once
#include <algorithm>
#include <array>
#include <cstddef>
#include <cstdint>
#include <functional>
// A fixed-size cache that erases least-recently-used items to make room for new ones.
template<typename Key, typename Val, std::size_t N>
class tr_lru_cache
{
public:
[[nodiscard]] Val* get(Key const& key) noexcept
{
if (auto const found = find(key); found != nullptr)
{
found->sequence_ = next_sequence_++;
return &found->val_;
}
return nullptr;
}
[[nodiscard]] bool contains(Key const& key) const noexcept
{
return !!find(key);
}
Val& add(Key&& key)
{
auto& entry = getFreeSlot();
entry.key_ = std::move(key);
entry.sequence_ = next_sequence_++;
key = {};
return entry.val_;
}
void erase(Key const& key)
{
if (auto* const found = find(key); found != nullptr)
{
this->erase(*found);
}
}
void erase_if(std::function<bool(Key const&, Val const&)> test)
{
for (auto& entry : entries_)
{
if (entry.sequence_ != InvalidSeq && test(entry.key_, entry.val_))
{
erase(entry);
}
}
}
void clear()
{
for (auto& entry : entries_)
{
erase(entry);
}
}
using PreEraseCallback = std::function<void(Key const&, Val&)>;
void setPreErase(PreEraseCallback&& func)
{
pre_erase_cb_ = std::move(func);
}
private:
PreEraseCallback pre_erase_cb_ = [](Key const&, Val&) {
};
struct Entry
{
Key key_ = {};
Val val_ = {};
uint64_t sequence_ = InvalidSeq;
};
void erase(Entry& entry)
{
if (entry.sequence_ != InvalidSeq)
{
pre_erase_cb_(entry.key_, entry.val_);
}
entry.key_ = {};
entry.val_ = {};
entry.sequence_ = InvalidSeq;
}
[[nodiscard]] Entry* find(Key const& key) noexcept
{
for (auto& entry : entries_)
{
if (entry.sequence_ != InvalidSeq && entry.key_ == key)
{
return &entry;
}
}
return nullptr;
}
[[nodiscard]] Entry const* find(Key const& key) const noexcept
{
for (auto const& entry : entries_)
{
if (entry.sequence_ != InvalidSeq && entry.key_ == key)
{
return &entry;
}
}
return nullptr;
}
Entry& getFreeSlot()
{
auto const iter = std::min_element(
std::begin(entries_),
std::end(entries_),
[](auto const& a, auto const& b) { return a.sequence_ < b.sequence_; });
this->erase(*iter);
return *iter;
}
std::array<Entry, N> entries_;
uint64_t next_sequence_ = 1;
static uint64_t constexpr InvalidSeq = 0;
};

281
libtransmission/open-files.cc Normal file
View File

@ -0,0 +1,281 @@
// This file Copyright © 2005-2022 Mnemosyne LLC.
// It may be used under GPLv2 (SPDX: GPL-2.0-only), GPLv3 (SPDX: GPL-3.0-only),
// or any future license endorsed by Mnemosyne LLC.
// License text can be found in the licenses/ folder.
#include <algorithm>
#include <array>
#include <cerrno>
#include <cstdint> // uint8_t, uint64_t
#include <fmt/core.h>
#include "transmission.h"
#include "error-types.h"
#include "error.h"
#include "file.h"
#include "log.h"
#include "open-files.h"
#include "tr-assert.h"
#include "tr-strbuf.h"
#include "utils.h" // _()
namespace
{
[[nodiscard]] auto constexpr isOpen(tr_sys_file_t fd) noexcept
{
return fd != TR_BAD_SYS_FILE;
}
bool preallocate_file_sparse(tr_sys_file_t fd, uint64_t length, tr_error** error)
{
tr_error* my_error = nullptr;
if (length == 0)
{
return true;
}
if (tr_sys_file_preallocate(fd, length, TR_SYS_FILE_PREALLOC_SPARSE, &my_error))
{
return true;
}
tr_logAddDebug(fmt::format("Fast preallocation failed: {} ({})", my_error->message, my_error->code));
if (!TR_ERROR_IS_ENOSPC(my_error->code))
{
char const zero = '\0';
tr_error_clear(&my_error);
/* fallback: the old-style seek-and-write */
if (tr_sys_file_write_at(fd, &zero, 1, length - 1, nullptr, &my_error) && tr_sys_file_truncate(fd, length, &my_error))
{
return true;
}
tr_logAddDebug(fmt::format("Fast prellocation fallback failed: {} ({})", my_error->message, my_error->code));
}
tr_error_propagate(error, &my_error);
return false;
}
bool preallocate_file_full(tr_sys_file_t fd, uint64_t length, tr_error** error)
{
tr_error* my_error = nullptr;
if (length == 0)
{
return true;
}
if (tr_sys_file_preallocate(fd, length, 0, &my_error))
{
return true;
}
tr_logAddDebug(fmt::format("Full preallocation failed: {} ({})", my_error->message, my_error->code));
if (!TR_ERROR_IS_ENOSPC(my_error->code))
{
auto buf = std::array<uint8_t, 4096>{};
bool success = true;
tr_error_clear(&my_error);
/* fallback: the old-fashioned way */
while (success && length > 0)
{
uint64_t const thisPass = std::min(length, uint64_t{ std::size(buf) });
uint64_t bytes_written = 0;
success = tr_sys_file_write(fd, std::data(buf), thisPass, &bytes_written, &my_error);
length -= bytes_written;
}
if (success)
{
return true;
}
tr_logAddDebug(fmt::format("Full preallocation fallback failed: {} ({})", my_error->message, my_error->code));
}
tr_error_propagate(error, &my_error);
return false;
}
} // unnamed namespace
///
std::optional<tr_sys_file_t> tr_open_files::get(tr_torrent_id_t tor_id, tr_file_index_t file_num, bool writable)
{
if (auto* const found = pool_.get(makeKey(tor_id, file_num)); found != nullptr)
{
if (writable && !found->writable_)
{
return {};
}
return found->fd_;
}
return {};
}
std::optional<tr_sys_file_t> tr_open_files::get(
tr_torrent_id_t tor_id,
tr_file_index_t file_num,
bool writable,
std::string_view filename_in,
tr_preallocation_mode allocation,
uint64_t file_size)
{
// is there already an entry
auto key = makeKey(tor_id, file_num);
if (auto* const found = pool_.get(key); found != nullptr)
{
if (!writable || found->writable_)
{
return found->fd_;
}
pool_.erase(key); // close so we can re-open as writable
}
// create subfolders, if any
auto const filename = tr_pathbuf{ filename_in };
tr_error* error = nullptr;
if (writable)
{
auto const dir = tr_sys_path_dirname(filename, &error);
if (std::empty(dir))
{
tr_logAddError(fmt::format(
_("Couldn't create '{path}': {error} ({error_code})"),
fmt::arg("path", filename),
fmt::arg("error", error->message),
fmt::arg("error_code", error->code)));
tr_error_free(error);
return {};
}
if (!tr_sys_dir_create(dir.c_str(), TR_SYS_DIR_CREATE_PARENTS, 0777, &error))
{
tr_logAddError(fmt::format(
_("Couldn't create '{path}': {error} ({error_code})"),
fmt::arg("path", dir),
fmt::arg("error", error->message),
fmt::arg("error_code", error->code)));
tr_error_free(error);
return {};
}
}
auto info = tr_sys_path_info{};
bool const already_existed = tr_sys_path_get_info(filename, 0, &info) && info.type == TR_SYS_PATH_IS_FILE;
// we need write permissions to resize the file
bool const resize_needed = already_existed && (file_size < info.size);
writable |= resize_needed;
// open the file
int flags = writable ? (TR_SYS_FILE_WRITE | TR_SYS_FILE_CREATE) : 0;
flags |= TR_SYS_FILE_READ | TR_SYS_FILE_SEQUENTIAL;
auto const fd = tr_sys_file_open(filename, flags, 0666, &error);
if (!isOpen(fd))
{
tr_logAddError(fmt::format(
_("Couldn't open '{path}': {error} ({error_code})"),
fmt::arg("path", filename),
fmt::arg("error", error->message),
fmt::arg("error_code", error->code)));
tr_error_free(error);
return {};
}
if (writable && !already_existed && allocation != TR_PREALLOCATE_NONE)
{
bool success = false;
char const* type = nullptr;
if (allocation == TR_PREALLOCATE_FULL)
{
success = preallocate_file_full(fd, file_size, &error);
type = "full";
}
else if (allocation == TR_PREALLOCATE_SPARSE)
{
success = preallocate_file_sparse(fd, file_size, &error);
type = "sparse";
}
TR_ASSERT(type != nullptr);
if (!success)
{
tr_logAddError(fmt::format(
_("Couldn't preallocate '{path}': {error} ({error_code})"),
fmt::arg("path", filename),
fmt::arg("error", error->message),
fmt::arg("error_code", error->code)));
tr_sys_file_close(fd);
tr_error_free(error);
return {};
}
tr_logAddDebug(fmt::format("Preallocated file '{}' ({}, size: {})", filename, type, file_size));
}
// If the file already exists and it's too large, truncate it.
// This is a fringe case that happens if a torrent's been updated
// and one of the updated torrent's files is smaller.
// https://trac.transmissionbt.com/ticket/2228
// https://bugs.launchpad.net/ubuntu/+source/transmission/+bug/318249
if (resize_needed && !tr_sys_file_truncate(fd, file_size, &error))
{
tr_logAddWarn(fmt::format(
_("Couldn't truncate '{path}': {error} ({error_code})"),
fmt::arg("path", filename),
fmt::arg("error", error->message),
fmt::arg("error_code", error->code)));
tr_sys_file_close(fd);
tr_error_free(error);
return {};
}
// cache it
auto& entry = pool_.add(std::move(key));
entry.fd_ = fd;
entry.writable_ = writable;
return fd;
}
void tr_open_files::closeAll()
{
pool_.clear();
}
void tr_open_files::closeTorrent(tr_torrent_id_t tor_id)
{
return pool_.erase_if([&tor_id](Key const& key, Val const&) { return key.first == tor_id; });
}
void tr_open_files::closeFile(tr_torrent_id_t tor_id, tr_file_index_t file_num)
{
pool_.erase(makeKey(tor_id, file_num));
}
tr_open_files::Val::~Val()
{
if (isOpen(fd_))
{
tr_sys_file_close(fd_);
}
}

64
libtransmission/open-files.h Normal file
View File

@ -0,0 +1,64 @@
// This file Copyright © 2005-2022 Mnemosyne LLC.
// It may be used under GPLv2 (SPDX: GPL-2.0), GPLv3 (SPDX: GPL-3.0),
// or any future license endorsed by Mnemosyne LLC.
// License text can be found in the licenses/ folder.
#pragma once
#ifndef __TRANSMISSION__
#error only libtransmission should #include this header.
#endif
#include <cstdint> // uint64_t
#include <optional>
#include <utility> // std::pair
#include "transmission.h"
#include "file.h" // tr_sys_file_t
#include "lru-cache.h"
struct tr_session;
// A pool of open files that are cached while reading / writing torrents' data
class tr_open_files
{
public:
[[nodiscard]] std::optional<tr_sys_file_t> get(tr_torrent_id_t tor_id, tr_file_index_t file_num, bool writable);
[[nodiscard]] std::optional<tr_sys_file_t> get(
tr_torrent_id_t tor_id,
tr_file_index_t file_num,
bool writable,
std::string_view filename,
tr_preallocation_mode mode,
uint64_t file_size);
void closeAll();
void closeTorrent(tr_torrent_id_t tor_id);
void closeFile(tr_torrent_id_t tor_id, tr_file_index_t file_num);
private:
using Key = std::pair<tr_torrent_id_t, tr_file_index_t>;
[[nodiscard]] static Key makeKey(tr_torrent_id_t tor_id, tr_file_index_t file_num) noexcept
{
return std::make_pair(tor_id, file_num);
}
struct Val
{
Val& operator=(Val const&) = delete;
Val& operator=(Val&&) = default;
Val() = default;
Val(Val const&) = delete;
Val(Val&&) = default;
~Val();
tr_sys_file_t fd_ = TR_BAD_SYS_FILE;
bool writable_ = false;
};
static constexpr size_t MaxOpenFiles = 32;
tr_lru_cache<Key, Val, MaxOpenFiles> pool_;
};

16
libtransmission/session.cc
View File

@ -1924,7 +1924,7 @@ static void sessionCloseImplFinish(tr_session* session)
closeBlocklists(session);
tr_fdClose(session);
session->openFiles().closeAll();
session->isClosed = true;
}
@ -2946,3 +2946,17 @@ static int bandwidthGroupWrite(tr_session const* session, std::string_view confi
tr_variantFree(&groups_dict);
return ret;
}
///
void tr_session::closeTorrentFiles(tr_torrent* tor) noexcept
{
tr_cacheFlushTorrent(this->cache, tor);
openFiles().closeTorrent(tor->uniqueId);
}
void tr_session::closeTorrentFile(tr_torrent* tor, tr_file_index_t file_num) noexcept
{
tr_cacheFlushFile(this->cache, tor, file_num);
openFiles().closeFile(tor->uniqueId, file_num);
}

15
libtransmission/session.h
View File

@ -28,6 +28,7 @@
#include "bandwidth.h"
#include "interned-string.h"
#include "net.h" // tr_socket_t
#include "open-files.h"
#include "quark.h"
#include "torrents.h"
#include "web.h"
@ -276,6 +277,16 @@ public:
[[nodiscard]] Bandwidth& getBandwidthGroup(std::string_view name);
//
[[nodiscard]] constexpr auto& openFiles() noexcept
{
return open_files_;
}
void closeTorrentFiles(tr_torrent* tor) noexcept;
void closeTorrentFile(tr_torrent* tor, tr_file_index_t file_num) noexcept;
public:
static constexpr std::array<std::tuple<tr_quark, tr_quark, TrScript>, 3> Scripts{
{ { TR_KEY_script_torrent_added_enabled, TR_KEY_script_torrent_added_filename, TR_SCRIPT_ON_TORRENT_ADDED },
@ -314,8 +325,6 @@ public:
struct tr_turtle_info turtle;
struct tr_fdInfo* fdInfo;
int magicNumber;
tr_encryption_mode encryptionMode;
@ -450,6 +459,8 @@ private:
std::array<bool, TR_SCRIPT_N_TYPES> scripts_enabled_;
bool blocklist_enabled_ = false;
bool incomplete_dir_enabled_ = false;
tr_open_files open_files_;
};
bool tr_sessionAllowsDHT(tr_session const* session);

16
libtransmission/torrent.cc
View File

@ -34,11 +34,9 @@
#include "announcer.h"
#include "bandwidth.h"
#include "cache.h"
#include "completion.h"
#include "crypto-utils.h" /* for tr_sha1 */
#include "error.h"
#include "fdlimit.h" /* tr_fdTorrentClose */
#include "file.h"
#include "inout.h" /* tr_ioTestPiece() */
#include "log.h"
@ -1559,9 +1557,8 @@ static void stopTorrent(tr_torrent* const tor)
tr_verifyRemove(tor);
tr_peerMgrStopTorrent(tor);
tr_announcerTorrentStopped(tor);
tr_cacheFlushTorrent(tor->session->cache, tor);
tr_fdTorrentClose(tor->session, tor->uniqueId);
tor->session->closeTorrentFiles(tor);
if (!tor->isDeleting)
{
@ -1642,8 +1639,7 @@ static void removeTorrentInEventThread(tr_torrent* tor, bool delete_flag, tr_fil
if (delete_flag && tor->hasMetainfo())
{
// ensure the files are all closed and idle before moving
tr_cacheFlushTorrent(tor->session->cache, tor);
tr_fdTorrentClose(tor->session, tor->uniqueId);
tor->session->closeTorrentFiles(tor);
tr_verifyRemove(tor);
if (delete_func == nullptr)
@ -1837,7 +1833,7 @@ void tr_torrent::recheckCompleteness()
}
this->completeness = new_completeness;
tr_fdTorrentClose(this->session, this->uniqueId);
this->session->closeTorrentFiles(this);
if (this->isDone())
{
@ -2186,8 +2182,7 @@ static void setLocationInEventThread(
}
// ensure the files are all closed and idle before moving
tr_cacheFlushTorrent(tor->session->cache, tor);
tr_fdTorrentClose(tor->session, tor->uniqueId);
tor->session->closeTorrentFiles(tor);
tr_verifyRemove(tor);
tr_error* error = nullptr;
@ -2293,8 +2288,7 @@ std::string_view tr_torrent::primaryMimeType() const
static void tr_torrentFileCompleted(tr_torrent* tor, tr_file_index_t i)
{
/* close the file so that we can reopen in read-only mode as needed */
tr_cacheFlushFile(tor->session->cache, tor, i);
tr_fdFileClose(tor->session, tor, i);
tor->session->closeTorrentFile(tor, i);
/* now that the file is complete and closed, we can start watching its
* mtime timestamp for changes to know if we need to reverify pieces */

1
libtransmission/transmission.h
View File

@ -35,6 +35,7 @@ using tr_block_index_t = uint32_t;
using tr_tracker_tier_t = uint32_t;
using tr_tracker_id_t = uint32_t;
using tr_byte_index_t = uint64_t;
using tr_torrent_id_t = int;
struct tr_block_span_t
{

1
tests/libtransmission/CMakeLists.txt
View File

@ -20,6 +20,7 @@ add_executable(libtransmission-test
magnet-metainfo-test.cc
makemeta-test.cc
move-test.cc
open-files-test.cc
peer-mgr-active-requests-test.cc
peer-mgr-wishlist-test.cc
peer-msgs-test.cc

192
tests/libtransmission/open-files-test.cc Normal file
View File

@ -0,0 +1,192 @@
// This file copyright Transmission authors and contributors.
// It may be used under GPLv2 (SPDX: GPL-2.0), GPLv3 (SPDX: GPL-3.0),
// or any future license endorsed by Mnemosyne LLC.
// License text can be found in the licenses/ folder.
#include <algorithm>
#include <string_view>
#include "transmission.h"
#include "error.h"
#include "file.h"
#include "tr-strbuf.h"
#include "test-fixtures.h"
using namespace std::literals;
using OpenFilesTest = libtransmission::test::SessionTest;
TEST_F(OpenFilesTest, getCachedFailsIfNotCached)
{
auto const fd = session_->openFiles().get(0, 0, false);
EXPECT_FALSE(fd);
}
TEST_F(OpenFilesTest, getOpensIfNotCached)
{
static auto constexpr Contents = "Hello, World!\n"sv;
auto filename = tr_pathbuf{ sandboxDir(), "/test-file.txt" };
createFileWithContents(filename, Contents);
// confirm that it's not pre-cached
EXPECT_FALSE(session_->openFiles().get(0, 0, false));
// confirm that we can cache the file
auto fd = session_->openFiles().get(0, 0, false, filename, TR_PREALLOCATE_FULL, std::size(Contents));
EXPECT_TRUE(fd);
EXPECT_NE(TR_BAD_SYS_FILE, *fd);
// test the file contents to confirm that fd points to the right file
auto buf = std::array<char, std::size(Contents) + 1>{};
auto bytes_read = uint64_t{};
EXPECT_TRUE(tr_sys_file_read_at(*fd, std::data(buf), std::size(Contents), 0, &bytes_read));
buf[bytes_read] = '\0';
EXPECT_EQ(Contents, (char*)std::data(buf));
}
TEST_F(OpenFilesTest, getCacheSucceedsIfCached)
{
static auto constexpr Contents = "Hello, World!\n"sv;
auto filename = tr_pathbuf{ sandboxDir(), "/test-file.txt" };
createFileWithContents(filename, Contents);
EXPECT_FALSE(session_->openFiles().get(0, 0, false));
EXPECT_TRUE(session_->openFiles().get(0, 0, false, filename, TR_PREALLOCATE_FULL, std::size(Contents)));
EXPECT_TRUE(session_->openFiles().get(0, 0, false));
}
TEST_F(OpenFilesTest, getCachedReturnsTheSameFd)
{
static auto constexpr Contents = "Hello, World!\n"sv;
auto filename = tr_pathbuf{ sandboxDir(), "/test-file.txt" };
createFileWithContents(filename, Contents);
EXPECT_FALSE(session_->openFiles().get(0, 0, false));
auto const fd1 = session_->openFiles().get(0, 0, false, filename, TR_PREALLOCATE_FULL, std::size(Contents));
auto const fd2 = session_->openFiles().get(0, 0, false);
EXPECT_TRUE(fd1);
EXPECT_TRUE(fd2);
EXPECT_EQ(*fd1, *fd2);
}
TEST_F(OpenFilesTest, getCachedFailsIfWrongPermissions)
{
static auto constexpr Contents = "Hello, World!\n"sv;
auto filename = tr_pathbuf{ sandboxDir(), "/test-file.txt" };
createFileWithContents(filename, Contents);
// cache it in ro mode
EXPECT_FALSE(session_->openFiles().get(0, 0, false));
EXPECT_TRUE(session_->openFiles().get(0, 0, false, filename, TR_PREALLOCATE_FULL, std::size(Contents)));
// now try to get it in r/w mode
EXPECT_TRUE(session_->openFiles().get(0, 0, false));
EXPECT_FALSE(session_->openFiles().get(0, 0, true));
}
TEST_F(OpenFilesTest, opensInReadOnlyUnlessWritableIsRequested)
{
static auto constexpr Contents = "Hello, World!\n"sv;
auto filename = tr_pathbuf{ sandboxDir(), "/test-file.txt" };
createFileWithContents(filename, Contents);
// cache a file read-only mode
tr_error* error = nullptr;
auto fd = session_->openFiles().get(0, 0, false, filename, TR_PREALLOCATE_FULL, std::size(Contents));
// confirm that writing to it fails
EXPECT_FALSE(tr_sys_file_write(*fd, std::data(Contents), std::size(Contents), nullptr, &error));
EXPECT_NE(0, error->code);
tr_error_clear(&error);
}
TEST_F(OpenFilesTest, createsMissingFileIfWriteRequested)
{
static auto constexpr Contents = "Hello, World!\n"sv;
auto filename = tr_pathbuf{ sandboxDir(), "/test-file.txt" };
EXPECT_FALSE(tr_sys_path_exists(filename));
auto fd = session_->openFiles().get(0, 0, false);
EXPECT_FALSE(fd);
EXPECT_FALSE(tr_sys_path_exists(filename));
fd = session_->openFiles().get(0, 0, true, filename, TR_PREALLOCATE_FULL, std::size(Contents));
EXPECT_TRUE(fd);
EXPECT_NE(TR_BAD_SYS_FILE, *fd);
EXPECT_TRUE(tr_sys_path_exists(filename));
}
TEST_F(OpenFilesTest, closeFileClosesTheFile)
{
static auto constexpr Contents = "Hello, World!\n"sv;
auto filename = tr_pathbuf{ sandboxDir(), "/test-file.txt" };
createFileWithContents(filename, Contents);
// cache a file read-only mode
EXPECT_TRUE(session_->openFiles().get(0, 0, false, filename, TR_PREALLOCATE_FULL, std::size(Contents)));
EXPECT_TRUE(session_->openFiles().get(0, 0, false));
// close the file
session_->openFiles().closeFile(0, 0);
// confirm that its fd is no longer cached
EXPECT_FALSE(session_->openFiles().get(0, 0, false));
}
TEST_F(OpenFilesTest, closeTorrentClosesTheTorrentFiles)
{
static auto constexpr Contents = "Hello, World!\n"sv;
static auto constexpr TorId = tr_torrent_id_t{ 0 };
auto filename = tr_pathbuf{ sandboxDir(), "/a.txt" };
createFileWithContents(filename, Contents);
EXPECT_TRUE(session_->openFiles().get(TorId, 1, false, filename, TR_PREALLOCATE_FULL, std::size(Contents)));
filename.assign(sandboxDir(), "/b.txt");
createFileWithContents(filename, Contents);
EXPECT_TRUE(session_->openFiles().get(TorId, 3, false, filename, TR_PREALLOCATE_FULL, std::size(Contents)));
// confirm that closing a different torrent does not affect these files
session_->openFiles().closeTorrent(TorId + 1);
EXPECT_TRUE(session_->openFiles().get(TorId, 1, false));
EXPECT_TRUE(session_->openFiles().get(TorId, 3, false));
// confirm that closing this torrent closes and uncaches the files
session_->openFiles().closeTorrent(TorId);
EXPECT_FALSE(session_->openFiles().get(TorId, 1, false));
EXPECT_FALSE(session_->openFiles().get(TorId, 3, false));
}
TEST_F(OpenFilesTest, closesLeastRecentlyUsedFile)
{
static auto constexpr Contents = "Hello, World!\n"sv;
static auto constexpr TorId = tr_torrent_id_t{ 0 };
static auto constexpr LargerThanCacheLimit = 100;
// Walk through a number of files. Confirm that they all succeed
// even when the number exhausts the cache size, and newer files
// supplant older ones.
for (int i = 0; i < LargerThanCacheLimit; ++i)
{
auto filename = tr_pathbuf{ sandboxDir(), fmt::format("/file-{:d}.txt", i) };
EXPECT_TRUE(session_->openFiles().get(TorId, i, true, filename, TR_PREALLOCATE_FULL, std::size(Contents)));
}
// Do a lookup-only for the files again *in the same order*. By following the
// order, the first files we check will be the oldest from the last pass and
// should have aged out. So we should have a nonzero number of failures; but
// once we get a success, all the remaining should also succeed.
auto results = std::array<bool, LargerThanCacheLimit>{};
auto sorted = std::array<bool, LargerThanCacheLimit>{};
for (int i = 0; i < LargerThanCacheLimit; ++i)
{
auto filename = tr_pathbuf{ sandboxDir(), fmt::format("/file-{:d}.txt", i) };
results[i] = !!session_->openFiles().get(TorId, i, false);
}
sorted = results;
std::sort(std::begin(sorted), std::end(sorted));
EXPECT_EQ(sorted, results);
EXPECT_GT(std::count(std::begin(results), std::end(results), true), 0);
}

5
tests/libtransmission/test-fixtures.h
View File

@ -244,6 +244,11 @@ protected:
errno = tmperr;
}
void createFileWithContents(std::string_view path, std::string_view payload) const
{
createFileWithContents(path, std::data(payload), std::size(payload));
}
void createFileWithContents(std::string_view path, void const* payload) const
{
createFileWithContents(path, payload, strlen(static_cast<char const*>(payload)));