From f1a53840f924c7bb4194ab6971c1a2d57f525ade Mon Sep 17 00:00:00 2001 From: Charles Kerr Date: Wed, 16 Feb 2022 12:33:50 -0600 Subject: [PATCH] refactor: tr_web (#2633) * refactor: fix tr_web rate limiting and dns caching --- cli/cli.cc | 16 +- gtk/FaviconCache.cc | 21 +- libtransmission/announcer-http.cc | 123 ++--- libtransmission/rpcimpl.cc | 82 +-- libtransmission/session.cc | 76 ++- libtransmission/session.h | 26 +- libtransmission/web.cc | 811 +++++++++++++++--------------- libtransmission/web.h | 160 ++++-- libtransmission/webseed.cc | 21 +- 9 files changed, 736 insertions(+), 600 deletions(-) diff --git a/cli/cli.cc b/cli/cli.cc index 10ccd009c..a5288b029 100644 --- a/cli/cli.cc +++ b/cli/cli.cc @@ -19,7 +19,7 @@ #include #include #include -#include /* tr_webRun */ +#include // tr_sessionFetch() /*** **** @@ -124,16 +124,10 @@ static char* tr_strlratio(char* buf, double ratio, size_t buflen) static bool waitingOnWeb; -static void onTorrentFileDownloaded( - tr_session* /*session*/, - bool /*did_connect*/, - bool /*did_timeout*/, - long /*response_code*/, - std::string_view response, - void* vctor) +static void onTorrentFileDownloaded(tr_web::FetchResponse&& response) { - auto* ctor = static_cast(vctor); - tr_ctorSetMetainfo(ctor, std::data(response), std::size(response), nullptr); + auto* ctor = static_cast(response.user_data); + tr_ctorSetMetainfo(ctor, std::data(response.body), std::size(response.body), nullptr); waitingOnWeb = false; } @@ -286,7 +280,7 @@ int tr_main(int argc, char* argv[]) else if (tr_urlIsValid(torrentPath)) { // fetch it - tr_webRun(h, { torrentPath, onTorrentFileDownloaded, ctor }); + tr_sessionFetch(h, { torrentPath, onTorrentFileDownloaded, ctor }); waitingOnWeb = true; while (waitingOnWeb) { diff --git a/gtk/FaviconCache.cc b/gtk/FaviconCache.cc index 89741bcb9..cac648240 100644 --- a/gtk/FaviconCache.cc +++ b/gtk/FaviconCache.cc @@ -10,7 +10,7 @@ #include /* g_remove() */ #include -#include /* tr_webRun() */ +#include // tr_sessionFetch() #include #include "FaviconCache.h" @@ -73,7 +73,7 @@ Glib::RefPtr favicon_load_from_cache(std::string const& host) } } -void favicon_web_done_cb(tr_session*, bool, bool, long, std::string_view, gpointer); +void favicon_web_done_cb(tr_web::FetchResponse&& response); bool favicon_web_done_idle_cb(std::unique_ptr fav) { @@ -90,7 +90,7 @@ bool favicon_web_done_idle_cb(std::unique_ptr fav) fav->contents.clear(); auto* const session = fav->session; auto const next_url = get_url(fav->host, fav->type); - tr_webRun(session, { next_url.raw(), favicon_web_done_cb, fav.release() }); + tr_sessionFetch(session, { next_url.raw(), favicon_web_done_cb, fav.release() }); } // Not released into the next web request, means we're done trying (even if `pixbuf` is still invalid) @@ -102,17 +102,10 @@ bool favicon_web_done_idle_cb(std::unique_ptr fav) return false; } -void favicon_web_done_cb( - tr_session* /*session*/, - bool /*did_connect*/, - bool /*did_timeout*/, - long /*code*/, - std::string_view data, - gpointer vfav) +void favicon_web_done_cb(tr_web::FetchResponse&& response) { - auto* fav = static_cast(vfav); - fav->contents.assign(std::data(data), std::size(data)); - + auto* const fav = static_cast(response.user_data); + fav->contents = response.body; Glib::signal_idle().connect([fav]() { return favicon_web_done_idle_cb(std::unique_ptr(fav)); }); } @@ -136,7 +129,7 @@ void gtr_get_favicon( data->func = pixbuf_ready_func; data->host = host; - tr_webRun(session, { get_url(host, 0).raw(), favicon_web_done_cb, data.release() }); + tr_sessionFetch(session, { get_url(host, 0).raw(), favicon_web_done_cb, data.release() }); } } diff --git a/libtransmission/announcer-http.cc b/libtransmission/announcer-http.cc index 9f0350b6c..5e2f99c1a 100644 --- a/libtransmission/announcer-http.cc +++ b/libtransmission/announcer-http.cc @@ -121,26 +121,6 @@ static std::string announce_url_new(tr_session const* session, tr_announce_reque return evbuffer_free_to_str(buf); } -struct announce_data -{ - tr_announce_response response; - tr_announce_response_func response_func; - void* response_func_user_data; - char log_name[128]; -}; - -static void on_announce_done_eventthread(void* vdata) -{ - auto* data = static_cast(vdata); - - if (data->response_func != nullptr) - { - data->response_func(&data->response, data->response_func_user_data); - } - - delete data; -} - static void verboseLog(std::string_view description, tr_direction direction, std::string_view message) { auto& out = std::cerr; @@ -293,14 +273,17 @@ void tr_announcerParseHttpAnnounceResponse(tr_announce_response& response, std:: } } -static void on_announce_done( - tr_session* session, - bool did_connect, - bool did_timeout, - long response_code, - std::string_view msg, - void* vdata) +struct announce_data { + tr_announce_response response; + tr_announce_response_func response_func; + void* response_func_user_data; + char log_name[128]; +}; + +static void onAnnounceDone(tr_web::FetchResponse&& web_response) +{ + auto const& [status, body, did_connect, did_timeout, vdata] = web_response; auto* data = static_cast(vdata); tr_announce_response* const response = &data->response; @@ -308,14 +291,14 @@ static void on_announce_done( response->did_timeout = did_timeout; dbgmsg(data->log_name, "Got announce response"); - if (response_code != HTTP_OK) + if (status != HTTP_OK) { - auto const* const response_str = tr_webGetResponseStr(response_code); - response->errmsg = tr_strvJoin("Tracker HTTP response "sv, std::to_string(response_code), " ("sv, response_str, ")"sv); + auto const* const response_str = tr_webGetResponseStr(status); + response->errmsg = tr_strvJoin("Tracker HTTP response "sv, std::to_string(status), " ("sv, response_str, ")"sv); } else { - tr_announcerParseHttpAnnounceResponse(*response, msg); + tr_announcerParseHttpAnnounceResponse(*response, body); } if (!std::empty(response->pex6)) @@ -328,7 +311,12 @@ static void on_announce_done( dbgmsg(data->log_name, "got a peers length of %zu", std::size(response->pex)); } - tr_runInEventThread(session, on_announce_done_eventthread, data); + if (data->response_func != nullptr) + { + data->response_func(&data->response, data->response_func_user_data); + } + + delete data; } void tr_tracker_http_announce( @@ -345,7 +333,12 @@ void tr_tracker_http_announce( auto const url = announce_url_new(session, request); dbgmsg(request->log_name, "Sending announce to libcurl: \"%" TR_PRIsv "\"", TR_PRIsv_ARG(url)); - tr_webRun(session, { url, on_announce_done, d }); + + auto options = tr_web::FetchOptions{ url, onAnnounceDone, d }; + options.timeout_secs = 90L; + options.sndbuf = 1024; + options.rcvbuf = 3072; + session->web->fetch(std::move(options)); } /**** @@ -354,26 +347,6 @@ void tr_tracker_http_announce( ***** ****/ -struct scrape_data -{ - tr_scrape_response response; - tr_scrape_response_func response_func; - void* response_func_user_data; - char log_name[128]; -}; - -static void on_scrape_done_eventthread(void* vdata) -{ - auto* data = static_cast(vdata); - - if (data->response_func != nullptr) - { - data->response_func(&data->response, data->response_func_user_data); - } - - delete data; -} - void tr_announcerParseHttpScrapeResponse(tr_scrape_response& response, std::string_view benc) { verboseLog("Scrape response:", TR_DOWN, benc); @@ -464,15 +437,18 @@ void tr_announcerParseHttpScrapeResponse(tr_scrape_response& response, std::stri } } -static void on_scrape_done( - tr_session* session, - bool did_connect, - bool did_timeout, - long response_code, - std::string_view msg, - void* vdata) +struct scrape_data { - auto* data = static_cast(vdata); + tr_scrape_response response; + tr_scrape_response_func response_func; + void* response_func_user_data; + char log_name[128]; +}; + +static void onScrapeDone(tr_web::FetchResponse&& web_response) +{ + auto const& [status, body, did_connect, did_timeout, vdata] = web_response; + auto* const data = static_cast(vdata); tr_scrape_response& response = data->response; response.did_connect = did_connect; @@ -481,20 +457,22 @@ static void on_scrape_done( auto const scrape_url_sv = response.scrape_url.sv(); dbgmsg(data->log_name, "Got scrape response for \"%" TR_PRIsv "\"", TR_PRIsv_ARG(scrape_url_sv)); - if (response_code != HTTP_OK) + if (status != HTTP_OK) { - char const* fmt = _("Tracker gave HTTP response code %1$ld (%2$s)"); - char const* response_str = tr_webGetResponseStr(response_code); - char buf[512]; - tr_snprintf(buf, sizeof(buf), fmt, response_code, response_str); - response.errmsg = buf; + auto const* const response_str = tr_webGetResponseStr(status); + response.errmsg = tr_strvJoin("Tracker HTTP response "sv, std::to_string(status), " ("sv, response_str, ")"sv); } else { - tr_announcerParseHttpScrapeResponse(response, msg); + tr_announcerParseHttpScrapeResponse(response, body); } - tr_runInEventThread(session, on_scrape_done_eventthread, data); + if (data->response_func != nullptr) + { + data->response_func(&data->response, data->response_func_user_data); + } + + delete data; } static std::string scrape_url_new(tr_scrape_request const* req) @@ -540,5 +518,10 @@ void tr_tracker_http_scrape( auto const url = scrape_url_new(request); dbgmsg(request->log_name, "Sending scrape to libcurl: \"%" TR_PRIsv "\"", TR_PRIsv_ARG(url)); - tr_webRun(session, { url, on_scrape_done, d }); + + auto options = tr_web::FetchOptions{ url, onScrapeDone, d }; + options.timeout_secs = 30L; + options.sndbuf = 4096; + options.rcvbuf = 4096; + session->web->fetch(std::move(options)); } diff --git a/libtransmission/rpcimpl.cc b/libtransmission/rpcimpl.cc index 71c696c65..2f97c0b6d 100644 --- a/libtransmission/rpcimpl.cc +++ b/libtransmission/rpcimpl.cc @@ -1340,29 +1340,19 @@ static char const* torrentRenamePath( **** ***/ -static void portTested( - tr_session* /*session*/, - bool /*did_connect*/, - bool /*did_timeout*/, - long response_code, - std::string_view response, - void* user_data) +static void portTested(tr_web::FetchResponse&& web_response) { + auto const& [status, body, did_connect, did_tmieout, user_data] = web_response; char result[1024]; auto* data = static_cast(user_data); - if (response_code != 200) + if (status != 200) { - tr_snprintf( - result, - sizeof(result), - "portTested: http error %ld: %s", - response_code, - tr_webGetResponseStr(response_code)); + tr_snprintf(result, sizeof(result), "portTested: http error %ld: %s", status, tr_webGetResponseStr(status)); } else /* success */ { - bool const isOpen = tr_strvStartsWith(response, '1'); + bool const isOpen = tr_strvStartsWith(body, '1'); tr_variantDictAddBool(data->args_out, TR_KEY_port_is_open, isOpen); tr_snprintf(result, sizeof(result), "success"); } @@ -1378,7 +1368,7 @@ static char const* portTest( { auto const port = tr_sessionGetPeerPort(session); auto const url = tr_strvJoin("https://portcheck.transmissionbt.com/"sv, std::to_string(port)); - tr_webRun(session, { url, portTested, idle_data }); + session->web->fetch({ url, portTested, idle_data }); return nullptr; } @@ -1386,28 +1376,18 @@ static char const* portTest( **** ***/ -static void gotNewBlocklist( - tr_session* session, - bool /*did_connect*/, - bool /*did_timeout*/, - long response_code, - std::string_view response, - void* user_data) +static void gotNewBlocklist(tr_web::FetchResponse&& web_response) { - char result[1024]; + auto const& [status, body, did_connect, did_timeout, user_data] = web_response; auto* data = static_cast(user_data); + auto* const session = data->session; - *result = '\0'; + char result[1024] = {}; - if (response_code != 200) + if (status != 200) { // we failed to download the blocklist... - tr_snprintf( - result, - sizeof(result), - "gotNewBlocklist: http error %ld: %s", - response_code, - tr_webGetResponseStr(response_code)); + tr_snprintf(result, sizeof(result), "gotNewBlocklist: http error %ld: %s", status, tr_webGetResponseStr(status)); tr_idle_function_done(data, result); return; } @@ -1422,8 +1402,8 @@ static void gotNewBlocklist( auto actual_size = size_t{}; auto const decompress_result = libdeflate_gzip_decompress( decompressor.get(), - std::data(response), - std::size(response), + std::data(body), + std::size(body), std::data(content), std::size(content), &actual_size); @@ -1436,7 +1416,7 @@ static void gotNewBlocklist( if (decompress_result == LIBDEFLATE_BAD_DATA) { // couldn't decompress it; maybe we downloaded an uncompressed file - content.assign(std::begin(response), std::end(response)); + content.assign(std::begin(body), std::end(body)); } break; } @@ -1466,7 +1446,7 @@ static char const* blocklistUpdate( tr_variant* /*args_out*/, struct tr_rpc_idle_data* idle_data) { - tr_webRun(session, { session->blocklistUrl(), gotNewBlocklist, idle_data }); + session->web->fetch({ session->blocklistUrl(), gotNewBlocklist, idle_data }); return nullptr; } @@ -1521,36 +1501,26 @@ struct add_torrent_idle_data tr_ctor* ctor; }; -static void gotMetadataFromURL( - tr_session* /*session*/, - bool /*did_connect*/, - bool /*did_timeout*/, - long response_code, - std::string_view response, - void* user_data) +static void gotMetadataFromURL(tr_web::FetchResponse&& web_response) { + auto const& [status, body, did_connect, did_timeout, user_data] = web_response; auto* data = static_cast(user_data); dbgmsg( "torrentAdd: HTTP response code was %ld (%s); response length was %zu bytes", - response_code, - tr_webGetResponseStr(response_code), - std::size(response)); + status, + tr_webGetResponseStr(status), + std::size(body)); - if (response_code == 200 || response_code == 221) /* http or ftp success.. */ + if (status == 200 || status == 221) /* http or ftp success.. */ { - tr_ctorSetMetainfo(data->ctor, std::data(response), std::size(response), nullptr); + tr_ctorSetMetainfo(data->ctor, std::data(body), std::size(body), nullptr); addTorrentImpl(data->data, data->ctor); } else { char result[1024]; - tr_snprintf( - result, - sizeof(result), - "gotMetadataFromURL: http error %ld: %s", - response_code, - tr_webGetResponseStr(response_code)); + tr_snprintf(result, sizeof(result), "gotMetadataFromURL: http error %ld: %s", status, tr_webGetResponseStr(status)); tr_idle_function_done(data->data, result); } @@ -1686,9 +1656,9 @@ static char const* torrentAdd(tr_session* session, tr_variant* args_in, tr_varia d->data = idle_data; d->ctor = ctor; - auto options = tr_web_options{ filename, gotMetadataFromURL, d }; + auto options = tr_web::FetchOptions{ filename, gotMetadataFromURL, d }; options.cookies = cookies; - tr_webRun(session, std::move(options)); + session->web->fetch(std::move(options)); } else { diff --git a/libtransmission/session.cc b/libtransmission/session.cc index 0ae03310a..2aa978380 100644 --- a/libtransmission/session.cc +++ b/libtransmission/session.cc @@ -118,6 +118,74 @@ tr_peer_id_t tr_peerIdInit() **** ***/ +std::optional tr_session::WebController::cookieFile() const +{ + auto const str = tr_strvPath(session_->config_dir, "cookies.txt"); + return tr_sys_path_exists(str.c_str(), nullptr) ? std::optional{ str } : std::nullopt; +} + +std::optional tr_session::WebController::userAgent() const +{ + return tr_strvJoin(TR_NAME, "/"sv, SHORT_VERSION_STRING); +} + +std::optional tr_session::WebController::publicAddress() const +{ + for (auto const type : { TR_AF_INET, TR_AF_INET6 }) + { + auto is_default_value = bool{}; + tr_address const* addr = tr_sessionGetPublicAddress(session_, type, &is_default_value); + if (addr != nullptr && !is_default_value) + { + return tr_address_to_string(addr); + } + } + + return std::nullopt; +} + +unsigned int tr_session::WebController::clamp(int torrent_id, unsigned int byte_count) const +{ + auto const lock = session_->unique_lock(); + auto const it = session_->torrentsById.find(torrent_id); + return it == std::end(session_->torrentsById) ? 0U : it->second->bandwidth->clamp(TR_DOWN, byte_count); +} + +void tr_session::WebController::notifyBandwidthConsumed(int torrent_id, size_t byte_count) +{ + auto const lock = session_->unique_lock(); + auto const it = session_->torrentsById.find(torrent_id); + if (it != std::end(session_->torrentsById)) + { + it->second->bandwidth->notifyBandwidthConsumed(TR_DOWN, byte_count, true, tr_time_msec()); + } +} + +void tr_session::WebController::run(tr_web::FetchDoneFunc func, tr_web::FetchResponse&& response) const +{ + // marshall the `func` call into the libtransmission thread + + using wrapper_t = std::pair; + + auto constexpr callback = [](void* vwrapped) + { + auto* const wrapped = static_cast(vwrapped); + wrapped->first(std::move(wrapped->second)); + delete wrapped; + }; + + tr_runInEventThread(session_, callback, new wrapper_t{ func, std::move(response) }); +} + +void tr_sessionFetch(tr_session* session, tr_web::FetchOptions&& options) +{ + session->web->fetch(std::move(options)); +} + +/*** +**** +***/ + tr_encryption_mode tr_sessionGetEncryption(tr_session* session) { TR_ASSERT(session != nullptr); @@ -693,6 +761,8 @@ static void tr_sessionInitImpl(void* vdata) tr_udpInit(session); + session->web = tr_web::create(session->web_controller); + if (session->isLPDEnabled) { tr_lpdInit(session, &session->bind_ipv4->addr); @@ -1799,7 +1869,7 @@ static void sessionCloseImplStart(tr_session* session) /* and this goes *after* announcer close so that it won't be idle until the announce events are sent... */ - tr_webClose(session, TR_WEB_CLOSE_WHEN_IDLE); + session->web->closeSoon(); tr_cacheFree(session->cache); session->cache = nullptr; @@ -1890,7 +1960,7 @@ void tr_sessionClose(tr_session* session) * so we need to keep the transmission thread alive * for a bit while they tell the router & tracker * that we're closing now */ - while ((session->shared != nullptr || session->web != nullptr || session->announcer != nullptr || + while ((session->shared != nullptr || !session->web->isClosed() || session->announcer != nullptr || session->announcer_udp != nullptr) && !deadlineReached(deadline)) { @@ -1903,7 +1973,7 @@ void tr_sessionClose(tr_session* session) tr_wait_msec(50); } - tr_webClose(session, TR_WEB_CLOSE_NOW); + session->web.reset(); /* close the libtransmission thread */ tr_eventClose(session); diff --git a/libtransmission/session.h b/libtransmission/session.h index 1e01d5702..738ca5714 100644 --- a/libtransmission/session.h +++ b/libtransmission/session.h @@ -29,6 +29,7 @@ #include "net.h" // tr_socket_t #include "quark.h" +#include "web.h" enum tr_auto_switch_state_t { @@ -44,6 +45,7 @@ struct evdns_base; class tr_bitfield; class tr_rpc_server; +class tr_web; struct Bandwidth; struct tr_address; struct tr_announcer; @@ -334,7 +336,29 @@ public: struct tr_cache* cache; - struct tr_web* web; + class WebController final : public tr_web::Controller + { + public: + explicit WebController(tr_session* session) + : session_{ session } + { + } + ~WebController() override = default; + + [[nodiscard]] std::optional cookieFile() const override; + [[nodiscard]] std::optional publicAddress() const override; + [[nodiscard]] std::optional userAgent() const override; + [[nodiscard]] unsigned int clamp(int bandwidth_tag, unsigned int byte_count) const override; + void notifyBandwidthConsumed(int torrent_id, size_t byte_count) override; + // runs the tr_web::fetch response callback in the libtransmission thread + void run(tr_web::FetchDoneFunc func, tr_web::FetchResponse&& response) const override; + + private: + tr_session* const session_; + }; + + WebController web_controller{ this }; + std::unique_ptr web; struct tr_session_id* session_id; diff --git a/libtransmission/web.cc b/libtransmission/web.cc index 7a1d2e72e..f14d74877 100644 --- a/libtransmission/web.cc +++ b/libtransmission/web.cc @@ -4,8 +4,9 @@ // License text can be found in the licenses/ folder. #include -#include -#include +#include +#include +#include #include #include #include @@ -19,174 +20,24 @@ #include -#include "transmission.h" #include "crypto-utils.h" -#include "file.h" #include "log.h" -#include "net.h" /* tr_address */ -#include "torrent.h" -#include "session.h" #include "tr-assert.h" -#include "tr-macros.h" -#include "trevent.h" /* tr_runInEventThread() */ #include "utils.h" -#include "version.h" /* User-Agent */ #include "web.h" using namespace std::literals; -#if LIBCURL_VERSION_NUM >= 0x070F06 /* CURLOPT_SOCKOPT* was added in 7.15.6 */ +#if LIBCURL_VERSION_NUM >= 0x070F06 // CURLOPT_SOCKOPT* was added in 7.15.6 #define USE_LIBCURL_SOCKOPT #endif -static auto constexpr ThreadfuncMaxSleepMsec = int{ 200 }; - #define dbgmsg(...) tr_logAddDeepNamed("web", __VA_ARGS__) /*** **** ***/ -struct tr_web_task -{ -private: - std::shared_ptr const privbuf{ evbuffer_new(), evbuffer_free }; - tr_web_options const options; - -public: - tr_web_task(tr_session* session_in, tr_web_options&& options_in) - : options{ std::move(options_in) } - , session{ session_in } - { - } - - [[nodiscard]] auto* response() const - { - return options.buffer != nullptr ? options.buffer : privbuf.get(); - } - - [[nodiscard]] auto const& torrent_id() const - { - return options.torrent_id; - } - - [[nodiscard]] auto const& url() const - { - return options.url; - } - - [[nodiscard]] auto const& range() const - { - return options.range; - } - - [[nodiscard]] auto const& cookies() const - { - return options.cookies; - } - - void done() const - { - if (options.done_func == nullptr) - { - return; - } - - auto const sv = std::string_view{ reinterpret_cast(evbuffer_pullup(response(), -1)), - evbuffer_get_length(response()) }; - options.done_func(session, did_connect, did_timeout, response_code, sv, options.done_func_user_data); - } - - tr_session* const session; - - CURL* curl_easy = nullptr; - tr_web_task* next = nullptr; - - long response_code = 0; - long timeout_secs = 0; - bool did_connect = false; - bool did_timeout = false; -}; - -/*** -**** -***/ - -struct tr_web -{ - bool const curl_verbose = tr_env_key_exists("TR_CURL_VERBOSE"); - bool const curl_ssl_verify = !tr_env_key_exists("TR_CURL_SSL_NO_VERIFY"); - bool const curl_proxy_ssl_verify = !tr_env_key_exists("TR_CURL_PROXY_SSL_NO_VERIFY"); - - char* curl_ca_bundle; - int close_mode = ~0; - - std::recursive_mutex web_tasks_mutex; - tr_web_task* tasks = nullptr; - - std::string cookie_filename; - std::set paused_easy_handles; -}; - -/*** -**** -***/ - -static size_t writeFunc(void* ptr, size_t size, size_t nmemb, void* vtask) -{ - size_t const byteCount = size * nmemb; - auto* task = static_cast(vtask); - - /* webseed downloads should be speed limited */ - if (auto const& torrent_id = task->torrent_id(); torrent_id) - { - tr_torrent const* const tor = tr_torrentFindFromId(task->session, *torrent_id); - - if (tor != nullptr && tor->bandwidth->clamp(TR_DOWN, nmemb) == 0) - { - task->session->web->paused_easy_handles.insert(task->curl_easy); - return CURL_WRITEFUNC_PAUSE; - } - } - - evbuffer_add(task->response(), ptr, byteCount); - dbgmsg("wrote %zu bytes to task %p's buffer", byteCount, (void*)task); - return byteCount; -} - -#ifdef USE_LIBCURL_SOCKOPT - -static int sockoptfunction(void* vtask, curl_socket_t fd, curlsocktype /*purpose*/) -{ - auto* task = static_cast(vtask); - - // Announce and scrape requests have tiny payloads. - // Ignore the sockopt() return values -- these are suggestions - // rather than hard requirements & it's OK for them to fail - - auto const& url = task->url(); - - if (tr_strvContains(url, "scrape"sv)) - { - int const sndbuf = 4096; - int const rcvbuf = 4096; - (void)setsockopt(fd, SOL_SOCKET, SO_SNDBUF, reinterpret_cast(&sndbuf), sizeof(sndbuf)); - (void)setsockopt(fd, SOL_SOCKET, SO_RCVBUF, reinterpret_cast(&rcvbuf), sizeof(rcvbuf)); - } - else if (tr_strvContains(url, "announce"sv)) - { - int const sndbuf = 1024; - int const rcvbuf = 3072; - (void)setsockopt(fd, SOL_SOCKET, SO_SNDBUF, reinterpret_cast(&sndbuf), sizeof(sndbuf)); - (void)setsockopt(fd, SOL_SOCKET, SO_RCVBUF, reinterpret_cast(&rcvbuf), sizeof(rcvbuf)); - } - - /* return nonzero if this function encountered an error */ - return 0; -} - -#endif - static CURLcode ssl_context_func(CURL* /*curl*/, void* ssl_ctx, void* /*user_data*/) { auto const cert_store = tr_ssl_get_x509_store(ssl_ctx); @@ -244,322 +95,482 @@ static CURLcode ssl_context_func(CURL* /*curl*/, void* ssl_ctx, void* /*user_dat return CURLE_OK; } -static long getTimeoutFromURL(struct tr_web_task const* task) +/*** +**** +***/ + +class tr_web::Impl { - if (auto const* const session = task->session; session == nullptr || session->isClosed) +public: + Impl(Controller& controller_in) + : controller{ controller_in } { - return 20L; + std::call_once(curl_init_flag, curlInit); + + if (auto* bundle = tr_env_get_string("CURL_CA_BUNDLE", nullptr); bundle != nullptr) + { + curl_ca_bundle = bundle; + tr_free(bundle); + } + + if (curl_ssl_verify) + { + auto const* bundle = std::empty(curl_ca_bundle) ? "none" : curl_ca_bundle.c_str(); + tr_logAddNamedInfo("web", "will verify tracker certs using envvar CURL_CA_BUNDLE: %s", bundle); + tr_logAddNamedInfo("web", "NB: this only works if you built against libcurl with openssl or gnutls, NOT nss"); + tr_logAddNamedInfo("web", "NB: Invalid certs will appear as 'Could not connect to tracker' like many other errors"); + } + + if (auto const& file = controller.cookieFile(); file) + { + this->cookie_file = *file; + } + + if (auto const& ua = controller.userAgent(); ua) + { + this->user_agent = *ua; + } + + curl_thread = std::make_unique(tr_webThreadFunc, this); } - if (tr_strvContains(task->url(), "scrape"sv)) + ~Impl() { - return 30L; + run_mode = RunMode::CloseNow; + curl_thread->join(); } - if (tr_strvContains(task->url(), "announce"sv)) + void closeSoon() { - return 90L; + run_mode = RunMode::CloseSoon; } - return 240L; -} + [[nodiscard]] bool isClosed() const + { + return is_closed_; + } -static CURL* createEasy(tr_session* s, struct tr_web* web, struct tr_web_task* task) -{ - CURL* const e = curl_easy_init(); + void fetch(FetchOptions&& options) + { + if (run_mode != RunMode::Run) + { + return; + } - task->curl_easy = e; - task->timeout_secs = getTimeoutFromURL(task); + auto const lock = std::unique_lock(web_tasks_mutex); + auto* const task = new Task{ *this, std::move(options) }; + task->next = tasks; + tasks = task; + } - curl_easy_setopt(e, CURLOPT_AUTOREFERER, 1L); - curl_easy_setopt(e, CURLOPT_ENCODING, ""); - curl_easy_setopt(e, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(e, CURLOPT_MAXREDIRS, -1L); - curl_easy_setopt(e, CURLOPT_NOSIGNAL, 1L); - curl_easy_setopt(e, CURLOPT_PRIVATE, task); +private: + class Task + { + private: + std::shared_ptr const privbuf{ evbuffer_new(), evbuffer_free }; + std::shared_ptr const easy_handle{ curl_easy_init(), curl_easy_cleanup }; + tr_web::FetchOptions const options; + + public: + Task(tr_web::Impl& impl_in, tr_web::FetchOptions&& options_in) + : options{ std::move(options_in) } + , impl{ impl_in } + { + response.user_data = options.done_func_user_data; + } + + [[nodiscard]] auto* easy() const + { + return easy_handle.get(); + } + + [[nodiscard]] auto* body() const + { + return options.buffer != nullptr ? options.buffer : privbuf.get(); + } + + [[nodiscard]] auto const& speedLimitTag() const + { + return options.speed_limit_tag; + } + + [[nodiscard]] auto const& url() const + { + return options.url; + } + + [[nodiscard]] auto const& range() const + { + return options.range; + } + + [[nodiscard]] auto const& cookies() const + { + return options.cookies; + } + + [[nodiscard]] auto const& sndbuf() const + { + return options.sndbuf; + } + + [[nodiscard]] auto const& rcvbuf() const + { + return options.rcvbuf; + } + + [[nodiscard]] auto const& timeoutSecs() const + { + return options.timeout_secs; + } + + void done() + { + if (options.done_func == nullptr) + { + return; + } + + response.body.assign(reinterpret_cast(evbuffer_pullup(body(), -1)), evbuffer_get_length(body())); + impl.controller.run(options.done_func, std::move(this->response)); + } + + tr_web::Impl& impl; + tr_web::FetchResponse response; + Task* next = nullptr; + }; + + static auto constexpr BandwidthPauseMsec = long{ 500 }; + static auto constexpr DnsCacheTimeoutSecs = long{ 60 * 60 }; + + bool const curl_verbose = tr_env_key_exists("TR_CURL_VERBOSE"); + bool const curl_ssl_verify = !tr_env_key_exists("TR_CURL_SSL_NO_VERIFY"); + bool const curl_proxy_ssl_verify = !tr_env_key_exists("TR_CURL_PROXY_SSL_NO_VERIFY"); + + Controller& controller; + + std::string curl_ca_bundle; + + std::recursive_mutex web_tasks_mutex; + Task* tasks = nullptr; + + std::string cookie_file; + std::string user_agent; + + std::unique_ptr curl_thread; + + enum class RunMode + { + Run, + CloseSoon, // no new tasks; exit when running tasks finish + CloseNow // exit now even if tasks are running + }; + + RunMode run_mode = RunMode::Run; + + static size_t onDataReceived(void* data, size_t size, size_t nmemb, void* vtask) + { + size_t const bytes_used = size * nmemb; + auto* task = static_cast(vtask); + TR_ASSERT(std::this_thread::get_id() == task->impl.curl_thread->get_id()); + + if (auto const& tag = task->speedLimitTag(); tag) + { + // If this is more bandwidth than is allocated for this tag, + // then pause the torrent for a tick. curl will deliver `data` + // again when the transfer is unpaused. + if (task->impl.controller.clamp(*tag, bytes_used) < bytes_used) + { + task->impl.paused_easy_handles.emplace(tr_time_msec(), task->easy()); + return CURL_WRITEFUNC_PAUSE; + } + + task->impl.controller.notifyBandwidthConsumed(*tag, bytes_used); + } + + evbuffer_add(task->body(), data, bytes_used); + dbgmsg("wrote %zu bytes to task %p's buffer", bytes_used, (void*)task); + return bytes_used; + } #ifdef USE_LIBCURL_SOCKOPT - curl_easy_setopt(e, CURLOPT_SOCKOPTFUNCTION, sockoptfunction); - curl_easy_setopt(e, CURLOPT_SOCKOPTDATA, task); + static int onSocketCreated(void* vtask, curl_socket_t fd, curlsocktype /*purpose*/) + { + auto const* const task = static_cast(vtask); + TR_ASSERT(std::this_thread::get_id() == task->impl.curl_thread->get_id()); + + // Ignore the sockopt() return values -- these are suggestions + // rather than hard requirements & it's OK for them to fail + + if (auto const& buf = task->sndbuf(); buf) + { + (void)setsockopt(fd, SOL_SOCKET, SO_SNDBUF, reinterpret_cast(&*buf), sizeof(*buf)); + } + if (auto const& buf = task->rcvbuf(); buf) + { + (void)setsockopt(fd, SOL_SOCKET, SO_RCVBUF, reinterpret_cast(&*buf), sizeof(*buf)); + } + + // return nonzero if this function encountered an error + return 0; + } #endif - if (web->curl_ssl_verify) + static void initEasy(tr_web::Impl* impl, Task* task) { - if (web->curl_ca_bundle != nullptr) + TR_ASSERT(std::this_thread::get_id() == impl->curl_thread->get_id()); + auto* const e = task->easy(); + + curl_easy_setopt(e, CURLOPT_SHARE, impl->shared()); + curl_easy_setopt(e, CURLOPT_DNS_CACHE_TIMEOUT, DnsCacheTimeoutSecs); + curl_easy_setopt(e, CURLOPT_AUTOREFERER, 1L); + curl_easy_setopt(e, CURLOPT_ENCODING, ""); + curl_easy_setopt(e, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(e, CURLOPT_MAXREDIRS, -1L); + curl_easy_setopt(e, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(e, CURLOPT_PRIVATE, task); + +#ifdef USE_LIBCURL_SOCKOPT + curl_easy_setopt(e, CURLOPT_SOCKOPTFUNCTION, onSocketCreated); + curl_easy_setopt(e, CURLOPT_SOCKOPTDATA, task); +#endif + + if (!impl->curl_ssl_verify) { - curl_easy_setopt(e, CURLOPT_CAINFO, web->curl_ca_bundle); + curl_easy_setopt(e, CURLOPT_SSL_VERIFYHOST, 0L); + curl_easy_setopt(e, CURLOPT_SSL_VERIFYPEER, 0L); + } + else if (!std::empty(impl->curl_ca_bundle)) + { + curl_easy_setopt(e, CURLOPT_CAINFO, impl->curl_ca_bundle.c_str()); } else { curl_easy_setopt(e, CURLOPT_SSL_CTX_FUNCTION, ssl_context_func); } - } - else - { - curl_easy_setopt(e, CURLOPT_SSL_VERIFYHOST, 0L); - curl_easy_setopt(e, CURLOPT_SSL_VERIFYPEER, 0L); - } - if (web->curl_proxy_ssl_verify) - { - if (web->curl_ca_bundle != nullptr) + if (!impl->curl_proxy_ssl_verify) { - curl_easy_setopt(e, CURLOPT_PROXY_CAINFO, web->curl_ca_bundle); + curl_easy_setopt(e, CURLOPT_PROXY_SSL_VERIFYHOST, 0L); + curl_easy_setopt(e, CURLOPT_PROXY_SSL_VERIFYPEER, 0L); } - } - else - { - curl_easy_setopt(e, CURLOPT_PROXY_SSL_VERIFYHOST, 0L); - curl_easy_setopt(e, CURLOPT_PROXY_SSL_VERIFYPEER, 0L); - } - - curl_easy_setopt(e, CURLOPT_TIMEOUT, task->timeout_secs); - curl_easy_setopt(e, CURLOPT_URL, task->url().c_str()); - curl_easy_setopt(e, CURLOPT_USERAGENT, TR_NAME "/" SHORT_VERSION_STRING); - curl_easy_setopt(e, CURLOPT_VERBOSE, (long)(web->curl_verbose ? 1 : 0)); - curl_easy_setopt(e, CURLOPT_WRITEDATA, task); - curl_easy_setopt(e, CURLOPT_WRITEFUNCTION, writeFunc); - - auto is_default_value = bool{}; - tr_address const* addr = tr_sessionGetPublicAddress(s, TR_AF_INET, &is_default_value); - if (addr != nullptr && !is_default_value) - { - (void)curl_easy_setopt(e, CURLOPT_INTERFACE, tr_address_to_string(addr)); - } - - addr = tr_sessionGetPublicAddress(s, TR_AF_INET6, &is_default_value); - if (addr != nullptr && !is_default_value) - { - (void)curl_easy_setopt(e, CURLOPT_INTERFACE, tr_address_to_string(addr)); - } - - if (auto const& cookies = task->cookies(); !std::empty(cookies)) - { - (void)curl_easy_setopt(e, CURLOPT_COOKIE, cookies.c_str()); - } - - if (auto const& filename = web->cookie_filename; !std::empty(filename)) - { - (void)curl_easy_setopt(e, CURLOPT_COOKIEFILE, filename.c_str()); - } - - if (auto const& range = task->range(); !std::empty(range)) - { - curl_easy_setopt(e, CURLOPT_RANGE, range.c_str()); - /* don't bother asking the server to compress webseed fragments */ - curl_easy_setopt(e, CURLOPT_ENCODING, "identity"); - } - - return e; -} - -static void task_finish_func(void* vtask) -{ - auto* task = static_cast(vtask); - task->done(); - delete task; -} - -/**** -***** -****/ - -static void tr_webThreadFunc(void* vsession); - -void tr_webRun(tr_session* session, tr_web_options&& options) -{ - if (session->isClosing()) - { - return; - } - - if (session->web == nullptr) - { - std::thread(tr_webThreadFunc, session).detach(); - while (session->web == nullptr) + else if (!std::empty(impl->curl_ca_bundle)) { - tr_wait_msec(20); + curl_easy_setopt(e, CURLOPT_PROXY_CAINFO, impl->curl_ca_bundle.c_str()); + } + + if (auto const& ua = impl->user_agent; !std::empty(ua)) + { + curl_easy_setopt(e, CURLOPT_USERAGENT, ua.c_str()); + } + + curl_easy_setopt(e, CURLOPT_TIMEOUT, task->timeoutSecs()); + curl_easy_setopt(e, CURLOPT_URL, task->url().c_str()); + curl_easy_setopt(e, CURLOPT_VERBOSE, impl->curl_verbose ? 1L : 0L); + curl_easy_setopt(e, CURLOPT_WRITEDATA, task); + curl_easy_setopt(e, CURLOPT_WRITEFUNCTION, onDataReceived); + + if (auto const addrstr = impl->controller.publicAddress(); addrstr) + { + (void)curl_easy_setopt(e, CURLOPT_INTERFACE, addrstr->c_str()); + } + + if (auto const& cookies = task->cookies(); cookies) + { + (void)curl_easy_setopt(e, CURLOPT_COOKIE, cookies->c_str()); + } + + if (auto const& file = impl->cookie_file; !std::empty(file)) + { + (void)curl_easy_setopt(e, CURLOPT_COOKIEFILE, file.c_str()); + } + + if (auto const& range = task->range(); range) + { + /* don't bother asking the server to compress webseed fragments */ + curl_easy_setopt(e, CURLOPT_ENCODING, "identity"); + curl_easy_setopt(e, CURLOPT_RANGE, range->c_str()); } } - auto const lock = std::unique_lock(session->web->web_tasks_mutex); - auto* const task = new tr_web_task{ session, std::move(options) }; - task->next = session->web->tasks; - session->web->tasks = task; -} - -static void tr_webThreadFunc(void* vsession) -{ - auto* session = static_cast(vsession); - - /* try to enable ssl for https support; but if that fails, - * try a plain vanilla init */ - if (curl_global_init(CURL_GLOBAL_SSL) != CURLE_OK) + void resumePausedTasks() { - curl_global_init(0); - } + TR_ASSERT(std::this_thread::get_id() == curl_thread->get_id()); - auto* const web = new tr_web{}; - web->curl_ca_bundle = tr_env_get_string("CURL_CA_BUNDLE", nullptr); - - if (web->curl_ssl_verify) - { - tr_logAddNamedInfo( - "web", - "will verify tracker certs using envvar CURL_CA_BUNDLE: %s", - web->curl_ca_bundle == nullptr ? "none" : web->curl_ca_bundle); - tr_logAddNamedInfo("web", "NB: this only works if you built against libcurl with openssl or gnutls, NOT nss"); - tr_logAddNamedInfo("web", "NB: invalid certs will show up as 'Could not connect to tracker' like many other errors"); - } - - auto const str = tr_strvPath(session->config_dir, "cookies.txt"); - if (tr_sys_path_exists(str.c_str(), nullptr)) - { - web->cookie_filename = str; - } - - auto* const multi = curl_multi_init(); - session->web = web; - - auto repeats = uint32_t{}; - for (;;) - { - if (web->close_mode == TR_WEB_CLOSE_NOW) + auto& paused = paused_easy_handles; + if (std::empty(paused)) { - break; + return; } - if (web->close_mode == TR_WEB_CLOSE_WHEN_IDLE && web->tasks == nullptr) - { - break; - } + auto const now = tr_time_msec(); - /* add tasks from the queue */ + for (auto it = std::begin(paused); it != std::end(paused);) { - auto const lock = std::unique_lock(web->web_tasks_mutex); - - while (web->tasks != nullptr) + if (it->first + BandwidthPauseMsec < now) { - /* pop the task */ - auto* const task = web->tasks; - web->tasks = task->next; - task->next = nullptr; - - dbgmsg("adding task to curl: [%s]", task->url().c_str()); - curl_multi_add_handle(multi, createEasy(session, web, task)); + curl_easy_pause(it->second, CURLPAUSE_CONT); + it = paused.erase(it); + } + else + { + ++it; } } + } - /* resume any paused curl handles. - swap paused_easy_handles to prevent oscillation - between writeFunc this while loop */ - auto paused = decltype(web->paused_easy_handles){}; - std::swap(paused, web->paused_easy_handles); - std::for_each(std::begin(paused), std::end(paused), [](auto* curl) { curl_easy_pause(curl, CURLPAUSE_CONT); }); + // the thread started by Impl.curl_thread runs this function + static void tr_webThreadFunc(void* vimpl) + { + auto* impl = static_cast(vimpl); + TR_ASSERT(std::this_thread::get_id() == impl->curl_thread->get_id()); - /* maybe wait a little while before calling curl_multi_perform() */ - auto msec = long{}; - curl_multi_timeout(multi, &msec); - if (msec < 0) + auto const multi = std::shared_ptr(curl_multi_init(), curl_multi_cleanup); + + auto repeats = unsigned{}; + for (;;) { - msec = ThreadfuncMaxSleepMsec; - } - - if (session->isClosed) - { - msec = 100; /* on shutdown, call perform() more frequently */ - } - - if (msec > 0) - { - if (msec > ThreadfuncMaxSleepMsec) + if (impl->run_mode == RunMode::CloseNow) { - msec = ThreadfuncMaxSleepMsec; + break; } + if (impl->run_mode == RunMode::CloseSoon && impl->tasks == nullptr) + { + break; + } + + /* add tasks from the queue */ + { + auto const lock = std::unique_lock(impl->web_tasks_mutex); + + while (impl->tasks != nullptr) + { + /* pop the task */ + auto* const task = impl->tasks; + impl->tasks = task->next; + task->next = nullptr; + + dbgmsg("adding task to curl: [%s]", task->url().c_str()); + initEasy(impl, task); + curl_multi_add_handle(multi.get(), task->easy()); + } + } + + impl->resumePausedTasks(); + + // Adapted from https://curl.se/libcurl/c/curl_multi_wait.html docs. + // 'numfds' being zero means either a timeout or no file descriptors to + // wait for. Try timeout on first occurrence, then assume no file + // descriptors and no file descriptors to wait for means wait for 100 + // milliseconds. auto numfds = int{}; - curl_multi_wait(multi, nullptr, 0, msec, &numfds); + curl_multi_wait(multi.get(), nullptr, 0, 1000, &numfds); if (numfds == 0) { - repeats++; - if (repeats > 1) + ++repeats; + if (repeats > 1U) { - /* curl_multi_wait() returns immediately if there are - * no fds to wait for, so we need an explicit wait here - * to emulate select() behavior */ - tr_wait_msec(std::min(msec, ThreadfuncMaxSleepMsec / 2L)); + tr_wait_msec(100); } } else { repeats = 0; } - } - /* call curl_multi_perform() */ - auto mcode = CURLMcode{}; - auto unused = int{}; - do - { - mcode = curl_multi_perform(multi, &unused); - } while (mcode == CURLM_CALL_MULTI_PERFORM); + /* call curl_multi_perform() */ + auto unused = int{}; + curl_multi_perform(multi.get(), &unused); - /* pump completed tasks from the multi */ - CURLMsg* msg = nullptr; - while ((msg = curl_multi_info_read(multi, &unused)) != nullptr) - { - if (msg->msg == CURLMSG_DONE && msg->easy_handle != nullptr) + /* pump completed tasks from the multi */ + CURLMsg* msg = nullptr; + while ((msg = curl_multi_info_read(multi.get(), &unused)) != nullptr) { - auto* const e = msg->easy_handle; + if (msg->msg == CURLMSG_DONE && msg->easy_handle != nullptr) + { + auto* const e = msg->easy_handle; - tr_web_task* task = nullptr; - curl_easy_getinfo(e, CURLINFO_PRIVATE, (void*)&task); - TR_ASSERT(e == task->curl_easy); + Task* task = nullptr; + curl_easy_getinfo(e, CURLINFO_PRIVATE, (void*)&task); - auto req_bytes_sent = long{}; - auto total_time = double{}; - curl_easy_getinfo(e, CURLINFO_RESPONSE_CODE, &task->response_code); - curl_easy_getinfo(e, CURLINFO_REQUEST_SIZE, &req_bytes_sent); - curl_easy_getinfo(e, CURLINFO_TOTAL_TIME, &total_time); - task->did_connect = task->response_code > 0 || req_bytes_sent > 0; - task->did_timeout = task->response_code == 0 && total_time >= task->timeout_secs; - curl_multi_remove_handle(multi, e); - web->paused_easy_handles.erase(e); - curl_easy_cleanup(e); - tr_runInEventThread(task->session, task_finish_func, task); + auto req_bytes_sent = long{}; + auto total_time = double{}; + curl_easy_getinfo(e, CURLINFO_REQUEST_SIZE, &req_bytes_sent); + curl_easy_getinfo(e, CURLINFO_TOTAL_TIME, &total_time); + curl_easy_getinfo(e, CURLINFO_RESPONSE_CODE, &task->response.status); + task->response.did_connect = task->response.status > 0 || req_bytes_sent > 0; + task->response.did_timeout = task->response.status == 0 && total_time >= task->timeoutSecs(); + curl_multi_remove_handle(multi.get(), e); + task->done(); + delete task; + } } } + + /* Discard any remaining tasks. + * This is rare, but can happen on shutdown with unresponsive trackers. */ + while (impl->tasks != nullptr) + { + auto* const task = impl->tasks; + impl->tasks = task->next; + dbgmsg("Discarding task \"%s\"", task->url().c_str()); + delete task; + } + + impl->is_closed_ = true; } - /* Discard any remaining tasks. - * This is rare, but can happen on shutdown with unresponsive trackers. */ - while (web->tasks != nullptr) +private: + std::shared_ptr const curlsh_{ curl_share_init(), curl_share_cleanup }; + + CURLSH* shared() { - auto* const task = web->tasks; - web->tasks = task->next; - dbgmsg("Discarding task \"%s\"", task->url().c_str()); - delete task; + return curlsh_.get(); } - /* cleanup */ - curl_multi_cleanup(multi); - tr_free(web->curl_ca_bundle); - delete web; - session->web = nullptr; -} + static std::once_flag curl_init_flag; -void tr_webClose(tr_session* session, tr_web_close_mode close_mode) + bool is_closed_ = false; + + std::multimap paused_easy_handles; + + static void curlInit() + { + // try to enable ssl for https support; + // but if that fails, try a plain vanilla init + if (curl_global_init(CURL_GLOBAL_SSL) != CURLE_OK) + { + curl_global_init(0); + } + } +}; + +std::once_flag tr_web::Impl::curl_init_flag; + +tr_web::tr_web(Controller& controller) + : impl_{ std::make_unique(controller) } { - if (session->web != nullptr) - { - session->web->close_mode = close_mode; - - if (close_mode == TR_WEB_CLOSE_NOW) - { - while (session->web != nullptr) - { - tr_wait_msec(100); - } - } - } +} + +tr_web::~tr_web() = default; + +std::unique_ptr tr_web::create(Controller& controller) +{ + return std::unique_ptr(new tr_web(controller)); +} + +void tr_web::fetch(FetchOptions&& options) +{ + impl_->fetch(std::move(options)); +} + +bool tr_web::isClosed() const +{ + return impl_->isClosed(); +} + +void tr_web::closeSoon() +{ + impl_->closeSoon(); } diff --git a/libtransmission/web.h b/libtransmission/web.h index 99d935887..c6b53ae81 100644 --- a/libtransmission/web.h +++ b/libtransmission/web.h @@ -5,48 +5,142 @@ #pragma once -#include #include +#include #include #include -#include "transmission.h" - struct evbuffer; -enum tr_web_close_mode -{ - TR_WEB_CLOSE_WHEN_IDLE, - TR_WEB_CLOSE_NOW -}; - -void tr_webClose(tr_session* session, tr_web_close_mode close_mode); - -using tr_web_done_func = void (*)( - tr_session* session, - bool did_connect_flag, - bool timeout_flag, - long response_code, - std::string_view response, - void* user_data); - -class tr_web_options +class tr_web { public: - tr_web_options(std::string_view url_in, tr_web_done_func done_func_in, void* done_func_user_data_in) - : url{ url_in } - , done_func{ done_func_in } - , done_func_user_data{ done_func_user_data_in } + // The response struct passed to the user's FetchDoneFunc callback + // when a fetch() finishes. + struct FetchResponse { - } + long status; // http server response, e.g. 200 + std::string body; + bool did_connect; + bool did_timeout; + void* user_data; + }; - std::string url; - std::optional torrent_id; - tr_web_done_func done_func = nullptr; - void* done_func_user_data = nullptr; - std::string range; - std::string cookies; - evbuffer* buffer = nullptr; + using FetchDoneFunc = void (*)(FetchResponse&& response); + + class FetchOptions + { + public: + FetchOptions(std::string_view url_in, FetchDoneFunc done_func_in, void* done_func_user_data_in) + : url{ url_in } + , done_func{ done_func_in } + , done_func_user_data{ done_func_user_data_in } + { + } + + // the URL to fetch + std::string url; + + // Callback to invoke with a FetchResponse when done + FetchDoneFunc done_func = nullptr; + void* done_func_user_data = nullptr; + + // If you need to set multiple cookies, set them all using a single + // option concatenated like this: "name1=content1; name2=content2;" + std::optional cookies; + + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests + std::optional range; + + // Tag used by tr_web::Controller to limit some transfers' bandwidth + std::optional speed_limit_tag; + + // Optionaly set the underlying sockets' send/receive buffers' size. + // Can be useful for scrapes / announces where the payload is known + // to be small. + std::optional sndbuf; + std::optional rcvbuf; + + // Maximum time to wait before timeout + int timeout_secs = DefaultTimeoutSecs; + + // If provided, this buffer will be used to hold the response body. + // Provided for webseeds, which need to set low-level callbacks on + // the buffer itself. + evbuffer* buffer = nullptr; + + static constexpr int DefaultTimeoutSecs = 120; + }; + + void fetch(FetchOptions&& options); + + // Notify tr_web that it's going to be destroyed sooon. + // New fetch() tasks will be rejected, but already-running tasks + // are left alone so that they can finish. + void closeSoon(); + + // True when tr_web is ready to be destroyed. + // Will never be true until after closeSoon() is called. + [[nodiscard]] bool isClosed() const; + + // closeSoon() *should* be called first, but OK to destroy tr_web before + // isClosed() is true, e.g. there could be a hung fetch task that hasn't + // timmed out yet. Deleting the tr_web object will force-terminate any + // pending tasks. + ~tr_web(); + + /** + * Mediates between tr_web and its clients. + * + * NB: Note that tr_web calls all these methods in the web thread. + */ + class Controller + { + public: + virtual ~Controller() = default; + + // Return the location of the cookie file, or nullopt to not use one + [[nodiscard]] virtual std::optional cookieFile() const + { + return std::nullopt; + } + + // Return the preferred user public address string, or nullopt to not use one + [[nodiscard]] virtual std::optional publicAddress() const + { + return std::nullopt; + } + + // Return the preferred user aagent, or nullopt to not use one + [[nodiscard]] virtual std::optional userAgent() const + { + return std::nullopt; + } + + // Notify the system that `byte_count` of download bandwidth was used + virtual void notifyBandwidthConsumed([[maybe_unused]] int bandwidth_tag, [[maybe_unused]] size_t byte_count) + { + } + + // Return the number of bytes that should be allowed. See Bandwidth::clamp() + [[nodiscard]] virtual unsigned int clamp([[maybe_unused]] int bandwidth_tag, unsigned int byte_count) const + { + return byte_count; + } + + // Invoke the user-provided fetch callback + virtual void run(FetchDoneFunc func, FetchResponse&& response) const + { + func(std::move(response)); + } + }; + + static std::unique_ptr create(Controller& controller); + +private: + class Impl; + std::unique_ptr const impl_; + explicit tr_web(Controller& controller); }; -void tr_webRun(tr_session* session, tr_web_options&& options); +void tr_sessionFetch(struct tr_session* session, tr_web::FetchOptions&& options); diff --git a/libtransmission/webseed.cc b/libtransmission/webseed.cc index 8da2d6000..b0caf9eec 100644 --- a/libtransmission/webseed.cc +++ b/libtransmission/webseed.cc @@ -405,17 +405,14 @@ void on_idle(tr_webseed* w) } } -void web_response_func( - tr_session* session, - bool /*did_connect*/, - bool /*did_timeout*/, - long response_code, - std::string_view /*response*/, - void* vtask) +void onWebResponse(tr_web::FetchResponse&& web_response) { + auto const& [status, body, did_connect, did_timeout, vtask] = web_response; + bool const success = status == 206; + auto* const t = static_cast(vtask); - bool const success = response_code == 206; - tr_webseed* w = t->webseed; + auto* const session = t->session; + auto* const w = t->webseed; w->connection_limiter.taskFinished(success); @@ -511,11 +508,11 @@ void task_request_next_chunk(tr_webseed_task* t) uint64_t this_pass = std::min(remain, tor->fileSize(file_index) - file_offset); auto const url = make_url(t->webseed, tor->fileSubpath(file_index)); - auto options = tr_web_options{ url, web_response_func, t }; + auto options = tr_web::FetchOptions{ url, onWebResponse, t }; options.range = tr_strvJoin(std::to_string(file_offset), "-"sv, std::to_string(file_offset + this_pass - 1)); - options.torrent_id = tor->uniqueId; + options.speed_limit_tag = tor->uniqueId; options.buffer = t->content(); - tr_webRun(tor->session, std::move(options)); + tor->session->web->fetch(std::move(options)); } } // namespace