From d6032f829b7baf43c4eb79f60f1f2827c2668db0 Mon Sep 17 00:00:00 2001 From: Charles Kerr Date: Fri, 29 Oct 2021 13:24:30 -0500 Subject: [PATCH] refactor: remove tr_piece struct (#2059) * refactor: remove tr_piece struct --- libtransmission/completion.cc | 4 +- libtransmission/inout.cc | 49 +++---- libtransmission/metainfo.cc | 9 +- libtransmission/peer-mgr.cc | 10 +- libtransmission/peer-msgs.cc | 5 +- libtransmission/resume.cc | 236 ++++++++++++++------------------- libtransmission/torrent.cc | 114 +++------------- libtransmission/torrent.h | 114 ++++++++++++++-- libtransmission/tr-macros.h | 15 ++- libtransmission/transmission.h | 18 +-- libtransmission/verify.cc | 14 +- qt/OptionsDialog.cc | 2 +- 12 files changed, 281 insertions(+), 309 deletions(-) diff --git a/libtransmission/completion.cc b/libtransmission/completion.cc index df5ba3f77..752382e63 100644 --- a/libtransmission/completion.cc +++ b/libtransmission/completion.cc @@ -113,7 +113,7 @@ void tr_cpBlockAdd(tr_completion* cp, tr_block_index_t block) cp->sizeNow += tr_torBlockCountBytes(tor, block); cp->haveValidIsDirty = true; - cp->sizeWhenDoneIsDirty = cp->sizeWhenDoneIsDirty || tor->info.pieces[piece].dnd; + cp->sizeWhenDoneIsDirty = cp->sizeWhenDoneIsDirty || tor->pieceIsDnd(piece); } } @@ -165,7 +165,7 @@ uint64_t tr_cpSizeWhenDone(tr_completion const* ccp) uint64_t n = 0; uint64_t const pieceSize = tr_torPieceCountBytes(tor, p); - if (!inf->pieces[p].dnd) + if (!tor->pieceIsDnd(p)) { n = pieceSize; } diff --git a/libtransmission/inout.cc b/libtransmission/inout.cc index 786943847..f4bb34fc5 100644 --- a/libtransmission/inout.cc +++ b/libtransmission/inout.cc @@ -10,6 +10,7 @@ #include #include /* bsearch() */ #include /* memcmp() */ +#include #include "transmission.h" #include "cache.h" /* tr_cacheReadBlock() */ @@ -256,49 +257,39 @@ int tr_ioWrite(tr_torrent* tor, tr_piece_index_t pieceIndex, uint32_t begin, uin ***** ****/ -static bool recalculateHash(tr_torrent* tor, tr_piece_index_t pieceIndex, uint8_t* setme) +static std::optional recalculateHash(tr_torrent* tor, tr_piece_index_t piece) { TR_ASSERT(tor != nullptr); - TR_ASSERT(pieceIndex < tor->info.pieceCount); - TR_ASSERT(setme != nullptr); + TR_ASSERT(piece < tor->info.pieceCount); - uint32_t offset = 0; - bool success = true; - size_t const buflen = tor->blockSize; - void* const buffer = tr_malloc(buflen); + auto bytes_left = size_t{ tr_torPieceCountBytes(tor, piece) }; + auto offset = uint32_t{}; + tr_ioPrefetch(tor, piece, offset, bytes_left); - TR_ASSERT(buffer != nullptr); - TR_ASSERT(buflen > 0); - - tr_sha1_ctx_t sha = tr_sha1_init(); - size_t bytesLeft = tr_torPieceCountBytes(tor, pieceIndex); - - tr_ioPrefetch(tor, pieceIndex, offset, bytesLeft); - - while (bytesLeft != 0) + auto sha = tr_sha1_init(); + auto buffer = std::vector(tor->blockSize); + while (bytes_left != 0) { - size_t const len = std::min(bytesLeft, buflen); - success = tr_cacheReadBlock(tor->session->cache, tor, pieceIndex, offset, len, static_cast(buffer)) == 0; - + size_t const len = std::min(bytes_left, std::size(buffer)); + auto const success = tr_cacheReadBlock(tor->session->cache, tor, piece, offset, len, std::data(buffer)) == 0; if (!success) { - break; + tr_sha1_final(sha, nullptr); + return {}; } - tr_sha1_update(sha, buffer, len); + tr_sha1_update(sha, std::data(buffer), len); offset += len; - bytesLeft -= len; + bytes_left -= len; } - tr_sha1_final(sha, success ? setme : nullptr); - - tr_free(buffer); - return success; + auto digest = tr_sha1_digest_t{}; + tr_sha1_final(sha, std::data(digest)); + return digest; } bool tr_ioTestPiece(tr_torrent* tor, tr_piece_index_t piece) { - uint8_t hash[SHA_DIGEST_LENGTH]; - - return recalculateHash(tor, piece, hash) && memcmp(hash, tor->info.pieces[piece].hash, SHA_DIGEST_LENGTH) == 0; + auto const hash = recalculateHash(tor, piece); + return hash && *hash == tor->info.pieces[piece]; } diff --git a/libtransmission/metainfo.cc b/libtransmission/metainfo.cc index 143315633..e292b12d0 100644 --- a/libtransmission/metainfo.cc +++ b/libtransmission/metainfo.cc @@ -14,6 +14,7 @@ #include #include "transmission.h" + #include "crypto-utils.h" /* tr_sha1 */ #include "file.h" #include "log.h" @@ -709,12 +710,8 @@ static char const* tr_metainfoParseImpl( } inf->pieceCount = len / SHA_DIGEST_LENGTH; - inf->pieces = tr_new0(tr_piece, inf->pieceCount); - - for (tr_piece_index_t pi = 0; pi < inf->pieceCount; ++pi) - { - memcpy(inf->pieces[pi].hash, &raw[pi * SHA_DIGEST_LENGTH], SHA_DIGEST_LENGTH); - } + inf->pieces = tr_new0(tr_sha1_digest_t, inf->pieceCount); + std::copy_n(raw, len, (uint8_t*)(inf->pieces)); } /* files */ diff --git a/libtransmission/peer-mgr.cc b/libtransmission/peer-mgr.cc index 8224115b1..642945f63 100644 --- a/libtransmission/peer-mgr.cc +++ b/libtransmission/peer-mgr.cc @@ -801,8 +801,8 @@ static int comparePieceByWeight(void const* va, void const* vb) } /* secondary key: higher priorities go first */ - ia = tor->info.pieces[a->index].priority; - ib = tor->info.pieces[b->index].priority; + ia = tor->piecePriority(a->index); + ib = tor->piecePriority(b->index); if (ia != ib) { return ia > ib ? -1 : 1; @@ -900,7 +900,7 @@ static void pieceListRebuild(tr_swarm* s) auto* const pool = tr_new(tr_piece_index_t, inf->pieceCount); for (tr_piece_index_t i = 0; i < inf->pieceCount; ++i) { - if (!inf->pieces[i].dnd && !tr_torrentPieceIsComplete(tor, i)) + if (!tor->pieceIsDnd(i) && !tr_torrentPieceIsComplete(tor, i)) { pool[poolCount++] = i; } @@ -2378,7 +2378,7 @@ uint64_t tr_peerMgrGetDesiredAvailable(tr_torrent const* tor) for (size_t i = 0; i < n_pieces; ++i) { - if (!tor->info.pieces[i].dnd && have.at(i)) + if (!tor->pieceIsDnd(i) && have.at(i)) { desired_available += tr_torrentMissingBytesInPiece(tor, i); } @@ -2706,7 +2706,7 @@ static void rechokeDownloads(tr_swarm* s) for (int i = 0; i < n; ++i) { - piece_is_interesting[i] = !tor->info.pieces[i].dnd && !tr_torrentPieceIsComplete(tor, i); + piece_is_interesting[i] = !tor->pieceIsDnd(i) && !tr_torrentPieceIsComplete(tor, i); } /* decide WHICH peers to be interested in (based on their cancel-to-block ratio) */ diff --git a/libtransmission/peer-msgs.cc b/libtransmission/peer-msgs.cc index 86e6b0747..d810591f1 100644 --- a/libtransmission/peer-msgs.cc +++ b/libtransmission/peer-msgs.cc @@ -2216,10 +2216,9 @@ static size_t fillOutputBuffer(tr_peerMsgsImpl* msgs, time_t now) evbuffer_commit_space(out, iovec, 1); /* check the piece if it needs checking... */ - if (!err && tr_torrentPieceNeedsCheck(msgs->torrent, req.index)) + if (!err) { - err = !tr_torrentCheckPiece(msgs->torrent, req.index); - + err = !msgs->torrent->ensurePieceIsChecked(req.index); if (err) { tr_torrentSetLocalError( diff --git a/libtransmission/resume.cc b/libtransmission/resume.cc index 8d8445751..bc3cee59f 100644 --- a/libtransmission/resume.cc +++ b/libtransmission/resume.cc @@ -465,17 +465,16 @@ static uint64_t loadFilenames(tr_variant* dict, tr_torrent* tor) **** ***/ -// TODO: Refactor this into a constructor for tr_variant -static void bitfieldToBenc(tr_bitfield const* b, tr_variant* benc) +static void bitfieldToRaw(tr_bitfield const* b, tr_variant* benc) { - if (b->hasAll()) - { - tr_variantInitStr(benc, "all"sv); - } - else if (b->hasNone()) + if (b->hasNone() || std::size(*b) == 0) { tr_variantInitStr(benc, "none"sv); } + else if (b->hasAll()) + { + tr_variantInitStr(benc, "all"sv); + } else { auto const raw = b->raw(); @@ -483,76 +482,39 @@ static void bitfieldToBenc(tr_bitfield const* b, tr_variant* benc) } } +static void rawToBitfield(tr_bitfield& bitfield, uint8_t const* raw, size_t rawlen) +{ + if (raw == nullptr || rawlen == 0 || (rawlen == 4 && memcmp(raw, "none", 4) == 0)) + { + bitfield.setHasNone(); + } + else if (rawlen == 3 && memcmp(raw, "all", 3) == 0) + { + bitfield.setHasAll(); + } + else + { + bitfield.setRaw(raw, rawlen, true); + } +} + static void saveProgress(tr_variant* dict, tr_torrent* tor) { tr_info const* inf = tr_torrentInfo(tor); - time_t const now = tr_time(); - tr_variant* const prog = tr_variantDictAddDict(dict, TR_KEY_progress, 3); + tr_variant* const prog = tr_variantDictAddDict(dict, TR_KEY_progress, 4); - /* add the file/piece check timestamps... */ - tr_variant* const l = tr_variantDictAddList(prog, TR_KEY_time_checked, inf->fileCount); - - for (tr_file_index_t fi = 0; fi < inf->fileCount; ++fi) + // add the mtimes + size_t const n = inf->fileCount; + tr_variant* const l = tr_variantDictAddList(prog, TR_KEY_mtimes, n); + for (auto const *file = inf->files, *end = file + inf->fileCount; file != end; ++file) { - time_t oldest_nonzero = now; - time_t newest = 0; - bool has_zero = false; - time_t const mtime = tr_torrentGetFileMTime(tor, fi); - tr_file const* f = &inf->files[fi]; - - /* get the oldest and newest nonzero timestamps for pieces in this file */ - for (tr_piece_index_t i = f->firstPiece; i <= f->lastPiece; ++i) - { - tr_piece const* const p = &inf->pieces[i]; - - if (p->timeChecked == 0) - { - has_zero = true; - } - else if (oldest_nonzero > p->timeChecked) - { - oldest_nonzero = p->timeChecked; - } - - if (newest < p->timeChecked) - { - newest = p->timeChecked; - } - } - - /* If some of a file's pieces have been checked more recently than - the file's mtime, and some less recently, then that file will - have a list containing timestamps for each piece. - - However, the most common use case is that the file doesn't change - after it's downloaded. To reduce overhead in the .resume file, - only a single timestamp is saved for the file if *all* or *none* - of the pieces were tested more recently than the file's mtime. */ - - if (!has_zero && mtime <= oldest_nonzero) /* all checked */ - { - tr_variantListAddInt(l, oldest_nonzero); - } - else if (newest < mtime) /* none checked */ - { - tr_variantListAddInt(l, newest); - } - else /* some are checked, some aren't... so list piece by piece */ - { - int const offset = oldest_nonzero - 1; - tr_variant* ll = tr_variantListAddList(l, 2 + f->lastPiece - f->firstPiece); - tr_variantListAddInt(ll, offset); - - for (tr_piece_index_t i = f->firstPiece; i <= f->lastPiece; ++i) - { - tr_piece const* const p = &inf->pieces[i]; - - tr_variantListAddInt(ll, p->timeChecked != 0 ? p->timeChecked - offset : 0); - } - } + tr_variantListAddInt(l, file->mtime); } + // add the 'checked pieces' bitfield + bitfieldToRaw(&tor->checked_pieces_, tr_variantDictAdd(prog, TR_KEY_pieces)); + /* add the progress */ if (tor->completeness == TR_SEED) { @@ -560,97 +522,113 @@ static void saveProgress(tr_variant* dict, tr_torrent* tor) } /* add the blocks bitfield */ - bitfieldToBenc(tor->completion.blockBitfield, tr_variantDictAdd(prog, TR_KEY_blocks)); + bitfieldToRaw(tor->completion.blockBitfield, tr_variantDictAdd(prog, TR_KEY_blocks)); } +/* + * Transmisison has iterated through a few strategies here, so the + * code has some added complexity to support older approaches. + * + * Current approach: 'progress' is a dict with two entries: + * - 'pieces' a bitfield for whether each piece has been checked. + * - 'mtimes', an array of per-file timestamps + * On startup, 'pieces' is loaded. Then we check to see if the disk + * mtimes differ from the 'mtimes' list. Changed files have their + * pieces cleared from the bitset. + * + * Second approach (2.20 - 3.00): the 'progress' dict had a + * 'time_checked' entry which was a list with fileCount items. + * Each item was either a list of per-piece timestamps, or a + * single timestamp if either all or none of the pieces had been + * tested more recently than the file's mtime. + * + * First approach (pre-2.20) had an "mtimes" list identical to + * 3.10, but not the 'pieces' bitfield. + */ static uint64_t loadProgress(tr_variant* dict, tr_torrent* tor) { auto ret = uint64_t{}; tr_info const* inf = tr_torrentInfo(tor); - for (size_t i = 0; i < inf->pieceCount; ++i) - { - inf->pieces[i].timeChecked = 0; - } - tr_variant* prog = nullptr; if (tr_variantDictFindDict(dict, TR_KEY_progress, &prog)) { + /// CHECKED PIECES + + auto checked = tr_bitfield(inf->pieceCount); + auto mtimes = std::vector{}; + mtimes.reserve(inf->fileCount); + + // try to load mtimes tr_variant* l = nullptr; + if (tr_variantDictFindList(prog, TR_KEY_mtimes, &l)) + { + auto fi = size_t{}; + auto t = int64_t{}; + while (tr_variantGetInt(tr_variantListChild(l, fi++), &t)) + { + mtimes.push_back(t); + } + } + + // try to load the piece-checked bitfield + uint8_t const* raw = nullptr; + auto rawlen = size_t{}; + if (tr_variantDictFindRaw(prog, TR_KEY_pieces, &raw, &rawlen)) + { + rawToBitfield(checked, raw, rawlen); + } + + // maybe it's a .resume file from [2.20 - 3.00] with the per-piece mtimes if (tr_variantDictFindList(prog, TR_KEY_time_checked, &l)) { - /* per-piece timestamps were added in 2.20. - - If some of a file's pieces have been checked more recently than - the file's mtime, and some lest recently, then that file will - have a list containing timestamps for each piece. - - However, the most common use case is that the file doesn't change - after it's downloaded. To reduce overhead in the .resume file, - only a single timestamp is saved for the file if *all* or *none* - of the pieces were tested more recently than the file's mtime. */ - for (tr_file_index_t fi = 0; fi < inf->fileCount; ++fi) { - tr_variant* b = tr_variantListChild(l, fi); - tr_file const* f = &inf->files[fi]; + tr_variant* const b = tr_variantListChild(l, fi); + tr_file* const f = &inf->files[fi]; + auto time_checked = time_t{}; if (tr_variantIsInt(b)) { auto t = int64_t{}; tr_variantGetInt(b, &t); - - for (tr_piece_index_t i = f->firstPiece; i <= f->lastPiece; ++i) - { - inf->pieces[i].timeChecked = (time_t)t; - } + time_checked = time_t(t); } else if (tr_variantIsList(b)) { - int64_t offset = 0; - int const pieces = f->lastPiece + 1 - f->firstPiece; - + auto offset = int64_t{}; tr_variantGetInt(tr_variantListChild(b, 0), &offset); - for (int i = 0; i < pieces; ++i) + time_checked = tr_time(); + size_t const pieces = f->lastPiece + 1 - f->firstPiece; + for (size_t i = 0; i < pieces; ++i) { - int64_t t = 0; - tr_variantGetInt(tr_variantListChild(b, i + 1), &t); - inf->pieces[f->firstPiece + i].timeChecked = (time_t)(t != 0 ? t + offset : 0); + int64_t piece_time = 0; + tr_variantGetInt(tr_variantListChild(b, i + 1), &piece_time); + time_checked = std::min(time_checked, time_t(piece_time)); } } + + mtimes.push_back(time_checked); } } - else if (tr_variantDictFindList(prog, TR_KEY_mtimes, &l)) + + if (std::size(mtimes) != tor->info.fileCount) { - /* Before 2.20, we stored the files' mtimes in the .resume file. - When loading the .resume file, a torrent's file would be flagged - as untested if its stored mtime didn't match its real mtime. */ - - for (tr_file_index_t fi = 0; fi < inf->fileCount; ++fi) - { - auto t = int64_t{}; - - if (tr_variantGetInt(tr_variantListChild(l, fi), &t)) - { - tr_file const* f = &inf->files[fi]; - time_t const mtime = tr_torrentGetFileMTime(tor, fi); - time_t const timeChecked = mtime == t ? mtime : 0; - - for (tr_piece_index_t i = f->firstPiece; i <= f->lastPiece; ++i) - { - inf->pieces[i].timeChecked = timeChecked; - } - } - } + tr_logAddTorErr(tor, "got %zu mtimes; expected %zu", std::size(mtimes), size_t(tor->info.fileCount)); + // if resizing grows the vector, we'll get 0 mtimes for the + // new items which is exactly what we want since the pieces + // in an unknown state should be treated as untested + mtimes.resize(tor->info.fileCount); } + tor->initCheckedPieces(checked, std::data(mtimes)); + + /// COMPLETION + auto blocks = tr_bitfield{ tor->blockCount }; - - auto rawlen = size_t{}; char const* err = nullptr; char const* str = nullptr; - uint8_t const* raw = nullptr; tr_variant const* const b = tr_variantDictFind(prog, TR_KEY_blocks); if (b != nullptr) { @@ -661,17 +639,9 @@ static uint64_t loadProgress(tr_variant* dict, tr_torrent* tor) { err = "Invalid value for \"blocks\""; } - else if (buflen == 3 && memcmp(buf, "all", 3) == 0) - { - blocks.setHasAll(); - } - else if (buflen == 4 && memcmp(buf, "none", 4) == 0) - { - blocks.setHasNone(); - } else { - blocks.setRaw(buf, buflen, true); + rawToBitfield(blocks, buf, buflen); } } else if (tr_variantDictFindStr(prog, TR_KEY_have, &str, nullptr)) diff --git a/libtransmission/torrent.cc b/libtransmission/torrent.cc index 63d7ee83d..be04f7bc2 100644 --- a/libtransmission/torrent.cc +++ b/libtransmission/torrent.cc @@ -700,9 +700,10 @@ static void tr_torrentInitFilePieces(tr_torrent* tor) #endif + tor->piece_priorities_.clear(); for (tr_piece_index_t p = 0; p < inf->pieceCount; ++p) { - inf->pieces[p].priority = calculatePiecePriority(*inf, p, firstFiles[p]); + tor->setPiecePriority(p, calculatePiecePriority(*inf, p, firstFiles[p])); } tr_free(firstFiles); @@ -862,6 +863,9 @@ static void torrentInit(tr_torrent* tor, tr_ctor const* ctor) tor->magicNumber = TORRENT_MAGIC_NUMBER; tor->queuePosition = tr_sessionCountTorrents(session); + tor->dnd_pieces_ = tr_bitfield{ tor->info.pieceCount }; + tor->checked_pieces_ = tr_bitfield{ tor->info.pieceCount }; + tr_sha1(tor->obfuscatedHash, "req2", 4, tor->info.hash, SHA_DIGEST_LENGTH, nullptr); char const* dir = nullptr; @@ -1234,24 +1238,7 @@ static inline bool tr_torrentIsStalled(tr_torrent const* tor, int idle_secs) static double getVerifyProgress(tr_torrent const* tor) { - double d = 0; - - if (tr_torrentHasMetadata(tor)) - { - tr_piece_index_t checked = 0; - - for (tr_piece_index_t i = 0; i < tor->info.pieceCount; ++i) - { - if (tor->info.pieces[i].timeChecked != 0) - { - ++checked; - } - } - - d = checked / (double)tor->info.pieceCount; - } - - return d; + return tor->verify_progress ? *tor->verify_progress : 0.0; } tr_stat const* tr_torrentStat(tr_torrent* tor) @@ -2262,7 +2249,7 @@ void tr_torrentInitFilePriority(tr_torrent* tor, tr_file_index_t fileIndex, tr_p for (tr_piece_index_t i = file->firstPiece; i <= file->lastPiece; ++i) { - info.pieces[i].priority = calculatePiecePriority(info, i, fileIndex); + tor->setPiecePriority(i, calculatePiecePriority(info, i, fileIndex)); } } @@ -2353,18 +2340,19 @@ static void setFileDND(tr_torrent* tor, tr_file_index_t fileIndex, bool doDownlo lastPieceDND = tor->info.files[i].dnd; } + // update dnd_pieces_ + if (firstPiece == lastPiece) { - tor->info.pieces[firstPiece].dnd = firstPieceDND && lastPieceDND; + tor->dnd_pieces_.set(firstPiece, firstPieceDND && lastPieceDND); } else { - tor->info.pieces[firstPiece].dnd = firstPieceDND; - tor->info.pieces[lastPiece].dnd = lastPieceDND; - + tor->dnd_pieces_.set(firstPiece, firstPieceDND); + tor->dnd_pieces_.set(lastPiece, lastPieceDND); for (tr_piece_index_t pp = firstPiece + 1; pp < lastPiece; ++pp) { - tor->info.pieces[pp].dnd = dnd; + tor->dnd_pieces_.set(pp, dnd); } } } @@ -2590,34 +2578,11 @@ tr_block_range tr_torGetPieceBlockRange(tr_torrent const* tor, tr_piece_index_t **** ***/ -void tr_torrentSetPieceChecked(tr_torrent* tor, tr_piece_index_t pieceIndex) +// TODO: should be const after tr_ioTestPiece() is const +bool tr_torrent::checkPiece(tr_piece_index_t piece) { - TR_ASSERT(tr_isTorrent(tor)); - TR_ASSERT(pieceIndex < tor->info.pieceCount); - - tor->info.pieces[pieceIndex].timeChecked = tr_time(); -} - -void tr_torrentSetChecked(tr_torrent* tor, time_t when) -{ - TR_ASSERT(tr_isTorrent(tor)); - - for (tr_piece_index_t i = 0; i < tor->info.pieceCount; ++i) - { - tor->info.pieces[i].timeChecked = when; - } -} - -bool tr_torrentCheckPiece(tr_torrent* tor, tr_piece_index_t pieceIndex) -{ - bool const pass = tr_ioTestPiece(tor, pieceIndex); - - tr_deeplog_tor(tor, "[LAZY] tr_torrentCheckPiece tested piece %zu, pass==%d", (size_t)pieceIndex, (int)pass); - tr_torrentSetHasPiece(tor, pieceIndex, pass); - tr_torrentSetPieceChecked(tor, pieceIndex); - tor->anyDate = tr_time(); - tr_torrentSetDirty(tor); - + bool const pass = tr_ioTestPiece(this, piece); + tr_logAddTorDbg(this, "[LAZY] tr_torrent.checkPiece tested piece %zu, pass==%d", size_t(piece), int(pass)); return pass; } @@ -2633,38 +2598,6 @@ time_t tr_torrentGetFileMTime(tr_torrent const* tor, tr_file_index_t i) return mtime; } -bool tr_torrentPieceNeedsCheck(tr_torrent const* tor, tr_piece_index_t p) -{ - tr_info const* const inf = tr_torrentInfo(tor); - if (inf == nullptr) - { - return false; - } - - /* if we've never checked this piece, then it needs to be checked */ - if (inf->pieces[p].timeChecked == 0) - { - return true; - } - - /* If we think we've completed one of the files in this piece, - * but it's been modified since we last checked it, - * then it needs to be rechecked */ - auto f = tr_file_index_t{}; - auto unused = uint64_t{}; - tr_ioFindFileLocation(tor, p, 0, &f, &unused); - - for (tr_file_index_t i = f; i < inf->fileCount && pieceHasFile(p, &inf->files[i]); ++i) - { - if (tr_cpFileIsComplete(&tor->completion, i) && (tr_torrentGetFileMTime(tor, i) > inf->pieces[p].timeChecked)) - { - return true; - } - } - - return false; -} - /*** **** ***/ @@ -3277,8 +3210,8 @@ std::string_view tr_torrentPrimaryMimeType(tr_torrent const* tor) static void tr_torrentFileCompleted(tr_torrent* tor, tr_file_index_t fileIndex) { - tr_info const* inf = &tor->info; - tr_file const* f = &inf->files[fileIndex]; + tr_info const* const inf = &tor->info; + tr_file* const f = &inf->files[fileIndex]; time_t const now = tr_time(); /* close the file so that we can reopen in read-only mode as needed */ @@ -3287,10 +3220,7 @@ static void tr_torrentFileCompleted(tr_torrent* tor, tr_file_index_t fileIndex) /* now that the file is complete and closed, we can start watching its * mtime timestamp for changes to know if we need to reverify pieces */ - for (tr_piece_index_t i = f->firstPiece; i <= f->lastPiece; ++i) - { - inf->pieces[i].timeChecked = now; - } + f->mtime = now; /* if the torrent's current filename isn't the same as the one in the * metadata -- for example, if it had the ".part" suffix appended to @@ -3351,9 +3281,7 @@ void tr_torrentGotBlock(tr_torrent* tor, tr_block_index_t block) if (tr_torrentPieceIsComplete(tor, p)) { - tr_logAddTorDbg(tor, "[LAZY] checking just-completed piece %zu", (size_t)p); - - if (tr_torrentCheckPiece(tor, p)) + if (tor->checkPiece(p)) { tr_torrentPieceCompleted(tor, p); } diff --git a/libtransmission/torrent.h b/libtransmission/torrent.h index 6c029a2a6..cb7ff4d14 100644 --- a/libtransmission/torrent.h +++ b/libtransmission/torrent.h @@ -1,5 +1,5 @@ /* - * This file Copyright (C) 2009-2014 Mnemosyne LLC + * This file Copyright (C) 2009-2021 Mnemosyne LLC * * It may be used under the GNU GPL versions 2 or 3 * or any future license endorsed by Mnemosyne LLC. @@ -15,10 +15,12 @@ #include #include #include +#include #include #include #include "bandwidth.h" /* tr_bandwidth */ +#include "bitfield.h" #include "completion.h" /* tr_completion */ #include "session.h" /* tr_sessionLock(), tr_sessionUnlock() */ #include "tr-assert.h" @@ -26,6 +28,7 @@ #include "utils.h" /* TR_GNUC_PRINTF */ class tr_swarm; +struct tr_torrent; struct tr_torrent_tiers; struct tr_magnet_info; @@ -93,10 +96,6 @@ tr_block_range tr_torGetPieceBlockRange(tr_torrent const* tor, tr_piece_index_t void tr_torrentInitFilePriority(tr_torrent* tor, tr_file_index_t fileIndex, tr_priority_t priority); -void tr_torrentSetPieceChecked(tr_torrent* tor, tr_piece_index_t piece); - -void tr_torrentSetChecked(tr_torrent* tor, time_t when); - void tr_torrentCheckSeedLimit(tr_torrent* tor); /** save a torrent's .resume file if it's changed since the last time it was saved */ @@ -110,6 +109,8 @@ void tr_torrentSetDateActive(tr_torrent* torrent, time_t activityDate); void tr_torrentSetDateDone(tr_torrent* torrent, time_t doneDate); +time_t tr_torrentGetFileMTime(tr_torrent const* tor, tr_file_index_t i); + /** Return the mime-type (e.g. "audio/x-flac") that matches more of the torrent's content than any other mime-type. */ std::string_view tr_torrentPrimaryMimeType(tr_torrent const* tor); @@ -135,10 +136,100 @@ struct tr_torrent int magicNumber; + std::optional verify_progress; + std::vector piece_checksums; + tr_stat_errtype error; char errorString[128]; char errorTracker[128]; + /// DND + + tr_bitfield dnd_pieces_ = tr_bitfield{ 0 }; + + bool pieceIsDnd(tr_piece_index_t piece) const + { + return dnd_pieces_.test(piece); + } + + /// PRIORITIES + + // since 'TR_PRI_NORMAL' is by far the most common, save some + // space by treating anything not in the map as normal + std::unordered_map piece_priorities_; + + void setPiecePriority(tr_piece_index_t piece, tr_priority_t priority) + { + if (priority == TR_PRI_NORMAL) + { + piece_priorities_.erase(piece); + + if (std::empty(piece_priorities_)) + { + // ensure we release piece_priorities_' internal memory + piece_priorities_ = decltype(piece_priorities_){}; + } + } + else + { + piece_priorities_[piece] = priority; + } + } + + tr_priority_t piecePriority(tr_piece_index_t piece) const + { + auto const it = piece_priorities_.find(piece); + if (it == std::end(piece_priorities_)) + { + return TR_PRI_NORMAL; + } + return it->second; + } + + /// CHECKSUMS + + tr_bitfield checked_pieces_ = tr_bitfield{ 0 }; + + bool checkPiece(tr_piece_index_t piece); + + bool ensurePieceIsChecked(tr_piece_index_t piece) + { + TR_ASSERT(piece < info.pieceCount); + + if (checked_pieces_.test(piece)) + { + return true; + } + + bool const checked = checkPiece(piece); + this->anyDate = tr_time(); + this->setDirty(); + + checked_pieces_.set(piece, checked); + return checked; + } + + void initCheckedPieces(tr_bitfield const& checked, time_t const* mtimes /*fileCount*/) + { + TR_ASSERT(std::size(checked) == info.pieceCount); + checked_pieces_ = checked; + + for (size_t i = 0; i < info.fileCount; ++i) + { + auto const mtime = tr_torrentGetFileMTime(this, i); + + info.files[i].mtime = mtime; + + // if a file has changed, mark its pieces as unchecked + if (mtime != mtimes[i]) + { + checked_pieces_.unsetRange(info.files[i].firstPiece, info.files[i].lastPiece); + } + } + } + + /// + uint8_t obfuscatedHash[SHA_DIGEST_LENGTH]; /* Used when the torrent has been created with a magnet link @@ -243,6 +334,12 @@ struct tr_torrent bool isDeleting; bool startAfterVerify; bool isDirty; + + void setDirty() + { + this->isDirty = true; + } + bool isQueued; bool prefetchMagnetMetadata; @@ -410,19 +507,12 @@ void tr_torrentGotNewInfoDict(tr_torrent* tor); void tr_torrentSetSpeedLimit_Bps(tr_torrent*, tr_direction, unsigned int Bps); unsigned int tr_torrentGetSpeedLimit_Bps(tr_torrent const*, tr_direction); -/** - * @return true if this piece needs to be tested - */ -bool tr_torrentPieceNeedsCheck(tr_torrent const* tor, tr_piece_index_t pieceIndex); - /** * @brief Test a piece against its info dict checksum * @return true if the piece's passes the checksum test */ bool tr_torrentCheckPiece(tr_torrent* tor, tr_piece_index_t pieceIndex); -time_t tr_torrentGetFileMTime(tr_torrent const* tor, tr_file_index_t i); - uint64_t tr_torrentGetCurrentSizeOnDisk(tr_torrent const* tor); tr_peer_id_t const& tr_torrentGetPeerId(tr_torrent* tor); diff --git a/libtransmission/tr-macros.h b/libtransmission/tr-macros.h index e85eaa0f3..317035ecf 100644 --- a/libtransmission/tr-macros.h +++ b/libtransmission/tr-macros.h @@ -10,6 +10,7 @@ #include #include +#include /*** **** @@ -112,8 +113,6 @@ /* Only use this macro to suppress false-positive alignment warnings */ #define TR_DISCARD_ALIGN(ptr, type) ((type)(void*)(ptr)) -#define SHA_DIGEST_LENGTH 20 - #define TR_INET6_ADDRSTRLEN 46 #define TR_ADDRSTRLEN 64 @@ -123,10 +122,16 @@ // Mostly to enforce better formatting #define TR_ARG_TUPLE(...) __VA_ARGS__ -auto inline constexpr PEER_ID_LEN = size_t{ 20 }; - // https://www.bittorrent.org/beps/bep_0003.html // A string of length 20 which this downloader uses as its id. Each // downloader generates its own id at random at the start of a new // download. This value will also almost certainly have to be escaped. -using tr_peer_id_t = std::array; +auto inline constexpr PEER_ID_LEN = size_t{ 20 }; +using tr_peer_id_t = std::array; + +#define SHA_DIGEST_LENGTH 20 + +// TODO #1: all arrays of SHA_DIGEST_LENGTH should be replaced with tr_sha1_digest_t +// TODO #2: tr_peer_id_t, tr_sha1_digest_t should be moved into a new 'types.h' header +// TODO #3: this should be an array of std::byte +using tr_sha1_digest_t = std::array; diff --git a/libtransmission/transmission.h b/libtransmission/transmission.h index 387bf4e03..584c67d4e 100644 --- a/libtransmission/transmission.h +++ b/libtransmission/transmission.h @@ -1584,23 +1584,15 @@ void tr_torrentVerify(tr_torrent* torrent, tr_verify_done_func callback_func_or_ /** @brief a part of tr_info that represents a single file of the torrent's content */ struct tr_file { + time_t mtime; uint64_t length; /* Length of the file, in bytes */ + uint64_t offset; /* file begins at the torrent's nth byte */ char* name; /* Path to the file */ + tr_piece_index_t firstPiece; /* We need pieces [firstPiece... */ + tr_piece_index_t lastPiece; /* ...lastPiece] to dl this file */ int8_t priority; /* TR_PRI_HIGH, _NORMAL, or _LOW */ bool dnd; /* "do not download" flag */ bool is_renamed; /* true if we're using a different path from the one in the metainfo; ie, if the user has renamed it */ - tr_piece_index_t firstPiece; /* We need pieces [firstPiece... */ - tr_piece_index_t lastPiece; /* ...lastPiece] to dl this file */ - uint64_t offset; /* file begins at the torrent's nth byte */ -}; - -/** @brief a part of tr_info that represents a single piece of the torrent's content */ -struct tr_piece -{ - time_t timeChecked; /* the last time we tested this piece */ - uint8_t hash[SHA_DIGEST_LENGTH]; /* pieces hash */ - int8_t priority; /* TR_PRI_HIGH, _NORMAL, or _LOW */ - bool dnd; /* "do not download" flag */ }; /** @brief information about a torrent that comes from its metainfo file */ @@ -1628,7 +1620,7 @@ struct tr_info char* source; tr_file* files; - tr_piece* pieces; + tr_sha1_digest_t* pieces; /* these trackers are sorted by tier */ tr_tracker_info* trackers; diff --git a/libtransmission/verify.cc b/libtransmission/verify.cc index 49c741170..40fec1ba8 100644 --- a/libtransmission/verify.cc +++ b/libtransmission/verify.cc @@ -46,7 +46,7 @@ static bool verifyTorrent(tr_torrent* tor, bool* stopFlag) tr_sha1_ctx_t sha = tr_sha1_init(); tr_logAddTorDbg(tor, "%s", "verifying torrent..."); - tr_torrentSetChecked(tor, 0); + tor->verify_progress = 0; while (!*stopFlag && pieceIndex < tor->info.pieceCount) { @@ -95,10 +95,9 @@ static bool verifyTorrent(tr_torrent* tor, bool* stopFlag) /* if we're finishing a piece... */ if (leftInPiece == 0) { - uint8_t hash[SHA_DIGEST_LENGTH]; - - tr_sha1_final(sha, hash); - bool const hasPiece = memcmp(hash, tor->info.pieces[pieceIndex].hash, SHA_DIGEST_LENGTH) == 0; + auto hash = tr_sha1_digest_t{}; + tr_sha1_final(sha, std::data(hash)); + bool const hasPiece = hash == tor->info.pieces[pieceIndex]; if (hasPiece || hadPiece) { @@ -106,7 +105,6 @@ static bool verifyTorrent(tr_torrent* tor, bool* stopFlag) changed |= hasPiece != hadPiece; } - tr_torrentSetPieceChecked(tor, pieceIndex); time_t const now = tr_time(); tor->anyDate = now; @@ -119,7 +117,8 @@ static bool verifyTorrent(tr_torrent* tor, bool* stopFlag) } sha = tr_sha1_init(); - pieceIndex++; + ++pieceIndex; + tor->verify_progress = pieceIndex / double(tor->info.pieceCount); piecePos = 0; } @@ -143,6 +142,7 @@ static bool verifyTorrent(tr_torrent* tor, bool* stopFlag) tr_sys_file_close(fd, nullptr); } + tor->verify_progress.reset(); tr_sha1_final(sha, nullptr); free(buffer); diff --git a/qt/OptionsDialog.cc b/qt/OptionsDialog.cc index c6698b299..d629b1add 100644 --- a/qt/OptionsDialog.cc +++ b/qt/OptionsDialog.cc @@ -442,7 +442,7 @@ void OptionsDialog::onTimeout() if (left_in_piece == 0) { QByteArray const result(verify_hash_.result()); - bool const matches = memcmp(result.constData(), info_.pieces[verify_piece_index_].hash, SHA_DIGEST_LENGTH) == 0; + bool const matches = memcmp(result.constData(), std::data(info_.pieces[verify_piece_index_]), SHA_DIGEST_LENGTH) == 0; verify_flags_[verify_piece_index_] = matches; verify_piece_pos_ = 0; ++verify_piece_index_;