(libT) add package-visible API hook for when a block is downloaded. Add unit test to confirm that when the last file finishes downloading, its .part suffix is removed and it's moved from the incomplete to complete dir

This commit is contained in:
Jordan Lee 2013-01-26 23:08:51 +00:00
parent a3eafbb742
commit 388da24dd0
9 changed files with 354 additions and 207 deletions

View File

@ -134,6 +134,7 @@ TESTS = \
json-test \
magnet-test \
metainfo-test \
move-test \
peer-msgs-test \
quark-test \
rename-test \
@ -194,6 +195,10 @@ metainfo_test_SOURCES = metainfo-test.c $(TEST_SOURCES)
metainfo_test_LDADD = ${apps_ldadd}
metainfo_test_LDFLAGS = ${apps_ldflags}
move_test_SOURCES = move-test.c $(TEST_SOURCES)
move_test_LDADD = ${apps_ldadd}
move_test_LDFLAGS = ${apps_ldflags}
peer_msgs_test_SOURCES = peer-msgs-test.c $(TEST_SOURCES)
peer_msgs_test_LDADD = ${apps_ldadd}
peer_msgs_test_LDFLAGS = ${apps_ldflags}

View File

@ -2,6 +2,7 @@
#include <stdio.h>
#include "transmission.h"
#include "torrent.h"
#include "libtransmission-test.h"
bool verbose = false;
@ -320,3 +321,69 @@ libtransmission_test_zero_torrent_init (void)
tr_ctorFree (ctor);
return tor;
}
#define verify_and_block_until_done(tor) \
do { \
tr_torrentVerify (tor); \
do { \
tr_wait_msec (10); \
} while (tor->verifyState != TR_VERIFY_NONE); \
} while (0)
void
libtransmission_test_zero_torrent_populate (tr_torrent * tor, bool complete)
{
tr_file_index_t i;
for (i=0; i<tor->info.fileCount; ++i)
{
int rv;
uint64_t j;
FILE * fp;
char * path;
char * dirname;
const tr_file * file = &tor->info.files[i];
struct stat sb;
path = tr_buildPath (tor->currentDir, file->name, NULL);
dirname = tr_dirname (path);
tr_mkdirp (dirname, 0700);
fp = fopen (path, "wb+");
for (j=0; j<file->length; ++j)
fputc ('\0', fp);
fclose (fp);
tr_free (dirname);
tr_free (path);
path = tr_torrentFindFile (tor, i);
assert (path != NULL);
rv = stat (path, &sb);
assert (rv == 0);
tr_free (path);
}
sync ();
if (!complete)
{
FILE * fp;
char * oldpath = tr_torrentFindFile (tor, 0);
char * newpath = tr_strdup_printf ("%s.part", oldpath);
rename (oldpath, newpath);
/* invalidate one piece */
fp = fopen (newpath, "rb+");
fputc ('\1', fp);
fclose (fp);
tr_free (newpath);
tr_free (oldpath);
sync ();
verify_and_block_until_done (tor);
assert (tr_torrentStat(tor)->leftUntilDone == tor->info.pieceSize);
}
}

View File

@ -77,6 +77,8 @@ void libtransmission_test_session_init_session (void);
void libtransmission_test_session_init (void); /* utility; calls the other 3 */
void libtransmission_test_session_close (void);
void libtransmission_test_zero_torrent_populate (tr_torrent * tor, bool complete);
tr_torrent * libtransmission_test_zero_torrent_init (void);

128
libtransmission/move-test.c Normal file
View File

@ -0,0 +1,128 @@
#include <assert.h>
#include <errno.h>
#include <stdio.h> /* remove() */
#include <string.h> /* strcmp() */
#include <stdio.h>
#include <sys/types.h> /* stat() */
#include <sys/stat.h> /* stat() */
#include <unistd.h> /* stat(), sync() */
#include <event2/buffer.h>
#include "transmission.h"
#include "cache.h"
#include "resume.h"
#include "torrent.h" /* tr_isTorrent() */
#include "utils.h" /* tr_mkdirp() */
#include "variant.h"
#include "libtransmission-test.h"
/***
****
***/
static void
zeroes_completeness_func (tr_torrent * torrent UNUSED,
tr_completeness completeness,
bool wasRunning UNUSED,
void * user_data)
{
*(tr_completeness*)user_data = completeness;
}
static int
test_incomplete_dir_is_subdir_of_download_dir (void)
{
tr_file_index_t i;
char * path;
char * incomplete_dir;
char * expected_path;
tr_torrent * tor;
tr_completeness completeness;
/* init the session */
libtransmission_test_session_init ();
incomplete_dir = tr_buildPath (downloadDir, "incomplete", NULL);
tr_sessionSetIncompleteDir (session, incomplete_dir);
tr_sessionSetIncompleteDirEnabled (session, true);
/* init an incomplete torrent */
tor = libtransmission_test_zero_torrent_init ();
libtransmission_test_zero_torrent_populate (tor, false);
check (tr_torrentStat(tor)->leftUntilDone == tor->info.pieceSize);
path = tr_torrentFindFile (tor, 0);
expected_path = tr_strdup_printf ("%s/%s.part", incomplete_dir, tor->info.files[0].name);
check_streq (expected_path, path);
tr_free (expected_path);
tr_free (path);
path = tr_torrentFindFile (tor, 1);
expected_path = tr_buildPath (incomplete_dir, tor->info.files[1].name, NULL);
check_streq (expected_path, path);
tr_free (expected_path);
tr_free (path);
check_int_eq (tor->info.pieceSize, tr_torrentStat(tor)->leftUntilDone);
/* now finish writing it */
{
//char * block;
uint32_t offset;
tr_block_index_t i;
tr_block_index_t first;
tr_block_index_t last;
char * tobuf;
struct evbuffer * buf;
tobuf = tr_new0 (char, tor->blockSize);
buf = evbuffer_new ();
tr_torGetPieceBlockRange (tor, 0, &first, &last);
for (offset=0, i=first; i<=last; ++i, offset+=tor->blockSize)
{
evbuffer_add (buf, tobuf, tor->blockSize);
tr_cacheWriteBlock (session->cache, tor, 0, offset, tor->blockSize, buf);
tr_torrentGotBlock (tor, i);
}
evbuffer_free (buf);
tr_free (tobuf);
}
completeness = -1;
tr_torrentSetCompletenessCallback (tor, zeroes_completeness_func, &completeness);
tr_torrentRecheckCompleteness (tor);
check_int_eq (TR_SEED, completeness);
sync ();
for (i=0; i<tor->info.fileCount; ++i)
{
path = tr_torrentFindFile (tor, i);
expected_path = tr_buildPath (downloadDir, tor->info.files[i].name, NULL);
check_streq (expected_path, path);
tr_free (expected_path);
tr_free (path);
}
/* cleanup */
tr_torrentRemove (tor, true, remove);
libtransmission_test_session_close ();
tr_free (incomplete_dir);
return 0;
}
/***
****
***/
int
main (void)
{
const testFunc tests[] = { test_incomplete_dir_is_subdir_of_download_dir };
return runTests (tests, NUM_TESTS (tests));
}

View File

@ -1557,18 +1557,6 @@ addStrike (Torrent * t, tr_peer * peer)
}
}
static void
gotBadPiece (Torrent * t, tr_piece_index_t pieceIndex)
{
tr_torrent * tor = t->tor;
const uint32_t byteCount = tr_torPieceCountBytes (tor, pieceIndex);
tor->corruptCur += byteCount;
tor->downloadedCur -= MIN (tor->downloadedCur, byteCount);
tr_announcerAddBytes (tor, TR_ANN_CORRUPT, byteCount);
}
static void
peerSuggestedPiece (Torrent * t UNUSED,
tr_peer * peer UNUSED,
@ -1644,7 +1632,53 @@ peerDeclinedAllRequests (Torrent * t, const tr_peer * peer)
tr_free (blocks);
}
static void tr_peerMgrSetBlame (tr_torrent *, tr_piece_index_t, int);
static void
cancelAllRequestsForBlock (struct tr_torrent_peers * t,
tr_block_index_t block,
tr_peer * no_notify)
{
int i;
int peerCount;
tr_peer ** peers;
tr_ptrArray peerArr;
peerArr = TR_PTR_ARRAY_INIT;
getBlockRequestPeers (t, block, &peerArr);
peers = (tr_peer **) tr_ptrArrayPeek (&peerArr, &peerCount);
for (i=0; i<peerCount; ++i)
{
tr_peer * p = peers[i];
if ((p != no_notify) && (p->msgs != NULL))
{
tr_historyAdd (&p->cancelsSentToPeer, tr_time (), 1);
tr_peerMsgsCancel (p->msgs, block);
}
removeRequestFromTables (t, block, p);
}
tr_ptrArrayDestruct (&peerArr, NULL);
}
void
tr_peerMgrPieceCompleted (tr_torrent * tor, tr_piece_index_t p)
{
int i;
int peerCount;
tr_peer ** peers;
struct tr_torrent_peers * t = tor->torrentPeers;
/* notify the peers that we now have this piece */
peerCount = tr_ptrArraySize (&t->peers);
peers = (tr_peer**) tr_ptrArrayBase (&t->peers);
for (i=0; i<peerCount; ++i)
tr_peerMsgsHave (peers[i]->msgs, p);
/* bookkeeping */
pieceListRemovePiece (t, p);
t->needsCompletenessCheck = true;
}
static void
peerCallbackFunc (tr_peer * peer, const tr_peer_event * e, void * vt)
@ -1758,105 +1792,14 @@ peerCallbackFunc (tr_peer * peer, const tr_peer_event * e, void * vt)
}
case TR_PEER_CLIENT_GOT_BLOCK:
{
tr_torrent * tor = t->tor;
tr_block_index_t block = _tr_block (tor, e->pieceIndex, e->offset);
int i, peerCount;
tr_peer ** peers;
tr_ptrArray peerArr = TR_PTR_ARRAY_INIT;
removeRequestFromTables (t, block, peer);
getBlockRequestPeers (t, block, &peerArr);
peers = (tr_peer **) tr_ptrArrayPeek (&peerArr, &peerCount);
/* remove additional block requests and send cancel to peers */
for (i=0; i<peerCount; i++) {
tr_peer * p = peers[i];
assert (p != peer);
if (p->msgs) {
tr_historyAdd (&p->cancelsSentToPeer, tr_time (), 1);
tr_peerMsgsCancel (p->msgs, block);
}
removeRequestFromTables (t, block, p);
}
tr_ptrArrayDestruct (&peerArr, false);
tr_historyAdd (&peer->blocksSentToClient, tr_time (), 1);
if (tr_cpBlockIsComplete (&tor->completion, block))
{
/* we already have this block... */
const uint32_t n = tr_torBlockCountBytes (tor, block);
tor->downloadedCur -= MIN (tor->downloadedCur, n);
tordbg (t, "we have this block already...");
}
else
{
tr_cpBlockAdd (&tor->completion, block);
pieceListResortPiece (t, pieceListLookup (t, e->pieceIndex));
tr_torrentSetDirty (tor);
if (tr_cpPieceIsComplete (&tor->completion, e->pieceIndex))
{
const tr_piece_index_t p = e->pieceIndex;
const bool ok = tr_torrentCheckPiece (tor, p);
tordbg (t, "[LAZY] checked just-completed piece %zu", (size_t)p);
if (!ok)
{
tr_logAddTorErr (tor, _("Piece %lu, which was just downloaded, failed its checksum test"),
(unsigned long)p);
}
tr_peerMgrSetBlame (tor, p, ok);
if (!ok)
{
gotBadPiece (t, p);
}
else
{
int i;
int peerCount;
tr_peer ** peers;
tr_file_index_t fileIndex;
/* only add this to downloadedCur if we got it from a peer --
* webseeds shouldn't count against our ratio. As one tracker
* admin put it, "Those pieces are downloaded directly from the
* content distributor, not the peers, it is the tracker's job
* to manage the swarms, not the web server and does not fit
* into the jurisdiction of the tracker." */
if (peer->msgs != NULL) {
const uint32_t n = tr_torPieceCountBytes (tor, p);
tr_announcerAddBytes (tor, TR_ANN_DOWN, n);
}
peerCount = tr_ptrArraySize (&t->peers);
peers = (tr_peer**) tr_ptrArrayBase (&t->peers);
for (i=0; i<peerCount; ++i)
tr_peerMsgsHave (peers[i]->msgs, p);
for (fileIndex=0; fileIndex<tor->info.fileCount; ++fileIndex) {
const tr_file * file = &tor->info.files[fileIndex];
if ((file->firstPiece <= p) && (p <= file->lastPiece)) {
if (tr_cpFileIsComplete (&tor->completion, fileIndex)) {
tr_cacheFlushFile (tor->session->cache, tor, fileIndex);
tr_torrentFileCompleted (tor, fileIndex);
}
}
}
pieceListRemovePiece (t, p);
}
}
t->needsCompletenessCheck = true;
}
{
const tr_block_index_t block = _tr_block (t->tor, e->pieceIndex, e->offset);
cancelAllRequestsForBlock (t, block, peer);
tr_historyAdd (&peer->blocksSentToClient, tr_time(), 1);
pieceListResortPiece (t, pieceListLookup (t, e->pieceIndex));
tr_torrentGotBlock (t->tor, block);
break;
}
}
case TR_PEER_ERROR:
if ((e->err == ERANGE) || (e->err == EMSGSIZE) || (e->err == ENOTCONN))
@ -2231,32 +2174,28 @@ tr_peerMgrArrayToPex (const void * array,
***
**/
static void
tr_peerMgrSetBlame (tr_torrent * tor,
tr_piece_index_t pieceIndex,
int success)
void
tr_peerMgrGotBadPiece (tr_torrent * tor, tr_piece_index_t pieceIndex)
{
if (!success)
int i;
int n;
Torrent * t = tor->torrentPeers;
const uint32_t byteCount = tr_torPieceCountBytes (tor, pieceIndex);
for (i=0, n=tr_ptrArraySize(&t->peers); i!=n; ++i)
{
int peerCount, i;
Torrent * t = tor->torrentPeers;
tr_peer ** peers;
tr_peer * peer = tr_ptrArrayNth (&t->peers, i);
assert (torrentIsLocked (t));
peers = (tr_peer **) tr_ptrArrayPeek (&t->peers, &peerCount);
for (i = 0; i < peerCount; ++i)
if (tr_bitfieldHas (&peer->blame, pieceIndex))
{
tr_peer * peer = peers[i];
if (tr_bitfieldHas (&peer->blame, pieceIndex))
{
tordbg (t, "peer %s contributed to corrupt piece (%d); now has %d strikes",
tr_atomAddrStr (peer->atom),
pieceIndex, (int)peer->strikes + 1);
addStrike (t, peer);
}
tordbg (t, "peer %s contributed to corrupt piece (%d); now has %d strikes",
tr_atomAddrStr(peer->atom), pieceIndex, (int)peer->strikes + 1);
addStrike (t, peer);
}
}
tr_announcerAddBytes (tor, TR_ANN_CORRUPT, byteCount);
}
int

View File

@ -255,6 +255,12 @@ unsigned int tr_peerGetPieceSpeed_Bps (const tr_peer * peer,
void tr_peerMgrClearInterest (tr_torrent * tor);
void tr_peerMgrGotBadPiece (tr_torrent * tor, tr_piece_index_t pieceIndex);
void tr_peerMgrPieceCompleted (tr_torrent * tor, tr_piece_index_t pieceIndex);
/* @} */
#endif

View File

@ -187,7 +187,7 @@ test_single_filename_torrent (void)
verify_and_block_until_done (tor);
check_have_none (tor, totalSize);
create_single_file_torrent_contents (tor->downloadDir);
create_single_file_torrent_contents (tor->currentDir);
/* sanity check the stats again, now that we've added the file */
verify_and_block_until_done (tor);
@ -214,7 +214,7 @@ test_single_filename_torrent (void)
**** Now try a rename that should succeed
***/
tmpstr = tr_buildPath (tor->downloadDir, "hello-world.txt", NULL);
tmpstr = tr_buildPath (tor->currentDir, "hello-world.txt", NULL);
check (tr_fileExists (tmpstr, NULL));
check_streq ("hello-world.txt", tr_torrentName(tor));
check_int_eq (0, torrentRenameAndWait (tor, tor->info.name, "foobar"));
@ -237,7 +237,7 @@ test_single_filename_torrent (void)
**** ...and rename it back again
***/
tmpstr = tr_buildPath (tor->downloadDir, "foobar", NULL);
tmpstr = tr_buildPath (tor->currentDir, "foobar", NULL);
check (tr_fileExists (tmpstr, NULL));
check_int_eq (0, torrentRenameAndWait (tor, "foobar", "hello-world.txt"));
check (!tr_fileExists (tmpstr, NULL));
@ -333,7 +333,7 @@ test_multifile_torrent (void)
check_have_none (tor, totalSize);
/* build the local data */
create_multifile_torrent_contents (tor->downloadDir);
create_multifile_torrent_contents (tor->currentDir);
/* sanity check the (full) stats */
verify_and_block_until_done (tor);
@ -395,10 +395,10 @@ test_multifile_torrent (void)
***/
/* remove the directory Felidae/Felinae/Felis/catus */
str = tr_buildPath (tor->downloadDir, files[1].name, NULL);
str = tr_buildPath (tor->currentDir, files[1].name, NULL);
remove (str);
tr_free (str);
str = tr_buildPath (tor->downloadDir, files[2].name, NULL);
str = tr_buildPath (tor->currentDir, files[2].name, NULL);
remove (str);
tmp = tr_dirname (str);
remove (tmp);
@ -461,65 +461,6 @@ test_multifile_torrent (void)
****
***/
static void
create_zero_torrent_partial_contents (tr_torrent * tor, bool incomplete)
{
tr_file_index_t i;
for (i=0; i<tor->info.fileCount; ++i)
{
int rv;
uint64_t j;
FILE * fp;
char * path;
char * dirname;
const tr_file * file = &tor->info.files[i];
struct stat sb;
path = tr_buildPath (tor->downloadDir, file->name, NULL);
dirname = tr_dirname (path);
tr_mkdirp (dirname, 0700);
fp = fopen (path, "wb+");
for (j=0; j<file->length; ++j)
fputc ('\0', fp);
fclose (fp);
tr_free (dirname);
tr_free (path);
path = tr_torrentFindFile (tor, i);
assert (path != NULL);
rv = stat (path, &sb);
assert (rv == 0);
tr_free (path);
}
sync ();
verify_and_block_until_done (tor);
assert (tr_torrentStat(tor)->leftUntilDone == 0);
if (incomplete)
{
FILE * fp;
char * oldpath = tr_torrentFindFile (tor, 0);
char * newpath = tr_strdup_printf ("%s.part", oldpath);
rename (oldpath, newpath);
/* invalidate one piece */
fp = fopen (newpath, "rb+");
fputc ('\1', fp);
fclose (fp);
tr_free (newpath);
tr_free (oldpath);
sync ();
verify_and_block_until_done (tor);
assert (tr_torrentStat(tor)->leftUntilDone == tor->info.pieceSize);
}
}
static int
test_partial_file (void)
{
@ -545,7 +486,7 @@ test_partial_file (void)
check_streq ("files-filled-with-zeroes/4096", tor->info.files[1].name);
check_streq ("files-filled-with-zeroes/512", tor->info.files[2].name);
create_zero_torrent_partial_contents (tor, true);
libtransmission_test_zero_torrent_populate (tor, false);
fst = tr_torrentFiles (tor, NULL);
check_int_eq (length[0] - pieceSize, fst[0].bytesCompleted);
check_int_eq (length[1], fst[1].bytesCompleted);
@ -572,7 +513,7 @@ test_partial_file (void)
strings[0] = "foo/bar.part";
for (i=0; i<3; ++i)
{
char * expected = tr_buildPath (tor->downloadDir, strings[i], NULL);
char * expected = tr_buildPath (tor->currentDir, strings[i], NULL);
char * path = tr_torrentFindFile (tor, i);
check_streq (expected, path);
tr_free (path);

View File

@ -1328,7 +1328,7 @@ tr_torrentStat (tr_torrent * tor)
***/
static uint64_t
fileBytesCompleted (const tr_torrent * tor, tr_file_index_t index)
countFileBytesCompleted (const tr_torrent * tor, tr_file_index_t index)
{
uint64_t total = 0;
const tr_file * f = &tor->info.files[index];
@ -1379,7 +1379,7 @@ tr_torrentFiles (const tr_torrent * tor,
assert (tr_isTorrent (tor));
for (i=0; i<n; ++i, ++walk) {
const uint64_t b = isSeed ? tor->info.files[i].length : fileBytesCompleted (tor, i);
const uint64_t b = isSeed ? tor->info.files[i].length : countFileBytesCompleted (tor, i);
walk->bytesCompleted = b;
walk->progress = tor->info.files[i].length > 0 ? ((float)b / tor->info.files[i].length) : 1.0f;
}
@ -3000,19 +3000,20 @@ tr_torrentSetLocation (tr_torrent * tor,
****
***/
void
tr_torrentFileCompleted (tr_torrent * tor, tr_file_index_t fileNum)
static void
tr_torrentFileCompleted (tr_torrent * tor, tr_file_index_t fileIndex)
{
char * sub;
const char * base;
const tr_info * inf = &tor->info;
const tr_file * f = &inf->files[fileNum];
const tr_file * f = &inf->files[fileIndex];
tr_piece * p;
const tr_piece * pend;
const time_t now = tr_time ();
/* close the file so that we can reopen in read-only mode as needed */
tr_fdFileClose (tor->session, tor, fileNum);
tr_cacheFlushFile (tor->session->cache, tor, fileIndex);
tr_fdFileClose (tor->session, tor, fileIndex);
/* now that the file is complete and closed, we can start watching its
* mtime timestamp for changes to know if we need to reverify pieces */
@ -3022,7 +3023,7 @@ tr_torrentFileCompleted (tr_torrent * tor, tr_file_index_t fileNum)
/* if the torrent's current filename isn't the same as the one in the
* metadata -- for example, if it had the ".part" suffix appended to
* it until now -- then rename it to match the one in the metadata */
if (tr_torrentFindFile2 (tor, fileNum, &base, &sub, NULL))
if (tr_torrentFindFile2 (tor, fileIndex, &base, &sub, NULL))
{
if (strcmp (sub, f->name))
{
@ -3040,6 +3041,63 @@ tr_torrentFileCompleted (tr_torrent * tor, tr_file_index_t fileNum)
}
}
static void
tr_torrentPieceCompleted (tr_torrent * tor, tr_piece_index_t pieceIndex)
{
tr_file_index_t i;
tr_peerMgrPieceCompleted (tor, pieceIndex);
/* if this piece completes any file, invoke the fileCompleted func for it */
for (i=0; i<tor->info.fileCount; ++i)
{
const tr_file * file = &tor->info.files[i];
if ((file->firstPiece <= pieceIndex) && (pieceIndex <= file->lastPiece))
if (tr_cpFileIsComplete (&tor->completion, i))
tr_torrentFileCompleted (tor, i);
}
}
void
tr_torrentGotBlock (tr_torrent * tor, tr_block_index_t block)
{
const bool block_is_new = !tr_cpBlockIsComplete (&tor->completion, block);
if (block_is_new)
{
tr_piece_index_t p;
tr_cpBlockAdd (&tor->completion, block);
tr_torrentSetDirty (tor);
p = tr_torBlockPiece (tor, block);
if (tr_cpPieceIsComplete (&tor->completion, p))
{
tr_logAddTorDbg (tor, "[LAZY] checking just-completed piece %zu", (size_t)p);
if (tr_torrentCheckPiece (tor, p))
{
tr_torrentPieceCompleted (tor, p);
}
else
{
const uint32_t n = tr_torPieceCountBytes (tor, p);
tr_logAddTorErr (tor, _("Piece %"PRIu32", which was just downloaded, failed its checksum test"), p);
tor->corruptCur += n;
tor->downloadedCur -= MIN (tor->downloadedCur, n);
tr_peerMgrGotBadPiece (tor, p);
}
}
}
else
{
const uint32_t n = tr_torBlockCountBytes (tor, block);
tor->downloadedCur -= MIN (tor->downloadedCur, n);
tr_logAddTorDbg (tor, "we have this block already...");
}
}
/***
****
***/

View File

@ -382,9 +382,10 @@ void tr_torrentSetDirty (tr_torrent * tor)
uint32_t tr_getBlockSize (uint32_t pieceSize);
/**
* Tell the tr_torrent that one of its files has become complete
* Tell the tr_torrent that it's gotten a block
*/
void tr_torrentFileCompleted (tr_torrent * tor, tr_file_index_t fileNo);
void tr_torrentGotBlock (tr_torrent * tor, tr_block_index_t blockIndex);
/**