mirror
/
borg
mirror of https://github.com/borgbackup/borg.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'master' of github.com:jborg/attic

This commit is contained in:
Thomas Waldmann 2015-05-31 17:37:02 +02:00
parent 374fb94aa2 2b348104f6
commit 91e10fec5f
2 changed files with 52 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
CHANGES
View File

@ -58,6 +58,13 @@ Attic Changelog
Here you can see the full list of changes between each Attic release until Borg Here you can see the full list of changes between each Attic release until Borg
forked from Attic: forked from Attic:
Version 0.17
------------
(bugfix release, released on X)
- Fix hashindex ARM memory alignment issue (#309)
- Improve hashindex error messages (#298)
Version 0.16 Version 0.16
------------ ------------

80
borg/_hashindex.c
View File

@ -27,7 +27,6 @@ typedef struct {
} __attribute__((__packed__)) HashHeader; } __attribute__((__packed__)) HashHeader;
typedef struct { typedef struct {
void *data;
void *buckets; void *buckets;
int num_entries; int num_entries;
int num_buckets; int num_buckets;
@ -36,7 +35,6 @@ typedef struct {
off_t bucket_size; off_t bucket_size;
int lower_limit; int lower_limit;
int upper_limit; int upper_limit;
off_t data_len;
} HashIndex; } HashIndex;
#define MAGIC "BORG_IDX" #define MAGIC "BORG_IDX"
@ -57,8 +55,10 @@ typedef struct {
#define BUCKET_MARK_DELETED(index, idx) (*((uint32_t *)(BUCKET_ADDR(index, idx) + index->key_size)) = DELETED) #define BUCKET_MARK_DELETED(index, idx) (*((uint32_t *)(BUCKET_ADDR(index, idx) + index->key_size)) = DELETED)
#define BUCKET_MARK_EMPTY(index, idx) (*((uint32_t *)(BUCKET_ADDR(index, idx) + index->key_size)) = EMPTY) #define BUCKET_MARK_EMPTY(index, idx) (*((uint32_t *)(BUCKET_ADDR(index, idx) + index->key_size)) = EMPTY)
#define EPRINTF(msg, ...) fprintf(stderr, "hashindex: " msg "\n", ##__VA_ARGS__) #define EPRINTF_MSG(msg, ...) fprintf(stderr, "hashindex: " msg "\n", ##__VA_ARGS__)
#define EPRINTF_PATH(path, msg, ...) fprintf(stderr, "hashindex: %s: " msg "\n", path, ##__VA_ARGS__) #define EPRINTF_MSG_PATH(path, msg, ...) fprintf(stderr, "hashindex: %s: " msg "\n", path, ##__VA_ARGS__)
#define EPRINTF(msg, ...) fprintf(stderr, "hashindex: " msg "(%s)\n", ##__VA_ARGS__, strerror(errno))
#define EPRINTF_PATH(path, msg, ...) fprintf(stderr, "hashindex: %s: " msg " (%s)\n", path, ##__VA_ARGS__, strerror(errno))
static HashIndex *hashindex_read(const char *path); static HashIndex *hashindex_read(const char *path);
static int hashindex_write(HashIndex *index, const char *path); static int hashindex_write(HashIndex *index, const char *path);
@ -118,13 +118,11 @@ hashindex_resize(HashIndex *index, int capacity)
while((key = hashindex_next_key(index, key))) { while((key = hashindex_next_key(index, key))) {
hashindex_set(new, key, hashindex_get(index, key)); hashindex_set(new, key, hashindex_get(index, key));
} }
free(index->data); free(index->buckets);
index->data = new->data; index->buckets = new->buckets;
index->data_len = new->data_len;
index->num_buckets = new->num_buckets; index->num_buckets = new->num_buckets;
index->lower_limit = new->lower_limit; index->lower_limit = new->lower_limit;
index->upper_limit = new->upper_limit; index->upper_limit = new->upper_limit;
index->buckets = new->buckets;
free(new); free(new);
return 1; return 1;
} }
@ -134,8 +132,7 @@ static HashIndex *
hashindex_read(const char *path) hashindex_read(const char *path)
{ {
FILE *fd; FILE *fd;
off_t length; off_t length, buckets_length, bytes_read;
off_t bytes_read;
HashHeader header; HashHeader header;
HashIndex *index = NULL; HashIndex *index = NULL;
@ -145,7 +142,12 @@ hashindex_read(const char *path)
} }
bytes_read = fread(&header, 1, sizeof(HashHeader), fd); bytes_read = fread(&header, 1, sizeof(HashHeader), fd);
if(bytes_read != sizeof(HashHeader)) { if(bytes_read != sizeof(HashHeader)) {
EPRINTF_PATH(path, "fread header failed (expected %ld, got %ld)", sizeof(HashHeader), bytes_read); if(ferror(fd)) {
EPRINTF_PATH(path, "fread header failed (expected %ld, got %ld)", sizeof(HashHeader), bytes_read);
}
else {
EPRINTF_MSG_PATH(path, "fread header failed (expected %ld, got %ld)", sizeof(HashHeader), bytes_read);
}
goto fail; goto fail;
} }
if(fseek(fd, 0, SEEK_END) < 0) { if(fseek(fd, 0, SEEK_END) < 0) {
@ -156,43 +158,47 @@ hashindex_read(const char *path)
EPRINTF_PATH(path, "ftell failed"); EPRINTF_PATH(path, "ftell failed");
goto fail; goto fail;
} }
if(fseek(fd, 0, SEEK_SET) < 0) { if(fseek(fd, sizeof(HashHeader), SEEK_SET) < 0) {
EPRINTF_PATH(path, "fseek failed"); EPRINTF_PATH(path, "fseek failed");
goto fail; goto fail;
} }
if(memcmp(header.magic, MAGIC, 8)) { if(memcmp(header.magic, MAGIC, 8)) {
EPRINTF_PATH(path, "Unknown file header"); EPRINTF_MSG_PATH(path, "Unknown file header");
goto fail; goto fail;
} }
if(length != sizeof(HashHeader) + (off_t)_le32toh(header.num_buckets) * (header.key_size + header.value_size)) { buckets_length = (off_t)_le32toh(header.num_buckets) * (header.key_size + header.value_size);
EPRINTF_PATH(path, "Incorrect file length"); if(length != sizeof(HashHeader) + buckets_length) {
EPRINTF_MSG_PATH(path, "Incorrect file length");
goto fail; goto fail;
} }
if(!(index = malloc(sizeof(HashIndex)))) { if(!(index = malloc(sizeof(HashIndex)))) {
EPRINTF_PATH(path, "malloc failed"); EPRINTF_PATH(path, "malloc failed");
goto fail; goto fail;
} }
if(!(index->data = malloc(length))) { if(!(index->buckets = malloc(buckets_length))) {
EPRINTF_PATH(path, "malloc failed"); EPRINTF_PATH(path, "malloc failed");
free(index); free(index);
index = NULL; index = NULL;
goto fail; goto fail;
} }
bytes_read = fread(index->data, 1, length, fd); bytes_read = fread(index->buckets, 1, buckets_length, fd);
if(bytes_read != length) { if(bytes_read != buckets_length) {
EPRINTF_PATH(path, "fread hashindex failed (expected %ld, got %ld)", length, bytes_read); if(ferror(fd)) {
free(index->data); EPRINTF_PATH(path, "fread buckets failed (expected %ld, got %ld)", buckets_length, bytes_read);
}
else {
EPRINTF_MSG_PATH(path, "fread buckets failed (expected %ld, got %ld)", buckets_length, bytes_read);
}
free(index->buckets);
free(index); free(index);
index = NULL; index = NULL;
goto fail; goto fail;
} }
index->data_len = length;
index->num_entries = _le32toh(header.num_entries); index->num_entries = _le32toh(header.num_entries);
index->num_buckets = _le32toh(header.num_buckets); index->num_buckets = _le32toh(header.num_buckets);
index->key_size = header.key_size; index->key_size = header.key_size;
index->value_size = header.value_size; index->value_size = header.value_size;
index->bucket_size = index->key_size + index->value_size; index->bucket_size = index->key_size + index->value_size;
index->buckets = index->data + sizeof(HashHeader);
index->lower_limit = index->num_buckets > MIN_BUCKETS ? ((int)(index->num_buckets * BUCKET_LOWER_LIMIT)) : 0; index->lower_limit = index->num_buckets > MIN_BUCKETS ? ((int)(index->num_buckets * BUCKET_LOWER_LIMIT)) : 0;
index->upper_limit = (int)(index->num_buckets * BUCKET_UPPER_LIMIT); index->upper_limit = (int)(index->num_buckets * BUCKET_UPPER_LIMIT);
fail: fail:
@ -205,10 +211,8 @@ fail:
static HashIndex * static HashIndex *
hashindex_init(int capacity, int key_size, int value_size) hashindex_init(int capacity, int key_size, int value_size)
{ {
off_t buckets_length;
HashIndex *index; HashIndex *index;
HashHeader header = {
.magic = MAGIC, .num_entries = 0, .key_size = key_size, .value_size = value_size
};
int i; int i;
capacity = MAX(MIN_BUCKETS, capacity); capacity = MAX(MIN_BUCKETS, capacity);
@ -216,8 +220,8 @@ hashindex_init(int capacity, int key_size, int value_size)
EPRINTF("malloc failed"); EPRINTF("malloc failed");
return NULL; return NULL;
} }
index->data_len = sizeof(HashHeader) + (off_t)capacity * (key_size + value_size); buckets_length = (off_t)capacity * (key_size + value_size);
if(!(index->data = calloc(index->data_len, 1))) { if(!(index->buckets = calloc(buckets_length, 1))) {
EPRINTF("malloc failed"); EPRINTF("malloc failed");
free(index); free(index);
return NULL; return NULL;
@ -229,8 +233,6 @@ hashindex_init(int capacity, int key_size, int value_size)
index->bucket_size = index->key_size + index->value_size; index->bucket_size = index->key_size + index->value_size;
index->lower_limit = index->num_buckets > MIN_BUCKETS ? ((int)(index->num_buckets * BUCKET_LOWER_LIMIT)) : 0; index->lower_limit = index->num_buckets > MIN_BUCKETS ? ((int)(index->num_buckets * BUCKET_LOWER_LIMIT)) : 0;
index->upper_limit = (int)(index->num_buckets * BUCKET_UPPER_LIMIT); index->upper_limit = (int)(index->num_buckets * BUCKET_UPPER_LIMIT);
index->buckets = index->data + sizeof(HashHeader);
memcpy(index->data, &header, sizeof(HashHeader));
for(i = 0; i < capacity; i++) { for(i = 0; i < capacity; i++) {
BUCKET_MARK_EMPTY(index, i); BUCKET_MARK_EMPTY(index, i);
} }
@ -240,24 +242,33 @@ hashindex_init(int capacity, int key_size, int value_size)
static void static void
hashindex_free(HashIndex *index) hashindex_free(HashIndex *index)
{ {
free(index->data); free(index->buckets);
free(index); free(index);
} }
static int static int
hashindex_write(HashIndex *index, const char *path) hashindex_write(HashIndex *index, const char *path)
{ {
off_t buckets_length = (off_t)index->num_buckets * index->bucket_size;
FILE *fd; FILE *fd;
HashHeader header = {
.magic = MAGIC,
.num_entries = _htole32(index->num_entries),
.num_buckets = _htole32(index->num_buckets),
.key_size = index->key_size,
.value_size = index->value_size
};
int ret = 1; int ret = 1;
if((fd = fopen(path, "w")) == NULL) { if((fd = fopen(path, "w")) == NULL) {
EPRINTF_PATH(path, "open failed"); EPRINTF_PATH(path, "open failed");
fprintf(stderr, "Failed to open %s for writing\n", path);
return 0; return 0;
} }
*((uint32_t *)(index->data + 8)) = _htole32(index->num_entries); if(fwrite(&header, 1, sizeof(header), fd) != sizeof(header)) {
*((uint32_t *)(index->data + 12)) = _htole32(index->num_buckets); EPRINTF_PATH(path, "fwrite failed");
if(fwrite(index->data, 1, index->data_len, fd) != index->data_len) { ret = 0;
}
if(fwrite(index->buckets, 1, buckets_length, fd) != buckets_length) {
EPRINTF_PATH(path, "fwrite failed"); EPRINTF_PATH(path, "fwrite failed");
ret = 0; ret = 0;
} }
@ -366,4 +377,3 @@ hashindex_summarize(HashIndex *index, long long *total_size, long long *total_cs
*total_unique_size = unique_size; *total_unique_size = unique_size;
*total_unique_csize = unique_csize; *total_unique_csize = unique_csize;
} }