borg/attic/_hashindex.c

388 lines
10 KiB
C
Raw Normal View History

2010-12-16 19:23:22 +00:00
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/mman.h>
2013-07-03 10:19:16 +00:00
#if defined(__BYTE_ORDER)&&(__BYTE_ORDER == __BIG_ENDIAN)
#error This code is not big endian safe yet
#endif
2010-12-16 19:23:22 +00:00
typedef struct {
char magic[8];
int32_t num_entries;
int32_t num_buckets;
int8_t key_size;
int8_t value_size;
} __attribute__((__packed__)) HashHeader;
2013-05-28 12:35:55 +00:00
typedef struct {
char *path;
void *map_addr;
off_t map_length;
void *buckets;
int num_entries;
int num_buckets;
int key_size;
int value_size;
int bucket_size;
2013-07-03 10:19:16 +00:00
int lower_limit;
int upper_limit;
int readonly;
2013-05-28 12:35:55 +00:00
} HashIndex;
2010-12-16 19:23:22 +00:00
2013-07-08 21:38:27 +00:00
#define MAGIC "ATTICIDX"
2010-12-16 19:23:22 +00:00
#define EMPTY ((int32_t)-1)
#define DELETED ((int32_t)-2)
2013-07-03 10:19:16 +00:00
#define MAX_BUCKET_SIZE 512
#define BUCKET_LOWER_LIMIT .25
#define BUCKET_UPPER_LIMIT .90
#define MIN_BUCKETS 1024
#define MAX(x, y) ((x) > (y) ? (x): (y))
2011-08-14 18:06:30 +00:00
#define BUCKET_ADDR_READ(index, idx) (index->buckets + (idx * index->bucket_size))
#define BUCKET_ADDR_WRITE(index, idx) (index->buckets + (idx * index->bucket_size))
2010-12-16 19:23:22 +00:00
2011-08-14 18:06:30 +00:00
#define BUCKET_IS_DELETED(index, idx) (*((int32_t *)(BUCKET_ADDR_READ(index, idx) + index->key_size)) == DELETED)
#define BUCKET_IS_EMPTY(index, idx) (*((int32_t *)(BUCKET_ADDR_READ(index, idx) + index->key_size)) == EMPTY)
2010-12-16 19:23:22 +00:00
2011-08-14 18:06:30 +00:00
#define BUCKET_MATCHES_KEY(index, idx, key) (memcmp(key, BUCKET_ADDR_READ(index, idx), index->key_size) == 0)
2010-12-16 19:23:22 +00:00
2011-08-14 18:06:30 +00:00
#define BUCKET_MARK_DELETED(index, idx) (*((int32_t *)(BUCKET_ADDR_WRITE(index, idx) + index->key_size)) = DELETED)
2010-12-16 19:23:22 +00:00
2013-07-03 10:19:16 +00:00
#define EPRINTF(msg, ...) EPRINTF_PATH(index->path, msg, ##__VA_ARGS__)
#define EPRINTF_PATH(path, msg, ...) fprintf(stderr, "hashindex: %s: " msg "\n", path, ##__VA_ARGS__)
static HashIndex *hashindex_open(const char *path, int readonly);
2013-07-03 10:19:16 +00:00
static int hashindex_close(HashIndex *index);
static int hashindex_clear(HashIndex *index);
static int hashindex_flush(HashIndex *index);
2013-05-28 12:35:55 +00:00
static HashIndex *hashindex_create(const char *path, int capacity, int key_size, int value_size);
static const void *hashindex_get(HashIndex *index, const void *key);
2013-07-03 10:19:16 +00:00
static int hashindex_set(HashIndex *index, const void *key, const void *value);
static int hashindex_delete(HashIndex *index, const void *key);
2013-05-28 12:35:55 +00:00
static void *hashindex_next_key(HashIndex *index, const void *key);
2010-12-16 19:23:22 +00:00
/* Private API */
static int
hashindex_index(HashIndex *index, const void *key)
{
return *((uint32_t *)key) % index->num_buckets;
}
static int
hashindex_lookup(HashIndex *index, const void *key)
{
int didx = -1;
int start = hashindex_index(index, key);
int idx = start;
2010-12-16 19:23:22 +00:00
for(;;) {
if(BUCKET_IS_EMPTY(index, idx))
{
return -1;
}
if(BUCKET_IS_DELETED(index, idx)) {
if(didx == -1) {
didx = idx;
}
}
else if(BUCKET_MATCHES_KEY(index, idx, key)) {
2013-10-04 20:06:16 +00:00
if (didx != -1 && !index->readonly) {
2011-08-14 18:06:30 +00:00
memcpy(BUCKET_ADDR_WRITE(index, didx), BUCKET_ADDR_READ(index, idx), index->bucket_size);
2010-12-16 19:23:22 +00:00
BUCKET_MARK_DELETED(index, idx);
idx = didx;
}
return idx;
}
idx = (idx + 1) % index->num_buckets;
if(idx == start) {
return -1;
}
2010-12-16 19:23:22 +00:00
}
}
2013-07-03 10:19:16 +00:00
static int
2010-12-16 19:23:22 +00:00
hashindex_resize(HashIndex *index, int capacity)
{
char *new_path = malloc(strlen(index->path) + 5);
2013-10-02 18:42:26 +00:00
int ret = 0;
2010-12-16 19:23:22 +00:00
strcpy(new_path, index->path);
strcat(new_path, ".tmp");
2013-07-03 10:19:16 +00:00
HashIndex *new;
if(!(new = hashindex_create(new_path, capacity, index->key_size, index->value_size))) {
free(new_path);
return 0;
}
2010-12-16 19:23:22 +00:00
void *key = NULL;
while((key = hashindex_next_key(index, key))) {
hashindex_set(new, key, hashindex_get(index, key));
}
munmap(index->map_addr, index->map_length);
index->map_addr = new->map_addr;
index->map_length = new->map_length;
index->num_buckets = new->num_buckets;
2013-07-03 10:19:16 +00:00
index->lower_limit = new->lower_limit;
index->upper_limit = new->upper_limit;
2010-12-16 19:23:22 +00:00
index->buckets = new->buckets;
2013-10-02 18:42:26 +00:00
if(unlink(index->path) < 0) {
EPRINTF("unlink failed");
goto out;
}
if(rename(new_path, index->path) < 0) {
EPRINTF_PATH(new_path, "rename failed");
goto out;
}
ret = 1;
out:
2010-12-16 19:23:22 +00:00
free(new_path);
free(new->path);
free(new);
2013-10-02 18:42:26 +00:00
return ret;
2010-12-16 19:23:22 +00:00
}
/* Public API */
2013-05-28 12:35:55 +00:00
static HashIndex *
hashindex_open(const char *path, int readonly)
2010-12-16 19:23:22 +00:00
{
2013-07-03 10:19:16 +00:00
void *addr;
int fd, oflags, prot;
2013-07-03 10:19:16 +00:00
off_t length;
HashHeader *header;
HashIndex *index;
if(readonly) {
oflags = O_RDONLY;
prot = PROT_READ;
}
else {
oflags = O_RDWR;
prot = PROT_READ | PROT_WRITE;
}
if((fd = open(path, oflags)) < 0) {
2013-07-03 10:19:16 +00:00
EPRINTF_PATH(path, "open failed");
2010-12-17 21:14:36 +00:00
fprintf(stderr, "Failed to open %s\n", path);
2010-12-16 19:23:22 +00:00
return NULL;
}
2013-07-03 10:19:16 +00:00
if((length = lseek(fd, 0, SEEK_END)) < 0) {
EPRINTF_PATH(path, "lseek failed");
if(close(fd) < 0) {
EPRINTF_PATH(path, "close failed");
}
return NULL;
}
addr = mmap(0, length, prot, MAP_SHARED, fd, 0);
if(close(fd) < 0) {
EPRINTF_PATH(path, "close failed");
2013-07-03 10:19:16 +00:00
return NULL;
}
if(addr == MAP_FAILED) {
2013-07-03 10:19:16 +00:00
EPRINTF_PATH(path, "mmap failed");
return NULL;
}
header = (HashHeader *)addr;
if(memcmp(header->magic, MAGIC, 8)) {
EPRINTF_PATH(path, "Unknown file header");
return NULL;
}
if(length != sizeof(HashHeader) + header->num_buckets * (header->key_size + header->value_size)) {
EPRINTF_PATH(path, "Incorrect file length");
return NULL;
}
if(!(index = malloc(sizeof(HashIndex)))) {
EPRINTF_PATH(path, "malloc failed");
return NULL;
2010-12-16 19:23:22 +00:00
}
index->readonly = readonly;
2010-12-16 19:23:22 +00:00
index->map_addr = addr;
index->map_length = length;
index->num_entries = header->num_entries;
index->num_buckets = header->num_buckets;
index->key_size = header->key_size;
index->value_size = header->value_size;
index->bucket_size = index->key_size + index->value_size;
index->buckets = (addr + sizeof(HashHeader));
2013-07-03 10:19:16 +00:00
index->lower_limit = index->num_buckets > MIN_BUCKETS ? ((int)(index->num_buckets * BUCKET_LOWER_LIMIT)) : 0;
index->upper_limit = (int)(index->num_buckets * BUCKET_UPPER_LIMIT);
if(!(index->path = strdup(path))) {
EPRINTF_PATH(path, "strdup failed");
free(index);
return NULL;
}
2010-12-16 19:23:22 +00:00
return index;
}
2013-05-28 12:35:55 +00:00
static HashIndex *
2010-12-16 19:23:22 +00:00
hashindex_create(const char *path, int capacity, int key_size, int value_size)
{
FILE *fd;
2013-07-03 10:19:16 +00:00
char bucket[MAX_BUCKET_SIZE] = {};
int i, bucket_size;
HashHeader header = {
.magic = MAGIC, .num_entries = 0, .key_size = key_size, .value_size = value_size
};
capacity = MAX(MIN_BUCKETS, capacity);
header.num_buckets = capacity;
2010-12-16 19:23:22 +00:00
if(!(fd = fopen(path, "w"))) {
2013-07-03 10:19:16 +00:00
EPRINTF_PATH(path, "fopen failed");
2010-12-16 19:23:22 +00:00
return NULL;
}
2013-07-03 10:19:16 +00:00
bucket_size = key_size + value_size;
if(fwrite(&header, 1, sizeof(header), fd) != sizeof(header)) {
2010-12-16 19:23:22 +00:00
goto error;
2013-07-03 10:19:16 +00:00
}
2010-12-16 19:23:22 +00:00
*((int32_t *)(bucket + key_size)) = EMPTY;
for(i = 0; i < capacity; i++) {
2013-07-03 10:19:16 +00:00
if(fwrite(bucket, 1, bucket_size, fd) != bucket_size) {
2010-12-16 19:23:22 +00:00
goto error;
2013-07-03 10:19:16 +00:00
}
}
if(fclose(fd) < 0) {
EPRINTF_PATH(path, "fclose failed");
2013-10-02 18:42:26 +00:00
if(unlink(path) < 0) {
EPRINTF_PATH(path, "unlink failed");
}
2013-07-03 10:19:16 +00:00
return NULL;
2010-12-16 19:23:22 +00:00
}
return hashindex_open(path, 0);
2010-12-16 19:23:22 +00:00
error:
2013-10-02 18:42:26 +00:00
if(unlink(path) < 0) {
EPRINTF_PATH(path, "unlink failed");
}
2013-07-03 10:19:16 +00:00
EPRINTF_PATH(path, "fwrite failed");
if(fclose(fd) < 0) {
EPRINTF_PATH(path, "fclose failed");
}
2010-12-16 19:23:22 +00:00
return NULL;
}
2013-07-03 10:19:16 +00:00
static int
hashindex_clear(HashIndex *index)
{
int i;
for(i = 0; i < index->num_buckets; i++) {
BUCKET_MARK_DELETED(index, i);
}
index->num_entries = 0;
2013-07-03 10:19:16 +00:00
return hashindex_resize(index, MIN_BUCKETS);
}
2013-07-03 10:19:16 +00:00
static int
2010-12-16 19:23:22 +00:00
hashindex_flush(HashIndex *index)
{
if(index->readonly) {
return 1;
}
2010-12-16 19:23:22 +00:00
*((int32_t *)(index->map_addr + 8)) = index->num_entries;
*((int32_t *)(index->map_addr + 12)) = index->num_buckets;
2013-07-03 10:19:16 +00:00
if(msync(index->map_addr, index->map_length, MS_SYNC) < 0) {
EPRINTF("msync failed");
return 0;
}
return 1;
2010-12-16 19:23:22 +00:00
}
2013-07-03 10:19:16 +00:00
static int
2010-12-16 19:23:22 +00:00
hashindex_close(HashIndex *index)
{
2013-07-03 10:19:16 +00:00
int rv = 1;
if(hashindex_flush(index) < 0) {
rv = 0;
}
if(munmap(index->map_addr, index->map_length) < 0) {
EPRINTF("munmap failed");
rv = 0;
}
2010-12-16 19:23:22 +00:00
free(index->path);
free(index);
2013-07-03 10:19:16 +00:00
return rv;
2010-12-16 19:23:22 +00:00
}
2013-05-28 12:35:55 +00:00
static const void *
2010-12-16 19:23:22 +00:00
hashindex_get(HashIndex *index, const void *key)
{
int idx = hashindex_lookup(index, key);
if(idx < 0) {
return NULL;
}
2011-08-14 18:06:30 +00:00
return BUCKET_ADDR_READ(index, idx) + index->key_size;
2010-12-16 19:23:22 +00:00
}
2013-07-03 10:19:16 +00:00
static int
2010-12-16 19:23:22 +00:00
hashindex_set(HashIndex *index, const void *key, const void *value)
{
int idx = hashindex_lookup(index, key);
2011-08-14 18:06:30 +00:00
uint8_t *ptr;
2010-12-16 19:23:22 +00:00
if(idx < 0)
{
2013-07-03 10:19:16 +00:00
if(index->num_entries > index->upper_limit) {
if(!hashindex_resize(index, index->num_buckets * 2)) {
return 0;
}
2010-12-16 19:23:22 +00:00
}
idx = hashindex_index(index, key);
while(!BUCKET_IS_EMPTY(index, idx) && !BUCKET_IS_DELETED(index, idx)) {
idx = (idx + 1) % index->num_buckets;
}
2011-08-14 18:06:30 +00:00
ptr = BUCKET_ADDR_WRITE(index, idx);
memcpy(ptr, key, index->key_size);
memcpy(ptr + index->key_size, value, index->value_size);
2010-12-16 19:23:22 +00:00
index->num_entries += 1;
}
else
{
2011-08-14 18:06:30 +00:00
memcpy(BUCKET_ADDR_WRITE(index, idx) + index->key_size, value, index->value_size);
2010-12-16 19:23:22 +00:00
}
2013-07-03 10:19:16 +00:00
return 1;
2010-12-16 19:23:22 +00:00
}
2013-07-03 10:19:16 +00:00
static int
2010-12-16 19:23:22 +00:00
hashindex_delete(HashIndex *index, const void *key)
{
int idx = hashindex_lookup(index, key);
if (idx < 0) {
2013-07-03 10:19:16 +00:00
return 1;
2010-12-16 19:23:22 +00:00
}
BUCKET_MARK_DELETED(index, idx);
index->num_entries -= 1;
2013-07-03 10:19:16 +00:00
if(index->num_entries < index->lower_limit) {
2013-10-02 18:42:26 +00:00
if(!hashindex_resize(index, index->num_buckets / 2)) {
2013-07-03 10:19:16 +00:00
return 0;
}
}
return 1;
2010-12-16 19:23:22 +00:00
}
2013-05-28 12:35:55 +00:00
static void *
2010-12-16 19:23:22 +00:00
hashindex_next_key(HashIndex *index, const void *key)
{
int idx = 0;
if(key) {
idx = 1 + (key - index->buckets) / index->bucket_size;
}
2013-07-03 10:19:16 +00:00
if (idx == index->num_buckets) {
2010-12-16 19:23:22 +00:00
return NULL;
2013-07-03 10:19:16 +00:00
}
2010-12-16 19:23:22 +00:00
while(BUCKET_IS_EMPTY(index, idx) || BUCKET_IS_DELETED(index, idx)) {
idx ++;
2013-07-03 10:19:16 +00:00
if (idx == index->num_buckets) {
2010-12-16 19:23:22 +00:00
return NULL;
2013-07-03 10:19:16 +00:00
}
2010-12-16 19:23:22 +00:00
}
2011-08-14 18:06:30 +00:00
return BUCKET_ADDR_READ(index, idx);
2010-12-16 19:23:22 +00:00
}
2013-05-28 12:35:55 +00:00
static int
2010-12-16 19:23:22 +00:00
hashindex_get_size(HashIndex *index)
{
return index->num_entries;
}