2010-12-16 19:23:22 +00:00
|
|
|
# -*- coding: utf-8 -*-
|
2013-06-29 12:22:05 +00:00
|
|
|
import os
|
|
|
|
|
2016-04-11 22:10:44 +00:00
|
|
|
cimport cython
|
|
|
|
from libc.stdint cimport uint32_t, UINT32_MAX, uint64_t
|
|
|
|
|
2014-07-10 13:32:12 +00:00
|
|
|
API_VERSION = 2
|
2014-03-18 21:04:08 +00:00
|
|
|
|
2010-12-16 19:23:22 +00:00
|
|
|
|
2013-05-28 12:35:55 +00:00
|
|
|
cdef extern from "_hashindex.c":
|
2010-12-16 19:23:22 +00:00
|
|
|
ctypedef struct HashIndex:
|
|
|
|
pass
|
|
|
|
|
2014-07-10 13:32:12 +00:00
|
|
|
HashIndex *hashindex_read(char *path)
|
|
|
|
HashIndex *hashindex_init(int capacity, int key_size, int value_size)
|
|
|
|
void hashindex_free(HashIndex *index)
|
2015-08-06 21:32:53 +00:00
|
|
|
void hashindex_merge(HashIndex *index, HashIndex *other)
|
2015-12-07 18:13:58 +00:00
|
|
|
void hashindex_add(HashIndex *index, void *key, void *value)
|
2010-12-16 19:23:22 +00:00
|
|
|
int hashindex_get_size(HashIndex *index)
|
2014-07-10 13:32:12 +00:00
|
|
|
int hashindex_write(HashIndex *index, char *path)
|
2010-12-16 19:23:22 +00:00
|
|
|
void *hashindex_get(HashIndex *index, void *key)
|
|
|
|
void *hashindex_next_key(HashIndex *index, void *key)
|
2013-07-03 10:19:16 +00:00
|
|
|
int hashindex_delete(HashIndex *index, void *key)
|
|
|
|
int hashindex_set(HashIndex *index, void *key, void *value)
|
2016-04-11 22:10:44 +00:00
|
|
|
uint32_t _htole32(uint32_t v)
|
|
|
|
uint32_t _le32toh(uint32_t v)
|
2010-12-16 19:23:22 +00:00
|
|
|
|
2013-05-28 12:35:55 +00:00
|
|
|
|
2015-08-04 11:30:35 +00:00
|
|
|
cdef _NoDefault = object()
|
2010-12-16 19:23:22 +00:00
|
|
|
|
2016-04-11 22:10:44 +00:00
|
|
|
"""
|
|
|
|
The HashIndex is *not* a general purpose data structure. The value size must be at least 4 bytes, and these
|
|
|
|
first bytes are used for in-band signalling in the data structure itself.
|
|
|
|
|
|
|
|
The constant MAX_VALUE defines the valid range for these 4 bytes when interpreted as an uint32_t from 0
|
|
|
|
to MAX_VALUE (inclusive). The following reserved values beyond MAX_VALUE are currently in use
|
|
|
|
(byte order is LE)::
|
|
|
|
|
|
|
|
0xffffffff marks empty entries in the hashtable
|
|
|
|
0xfffffffe marks deleted entries in the hashtable
|
|
|
|
|
|
|
|
None of the publicly available classes in this module will accept nor return a reserved value;
|
|
|
|
AssertionError is raised instead.
|
|
|
|
"""
|
|
|
|
|
|
|
|
assert UINT32_MAX == 2**32-1
|
|
|
|
|
|
|
|
# module-level constant because cdef's in classes can't have default values
|
|
|
|
cdef uint32_t _MAX_VALUE = 2**32-1025
|
|
|
|
MAX_VALUE = _MAX_VALUE
|
|
|
|
|
|
|
|
assert _MAX_VALUE % 2 == 1
|
2015-08-04 11:30:35 +00:00
|
|
|
|
|
|
|
@cython.internal
|
2010-12-16 19:23:22 +00:00
|
|
|
cdef class IndexBase:
|
|
|
|
cdef HashIndex *index
|
2015-08-16 12:51:15 +00:00
|
|
|
cdef int key_size
|
2010-12-16 19:23:22 +00:00
|
|
|
|
2015-08-16 12:51:15 +00:00
|
|
|
def __cinit__(self, capacity=0, path=None, key_size=32):
|
|
|
|
self.key_size = key_size
|
2014-07-10 13:32:12 +00:00
|
|
|
if path:
|
2015-09-12 21:21:49 +00:00
|
|
|
path = os.fsencode(path)
|
|
|
|
self.index = hashindex_read(path)
|
2014-07-10 13:32:12 +00:00
|
|
|
if not self.index:
|
|
|
|
raise Exception('hashindex_read failed')
|
|
|
|
else:
|
|
|
|
self.index = hashindex_init(capacity, self.key_size, self.value_size)
|
|
|
|
if not self.index:
|
|
|
|
raise Exception('hashindex_init failed')
|
2010-12-16 19:23:22 +00:00
|
|
|
|
|
|
|
def __dealloc__(self):
|
2013-06-03 11:45:48 +00:00
|
|
|
if self.index:
|
2014-07-10 13:32:12 +00:00
|
|
|
hashindex_free(self.index)
|
2013-07-03 10:19:16 +00:00
|
|
|
|
|
|
|
@classmethod
|
2014-07-10 13:32:12 +00:00
|
|
|
def read(cls, path):
|
|
|
|
return cls(path=path)
|
2010-12-16 19:23:22 +00:00
|
|
|
|
2014-07-10 13:32:12 +00:00
|
|
|
def write(self, path):
|
2015-09-12 21:21:49 +00:00
|
|
|
path = os.fsencode(path)
|
|
|
|
if not hashindex_write(self.index, path):
|
2014-07-10 13:32:12 +00:00
|
|
|
raise Exception('hashindex_write failed')
|
2011-06-18 09:26:20 +00:00
|
|
|
|
2014-07-10 13:32:12 +00:00
|
|
|
def clear(self):
|
|
|
|
hashindex_free(self.index)
|
|
|
|
self.index = hashindex_init(0, self.key_size, self.value_size)
|
|
|
|
if not self.index:
|
|
|
|
raise Exception('hashindex_init failed')
|
2010-12-16 19:23:22 +00:00
|
|
|
|
|
|
|
def setdefault(self, key, value):
|
|
|
|
if not key in self:
|
|
|
|
self[key] = value
|
|
|
|
|
2013-07-03 10:19:16 +00:00
|
|
|
def __delitem__(self, key):
|
2015-08-16 12:51:15 +00:00
|
|
|
assert len(key) == self.key_size
|
2013-07-03 10:19:16 +00:00
|
|
|
if not hashindex_delete(self.index, <char *>key):
|
|
|
|
raise Exception('hashindex_delete failed')
|
|
|
|
|
2010-12-21 20:29:09 +00:00
|
|
|
def get(self, key, default=None):
|
|
|
|
try:
|
|
|
|
return self[key]
|
|
|
|
except KeyError:
|
|
|
|
return default
|
|
|
|
|
2011-06-18 09:26:20 +00:00
|
|
|
def pop(self, key, default=_NoDefault):
|
|
|
|
try:
|
|
|
|
value = self[key]
|
|
|
|
del self[key]
|
|
|
|
return value
|
|
|
|
except KeyError:
|
|
|
|
if default != _NoDefault:
|
|
|
|
return default
|
|
|
|
raise
|
2010-12-16 19:23:22 +00:00
|
|
|
|
|
|
|
def __len__(self):
|
|
|
|
return hashindex_get_size(self.index)
|
|
|
|
|
|
|
|
|
|
|
|
cdef class NSIndex(IndexBase):
|
|
|
|
|
2013-07-03 10:19:16 +00:00
|
|
|
value_size = 8
|
2010-12-16 19:23:22 +00:00
|
|
|
|
|
|
|
def __getitem__(self, key):
|
2015-08-16 12:51:15 +00:00
|
|
|
assert len(key) == self.key_size
|
2016-04-11 22:10:44 +00:00
|
|
|
data = <uint32_t *>hashindex_get(self.index, <char *>key)
|
2010-12-16 19:23:22 +00:00
|
|
|
if not data:
|
2016-04-11 22:10:44 +00:00
|
|
|
raise KeyError(key)
|
|
|
|
cdef uint32_t segment = _le32toh(data[0])
|
|
|
|
assert segment <= _MAX_VALUE, "maximum number of segments reached"
|
|
|
|
return segment, _le32toh(data[1])
|
2010-12-16 19:23:22 +00:00
|
|
|
|
|
|
|
def __setitem__(self, key, value):
|
2015-08-16 12:51:15 +00:00
|
|
|
assert len(key) == self.key_size
|
2016-04-11 22:10:44 +00:00
|
|
|
cdef uint32_t[2] data
|
|
|
|
cdef uint32_t segment = value[0]
|
|
|
|
assert segment <= _MAX_VALUE, "maximum number of segments reached"
|
|
|
|
data[0] = _htole32(segment)
|
2014-01-29 20:34:21 +00:00
|
|
|
data[1] = _htole32(value[1])
|
2013-07-03 10:19:16 +00:00
|
|
|
if not hashindex_set(self.index, <char *>key, data):
|
|
|
|
raise Exception('hashindex_set failed')
|
2010-12-16 19:23:22 +00:00
|
|
|
|
|
|
|
def __contains__(self, key):
|
2016-04-11 22:10:44 +00:00
|
|
|
cdef uint32_t segment
|
2015-08-16 12:51:15 +00:00
|
|
|
assert len(key) == self.key_size
|
2016-04-11 22:10:44 +00:00
|
|
|
data = <uint32_t *>hashindex_get(self.index, <char *>key)
|
|
|
|
if data != NULL:
|
|
|
|
segment = _le32toh(data[0])
|
|
|
|
assert segment <= _MAX_VALUE, "maximum number of segments reached"
|
2010-12-16 19:23:22 +00:00
|
|
|
return data != NULL
|
|
|
|
|
2014-02-09 21:05:33 +00:00
|
|
|
def iteritems(self, marker=None):
|
|
|
|
cdef const void *key
|
2015-08-16 12:51:15 +00:00
|
|
|
iter = NSKeyIterator(self.key_size)
|
2014-02-04 22:49:10 +00:00
|
|
|
iter.idx = self
|
2010-12-16 19:23:22 +00:00
|
|
|
iter.index = self.index
|
2014-02-09 21:05:33 +00:00
|
|
|
if marker:
|
|
|
|
key = hashindex_get(self.index, <char *>marker)
|
|
|
|
if marker is None:
|
|
|
|
raise IndexError
|
2015-08-16 12:51:15 +00:00
|
|
|
iter.key = key - self.key_size
|
2010-12-16 19:23:22 +00:00
|
|
|
return iter
|
|
|
|
|
|
|
|
|
|
|
|
cdef class NSKeyIterator:
|
2014-02-04 22:49:10 +00:00
|
|
|
cdef NSIndex idx
|
2010-12-16 19:23:22 +00:00
|
|
|
cdef HashIndex *index
|
2014-02-09 21:05:33 +00:00
|
|
|
cdef const void *key
|
2015-08-16 12:51:15 +00:00
|
|
|
cdef int key_size
|
2010-12-16 19:23:22 +00:00
|
|
|
|
2015-08-16 12:51:15 +00:00
|
|
|
def __cinit__(self, key_size):
|
2010-12-16 19:23:22 +00:00
|
|
|
self.key = NULL
|
2015-08-16 12:51:15 +00:00
|
|
|
self.key_size = key_size
|
2010-12-16 19:23:22 +00:00
|
|
|
|
|
|
|
def __iter__(self):
|
|
|
|
return self
|
|
|
|
|
|
|
|
def __next__(self):
|
2014-02-09 21:05:33 +00:00
|
|
|
self.key = hashindex_next_key(self.index, <char *>self.key)
|
2010-12-16 19:23:22 +00:00
|
|
|
if not self.key:
|
|
|
|
raise StopIteration
|
2016-04-11 22:10:44 +00:00
|
|
|
cdef uint32_t *value = <uint32_t *>(self.key + self.key_size)
|
|
|
|
cdef uint32_t segment = _le32toh(value[0])
|
|
|
|
assert segment <= _MAX_VALUE, "maximum number of segments reached"
|
|
|
|
return (<char *>self.key)[:self.key_size], (segment, _le32toh(value[1]))
|
2010-12-16 19:23:22 +00:00
|
|
|
|
|
|
|
|
2011-07-30 19:13:48 +00:00
|
|
|
cdef class ChunkIndex(IndexBase):
|
2016-04-11 22:10:44 +00:00
|
|
|
"""
|
|
|
|
Mapping of 32 byte keys to (refcount, size, csize), which are all 32-bit unsigned.
|
|
|
|
|
|
|
|
The reference count cannot overflow. If an overflow would occur, the refcount
|
|
|
|
is fixed to MAX_VALUE and will neither increase nor decrease by incref(), decref()
|
|
|
|
or add().
|
|
|
|
|
|
|
|
Prior signed 32-bit overflow is handled correctly for most cases: All values
|
|
|
|
from UINT32_MAX (2**32-1, inclusive) to MAX_VALUE (exclusive) are reserved and either
|
|
|
|
cause silent data loss (-1, -2) or will raise an AssertionError when accessed.
|
|
|
|
Other values are handled correctly. Note that previously the refcount could also reach
|
|
|
|
0 by *increasing* it.
|
|
|
|
|
|
|
|
Assigning refcounts in this reserved range is an invalid operation and raises AssertionError.
|
|
|
|
"""
|
2011-07-30 19:13:48 +00:00
|
|
|
|
2013-07-03 10:19:16 +00:00
|
|
|
value_size = 12
|
2011-07-30 19:13:48 +00:00
|
|
|
|
|
|
|
def __getitem__(self, key):
|
2015-08-16 12:51:15 +00:00
|
|
|
assert len(key) == self.key_size
|
2016-04-11 22:10:44 +00:00
|
|
|
data = <uint32_t *>hashindex_get(self.index, <char *>key)
|
2011-07-30 19:13:48 +00:00
|
|
|
if not data:
|
2016-04-11 22:10:44 +00:00
|
|
|
raise KeyError(key)
|
|
|
|
cdef uint32_t refcount = _le32toh(data[0])
|
|
|
|
assert refcount <= _MAX_VALUE
|
|
|
|
return refcount, _le32toh(data[1]), _le32toh(data[2])
|
2011-07-30 19:13:48 +00:00
|
|
|
|
|
|
|
def __setitem__(self, key, value):
|
2015-08-16 12:51:15 +00:00
|
|
|
assert len(key) == self.key_size
|
2016-04-11 22:10:44 +00:00
|
|
|
cdef uint32_t[3] data
|
|
|
|
cdef uint32_t refcount = value[0]
|
|
|
|
assert refcount <= _MAX_VALUE, "invalid reference count"
|
|
|
|
data[0] = _htole32(refcount)
|
2014-01-29 20:34:21 +00:00
|
|
|
data[1] = _htole32(value[1])
|
|
|
|
data[2] = _htole32(value[2])
|
2013-07-03 10:19:16 +00:00
|
|
|
if not hashindex_set(self.index, <char *>key, data):
|
|
|
|
raise Exception('hashindex_set failed')
|
2011-07-30 19:13:48 +00:00
|
|
|
|
|
|
|
def __contains__(self, key):
|
2015-08-16 12:51:15 +00:00
|
|
|
assert len(key) == self.key_size
|
2016-04-11 22:10:44 +00:00
|
|
|
data = <uint32_t *>hashindex_get(self.index, <char *>key)
|
|
|
|
if data != NULL:
|
|
|
|
assert data[0] <= _MAX_VALUE
|
2011-07-30 19:13:48 +00:00
|
|
|
return data != NULL
|
|
|
|
|
2016-04-11 22:10:44 +00:00
|
|
|
def incref(self, key):
|
|
|
|
"""Increase refcount for 'key', return (refcount, size, csize)"""
|
|
|
|
assert len(key) == self.key_size
|
|
|
|
data = <uint32_t *>hashindex_get(self.index, <char *>key)
|
|
|
|
if not data:
|
|
|
|
raise KeyError(key)
|
|
|
|
cdef uint32_t refcount = _le32toh(data[0])
|
|
|
|
assert refcount <= _MAX_VALUE, "invalid reference count"
|
|
|
|
if refcount != _MAX_VALUE:
|
|
|
|
refcount += 1
|
|
|
|
data[0] = _htole32(refcount)
|
|
|
|
return refcount, _le32toh(data[1]), _le32toh(data[2])
|
|
|
|
|
|
|
|
def decref(self, key):
|
|
|
|
"""Decrease refcount for 'key', return (refcount, size, csize)"""
|
|
|
|
assert len(key) == self.key_size
|
|
|
|
data = <uint32_t *>hashindex_get(self.index, <char *>key)
|
|
|
|
if not data:
|
|
|
|
raise KeyError(key)
|
|
|
|
cdef uint32_t refcount = _le32toh(data[0])
|
|
|
|
# Never decrease a reference count of zero
|
|
|
|
assert 0 < refcount <= _MAX_VALUE, "invalid reference count"
|
|
|
|
if refcount != _MAX_VALUE:
|
|
|
|
refcount -= 1
|
|
|
|
data[0] = _htole32(refcount)
|
|
|
|
return refcount, _le32toh(data[1]), _le32toh(data[2])
|
|
|
|
|
2014-02-09 21:05:33 +00:00
|
|
|
def iteritems(self, marker=None):
|
|
|
|
cdef const void *key
|
2015-08-16 12:51:15 +00:00
|
|
|
iter = ChunkKeyIterator(self.key_size)
|
2014-02-04 22:49:10 +00:00
|
|
|
iter.idx = self
|
2011-07-30 19:13:48 +00:00
|
|
|
iter.index = self.index
|
2014-02-09 21:05:33 +00:00
|
|
|
if marker:
|
|
|
|
key = hashindex_get(self.index, <char *>marker)
|
|
|
|
if marker is None:
|
|
|
|
raise IndexError
|
2015-08-16 12:51:15 +00:00
|
|
|
iter.key = key - self.key_size
|
2011-07-30 19:13:48 +00:00
|
|
|
return iter
|
|
|
|
|
2014-03-18 20:42:03 +00:00
|
|
|
def summarize(self):
|
2016-04-11 22:10:44 +00:00
|
|
|
cdef uint64_t size = 0, csize = 0, unique_size = 0, unique_csize = 0, chunks = 0, unique_chunks = 0
|
|
|
|
cdef uint32_t *values
|
|
|
|
cdef uint32_t refcount
|
2016-04-11 21:48:01 +00:00
|
|
|
cdef void *key = NULL
|
|
|
|
|
|
|
|
while True:
|
|
|
|
key = hashindex_next_key(self.index, key)
|
|
|
|
if not key:
|
|
|
|
break
|
|
|
|
unique_chunks += 1
|
2016-04-11 22:10:44 +00:00
|
|
|
values = <uint32_t*> (key + self.key_size)
|
|
|
|
refcount = _le32toh(values[0])
|
|
|
|
assert refcount <= MAX_VALUE, "invalid reference count"
|
|
|
|
chunks += refcount
|
2016-04-11 21:48:01 +00:00
|
|
|
unique_size += _le32toh(values[1])
|
|
|
|
unique_csize += _le32toh(values[2])
|
2016-04-11 22:10:44 +00:00
|
|
|
size += <uint64_t> _le32toh(values[1]) * _le32toh(values[0])
|
|
|
|
csize += <uint64_t> _le32toh(values[2]) * _le32toh(values[0])
|
2016-04-11 21:48:01 +00:00
|
|
|
|
|
|
|
return size, csize, unique_size, unique_csize, unique_chunks, chunks
|
2014-03-18 20:42:03 +00:00
|
|
|
|
2015-12-07 18:13:58 +00:00
|
|
|
def add(self, key, refs, size, csize):
|
|
|
|
assert len(key) == self.key_size
|
2016-04-11 22:10:44 +00:00
|
|
|
cdef uint32_t[3] data
|
2015-12-07 18:13:58 +00:00
|
|
|
data[0] = _htole32(refs)
|
|
|
|
data[1] = _htole32(size)
|
|
|
|
data[2] = _htole32(csize)
|
2016-04-11 22:10:44 +00:00
|
|
|
self._add(<char*> key, data)
|
|
|
|
|
|
|
|
cdef _add(self, void *key, uint32_t *data):
|
|
|
|
cdef uint64_t refcount1, refcount2, result64
|
|
|
|
values = <uint32_t*> hashindex_get(self.index, key)
|
|
|
|
if values:
|
|
|
|
refcount1 = _le32toh(values[0])
|
|
|
|
refcount2 = _le32toh(data[0])
|
|
|
|
assert refcount1 <= _MAX_VALUE
|
|
|
|
assert refcount2 <= _MAX_VALUE
|
|
|
|
result64 = refcount1 + refcount2
|
|
|
|
values[0] = _htole32(min(result64, _MAX_VALUE))
|
|
|
|
else:
|
|
|
|
hashindex_set(self.index, key, data)
|
2015-12-07 18:13:58 +00:00
|
|
|
|
2015-08-06 21:32:53 +00:00
|
|
|
def merge(self, ChunkIndex other):
|
2016-04-11 22:10:44 +00:00
|
|
|
cdef void *key = NULL
|
|
|
|
|
|
|
|
while True:
|
|
|
|
key = hashindex_next_key(other.index, key)
|
|
|
|
if not key:
|
|
|
|
break
|
|
|
|
self._add(key, <uint32_t*> (key + self.key_size))
|
2015-08-06 21:32:53 +00:00
|
|
|
|
2011-07-30 19:13:48 +00:00
|
|
|
|
|
|
|
cdef class ChunkKeyIterator:
|
2014-02-04 22:49:10 +00:00
|
|
|
cdef ChunkIndex idx
|
2011-07-30 19:13:48 +00:00
|
|
|
cdef HashIndex *index
|
2014-02-09 21:05:33 +00:00
|
|
|
cdef const void *key
|
2015-08-16 12:51:15 +00:00
|
|
|
cdef int key_size
|
2011-07-30 19:13:48 +00:00
|
|
|
|
2015-08-16 12:51:15 +00:00
|
|
|
def __cinit__(self, key_size):
|
2011-07-30 19:13:48 +00:00
|
|
|
self.key = NULL
|
2015-08-16 12:51:15 +00:00
|
|
|
self.key_size = key_size
|
2011-07-30 19:13:48 +00:00
|
|
|
|
|
|
|
def __iter__(self):
|
|
|
|
return self
|
|
|
|
|
|
|
|
def __next__(self):
|
2014-02-09 21:05:33 +00:00
|
|
|
self.key = hashindex_next_key(self.index, <char *>self.key)
|
2011-07-30 19:13:48 +00:00
|
|
|
if not self.key:
|
|
|
|
raise StopIteration
|
2016-04-11 22:10:44 +00:00
|
|
|
cdef uint32_t *value = <uint32_t *>(self.key + self.key_size)
|
|
|
|
cdef uint32_t refcount = _le32toh(value[0])
|
|
|
|
assert refcount <= MAX_VALUE, "invalid reference count"
|
|
|
|
return (<char *>self.key)[:self.key_size], (refcount, _le32toh(value[1]), _le32toh(value[2]))
|