borg/attic/chunker.pyx

63 lines
1.9 KiB
Cython
Raw Normal View History

2013-05-28 12:35:55 +00:00
# -*- coding: utf-8 -*-
2014-08-03 13:04:41 +00:00
API_VERSION = 2
2013-05-28 12:35:55 +00:00
from libc.stdlib cimport free
cdef extern from "_chunker.c":
ctypedef int uint32_t
2014-08-03 13:04:41 +00:00
ctypedef struct _Chunker "Chunker":
2013-05-28 12:35:55 +00:00
pass
_Chunker *chunker_init(int window_size, int chunk_mask, int min_size, int max_size, uint32_t seed)
void chunker_set_fd(_Chunker *chunker, object f, int fd)
2014-08-03 13:04:41 +00:00
void chunker_free(_Chunker *chunker)
object chunker_process(_Chunker *chunker)
2013-05-28 12:35:55 +00:00
uint32_t *buzhash_init_table(uint32_t seed)
uint32_t c_buzhash "buzhash"(unsigned char *data, size_t len, uint32_t *h)
uint32_t c_buzhash_update "buzhash_update"(uint32_t sum, unsigned char remove, unsigned char add, size_t len, uint32_t *h)
2013-05-28 12:35:55 +00:00
2014-08-03 13:04:41 +00:00
cdef class Chunker:
cdef _Chunker *chunker
2013-05-28 12:35:55 +00:00
def __cinit__(self, window_size, chunk_mask, min_size, max_size, seed):
self.chunker = chunker_init(window_size, chunk_mask, min_size, max_size, seed & 0xffffffff)
2014-08-03 13:04:41 +00:00
def chunkify(self, fd, fh=-1):
"""
Cut a file into chunks.
:param fd: Python file object
:param fh: OS-level file handle (if available),
defaults to -1 which means not to use OS-level fd.
"""
chunker_set_fd(self.chunker, fd, fh)
2014-08-03 13:04:41 +00:00
return self
2013-05-28 12:35:55 +00:00
def __dealloc__(self):
if self.chunker:
chunker_free(self.chunker)
def __iter__(self):
return self
def __next__(self):
return chunker_process(self.chunker)
def buzhash(unsigned char *data, unsigned long seed):
cdef uint32_t *table
cdef uint32_t sum
table = buzhash_init_table(seed & 0xffffffff)
sum = c_buzhash(data, len(data), table)
free(table)
return sum
def buzhash_update(uint32_t sum, unsigned char remove, unsigned char add, size_t len, unsigned long seed):
cdef uint32_t *table
table = buzhash_init_table(seed & 0xffffffff)
sum = c_buzhash_update(sum, remove, add, len, table)
free(table)
return sum