1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2024-12-23 08:16:54 +00:00
borg/dedupestore/_speedups.c

264 lines
7.4 KiB
C
Raw Normal View History

2010-02-28 19:22:45 +00:00
#include <Python/Python.h>
#include <Python/structmember.h>
2010-03-03 21:52:57 +00:00
static unsigned long int
checksum(const unsigned char *data, int len, unsigned long int sum)
{
unsigned long int s1, s2, i;
s1 = sum & 0xffff;
s2 = sum >> 16;
for(i=0; i < len; i++)
{
s1 += data[i] + 1;
s2 += s1;
}
return ((s2 & 0xffff) << 16) | (s1 & 0xffff);
}
static unsigned long int
roll_checksum(unsigned long int sum, unsigned char remove, unsigned char add, int len)
{
unsigned long int s1, s2;
s1 = sum & 0xffff;
s2 = sum >> 16;
s1 -= remove - add;
s2 -= len * (remove + 1) - s1;
return ((s2 & 0xffff) << 16) | (s1 & 0xffff);
}
2010-02-28 19:22:45 +00:00
typedef struct {
2010-03-03 21:52:57 +00:00
PyObject_HEAD
int chunk_size, i, full_sum, done, buf_size, data_len;
PyObject *chunks, *fd, *extra;
unsigned long sum;
unsigned char *data, add, remove;
2010-02-28 19:22:45 +00:00
} ChunkifyIter;
2010-03-03 21:52:57 +00:00
static PyObject*
ChunkifyIter_iter(PyObject *self)
{
Py_INCREF(self);
return self;
}
static void
ChunkifyIter_dealloc(PyObject *self)
2010-02-28 19:22:45 +00:00
{
2010-03-03 21:52:57 +00:00
ChunkifyIter *c = (ChunkifyIter *)self;
Py_DECREF(c->fd);
2010-03-09 21:27:37 +00:00
Py_XDECREF(c->chunks);
2010-03-03 21:52:57 +00:00
free(c->data);
self->ob_type->tp_free(self);
2010-02-28 19:22:45 +00:00
}
2010-03-03 21:52:57 +00:00
static PyObject*
ChunkifyIter_iternext(PyObject *self)
2010-02-28 19:22:45 +00:00
{
2010-03-03 21:52:57 +00:00
ChunkifyIter *c = (ChunkifyIter *)self;
PyObject *pysum;
2010-03-03 21:52:57 +00:00
int o = 0;
if(c->done)
{
PyErr_SetNone(PyExc_StopIteration);
return NULL;
}
if(c->extra)
{
c->done = 1;
2010-03-09 21:27:37 +00:00
Py_INCREF(c->extra);
2010-03-03 21:52:57 +00:00
return c->extra;
}
for(;;)
{
if(c->i > c->buf_size - c->chunk_size)
{
memmove(c->data, c->data + c->i - o, c->data_len - c->i + o);
c->data_len -= c->i - o;
c->i = o;
}
if(c->data_len - c->i < c->chunk_size)
{
PyObject *data = PyObject_CallMethod(c->fd, "read", "i", c->buf_size - c->data_len);
int n = PyString_Size(data);
memcpy(c->data + c->data_len, PyString_AsString(data), n);
c->data_len += n;
Py_DECREF(data);
}
if(c->i == c->data_len)
{
PyErr_SetNone(PyExc_StopIteration);
return NULL;
}
if(c->data_len - c->i < c->chunk_size) /* EOF ? */
{
if(o == 1)
{
c->done = 1;
return PyString_FromStringAndSize((char *)(c->data + c->i - 1), c->data_len - c->i + 1);
}
else if(o > 1)
{
c->extra = PyString_FromStringAndSize((char *)(c->data + c->i - 1), c->chunk_size);
return PyString_FromStringAndSize((char *)(c->data + c->i - o), o - 1);
}
else
{
c->done = 1;
return PyString_FromStringAndSize((char *)(c->data + c->i), c->data_len - c->i);
}
}
if(o == c->chunk_size)
{
return PyString_FromStringAndSize((char *)(c->data + c->i - c->chunk_size), c->chunk_size);
}
if(c->full_sum || c->i + c->chunk_size > c->data_len)
{
c->full_sum = 0;
c->sum = checksum(c->data + c->i, c->chunk_size, 0);
2010-03-03 21:52:57 +00:00
}
else
{
c->sum = roll_checksum(c->sum, c->remove, c->data[c->i + c->chunk_size - 1], c->chunk_size);
2010-03-03 21:52:57 +00:00
}
c->remove = c->data[c->i];
pysum = PyInt_FromLong(c->sum);
if(PySequence_Contains(c->chunks, pysum) == 1)
{
Py_DECREF(pysum);
c->full_sum = 1;
if(o > 0)
{
return PyString_FromStringAndSize((char *)(c->data + c->i - o), o);
}
else
{
c->i += c->chunk_size;
return PyString_FromStringAndSize((char *)(c->data + c->i - c->chunk_size), c->chunk_size);
}
}
Py_DECREF(pysum);
2010-03-03 21:52:57 +00:00
o++;
c->i++;
}
2010-02-28 19:22:45 +00:00
PyErr_SetNone(PyExc_StopIteration);
return NULL;
}
static PyTypeObject ChunkifyIterType = {
PyObject_HEAD_INIT(NULL)
0, /*ob_size*/
"_chunkifier._ChunkifyIter", /*tp_name*/
sizeof(ChunkifyIter), /*tp_basicsize*/
0, /*tp_itemsize*/
2010-03-03 21:52:57 +00:00
ChunkifyIter_dealloc, /*tp_dealloc*/
2010-02-28 19:22:45 +00:00
0, /*tp_print*/
0, /*tp_getattr*/
0, /*tp_setattr*/
0, /*tp_compare*/
0, /*tp_repr*/
0, /*tp_as_number*/
0, /*tp_as_sequence*/
0, /*tp_as_mapping*/
0, /*tp_hash */
0, /*tp_call*/
0, /*tp_str*/
0, /*tp_getattro*/
0, /*tp_setattro*/
0, /*tp_as_buffer*/
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_ITER,
/* tp_flags: Py_TPFLAGS_HAVE_ITER tells python to
use tp_iter and tp_iternext fields. */
"", /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
ChunkifyIter_iter, /* tp_iter: __iter__() method */
ChunkifyIter_iternext /* tp_iternext: next() method */
};
static PyObject *
chunkify(PyObject *self, PyObject *args)
{
2010-03-03 21:52:57 +00:00
PyObject *fd, *chunks;
long int chunk_size;
ChunkifyIter *c;
2010-02-28 19:22:45 +00:00
2010-03-03 21:52:57 +00:00
if (!PyArg_ParseTuple(args, "OiO", &fd, &chunk_size, &chunks))
{
return NULL;
}
if (!(c = PyObject_New(ChunkifyIter, &ChunkifyIterType)))
{
return NULL;
}
PyObject_Init((PyObject *)c, &ChunkifyIterType);
c->buf_size = chunk_size * 10;
c->data = malloc(c->buf_size);
c->data_len = 0;
c->i = 0;
c->full_sum = 1;
c->done = 0;
c->extra = NULL;
c->fd = fd;
c->chunk_size = chunk_size;
c->chunks = chunks;
Py_INCREF(fd);
Py_INCREF(chunks);
return (PyObject *)c;
}
2010-03-01 22:39:14 +00:00
static PyObject *
2010-03-03 21:52:57 +00:00
py_checksum(PyObject *self, PyObject *args)
{
2010-03-01 22:39:14 +00:00
PyObject *data;
2010-03-03 21:52:57 +00:00
unsigned long int sum = 0;
if(!PyArg_ParseTuple(args, "O|k", &data, &sum)) return NULL;
if(!PyString_Check(data))
{
PyErr_SetNone(PyExc_TypeError);
return NULL;
}
2010-03-03 21:52:57 +00:00
return PyInt_FromLong(checksum((unsigned char *)PyString_AsString(data),
PyString_Size(data), sum));
2010-02-28 19:22:45 +00:00
}
static PyObject *
2010-03-03 21:52:57 +00:00
py_roll_checksum(PyObject *self, PyObject *args)
{
2010-03-03 21:52:57 +00:00
unsigned long int sum = 0, len, a, r;
PyObject *add, *remove;
2010-03-03 21:52:57 +00:00
if (!PyArg_ParseTuple(args, "kOOk", &sum, &remove, &add, &len)) return NULL;
if(!PyString_Check(remove) || !PyString_Check(add) ||
PyString_Size(remove) != 1 || PyString_Size(add) != 1)
{
PyErr_SetNone(PyExc_TypeError);
return NULL;
}
a = *((const unsigned char *)PyString_AsString(add));
r = *((const unsigned char *)PyString_AsString(remove));
2010-03-03 21:52:57 +00:00
return PyInt_FromLong(roll_checksum(sum, r, a, len));
}
2010-03-03 21:52:57 +00:00
2010-02-28 19:22:45 +00:00
static PyMethodDef ChunkifierMethods[] = {
{"chunkify", chunkify, METH_VARARGS, ""},
2010-03-03 21:52:57 +00:00
{"checksum", py_checksum, METH_VARARGS, ""},
{"roll_checksum", py_roll_checksum, METH_VARARGS, ""},
2010-02-28 19:22:45 +00:00
{NULL, NULL, 0, NULL} /* Sentinel */
};
PyMODINIT_FUNC
2010-03-01 22:39:14 +00:00
init_speedups(void)
2010-02-28 19:22:45 +00:00
{
PyObject* m;
ChunkifyIterType.tp_new = PyType_GenericNew;
if (PyType_Ready(&ChunkifyIterType) < 0) return;
2010-03-01 22:39:14 +00:00
m = Py_InitModule("_speedups", ChunkifierMethods);
2010-02-28 19:22:45 +00:00
Py_INCREF(&ChunkifyIterType);
PyModule_AddObject(m, "_ChunkifyIter", (PyObject *)&ChunkifyIterType);
2010-03-09 21:27:37 +00:00
}