mirror of https://github.com/borgbackup/borg.git
CacheSynchronizer: redo as quasi FSM on top of unpack.h
This is a (relatively) simple state machine running in the data callbacks invoked by the msgpack unpacking stack machine (the same machine is used in msgpack-c and msgpack-python, changes are minor and cosmetic, e.g. removal of msgpack_unpack_object, removal of the C++ template thus porting to C and so on). Compared to the previous solution this has multiple advantages - msgpack-c dependency is removed - this approach is faster and requires fewer and smaller memory allocations Testability of the two solutions does not differ in my professional opinion(tm). Two other changes were rolled up; _hashindex.c can be compiled without Python.h again (handy for fuzzing and testing); a "small" bug in the cache sync was fixed which allocated too large archive indices, leading to excessive archive.chunks.d disk usage (that actually gave me an idea).
This commit is contained in:
Marian Beermann
parent
bf895950ac
commit
c786a5941e
|
|
@ -0,0 +1,10 @@
|
|||
- Install AFL and the requirements for LLVM mode (see docs)
|
||||
- Compile the fuzzing target, e.g.
|
||||
|
||||
AFL_HARDEN=1 afl-clang-fast main.c -o fuzz-target -O3
|
||||
|
||||
(other options, like using ASan or MSan are possible as well)
|
||||
- Add additional test cases to testcase_dir
|
||||
- Run afl, easiest (but inefficient) way;
|
||||
|
||||
afl-fuzz -i testcase_dir -o findings_dir ./fuzz-target
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
#include "../../src/borg/_hashindex.c"
|
||||
#include "../../src/borg/cache_sync/cache_sync.c"
|
||||
|
||||
#define BUFSZ 32768
|
||||
|
||||
int main() {
|
||||
char buf[BUFSZ];
|
||||
int len, ret;
|
||||
CacheSyncCtx *ctx;
|
||||
HashIndex *idx;
|
||||
|
||||
/* capacity, key size, value size */
|
||||
idx = hashindex_init(0, 32, 12);
|
||||
ctx = cache_sync_init(idx);
|
||||
|
||||
while (1) {
|
||||
len = read(0, buf, BUFSZ);
|
||||
if (!len) {
|
||||
break;
|
||||
}
|
||||
ret = cache_sync_feed(ctx, buf, len);
|
||||
if(!ret && cache_sync_error(ctx)) {
|
||||
fprintf(stderr, "error: %s\n", cache_sync_error(ctx));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
hashindex_free(idx);
|
||||
cache_sync_free(ctx);
|
||||
return 0;
|
||||
}
|
||||
Binary file not shown.
4
setup.py
4
setup.py
|
|
@ -91,6 +91,8 @@ try:
|
|||
'src/borg/crypto/low_level.c',
|
||||
'src/borg/algorithms/chunker.c', 'src/borg/algorithms/buzhash.c',
|
||||
'src/borg/hashindex.c', 'src/borg/_hashindex.c',
|
||||
'src/borg/cache_sync/cache_sync.c', 'src/borg/cache_sync/sysdep.h', 'src/borg/cache_sync/unpack.h',
|
||||
'src/borg/cache_sync/unpack_define.h', 'src/borg/cache_sync/unpack_template.h',
|
||||
'src/borg/item.c',
|
||||
'src/borg/algorithms/checksums.c',
|
||||
'src/borg/algorithms/crc32_dispatch.c', 'src/borg/algorithms/crc32_clmul.c', 'src/borg/algorithms/crc32_slice_by_8.c',
|
||||
|
|
@ -600,7 +602,7 @@ if not on_rtd:
|
|||
ext_modules += [
|
||||
Extension('borg.compress', [compress_source], libraries=['lz4'], include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
|
||||
Extension('borg.crypto.low_level', [crypto_ll_source], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
|
||||
Extension('borg.hashindex', [hashindex_source], libraries=['msgpackc']),
|
||||
Extension('borg.hashindex', [hashindex_source]),
|
||||
Extension('borg.item', [item_source]),
|
||||
Extension('borg.algorithms.chunker', [chunker_source]),
|
||||
Extension('borg.algorithms.checksums', [checksums_source]),
|
||||
|
|
|
|||
|
|
@ -1,159 +0,0 @@
|
|||
|
||||
#include <msgpack.h>
|
||||
|
||||
// 2**32 - 1025
|
||||
#define _MAX_VALUE ( (uint32_t) 4294966271 )
|
||||
|
||||
#define MIN(x, y) ((x) < (y) ? (x): (y))
|
||||
|
||||
typedef struct {
|
||||
HashIndex *chunks;
|
||||
|
||||
msgpack_unpacker unpacker;
|
||||
msgpack_unpacked unpacked;
|
||||
const char *error;
|
||||
} CacheSyncCtx;
|
||||
|
||||
static CacheSyncCtx *
|
||||
cache_sync_init(HashIndex *chunks)
|
||||
{
|
||||
CacheSyncCtx *ctx;
|
||||
if (!(ctx = malloc(sizeof(CacheSyncCtx)))) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
ctx->chunks = chunks;
|
||||
ctx->error = NULL;
|
||||
|
||||
if(!msgpack_unpacker_init(&ctx->unpacker, MSGPACK_UNPACKER_INIT_BUFFER_SIZE)) {
|
||||
free(ctx);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
msgpack_unpacked_init(&ctx->unpacked);
|
||||
|
||||
return ctx;
|
||||
}
|
||||
|
||||
static void
|
||||
cache_sync_free(CacheSyncCtx *ctx)
|
||||
{
|
||||
msgpack_unpacker_destroy(&ctx->unpacker);
|
||||
msgpack_unpacked_destroy(&ctx->unpacked);
|
||||
free(ctx);
|
||||
}
|
||||
|
||||
static const char *
|
||||
cache_sync_error(CacheSyncCtx *ctx)
|
||||
{
|
||||
return ctx->error;
|
||||
}
|
||||
|
||||
static int
|
||||
cache_process_chunks(CacheSyncCtx *ctx, msgpack_object_array *array)
|
||||
{
|
||||
uint32_t i;
|
||||
const char *key;
|
||||
uint32_t cache_values[3];
|
||||
uint32_t *cache_entry;
|
||||
uint64_t refcount;
|
||||
msgpack_object *current;
|
||||
for (i = 0; i < array->size; i++) {
|
||||
current = &array->ptr[i];
|
||||
|
||||
if (current->type != MSGPACK_OBJECT_ARRAY || current->via.array.size != 3
|
||||
|| current->via.array.ptr[0].type != MSGPACK_OBJECT_STR || current->via.array.ptr[0].via.str.size != 32
|
||||
|| current->via.array.ptr[1].type != MSGPACK_OBJECT_POSITIVE_INTEGER
|
||||
|| current->via.array.ptr[2].type != MSGPACK_OBJECT_POSITIVE_INTEGER) {
|
||||
ctx->error = "Malformed chunk list entry";
|
||||
return 0;
|
||||
}
|
||||
|
||||
key = current->via.array.ptr[0].via.str.ptr;
|
||||
cache_entry = (uint32_t*) hashindex_get(ctx->chunks, key);
|
||||
if (cache_entry) {
|
||||
refcount = _le32toh(cache_entry[0]);
|
||||
refcount += 1;
|
||||
cache_entry[0] = _htole32(MIN(refcount, _MAX_VALUE));
|
||||
} else {
|
||||
/* refcount, size, csize */
|
||||
cache_values[0] = 1;
|
||||
cache_values[1] = current->via.array.ptr[1].via.u64;
|
||||
cache_values[2] = current->via.array.ptr[2].via.u64;
|
||||
if (!hashindex_set(ctx->chunks, key, cache_values)) {
|
||||
ctx->error = "hashindex_set failed";
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* feed data to the cache synchronizer
|
||||
* 0 = abort, 1 = continue
|
||||
* abort is a regular condition, check cache_sync_error
|
||||
*/
|
||||
static int
|
||||
cache_sync_feed(CacheSyncCtx *ctx, void *data, uint32_t length)
|
||||
{
|
||||
msgpack_unpack_return unpack_status;
|
||||
|
||||
/* grow buffer if necessary */
|
||||
if (msgpack_unpacker_buffer_capacity(&ctx->unpacker) < length) {
|
||||
if (!msgpack_unpacker_reserve_buffer(&ctx->unpacker, length)) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
memcpy(msgpack_unpacker_buffer(&ctx->unpacker), data, length);
|
||||
msgpack_unpacker_buffer_consumed(&ctx->unpacker, length);
|
||||
|
||||
do {
|
||||
unpack_status = msgpack_unpacker_next(&ctx->unpacker, &ctx->unpacked);
|
||||
|
||||
switch (unpack_status) {
|
||||
case MSGPACK_UNPACK_SUCCESS:
|
||||
{
|
||||
uint32_t i;
|
||||
msgpack_object *item = &ctx->unpacked.data;
|
||||
msgpack_object_kv *current;
|
||||
|
||||
if (item->type != MSGPACK_OBJECT_MAP) {
|
||||
ctx->error = "Unexpected data type in item stream";
|
||||
return 0;
|
||||
}
|
||||
|
||||
for (i = 0; i < item->via.map.size; i++) {
|
||||
current = &item->via.map.ptr[i];
|
||||
|
||||
if (current->key.type != MSGPACK_OBJECT_STR) {
|
||||
ctx->error = "Invalid key data type in item";
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (current->key.via.str.size == 6
|
||||
&& !memcmp(current->key.via.str.ptr, "chunks", 6)) {
|
||||
|
||||
if (current->val.type != MSGPACK_OBJECT_ARRAY) {
|
||||
ctx->error = "Unexpected value type of item chunks";
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!cache_process_chunks(ctx, ¤t->val.via.array)) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
case MSGPACK_UNPACK_PARSE_ERROR:
|
||||
ctx->error = "Malformed msgpack";
|
||||
return 0;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
} while (unpack_status != MSGPACK_UNPACK_CONTINUE);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
#include <Python.h>
|
||||
|
||||
#include <assert.h>
|
||||
#include <errno.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <stdint.h>
|
||||
|
|
@ -56,8 +56,10 @@ typedef struct {
|
|||
int lower_limit;
|
||||
int upper_limit;
|
||||
int min_empty;
|
||||
#ifdef Py_PYTHON_H
|
||||
/* buckets may be backed by a Python buffer. If buckets_buffer.buf is NULL then this is not used. */
|
||||
Py_buffer buckets_buffer;
|
||||
#endif
|
||||
} HashIndex;
|
||||
|
||||
/* prime (or w/ big prime factors) hash table sizes
|
||||
|
|
@ -106,8 +108,11 @@ static int hash_sizes[] = {
|
|||
#define EPRINTF(msg, ...) fprintf(stderr, "hashindex: " msg "(%s)\n", ##__VA_ARGS__, strerror(errno))
|
||||
#define EPRINTF_PATH(path, msg, ...) fprintf(stderr, "hashindex: %s: " msg " (%s)\n", path, ##__VA_ARGS__, strerror(errno))
|
||||
|
||||
#ifdef Py_PYTHON_H
|
||||
static HashIndex *hashindex_read(PyObject *file_py);
|
||||
static void hashindex_write(HashIndex *index, PyObject *file_py);
|
||||
#endif
|
||||
|
||||
static HashIndex *hashindex_init(int capacity, int key_size, int value_size);
|
||||
static const void *hashindex_get(HashIndex *index, const void *key);
|
||||
static int hashindex_set(HashIndex *index, const void *key, const void *value);
|
||||
|
|
@ -120,9 +125,12 @@ static void hashindex_free(HashIndex *index);
|
|||
static void
|
||||
hashindex_free_buckets(HashIndex *index)
|
||||
{
|
||||
#ifdef Py_PYTHON_H
|
||||
if(index->buckets_buffer.buf) {
|
||||
PyBuffer_Release(&index->buckets_buffer);
|
||||
} else {
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
free(index->buckets);
|
||||
}
|
||||
}
|
||||
|
|
@ -263,6 +271,7 @@ count_empty(HashIndex *index)
|
|||
|
||||
/* Public API */
|
||||
|
||||
#ifdef Py_PYTHON_H
|
||||
static HashIndex *
|
||||
hashindex_read(PyObject *file_py)
|
||||
{
|
||||
|
|
@ -418,6 +427,7 @@ fail_decref_header:
|
|||
fail:
|
||||
return index;
|
||||
}
|
||||
#endif
|
||||
|
||||
static HashIndex *
|
||||
hashindex_init(int capacity, int key_size, int value_size)
|
||||
|
|
@ -444,7 +454,9 @@ hashindex_init(int capacity, int key_size, int value_size)
|
|||
index->lower_limit = get_lower_limit(index->num_buckets);
|
||||
index->upper_limit = get_upper_limit(index->num_buckets);
|
||||
index->min_empty = get_min_empty(index->num_buckets);
|
||||
#ifdef Py_PYTHON_H
|
||||
index->buckets_buffer.buf = NULL;
|
||||
#endif
|
||||
for(i = 0; i < capacity; i++) {
|
||||
BUCKET_MARK_EMPTY(index, i);
|
||||
}
|
||||
|
|
@ -458,7 +470,7 @@ hashindex_free(HashIndex *index)
|
|||
free(index);
|
||||
}
|
||||
|
||||
|
||||
#ifdef Py_PYTHON_H
|
||||
static void
|
||||
hashindex_write(HashIndex *index, PyObject *file_py)
|
||||
{
|
||||
|
|
@ -521,6 +533,7 @@ hashindex_write(HashIndex *index, PyObject *file_py)
|
|||
return;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
static const void *
|
||||
hashindex_get(HashIndex *index, const void *key)
|
||||
|
|
|
|||
|
|
@ -640,7 +640,7 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
|
|||
# Do not make this an else branch; the FileIntegrityError exception handler
|
||||
# above can remove *archive_id* from *cached_ids*.
|
||||
logger.info('Fetching and building archive index for %s ...', archive_name)
|
||||
archive_chunk_idx = ChunkIndex(master_index_capacity)
|
||||
archive_chunk_idx = ChunkIndex()
|
||||
fetch_and_build_idx(archive_id, repository, self.key, archive_chunk_idx)
|
||||
logger.info("Merging into master chunks index ...")
|
||||
if chunk_idx is None:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,108 @@
|
|||
|
||||
#include "unpack.h"
|
||||
|
||||
typedef struct {
|
||||
unpack_context ctx;
|
||||
|
||||
char *buf;
|
||||
size_t head;
|
||||
size_t tail;
|
||||
size_t size;
|
||||
} CacheSyncCtx;
|
||||
|
||||
static CacheSyncCtx *
|
||||
cache_sync_init(HashIndex *chunks)
|
||||
{
|
||||
CacheSyncCtx *ctx;
|
||||
if (!(ctx = (CacheSyncCtx*)malloc(sizeof(CacheSyncCtx)))) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
unpack_init(&ctx->ctx);
|
||||
|
||||
ctx->ctx.user.chunks = chunks;
|
||||
ctx->ctx.user.last_error = NULL;
|
||||
ctx->ctx.user.level = 0;
|
||||
ctx->ctx.user.inside_chunks = false;
|
||||
ctx->buf = NULL;
|
||||
ctx->head = 0;
|
||||
ctx->tail = 0;
|
||||
ctx->size = 0;
|
||||
|
||||
return ctx;
|
||||
}
|
||||
|
||||
static void
|
||||
cache_sync_free(CacheSyncCtx *ctx)
|
||||
{
|
||||
if(ctx->buf) {
|
||||
free(ctx->buf);
|
||||
}
|
||||
free(ctx);
|
||||
}
|
||||
|
||||
static const char *
|
||||
cache_sync_error(CacheSyncCtx *ctx)
|
||||
{
|
||||
return ctx->ctx.user.last_error;
|
||||
}
|
||||
|
||||
/**
|
||||
* feed data to the cache synchronizer
|
||||
* 0 = abort, 1 = continue
|
||||
* abort is a regular condition, check cache_sync_error
|
||||
*/
|
||||
static int
|
||||
cache_sync_feed(CacheSyncCtx *ctx, void *data, uint32_t length)
|
||||
{
|
||||
size_t new_size;
|
||||
int ret;
|
||||
char *new_buf;
|
||||
|
||||
if(ctx->tail + length > ctx->size) {
|
||||
if((ctx->tail - ctx->head) + length <= ctx->size) {
|
||||
/* | XXXXX| -> move data in buffer backwards -> |XXXXX | */
|
||||
memmove(ctx->buf, ctx->buf + ctx->head, ctx->tail - ctx->head);
|
||||
ctx->tail -= ctx->head;
|
||||
ctx->head = 0;
|
||||
} else {
|
||||
/* must expand buffer to fit all data */
|
||||
new_size = (ctx->tail - ctx->head) + length;
|
||||
new_buf = (char*) malloc(new_size);
|
||||
if(!new_buf) {
|
||||
ctx->ctx.user.last_error = "cache_sync_feed: unable to allocate buffer";
|
||||
return 0;
|
||||
}
|
||||
memcpy(new_buf, ctx->buf + ctx->head, ctx->tail - ctx->head);
|
||||
free(ctx->buf);
|
||||
ctx->buf = new_buf;
|
||||
ctx->tail -= ctx->head;
|
||||
ctx->head = 0;
|
||||
ctx->size = new_size;
|
||||
}
|
||||
}
|
||||
|
||||
memcpy(ctx->buf + ctx->tail, data, length);
|
||||
ctx->tail += length;
|
||||
|
||||
while(1) {
|
||||
if(ctx->head >= ctx->tail) {
|
||||
return 1; /* request more bytes */
|
||||
}
|
||||
|
||||
ret = unpack_execute(&ctx->ctx, ctx->buf, ctx->tail, &ctx->head);
|
||||
if(ret == 1) {
|
||||
unpack_init(&ctx->ctx);
|
||||
continue;
|
||||
} else if(ret == 0) {
|
||||
return 1;
|
||||
} else {
|
||||
if(!ctx->ctx.user.last_error) {
|
||||
ctx->ctx.user.last_error = "Unknown error";
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
/* unreachable */
|
||||
return 1;
|
||||
}
|
||||
|
|
@ -0,0 +1,194 @@
|
|||
/*
|
||||
* MessagePack system dependencies
|
||||
*
|
||||
* Copyright (C) 2008-2010 FURUHASHI Sadayuki
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#ifndef MSGPACK_SYSDEP_H__
|
||||
#define MSGPACK_SYSDEP_H__
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <stddef.h>
|
||||
#if defined(_MSC_VER) && _MSC_VER < 1600
|
||||
typedef __int8 int8_t;
|
||||
typedef unsigned __int8 uint8_t;
|
||||
typedef __int16 int16_t;
|
||||
typedef unsigned __int16 uint16_t;
|
||||
typedef __int32 int32_t;
|
||||
typedef unsigned __int32 uint32_t;
|
||||
typedef __int64 int64_t;
|
||||
typedef unsigned __int64 uint64_t;
|
||||
#elif defined(_MSC_VER) // && _MSC_VER >= 1600
|
||||
#include <stdint.h>
|
||||
#else
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#endif
|
||||
|
||||
#ifdef _WIN32
|
||||
#define _msgpack_atomic_counter_header <windows.h>
|
||||
typedef long _msgpack_atomic_counter_t;
|
||||
#define _msgpack_sync_decr_and_fetch(ptr) InterlockedDecrement(ptr)
|
||||
#define _msgpack_sync_incr_and_fetch(ptr) InterlockedIncrement(ptr)
|
||||
#elif defined(__GNUC__) && ((__GNUC__*10 + __GNUC_MINOR__) < 41)
|
||||
#define _msgpack_atomic_counter_header "gcc_atomic.h"
|
||||
#else
|
||||
typedef unsigned int _msgpack_atomic_counter_t;
|
||||
#define _msgpack_sync_decr_and_fetch(ptr) __sync_sub_and_fetch(ptr, 1)
|
||||
#define _msgpack_sync_incr_and_fetch(ptr) __sync_add_and_fetch(ptr, 1)
|
||||
#endif
|
||||
|
||||
#ifdef _WIN32
|
||||
|
||||
#ifdef __cplusplus
|
||||
/* numeric_limits<T>::min,max */
|
||||
#ifdef max
|
||||
#undef max
|
||||
#endif
|
||||
#ifdef min
|
||||
#undef min
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#else
|
||||
#include <arpa/inet.h> /* __BYTE_ORDER */
|
||||
#endif
|
||||
|
||||
#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
|
||||
#if __BYTE_ORDER == __LITTLE_ENDIAN
|
||||
#define __LITTLE_ENDIAN__
|
||||
#elif __BYTE_ORDER == __BIG_ENDIAN
|
||||
#define __BIG_ENDIAN__
|
||||
#elif _WIN32
|
||||
#define __LITTLE_ENDIAN__
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
||||
#ifdef __LITTLE_ENDIAN__
|
||||
|
||||
#ifdef _WIN32
|
||||
# if defined(ntohs)
|
||||
# define _msgpack_be16(x) ntohs(x)
|
||||
# elif defined(_byteswap_ushort) || (defined(_MSC_VER) && _MSC_VER >= 1400)
|
||||
# define _msgpack_be16(x) ((uint16_t)_byteswap_ushort((unsigned short)x))
|
||||
# else
|
||||
# define _msgpack_be16(x) ( \
|
||||
((((uint16_t)x) << 8) ) | \
|
||||
((((uint16_t)x) >> 8) ) )
|
||||
# endif
|
||||
#else
|
||||
# define _msgpack_be16(x) ntohs(x)
|
||||
#endif
|
||||
|
||||
#ifdef _WIN32
|
||||
# if defined(ntohl)
|
||||
# define _msgpack_be32(x) ntohl(x)
|
||||
# elif defined(_byteswap_ulong) || (defined(_MSC_VER) && _MSC_VER >= 1400)
|
||||
# define _msgpack_be32(x) ((uint32_t)_byteswap_ulong((unsigned long)x))
|
||||
# else
|
||||
# define _msgpack_be32(x) \
|
||||
( ((((uint32_t)x) << 24) ) | \
|
||||
((((uint32_t)x) << 8) & 0x00ff0000U ) | \
|
||||
((((uint32_t)x) >> 8) & 0x0000ff00U ) | \
|
||||
((((uint32_t)x) >> 24) ) )
|
||||
# endif
|
||||
#else
|
||||
# define _msgpack_be32(x) ntohl(x)
|
||||
#endif
|
||||
|
||||
#if defined(_byteswap_uint64) || (defined(_MSC_VER) && _MSC_VER >= 1400)
|
||||
# define _msgpack_be64(x) (_byteswap_uint64(x))
|
||||
#elif defined(bswap_64)
|
||||
# define _msgpack_be64(x) bswap_64(x)
|
||||
#elif defined(__DARWIN_OSSwapInt64)
|
||||
# define _msgpack_be64(x) __DARWIN_OSSwapInt64(x)
|
||||
#else
|
||||
#define _msgpack_be64(x) \
|
||||
( ((((uint64_t)x) << 56) ) | \
|
||||
((((uint64_t)x) << 40) & 0x00ff000000000000ULL ) | \
|
||||
((((uint64_t)x) << 24) & 0x0000ff0000000000ULL ) | \
|
||||
((((uint64_t)x) << 8) & 0x000000ff00000000ULL ) | \
|
||||
((((uint64_t)x) >> 8) & 0x00000000ff000000ULL ) | \
|
||||
((((uint64_t)x) >> 24) & 0x0000000000ff0000ULL ) | \
|
||||
((((uint64_t)x) >> 40) & 0x000000000000ff00ULL ) | \
|
||||
((((uint64_t)x) >> 56) ) )
|
||||
#endif
|
||||
|
||||
#define _msgpack_load16(cast, from) ((cast)( \
|
||||
(((uint16_t)((uint8_t*)(from))[0]) << 8) | \
|
||||
(((uint16_t)((uint8_t*)(from))[1]) ) ))
|
||||
|
||||
#define _msgpack_load32(cast, from) ((cast)( \
|
||||
(((uint32_t)((uint8_t*)(from))[0]) << 24) | \
|
||||
(((uint32_t)((uint8_t*)(from))[1]) << 16) | \
|
||||
(((uint32_t)((uint8_t*)(from))[2]) << 8) | \
|
||||
(((uint32_t)((uint8_t*)(from))[3]) ) ))
|
||||
|
||||
#define _msgpack_load64(cast, from) ((cast)( \
|
||||
(((uint64_t)((uint8_t*)(from))[0]) << 56) | \
|
||||
(((uint64_t)((uint8_t*)(from))[1]) << 48) | \
|
||||
(((uint64_t)((uint8_t*)(from))[2]) << 40) | \
|
||||
(((uint64_t)((uint8_t*)(from))[3]) << 32) | \
|
||||
(((uint64_t)((uint8_t*)(from))[4]) << 24) | \
|
||||
(((uint64_t)((uint8_t*)(from))[5]) << 16) | \
|
||||
(((uint64_t)((uint8_t*)(from))[6]) << 8) | \
|
||||
(((uint64_t)((uint8_t*)(from))[7]) ) ))
|
||||
|
||||
#else
|
||||
|
||||
#define _msgpack_be16(x) (x)
|
||||
#define _msgpack_be32(x) (x)
|
||||
#define _msgpack_be64(x) (x)
|
||||
|
||||
#define _msgpack_load16(cast, from) ((cast)( \
|
||||
(((uint16_t)((uint8_t*)from)[0]) << 8) | \
|
||||
(((uint16_t)((uint8_t*)from)[1]) ) ))
|
||||
|
||||
#define _msgpack_load32(cast, from) ((cast)( \
|
||||
(((uint32_t)((uint8_t*)from)[0]) << 24) | \
|
||||
(((uint32_t)((uint8_t*)from)[1]) << 16) | \
|
||||
(((uint32_t)((uint8_t*)from)[2]) << 8) | \
|
||||
(((uint32_t)((uint8_t*)from)[3]) ) ))
|
||||
|
||||
#define _msgpack_load64(cast, from) ((cast)( \
|
||||
(((uint64_t)((uint8_t*)from)[0]) << 56) | \
|
||||
(((uint64_t)((uint8_t*)from)[1]) << 48) | \
|
||||
(((uint64_t)((uint8_t*)from)[2]) << 40) | \
|
||||
(((uint64_t)((uint8_t*)from)[3]) << 32) | \
|
||||
(((uint64_t)((uint8_t*)from)[4]) << 24) | \
|
||||
(((uint64_t)((uint8_t*)from)[5]) << 16) | \
|
||||
(((uint64_t)((uint8_t*)from)[6]) << 8) | \
|
||||
(((uint64_t)((uint8_t*)from)[7]) ) ))
|
||||
#endif
|
||||
|
||||
|
||||
#define _msgpack_store16(to, num) \
|
||||
do { uint16_t val = _msgpack_be16(num); memcpy(to, &val, 2); } while(0)
|
||||
#define _msgpack_store32(to, num) \
|
||||
do { uint32_t val = _msgpack_be32(num); memcpy(to, &val, 4); } while(0)
|
||||
#define _msgpack_store64(to, num) \
|
||||
do { uint64_t val = _msgpack_be64(num); memcpy(to, &val, 8); } while(0)
|
||||
|
||||
/*
|
||||
#define _msgpack_load16(cast, from) \
|
||||
({ cast val; memcpy(&val, (char*)from, 2); _msgpack_be16(val); })
|
||||
#define _msgpack_load32(cast, from) \
|
||||
({ cast val; memcpy(&val, (char*)from, 4); _msgpack_be32(val); })
|
||||
#define _msgpack_load64(cast, from) \
|
||||
({ cast val; memcpy(&val, (char*)from, 8); _msgpack_be64(val); })
|
||||
*/
|
||||
|
||||
|
||||
#endif /* msgpack/sysdep.h */
|
||||
|
|
@ -0,0 +1,374 @@
|
|||
/*
|
||||
* Borg cache synchronizer,
|
||||
* based on a MessagePack for Python unpacking routine
|
||||
*
|
||||
* Copyright (C) 2009 Naoki INADA
|
||||
* Copyright (c) 2017 Marian Beermann
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/*
|
||||
* This limits the depth of the structures we can unpack, i.e. how many containers
|
||||
* are nestable.
|
||||
*/
|
||||
#define MSGPACK_EMBED_STACK_SIZE (16)
|
||||
#include "unpack_define.h"
|
||||
|
||||
// 2**32 - 1025
|
||||
#define _MAX_VALUE ( (uint32_t) 4294966271 )
|
||||
|
||||
#define MIN(x, y) ((x) < (y) ? (x): (y))
|
||||
|
||||
#ifdef DEBUG
|
||||
#define set_last_error(msg) \
|
||||
fprintf(stderr, "cache_sync parse error: %s\n", (msg)); \
|
||||
u->last_error = (msg);
|
||||
#else
|
||||
#define set_last_error(msg) \
|
||||
u->last_error = (msg);
|
||||
#endif
|
||||
|
||||
typedef struct unpack_user {
|
||||
/* Item.chunks is at the top level; we don't care about anything else,
|
||||
* only need to track the current level to navigate arbitrary and unknown structure.
|
||||
* To discern keys from everything else on the top level we use expect_map_item_end.
|
||||
*/
|
||||
int level;
|
||||
|
||||
const char *last_error;
|
||||
|
||||
HashIndex *chunks;
|
||||
|
||||
/*
|
||||
* We don't care about most stuff. This flag tells us whether we're at the chunks structure,
|
||||
* meaning:
|
||||
* {'foo': 'bar', 'chunks': [...], 'stuff': ... }
|
||||
* ^-HERE-^
|
||||
*/
|
||||
int inside_chunks;
|
||||
enum {
|
||||
/* the next thing is a map key at the Item root level,
|
||||
* and it might be the "chunks" key we're looking for */
|
||||
expect_chunks_map_key,
|
||||
|
||||
/* blocking state to expect_chunks_map_key
|
||||
* { 'stuff': <complex and arbitrary structure>, 'chunks': [
|
||||
* ecmk -> emie -> -> -> -> ecmk ecb eeboce
|
||||
* (nested containers are tracked via level)
|
||||
* ecmk=expect_chunks_map_key, emie=expect_map_item_end, ecb=expect_chunks_begin,
|
||||
* eeboce=expect_entry_begin_or_chunks_end
|
||||
*/
|
||||
expect_map_item_end,
|
||||
|
||||
/* next thing must be the chunks array (array) */
|
||||
expect_chunks_begin,
|
||||
|
||||
/* next thing must either be another CLE (array) or end of Item.chunks (array_end) */
|
||||
expect_entry_begin_or_chunks_end,
|
||||
|
||||
/*
|
||||
* processing ChunkListEntry tuple:
|
||||
* expect_key, expect_size, expect_csize, expect_entry_end
|
||||
*/
|
||||
/* next thing must be the key (raw, l=32) */
|
||||
expect_key,
|
||||
/* next thing must be the size (int) */
|
||||
expect_size,
|
||||
/* next thing must be the csize (int) */
|
||||
expect_csize,
|
||||
/* next thing must be the end of the CLE (array_end) */
|
||||
expect_entry_end,
|
||||
} expect;
|
||||
|
||||
struct {
|
||||
char key[32];
|
||||
uint32_t csize;
|
||||
uint32_t size;
|
||||
} current;
|
||||
} unpack_user;
|
||||
|
||||
struct unpack_context;
|
||||
typedef struct unpack_context unpack_context;
|
||||
typedef int (*execute_fn)(unpack_context *ctx, const char* data, size_t len, size_t* off);
|
||||
|
||||
#define unexpected(what) \
|
||||
if(u->inside_chunks || u->expect == expect_chunks_map_key) { \
|
||||
set_last_error("Unexpected object: " what); \
|
||||
return -1; \
|
||||
}
|
||||
|
||||
static inline int unpack_callback_int64(unpack_user* u, int64_t d)
|
||||
{
|
||||
switch(u->expect) {
|
||||
case expect_size:
|
||||
u->current.size = d;
|
||||
u->expect = expect_csize;
|
||||
break;
|
||||
case expect_csize:
|
||||
u->current.csize = d;
|
||||
u->expect = expect_entry_end;
|
||||
break;
|
||||
default:
|
||||
unexpected("integer");
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline int unpack_callback_uint16(unpack_user* u, uint16_t d)
|
||||
{
|
||||
return unpack_callback_int64(u, d);
|
||||
}
|
||||
|
||||
static inline int unpack_callback_uint8(unpack_user* u, uint8_t d)
|
||||
{
|
||||
return unpack_callback_int64(u, d);
|
||||
}
|
||||
|
||||
|
||||
static inline int unpack_callback_uint32(unpack_user* u, uint32_t d)
|
||||
{
|
||||
return unpack_callback_int64(u, d);
|
||||
}
|
||||
|
||||
static inline int unpack_callback_uint64(unpack_user* u, uint64_t d)
|
||||
{
|
||||
return unpack_callback_int64(u, d);
|
||||
}
|
||||
|
||||
static inline int unpack_callback_int32(unpack_user* u, int32_t d)
|
||||
{
|
||||
return unpack_callback_int64(u, d);
|
||||
}
|
||||
|
||||
static inline int unpack_callback_int16(unpack_user* u, int16_t d)
|
||||
{
|
||||
return unpack_callback_int64(u, d);
|
||||
}
|
||||
|
||||
static inline int unpack_callback_int8(unpack_user* u, int8_t d)
|
||||
{
|
||||
return unpack_callback_int64(u, d);
|
||||
}
|
||||
|
||||
/* Ain't got anything to do with those floats */
|
||||
static inline int unpack_callback_double(unpack_user* u, double d)
|
||||
{
|
||||
(void)d;
|
||||
unexpected("double");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline int unpack_callback_float(unpack_user* u, float d)
|
||||
{
|
||||
(void)d;
|
||||
unexpected("float");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* nil/true/false — I/don't/care */
|
||||
static inline int unpack_callback_nil(unpack_user* u)
|
||||
{
|
||||
unexpected("nil");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline int unpack_callback_true(unpack_user* u)
|
||||
{
|
||||
unexpected("true");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline int unpack_callback_false(unpack_user* u)
|
||||
{
|
||||
unexpected("false");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline int unpack_callback_array(unpack_user* u, unsigned int n)
|
||||
{
|
||||
switch(u->expect) {
|
||||
case expect_chunks_begin:
|
||||
/* b'chunks': [
|
||||
* ^ */
|
||||
u->expect = expect_entry_begin_or_chunks_end;
|
||||
break;
|
||||
case expect_entry_begin_or_chunks_end:
|
||||
/* b'chunks': [ (
|
||||
* ^ */
|
||||
if(n != 3) {
|
||||
set_last_error("Invalid chunk list entry length");
|
||||
return -1;
|
||||
}
|
||||
u->expect = expect_key;
|
||||
break;
|
||||
default:
|
||||
if(u->inside_chunks) {
|
||||
set_last_error("Unexpected array start");
|
||||
return -1;
|
||||
} else {
|
||||
u->level++;
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline int unpack_callback_array_item(unpack_user* u, unsigned int current)
|
||||
{
|
||||
(void)u; (void)current;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline int unpack_callback_array_end(unpack_user* u)
|
||||
{
|
||||
uint32_t *cache_entry;
|
||||
uint32_t cache_values[3];
|
||||
uint64_t refcount;
|
||||
|
||||
switch(u->expect) {
|
||||
case expect_entry_end:
|
||||
/* b'chunks': [ ( b'1234...', 123, 345 )
|
||||
* ^ */
|
||||
cache_entry = (uint32_t*) hashindex_get(u->chunks, u->current.key);
|
||||
if (cache_entry) {
|
||||
refcount = _le32toh(cache_entry[0]);
|
||||
refcount += 1;
|
||||
cache_entry[0] = _htole32(MIN(refcount, _MAX_VALUE));
|
||||
} else {
|
||||
/* refcount, size, csize */
|
||||
cache_values[0] = _htole32(1);
|
||||
cache_values[1] = _htole32(u->current.size);
|
||||
cache_values[2] = _htole32(u->current.csize);
|
||||
if (!hashindex_set(u->chunks, u->current.key, cache_values)) {
|
||||
set_last_error("hashindex_set failed");
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
u->expect = expect_entry_begin_or_chunks_end;
|
||||
break;
|
||||
case expect_entry_begin_or_chunks_end:
|
||||
/* b'chunks': [ ]
|
||||
* ^ */
|
||||
/* end of Item.chunks */
|
||||
u->inside_chunks = 0;
|
||||
u->expect = expect_map_item_end;
|
||||
break;
|
||||
default:
|
||||
if(u->inside_chunks) {
|
||||
set_last_error("Invalid state transition (unexpected array end)");
|
||||
return -1;
|
||||
} else {
|
||||
u->level--;
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline int unpack_callback_map(unpack_user* u, unsigned int n)
|
||||
{
|
||||
(void)n;
|
||||
|
||||
|
||||
if(u->level == 0) {
|
||||
/* This begins a new Item */
|
||||
u->expect = expect_chunks_map_key;
|
||||
}
|
||||
|
||||
if(u->inside_chunks) {
|
||||
unexpected("map");
|
||||
}
|
||||
|
||||
u->level++;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline int unpack_callback_map_item(unpack_user* u, unsigned int current)
|
||||
{
|
||||
(void)u; (void)current;
|
||||
if(u->level == 1) {
|
||||
switch(u->expect) {
|
||||
case expect_map_item_end:
|
||||
u->expect = expect_chunks_map_key;
|
||||
break;
|
||||
default:
|
||||
set_last_error("Unexpected map item");
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline int unpack_callback_map_end(unpack_user* u)
|
||||
{
|
||||
u->level--;
|
||||
if(u->inside_chunks) {
|
||||
set_last_error("Unexpected map end");
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline int unpack_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l)
|
||||
{
|
||||
/* raw = what Borg uses for binary stuff and strings as well */
|
||||
/* Note: p points to an internal buffer which contains l bytes. */
|
||||
(void)b;
|
||||
|
||||
switch(u->expect) {
|
||||
case expect_key:
|
||||
if(l != 32) {
|
||||
set_last_error("Incorrect key length");
|
||||
return -1;
|
||||
}
|
||||
memcpy(u->current.key, p, 32);
|
||||
u->expect = expect_size;
|
||||
break;
|
||||
case expect_chunks_map_key:
|
||||
if(l == 6 && !memcmp("chunks", p, 6)) {
|
||||
u->expect = expect_chunks_begin;
|
||||
u->inside_chunks = 1;
|
||||
} else {
|
||||
u->expect = expect_map_item_end;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
if(u->inside_chunks) {
|
||||
set_last_error("Unexpected bytes in chunks structure");
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline int unpack_callback_bin(unpack_user* u, const char* b, const char* p, unsigned int l)
|
||||
{
|
||||
(void)u; (void)b; (void)p; (void)l;
|
||||
unexpected("bin");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline int unpack_callback_ext(unpack_user* u, const char* base, const char* pos,
|
||||
unsigned int length)
|
||||
{
|
||||
(void)u; (void)base; (void)pos; (void)length;
|
||||
unexpected("ext");
|
||||
return 0;
|
||||
}
|
||||
|
||||
#include "unpack_template.h"
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
/*
|
||||
* MessagePack unpacking routine template
|
||||
*
|
||||
* Copyright (C) 2008-2010 FURUHASHI Sadayuki
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#ifndef MSGPACK_UNPACK_DEFINE_H__
|
||||
#define MSGPACK_UNPACK_DEFINE_H__
|
||||
|
||||
#include "sysdep.h"
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <assert.h>
|
||||
#include <stdio.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
#ifndef MSGPACK_EMBED_STACK_SIZE
|
||||
#define MSGPACK_EMBED_STACK_SIZE 32
|
||||
#endif
|
||||
|
||||
|
||||
// CS is first byte & 0x1f
|
||||
typedef enum {
|
||||
CS_HEADER = 0x00, // nil
|
||||
|
||||
//CS_ = 0x01,
|
||||
//CS_ = 0x02, // false
|
||||
//CS_ = 0x03, // true
|
||||
|
||||
CS_BIN_8 = 0x04,
|
||||
CS_BIN_16 = 0x05,
|
||||
CS_BIN_32 = 0x06,
|
||||
|
||||
CS_EXT_8 = 0x07,
|
||||
CS_EXT_16 = 0x08,
|
||||
CS_EXT_32 = 0x09,
|
||||
|
||||
CS_FLOAT = 0x0a,
|
||||
CS_DOUBLE = 0x0b,
|
||||
CS_UINT_8 = 0x0c,
|
||||
CS_UINT_16 = 0x0d,
|
||||
CS_UINT_32 = 0x0e,
|
||||
CS_UINT_64 = 0x0f,
|
||||
CS_INT_8 = 0x10,
|
||||
CS_INT_16 = 0x11,
|
||||
CS_INT_32 = 0x12,
|
||||
CS_INT_64 = 0x13,
|
||||
|
||||
//CS_FIXEXT1 = 0x14,
|
||||
//CS_FIXEXT2 = 0x15,
|
||||
//CS_FIXEXT4 = 0x16,
|
||||
//CS_FIXEXT8 = 0x17,
|
||||
//CS_FIXEXT16 = 0x18,
|
||||
|
||||
CS_RAW_8 = 0x19,
|
||||
CS_RAW_16 = 0x1a,
|
||||
CS_RAW_32 = 0x1b,
|
||||
CS_ARRAY_16 = 0x1c,
|
||||
CS_ARRAY_32 = 0x1d,
|
||||
CS_MAP_16 = 0x1e,
|
||||
CS_MAP_32 = 0x1f,
|
||||
|
||||
ACS_RAW_VALUE,
|
||||
ACS_BIN_VALUE,
|
||||
ACS_EXT_VALUE,
|
||||
} msgpack_unpack_state;
|
||||
|
||||
|
||||
typedef enum {
|
||||
CT_ARRAY_ITEM,
|
||||
CT_MAP_KEY,
|
||||
CT_MAP_VALUE,
|
||||
} msgpack_container_type;
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* msgpack/unpack_define.h */
|
||||
|
|
@ -0,0 +1,359 @@
|
|||
/*
|
||||
* MessagePack unpacking routine template
|
||||
*
|
||||
* Copyright (C) 2008-2010 FURUHASHI Sadayuki
|
||||
* Copyright (c) 2017 Marian Beermann
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef USE_CASE_RANGE
|
||||
#if !defined(_MSC_VER)
|
||||
#define USE_CASE_RANGE
|
||||
#endif
|
||||
#endif
|
||||
|
||||
typedef struct unpack_stack {
|
||||
size_t size;
|
||||
size_t count;
|
||||
unsigned int ct;
|
||||
} unpack_stack;
|
||||
|
||||
struct unpack_context {
|
||||
unpack_user user;
|
||||
unsigned int cs;
|
||||
unsigned int trail;
|
||||
unsigned int top;
|
||||
unpack_stack stack[MSGPACK_EMBED_STACK_SIZE];
|
||||
};
|
||||
|
||||
static inline void unpack_init(unpack_context* ctx)
|
||||
{
|
||||
ctx->cs = CS_HEADER;
|
||||
ctx->trail = 0;
|
||||
ctx->top = 0;
|
||||
}
|
||||
|
||||
#define construct 1
|
||||
|
||||
static inline int unpack_execute(unpack_context* ctx, const char* data, size_t len, size_t* off)
|
||||
{
|
||||
assert(len >= *off);
|
||||
|
||||
const unsigned char* p = (unsigned char*)data + *off;
|
||||
const unsigned char* const pe = (unsigned char*)data + len;
|
||||
const void* n = NULL;
|
||||
|
||||
unsigned int trail = ctx->trail;
|
||||
unsigned int cs = ctx->cs;
|
||||
unsigned int top = ctx->top;
|
||||
unpack_stack* stack = ctx->stack;
|
||||
unpack_user* user = &ctx->user;
|
||||
|
||||
unpack_stack* c = NULL;
|
||||
|
||||
int ret;
|
||||
|
||||
#define construct_cb(name) \
|
||||
construct && unpack_callback ## name
|
||||
|
||||
#define push_simple_value(func) \
|
||||
if(construct_cb(func)(user) < 0) { goto _failed; } \
|
||||
goto _push
|
||||
#define push_fixed_value(func, arg) \
|
||||
if(construct_cb(func)(user, arg) < 0) { goto _failed; } \
|
||||
goto _push
|
||||
#define push_variable_value(func, base, pos, len) \
|
||||
if(construct_cb(func)(user, \
|
||||
(const char*)base, (const char*)pos, len) < 0) { goto _failed; } \
|
||||
goto _push
|
||||
|
||||
#define again_fixed_trail(_cs, trail_len) \
|
||||
trail = trail_len; \
|
||||
cs = _cs; \
|
||||
goto _fixed_trail_again
|
||||
#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \
|
||||
trail = trail_len; \
|
||||
if(trail == 0) { goto ifzero; } \
|
||||
cs = _cs; \
|
||||
goto _fixed_trail_again
|
||||
|
||||
#define start_container(func, count_, ct_) \
|
||||
if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \
|
||||
if(construct_cb(func)(user, count_) < 0) { goto _failed; } \
|
||||
if((count_) == 0) { \
|
||||
if (construct_cb(func##_end)(user) < 0) { goto _failed; } \
|
||||
goto _push; } \
|
||||
stack[top].ct = ct_; \
|
||||
stack[top].size = count_; \
|
||||
stack[top].count = 0; \
|
||||
++top; \
|
||||
goto _header_again
|
||||
|
||||
#define NEXT_CS(p) ((unsigned int)*p & 0x1f)
|
||||
|
||||
#ifdef USE_CASE_RANGE
|
||||
#define SWITCH_RANGE_BEGIN switch(*p) {
|
||||
#define SWITCH_RANGE(FROM, TO) case FROM ... TO:
|
||||
#define SWITCH_RANGE_DEFAULT default:
|
||||
#define SWITCH_RANGE_END }
|
||||
#else
|
||||
#define SWITCH_RANGE_BEGIN { if(0) {
|
||||
#define SWITCH_RANGE(FROM, TO) } else if(FROM <= *p && *p <= TO) {
|
||||
#define SWITCH_RANGE_DEFAULT } else {
|
||||
#define SWITCH_RANGE_END } }
|
||||
#endif
|
||||
|
||||
if(p == pe) { goto _out; }
|
||||
do {
|
||||
switch(cs) {
|
||||
case CS_HEADER:
|
||||
SWITCH_RANGE_BEGIN
|
||||
SWITCH_RANGE(0x00, 0x7f) // Positive Fixnum
|
||||
push_fixed_value(_uint8, *(uint8_t*)p);
|
||||
SWITCH_RANGE(0xe0, 0xff) // Negative Fixnum
|
||||
push_fixed_value(_int8, *(int8_t*)p);
|
||||
SWITCH_RANGE(0xc0, 0xdf) // Variable
|
||||
switch(*p) {
|
||||
case 0xc0: // nil
|
||||
push_simple_value(_nil);
|
||||
//case 0xc1: // never used
|
||||
case 0xc2: // false
|
||||
push_simple_value(_false);
|
||||
case 0xc3: // true
|
||||
push_simple_value(_true);
|
||||
case 0xc4: // bin 8
|
||||
again_fixed_trail(NEXT_CS(p), 1);
|
||||
case 0xc5: // bin 16
|
||||
again_fixed_trail(NEXT_CS(p), 2);
|
||||
case 0xc6: // bin 32
|
||||
again_fixed_trail(NEXT_CS(p), 4);
|
||||
case 0xc7: // ext 8
|
||||
again_fixed_trail(NEXT_CS(p), 1);
|
||||
case 0xc8: // ext 16
|
||||
again_fixed_trail(NEXT_CS(p), 2);
|
||||
case 0xc9: // ext 32
|
||||
again_fixed_trail(NEXT_CS(p), 4);
|
||||
case 0xca: // float
|
||||
case 0xcb: // double
|
||||
case 0xcc: // unsigned int 8
|
||||
case 0xcd: // unsigned int 16
|
||||
case 0xce: // unsigned int 32
|
||||
case 0xcf: // unsigned int 64
|
||||
case 0xd0: // signed int 8
|
||||
case 0xd1: // signed int 16
|
||||
case 0xd2: // signed int 32
|
||||
case 0xd3: // signed int 64
|
||||
again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03));
|
||||
case 0xd4: // fixext 1
|
||||
case 0xd5: // fixext 2
|
||||
case 0xd6: // fixext 4
|
||||
case 0xd7: // fixext 8
|
||||
again_fixed_trail_if_zero(ACS_EXT_VALUE,
|
||||
(1 << (((unsigned int)*p) & 0x03))+1,
|
||||
_ext_zero);
|
||||
case 0xd8: // fixext 16
|
||||
again_fixed_trail_if_zero(ACS_EXT_VALUE, 16+1, _ext_zero);
|
||||
case 0xd9: // str 8
|
||||
again_fixed_trail(NEXT_CS(p), 1);
|
||||
case 0xda: // raw 16
|
||||
case 0xdb: // raw 32
|
||||
case 0xdc: // array 16
|
||||
case 0xdd: // array 32
|
||||
case 0xde: // map 16
|
||||
case 0xdf: // map 32
|
||||
again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01));
|
||||
default:
|
||||
goto _failed;
|
||||
}
|
||||
SWITCH_RANGE(0xa0, 0xbf) // FixRaw
|
||||
again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero);
|
||||
SWITCH_RANGE(0x90, 0x9f) // FixArray
|
||||
start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM);
|
||||
SWITCH_RANGE(0x80, 0x8f) // FixMap
|
||||
start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY);
|
||||
|
||||
SWITCH_RANGE_DEFAULT
|
||||
goto _failed;
|
||||
SWITCH_RANGE_END
|
||||
// end CS_HEADER
|
||||
|
||||
|
||||
_fixed_trail_again:
|
||||
++p;
|
||||
|
||||
default:
|
||||
if((size_t)(pe - p) < trail) { goto _out; }
|
||||
n = p; p += trail - 1;
|
||||
switch(cs) {
|
||||
case CS_EXT_8:
|
||||
again_fixed_trail_if_zero(ACS_EXT_VALUE, *(uint8_t*)n+1, _ext_zero);
|
||||
case CS_EXT_16:
|
||||
again_fixed_trail_if_zero(ACS_EXT_VALUE,
|
||||
_msgpack_load16(uint16_t,n)+1,
|
||||
_ext_zero);
|
||||
case CS_EXT_32:
|
||||
again_fixed_trail_if_zero(ACS_EXT_VALUE,
|
||||
_msgpack_load32(uint32_t,n)+1,
|
||||
_ext_zero);
|
||||
case CS_FLOAT: {
|
||||
union { uint32_t i; float f; } mem;
|
||||
mem.i = _msgpack_load32(uint32_t,n);
|
||||
push_fixed_value(_float, mem.f); }
|
||||
case CS_DOUBLE: {
|
||||
union { uint64_t i; double f; } mem;
|
||||
mem.i = _msgpack_load64(uint64_t,n);
|
||||
#if defined(__arm__) && !(__ARM_EABI__) // arm-oabi
|
||||
// https://github.com/msgpack/msgpack-perl/pull/1
|
||||
mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL);
|
||||
#endif
|
||||
push_fixed_value(_double, mem.f); }
|
||||
case CS_UINT_8:
|
||||
push_fixed_value(_uint8, *(uint8_t*)n);
|
||||
case CS_UINT_16:
|
||||
push_fixed_value(_uint16, _msgpack_load16(uint16_t,n));
|
||||
case CS_UINT_32:
|
||||
push_fixed_value(_uint32, _msgpack_load32(uint32_t,n));
|
||||
case CS_UINT_64:
|
||||
push_fixed_value(_uint64, _msgpack_load64(uint64_t,n));
|
||||
|
||||
case CS_INT_8:
|
||||
push_fixed_value(_int8, *(int8_t*)n);
|
||||
case CS_INT_16:
|
||||
push_fixed_value(_int16, _msgpack_load16(int16_t,n));
|
||||
case CS_INT_32:
|
||||
push_fixed_value(_int32, _msgpack_load32(int32_t,n));
|
||||
case CS_INT_64:
|
||||
push_fixed_value(_int64, _msgpack_load64(int64_t,n));
|
||||
|
||||
case CS_BIN_8:
|
||||
again_fixed_trail_if_zero(ACS_BIN_VALUE, *(uint8_t*)n, _bin_zero);
|
||||
case CS_BIN_16:
|
||||
again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load16(uint16_t,n), _bin_zero);
|
||||
case CS_BIN_32:
|
||||
again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load32(uint32_t,n), _bin_zero);
|
||||
case ACS_BIN_VALUE:
|
||||
_bin_zero:
|
||||
push_variable_value(_bin, data, n, trail);
|
||||
|
||||
case CS_RAW_8:
|
||||
again_fixed_trail_if_zero(ACS_RAW_VALUE, *(uint8_t*)n, _raw_zero);
|
||||
case CS_RAW_16:
|
||||
again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero);
|
||||
case CS_RAW_32:
|
||||
again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero);
|
||||
case ACS_RAW_VALUE:
|
||||
_raw_zero:
|
||||
push_variable_value(_raw, data, n, trail);
|
||||
|
||||
case ACS_EXT_VALUE:
|
||||
_ext_zero:
|
||||
push_variable_value(_ext, data, n, trail);
|
||||
|
||||
case CS_ARRAY_16:
|
||||
start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM);
|
||||
case CS_ARRAY_32:
|
||||
/* FIXME security guard */
|
||||
start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM);
|
||||
|
||||
case CS_MAP_16:
|
||||
start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY);
|
||||
case CS_MAP_32:
|
||||
/* FIXME security guard */
|
||||
start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY);
|
||||
|
||||
default:
|
||||
goto _failed;
|
||||
}
|
||||
}
|
||||
|
||||
_push:
|
||||
if(top == 0) { goto _finish; }
|
||||
c = &stack[top-1];
|
||||
switch(c->ct) {
|
||||
case CT_ARRAY_ITEM:
|
||||
if(construct_cb(_array_item)(user, c->count) < 0) { goto _failed; }
|
||||
if(++c->count == c->size) {
|
||||
if (construct_cb(_array_end)(user) < 0) { goto _failed; }
|
||||
--top;
|
||||
/*printf("stack pop %d\n", top);*/
|
||||
goto _push;
|
||||
}
|
||||
goto _header_again;
|
||||
case CT_MAP_KEY:
|
||||
c->ct = CT_MAP_VALUE;
|
||||
goto _header_again;
|
||||
case CT_MAP_VALUE:
|
||||
if(construct_cb(_map_item)(user, c->count) < 0) { goto _failed; }
|
||||
if(++c->count == c->size) {
|
||||
if (construct_cb(_map_end)(user) < 0) { goto _failed; }
|
||||
--top;
|
||||
/*printf("stack pop %d\n", top);*/
|
||||
goto _push;
|
||||
}
|
||||
c->ct = CT_MAP_KEY;
|
||||
goto _header_again;
|
||||
|
||||
default:
|
||||
goto _failed;
|
||||
}
|
||||
|
||||
_header_again:
|
||||
cs = CS_HEADER;
|
||||
++p;
|
||||
} while(p != pe);
|
||||
goto _out;
|
||||
|
||||
|
||||
_finish:
|
||||
if (!construct)
|
||||
unpack_callback_nil(user);
|
||||
++p;
|
||||
ret = 1;
|
||||
/* printf("-- finish --\n"); */
|
||||
goto _end;
|
||||
|
||||
_failed:
|
||||
/* printf("** FAILED **\n"); */
|
||||
ret = -1;
|
||||
goto _end;
|
||||
|
||||
_out:
|
||||
ret = 0;
|
||||
goto _end;
|
||||
|
||||
_end:
|
||||
ctx->cs = cs;
|
||||
ctx->trail = trail;
|
||||
ctx->top = top;
|
||||
*off = p - (const unsigned char*)data;
|
||||
|
||||
return ret;
|
||||
#undef construct_cb
|
||||
}
|
||||
|
||||
#undef SWITCH_RANGE_BEGIN
|
||||
#undef SWITCH_RANGE
|
||||
#undef SWITCH_RANGE_DEFAULT
|
||||
#undef SWITCH_RANGE_END
|
||||
#undef push_simple_value
|
||||
#undef push_fixed_value
|
||||
#undef push_variable_value
|
||||
#undef again_fixed_trail
|
||||
#undef again_fixed_trail_if_zero
|
||||
#undef start_container
|
||||
#undef construct
|
||||
|
||||
#undef NEXT_CS
|
||||
|
||||
/* vim: set ts=4 sw=4 sts=4 expandtab */
|
||||
|
|
@ -31,7 +31,7 @@ cdef extern from "_hashindex.c":
|
|||
double HASH_MAX_LOAD
|
||||
|
||||
|
||||
cdef extern from "_cache.c":
|
||||
cdef extern from "cache_sync/cache_sync.c":
|
||||
ctypedef struct CacheSyncCtx:
|
||||
pass
|
||||
|
||||
|
|
@ -403,5 +403,5 @@ cdef class CacheSynchronizer:
|
|||
def feed(self, chunk):
|
||||
if not cache_sync_feed(self.sync, <char *>chunk, len(chunk)):
|
||||
error = cache_sync_error(self.sync)
|
||||
if error is not None:
|
||||
raise Exception('cache_sync_feed failed: ' + error.decode('ascii'))
|
||||
if error != NULL:
|
||||
raise ValueError('cache_sync_feed failed: ' + error.decode('ascii'))
|
||||
|
|
|
|||
|
|
@ -0,0 +1,139 @@
|
|||
|
||||
from msgpack import packb
|
||||
|
||||
import pytest
|
||||
|
||||
from ..hashindex import ChunkIndex, CacheSynchronizer
|
||||
from .hashindex import H
|
||||
|
||||
|
||||
class TestCacheSynchronizer:
|
||||
@pytest.fixture
|
||||
def index(self):
|
||||
return ChunkIndex()
|
||||
|
||||
@pytest.fixture
|
||||
def sync(self, index):
|
||||
return CacheSynchronizer(index)
|
||||
|
||||
def test_no_chunks(self, index, sync):
|
||||
data = packb({
|
||||
'foo': 'bar',
|
||||
'baz': 1234,
|
||||
'bar': 5678,
|
||||
'user': 'chunks',
|
||||
'chunks': []
|
||||
})
|
||||
sync.feed(data)
|
||||
assert not len(index)
|
||||
|
||||
def test_simple(self, index, sync):
|
||||
data = packb({
|
||||
'foo': 'bar',
|
||||
'baz': 1234,
|
||||
'bar': 5678,
|
||||
'user': 'chunks',
|
||||
'chunks': [
|
||||
(H(1), 1, 2),
|
||||
(H(2), 2, 3),
|
||||
]
|
||||
})
|
||||
sync.feed(data)
|
||||
assert len(index) == 2
|
||||
assert index[H(1)] == (1, 1, 2)
|
||||
assert index[H(2)] == (1, 2, 3)
|
||||
|
||||
def test_multiple(self, index, sync):
|
||||
data = packb({
|
||||
'foo': 'bar',
|
||||
'baz': 1234,
|
||||
'bar': 5678,
|
||||
'user': 'chunks',
|
||||
'chunks': [
|
||||
(H(1), 1, 2),
|
||||
(H(2), 2, 3),
|
||||
]
|
||||
})
|
||||
data += packb({
|
||||
'xattrs': {
|
||||
'security.foo': 'bar',
|
||||
'chunks': '123456',
|
||||
},
|
||||
'stuff': [
|
||||
(1, 2, 3),
|
||||
]
|
||||
})
|
||||
data += packb({
|
||||
'xattrs': {
|
||||
'security.foo': 'bar',
|
||||
'chunks': '123456',
|
||||
},
|
||||
'chunks': [
|
||||
(H(1), 1, 2),
|
||||
(H(2), 2, 3),
|
||||
],
|
||||
'stuff': [
|
||||
(1, 2, 3),
|
||||
]
|
||||
})
|
||||
data += packb({
|
||||
'chunks': [
|
||||
(H(3), 1, 2),
|
||||
],
|
||||
})
|
||||
data += packb({
|
||||
'chunks': [
|
||||
(H(1), 1, 2),
|
||||
],
|
||||
})
|
||||
|
||||
part1 = data[:70]
|
||||
part2 = data[70:120]
|
||||
part3 = data[120:]
|
||||
sync.feed(part1)
|
||||
sync.feed(part2)
|
||||
sync.feed(part3)
|
||||
assert len(index) == 3
|
||||
assert index[H(1)] == (3, 1, 2)
|
||||
assert index[H(2)] == (2, 2, 3)
|
||||
assert index[H(3)] == (1, 1, 2)
|
||||
|
||||
@pytest.mark.parametrize('elem,error', (
|
||||
({1: 2}, 'Unexpected object: map'),
|
||||
(bytes(213), [
|
||||
'Unexpected bytes in chunks structure', # structure 2/3
|
||||
'Incorrect key length']), # structure 3/3
|
||||
(1, 'Unexpected object: integer'),
|
||||
(1.0, 'Unexpected object: double'),
|
||||
(True, 'Unexpected object: true'),
|
||||
(False, 'Unexpected object: false'),
|
||||
(None, 'Unexpected object: nil'),
|
||||
))
|
||||
@pytest.mark.parametrize('structure', (
|
||||
lambda elem: {'chunks': elem},
|
||||
lambda elem: {'chunks': [elem]},
|
||||
lambda elem: {'chunks': [(elem, 1, 2)]},
|
||||
))
|
||||
def test_corrupted(self, sync, structure, elem, error):
|
||||
packed = packb(structure(elem))
|
||||
with pytest.raises(ValueError) as excinfo:
|
||||
sync.feed(packed)
|
||||
if isinstance(error, str):
|
||||
error = [error]
|
||||
possible_errors = ['cache_sync_feed failed: ' + error for error in error]
|
||||
assert str(excinfo.value) in possible_errors
|
||||
|
||||
@pytest.mark.parametrize('data,error', (
|
||||
# Incorrect tuple length
|
||||
({'chunks': [(bytes(32), 2, 3, 4)]}, 'Invalid chunk list entry length'),
|
||||
({'chunks': [(bytes(32), 2)]}, 'Invalid chunk list entry length'),
|
||||
# Incorrect types
|
||||
({'chunks': [(1, 2, 3)]}, 'Unexpected object: integer'),
|
||||
({'chunks': [(1, bytes(32), 2)]}, 'Unexpected object: integer'),
|
||||
({'chunks': [(bytes(32), 1.0, 2)]}, 'Unexpected object: double'),
|
||||
))
|
||||
def test_corrupted_ancillary(self, index, sync, data, error):
|
||||
packed = packb(data)
|
||||
with pytest.raises(ValueError) as excinfo:
|
||||
sync.feed(packed)
|
||||
assert str(excinfo.value) == 'cache_sync_feed failed: ' + error
|
||||
Loading…
Reference in New Issue