Merge pull request #2663 from enkore/f/fcs-changes

cache sync changes

scripts/fuzz-cache-sync/main.c

@@ -1,3 +1,6 @@
+
+#define BORG_NO_PYTHON
+
 #include "../../src/borg/_hashindex.c"
 #include "../../src/borg/cache_sync/cache_sync.c"
 

src/borg/_hashindex.c

@@ -56,7 +56,7 @@ typedef struct {
     int lower_limit;
     int upper_limit;
     int min_empty;
-#ifdef Py_PYTHON_H
+#ifndef BORG_NO_PYTHON
     /* buckets may be backed by a Python buffer. If buckets_buffer.buf is NULL then this is not used. */
     Py_buffer buckets_buffer;
 #endif
@@ -108,7 +108,7 @@ static int hash_sizes[] = {
 #define EPRINTF(msg, ...) fprintf(stderr, "hashindex: " msg "(%s)\n", ##__VA_ARGS__, strerror(errno))
 #define EPRINTF_PATH(path, msg, ...) fprintf(stderr, "hashindex: %s: " msg " (%s)\n", path, ##__VA_ARGS__, strerror(errno))
 
-#ifdef Py_PYTHON_H
+#ifndef BORG_NO_PYTHON
 static HashIndex *hashindex_read(PyObject *file_py, int permit_compact);
 static void hashindex_write(HashIndex *index, PyObject *file_py);
 #endif
@@ -126,7 +126,7 @@ static void hashindex_free(HashIndex *index);
 static void
 hashindex_free_buckets(HashIndex *index)
 {
-#ifdef Py_PYTHON_H
+#ifndef BORG_NO_PYTHON
     if(index->buckets_buffer.buf) {
         PyBuffer_Release(&index->buckets_buffer);
     } else
@@ -272,7 +272,7 @@ count_empty(HashIndex *index)
 
 /* Public API */
 
-#ifdef Py_PYTHON_H
+#ifndef BORG_NO_PYTHON
 static HashIndex *
 hashindex_read(PyObject *file_py, int permit_compact)
 {
@@ -457,7 +457,7 @@ hashindex_init(int capacity, int key_size, int value_size)
     index->lower_limit = get_lower_limit(index->num_buckets);
     index->upper_limit = get_upper_limit(index->num_buckets);
     index->min_empty = get_min_empty(index->num_buckets);
-#ifdef Py_PYTHON_H
+#ifndef BORG_NO_PYTHON
     index->buckets_buffer.buf = NULL;
 #endif
     for(i = 0; i < capacity; i++) {
@@ -473,7 +473,7 @@ hashindex_free(HashIndex *index)
     free(index);
 }
 
-#ifdef Py_PYTHON_H
+#ifndef BORG_NO_PYTHON
 static void
 hashindex_write(HashIndex *index, PyObject *file_py)
 {

src/borg/cache_sync/unpack.h

@@ -118,7 +118,7 @@ static inline void unpack_init_user_state(unpack_user *u)
     u->expect = expect_item_begin;
 }
 
-static inline int unpack_callback_int64(unpack_user* u, int64_t d)
+static inline int unpack_callback_uint64(unpack_user* u, int64_t d)
 {
     switch(u->expect) {
         case expect_size:
@@ -135,40 +135,39 @@ static inline int unpack_callback_int64(unpack_user* u, int64_t d)
     return 0;
 }
 
+static inline int unpack_callback_uint32(unpack_user* u, uint32_t d)
+{
+    return unpack_callback_uint64(u, d);
+}
+
 static inline int unpack_callback_uint16(unpack_user* u, uint16_t d)
 {
-    return unpack_callback_int64(u, d);
+    return unpack_callback_uint64(u, d);
 }
 
 static inline int unpack_callback_uint8(unpack_user* u, uint8_t d)
 {
-    return unpack_callback_int64(u, d);
+    return unpack_callback_uint64(u, d);
 }
 
-
-static inline int unpack_callback_uint32(unpack_user* u, uint32_t d)
+static inline int unpack_callback_int64(unpack_user* u, uint64_t d)
 {
-    return unpack_callback_int64(u, d);
-}
-
-static inline int unpack_callback_uint64(unpack_user* u, uint64_t d)
-{
-    return unpack_callback_int64(u, d);
+    return unpack_callback_uint64(u, d);
 }
 
 static inline int unpack_callback_int32(unpack_user* u, int32_t d)
 {
-    return unpack_callback_int64(u, d);
+    return unpack_callback_uint64(u, d);
 }
 
 static inline int unpack_callback_int16(unpack_user* u, int16_t d)
 {
-    return unpack_callback_int64(u, d);
+    return unpack_callback_uint64(u, d);
 }
 
 static inline int unpack_callback_int8(unpack_user* u, int8_t d)
 {
-    return unpack_callback_int64(u, d);
+    return unpack_callback_uint64(u, d);
 }
 
 /* Ain't got anything to do with those floats */

src/borg/remote.py

@@ -27,6 +27,7 @@ from .helpers import hostname_is_unique
 from .helpers import replace_placeholders
 from .helpers import sysinfo
 from .helpers import format_file_size
+from .helpers import truncate_and_unlink
 from .logger import create_logger, setup_logging
 from .repository import Repository, MAX_OBJECT_SIZE, LIST_SCAN_LIMIT
 from .version import parse_version, format_version
@@ -1144,6 +1145,10 @@ class RepositoryCache(RepositoryNoCache):
             with open(file, 'wb') as fd:
                 fd.write(packed)
         except OSError as os_error:
+            try:
+                truncate_and_unlink(file)
+            except FileNotFoundError:
+                pass  # open() could have failed as well
             if os_error.errno == errno.ENOSPC:
                 self.enospc += 1
                 self.backoff()

src/borg/testsuite/remote.py

@@ -141,6 +141,9 @@ class TestRepositoryCache:
             def write(self, data):
                 raise OSError(errno.ENOSPC, 'foo')
 
+            def truncate(self, n=None):
+                pass
+
         iterator = cache.get_many([H(1), H(2), H(3)])
         assert next(iterator) == b'1234'