Merge pull request #819 from enkore/fix/chunker-io2

Chunker: fix wrong EOF assumption, check for return type

borg/_chunker.c

@@ -174,6 +174,10 @@ chunker_fill(Chunker *c)
             return 0;
         }
         n = PyBytes_Size(data);
+        if(PyErr_Occurred()) {
+            // we wanted bytes(), but got something else
+            return 0;
+        }
         if(n) {
             memcpy(c->data + c->position + c->remaining, PyBytes_AsString(data), n);
             c->remaining += n;
@@ -200,12 +204,12 @@ chunker_process(Chunker *c)
             PyErr_SetString(PyExc_Exception, "chunkifier byte count mismatch");
         return NULL;
     }
-    if(c->remaining <= window_size) {
+    while(c->remaining <= window_size && !c->eof) {
         if(!chunker_fill(c)) {
             return NULL;
         }
     }
-    if(c->remaining < window_size) {
+    if(c->eof) {
         c->done = 1;
         if(c->remaining) {
             c->bytes_yielded += c->remaining;

borg/testsuite/chunker.py

@@ -1,7 +1,7 @@
 from io import BytesIO
 
 from ..chunker import Chunker, buzhash, buzhash_update
-from ..archive import CHUNK_MAX_EXP
+from ..archive import CHUNK_MAX_EXP, CHUNKER_PARAMS
 from . import BaseTestCase
 
 
@@ -29,3 +29,14 @@ class ChunkerTestCase(BaseTestCase):
         self.assert_equal(buzhash(b'abcdefghijklmnop', 1), buzhash_update(buzhash(b'Xabcdefghijklmno', 1), ord('X'), ord('p'), 16, 1))
         # Test with more than 31 bytes to make sure our barrel_shift macro works correctly
         self.assert_equal(buzhash(b'abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz', 0), 566521248)
+
+    def test_small_reads(self):
+        class SmallReadFile:
+            input = b'a' * (20 + 1)
+
+            def read(self, nbytes):
+                self.input = self.input[:-1]
+                return self.input[:1]
+
+        reconstructed = b''.join(Chunker(0, *CHUNKER_PARAMS).chunkify(SmallReadFile()))
+        assert reconstructed == b'a' * 20