Merge branch '1.0-maint' into master

# Conflicts:
#	src/borg/repository.py
#	src/borg/testsuite/repository.py

docs/changes.rst

@@ -131,21 +131,47 @@ Security fixes:
 - fix security issue with remote repository access, #1428
 
 
-Version 1.0.7rc2 (not released yet)
+Version 1.0.7rc2 (2016-08-13)
------------------------------------
+-----------------------------
 
 Bug fixes:
 
 - do not write objects to repository that are bigger than the allowed size,
   borg will reject reading them, #1451.
-  IMPORTANT: if you created archives with many millions of files or
+
-             directories, please verify if you can open them successfully,
+  Important: if you created archives with many millions of files or
-             e.g. try a "borg list REPO::ARCHIVE".
+  directories, please verify if you can open them successfully,
-- fixed a race condition in extended attributes querying that led to the
+  e.g. try a "borg list REPO::ARCHIVE".
-  entire file not being backed up (while logging the error, exit code = 1),
+- lz4 compression: dynamically enlarge the (de)compression buffer, the static
-  #1469
+  buffer was not big enough for archives with extremely many items, #1453
-- fixed a race condition in extended attributes querying that led to a crash,
+- larger item metadata stream chunks, raise archive item limit by 8x, #1452
-  #1462
+- fix untracked segments made by moved DELETEs, #1442
+
+  Impact: Previously (metadata) segments could become untracked when deleting data,
+  these would never be cleaned up.
+- extended attributes (xattrs) related fixes:
+
+  - fixed a race condition in xattrs querying that led to the entire file not
+    being backed up (while logging the error, exit code = 1), #1469
+  - fixed a race condition in xattrs querying that led to a crash, #1462
+  - raise OSError including the error message derived from errno, deal with
+    path being a integer FD
+
+Other changes:
+
+- print active env var override by default, #1467
+- xattr module: refactor code, deduplicate, clean up
+- repository: split object size check into too small and too big
+- add a transaction_id assertion, so borg init on a broken (inconsistent)
+  filesystem does not look like a coding error in borg, but points to the
+  real problem.
+- explain confusing TypeError caused by compat support for old servers, #1456
+- add forgotten usage help file from build_usage
+- refactor/unify buffer code into helpers.Buffer class, add tests
+- docs:
+
+  - document archive limitation, #1452
+  - improve prune examples
 
 
 Version 1.0.7rc1 (2016-08-05)

src/borg/repository.py

@@ -449,11 +449,46 @@ def complete_xfer(intermediate=True):
                     segments[segment] -= 1
                 elif tag == TAG_DELETE:
                     if index_transaction_id is None or segment > index_transaction_id:
+                        # (introduced in 6425d16aa84be1eaaf88)
+                        # This is needed to avoid object un-deletion if we crash between the commit and the deletion
+                        # of old segments in complete_xfer().
+                        #
+                        # However, this only happens if the crash also affects the FS to the effect that file deletions
+                        # did not materialize consistently after journal recovery. If they always materialize in-order
+                        # then this is not a problem, because the old segment containing a deleted object would be deleted
+                        # before the segment containing the delete.
+                        #
+                        # Consider the following series of operations if we would not do this, ie. this entire if:
+                        # would be removed.
+                        # Columns are segments, lines are different keys (line 1 = some key, line 2 = some other key)
+                        # Legend: P=TAG_PUT, D=TAG_DELETE, c=commit, i=index is written for latest commit
+                        #
+                        # Segment | 1     | 2   | 3
+                        # --------+-------+-----+------
+                        # Key 1   | P     | D   |
+                        # Key 2   | P     |     | P
+                        # commits |   c i |   c |   c i
+                        # --------+-------+-----+------
+                        #                       ^- compact_segments starts
+                        #                           ^- complete_xfer commits, after that complete_xfer deletes
+                        #                              segments 1 and 2 (and then the index would be written).
+                        #
+                        # Now we crash. But only segment 2 gets deleted, while segment 1 is still around. Now key 1
+                        # is suddenly undeleted (because the delete in segment 2 is now missing).
+                        # Again, note the requirement here. We delete these in the correct order that this doesn't happen,
+                        # and only if the FS materialization of these deletes is reordered or parts dropped this can happen.
+                        # In this case it doesn't cause outright corruption, 'just' an index count mismatch, which will be
+                        # fixed by borg-check --repair.
+                        #
+                        # Note that in this check the index state is the proxy for a "most definitely settled" repository state,
+                        # ie. the assumption is that *all* operations on segments <= index state are completed and stable.
                         try:
-                            self.io.write_delete(key, raise_full=True)
+                            new_segment, size = self.io.write_delete(key, raise_full=True)
                         except LoggedIO.SegmentFull:
                             complete_xfer()
-                            self.io.write_delete(key)
+                            new_segment, size = self.io.write_delete(key)
+                        self.compact[new_segment] += size
+                        segments.setdefault(new_segment, 0)
             assert segments[segment] == 0
             unused.append(segment)
         complete_xfer(intermediate=False)

src/borg/testsuite/repository.py

@@ -13,8 +13,9 @@
 from ..helpers import IntegrityError
 from ..locking import Lock, LockFailed
 from ..remote import RemoteRepository, InvalidRPCMethod, ConnectionClosedWithHint, handle_remote_line
-from ..repository import Repository, LoggedIO, MAGIC, MAX_DATA_SIZE
+from ..repository import Repository, LoggedIO, MAGIC, MAX_DATA_SIZE, TAG_DELETE
 from . import BaseTestCase
+from .hashindex import H
 
 
 UNSPECIFIED = object()  # for default values where we can't use None
@@ -272,6 +273,28 @@ def test_ignores_commit_tag_in_data(self):
             io = self.repository.io
             assert not io.is_committed_segment(io.get_latest_segment())
 
+    def test_moved_deletes_are_tracked(self):
+        self.repository.put(H(1), b'1')
+        self.repository.put(H(2), b'2')
+        self.repository.commit()
+        self.repository.delete(H(1))
+        self.repository.commit()
+        last_segment = self.repository.io.get_latest_segment() - 1
+        num_deletes = 0
+        for tag, key, offset, size in self.repository.io.iter_objects(last_segment):
+            if tag == TAG_DELETE:
+                assert key == H(1)
+                num_deletes += 1
+        assert num_deletes == 1
+        assert last_segment in self.repository.compact
+        self.repository.put(H(3), b'3')
+        self.repository.commit()
+        assert last_segment not in self.repository.compact
+        assert not self.repository.io.segment_exists(last_segment)
+        for segment, _ in self.repository.io.segment_iterator():
+            for tag, key, offset, size in self.repository.io.iter_objects(segment):
+                assert tag != TAG_DELETE
+
 
 class RepositoryAppendOnlyTestCase(RepositoryTestCaseBase):
     def open(self, create=False):