diff --git a/docs/changes.rst b/docs/changes.rst index 4990a1e73..3af543990 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -131,21 +131,47 @@ Security fixes: - fix security issue with remote repository access, #1428 -Version 1.0.7rc2 (not released yet) ------------------------------------ +Version 1.0.7rc2 (2016-08-13) +----------------------------- Bug fixes: - do not write objects to repository that are bigger than the allowed size, borg will reject reading them, #1451. - IMPORTANT: if you created archives with many millions of files or - directories, please verify if you can open them successfully, - e.g. try a "borg list REPO::ARCHIVE". -- fixed a race condition in extended attributes querying that led to the - entire file not being backed up (while logging the error, exit code = 1), - #1469 -- fixed a race condition in extended attributes querying that led to a crash, - #1462 + + Important: if you created archives with many millions of files or + directories, please verify if you can open them successfully, + e.g. try a "borg list REPO::ARCHIVE". +- lz4 compression: dynamically enlarge the (de)compression buffer, the static + buffer was not big enough for archives with extremely many items, #1453 +- larger item metadata stream chunks, raise archive item limit by 8x, #1452 +- fix untracked segments made by moved DELETEs, #1442 + + Impact: Previously (metadata) segments could become untracked when deleting data, + these would never be cleaned up. +- extended attributes (xattrs) related fixes: + + - fixed a race condition in xattrs querying that led to the entire file not + being backed up (while logging the error, exit code = 1), #1469 + - fixed a race condition in xattrs querying that led to a crash, #1462 + - raise OSError including the error message derived from errno, deal with + path being a integer FD + +Other changes: + +- print active env var override by default, #1467 +- xattr module: refactor code, deduplicate, clean up +- repository: split object size check into too small and too big +- add a transaction_id assertion, so borg init on a broken (inconsistent) + filesystem does not look like a coding error in borg, but points to the + real problem. +- explain confusing TypeError caused by compat support for old servers, #1456 +- add forgotten usage help file from build_usage +- refactor/unify buffer code into helpers.Buffer class, add tests +- docs: + + - document archive limitation, #1452 + - improve prune examples Version 1.0.7rc1 (2016-08-05) diff --git a/src/borg/repository.py b/src/borg/repository.py index 4823c0c17..794f6fdc0 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -449,11 +449,46 @@ class Repository: segments[segment] -= 1 elif tag == TAG_DELETE: if index_transaction_id is None or segment > index_transaction_id: + # (introduced in 6425d16aa84be1eaaf88) + # This is needed to avoid object un-deletion if we crash between the commit and the deletion + # of old segments in complete_xfer(). + # + # However, this only happens if the crash also affects the FS to the effect that file deletions + # did not materialize consistently after journal recovery. If they always materialize in-order + # then this is not a problem, because the old segment containing a deleted object would be deleted + # before the segment containing the delete. + # + # Consider the following series of operations if we would not do this, ie. this entire if: + # would be removed. + # Columns are segments, lines are different keys (line 1 = some key, line 2 = some other key) + # Legend: P=TAG_PUT, D=TAG_DELETE, c=commit, i=index is written for latest commit + # + # Segment | 1 | 2 | 3 + # --------+-------+-----+------ + # Key 1 | P | D | + # Key 2 | P | | P + # commits | c i | c | c i + # --------+-------+-----+------ + # ^- compact_segments starts + # ^- complete_xfer commits, after that complete_xfer deletes + # segments 1 and 2 (and then the index would be written). + # + # Now we crash. But only segment 2 gets deleted, while segment 1 is still around. Now key 1 + # is suddenly undeleted (because the delete in segment 2 is now missing). + # Again, note the requirement here. We delete these in the correct order that this doesn't happen, + # and only if the FS materialization of these deletes is reordered or parts dropped this can happen. + # In this case it doesn't cause outright corruption, 'just' an index count mismatch, which will be + # fixed by borg-check --repair. + # + # Note that in this check the index state is the proxy for a "most definitely settled" repository state, + # ie. the assumption is that *all* operations on segments <= index state are completed and stable. try: - self.io.write_delete(key, raise_full=True) + new_segment, size = self.io.write_delete(key, raise_full=True) except LoggedIO.SegmentFull: complete_xfer() - self.io.write_delete(key) + new_segment, size = self.io.write_delete(key) + self.compact[new_segment] += size + segments.setdefault(new_segment, 0) assert segments[segment] == 0 unused.append(segment) complete_xfer(intermediate=False) diff --git a/src/borg/testsuite/repository.py b/src/borg/testsuite/repository.py index 0fc1461de..25e93101a 100644 --- a/src/borg/testsuite/repository.py +++ b/src/borg/testsuite/repository.py @@ -13,8 +13,9 @@ from ..helpers import Location from ..helpers import IntegrityError from ..locking import Lock, LockFailed from ..remote import RemoteRepository, InvalidRPCMethod, ConnectionClosedWithHint, handle_remote_line -from ..repository import Repository, LoggedIO, MAGIC, MAX_DATA_SIZE +from ..repository import Repository, LoggedIO, MAGIC, MAX_DATA_SIZE, TAG_DELETE from . import BaseTestCase +from .hashindex import H UNSPECIFIED = object() # for default values where we can't use None @@ -272,6 +273,28 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase): io = self.repository.io assert not io.is_committed_segment(io.get_latest_segment()) + def test_moved_deletes_are_tracked(self): + self.repository.put(H(1), b'1') + self.repository.put(H(2), b'2') + self.repository.commit() + self.repository.delete(H(1)) + self.repository.commit() + last_segment = self.repository.io.get_latest_segment() - 1 + num_deletes = 0 + for tag, key, offset, size in self.repository.io.iter_objects(last_segment): + if tag == TAG_DELETE: + assert key == H(1) + num_deletes += 1 + assert num_deletes == 1 + assert last_segment in self.repository.compact + self.repository.put(H(3), b'3') + self.repository.commit() + assert last_segment not in self.repository.compact + assert not self.repository.io.segment_exists(last_segment) + for segment, _ in self.repository.io.segment_iterator(): + for tag, key, offset, size in self.repository.io.iter_objects(segment): + assert tag != TAG_DELETE + class RepositoryAppendOnlyTestCase(RepositoryTestCaseBase): def open(self, create=False):