Merge pull request #817 from enkore/feature/append-only

Feature append only

borg/repository.py

@@ -1,5 +1,6 @@
 from configparser import ConfigParser
 from binascii import hexlify, unhexlify
+from datetime import datetime
 from itertools import islice
 import errno
 import logging
@@ -84,6 +85,7 @@ def create(self, path):
         config.set('repository', 'version', '1')
         config.set('repository', 'segments_per_dir', str(self.DEFAULT_SEGMENTS_PER_DIR))
         config.set('repository', 'max_segment_size', str(self.DEFAULT_MAX_SEGMENT_SIZE))
+        config.set('repository', 'append_only', '0')
         config.set('repository', 'id', hexlify(os.urandom(32)).decode('ascii'))
         self.save_config(path, config)
 
@@ -105,6 +107,8 @@ def load_key(self):
     def destroy(self):
         """Destroy the repository at `self.path`
         """
+        if self.append_only:
+            raise ValueError(self.path + " is in append-only mode")
         self.close()
         os.remove(os.path.join(self.path, 'config'))  # kill config first
         shutil.rmtree(self.path)
@@ -148,6 +152,7 @@ def open(self, path, exclusive, lock_wait=None, lock=True):
             raise self.InvalidRepository(path)
         self.max_segment_size = self.config.getint('repository', 'max_segment_size')
         self.segments_per_dir = self.config.getint('repository', 'segments_per_dir')
+        self.append_only = self.config.getboolean('repository', 'append_only', fallback=False)
         self.id = unhexlify(self.config.get('repository', 'id').strip())
         self.io = LoggedIO(self.path, self.max_segment_size, self.segments_per_dir)
 
@@ -163,6 +168,7 @@ def commit(self, save_space=False):
         """Commit transaction
         """
         self.io.write_commit()
+        if not self.append_only:
             self.compact_segments(save_space=save_space)
         self.write_index()
         self.rollback()
@@ -211,6 +217,9 @@ def write_index(self):
         self.index.write(os.path.join(self.path, 'index.tmp'))
         os.rename(os.path.join(self.path, 'index.tmp'),
                   os.path.join(self.path, 'index.%d' % transaction_id))
+        if self.append_only:
+            with open(os.path.join(self.path, 'transactions'), 'a') as log:
+                print('transaction %d, UTC time %s' % (transaction_id, datetime.utcnow().isoformat()), file=log)
         # Remove old indices
         current = '.%d' % transaction_id
         for name in os.listdir(self.path):
@@ -323,6 +332,8 @@ def check(self, repair=False, save_space=False):
         This method verifies all segment checksums and makes sure
         the index is consistent with the data stored in the segments.
         """
+        if self.append_only and repair:
+            raise ValueError(self.path + " is in append-only mode")
         error_found = False
 
         def report_error(msg):

borg/testsuite/repository.py

@@ -187,6 +187,34 @@ def test_crash_before_deleting_compacted_segments(self):
         self.assert_equal(len(self.repository), 3)
 
 
+class RepositoryAppendOnlyTestCase(RepositoryTestCaseBase):
+    def test_destroy_append_only(self):
+        # Can't destroy append only repo (via the API)
+        self.repository.append_only = True
+        with self.assert_raises(ValueError):
+            self.repository.destroy()
+
+    def test_append_only(self):
+        def segments_in_repository():
+            return len(list(self.repository.io.segment_iterator()))
+        self.repository.put(b'00000000000000000000000000000000', b'foo')
+        self.repository.commit()
+
+        self.repository.append_only = False
+        assert segments_in_repository() == 1
+        self.repository.put(b'00000000000000000000000000000000', b'foo')
+        self.repository.commit()
+        # normal: compact squashes the data together, only one segment
+        assert segments_in_repository() == 1
+
+        self.repository.append_only = True
+        assert segments_in_repository() == 1
+        self.repository.put(b'00000000000000000000000000000000', b'foo')
+        self.repository.commit()
+        # append only: does not compact, only new segments written
+        assert segments_in_repository() == 2
+
+
 class RepositoryCheckTestCase(RepositoryTestCaseBase):
 
     def list_indices(self):

docs/usage.rst

@@ -694,3 +694,75 @@ Now, let's see how to restore some LVs from such a backup. ::
     $ # we assume that you created an empty root and home LV and overwrite it now:
     $ borg extract --stdout /mnt/backup::repo dev/vg0/root-snapshot > /dev/vg0/root
     $ borg extract --stdout /mnt/backup::repo dev/vg0/home-snapshot > /dev/vg0/home
+
+
+Append-only mode
+~~~~~~~~~~~~~~~~
+
+A repository can be made "append-only", which means that Borg will never overwrite or
+delete committed data. This is useful for scenarios where multiple machines back up to
+a central backup server using ``borg serve``, since a hacked machine cannot delete
+backups permanently.
+
+To activate append-only mode, edit the repository ``config`` file and add a line
+``append_only=1`` to the ``[repository]`` section (or edit the line if it exists).
+
+In append-only mode Borg will create a transaction log in the ``transactions`` file,
+where each line is a transaction and a UTC timestamp.
+
+Example
++++++++
+
+Suppose an attacker remotely deleted all backups, but your repository was in append-only
+mode. A transaction log in this situation might look like this: ::
+
+    transaction 1, UTC time 2016-03-31T15:53:27.383532
+    transaction 5, UTC time 2016-03-31T15:53:52.588922
+    transaction 11, UTC time 2016-03-31T15:54:23.887256
+    transaction 12, UTC time 2016-03-31T15:55:54.022540
+    transaction 13, UTC time 2016-03-31T15:55:55.472564
+
+From your security logs you conclude the attacker gained access at 15:54:00 and all
+the backups where deleted or replaced by compromised backups. From the log you know
+that transactions 11 and later are compromised. Note that the transaction ID is the
+name of the *last* file in the transaction. For example, transaction 11 spans files 6
+to 11.
+
+In a real attack you'll likely want to keep the compromised repository
+intact to analyze what the attacker tried to achieve. It's also a good idea to make this
+copy just in case something goes wrong during the recovery. Since recovery is done by
+deleting some files, a hard link copy (``cp -al``) is sufficient.
+
+The first step to reset the repository to transaction 5, the last uncompromised transaction,
+is to remove the ``hints.N`` and ``index.N`` files in the repository (these two files are
+always expendable). In this example N is 13.
+
+Then remove or move all segment files from the segment directories in ``data/`` starting
+with file 6::
+
+    rm data/**/{6..13}
+
+That's all to it.
+
+Drawbacks
++++++++++
+
+As data is only appended, and nothing deleted, commands like ``prune`` or ``delete``
+won't free disk space, they merely tag data as deleted in a new transaction.
+
+Note that you can go back-and-forth between normal and append-only operation by editing
+the configuration file, it's not a "one way trip".
+
+Further considerations
+++++++++++++++++++++++
+
+Append-only mode is not respected by tools other than Borg. ``rm`` still works on the
+repository. Make sure that backup client machines only get to access the repository via
+``borg serve``.
+
+Ensure that no remote access is possible if the repository is temporarily set to normal mode
+for e.g. regular pruning.
+
+Further protections can be implemented, but are outside of Borgs scope. For example,
+file system snapshots or wrapping ``borg serve`` to set special permissions or ACLs on
+new data files.