mirror of
https://github.com/borgbackup/borg.git
synced 2024-12-25 01:06:50 +00:00
Merge pull request #817 from enkore/feature/append-only
Feature append only
This commit is contained in:
commit
8cfc930066
3 changed files with 112 additions and 1 deletions
|
@ -1,5 +1,6 @@
|
|||
from configparser import ConfigParser
|
||||
from binascii import hexlify, unhexlify
|
||||
from datetime import datetime
|
||||
from itertools import islice
|
||||
import errno
|
||||
import logging
|
||||
|
@ -84,6 +85,7 @@ def create(self, path):
|
|||
config.set('repository', 'version', '1')
|
||||
config.set('repository', 'segments_per_dir', str(self.DEFAULT_SEGMENTS_PER_DIR))
|
||||
config.set('repository', 'max_segment_size', str(self.DEFAULT_MAX_SEGMENT_SIZE))
|
||||
config.set('repository', 'append_only', '0')
|
||||
config.set('repository', 'id', hexlify(os.urandom(32)).decode('ascii'))
|
||||
self.save_config(path, config)
|
||||
|
||||
|
@ -105,6 +107,8 @@ def load_key(self):
|
|||
def destroy(self):
|
||||
"""Destroy the repository at `self.path`
|
||||
"""
|
||||
if self.append_only:
|
||||
raise ValueError(self.path + " is in append-only mode")
|
||||
self.close()
|
||||
os.remove(os.path.join(self.path, 'config')) # kill config first
|
||||
shutil.rmtree(self.path)
|
||||
|
@ -148,6 +152,7 @@ def open(self, path, exclusive, lock_wait=None, lock=True):
|
|||
raise self.InvalidRepository(path)
|
||||
self.max_segment_size = self.config.getint('repository', 'max_segment_size')
|
||||
self.segments_per_dir = self.config.getint('repository', 'segments_per_dir')
|
||||
self.append_only = self.config.getboolean('repository', 'append_only', fallback=False)
|
||||
self.id = unhexlify(self.config.get('repository', 'id').strip())
|
||||
self.io = LoggedIO(self.path, self.max_segment_size, self.segments_per_dir)
|
||||
|
||||
|
@ -163,7 +168,8 @@ def commit(self, save_space=False):
|
|||
"""Commit transaction
|
||||
"""
|
||||
self.io.write_commit()
|
||||
self.compact_segments(save_space=save_space)
|
||||
if not self.append_only:
|
||||
self.compact_segments(save_space=save_space)
|
||||
self.write_index()
|
||||
self.rollback()
|
||||
|
||||
|
@ -211,6 +217,9 @@ def write_index(self):
|
|||
self.index.write(os.path.join(self.path, 'index.tmp'))
|
||||
os.rename(os.path.join(self.path, 'index.tmp'),
|
||||
os.path.join(self.path, 'index.%d' % transaction_id))
|
||||
if self.append_only:
|
||||
with open(os.path.join(self.path, 'transactions'), 'a') as log:
|
||||
print('transaction %d, UTC time %s' % (transaction_id, datetime.utcnow().isoformat()), file=log)
|
||||
# Remove old indices
|
||||
current = '.%d' % transaction_id
|
||||
for name in os.listdir(self.path):
|
||||
|
@ -323,6 +332,8 @@ def check(self, repair=False, save_space=False):
|
|||
This method verifies all segment checksums and makes sure
|
||||
the index is consistent with the data stored in the segments.
|
||||
"""
|
||||
if self.append_only and repair:
|
||||
raise ValueError(self.path + " is in append-only mode")
|
||||
error_found = False
|
||||
|
||||
def report_error(msg):
|
||||
|
|
|
@ -187,6 +187,34 @@ def test_crash_before_deleting_compacted_segments(self):
|
|||
self.assert_equal(len(self.repository), 3)
|
||||
|
||||
|
||||
class RepositoryAppendOnlyTestCase(RepositoryTestCaseBase):
|
||||
def test_destroy_append_only(self):
|
||||
# Can't destroy append only repo (via the API)
|
||||
self.repository.append_only = True
|
||||
with self.assert_raises(ValueError):
|
||||
self.repository.destroy()
|
||||
|
||||
def test_append_only(self):
|
||||
def segments_in_repository():
|
||||
return len(list(self.repository.io.segment_iterator()))
|
||||
self.repository.put(b'00000000000000000000000000000000', b'foo')
|
||||
self.repository.commit()
|
||||
|
||||
self.repository.append_only = False
|
||||
assert segments_in_repository() == 1
|
||||
self.repository.put(b'00000000000000000000000000000000', b'foo')
|
||||
self.repository.commit()
|
||||
# normal: compact squashes the data together, only one segment
|
||||
assert segments_in_repository() == 1
|
||||
|
||||
self.repository.append_only = True
|
||||
assert segments_in_repository() == 1
|
||||
self.repository.put(b'00000000000000000000000000000000', b'foo')
|
||||
self.repository.commit()
|
||||
# append only: does not compact, only new segments written
|
||||
assert segments_in_repository() == 2
|
||||
|
||||
|
||||
class RepositoryCheckTestCase(RepositoryTestCaseBase):
|
||||
|
||||
def list_indices(self):
|
||||
|
|
|
@ -694,3 +694,75 @@ Now, let's see how to restore some LVs from such a backup. ::
|
|||
$ # we assume that you created an empty root and home LV and overwrite it now:
|
||||
$ borg extract --stdout /mnt/backup::repo dev/vg0/root-snapshot > /dev/vg0/root
|
||||
$ borg extract --stdout /mnt/backup::repo dev/vg0/home-snapshot > /dev/vg0/home
|
||||
|
||||
|
||||
Append-only mode
|
||||
~~~~~~~~~~~~~~~~
|
||||
|
||||
A repository can be made "append-only", which means that Borg will never overwrite or
|
||||
delete committed data. This is useful for scenarios where multiple machines back up to
|
||||
a central backup server using ``borg serve``, since a hacked machine cannot delete
|
||||
backups permanently.
|
||||
|
||||
To activate append-only mode, edit the repository ``config`` file and add a line
|
||||
``append_only=1`` to the ``[repository]`` section (or edit the line if it exists).
|
||||
|
||||
In append-only mode Borg will create a transaction log in the ``transactions`` file,
|
||||
where each line is a transaction and a UTC timestamp.
|
||||
|
||||
Example
|
||||
+++++++
|
||||
|
||||
Suppose an attacker remotely deleted all backups, but your repository was in append-only
|
||||
mode. A transaction log in this situation might look like this: ::
|
||||
|
||||
transaction 1, UTC time 2016-03-31T15:53:27.383532
|
||||
transaction 5, UTC time 2016-03-31T15:53:52.588922
|
||||
transaction 11, UTC time 2016-03-31T15:54:23.887256
|
||||
transaction 12, UTC time 2016-03-31T15:55:54.022540
|
||||
transaction 13, UTC time 2016-03-31T15:55:55.472564
|
||||
|
||||
From your security logs you conclude the attacker gained access at 15:54:00 and all
|
||||
the backups where deleted or replaced by compromised backups. From the log you know
|
||||
that transactions 11 and later are compromised. Note that the transaction ID is the
|
||||
name of the *last* file in the transaction. For example, transaction 11 spans files 6
|
||||
to 11.
|
||||
|
||||
In a real attack you'll likely want to keep the compromised repository
|
||||
intact to analyze what the attacker tried to achieve. It's also a good idea to make this
|
||||
copy just in case something goes wrong during the recovery. Since recovery is done by
|
||||
deleting some files, a hard link copy (``cp -al``) is sufficient.
|
||||
|
||||
The first step to reset the repository to transaction 5, the last uncompromised transaction,
|
||||
is to remove the ``hints.N`` and ``index.N`` files in the repository (these two files are
|
||||
always expendable). In this example N is 13.
|
||||
|
||||
Then remove or move all segment files from the segment directories in ``data/`` starting
|
||||
with file 6::
|
||||
|
||||
rm data/**/{6..13}
|
||||
|
||||
That's all to it.
|
||||
|
||||
Drawbacks
|
||||
+++++++++
|
||||
|
||||
As data is only appended, and nothing deleted, commands like ``prune`` or ``delete``
|
||||
won't free disk space, they merely tag data as deleted in a new transaction.
|
||||
|
||||
Note that you can go back-and-forth between normal and append-only operation by editing
|
||||
the configuration file, it's not a "one way trip".
|
||||
|
||||
Further considerations
|
||||
++++++++++++++++++++++
|
||||
|
||||
Append-only mode is not respected by tools other than Borg. ``rm`` still works on the
|
||||
repository. Make sure that backup client machines only get to access the repository via
|
||||
``borg serve``.
|
||||
|
||||
Ensure that no remote access is possible if the repository is temporarily set to normal mode
|
||||
for e.g. regular pruning.
|
||||
|
||||
Further protections can be implemented, but are outside of Borgs scope. For example,
|
||||
file system snapshots or wrapping ``borg serve`` to set special permissions or ACLs on
|
||||
new data files.
|
Loading…
Reference in a new issue