Merge pull request #1716 from ThomasWaldmann/merge-1.0-maint

Merge 1.0-maint
This commit is contained in:
enkore 2016-10-14 21:18:30 +02:00 committed by GitHub
commit 34bd7cb4e2
13 changed files with 192 additions and 51 deletions

7
Vagrantfile vendored
View File

@ -114,7 +114,6 @@ def packages_openbsd
chsh -s /usr/local/bin/bash vagrant
pkg_add openssl
pkg_add lz4
# pkg_add fuse # does not install, sdl dependency missing
pkg_add git # no fakeroot
pkg_add py3-setuptools
ln -sf /usr/local/bin/python3.4 /usr/local/bin/python3
@ -265,7 +264,7 @@ def build_binary_with_pyinstaller(boxname)
cd /vagrant/borg
. borg-env/bin/activate
cd borg
pyinstaller -F -n borg.exe --distpath=/vagrant/borg --clean src/borg/__main__.py --hidden-import=borg.platform.posix
pyinstaller --clean --distpath=/vagrant/borg scripts/borg.exe.spec
EOF
end
@ -388,7 +387,7 @@ Vagrant.configure(2) do |config|
end
config.vm.define "wheezy32" do |b|
b.vm.box = "boxcutter/debian711-i386"
b.vm.box = "boxcutter/debian7-i386"
b.vm.provision "packages prepare wheezy", :type => :shell, :inline => packages_prepare_wheezy
b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid
b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("wheezy32")
@ -401,7 +400,7 @@ Vagrant.configure(2) do |config|
end
config.vm.define "wheezy64" do |b|
b.vm.box = "boxcutter/debian711"
b.vm.box = "boxcutter/debian7"
b.vm.provision "packages prepare wheezy", :type => :shell, :inline => packages_prepare_wheezy
b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid
b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("wheezy64")

View File

@ -54,6 +54,12 @@ Restrictions
Borg is instructed to restrict clients into their own paths:
``borg serve --restrict-to-path /home/backup/repos/<client fqdn>``
The client will be able to access any file or subdirectory inside of ``/home/backup/repos/<client fqdn>``
but no other directories. You can allow a client to access several separate directories by passing multiple
`--restrict-to-path` flags, for instance: ``borg serve --restrict-to-path /home/backup/repos/<client fqdn> --restrict-to-path /home/backup/repos/<other client fqdn>``,
which could make sense if multiple machines belong to one person which should then have access to all the
backups of their machines.
There is only one ssh key per client allowed. Keys are added for ``johndoe.clnt.local``, ``web01.srv.local`` and
``app01.srv.local``. But they will access the backup under only one UNIX user account as:
``backup@backup01.srv.local``. Every key in ``$HOME/.ssh/authorized_keys`` has a

View File

@ -12,6 +12,39 @@ Yes, the `deduplication`_ technique used by
|project_name| makes sure only the modified parts of the file are stored.
Also, we have optional simple sparse file support for extract.
If you use non-snapshotting backup tools like Borg to back up virtual machines,
then these should be turned off for doing so. Backing up live VMs this way can (and will)
result in corrupted or inconsistent backup contents: a VM image is just a regular file to
Borg with the same issues as regular files when it comes to concurrent reading and writing from
the same file.
For backing up live VMs use file system snapshots on the VM host, which establishes
crash-consistency for the VM images. This means that with most file systems
(that are journaling) the FS will always be fine in the backup (but may need a
journal replay to become accessible).
Usually this does not mean that file *contents* on the VM are consistent, since file
contents are normally not journaled. Notable exceptions are ext4 in data=journal mode,
ZFS and btrfs (unless nodatacow is used).
Applications designed with crash-consistency in mind (most relational databases
like PostgreSQL, SQLite etc. but also for example Borg repositories) should always
be able to recover to a consistent state from a backup created with
crash-consistent snapshots (even on ext4 with data=writeback or XFS).
Hypervisor snapshots capturing most of the VM's state can also be used for backups
and can be a better alternative to pure file system based snapshots of the VM's disk,
since no state is lost. Depending on the application this can be the easiest and most
reliable way to create application-consistent backups.
Other applications may require a lot of work to reach application-consistency:
It's a broad and complex issue that cannot be explained in entirety here.
Borg doesn't intend to address these issues due to their huge complexity
and platform/software dependency. Combining Borg with the mechanisms provided
by the platform (snapshots, hypervisor features) will be the best approach
to start tackling them.
Can I backup from multiple servers into a single repository?
------------------------------------------------------------

38
scripts/borg.exe.spec Normal file
View File

@ -0,0 +1,38 @@
# -*- mode: python -*-
# this pyinstaller spec file is used to build borg binaries on posix platforms
import os, sys
basepath = '/vagrant/borg/borg'
block_cipher = None
a = Analysis([os.path.join(basepath, 'src/borg/__main__.py'), ],
pathex=[basepath, ],
binaries=[],
datas=[],
hiddenimports=['borg.platform.posix'],
hookspath=[],
runtime_hooks=[],
excludes=[],
win_no_prefer_redirects=False,
win_private_assemblies=False,
cipher=block_cipher)
if sys.platform == 'darwin':
# do not bundle the osxfuse libraries, so we do not get a version
# mismatch to the installed kernel driver of osxfuse.
a.binaries = [b for b in a.binaries if 'libosxfuse' not in b[0]]
pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
exe = EXE(pyz,
a.scripts,
a.binaries,
a.zipfiles,
a.datas,
name='borg.exe',
debug=False,
strip=False,
upx=True,
console=True )

View File

@ -1528,7 +1528,9 @@ class Archiver:
help='start repository server process')
subparser.set_defaults(func=self.do_serve)
subparser.add_argument('--restrict-to-path', dest='restrict_to_paths', action='append',
metavar='PATH', help='restrict repository access to PATH')
metavar='PATH', help='restrict repository access to PATH. '
'Can be specified multiple times to allow the client access to several directories. '
'Access to all sub-directories is granted implicitly; PATH doesn\'t need to directly point to a repository.')
subparser.add_argument('--append-only', dest='append_only', action='store_true',
help='only allow appending to repository segment files')
init_epilog = textwrap.dedent("""

View File

@ -29,8 +29,11 @@ FileCacheEntry = namedtuple('FileCacheEntry', 'age inode size mtime chunk_ids')
class Cache:
"""Client Side cache
"""
class RepositoryIDNotUnique(Error):
"""Cache is newer than repository - do you have multiple, independently updated repos with same ID?"""
class RepositoryReplay(Error):
"""Cache is newer than repository, refusing to continue"""
"""Cache is newer than repository - this is either an attack or unsafe (multiple repos with same ID)"""
class CacheInitAbortedError(Error):
"""Cache initialization aborted"""
@ -92,10 +95,16 @@ class Cache:
if not yes(msg, false_msg="Aborting.", invalid_msg="Invalid answer, aborting.",
retry=False, env_var_override='BORG_RELOCATED_REPO_ACCESS_IS_OK'):
raise self.RepositoryAccessAborted()
# adapt on-disk config immediately if the new location was accepted
self.begin_txn()
self.commit()
if sync and self.manifest.id != self.manifest_id:
# If repository is older than the cache something fishy is going on
if self.timestamp and self.timestamp > manifest.timestamp:
if isinstance(key, PlaintextKey):
raise self.RepositoryIDNotUnique()
else:
raise self.RepositoryReplay()
# Make sure an encrypted repository has not been swapped for an unencrypted repository
if self.key_type is not None and self.key_type != str(key.TYPE):

View File

@ -68,7 +68,7 @@ class ErrorWithTraceback(Error):
class IntegrityError(ErrorWithTraceback):
"""Data integrity error"""
"""Data integrity error: {}"""
class ExtensionModuleError(Error):
@ -866,20 +866,49 @@ class Location:
"""Object representing a repository / archive location
"""
proto = user = host = port = path = archive = None
# path must not contain :: (it ends at :: or string end), but may contain single colons.
# to avoid ambiguities with other regexes, it must also not start with ":".
path_re = r"""
(?!:) # not starting with ":"
(?P<path>([^:]|(:(?!:)))+) # any chars, but no "::"
"""
# optional ::archive_name at the end, archive name must not contain "/".
# borg mount's FUSE filesystem creates one level of directories from
# the archive names. Thus, we must not accept "/" in archive names.
ssh_re = re.compile(r'(?P<proto>ssh)://(?:(?P<user>[^@]+)@)?'
r'(?P<host>[^:/#]+)(?::(?P<port>\d+))?'
r'(?P<path>[^:]+)(?:::(?P<archive>[^/]+))?$')
file_re = re.compile(r'(?P<proto>file)://'
r'(?P<path>[^:]+)(?:::(?P<archive>[^/]+))?$')
scp_re = re.compile(r'((?:(?P<user>[^@]+)@)?(?P<host>[^:/]+):)?'
r'(?P<path>[^:]+)(?:::(?P<archive>[^/]+))?$')
# get the repo from BORG_RE env and the optional archive from param.
# the archive names and of course "/" is not valid in a directory name.
optional_archive_re = r"""
(?:
:: # "::" as separator
(?P<archive>[^/]+) # archive name must not contain "/"
)?$""" # must match until the end
# regexes for misc. kinds of supported location specifiers:
ssh_re = re.compile(r"""
(?P<proto>ssh):// # ssh://
(?:(?P<user>[^@]+)@)? # user@ (optional)
(?P<host>[^:/]+)(?::(?P<port>\d+))? # host or host:port
""" + path_re + optional_archive_re, re.VERBOSE) # path or path::archive
file_re = re.compile(r"""
(?P<proto>file):// # file://
""" + path_re + optional_archive_re, re.VERBOSE) # path or path::archive
# note: scp_re is also use for local pathes
scp_re = re.compile(r"""
(
(?:(?P<user>[^@]+)@)? # user@ (optional)
(?P<host>[^:/]+): # host: (don't match / in host to disambiguate from file:)
)? # user@host: part is optional
""" + path_re + optional_archive_re, re.VERBOSE) # path with optional archive
# get the repo from BORG_REPO env and the optional archive from param.
# if the syntax requires giving REPOSITORY (see "borg mount"),
# use "::" to let it use the env var.
# if REPOSITORY argument is optional, it'll automatically use the env.
env_re = re.compile(r'(?:::(?P<archive>[^/]+)?)?$')
env_re = re.compile(r""" # the repo part is fetched from BORG_REPO
(?:::$) # just "::" is ok (when a pos. arg is required, no archive)
| # or
""" + optional_archive_re, re.VERBOSE) # archive name (optional, may be empty)
def __init__(self, text=''):
self.orig = text

View File

@ -113,7 +113,7 @@ class KeyBase:
if id:
id_computed = self.id_hash(data)
if not compare_digest(id_computed, id):
raise IntegrityError('Chunk id verification failed')
raise IntegrityError('Chunk %s: id verification failed' % bin_to_hex(id))
class PlaintextKey(KeyBase):
@ -140,7 +140,7 @@ class PlaintextKey(KeyBase):
def decrypt(self, id, data, decompress=True):
if data[0] != self.TYPE:
raise IntegrityError('Invalid encryption envelope')
raise IntegrityError('Chunk %s: Invalid encryption envelope' % bin_to_hex(id))
payload = memoryview(data)[1:]
if not decompress:
return Chunk(payload)
@ -180,12 +180,12 @@ class AESKeyBase(KeyBase):
def decrypt(self, id, data, decompress=True):
if not (data[0] == self.TYPE or
data[0] == PassphraseKey.TYPE and isinstance(self, RepoKey)):
raise IntegrityError('Invalid encryption envelope')
raise IntegrityError('Chunk %s: Invalid encryption envelope' % bin_to_hex(id))
data_view = memoryview(data)
hmac_given = data_view[1:33]
hmac_computed = memoryview(hmac_sha256(self.enc_hmac_key, data_view[33:]))
if not compare_digest(hmac_computed, hmac_given):
raise IntegrityError('Encryption envelope checksum mismatch')
raise IntegrityError('Chunk %s: Encryption envelope checksum mismatch' % bin_to_hex(id))
self.dec_cipher.reset(iv=PREFIX + data[33:41])
payload = self.dec_cipher.decrypt(data_view[41:])
if not decompress:
@ -197,7 +197,7 @@ class AESKeyBase(KeyBase):
def extract_nonce(self, payload):
if not (payload[0] == self.TYPE or
payload[0] == PassphraseKey.TYPE and isinstance(self, RepoKey)):
raise IntegrityError('Invalid encryption envelope')
raise IntegrityError('Manifest: Invalid encryption envelope')
nonce = bytes_to_long(payload[33:41])
return nonce

View File

@ -1,10 +1,10 @@
from binascii import hexlify, unhexlify, a2b_base64, b2a_base64
from binascii import unhexlify, a2b_base64, b2a_base64
import binascii
import textwrap
from hashlib import sha256
from .key import KeyfileKey, RepoKey, PassphraseKey, KeyfileNotFoundError, PlaintextKey
from .helpers import Manifest, NoManifestError, Error, yes
from .helpers import Manifest, NoManifestError, Error, yes, bin_to_hex
from .repository import Repository
@ -79,7 +79,7 @@ class KeyManager:
def store_keyfile(self, target):
with open(target, 'w') as fd:
fd.write('%s %s\n' % (KeyfileKey.FILE_ID, hexlify(self.repository.id).decode('ascii')))
fd.write('%s %s\n' % (KeyfileKey.FILE_ID, bin_to_hex(self.repository.id)))
fd.write(self.keyblob)
if not self.keyblob.endswith('\n'):
fd.write('\n')
@ -103,7 +103,7 @@ class KeyManager:
binary = a2b_base64(self.keyblob)
export += 'BORG PAPER KEY v1\n'
lines = (len(binary) + 17) // 18
repoid = hexlify(self.repository.id).decode('ascii')[:18]
repoid = bin_to_hex(self.repository.id)[:18]
complete_checksum = sha256_truncated(binary, 12)
export += 'id: {0:d} / {1} / {2} - {3}\n'.format(lines,
grouped(repoid),
@ -114,7 +114,7 @@ class KeyManager:
idx += 1
binline = binary[:18]
checksum = sha256_truncated(idx.to_bytes(2, byteorder='big') + binline, 2)
export += '{0:2d}: {1} - {2}\n'.format(idx, grouped(hexlify(binline).decode('ascii')), checksum)
export += '{0:2d}: {1} - {2}\n'.format(idx, grouped(bin_to_hex(binline)), checksum)
binary = binary[18:]
if path:
@ -125,7 +125,7 @@ class KeyManager:
def import_keyfile(self, args):
file_id = KeyfileKey.FILE_ID
first_line = file_id + ' ' + hexlify(self.repository.id).decode('ascii') + '\n'
first_line = file_id + ' ' + bin_to_hex(self.repository.id) + '\n'
with open(args.path, 'r') as fd:
file_first_line = fd.read(len(first_line))
if file_first_line != first_line:
@ -141,7 +141,7 @@ class KeyManager:
# imported here because it has global side effects
import readline
repoid = hexlify(self.repository.id).decode('ascii')[:18]
repoid = bin_to_hex(self.repository.id)[:18]
try:
while True: # used for repeating on overall checksum mismatch
# id line input

View File

@ -183,11 +183,9 @@ class BaseTestCase(unittest.TestCase):
self._assert_dirs_equal_cmp(sub_diff)
@contextmanager
def fuse_mount(self, location, mountpoint, mount_options=None):
def fuse_mount(self, location, mountpoint, *options):
os.mkdir(mountpoint)
args = ['mount', location, mountpoint]
if mount_options:
args += '-o', mount_options
args = ['mount', location, mountpoint] + list(options)
self.cmd(*args, fork=True)
self.wait_for_mount(mountpoint)
yield

View File

@ -1,4 +1,4 @@
from binascii import hexlify, unhexlify, b2a_base64
from binascii import unhexlify, b2a_base64
from configparser import ConfigParser
import errno
import os
@ -1473,7 +1473,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
self.cmd('create', self.repository_location + '::archive2', 'input')
mountpoint = os.path.join(self.tmpdir, 'mountpoint')
# mount the whole repository, archive contents shall show up in versioned view:
with self.fuse_mount(self.repository_location, mountpoint, 'versions'):
with self.fuse_mount(self.repository_location, mountpoint, '-o', 'versions'):
path = os.path.join(mountpoint, 'input', 'test') # filename shows up as directory ...
files = os.listdir(path)
assert all(f.startswith('test.') for f in files) # ... with files test.xxxxxxxx in there
@ -1505,7 +1505,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
with pytest.raises(OSError) as excinfo:
open(os.path.join(mountpoint, path))
assert excinfo.value.errno == errno.EIO
with self.fuse_mount(self.repository_location + '::archive', mountpoint, 'allow_damaged_files'):
with self.fuse_mount(self.repository_location + '::archive', mountpoint, '-o', 'allow_damaged_files'):
open(os.path.join(mountpoint, path)).close()
def verify_aes_counter_uniqueness(self, method):
@ -1835,7 +1835,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
with open(export_file, 'r') as fd:
export_contents = fd.read()
assert export_contents.startswith('BORG_KEY ' + hexlify(repo_id).decode() + '\n')
assert export_contents.startswith('BORG_KEY ' + bin_to_hex(repo_id) + '\n')
key_file = self.keys_path + '/' + os.listdir(self.keys_path)[0]
@ -1862,7 +1862,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
with open(export_file, 'r') as fd:
export_contents = fd.read()
assert export_contents.startswith('BORG_KEY ' + hexlify(repo_id).decode() + '\n')
assert export_contents.startswith('BORG_KEY ' + bin_to_hex(repo_id) + '\n')
with Repository(self.repository_path) as repository:
repo_key = RepoKey(repository)

View File

@ -54,6 +54,8 @@ class TestLocationWithoutEnv:
"Location(proto='ssh', user='user', host='host', port=1234, path='/some/path', archive='archive')"
assert repr(Location('ssh://user@host:1234/some/path')) == \
"Location(proto='ssh', user='user', host='host', port=1234, path='/some/path', archive=None)"
assert repr(Location('ssh://user@host/some/path')) == \
"Location(proto='ssh', user='user', host='host', port=None, path='/some/path', archive=None)"
def test_file(self, monkeypatch):
monkeypatch.delenv('BORG_REPO', raising=False)
@ -90,6 +92,15 @@ class TestLocationWithoutEnv:
assert repr(Location('some/relative/path')) == \
"Location(proto='file', user=None, host=None, port=None, path='some/relative/path', archive=None)"
def test_with_colons(self, monkeypatch):
monkeypatch.delenv('BORG_REPO', raising=False)
assert repr(Location('/abs/path:w:cols::arch:col')) == \
"Location(proto='file', user=None, host=None, port=None, path='/abs/path:w:cols', archive='arch:col')"
assert repr(Location('/abs/path:with:colons::archive')) == \
"Location(proto='file', user=None, host=None, port=None, path='/abs/path:with:colons', archive='archive')"
assert repr(Location('/abs/path:with:colons')) == \
"Location(proto='file', user=None, host=None, port=None, path='/abs/path:with:colons', archive=None)"
def test_underspecified(self, monkeypatch):
monkeypatch.delenv('BORG_REPO', raising=False)
with pytest.raises(ValueError):
@ -99,11 +110,6 @@ class TestLocationWithoutEnv:
with pytest.raises(ValueError):
Location()
def test_no_double_colon(self, monkeypatch):
monkeypatch.delenv('BORG_REPO', raising=False)
with pytest.raises(ValueError):
Location('ssh://localhost:22/path:archive')
def test_no_slashes(self, monkeypatch):
monkeypatch.delenv('BORG_REPO', raising=False)
with pytest.raises(ValueError):
@ -134,6 +140,8 @@ class TestLocationWithEnv:
monkeypatch.setenv('BORG_REPO', 'ssh://user@host:1234/some/path')
assert repr(Location('::archive')) == \
"Location(proto='ssh', user='user', host='host', port=1234, path='/some/path', archive='archive')"
assert repr(Location('::')) == \
"Location(proto='ssh', user='user', host='host', port=1234, path='/some/path', archive=None)"
assert repr(Location()) == \
"Location(proto='ssh', user='user', host='host', port=1234, path='/some/path', archive=None)"
@ -141,6 +149,8 @@ class TestLocationWithEnv:
monkeypatch.setenv('BORG_REPO', 'file:///some/path')
assert repr(Location('::archive')) == \
"Location(proto='file', user=None, host=None, port=None, path='/some/path', archive='archive')"
assert repr(Location('::')) == \
"Location(proto='file', user=None, host=None, port=None, path='/some/path', archive=None)"
assert repr(Location()) == \
"Location(proto='file', user=None, host=None, port=None, path='/some/path', archive=None)"
@ -148,6 +158,8 @@ class TestLocationWithEnv:
monkeypatch.setenv('BORG_REPO', 'user@host:/some/path')
assert repr(Location('::archive')) == \
"Location(proto='ssh', user='user', host='host', port=None, path='/some/path', archive='archive')"
assert repr(Location('::')) == \
"Location(proto='ssh', user='user', host='host', port=None, path='/some/path', archive=None)"
assert repr(Location()) == \
"Location(proto='ssh', user='user', host='host', port=None, path='/some/path', archive=None)"
@ -155,6 +167,8 @@ class TestLocationWithEnv:
monkeypatch.setenv('BORG_REPO', 'path')
assert repr(Location('::archive')) == \
"Location(proto='file', user=None, host=None, port=None, path='path', archive='archive')"
assert repr(Location('::')) == \
"Location(proto='file', user=None, host=None, port=None, path='path', archive=None)"
assert repr(Location()) == \
"Location(proto='file', user=None, host=None, port=None, path='path', archive=None)"
@ -162,6 +176,8 @@ class TestLocationWithEnv:
monkeypatch.setenv('BORG_REPO', '/some/absolute/path')
assert repr(Location('::archive')) == \
"Location(proto='file', user=None, host=None, port=None, path='/some/absolute/path', archive='archive')"
assert repr(Location('::')) == \
"Location(proto='file', user=None, host=None, port=None, path='/some/absolute/path', archive=None)"
assert repr(Location()) == \
"Location(proto='file', user=None, host=None, port=None, path='/some/absolute/path', archive=None)"
@ -169,9 +185,20 @@ class TestLocationWithEnv:
monkeypatch.setenv('BORG_REPO', 'some/relative/path')
assert repr(Location('::archive')) == \
"Location(proto='file', user=None, host=None, port=None, path='some/relative/path', archive='archive')"
assert repr(Location('::')) == \
"Location(proto='file', user=None, host=None, port=None, path='some/relative/path', archive=None)"
assert repr(Location()) == \
"Location(proto='file', user=None, host=None, port=None, path='some/relative/path', archive=None)"
def test_with_colons(self, monkeypatch):
monkeypatch.setenv('BORG_REPO', '/abs/path:w:cols')
assert repr(Location('::arch:col')) == \
"Location(proto='file', user=None, host=None, port=None, path='/abs/path:w:cols', archive='arch:col')"
assert repr(Location('::')) == \
"Location(proto='file', user=None, host=None, port=None, path='/abs/path:w:cols', archive=None)"
assert repr(Location()) == \
"Location(proto='file', user=None, host=None, port=None, path='/abs/path:w:cols', archive=None)"
def test_no_slashes(self, monkeypatch):
monkeypatch.setenv('BORG_REPO', '/some/absolute/path')
with pytest.raises(ValueError):

View File

@ -157,7 +157,7 @@ class TestKey:
data = bytearray(data)
data[offset] += 1
with pytest.raises(IntegrityError):
key.decrypt("", data)
key.decrypt(b'', data)
def test_decrypt_integrity(self, monkeypatch, keys_dir):
with keys_dir.join('keyfile').open('w') as fd: