From 9be32e97b120d889e41fddb7258fa2928118b01d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 15 Jun 2016 12:22:29 +0200 Subject: [PATCH 01/16] fix sparse file test, fixes #1170 found out that xfs is doing stuff behind the scenes: it is pre-allocating 16MB to prevent fragmentation (in my case, value depends on misc factors). fixed the test so it just checks that the extracted sparse file uses less (not necessary much less) space than a non-sparse file would use. another problem showed up when i tried to verify the holes in the sparse file via SEEK_HOLE, SEEK_DATA: after the few bytes of real data in the file, there was another 16MB preallocated space. So I ended up checking just the hole at the start of the file. tested on: ext4, xfs, zfs, btrfs --- borg/testsuite/archiver.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 6a89213e..a4548628 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -412,8 +412,21 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.assert_equal(fd.read(hole_size), b'\0' * hole_size) st = os.stat(filename) self.assert_equal(st.st_size, total_len) - if sparse_support and hasattr(st, 'st_blocks'): - self.assert_true(st.st_blocks * 512 < total_len / 9) # is output sparse? + if sparse_support: + if hasattr(st, 'st_blocks'): + # do only check if it is less, do NOT check if it is much less + # as that causes troubles on xfs and zfs: + self.assert_true(st.st_blocks * 512 < total_len) + if hasattr(os, 'SEEK_HOLE') and hasattr(os, 'SEEK_DATA'): + with open(filename, 'rb') as fd: + # only check if the first hole is as expected, because the 2nd hole check + # is problematic on xfs due to its "dynamic speculative EOF preallocation + try: + self.assert_equal(fd.seek(0, os.SEEK_HOLE), 0) + self.assert_equal(fd.seek(0, os.SEEK_DATA), hole_size) + except OSError: + # does not really support SEEK_HOLE/SEEK_DATA + pass def test_unusual_filenames(self): filenames = ['normal', 'with some blanks', '(with_parens)', ] From a19c9861bfad8b75fd3261046e5b9d9c27f8abcf Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 20 Jun 2016 23:40:30 +0200 Subject: [PATCH 02/16] crypto: use pointer to cipher context this is one of the steps needed to make borg compatible to openssl 1.1. --- borg/crypto.pyx | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/borg/crypto.pyx b/borg/crypto.pyx index 172fe074..0d408bb7 100644 --- a/borg/crypto.pyx +++ b/borg/crypto.pyx @@ -21,8 +21,8 @@ cdef extern from "openssl/evp.h": ctypedef struct ENGINE: pass const EVP_CIPHER *EVP_aes_256_ctr() - void EVP_CIPHER_CTX_init(EVP_CIPHER_CTX *a) - void EVP_CIPHER_CTX_cleanup(EVP_CIPHER_CTX *a) + EVP_CIPHER_CTX *EVP_CIPHER_CTX_new() + void EVP_CIPHER_CTX_free(EVP_CIPHER_CTX *a) int EVP_EncryptInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, ENGINE *impl, const unsigned char *key, const unsigned char *iv) @@ -56,24 +56,24 @@ def num_aes_blocks(int length): cdef class AES: """A thin wrapper around the OpenSSL EVP cipher API """ - cdef EVP_CIPHER_CTX ctx + cdef EVP_CIPHER_CTX *ctx cdef int is_encrypt def __cinit__(self, is_encrypt, key, iv=None): - EVP_CIPHER_CTX_init(&self.ctx) + self.ctx = EVP_CIPHER_CTX_new() self.is_encrypt = is_encrypt # Set cipher type and mode cipher_mode = EVP_aes_256_ctr() if self.is_encrypt: - if not EVP_EncryptInit_ex(&self.ctx, cipher_mode, NULL, NULL, NULL): + if not EVP_EncryptInit_ex(self.ctx, cipher_mode, NULL, NULL, NULL): raise Exception('EVP_EncryptInit_ex failed') else: # decrypt - if not EVP_DecryptInit_ex(&self.ctx, cipher_mode, NULL, NULL, NULL): + if not EVP_DecryptInit_ex(self.ctx, cipher_mode, NULL, NULL, NULL): raise Exception('EVP_DecryptInit_ex failed') self.reset(key, iv) def __dealloc__(self): - EVP_CIPHER_CTX_cleanup(&self.ctx) + EVP_CIPHER_CTX_free(self.ctx) def reset(self, key=None, iv=None): cdef const unsigned char *key2 = NULL @@ -84,10 +84,10 @@ cdef class AES: iv2 = iv # Initialise key and IV if self.is_encrypt: - if not EVP_EncryptInit_ex(&self.ctx, NULL, NULL, key2, iv2): + if not EVP_EncryptInit_ex(self.ctx, NULL, NULL, key2, iv2): raise Exception('EVP_EncryptInit_ex failed') else: # decrypt - if not EVP_DecryptInit_ex(&self.ctx, NULL, NULL, key2, iv2): + if not EVP_DecryptInit_ex(self.ctx, NULL, NULL, key2, iv2): raise Exception('EVP_DecryptInit_ex failed') @property @@ -103,10 +103,10 @@ cdef class AES: if not out: raise MemoryError try: - if not EVP_EncryptUpdate(&self.ctx, out, &outl, data, inl): + if not EVP_EncryptUpdate(self.ctx, out, &outl, data, inl): raise Exception('EVP_EncryptUpdate failed') ctl = outl - if not EVP_EncryptFinal_ex(&self.ctx, out+ctl, &outl): + if not EVP_EncryptFinal_ex(self.ctx, out+ctl, &outl): raise Exception('EVP_EncryptFinal failed') ctl += outl return out[:ctl] @@ -124,10 +124,10 @@ cdef class AES: if not out: raise MemoryError try: - if not EVP_DecryptUpdate(&self.ctx, out, &outl, data, inl): + if not EVP_DecryptUpdate(self.ctx, out, &outl, data, inl): raise Exception('EVP_DecryptUpdate failed') ptl = outl - if EVP_DecryptFinal_ex(&self.ctx, out+ptl, &outl) <= 0: + if EVP_DecryptFinal_ex(self.ctx, out+ptl, &outl) <= 0: # this error check is very important for modes with padding or # authentication. for them, a failure here means corrupted data. # CTR mode does not use padding nor authentication. From 27c0d0f0743c933b5c6efde72cffc0e8f0f05dcd Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 21 Jun 2016 20:06:36 +0200 Subject: [PATCH 03/16] move attic test dependency into own file so you can just empty that file to remove the attic-based tests when testing in a OpenSSL 1.1 environment. --- requirements.d/attic.txt | 5 +++++ tox.ini | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 requirements.d/attic.txt diff --git a/requirements.d/attic.txt b/requirements.d/attic.txt new file mode 100644 index 00000000..b5068ffd --- /dev/null +++ b/requirements.d/attic.txt @@ -0,0 +1,5 @@ +# Please note: +# attic only builds using OpenSSL 1.0.x, it can not be installed using OpenSSL >= 1.1.0. +# If attic is not installed, our unit tests will just skip the tests that require attic. +attic + diff --git a/tox.ini b/tox.ini index 0473cb27..c7b49bca 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ envlist = py{34,35},flake8 changedir = {toxworkdir} deps = -rrequirements.d/development.txt - attic + -rrequirements.d/attic.txt commands = py.test --cov=borg --cov-config=../.coveragerc --benchmark-skip --pyargs {posargs:borg.testsuite} # fakeroot -u needs some env vars: passenv = * From b5362fa5c88fb24d039b7d4293d63824189864c3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 21 Jun 2016 20:20:48 +0200 Subject: [PATCH 04/16] make borg build/work on OpenSSL 1.0 and 1.1, fixes #1187 in openssl 1.1, the cipher context is opaque, members can not be accessed directly. we only used this for ctx.iv to determine the current IV (counter value). now, we just remember the original IV, count the AES blocks we process and then compute iv = iv_orig + blocks. that way, it works on OpenSSL 1.0.x and >= 1.1 in the same way. --- borg/crypto.pyx | 39 +++++++++++++++++++++++++++++++++++++-- borg/testsuite/crypto.py | 22 ++++++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/borg/crypto.pyx b/borg/crypto.pyx index 0d408bb7..556f0fbc 100644 --- a/borg/crypto.pyx +++ b/borg/crypto.pyx @@ -16,7 +16,6 @@ cdef extern from "openssl/evp.h": ctypedef struct EVP_CIPHER: pass ctypedef struct EVP_CIPHER_CTX: - unsigned char *iv pass ctypedef struct ENGINE: pass @@ -40,12 +39,40 @@ import struct _int = struct.Struct('>I') _long = struct.Struct('>Q') +_2long = struct.Struct('>QQ') bytes_to_int = lambda x, offset=0: _int.unpack_from(x, offset)[0] bytes_to_long = lambda x, offset=0: _long.unpack_from(x, offset)[0] long_to_bytes = lambda x: _long.pack(x) +def bytes16_to_int(b, offset=0): + h, l = _2long.unpack_from(b, offset) + return (h << 64) + l + + +def int_to_bytes16(i): + max_uint64 = 0xffffffffffffffff + l = i & max_uint64 + h = (i >> 64) & max_uint64 + return _2long.pack(h, l) + + +def increment_iv(iv, amount=1): + """ + Increment the IV by the given amount (default 1). + + :param iv: input IV, 16 bytes (128 bit) + :param amount: increment value + :return: input_IV + amount, 16 bytes (128 bit) + """ + assert len(iv) == 16 + iv = bytes16_to_int(iv) + iv += amount + iv = int_to_bytes16(iv) + return iv + + def num_aes_blocks(int length): """Return the number of AES blocks required to encrypt/decrypt *length* bytes of data. Note: this is only correct for modes without padding, like AES-CTR. @@ -58,6 +85,8 @@ cdef class AES: """ cdef EVP_CIPHER_CTX *ctx cdef int is_encrypt + cdef unsigned char iv_orig[16] + cdef int blocks def __cinit__(self, is_encrypt, key, iv=None): self.ctx = EVP_CIPHER_CTX_new() @@ -82,6 +111,10 @@ cdef class AES: key2 = key if iv: iv2 = iv + assert isinstance(iv, bytes) and len(iv) == 16 + for i in range(16): + self.iv_orig[i] = iv[i] + self.blocks = 0 # number of AES blocks encrypted starting with iv_orig # Initialise key and IV if self.is_encrypt: if not EVP_EncryptInit_ex(self.ctx, NULL, NULL, key2, iv2): @@ -92,7 +125,7 @@ cdef class AES: @property def iv(self): - return self.ctx.iv[:16] + return increment_iv(self.iv_orig[:16], self.blocks) def encrypt(self, data): cdef int inl = len(data) @@ -109,6 +142,7 @@ cdef class AES: if not EVP_EncryptFinal_ex(self.ctx, out+ctl, &outl): raise Exception('EVP_EncryptFinal failed') ctl += outl + self.blocks += num_aes_blocks(ctl) return out[:ctl] finally: free(out) @@ -133,6 +167,7 @@ cdef class AES: # CTR mode does not use padding nor authentication. raise Exception('EVP_DecryptFinal failed') ptl += outl + self.blocks += num_aes_blocks(inl) return out[:ptl] finally: free(out) diff --git a/borg/testsuite/crypto.py b/borg/testsuite/crypto.py index 2d74493d..c6810194 100644 --- a/borg/testsuite/crypto.py +++ b/borg/testsuite/crypto.py @@ -1,6 +1,7 @@ from binascii import hexlify from ..crypto import AES, bytes_to_long, bytes_to_int, long_to_bytes +from ..crypto import increment_iv, bytes16_to_int, int_to_bytes16 from . import BaseTestCase @@ -13,6 +14,27 @@ class CryptoTestCase(BaseTestCase): self.assert_equal(bytes_to_long(b'\0\0\0\0\0\0\0\1'), 1) self.assert_equal(long_to_bytes(1), b'\0\0\0\0\0\0\0\1') + def test_bytes16_to_int(self): + self.assert_equal(bytes16_to_int(b'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1'), 1) + self.assert_equal(int_to_bytes16(1), b'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1') + self.assert_equal(bytes16_to_int(b'\0\0\0\0\0\0\0\1\0\0\0\0\0\0\0\0'), 2 ** 64) + self.assert_equal(int_to_bytes16(2 ** 64), b'\0\0\0\0\0\0\0\1\0\0\0\0\0\0\0\0') + + def test_increment_iv(self): + iv0 = b'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0' + iv1 = b'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1' + iv2 = b'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\2' + self.assert_equal(increment_iv(iv0, 0), iv0) + self.assert_equal(increment_iv(iv0, 1), iv1) + self.assert_equal(increment_iv(iv0, 2), iv2) + iva = b'\0\0\0\0\0\0\0\0\xff\xff\xff\xff\xff\xff\xff\xff' + ivb = b'\0\0\0\0\0\0\0\1\x00\x00\x00\x00\x00\x00\x00\x00' + ivc = b'\0\0\0\0\0\0\0\1\x00\x00\x00\x00\x00\x00\x00\x01' + self.assert_equal(increment_iv(iva, 0), iva) + self.assert_equal(increment_iv(iva, 1), ivb) + self.assert_equal(increment_iv(iva, 2), ivc) + self.assert_equal(increment_iv(iv0, 2**64), ivb) + def test_aes(self): key = b'X' * 32 data = b'foo' * 10 From 0f7eb871fd216d317252b88e75e50e4b3ea3d9fb Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 21 Jun 2016 21:52:20 +0200 Subject: [PATCH 05/16] slightly refactor placeholder related code - move from instance method to global function, so it can be used in other contexts also - rename preformat_text -> replace_placeholders --- borg/helpers.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index fad401a5..90f8f3a3 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -566,6 +566,20 @@ def format_line(format, data): return '' +def replace_placeholders(text): + """Replace placeholders in text with their values.""" + current_time = datetime.now() + data = { + 'pid': os.getpid(), + 'fqdn': socket.getfqdn(), + 'hostname': socket.gethostname(), + 'now': current_time.now(), + 'utcnow': current_time.utcnow(), + 'user': uid2user(os.getuid(), os.getuid()) + } + return format_line(text, data) + + def safe_timestamp(item_timestamp_ns): try: return datetime.fromtimestamp(bigint_to_int(item_timestamp_ns) / 1e9) @@ -720,21 +734,8 @@ class Location: if not self.parse(self.orig): raise ValueError - def preformat_text(self, text): - """Format repository and archive path with common tags""" - current_time = datetime.now() - data = { - 'pid': os.getpid(), - 'fqdn': socket.getfqdn(), - 'hostname': socket.gethostname(), - 'now': current_time.now(), - 'utcnow': current_time.utcnow(), - 'user': uid2user(os.getuid(), os.getuid()) - } - return format_line(text, data) - def parse(self, text): - text = self.preformat_text(text) + text = replace_placeholders(text) valid = self._parse(text) if valid: return True From 52007dbd43cad12ba4daf619d41d8e94a8e215ea Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 21 Jun 2016 22:02:13 +0200 Subject: [PATCH 06/16] add tests for format_line --- borg/testsuite/helpers.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index cdb96b96..15cd54a5 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -10,7 +10,7 @@ import msgpack import msgpack.fallback import time -from ..helpers import Location, format_file_size, format_timedelta, make_path_safe, \ +from ..helpers import Location, format_file_size, format_timedelta, format_line, make_path_safe, \ prune_within, prune_split, get_cache_dir, get_keys_dir, Statistics, is_slow_msgpack, \ yes, TRUISH, FALSISH, DEFAULTISH, \ StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \ @@ -877,3 +877,15 @@ def test_progress_endless_step(capfd): pi.show() out, err = capfd.readouterr() assert err == '.' + + +def test_format_line(): + data = dict(foo='bar baz') + assert format_line('', data) == '' + assert format_line('{foo}', data) == 'bar baz' + assert format_line('foo{foo}foo', data) == 'foobar bazfoo' + + +def test_format_line_erroneous(): + data = dict(foo='bar baz') + assert format_line('{invalid}', data) == '' # TODO: rather raise exception From ad1729401f1c33024a8a14aa9af893a2e4c9e257 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 21 Jun 2016 22:46:12 +0200 Subject: [PATCH 07/16] improve exception handling for placeholder replacement do not ignore bad placeholders and just return empty string, this could have bad consequences, e.g. with --prefix '{invalidplaceholder}': a typo in the placeholder name would cause the prefix to be the empty string. --- borg/helpers.py | 14 +++++--------- borg/testsuite/helpers.py | 9 ++++++--- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index 90f8f3a3..de609c8c 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -69,6 +69,10 @@ class NoManifestError(Error): """Repository has no manifest.""" +class PlaceholderError(Error): + """Formatting Error: "{}".format({}): {}({})""" + + def check_extension_modules(): from . import platform if hashindex.API_VERSION != 2: @@ -552,18 +556,10 @@ def dir_is_tagged(path, exclude_caches, exclude_if_present): def format_line(format, data): - # TODO: Filter out unwanted properties of str.format(), because "format" is user provided. - try: return format.format(**data) - except (KeyError, ValueError) as e: - # this should catch format errors - print('Error in lineformat: "{}" - reason "{}"'.format(format, str(e))) except Exception as e: - # something unexpected, print error and raise exception - print('Error in lineformat: "{}" - reason "{}"'.format(format, str(e))) - raise - return '' + raise PlaceholderError(format, data, e.__class__.__name__, str(e)) def replace_placeholders(text): diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index 15cd54a5..35993ef1 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -10,7 +10,7 @@ import msgpack import msgpack.fallback import time -from ..helpers import Location, format_file_size, format_timedelta, format_line, make_path_safe, \ +from ..helpers import Location, format_file_size, format_timedelta, format_line, PlaceholderError, make_path_safe, \ prune_within, prune_split, get_cache_dir, get_keys_dir, Statistics, is_slow_msgpack, \ yes, TRUISH, FALSISH, DEFAULTISH, \ StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \ @@ -887,5 +887,8 @@ def test_format_line(): def test_format_line_erroneous(): - data = dict(foo='bar baz') - assert format_line('{invalid}', data) == '' # TODO: rather raise exception + data = dict() + with pytest.raises(PlaceholderError): + assert format_line('{invalid}', data) + with pytest.raises(PlaceholderError): + assert format_line('{}', data) From b630ae92318f7566ef396bfb4356fe1e3646eeb4 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 21 Jun 2016 23:00:08 +0200 Subject: [PATCH 08/16] catch and format exceptions in arg parsing --- borg/archiver.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index bbd6a2e4..5f094b0e 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -1522,7 +1522,15 @@ def main(): # pragma: no cover setup_signal_handlers() archiver = Archiver() msg = None - args = archiver.get_args(sys.argv, os.environ.get('SSH_ORIGINAL_COMMAND')) + try: + args = archiver.get_args(sys.argv, os.environ.get('SSH_ORIGINAL_COMMAND')) + except Error as e: + msg = e.get_message() + if e.traceback: + msg += "\n%s\n%s" % (traceback.format_exc(), sysinfo()) + # we might not have logging setup yet, so get out quickly + print(msg, file=sys.stderr) + sys.exit(e.exit_code) try: exit_code = archiver.run(args) except Error as e: From b072e993948858d7cb191bb2588c14428ef18935 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 21 Jun 2016 23:14:44 +0200 Subject: [PATCH 09/16] fix invalid placeholder in unit test --- borg/testsuite/archiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 6a89213e..7c6657f6 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -933,7 +933,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.cmd('create', test_archive, src_dir) output_1 = self.cmd('list', test_archive) output_2 = self.cmd('list', '--list-format', '{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}', test_archive) - output_3 = self.cmd('list', '--list-format', '{mtime:%s} {path}{NL}', test_archive) + output_3 = self.cmd('list', '--list-format', '{mtime:%s} {path}{NEWLINE}', test_archive) self.assertEqual(output_1, output_2) self.assertNotEqual(output_1, output_3) From 6407742d789a1dc199eba0dd29b66a80d5b8dfe2 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 21 Jun 2016 23:36:30 +0200 Subject: [PATCH 10/16] support placeholders for --prefix (everywhere), fixes #1027 this fixes a ugly inconsistency: you could use placeholder for borg create's archivename. but you could not use them for borg prune's prefix option. --- borg/archiver.py | 8 ++++---- borg/helpers.py | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 5f094b0e..ef49e858 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -20,7 +20,7 @@ from .helpers import Error, location_validator, archivename_validator, format_li parse_pattern, PathPrefixPattern, to_localtime, timestamp, safe_timestamp, \ get_cache_dir, prune_within, prune_split, \ Manifest, NoManifestError, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ - dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, is_slow_msgpack, yes, sysinfo, \ + dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, PrefixSpec, is_slow_msgpack, yes, sysinfo, \ EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, log_multi, PatternMatcher from .logger import create_logger, setup_logging logger = create_logger() @@ -952,7 +952,7 @@ class Archiver: subparser.add_argument('--last', dest='last', type=int, default=None, metavar='N', help='only check last N archives (Default: all)') - subparser.add_argument('-P', '--prefix', dest='prefix', type=str, + subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec, help='only consider archive names starting with this prefix') change_passphrase_epilog = textwrap.dedent(""" @@ -1194,7 +1194,7 @@ class Archiver: help="""specify format for archive file listing (default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}") Special "{formatkeys}" exists to list available keys""") - subparser.add_argument('-P', '--prefix', dest='prefix', type=str, + subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec, help='only consider archive names starting with this prefix') subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', nargs='?', default='', type=location_validator(), @@ -1301,7 +1301,7 @@ class Archiver: help='number of monthly archives to keep') subparser.add_argument('-y', '--keep-yearly', dest='yearly', type=int, default=0, help='number of yearly archives to keep') - subparser.add_argument('-P', '--prefix', dest='prefix', type=str, + subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec, help='only consider archive names starting with this prefix') subparser.add_argument('--save-space', dest='save_space', action='store_true', default=False, diff --git a/borg/helpers.py b/borg/helpers.py index de609c8c..7c340760 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -518,6 +518,10 @@ def CompressionSpec(s): raise ValueError +def PrefixSpec(s): + return replace_placeholders(s) + + def dir_is_cachedir(path): """Determines whether the specified path is a cache directory (and therefore should potentially be excluded from the backup) according to From 5ae340998c85c14c8b8787569bd51441727d498d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 22 Jun 2016 00:31:31 +0200 Subject: [PATCH 11/16] update docs about placeholders --- borg/archiver.py | 34 ++++++++++++++++++++++++++++++++++ docs/quickstart.rst | 6 +++--- docs/usage.rst | 5 +++-- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index ef49e858..169e8bcb 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -803,6 +803,39 @@ class Archiver: EOF $ borg create --exclude-from exclude.txt backup / ''') + helptext['placeholders'] = textwrap.dedent(''' + Repository (or Archive) URLs and --prefix values support these placeholders: + + {hostname} + + The (short) hostname of the machine. + + {fqdn} + + The full name of the machine. + + {now} + + The current local date and time. + + {utcnow} + + The current UTC date and time. + + {user} + + The user name (or UID, if no name is available) of the user running borg. + + {pid} + + The current process ID. + + Examples:: + + borg create /path/to/repo::{hostname}-{user}-{utcnow} ... + borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S} ... + borg prune --prefix '{hostname}-' ... + ''') def do_help(self, parser, commands, args): if not args.topic: @@ -1013,6 +1046,7 @@ class Archiver: all files on these file systems. See the output of the "borg help patterns" command for more help on exclude patterns. + See the output of the "borg help placeholders" command for more help on placeholders. """) subparser = subparsers.add_parser('create', parents=[common_parser], diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 1d15f5d2..81d358d4 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -110,7 +110,7 @@ certain number of old archives:: # Backup all of /home and /var/www except a few # excluded directories borg create -v --stats \ - $REPOSITORY::`hostname`-`date +%Y-%m-%d` \ + $REPOSITORY::'{hostname}-{now:%Y-%m-%d}' \ /home \ /var/www \ --exclude '/home/*/.cache' \ @@ -118,10 +118,10 @@ certain number of old archives:: --exclude '*.pyc' # Use the `prune` subcommand to maintain 7 daily, 4 weekly and 6 monthly - # archives of THIS machine. --prefix `hostname`- is very important to + # archives of THIS machine. The '{hostname}-' prefix is very important to # limit prune's operation to this machine's archives and not apply to # other machine's archives also. - borg prune -v $REPOSITORY --prefix `hostname`- \ + borg prune -v $REPOSITORY --prefix '{hostname}-' \ --keep-daily=7 --keep-weekly=4 --keep-monthly=6 .. backup_compression: diff --git a/docs/usage.rst b/docs/usage.rst index 258f9a1a..acca302a 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -425,8 +425,9 @@ will see what it would do without it actually doing anything. # Do a dry-run without actually deleting anything. $ borg prune --dry-run --keep-daily=7 --keep-weekly=4 /path/to/repo - # Same as above but only apply to archive names starting with "foo": - $ borg prune --keep-daily=7 --keep-weekly=4 --prefix=foo /path/to/repo + # Same as above but only apply to archive names starting with the hostname + # of the machine followed by a "-" character: + $ borg prune --keep-daily=7 --keep-weekly=4 --prefix='{hostname}-' /path/to/repo # Keep 7 end of day, 4 additional end of week archives, # and an end of month archive for every month: From 12cb66b9f6f87c0e2b951348fea89ac6c8870110 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 22 Jun 2016 08:44:14 +0200 Subject: [PATCH 12/16] fix "patterns" help formatting, too this way it renders nicely in html (via sphinx) and on console --- borg/archiver.py | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 169e8bcb..5cac525c 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -773,35 +773,35 @@ class Archiver: whitespace removal paths with whitespace at the beginning or end can only be excluded using regular expressions. - Examples: + Examples:: - # Exclude '/home/user/file.o' but not '/home/user/file.odt': - $ borg create -e '*.o' backup / + # Exclude '/home/user/file.o' but not '/home/user/file.odt': + $ borg create -e '*.o' backup / - # Exclude '/home/user/junk' and '/home/user/subdir/junk' but - # not '/home/user/importantjunk' or '/etc/junk': - $ borg create -e '/home/*/junk' backup / + # Exclude '/home/user/junk' and '/home/user/subdir/junk' but + # not '/home/user/importantjunk' or '/etc/junk': + $ borg create -e '/home/*/junk' backup / - # Exclude the contents of '/home/user/cache' but not the directory itself: - $ borg create -e /home/user/cache/ backup / + # Exclude the contents of '/home/user/cache' but not the directory itself: + $ borg create -e /home/user/cache/ backup / - # The file '/home/user/cache/important' is *not* backed up: - $ borg create -e /home/user/cache/ backup / /home/user/cache/important + # The file '/home/user/cache/important' is *not* backed up: + $ borg create -e /home/user/cache/ backup / /home/user/cache/important - # The contents of directories in '/home' are not backed up when their name - # ends in '.tmp' - $ borg create --exclude 're:^/home/[^/]+\.tmp/' backup / + # The contents of directories in '/home' are not backed up when their name + # ends in '.tmp' + $ borg create --exclude 're:^/home/[^/]+\.tmp/' backup / - # Load exclusions from file - $ cat >exclude.txt <exclude.txt < Date: Thu, 23 Jun 2016 13:00:05 -0400 Subject: [PATCH 13/16] Ignore empty index file. Empty index file is most likely a result from an unclean shutdown in the middle of write, e.g. on ext4 with delayed allocation enabled (default). Ignoring such a file would get it recreated by other parts of code, where as not ignoring it leads to an exception about not being able to read enough bytes from the index. this commit fixes #1195 Signed-off-by: Oleg Drokin --- borg/repository.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/borg/repository.py b/borg/repository.py index 8c645eaa..25f9ccc1 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -129,7 +129,9 @@ class Repository: shutil.rmtree(self.path) def get_index_transaction_id(self): - indices = sorted((int(name[6:]) for name in os.listdir(self.path) if name.startswith('index.') and name[6:].isdigit())) + indices = sorted(int(fn[6:]) + for fn in os.listdir(self.path) + if fn.startswith('index.') and fn[6:].isdigit() and os.stat(os.path.join(self.path, fn)).st_size != 0) if indices: return indices[-1] else: From ad65c5ac16915f0476feec0cce2bc27cc9887add Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 24 Jun 2016 21:48:21 +0200 Subject: [PATCH 14/16] support docs: add freenode web chat link, fixes #1175 --- docs/support.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/support.rst b/docs/support.rst index 1547c666..7cd1890d 100644 --- a/docs/support.rst +++ b/docs/support.rst @@ -23,6 +23,12 @@ Join us on channel #borgbackup on chat.freenode.net. As usual on IRC, just ask or tell directly and then patiently wait for replies. Stay connected. +You could use the following link (after connecting, you can change the random +nickname you got by typing "/nick mydesirednickname"): + +http://webchat.freenode.net/?randomnick=1&channels=%23borgbackup&uio=MTY9dHJ1ZSY5PXRydWUa8 + + Mailing list ------------ From f54f159db88589256b2cc75a15d969899f4ed52c Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 24 Jun 2016 22:32:38 +0200 Subject: [PATCH 15/16] more details about checkpoints, add split trick, fixes #1171 --- docs/faq.rst | 46 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 0051a48c..0d6a1185 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -180,13 +180,53 @@ Yes, |project_name| supports resuming backups. During a backup a special checkpoint archive named ``.checkpoint`` is saved every checkpoint interval (the default value for this is 5 -minutes) containing all the data backed-up until that point. This means -that at most worth of data needs to be retransmitted -if a backup needs to be restarted. +minutes) containing all the data backed-up until that point. + +Checkpoints only happen between files (so they don't help for interruptions +happening while a very large file is being processed). + +This checkpoint archive is a valid archive (all files in it are valid and complete), +but it is only a partial backup (not all files that you wanted to backup are +contained in it). Having it in the repo until a successful, full backup is +completed is useful because it references all the transmitted chunks up +to the checkpoint. This means that in case of an interruption, you only need to +retransfer the data since the last checkpoint. + +If a backup was interrupted, you do not need to do any special considerations, +just invoke ``borg create`` as you always do. You may use the same archive name +as in previous attempt or a different one (e.g. if you always include the current +datetime), it does not matter. + +|project_name| always does full single-pass backups, so it will start again +from the beginning - but it will be much faster, because some of the data was +already stored into the repo (and is still referenced by the checkpoint +archive), so it does not need to get transmitted and stored again. Once your backup has finished successfully, you can delete all ``.checkpoint`` archives. +How can I backup huge file(s) over a instable connection? +--------------------------------------------------------- + +You can use this "split trick" as a workaround for the in-between-files-only +checkpoints (see above), huge files and a instable connection to the repository: + +Split the huge file(s) into parts of manageable size (e.g. 100MB) and create +a temporary archive of them. Borg will create checkpoints now more frequently +than if you try to backup the files in their original form (e.g. 100GB). + +After that, you can remove the parts again and backup the huge file(s) in +their original form. This will now work a lot faster as a lot of content chunks +are already in the repository. + +After you have successfully backed up the huge original file(s), you can remove +the temporary archive you made from the parts. + +We realize that this is just a better-than-nothing workaround, see :issue:`1198` +for a potential solution. + +Please note that this workaround only helps you for backup, not for restore. + If it crashes with a UnicodeError, what can I do? ------------------------------------------------- From b10025c6e535a3665cdb69ee0eb39ccf1c10f2ec Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 24 Jun 2016 23:30:27 +0200 Subject: [PATCH 16/16] document sshd settings, fixes #545 --- docs/faq.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/faq.rst b/docs/faq.rst index 0d6a1185..100abd89 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -143,6 +143,24 @@ stops after a while (some minutes, hours, ... - not immediately) with That's a good question and we are trying to find a good answer in `ticket 636 `_. +Why am I seeing idle borg serve processes on the repo server? +------------------------------------------------------------- + +Maybe the ssh connection between client and server broke down and that was not +yet noticed on the server. Try these settings: + +:: + + # /etc/ssh/sshd_config on borg repo server - kill connection to client + # after ClientAliveCountMax * ClientAliveInterval seconds with no response + ClientAliveInterval 20 + ClientAliveCountMax 3 + +If you have multiple borg create ... ; borg create ... commands in a already +serialized way in a single script, you need to give them --lock-wait N (with N +being a bit more than the time the server needs to terminate broken down +connections and release the lock). + The borg cache eats way too much disk space, what can I do? -----------------------------------------------------------