From 011e0fd3faf2730681a17403e8fd575bd3ea4b08 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 3 Oct 2017 21:11:43 +0200 Subject: [PATCH 01/22] auto compression: make sure expensive compression is actually better if it is not significantly better compressed, we just store lz4 compressed data (which we already have computed anyway), because that at least decompressed super fast. --- src/borg/compress.pyx | 21 ++++++++++++++++++--- src/borg/testsuite/compress.py | 16 +++++++++++----- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/borg/compress.pyx b/src/borg/compress.pyx index 16f7bcb54..e3c34e047 100644 --- a/src/borg/compress.pyx +++ b/src/borg/compress.pyx @@ -246,7 +246,7 @@ class Auto(CompressorBase): lz4_data = self.lz4.compress(data) ratio = len(lz4_data) / len(data) if ratio < 0.97: - return self.compressor, None + return self.compressor, lz4_data elif ratio < 1: return self.lz4, lz4_data else: @@ -257,9 +257,24 @@ class Auto(CompressorBase): def compress(self, data): compressor, lz4_data = self._decide(data) - if lz4_data is None: - return compressor.compress(data) + if compressor is self.lz4: + # we know that trying to compress with expensive compressor is likely pointless, + # but lz4 managed to at least squeeze the data a bit. + return lz4_data + if compressor is self.none: + # we know that trying to compress with expensive compressor is likely pointless + # and also lz4 did not manage to squeeze the data (not even a bit). + uncompressed_data = compressor.compress(data) + return uncompressed_data + # if we get here, the decider decided to try the expensive compressor. + # we also know that lz4_data is smaller than uncompressed data. + exp_compressed_data = compressor.compress(data) + ratio = len(exp_compressed_data) / len(lz4_data) + if ratio < 0.99: + # the expensive compressor managed to squeeze the data significantly better than lz4. + return exp_compressed_data else: + # otherwise let's just store the lz4 data, which decompresses extremely fast. return lz4_data def decompress(self, data): diff --git a/src/borg/testsuite/compress.py b/src/borg/testsuite/compress.py index ee6da55a1..f881ad2c7 100644 --- a/src/borg/testsuite/compress.py +++ b/src/borg/testsuite/compress.py @@ -110,12 +110,18 @@ def test_compressor(): def test_auto(): - compressor = CompressionSpec('auto,zlib,9').compressor + compressor_auto_zlib = CompressionSpec('auto,zlib,9').compressor + compressor_lz4 = CompressionSpec('lz4').compressor + compressor_zlib = CompressionSpec('zlib,9').compressor + data = bytes(500) + compressed_auto_zlib = compressor_auto_zlib.compress(data) + compressed_lz4 = compressor_lz4.compress(data) + compressed_zlib = compressor_zlib.compress(data) + ratio = len(compressed_zlib) / len(compressed_lz4) + assert Compressor.detect(compressed_auto_zlib) == ZLIB if ratio < 0.99 else LZ4 - compressed = compressor.compress(bytes(500)) - assert Compressor.detect(compressed) == ZLIB - - compressed = compressor.compress(b'\x00\xb8\xa3\xa2-O\xe1i\xb6\x12\x03\xc21\xf3\x8a\xf78\\\x01\xa5b\x07\x95\xbeE\xf8\xa3\x9ahm\xb1~') + data = b'\x00\xb8\xa3\xa2-O\xe1i\xb6\x12\x03\xc21\xf3\x8a\xf78\\\x01\xa5b\x07\x95\xbeE\xf8\xa3\x9ahm\xb1~' + compressed = compressor_auto_zlib.compress(data) assert Compressor.detect(compressed) == CNONE From 21601d5f53a3f395c27807d2d89206cdd472a600 Mon Sep 17 00:00:00 2001 From: Timothy Burchfield Date: Tue, 3 Oct 2017 17:18:33 -0400 Subject: [PATCH 02/22] Quickstart guide minor corrections (#3084) quickstart: minor grammar correction and added consistency with substitution of 'Borg' --- docs/quickstart.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 597929eac..a7a1ba0fb 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -21,14 +21,14 @@ a good amount of free space on the filesystem that has your backup repository (and also on ~/.cache). A few GB should suffice for most hard-drive sized repositories. See also :ref:`cache-memory-usage`. -Borg doesn't use space reserved for root on repository disks (even when run as root), +|project_name| doesn't use space reserved for root on repository disks (even when run as root), on file systems which do not support this mechanism (e.g. XFS) we recommend to -reserve some space in Borg itself just to be safe by adjusting the +reserve some space in |project_name| itself just to be safe by adjusting the ``additional_free_space`` setting in the ``[repository]`` section of a repositories ``config`` file. A good starting point is ``2G``. If |project_name| runs out of disk space, it tries to free as much space as it -can while aborting the current operation safely, which allows to free more space +can while aborting the current operation safely, which allows the user to free more space by deleting/pruning archives. This mechanism is not bullet-proof in some circumstances [1]_. @@ -153,14 +153,14 @@ backed up and that the ``prune`` command is keeping and deleting the correct bac Pitfalls with shell variables and environment variables ------------------------------------------------------- -This applies to all environment variables you want borg to see, not just +This applies to all environment variables you want |project_name| to see, not just ``BORG_PASSPHRASE``. The short explanation is: always ``export`` your variable, and use single quotes if you're unsure of the details of your shell's expansion behavior. E.g.:: export BORG_PASSPHRASE='complicated & long' -This is because ``export`` exposes variables to subprocesses, which borg may be +This is because ``export`` exposes variables to subprocesses, which |project_name| may be one of. More on ``export`` can be found in the "ENVIRONMENT" section of the bash(1) man page. From fe0843733767432d8af40423fd207231f4805ce1 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 5 Oct 2017 22:19:39 +0200 Subject: [PATCH 03/22] faq: we do not implement futile attempts of ETA / progress displays --- docs/faq.rst | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/faq.rst b/docs/faq.rst index fd9c6e50e..e23f45a30 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -682,6 +682,31 @@ the corruption is caused by a one time event such as a power outage, running `borg check --repair` will fix most problems. +Why isn't there more progress / ETA information displayed? +---------------------------------------------------------- + +Some borg runs take quite a bit, so it would be nice to see a progress display, +maybe even including a ETA (expected time of "arrival" [here rather "completion"]). + +For some functionality, this can be done: if the total amount of work is more or +less known, we can display progress. So check if there is a ``--progress`` option. + +But sometimes, the total amount is unknown (e.g. for ``borg create`` we just do +a single pass over the filesystem, so we do not know the total file count or data +volume before reaching the end). Adding another pass just to determine that would +take additional time and could be incorrect, if the filesystem is changing. + +Even if the fs does not change and we knew count and size of all files, we still +could not compute the ``borg create`` ETA as we do not know the amount of changed +chunks, how the bandwidth of source and destination or system performance might +fluctuate. + +You see, trying to display ETA would be futile. The borg developers prefer to +rather not implement progress / ETA display than doing futile attempts. + +See also: https://xkcd.com/612/ + + Miscellaneous ############# From be9fb349de3eebb326f9ec973a37d18fdaad7c76 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 6 Oct 2017 19:38:28 +0200 Subject: [PATCH 04/22] check for py 3.5 minimum requirement in setup.py we do not test on 3.4 any more and have dropped support for it. so we better enforce it in setup.py also. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d73fa089f..e0aeb14f0 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ from distutils.core import Command import textwrap -min_python = (3, 4) +min_python = (3, 5) my_python = sys.version_info if my_python < min_python: From ce956fa0afb3c16665b62d9d7dc0c0139f24b131 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 6 Oct 2017 20:01:17 +0200 Subject: [PATCH 05/22] add some comments about recent fuse versions to setup.py --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index d73fa089f..4cc7febdf 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,8 @@ extras_require = { # llfuse 0.42 (tested shortly, looks ok), needs FUSE version >= 2.8.0 # llfuse 1.0 (tested shortly, looks ok), needs FUSE version >= 2.8.0 # llfuse 1.1.1 (tested shortly, looks ok), needs FUSE version >= 2.8.0 + # llfuse 1.2 (tested shortly, looks ok), needs FUSE version >= 2.8.0 + # llfuse 1.3 (tested shortly, looks ok), needs FUSE version >= 2.8.0 # llfuse 2.0 will break API 'fuse': ['llfuse<2.0', ], } From 62e0f7a64e8950863ab53c05f9bf8c4597d16e6e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 7 Oct 2017 03:34:03 +0200 Subject: [PATCH 06/22] manpage: fix typos, update homepage --- docs/man_intro.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/man_intro.rst b/docs/man_intro.rst index 85ba9bd32..44dee959e 100644 --- a/docs/man_intro.rst +++ b/docs/man_intro.rst @@ -32,8 +32,8 @@ fully trusted targets. Borg stores a set of files in an *archive*. A *repository* is a collection of *archives*. The format of repositories is Borg-specific. Borg does not -distinguish archives from each other in a any way other than their name, -it does not matter when or where archives where created (eg. different hosts). +distinguish archives from each other in any way other than their name, +it does not matter when or where archives were created (e.g. different hosts). EXAMPLES -------- @@ -61,7 +61,7 @@ SEE ALSO `borg-compression(1)`, `borg-patterns(1)`, `borg-placeholders(1)` -* Main web site https://borgbackup.readthedocs.org/ +* Main web site https://www.borgbackup.org/ * Releases https://github.com/borgbackup/borg/releases * Changelog https://github.com/borgbackup/borg/blob/master/docs/changes.rst * GitHub https://github.com/borgbackup/borg From bf3f8e567283f43f297670554d158588df38bf78 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 7 Oct 2017 04:11:29 +0200 Subject: [PATCH 07/22] implement simple "issue" role for manpage generation, fixes #3075 --- setup.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/setup.py b/setup.py index 459c994a8..618594ea6 100644 --- a/setup.py +++ b/setup.py @@ -657,6 +657,13 @@ class build_man(Command): def gen_man_page(self, name, rst): from docutils.writers import manpage from docutils.core import publish_string + from docutils.nodes import inline + from docutils.parsers.rst import roles + + def issue(name, rawtext, text, lineno, inliner, options={}, content=[]): + return [inline(rawtext, '#' + text)], [] + + roles.register_local_role('issue', issue) # We give the source_path so that docutils can find relative includes # as-if the document where located in the docs/ directory. man_page = publish_string(source=rst, source_path='docs/virtmanpage.rst', writer=manpage.Writer()) From b00179ff784fbd959d7ee1e331ac62a1c9bbc25a Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 8 Oct 2017 12:28:05 +0200 Subject: [PATCH 08/22] init: fix wrong encryption choices in command line parser --- src/borg/archiver.py | 4 ++-- src/borg/crypto/key.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 039587ddb..1845d8840 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -40,7 +40,7 @@ from .archive import FilesystemObjectProcessors, MetadataCollector, ChunksProces from .cache import Cache, assert_secure from .constants import * # NOQA from .compress import CompressionSpec -from .crypto.key import key_creator, tam_required_file, tam_required, RepoKey, PassphraseKey +from .crypto.key import key_creator, key_argument_names, tam_required_file, tam_required, RepoKey, PassphraseKey from .crypto.keymanager import KeyManager from .helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR from .helpers import Error, NoManifestError, set_ec @@ -2383,7 +2383,7 @@ class Archiver: type=location_validator(archive=False), help='repository to create') subparser.add_argument('-e', '--encryption', metavar='MODE', dest='encryption', required=True, - choices=('none', 'keyfile', 'repokey', 'keyfile-blake2', 'repokey-blake2', 'authenticated'), + choices=key_argument_names(), help='select encryption key mode **(required)**') subparser.add_argument('--append-only', dest='append_only', action='store_true', help='create an append-only mode repository') diff --git a/src/borg/crypto/key.py b/src/borg/crypto/key.py index 7fc1a5a7a..21594e810 100644 --- a/src/borg/crypto/key.py +++ b/src/borg/crypto/key.py @@ -103,11 +103,16 @@ class KeyBlobStorage: def key_creator(repository, args): for key in AVAILABLE_KEY_TYPES: if key.ARG_NAME == args.encryption: + assert key.ARG_NAME is not None return key.create(repository, args) else: raise ValueError('Invalid encryption mode "%s"' % args.encryption) +def key_argument_names(): + return [key.ARG_NAME for key in AVAILABLE_KEY_TYPES if key.ARG_NAME] + + def identify_key(manifest_data): key_type = manifest_data[0] if key_type == PassphraseKey.TYPE: From bc42f58c04edf5d318ede88629f3b5edb98e367c Mon Sep 17 00:00:00 2001 From: Fabio Pedretti Date: Mon, 9 Oct 2017 14:09:43 +0200 Subject: [PATCH 09/22] use --format rather than --list-format in examples, the latter is deprecated --- docs/usage/list.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage/list.rst b/docs/usage/list.rst index 4268bff1a..9b84eb01a 100644 --- a/docs/usage/list.rst +++ b/docs/usage/list.rst @@ -19,7 +19,7 @@ Examples -rwxr-xr-x root root 2140 Fri, 2015-03-27 20:24:22 bin/bzdiff ... - $ borg list /path/to/repo::archiveA --list-format="{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}" + $ borg list /path/to/repo::archiveA --format="{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}" drwxrwxr-x user user 0 Sun, 2015-02-01 11:00:00 . drwxrwxr-x user user 0 Sun, 2015-02-01 11:00:00 code drwxrwxr-x user user 0 Sun, 2015-02-01 11:00:00 code/myproject From 9d3daebd5fa6a7929fc3a1e556a2b7931c97b354 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 10 Oct 2017 01:17:56 +0200 Subject: [PATCH 10/22] recreate: don't crash on attic archives w/o time_end, fixes #3109 --- src/borg/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 9bb2b8b1f..477cf8a18 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -1813,7 +1813,7 @@ class ArchiveRecreater: target.save(comment=comment, additional_metadata={ # keep some metadata as in original archive: 'time': archive.metadata.time, - 'time_end': archive.metadata.time_end, + 'time_end': archive.metadata.get('time_end') or archive.metadata.time, 'cmdline': archive.metadata.cmdline, # but also remember recreate metadata: 'recreate_cmdline': sys.argv, From 7a689b1295ca647a7f9008df508f303214930d08 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 10 Oct 2017 00:51:13 +0200 Subject: [PATCH 11/22] don't crash in first part of truncate_and_unlink, fixes #3117 --- src/borg/helpers/fs.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/borg/helpers/fs.py b/src/borg/helpers/fs.py index 783c14d7a..465c4c110 100644 --- a/src/borg/helpers/fs.py +++ b/src/borg/helpers/fs.py @@ -1,3 +1,4 @@ +import errno import os import os.path import re @@ -141,8 +142,13 @@ def truncate_and_unlink(path): recover. Refer to the "File system interaction" section in repository.py for further explanations. """ - with open(path, 'r+b') as fd: - fd.truncate() + try: + with open(path, 'r+b') as fd: + fd.truncate() + except OSError as err: + if err.errno != errno.ENOTSUP: + raise + # don't crash if the above ops are not supported. os.unlink(path) From 60e924910034b86d3d9c6e9af706e5559cdb4d19 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 10 Oct 2017 01:36:44 +0200 Subject: [PATCH 12/22] fix detection of non-local path, fixes #3108 filenames like ..foobar are valid, so, to detect stuff in upper dirs, we need to include the path separator and check if it starts with '../'. --- src/borg/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 9bb2b8b1f..9d2c78387 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -557,7 +557,7 @@ Utilization of max. archive size: {csize_max:.0%} original_path = original_path or item.path dest = self.cwd - if item.path.startswith(('/', '..')): + if item.path.startswith(('/', '../')): raise Exception('Path should be relative and local') path = os.path.join(dest, item.path) # Attempt to remove existing files, ignore errors on failure From 203a5c8f197904b51fbced32f42cf0c383130292 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 10 Oct 2017 01:57:58 +0200 Subject: [PATCH 13/22] catch ENOTSUP for os.link, fixes #3107 --- src/borg/repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/repository.py b/src/borg/repository.py index 5320b56a2..e327ddace 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -266,7 +266,7 @@ class Repository: try: os.link(config_path, old_config_path) except OSError as e: - if e.errno in (errno.EMLINK, errno.ENOSYS, errno.EPERM): + if e.errno in (errno.EMLINK, errno.ENOSYS, errno.EPERM, errno.ENOTSUP): logger.warning("Hardlink failed, cannot securely erase old config file") else: raise From afba813706e3f0db2688086fd553b40c770f4136 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 10 Oct 2017 02:18:13 +0200 Subject: [PATCH 14/22] logging with fileConfig: set json attr on "borg" logger, fixes #3114 --- src/borg/logger.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/borg/logger.py b/src/borg/logger.py index 69cb86f18..7edec891b 100644 --- a/src/borg/logger.py +++ b/src/borg/logger.py @@ -80,6 +80,8 @@ def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF', lev logging.config.fileConfig(f) configured = True logger = logging.getLogger(__name__) + borg_logger = logging.getLogger('borg') + borg_logger.json = json logger.debug('using logging configuration read from "{0}"'.format(conf_fname)) warnings.showwarning = _log_warning return None From 6049a07b74b55856d5be999bac4b4d49a5317d35 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 12 Oct 2017 03:37:48 +0200 Subject: [PATCH 15/22] don't brew update, hopefully fixes #2532 --- .travis/install.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis/install.sh b/.travis/install.sh index 4ffc99359..5bad12264 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -4,8 +4,6 @@ set -e set -x if [[ "$(uname -s)" == 'Darwin' ]]; then - brew update || brew update - if [[ "${OPENSSL}" != "0.9.8" ]]; then brew outdated openssl || brew upgrade openssl fi From 3be328ed709d7d893415ccc4ffc046835b74eb6b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 12 Oct 2017 05:40:52 +0200 Subject: [PATCH 16/22] don't crash if only a global option is given, show help, fixes #3142 --- src/borg/archiver.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 1845d8840..0df5d94a0 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1959,6 +1959,8 @@ class Archiver: parser.print_help() return EXIT_SUCCESS + do_maincommand_help = do_subcommand_help + def preprocess_args(self, args): deprecations = [ # ('--old', '--new' or None, 'Warning: "--old" has been deprecated. Use "--new" instead.'), @@ -2226,6 +2228,7 @@ class Archiver: parser = argparse.ArgumentParser(prog=self.prog, description='Borg - Deduplicated Backups', add_help=False) + parser.set_defaults(func=functools.partial(self.do_maincommand_help, parser)) parser.common_options = self.CommonOptions(define_common_options, suffix_precedence=('_maincommand', '_midcommand', '_subcommand')) parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + __version__, From ed1a8b5cf1009423cb34753e9c1e0dda3b4b7b91 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 12 Oct 2017 06:03:12 +0200 Subject: [PATCH 17/22] add example showing --show-version --show-rc --- docs/usage/general.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/usage/general.rst b/docs/usage/general.rst index 127ab6676..5629aa5b8 100644 --- a/docs/usage/general.rst +++ b/docs/usage/general.rst @@ -30,3 +30,11 @@ Common options All Borg commands share these options: .. include:: common-options.rst.inc + +Examples +~~~~~~~~ +:: + + # Create an archive and log: borg version, files list, return code + $ borg create --show-version --list --show-rc /path/to/repo::my-files files + From c8441b5b3da4413e63c9b856b2816a6758dae2f3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 12 Oct 2017 16:14:32 +0200 Subject: [PATCH 18/22] readme: -e is required in borg 1.1 --- README.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 6bc611787..6762017fb 100644 --- a/README.rst +++ b/README.rst @@ -89,9 +89,12 @@ Main features Easy to use ~~~~~~~~~~~ -Initialize a new backup repository and create a backup archive:: +Initialize a new backup repository (see ``borg init --help`` for encryption options):: + + $ borg init -e repokey /path/to/repo + +Create a backup archive:: - $ borg init /path/to/repo $ borg create /path/to/repo::Saturday1 ~/Documents Now doing another backup, just to show off the great deduplication:: From 19ed725a58b58901d22dbcdb74d22cf029c468d1 Mon Sep 17 00:00:00 2001 From: TW Date: Fri, 13 Oct 2017 20:16:30 +0200 Subject: [PATCH 19/22] move files-cache related options to borg create, fixes #3146 (#3147) move --no-files-cache from common to borg create options, fixes #3146 for borg prune, just use do_files=False (it only needs the chunks cache, not the files cache). --- src/borg/archiver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 0df5d94a0..8bc95c9d6 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1326,7 +1326,7 @@ class Archiver: keep += prune_split(archives, '%Y', args.yearly, keep) to_delete = (set(archives) | checkpoints) - (set(keep) | set(keep_checkpoints)) stats = Statistics() - with Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) as cache: + with Cache(repository, key, manifest, do_files=False, lock_wait=self.lock_wait) as cache: list_logger = logging.getLogger('borg.output.list') if args.output_list: # set up counters for the progress display @@ -2154,8 +2154,6 @@ class Archiver: help='show/log the borg version') add_common_option('--show-rc', dest='show_rc', action='store_true', help='show/log the return code (rc)') - add_common_option('--no-files-cache', dest='cache_files', action='store_false', - help='do not load/update the file metadata cache used to detect unchanged files') add_common_option('--umask', metavar='M', dest='umask', type=lambda s: int(s, 8), default=UMASK_DEFAULT, help='set umask to M (local and remote, default: %(default)04o)') add_common_option('--remote-path', metavar='PATH', dest='remote_path', @@ -2726,6 +2724,8 @@ class Archiver: help='output stats as JSON. Implies ``--stats``.') subparser.add_argument('--no-cache-sync', dest='no_cache_sync', action='store_true', help='experimental: do not synchronize the cache. Implies not using the files cache.') + subparser.add_argument('--no-files-cache', dest='cache_files', action='store_false', + help='do not load/update the file metadata cache used to detect unchanged files') define_exclusion_group(subparser, tag_files=True) From a6ee4e9aedd2eb124c3a486f12131215661f77e1 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 11 Oct 2017 03:32:33 +0200 Subject: [PATCH 20/22] bsdflags support: do not open BLK/CHR/LNK files, fixes #3130 opening a device file for a non-existing device can be very slow. symlinks will make the open() call fail as it is using O_NOFOLLOW. also: lstat -> stat(..., follow_symlinks=False) like everywhere else. --- src/borg/platform/linux.pyx | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/borg/platform/linux.pyx b/src/borg/platform/linux.pyx index 7c3cf1b0b..25f71fa18 100644 --- a/src/borg/platform/linux.pyx +++ b/src/borg/platform/linux.pyx @@ -72,8 +72,11 @@ BSD_TO_LINUX_FLAGS = { def set_flags(path, bsd_flags, fd=None): - if fd is None and stat.S_ISLNK(os.lstat(path).st_mode): - return + if fd is None: + st = os.stat(path, follow_symlinks=False) + if stat.S_ISBLK(st.st_mode) or stat.S_ISCHR(st.st_mode) or stat.S_ISLNK(st.st_mode): + # see comment in get_flags() + return cdef int flags = 0 for bsd_flag, linux_flag in BSD_TO_LINUX_FLAGS.items(): if bsd_flags & bsd_flag: @@ -92,6 +95,10 @@ def set_flags(path, bsd_flags, fd=None): def get_flags(path, st): + if stat.S_ISBLK(st.st_mode) or stat.S_ISCHR(st.st_mode) or stat.S_ISLNK(st.st_mode): + # avoid opening devices files - trying to open non-present devices can be rather slow. + # avoid opening symlinks, O_NOFOLLOW would make the open() fail anyway. + return 0 cdef int linux_flags try: fd = os.open(path, os.O_RDONLY|os.O_NONBLOCK|os.O_NOFOLLOW) From 9d6b125e989b02487cc05f6c5371ad2058cc9579 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 14 Oct 2017 04:24:26 +0200 Subject: [PATCH 21/22] borg recreate: correctly compute part file sizes, fixes #3157 when doing in-file checkpointing, borg creates *.borg_part_N files. complete_file = part_1 + part_2 + ... + part_N the source item for recreate already has a precomputed (total) size member, thus we must force recomputation from the (partial) chunks list to correct the size to be the part's size only. borg create avoided this problem by computing the size member after writing all the parts. this is now not required any more. the bug is mostly cosmetic, borg check will complain, borg extract on a part file would also complain. but all the complaints only refer to the wrong metadata of the part files, the part files' contents are correct. usually you will never extract or look at part files, but only deal with the full file, which will be completely valid, all metadata and content. you can get rid of the archives with these cosmetic errors by running borg recreate on them with a fixed borg version. the old part files will get dropped (because they are usually ignored) and any new part file created due to checkpointing will be correct. --- src/borg/archive.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 11e77440f..b8189096e 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -965,7 +965,9 @@ class ChunksProcessor: length = len(item.chunks) # the item should only have the *additional* chunks we processed after the last partial item: item.chunks = item.chunks[from_chunk:] - item.get_size(memorize=True) + # for borg recreate, we already have a size member in the source item (giving the total file size), + # but we consider only a part of the file here, thus we must recompute the size from the chunks: + item.get_size(memorize=True, from_chunks=True) item.path += '.borg_part_%d' % number item.part = number number += 1 From 0190abff814a8b5bc5cdb8342593ca3d4da321ce Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 14 Oct 2017 21:57:58 +0200 Subject: [PATCH 22/22] cache: use SaveFile for more safety, fixes #3158 Looks like under unfortunate circumstances, these files could become 0 byte files (see #3158). SaveFile usage should prevent that. --- src/borg/cache.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/borg/cache.py b/src/borg/cache.py index 227858bc1..e26e183a0 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -85,11 +85,11 @@ class SecurityManager: logger.debug('security: current location %s', current_location) logger.debug('security: key type %s', str(key.TYPE)) logger.debug('security: manifest timestamp %s', manifest.timestamp) - with open(self.location_file, 'w') as fd: + with SaveFile(self.location_file) as fd: fd.write(current_location) - with open(self.key_type_file, 'w') as fd: + with SaveFile(self.key_type_file) as fd: fd.write(str(key.TYPE)) - with open(self.manifest_ts_file, 'w') as fd: + with SaveFile(self.manifest_ts_file) as fd: fd.write(manifest.timestamp) def assert_location_matches(self, cache_config=None): @@ -119,7 +119,7 @@ class SecurityManager: raise Cache.RepositoryAccessAborted() # adapt on-disk config immediately if the new location was accepted logger.debug('security: updating location stored in cache and security dir') - with open(self.location_file, 'w') as fd: + with SaveFile(self.location_file) as fd: fd.write(repository_location) if cache_config: cache_config.save() @@ -470,7 +470,7 @@ class LocalCache(CacheStatsMixin): self.cache_config.create() ChunkIndex().write(os.path.join(self.path, 'chunks')) os.makedirs(os.path.join(self.path, 'chunks.archive.d')) - with SaveFile(os.path.join(self.path, 'files'), binary=True) as fd: + with SaveFile(os.path.join(self.path, 'files'), binary=True): pass # empty file def _do_open(self): @@ -844,7 +844,7 @@ class LocalCache(CacheStatsMixin): shutil.rmtree(os.path.join(self.path, 'chunks.archive.d')) os.makedirs(os.path.join(self.path, 'chunks.archive.d')) self.chunks = ChunkIndex() - with open(os.path.join(self.path, 'files'), 'wb'): + with SaveFile(os.path.join(self.path, 'files'), binary=True): pass # empty file self.cache_config.manifest_id = '' self.cache_config._config.set('cache', 'manifest', '')