From 7053a721409fcfe7cceec5934a7cc915e884b7d0 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Wed, 13 Jul 2016 20:04:20 +0200 Subject: [PATCH 1/7] Fix borg break-lock ignoring BORG_REPO env var --- borg/archiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index 5fc05ee2..cdf12ef7 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -1316,7 +1316,7 @@ class Archiver: formatter_class=argparse.RawDescriptionHelpFormatter, help='break repository and cache locks') subparser.set_defaults(func=self.do_break_lock) - subparser.add_argument('location', metavar='REPOSITORY', + subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', type=location_validator(archive=False), help='repository for which to break the locks') From b9952efd68163a909c05cdd3f2a5408e27dd5424 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Fri, 15 Jul 2016 00:43:49 +0200 Subject: [PATCH 2/7] faq: 'A' unchanged file; remove ambiguous entry age sentence. --- docs/faq.rst | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 87794493..eaeef714 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -336,11 +336,8 @@ I am seeing 'A' (added) status for a unchanged file!? The files cache is used to determine whether |project_name| already "knows" / has backed up a file and if so, to skip the file from -chunking. It does intentionally *not* contain files that: - -- have >= 10 as "entry age" (|project_name| has not seen this file for a while) -- have a modification time (mtime) same as the newest mtime in the created - archive +chunking. It does intentionally *not* contain files that have a modification +time (mtime) same as the newest mtime in the created archive. So, if you see an 'A' status for unchanged file(s), they are likely the files with the most recent mtime in that archive. From 5f713ec1c17edcc6daa79175198cd02197b1b6d3 Mon Sep 17 00:00:00 2001 From: Pankaj Garg Date: Fri, 15 Jul 2016 00:33:49 +0530 Subject: [PATCH 3/7] Fixed the issue of xenial64 box being unable to run because vagrant user is absent --- Vagrantfile | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index fc5fdacf..f89a1639 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -12,12 +12,19 @@ end def packages_debianoid return <<-EOF + if id "vagrant" >/dev/null 2>&1; then + username='vagrant' + home_dir=/home/vagrant + else + username='ubuntu' + home_dir=/home/ubuntu + fi apt-get update # install all the (security and other) updates apt-get dist-upgrade -y # for building borgbackup and dependencies: apt-get install -y libssl-dev libacl1-dev liblz4-dev libfuse-dev fuse pkg-config - usermod -a -G fuse vagrant + usermod -a -G fuse $username apt-get install -y fakeroot build-essential git apt-get install -y python3-dev python3-setuptools # for building python: @@ -27,7 +34,7 @@ def packages_debianoid # newer versions are not compatible with py 3.2 any more. easy_install3 'pip<8.0' pip3 install 'virtualenv<14.0' - touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile + touch $home_dir/.bash_profile ; chown $username $home_dir/.bash_profile EOF end @@ -283,7 +290,13 @@ end def fix_perms return <<-EOF # . ~/.profile - chown -R vagrant /vagrant/borg + + if id "vagrant" >/dev/null 2>&1; then + chown -R vagrant /vagrant/borg + else + chown -R ubuntu /vagrant/borg + fi + EOF end @@ -338,6 +351,17 @@ Vagrant.configure(2) do |config| b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("centos6_64") end + config.vm.define "xenial64" do |b| + b.vm.box = "ubuntu/xenial64" + b.vm.provider :virtualbox do |v| + v.memory = 768 + end + b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("xenial64") + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("xenial64") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("xenial64") + end + config.vm.define "trusty64" do |b| b.vm.box = "ubuntu/trusty64" b.vm.provider :virtualbox do |v| From 69f8d3c3f76316e6a5e1e86fb7b95e05889cb206 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Mon, 18 Jul 2016 12:14:48 +0200 Subject: [PATCH 4/7] Use an OrderedDict for helptext, making the build reproducible Closes #1346 --- borg/archiver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index cdf12ef7..02da0f1b 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -14,6 +14,7 @@ import stat import sys import textwrap import traceback +import collections from . import __version__ from .helpers import Error, location_validator, archivename_validator, format_line, format_time, format_file_size, \ @@ -736,7 +737,7 @@ class Archiver: Cache.break_lock(repository) return self.exit_code - helptext = {} + helptext = collections.OrderedDict() helptext['patterns'] = textwrap.dedent(''' Exclusion patterns support four separate styles, fnmatch, shell, regular expressions and path prefixes. By default, fnmatch is used. If followed From 9fe4473ff25df1c4c5519b65851afddbe2727ae9 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 26 Jul 2016 13:41:38 -0400 Subject: [PATCH 5/7] Ignore stdout/stderr broken pipe errors --- borg/archiver.py | 6 +++--- borg/helpers.py | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 02da0f1b..74c1d1a4 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -22,7 +22,7 @@ from .helpers import Error, location_validator, archivename_validator, format_li get_cache_dir, prune_within, prune_split, \ Manifest, NoManifestError, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, PrefixSpec, is_slow_msgpack, yes, sysinfo, \ - EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, log_multi, PatternMatcher + EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, log_multi, PatternMatcher, ErrorIgnoringTextIOWrapper from .logger import create_logger, setup_logging logger = create_logger() from .compress import Compressor, COMPR_BUFFER @@ -1592,8 +1592,8 @@ def setup_signal_handlers(): # pragma: no cover def main(): # pragma: no cover # Make sure stdout and stderr have errors='replace') to avoid unicode # issues when print()-ing unicode file names - sys.stdout = io.TextIOWrapper(sys.stdout.buffer, sys.stdout.encoding, 'replace', line_buffering=True) - sys.stderr = io.TextIOWrapper(sys.stderr.buffer, sys.stderr.encoding, 'replace', line_buffering=True) + sys.stdout = ErrorIgnoringTextIOWrapper(sys.stdout.buffer, sys.stdout.encoding, 'replace', line_buffering=True) + sys.stderr = ErrorIgnoringTextIOWrapper(sys.stderr.buffer, sys.stderr.encoding, 'replace', line_buffering=True) setup_signal_handlers() archiver = Archiver() msg = None diff --git a/borg/helpers.py b/borg/helpers.py index 56e336e0..1a7bf8af 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -12,6 +12,8 @@ import sys import platform import time import unicodedata +import io +import errno import logging from .logger import create_logger @@ -1089,3 +1091,27 @@ def log_multi(*msgs, level=logging.INFO): lines.extend(msg.splitlines()) for line in lines: logger.log(level, line) + + +class ErrorIgnoringTextIOWrapper(io.TextIOWrapper): + def read(self, n): + if not self.closed: + try: + return super().read(n) + except BrokenPipeError: + try: + super().close() + except OSError: + pass + return '' + + def write(self, s): + if not self.closed: + try: + return super().write(s) + except BrokenPipeError: + try: + super().close() + except OSError: + pass + return len(s) From 0a4a95eb1767a1dad84b7e29d4e951efa4e63d2c Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Wed, 27 Jul 2016 09:12:52 -0400 Subject: [PATCH 6/7] Remove trailing whitespace --- README.rst | 8 ++--- docs/changes.rst | 6 ++-- docs/installation.rst | 2 +- docs/internals.rst | 2 +- docs/misc/internals-picture.txt | 26 +++++++------- docs/misc/prune-example.txt | 64 ++++++++++++++++----------------- tox.ini | 2 +- 7 files changed, 55 insertions(+), 55 deletions(-) diff --git a/README.rst b/README.rst index 035a38d9..f6132773 100644 --- a/README.rst +++ b/README.rst @@ -33,14 +33,14 @@ Main features Compared to other deduplication approaches, this method does NOT depend on: - * file/directory names staying the same: So you can move your stuff around + * file/directory names staying the same: So you can move your stuff around without killing the deduplication, even between machines sharing a repo. - * complete files or time stamps staying the same: If a big file changes a - little, only a few new chunks need to be stored - this is great for VMs or + * complete files or time stamps staying the same: If a big file changes a + little, only a few new chunks need to be stored - this is great for VMs or raw disks. - * The absolute position of a data chunk inside a file: Stuff may get shifted + * The absolute position of a data chunk inside a file: Stuff may get shifted and will still be found by the deduplication algorithm. **Speed** diff --git a/docs/changes.rst b/docs/changes.rst index 0696da51..7d9b47ab 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -383,7 +383,7 @@ Compatibility notes: - disambiguate -p option, #563: - -p now is same as --progress - - -P now is same as --prefix + - -P now is same as --prefix - remove deprecated "borg verify", use "borg extract --dry-run" instead - cleanup environment variable semantics, #355 @@ -448,7 +448,7 @@ New features: - format options for location: user, pid, fqdn, hostname, now, utcnow, user - borg list --list-format - borg prune -v --list enables the keep/prune list output, #658 - + Bug fixes: - fix _open_rb noatime handling, #657 @@ -466,7 +466,7 @@ Other changes: - Vagrant: drop Ubuntu Precise (12.04) - does not have Python >= 3.4 - Vagrant: use pyinstaller v3.1.1 to build binaries - docs: - + - borg upgrade: add to docs that only LOCAL repos are supported - borg upgrade also handles borg 0.xx -> 1.0 - use pip extras or requirements file to install llfuse diff --git a/docs/installation.rst b/docs/installation.rst index b39bb0af..fec03b55 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -181,7 +181,7 @@ and commands to make fuse work for using the mount command. echo 'vfs.usermount=1' >> /etc/sysctl.conf kldload fuse sysctl vfs.usermount=1 - + Cygwin ++++++ diff --git a/docs/internals.rst b/docs/internals.rst index 61d84589..b088f68e 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -371,7 +371,7 @@ repository_id enc_key the key used to encrypt data with AES (256 bits) - + enc_hmac_key the key used to HMAC the encrypted data (256 bits) diff --git a/docs/misc/internals-picture.txt b/docs/misc/internals-picture.txt index ae76f0c1..01351a7b 100644 --- a/docs/misc/internals-picture.txt +++ b/docs/misc/internals-picture.txt @@ -11,22 +11,22 @@ BorgBackup from 10.000m | | | +------+-------+ | | | | | - /chunk\/chunk\/chunk\... /maybe different chunks lists\ + /chunk\/chunk\/chunk\... /maybe different chunks lists\ +-----------------------------------------------------------------+ |item list | +-----------------------------------------------------------------+ - | - +-------------------------------------+--------------+ - | | | - | | | -+-------------+ +-------------+ | -|item0 | |item1 | | -| - owner | | - owner | | -| - size | | - size | ... -| - ... | | - ... | -| - chunks | | - chunks | -+----+--------+ +-----+-------+ - | | + | + +-------------------------------------+--------------+ + | | | + | | | ++-------------+ +-------------+ | +|item0 | |item1 | | +| - owner | | - owner | | +| - size | | - size | ... +| - ... | | - ... | +| - chunks | | - chunks | ++----+--------+ +-----+-------+ + | | | +-----+----------------------------+-----------------+ | | | | +-o-----o------------+ | diff --git a/docs/misc/prune-example.txt b/docs/misc/prune-example.txt index 6c8f8e55..ac608b6a 100644 --- a/docs/misc/prune-example.txt +++ b/docs/misc/prune-example.txt @@ -14,41 +14,41 @@ Calendar view ------------- 2015 - January February March -Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su - 1 2 3 4 1 1 - 5 6 7 8 9 10 11 2 3 4 5 6 7 8 2 3 4 5 6 7 8 -12 13 14 15 16 17 18 9 10 11 12 13 14 15 9 10 11 12 13 14 15 -19 20 21 22 23 24 25 16 17 18 19 20 21 22 16 17 18 19 20 21 22 -26 27 28 29 30 31 23 24 25 26 27 28 23 24 25 26 27 28 29 - 30 31 + January February March +Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 3 4 1 1 + 5 6 7 8 9 10 11 2 3 4 5 6 7 8 2 3 4 5 6 7 8 +12 13 14 15 16 17 18 9 10 11 12 13 14 15 9 10 11 12 13 14 15 +19 20 21 22 23 24 25 16 17 18 19 20 21 22 16 17 18 19 20 21 22 +26 27 28 29 30 31 23 24 25 26 27 28 23 24 25 26 27 28 29 + 30 31 - April May June -Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su - 1 2 3 4 5 1 2 3 1 2 3 4 5 6 7 - 6 7 8 9 10 11 12 4 5 6 7 8 9 10 8 9 10 11 12 13 14 -13 14 15 16 17 18 19 11 12 13 14 15 16 17 15 16 17 18 19 20 21 -20 21 22 23 24 25 26 18 19 20 21 22 23 24 22 23 24 25 26 27 28 -27 28 29 30 25 26 27 28 29 30 31 29 30m - + April May June +Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 3 4 5 1 2 3 1 2 3 4 5 6 7 + 6 7 8 9 10 11 12 4 5 6 7 8 9 10 8 9 10 11 12 13 14 +13 14 15 16 17 18 19 11 12 13 14 15 16 17 15 16 17 18 19 20 21 +20 21 22 23 24 25 26 18 19 20 21 22 23 24 22 23 24 25 26 27 28 +27 28 29 30 25 26 27 28 29 30 31 29 30m - July August September -Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su - 1 2 3 4 5 1 2 1 2 3 4 5 6 - 6 7 8 9 10 11 12 3 4 5 6 7 8 9 7 8 9 10 11 12 13 -13 14 15 16 17 18 19 10 11 12 13 14 15 16 14 15 16 17 18 19 20 -20 21 22 23 24 25 26 17 18 19 20 21 22 23 21 22 23 24 25 26 27 -27 28 29 30 31m 24 25 26 27 28 29 30 28 29 30m - 31m - October November December -Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su - 1 2 3 4 1 1 2 3 4 5 6 - 5 6 7 8 9 10 11 2 3 4 5 6 7 8 7 8 9 10 11 12 13 -12 13 14 15 16 17 18 9 10 11 12 13 14 15 14 15 16 17d18d19d20 -19 20 21 22 23 24 25 16 17 18 19 20 21 22 21d22d23d24d25d26d27d -26 27 28 29 30 31m 23 24 25 26 27 28 29 28d29d30d31d - 30m + July August September +Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 3 4 5 1 2 1 2 3 4 5 6 + 6 7 8 9 10 11 12 3 4 5 6 7 8 9 7 8 9 10 11 12 13 +13 14 15 16 17 18 19 10 11 12 13 14 15 16 14 15 16 17 18 19 20 +20 21 22 23 24 25 26 17 18 19 20 21 22 23 21 22 23 24 25 26 27 +27 28 29 30 31m 24 25 26 27 28 29 30 28 29 30m + 31m + + October November December +Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 3 4 1 1 2 3 4 5 6 + 5 6 7 8 9 10 11 2 3 4 5 6 7 8 7 8 9 10 11 12 13 +12 13 14 15 16 17 18 9 10 11 12 13 14 15 14 15 16 17d18d19d20 +19 20 21 22 23 24 25 16 17 18 19 20 21 22 21d22d23d24d25d26d27d +26 27 28 29 30 31m 23 24 25 26 27 28 29 28d29d30d31d + 30m List view --------- diff --git a/tox.ini b/tox.ini index c7b49bca..bc6195d9 100644 --- a/tox.ini +++ b/tox.ini @@ -18,4 +18,4 @@ passenv = * [testenv:flake8] changedir = deps = flake8 -commands = flake8 +commands = flake8 From c5fffbefff28858fe5b44ca034f02c5ac627be93 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Wed, 27 Jul 2016 23:54:15 +0200 Subject: [PATCH 7/7] Platform feature matrix --- docs/development.rst | 10 +++++++ docs/faq.rst | 19 ------------ docs/installation.rst | 70 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 19 deletions(-) diff --git a/docs/development.rst b/docs/development.rst index 2dd47e70..05da98b4 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -19,6 +19,16 @@ instead of 79. We do *not* use form-feed (``^L``) characters to separate sections either. Compliance is tested automatically when you run the tests. +Continuous Integration +---------------------- + +All pull requests go through Travis-CI_, which runs the tests on Linux +and Mac OS X as well as the flake8 style checker. Additional Unix-like platforms +are tested on Golem_. + +.. _Golem: https://golem.enkore.de/view/Borg/ +.. _Travis-CI: https://travis-ci.org/borgbackup/borg + Output and Logging ------------------ When writing logger calls, always use correct log level (debug only for diff --git a/docs/faq.rst b/docs/faq.rst index eaeef714..0ea3f079 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -47,25 +47,6 @@ If you want to have 2 independent backups, it is better to do it like this: - client machine ---borg create---> repo1 - client machine ---borg create---> repo2 -Which file types, attributes, etc. are preserved? -------------------------------------------------- - - * Directories - * Regular files - * Hardlinks (considering all files in the same archive) - * Symlinks (stored as symlink, the symlink is not followed) - * Character and block device files - * FIFOs ("named pipes") - * Name - * Contents - * Timestamps in nanosecond precision: mtime, atime, ctime - * IDs of owning user and owning group - * Names of owning user and owning group (if the IDs can be resolved) - * Unix Mode/Permissions (u/g/o permissions, suid, sgid, sticky) - * Extended Attributes (xattrs) on Linux, OS X and FreeBSD - * Access Control Lists (ACL_) on Linux, OS X and FreeBSD - * BSD flags on OS X and FreeBSD - Which file types, attributes, etc. are *not* preserved? ------------------------------------------------------- diff --git a/docs/installation.rst b/docs/installation.rst index b39bb0af..ab57b7c8 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -89,6 +89,76 @@ the old version using the same steps as shown above. .. _pyinstaller: http://www.pyinstaller.org .. _releases: https://github.com/borgbackup/borg/releases +.. _platforms: + +Features & platforms +-------------------- + +Besides regular file and directory structures, |project_name| can preserve + + * Hardlinks (considering all files in the same archive) + * Symlinks (stored as symlink, the symlink is not followed) + * Special files: + + * Character and block device files (restored via mknod) + * FIFOs ("named pipes") + * Special file *contents* can be backed up in ``--read-special`` mode. + By default the metadata to create them with mknod(2), mkfifo(2) etc. is stored. + * Timestamps in nanosecond precision: mtime, atime, ctime + * Permissions: + + * IDs of owning user and owning group + * Names of owning user and owning group (if the IDs can be resolved) + * Unix Mode/Permissions (u/g/o permissions, suid, sgid, sticky) + +On some platforms additional features are supported: + +.. Yes/No's are grouped by reason/mechanism/reference. + ++------------------+----------+-----------+------------+ +| Platform | ACLs | xattr | Flags | +| | [#acls]_ | [#xattr]_ | [#flags]_ | ++==================+==========+===========+============+ +| Linux x86 | Yes | Yes | No | ++------------------+ | | | +| Linux PowerPC | | | | ++------------------+ | | | +| Linux ARM | | | | ++------------------+----------+-----------+------------+ +| Mac OS X | Yes | Yes | Yes (all) | ++------------------+----------+-----------+ | +| FreeBSD | Yes | Yes | | ++------------------+----------+-----------+ | +| OpenBSD | n/a | n/a | | ++------------------+----------+-----------+ | +| NetBSD | n/a | No [2]_ | | ++------------------+----------+-----------+------------+ +| Solaris 11 | No [3]_ | n/a | ++------------------+ | | +| OpenIndiana | | | ++------------------+----------+-----------+------------+ +| Windows (cygwin) | No [4]_ | No | No | ++------------------+----------+-----------+------------+ + +Some Distributions (e.g. Debian) run additional tests after each release, these +are not reflected here. + +Other Unix-like operating systems may work as well, but have not been tested at all. + +Note that most of the platform-dependent features also depend on the file system. +For example, ntfs-3g on Linux isn't able to convey NTFS ACLs. + + +.. [2] Feature request :issue:`1332` +.. [3] Feature request :issue:`1337` +.. [4] Cygwin tries to map NTFS ACLs to permissions with varying degress of success. + +.. [#acls] The native access control list mechanism of the OS. This normally limits access to + non-native ACLs. For example, NTFS ACLs aren't completely accessible on Linux with ntfs-3g. +.. [#xattr] extended attributes; key-value pairs attached to a file, mainly used by the OS. + This includes resource forks on Mac OS X. +.. [#flags] aka *BSD flags*. + .. _source-install: From Source