mirror
/
borg
mirror of https://github.com/borgbackup/borg.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch '1.0-maint' into master

This commit is contained in:
Marian Beermann 2016-07-28 00:29:23 +02:00
parent 3c2d41898f f811624fab
commit ac8c8119a7
No known key found for this signature in database
GPG Key ID: 9B8450B91D1362C1
12 changed files with 186 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
README.rst
View File

@ -33,14 +33,14 @@ Main features
Compared to other deduplication approaches, this method does NOT depend on: Compared to other deduplication approaches, this method does NOT depend on:
* file/directory names staying the same: So you can move your stuff around * file/directory names staying the same: So you can move your stuff around
without killing the deduplication, even between machines sharing a repo. without killing the deduplication, even between machines sharing a repo.
* complete files or time stamps staying the same: If a big file changes a * complete files or time stamps staying the same: If a big file changes a
little, only a few new chunks need to be stored - this is great for VMs or little, only a few new chunks need to be stored - this is great for VMs or
raw disks. raw disks.
* The absolute position of a data chunk inside a file: Stuff may get shifted * The absolute position of a data chunk inside a file: Stuff may get shifted
and will still be found by the deduplication algorithm. and will still be found by the deduplication algorithm.
**Speed** **Speed**

25
Vagrantfile vendored
View File

@ -12,12 +12,19 @@ end
def packages_debianoid def packages_debianoid
return <<-EOF return <<-EOF
if id "vagrant" >/dev/null 2>&1; then
username='vagrant'
home_dir=/home/vagrant
else
username='ubuntu'
home_dir=/home/ubuntu
fi
apt-get update apt-get update
# install all the (security and other) updates # install all the (security and other) updates
apt-get dist-upgrade -y apt-get dist-upgrade -y
# for building borgbackup and dependencies: # for building borgbackup and dependencies:
apt-get install -y libssl-dev libacl1-dev liblz4-dev libfuse-dev fuse pkg-config apt-get install -y libssl-dev libacl1-dev liblz4-dev libfuse-dev fuse pkg-config
usermod -a -G fuse vagrant usermod -a -G fuse $username
apt-get install -y fakeroot build-essential git apt-get install -y fakeroot build-essential git
apt-get install -y python3-dev python3-setuptools apt-get install -y python3-dev python3-setuptools
# for building python: # for building python:
@ -27,7 +34,7 @@ def packages_debianoid
# newer versions are not compatible with py 3.2 any more. # newer versions are not compatible with py 3.2 any more.
easy_install3 'pip<8.0' easy_install3 'pip<8.0'
pip3 install 'virtualenv<14.0' pip3 install 'virtualenv<14.0'
touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile touch $home_dir/.bash_profile ; chown $username $home_dir/.bash_profile
EOF EOF
end end
@ -283,7 +290,13 @@ end
def fix_perms def fix_perms
return <<-EOF return <<-EOF
# . ~/.profile # . ~/.profile
chown -R vagrant /vagrant/borg
if id "vagrant" >/dev/null 2>&1; then
chown -R vagrant /vagrant/borg
else
chown -R ubuntu /vagrant/borg
fi
EOF EOF
end end
@ -344,9 +357,9 @@ Vagrant.configure(2) do |config|
v.memory = 768 v.memory = 768
end end
b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("trusty64") b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("xenial64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("trusty64") b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("xenial64")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("trusty64") b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("xenial64")
end end
config.vm.define "trusty64" do |b| config.vm.define "trusty64" do |b|

6
docs/changes.rst
View File

@ -457,7 +457,7 @@ Compatibility notes:
- disambiguate -p option, #563: - disambiguate -p option, #563:
- -p now is same as --progress - -p now is same as --progress
- -P now is same as --prefix - -P now is same as --prefix
- remove deprecated "borg verify", - remove deprecated "borg verify",
use "borg extract --dry-run" instead use "borg extract --dry-run" instead
- cleanup environment variable semantics, #355 - cleanup environment variable semantics, #355
@ -522,7 +522,7 @@ New features:
- format options for location: user, pid, fqdn, hostname, now, utcnow, user - format options for location: user, pid, fqdn, hostname, now, utcnow, user
- borg list --list-format - borg list --list-format
- borg prune -v --list enables the keep/prune list output, #658 - borg prune -v --list enables the keep/prune list output, #658
Bug fixes: Bug fixes:
- fix _open_rb noatime handling, #657 - fix _open_rb noatime handling, #657
@ -540,7 +540,7 @@ Other changes:
- Vagrant: drop Ubuntu Precise (12.04) - does not have Python >= 3.4 - Vagrant: drop Ubuntu Precise (12.04) - does not have Python >= 3.4
- Vagrant: use pyinstaller v3.1.1 to build binaries - Vagrant: use pyinstaller v3.1.1 to build binaries
- docs: - docs:
- borg upgrade: add to docs that only LOCAL repos are supported - borg upgrade: add to docs that only LOCAL repos are supported
- borg upgrade also handles borg 0.xx -> 1.0 - borg upgrade also handles borg 0.xx -> 1.0
- use pip extras or requirements file to install llfuse - use pip extras or requirements file to install llfuse

10
docs/development.rst
View File

@ -19,6 +19,16 @@ instead of 79. We do *not* use form-feed (``^L``) characters to
separate sections either. Compliance is tested automatically when separate sections either. Compliance is tested automatically when
you run the tests. you run the tests.
Continuous Integration
----------------------
All pull requests go through Travis-CI_, which runs the tests on Linux
and Mac OS X as well as the flake8 style checker. Additional Unix-like platforms
are tested on Golem_.
.. _Golem: https://golem.enkore.de/view/Borg/
.. _Travis-CI: https://travis-ci.org/borgbackup/borg
Output and Logging Output and Logging
------------------ ------------------
When writing logger calls, always use correct log level (debug only for When writing logger calls, always use correct log level (debug only for

26
docs/faq.rst
View File

@ -47,25 +47,6 @@ If you want to have 2 independent backups, it is better to do it like this:
- client machine ---borg create---> repo1 - client machine ---borg create---> repo1
- client machine ---borg create---> repo2 - client machine ---borg create---> repo2
Which file types, attributes, etc. are preserved?
-------------------------------------------------
* Directories
* Regular files
* Hardlinks (considering all files in the same archive)
* Symlinks (stored as symlink, the symlink is not followed)
* Character and block device files
* FIFOs ("named pipes")
* Name
* Contents
* Timestamps in nanosecond precision: mtime, atime, ctime
* IDs of owning user and owning group
* Names of owning user and owning group (if the IDs can be resolved)
* Unix Mode/Permissions (u/g/o permissions, suid, sgid, sticky)
* Extended Attributes (xattrs) on Linux, OS X and FreeBSD
* Access Control Lists (ACL_) on Linux, OS X and FreeBSD
* BSD flags on OS X and FreeBSD
Which file types, attributes, etc. are *not* preserved? Which file types, attributes, etc. are *not* preserved?
------------------------------------------------------- -------------------------------------------------------
@ -337,11 +318,8 @@ I am seeing 'A' (added) status for a unchanged file!?
The files cache is used to determine whether |project_name| already The files cache is used to determine whether |project_name| already
"knows" / has backed up a file and if so, to skip the file from "knows" / has backed up a file and if so, to skip the file from
chunking. It does intentionally *not* contain files that: chunking. It does intentionally *not* contain files that have a modification
time (mtime) same as the newest mtime in the created archive.
- have >= 10 as "entry age" (|project_name| has not seen this file for a while)
- have a modification time (mtime) same as the newest mtime in the created
archive
So, if you see an 'A' status for unchanged file(s), they are likely the files So, if you see an 'A' status for unchanged file(s), they are likely the files
with the most recent mtime in that archive. with the most recent mtime in that archive.

72
docs/installation.rst
View File

@ -118,6 +118,76 @@ the old version using the same steps as shown above.
.. _pyinstaller: http://www.pyinstaller.org .. _pyinstaller: http://www.pyinstaller.org
.. _releases: https://github.com/borgbackup/borg/releases .. _releases: https://github.com/borgbackup/borg/releases
.. _platforms:
Features & platforms
--------------------
Besides regular file and directory structures, |project_name| can preserve
* Hardlinks (considering all files in the same archive)
* Symlinks (stored as symlink, the symlink is not followed)
* Special files:
* Character and block device files (restored via mknod)
* FIFOs ("named pipes")
* Special file *contents* can be backed up in ``--read-special`` mode.
By default the metadata to create them with mknod(2), mkfifo(2) etc. is stored.
* Timestamps in nanosecond precision: mtime, atime, ctime
* Permissions:
* IDs of owning user and owning group
* Names of owning user and owning group (if the IDs can be resolved)
* Unix Mode/Permissions (u/g/o permissions, suid, sgid, sticky)
On some platforms additional features are supported:
.. Yes/No's are grouped by reason/mechanism/reference.
+------------------+----------+-----------+------------+
| Platform | ACLs | xattr | Flags |
| | [#acls]_ | [#xattr]_ | [#flags]_ |
+==================+==========+===========+============+
| Linux x86 | Yes | Yes | No |
+------------------+ | | |
| Linux PowerPC | | | |
+------------------+ | | |
| Linux ARM | | | |
+------------------+----------+-----------+------------+
| Mac OS X | Yes | Yes | Yes (all) |
+------------------+----------+-----------+ |
| FreeBSD | Yes | Yes | |
+------------------+----------+-----------+ |
| OpenBSD | n/a | n/a | |
+------------------+----------+-----------+ |
| NetBSD | n/a | No [2]_ | |
+------------------+----------+-----------+------------+
| Solaris 11 | No [3]_ | n/a |
+------------------+ | |
| OpenIndiana | | |
+------------------+----------+-----------+------------+
| Windows (cygwin) | No [4]_ | No | No |
+------------------+----------+-----------+------------+
Some Distributions (e.g. Debian) run additional tests after each release, these
are not reflected here.
Other Unix-like operating systems may work as well, but have not been tested at all.
Note that most of the platform-dependent features also depend on the file system.
For example, ntfs-3g on Linux isn't able to convey NTFS ACLs.
.. [2] Feature request :issue:`1332`
.. [3] Feature request :issue:`1337`
.. [4] Cygwin tries to map NTFS ACLs to permissions with varying degress of success.
.. [#acls] The native access control list mechanism of the OS. This normally limits access to
non-native ACLs. For example, NTFS ACLs aren't completely accessible on Linux with ntfs-3g.
.. [#xattr] extended attributes; key-value pairs attached to a file, mainly used by the OS.
This includes resource forks on Mac OS X.
.. [#flags] aka *BSD flags*.
.. _source-install: .. _source-install:
From Source From Source
@ -211,7 +281,7 @@ and commands to make fuse work for using the mount command.
echo 'vfs.usermount=1' >> /etc/sysctl.conf echo 'vfs.usermount=1' >> /etc/sysctl.conf
kldload fuse kldload fuse
sysctl vfs.usermount=1 sysctl vfs.usermount=1
Cygwin Cygwin
++++++ ++++++

2
docs/internals.rst
View File

@ -371,7 +371,7 @@ repository_id
enc_key enc_key
the key used to encrypt data with AES (256 bits) the key used to encrypt data with AES (256 bits)
enc_hmac_key enc_hmac_key
the key used to HMAC the encrypted data (256 bits) the key used to HMAC the encrypted data (256 bits)

26
docs/misc/internals-picture.txt
View File

@ -11,22 +11,22 @@ BorgBackup from 10.000m
| | | | | |
+------+-------+ | +------+-------+ |
| | | | | | | |
/chunk\/chunk\/chunk\... /maybe different chunks lists\ /chunk\/chunk\/chunk\... /maybe different chunks lists\
+-----------------------------------------------------------------+ +-----------------------------------------------------------------+
|item list | |item list |
+-----------------------------------------------------------------+ +-----------------------------------------------------------------+
| |
+-------------------------------------+--------------+ +-------------------------------------+--------------+
| | | | | |
| | | | | |
+-------------+ +-------------+ | +-------------+ +-------------+ |
|item0 | |item1 | | |item0 | |item1 | |
| - owner | | - owner | | | - owner | | - owner | |
| - size | | - size | ... | - size | | - size | ...
| - ... | | - ... | | - ... | | - ... |
| - chunks | | - chunks | | - chunks | | - chunks |
+----+--------+ +-----+-------+ +----+--------+ +-----+-------+
| | | |
| +-----+----------------------------+-----------------+ | +-----+----------------------------+-----------------+
| | | | | | | |
+-o-----o------------+ | +-o-----o------------+ |

64
docs/misc/prune-example.txt
View File

@ -14,41 +14,41 @@ Calendar view
------------- -------------
2015 2015
January February March January February March
Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su
1 2 3 4 1 1 1 2 3 4 1 1
5 6 7 8 9 10 11 2 3 4 5 6 7 8 2 3 4 5 6 7 8 5 6 7 8 9 10 11 2 3 4 5 6 7 8 2 3 4 5 6 7 8
12 13 14 15 16 17 18 9 10 11 12 13 14 15 9 10 11 12 13 14 15 12 13 14 15 16 17 18 9 10 11 12 13 14 15 9 10 11 12 13 14 15
19 20 21 22 23 24 25 16 17 18 19 20 21 22 16 17 18 19 20 21 22 19 20 21 22 23 24 25 16 17 18 19 20 21 22 16 17 18 19 20 21 22
26 27 28 29 30 31 23 24 25 26 27 28 23 24 25 26 27 28 29 26 27 28 29 30 31 23 24 25 26 27 28 23 24 25 26 27 28 29
30 31 30 31
April May June April May June
Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su
1 2 3 4 5 1 2 3 1 2 3 4 5 6 7 1 2 3 4 5 1 2 3 1 2 3 4 5 6 7
6 7 8 9 10 11 12 4 5 6 7 8 9 10 8 9 10 11 12 13 14 6 7 8 9 10 11 12 4 5 6 7 8 9 10 8 9 10 11 12 13 14
13 14 15 16 17 18 19 11 12 13 14 15 16 17 15 16 17 18 19 20 21 13 14 15 16 17 18 19 11 12 13 14 15 16 17 15 16 17 18 19 20 21
20 21 22 23 24 25 26 18 19 20 21 22 23 24 22 23 24 25 26 27 28 20 21 22 23 24 25 26 18 19 20 21 22 23 24 22 23 24 25 26 27 28
27 28 29 30 25 26 27 28 29 30 31 29 30m 27 28 29 30 25 26 27 28 29 30 31 29 30m
July August September
Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su
1 2 3 4 5 1 2 1 2 3 4 5 6
6 7 8 9 10 11 12 3 4 5 6 7 8 9 7 8 9 10 11 12 13
13 14 15 16 17 18 19 10 11 12 13 14 15 16 14 15 16 17 18 19 20
20 21 22 23 24 25 26 17 18 19 20 21 22 23 21 22 23 24 25 26 27
27 28 29 30 31m 24 25 26 27 28 29 30 28 29 30m
31m
October November December July August September
Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su
1 2 3 4 1 1 2 3 4 5 6 1 2 3 4 5 1 2 1 2 3 4 5 6
5 6 7 8 9 10 11 2 3 4 5 6 7 8 7 8 9 10 11 12 13 6 7 8 9 10 11 12 3 4 5 6 7 8 9 7 8 9 10 11 12 13
12 13 14 15 16 17 18 9 10 11 12 13 14 15 14 15 16 17d18d19d20 13 14 15 16 17 18 19 10 11 12 13 14 15 16 14 15 16 17 18 19 20
19 20 21 22 23 24 25 16 17 18 19 20 21 22 21d22d23d24d25d26d27d 20 21 22 23 24 25 26 17 18 19 20 21 22 23 21 22 23 24 25 26 27
26 27 28 29 30 31m 23 24 25 26 27 28 29 28d29d30d31d 27 28 29 30 31m 24 25 26 27 28 29 30 28 29 30m
30m 31m
October November December
Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su
1 2 3 4 1 1 2 3 4 5 6
5 6 7 8 9 10 11 2 3 4 5 6 7 8 7 8 9 10 11 12 13
12 13 14 15 16 17 18 9 10 11 12 13 14 15 14 15 16 17d18d19d20
19 20 21 22 23 24 25 16 17 18 19 20 21 22 21d22d23d24d25d26d27d
26 27 28 29 30 31m 23 24 25 26 27 28 29 28d29d30d31d
30m
List view List view
--------- ---------

10
src/borg/archiver.py
View File

@ -3,7 +3,6 @@ import collections
import functools import functools
import hashlib import hashlib
import inspect import inspect
import io
import logging import logging
import os import os
import re import re
@ -41,6 +40,7 @@ from .helpers import dir_is_tagged, is_slow_msgpack, yes, sysinfo
from .helpers import log_multi from .helpers import log_multi
from .helpers import parse_pattern, PatternMatcher, PathPrefixPattern from .helpers import parse_pattern, PatternMatcher, PathPrefixPattern
from .helpers import signal_handler from .helpers import signal_handler
from .helpers import ErrorIgnoringTextIOWrapper
from .item import Item from .item import Item
from .key import key_creator, RepoKey, PassphraseKey from .key import key_creator, RepoKey, PassphraseKey
from .platform import get_flags from .platform import get_flags
@ -1047,7 +1047,7 @@ class Archiver:
Cache.break_lock(repository) Cache.break_lock(repository)
return self.exit_code return self.exit_code
helptext = {} helptext = collections.OrderedDict()
helptext['patterns'] = textwrap.dedent(''' helptext['patterns'] = textwrap.dedent('''
Exclusion patterns support four separate styles, fnmatch, shell, regular Exclusion patterns support four separate styles, fnmatch, shell, regular
expressions and path prefixes. By default, fnmatch is used. If followed expressions and path prefixes. By default, fnmatch is used. If followed
@ -1796,7 +1796,7 @@ class Archiver:
formatter_class=argparse.RawDescriptionHelpFormatter, formatter_class=argparse.RawDescriptionHelpFormatter,
help='break repository and cache locks') help='break repository and cache locks')
subparser.set_defaults(func=self.do_break_lock) subparser.set_defaults(func=self.do_break_lock)
subparser.add_argument('location', metavar='REPOSITORY', subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
type=location_validator(archive=False), type=location_validator(archive=False),
help='repository for which to break the locks') help='repository for which to break the locks')
@ -2251,8 +2251,8 @@ def main(): # pragma: no cover
# Make sure stdout and stderr have errors='replace' to avoid unicode # Make sure stdout and stderr have errors='replace' to avoid unicode
# issues when print()-ing unicode file names # issues when print()-ing unicode file names
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, sys.stdout.encoding, 'replace', line_buffering=True) sys.stdout = ErrorIgnoringTextIOWrapper(sys.stdout.buffer, sys.stdout.encoding, 'replace', line_buffering=True)
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, sys.stderr.encoding, 'replace', line_buffering=True) sys.stderr = ErrorIgnoringTextIOWrapper(sys.stderr.buffer, sys.stderr.encoding, 'replace', line_buffering=True)
setup_signal_handlers() setup_signal_handlers()
archiver = Archiver() archiver = Archiver()
msg = None msg = None

25
src/borg/helpers.py
View File

@ -2,6 +2,7 @@ import argparse
import grp import grp
import hashlib import hashlib
import logging import logging
import io
import os import os
import os.path import os.path
import platform import platform
@ -1546,3 +1547,27 @@ def signal_handler(signo, handler):
yield yield
finally: finally:
signal.signal(signo, old_signal_handler) signal.signal(signo, old_signal_handler)
class ErrorIgnoringTextIOWrapper(io.TextIOWrapper):
def read(self, n):
if not self.closed:
try:
return super().read(n)
except BrokenPipeError:
try:
super().close()
except OSError:
pass
return ''
def write(self, s):
if not self.closed:
try:
return super().write(s)
except BrokenPipeError:
try:
super().close()
except OSError:
pass
return len(s)

2
tox.ini
View File

@ -15,4 +15,4 @@ passenv = *
[testenv:flake8] [testenv:flake8]
changedir = changedir =
deps = flake8 deps = flake8
commands = flake8 commands = flake8