mirror
/
borg
mirror of https://github.com/borgbackup/borg.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch '1.0-maint'

This commit is contained in:
Thomas Waldmann 2016-10-24 21:41:20 +02:00
parent 0ff2352691 77110913c7
commit 09e74af7bf
14 changed files with 164 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
Vagrantfile vendored
View File

@ -25,6 +25,8 @@ def packages_debianoid
# for building borgbackup and dependencies:
apt-get install -y libssl-dev libacl1-dev liblz4-dev libfuse-dev fuse pkg-config
usermod -a -G fuse $username
chgrp fuse /dev/fuse
chmod 666 /dev/fuse
apt-get install -y fakeroot build-essential git
apt-get install -y python3-dev python3-setuptools
# for building python:
@ -45,6 +47,8 @@ def packages_redhatted
# for building borgbackup and dependencies:
yum install -y openssl-devel openssl libacl-devel libacl lz4-devel fuse-devel fuse pkgconfig
usermod -a -G fuse vagrant
chgrp fuse /dev/fuse
chmod 666 /dev/fuse
yum install -y fakeroot gcc git patch
# needed to compile msgpack-python (otherwise it will use slow fallback code):
yum install -y gcc-c++
@ -96,6 +100,8 @@ def packages_freebsd
kldload fuse
sysctl vfs.usermount=1
pw groupmod operator -M vagrant
# /dev/fuse has group operator
chmod 666 /dev/fuse
touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
# install all the (security and other) updates, packages
pkg update
@ -106,10 +112,6 @@ end
def packages_openbsd
return <<-EOF
. ~/.profile
mkdir -p /home/vagrant/borg
rsync -aH /vagrant/borg/ /home/vagrant/borg/
rm -rf /vagrant/borg
ln -sf /home/vagrant/borg /vagrant/
pkg_add bash
chsh -s /usr/local/bin/bash vagrant
pkg_add openssl
@ -121,6 +123,8 @@ def packages_openbsd
easy_install-3.4 pip
pip3 install virtualenv
touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
# avoid that breaking llfuse install breaks borgbackup install under tox:
sed -i.bak '/fuse.txt/d' /vagrant/borg/borg/tox.ini
EOF
end
@ -146,6 +150,8 @@ def packages_netbsd
easy_install-3.4 pip
pip install virtualenv
touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
# fuse does not work good enough (see above), do not install llfuse:
sed -i.bak '/fuse.txt/d' /vagrant/borg/borg/tox.ini
EOF
end
@ -273,13 +279,13 @@ def run_tests(boxname)
. ~/.bash_profile
cd /vagrant/borg/borg
. ../borg-env/bin/activate
if which pyenv > /dev/null; then
if which pyenv 2> /dev/null; then
# for testing, use the earliest point releases of the supported python versions:
pyenv global 3.4.0 3.5.0
pyenv local 3.4.0 3.5.0
fi
# otherwise: just use the system python
if which fakeroot > /dev/null; then
if which fakeroot 2> /dev/null; then
echo "Running tox WITH fakeroot -u"
fakeroot -u tox --skip-missing-interpreters
else
@ -304,7 +310,7 @@ end
Vagrant.configure(2) do |config|
# use rsync to copy content to the folder
config.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"]
config.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"], :rsync__chown => false
# do not let the VM access . on the host machine via the default shared folder!
config.vm.synced_folder ".", "/vagrant", disabled: true
@ -443,7 +449,7 @@ Vagrant.configure(2) do |config|
end
config.vm.define "openbsd64" do |b|
b.vm.box = "kaorimatz/openbsd-5.9-amd64"
b.vm.box = "openbsd60-64" # note: basic openbsd install for vagrant WITH sudo and rsync pre-installed
b.vm.provider :virtualbox do |v|
v.memory = 768
end
@ -454,7 +460,7 @@ Vagrant.configure(2) do |config|
end
config.vm.define "netbsd64" do |b|
b.vm.box = "alex-skimlinks/netbsd-6.1.5-amd64"
b.vm.box = "netbsd70-64"
b.vm.provider :virtualbox do |v|
v.memory = 768
end

4
docs/api.rst
View File

@ -26,6 +26,10 @@ API Documentation
:members:
:undoc-members:
.. automodule:: borg.keymanager
:members:
:undoc-members:
.. automodule:: borg.nonces
:members:
:undoc-members:

37
docs/changes.rst
View File

@ -218,8 +218,8 @@ Other changes:
- ChunkBuffer: add test for leaving partial chunk in buffer, fixes #945
Version 1.0.8rc1 (not released yet)
-----------------------------------
Version 1.0.8rc1 (2016-10-17)
-----------------------------
Bug fixes:
@ -231,15 +231,22 @@ Bug fixes:
also correctly processes broken symlinks. before this regressed to a crash
(5b45385) a broken symlink would've been skipped.
- process_symlink: fix missing backup_io()
Fixes a chmod/chown/chgrp/unlink/rename/... crash race between getting dirents
and dispatching to process_symlink.
- yes(): abort on wrong answers, saying so
Fixes a chmod/chown/chgrp/unlink/rename/... crash race between getting
dirents and dispatching to process_symlink.
- yes(): abort on wrong answers, saying so, #1622
- fixed exception borg serve raised when connection was closed before reposiory
was openend. add an error message for this.
- fix read-from-closed-FD issue, #1551
(this seems not to get triggered in 1.0.x, but was discovered in master)
- hashindex: fix iterators (always raise StopIteration when exhausted)
(this seems not to get triggered in 1.0.x, but was discovered in master)
- enable relative pathes in ssh:// repo URLs, via /./relpath hack, fixes #1655
- allow repo pathes with colons, fixes #1705
- update changed repo location immediately after acceptance, #1524
- fix debug get-obj / delete-obj crash if object not found and remote repo,
#1684
- pyinstaller: use a spec file to build borg.exe binary, exclude osxfuse dylib
on Mac OS X (avoids mismatch lib <-> driver), #1619
New features:
@ -250,6 +257,8 @@ New features:
special "paper" format with by line checksums for printed backups. For the
paper format, the import is an interactive process which checks each line as
soon as it is input.
- add "borg debug-refcount-obj" to determine a repo objects' referrer counts,
#1352
Other changes:
@ -258,10 +267,19 @@ Other changes:
- setup.py: Add subcommand support to build_usage.
- remote: change exception message for unexpected RPC data format to indicate
dataflow direction.
- vagrant:
- improved messages / error reporting:
- IntegrityError: add placeholder for message, so that the message we give
appears not only in the traceback, but also in the (short) error message,
#1572
- borg.key: include chunk id in exception msgs, #1571
- better messages for cache newer than repo, fixes #1700
- vagrant (testing/build VMs):
- upgrade OSXfuse / FUSE for macOS to 3.5.2
- update Debian Wheezy boxes to 7.11
- update Debian Wheezy boxes, #1686
- openbsd / netbsd: use own boxes, fixes misc rsync installation and
fuse/llfuse related testing issues, #1695 #1696 #1670 #1671 #1728
- docs:
- add docs for "key export" and "key import" commands, #1641
@ -277,12 +295,17 @@ Other changes:
- add debug-info usage help file
- internals.rst: fix typos
- setup.py: fix build_usage to always process all commands
- added docs explaining multiple --restrict-to-path flags, #1602
- add more specific warning about write-access debug commands, #1587
- clarify FAQ regarding backup of virtual machines, #1672
- tests:
- work around fuse xattr test issue with recent fakeroot
- simplify repo/hashindex tests
- travis: test fuse-enabled borg, use trusty to have a recent FUSE
- re-enable fuse tests for RemoteArchiver (no deadlocks any more)
- clean env for pytest based tests, #1714
- fuse_mount contextmanager: accept any options
Version 1.0.7 (2016-08-19)

4
docs/deployment.rst
View File

@ -149,10 +149,10 @@ package manager to install and keep borg up-to-date.
- authorized_key: user="{{ user }}"
key="{{ item.key }}"
key_options='command="cd {{ pool }}/{{ item.host }};borg serve --restrict-to-path {{ pool }}/{{ item.host }}",no-port-forwarding,no-X11-forwarding,no-pty,no-agent-forwarding,no-user-rc'
with_items: auth_users
with_items: "{{ auth_users }}"
- file: path="{{ home }}/.ssh/authorized_keys" owner="{{ user }}" group="{{ group }}" mode=0600 state=file
- file: path="{{ pool }}/{{ item.host }}" owner="{{ user }}" group="{{ group }}" mode=0700 state=directory
with_items: auth_users
with_items: "{{ auth_users }}"
Salt
----

7
docs/usage.rst
View File

@ -897,9 +897,14 @@ That's all to it.
Drawbacks
+++++++++
As data is only appended, and nothing deleted, commands like ``prune`` or ``delete``
As data is only appended, and nothing removed, commands like ``prune`` or ``delete``
won't free disk space, they merely tag data as deleted in a new transaction.
Be aware that as soon as you write to the repo in non-append-only mode (e.g. prune,
delete or create archives from an admin machine), it will remove the deleted objects
permanently (including the ones that were already marked as deleted, but not removed,
in append-only mode).
Note that you can go back-and-forth between normal and append-only operation by editing
the configuration file, it's not a "one way trip".

2
requirements.d/fuse.txt
View File

@ -1,4 +1,4 @@
# low-level FUSE support library for "borg mount"
# see comments setup.py about this version requirement.
# please see the comments in setup.py about llfuse.
llfuse<2.0

5
setup.py
View File

@ -23,12 +23,17 @@ on_rtd = os.environ.get('READTHEDOCS')
# Also, we might use some rather recent API features.
install_requires = ['msgpack-python>=0.4.6', ]
# note for package maintainers: if you package borgbackup for distribution,
# please add llfuse as a *requirement* on all platforms that have a working
# llfuse package. "borg mount" needs llfuse to work.
# if you do not have llfuse, do not require it, most of borgbackup will work.
extras_require = {
# llfuse 0.40 (tested, proven, ok), needs FUSE version >= 2.8.0
# llfuse 0.41 (tested shortly, looks ok), needs FUSE version >= 2.8.0
# llfuse 0.41.1 (tested shortly, looks ok), needs FUSE version >= 2.8.0
# llfuse 0.42 (tested shortly, looks ok), needs FUSE version >= 2.8.0
# llfuse 1.0 (tested shortly, looks ok), needs FUSE version >= 2.8.0
# llfuse 1.1.1 (tested shortly, looks ok), needs FUSE version >= 2.8.0
# llfuse 2.0 will break API
'fuse': ['llfuse<2.0', ],
}

5
src/borg/__init__.py
View File

@ -1,3 +1,6 @@
# This is a python package
from distutils.version import LooseVersion
from ._version import version as __version__
__version_tuple__ = tuple(LooseVersion(__version__).version[:3])

53
src/borg/archiver.py
View File

@ -1176,7 +1176,7 @@ class Archiver:
else:
try:
data = repository.get(id)
except repository.ObjectNotFound:
except Repository.ObjectNotFound:
print("object %s not found." % hex_id)
else:
with open(args.path, "wb") as f:
@ -1210,13 +1210,29 @@ class Archiver:
repository.delete(id)
modified = True
print("object %s deleted." % hex_id)
except repository.ObjectNotFound:
except Repository.ObjectNotFound:
print("object %s not found." % hex_id)
if modified:
repository.commit()
print('Done.')
return EXIT_SUCCESS
@with_repository(manifest=False, exclusive=True, cache=True)
def do_debug_refcount_obj(self, args, repository, manifest, key, cache):
"""display refcounts for the objects with the given IDs"""
for hex_id in args.ids:
try:
id = unhexlify(hex_id)
except ValueError:
print("object id %s is invalid." % hex_id)
else:
try:
refcount = cache.chunks[id][0]
print("object %s has %d referrers [info from chunks cache]." % (hex_id, refcount))
except KeyError:
print("object %s not found [info from chunks cache]." % hex_id)
return EXIT_SUCCESS
@with_repository(lock=False, manifest=False)
def do_break_lock(self, args, repository):
"""Break the repository lock (e.g. in case it was left by a dead borg."""
@ -1344,7 +1360,19 @@ class Archiver:
{borgversion}
The version of borg.
The version of borg, e.g.: 1.0.8rc1
{borgmajor}
The version of borg, only the major version, e.g.: 1
{borgminor}
The version of borg, only major and minor version, e.g.: 1.0
{borgpatch}
The version of borg, only major, minor and patch version, e.g.: 1.0.8
Examples::
@ -1777,8 +1805,8 @@ class Archiver:
'.checkpoint.N' (with N being a number), because these names are used for
checkpoints and treated in special ways.
In the archive name, you may use the following format tags:
{now}, {utcnow}, {fqdn}, {hostname}, {user}, {pid}, {uuid4}, {borgversion}
In the archive name, you may use the following placeholders:
{now}, {utcnow}, {fqdn}, {hostname}, {user} and some others.
To speed up pulling backups over sshfs and similar network file systems which do
not provide correct inode information the --ignore-inode flag can be used. This
@ -2541,6 +2569,21 @@ class Archiver:
subparser.add_argument('ids', metavar='IDs', nargs='+', type=str,
help='hex object ID(s) to delete from the repo')
debug_refcount_obj_epilog = textwrap.dedent("""
This command displays the reference count for objects from the repository.
""")
subparser = debug_parsers.add_parser('refcount-obj', parents=[common_parser], add_help=False,
description=self.do_debug_refcount_obj.__doc__,
epilog=debug_refcount_obj_epilog,
formatter_class=argparse.RawDescriptionHelpFormatter,
help='show refcount for object from repository (debug)')
subparser.set_defaults(func=self.do_debug_refcount_obj)
subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
type=location_validator(archive=False),
help='repository to use')
subparser.add_argument('ids', metavar='IDs', nargs='+', type=str,
help='hex object ID(s) to show refcounts for')
return parser
@staticmethod

20
src/borg/helpers.py
View File

@ -34,6 +34,7 @@ from .logger import create_logger
logger = create_logger()
from . import __version__ as borg_version
from . import __version_tuple__ as borg_version_tuple
from . import chunker
from . import crypto
from . import hashindex
@ -664,6 +665,9 @@ def replace_placeholders(text):
'user': uid2user(os.getuid(), os.getuid()),
'uuid4': str(uuid.uuid4()),
'borgversion': borg_version,
'borgmajor': '%d' % borg_version_tuple[:1],
'borgminor': '%d.%d' % borg_version_tuple[:2],
'borgpatch': '%d.%d.%d' % borg_version_tuple[:3],
}
return format_line(text, data)
@ -945,26 +949,32 @@ class Location:
return True
def _parse(self, text):
def normpath_special(p):
# avoid that normpath strips away our relative path hack and even makes p absolute
relative = p.startswith('/./')
p = os.path.normpath(p)
return ('/.' + p) if relative else p
m = self.ssh_re.match(text)
if m:
self.proto = m.group('proto')
self.user = m.group('user')
self.host = m.group('host')
self.port = m.group('port') and int(m.group('port')) or None
self.path = os.path.normpath(m.group('path'))
self.path = normpath_special(m.group('path'))
self.archive = m.group('archive')
return True
m = self.file_re.match(text)
if m:
self.proto = m.group('proto')
self.path = os.path.normpath(m.group('path'))
self.path = normpath_special(m.group('path'))
self.archive = m.group('archive')
return True
m = self.scp_re.match(text)
if m:
self.user = m.group('user')
self.host = m.group('host')
self.path = os.path.normpath(m.group('path'))
self.path = normpath_special(m.group('path'))
self.archive = m.group('archive')
self.proto = self.host and 'ssh' or 'file'
return True
@ -995,9 +1005,9 @@ class Location:
return self.path
else:
if self.path and self.path.startswith('~'):
path = '/' + self.path
path = '/' + self.path # /~/x = path x relative to home dir
elif self.path and not self.path.startswith('/'):
path = '/~/' + self.path
path = '/./' + self.path # /./x = path x relative to cwd
else:
path = self.path
return 'ssh://{}{}{}{}'.format('{}@'.format(self.user) if self.user else '',

6
src/borg/remote.py
View File

@ -149,8 +149,10 @@ class RepositoryServer: # pragma: no cover
def open(self, path, create=False, lock_wait=None, lock=True, exclusive=None, append_only=False):
path = os.fsdecode(path)
if path.startswith('/~'):
path = os.path.join(get_home_dir(), path[2:])
if path.startswith('/~'): # /~/x = path x relative to home dir, /~username/x = relative to "user" home dir
path = os.path.join(get_home_dir(), path[2:]) # XXX check this (see also 1.0-maint), is it correct for ~u?
elif path.startswith('/./'): # /./x = path x relative to cwd
path = path[3:]
path = os.path.realpath(path)
if self.restrict_to_paths:
# if --restrict-to-path P is given, we make sure that we only operate in/below path P.

13
src/borg/testsuite/__init__.py
View File

@ -117,6 +117,15 @@ def is_utime_fully_supported():
return False
def no_selinux(x):
# selinux fails our FUSE tests, thus ignore selinux xattrs
SELINUX_KEY = 'security.selinux'
if isinstance(x, dict):
return {k: v for k, v in x.items() if k != SELINUX_KEY}
if isinstance(x, list):
return [k for k in x if k != SELINUX_KEY]
class BaseTestCase(unittest.TestCase):
"""
"""
@ -176,8 +185,8 @@ class BaseTestCase(unittest.TestCase):
else:
d1.append(round(s1.st_mtime_ns, st_mtime_ns_round))
d2.append(round(s2.st_mtime_ns, st_mtime_ns_round))
d1.append(get_all(path1, follow_symlinks=False))
d2.append(get_all(path2, follow_symlinks=False))
d1.append(no_selinux(get_all(path1, follow_symlinks=False)))
d2.append(no_selinux(get_all(path2, follow_symlinks=False)))
self.assert_equal(d1, d2)
for sub_diff in diff.subdirs.values():
self._assert_dirs_equal_cmp(sub_diff)

12
src/borg/testsuite/archiver.py
View File

@ -39,8 +39,10 @@ from ..keymanager import RepoIdMismatch, NotABorgKeyFile
from ..remote import RemoteRepository, PathNotAllowed
from ..repository import Repository
from . import has_lchflags, has_llfuse
from . import BaseTestCase, changedir, environment_variable
from . import BaseTestCase, changedir, environment_variable, no_selinux
from . import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported, is_utime_fully_supported
from .platform import fakeroot_detected
src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
@ -1428,7 +1430,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
in_fn = 'input/fusexattr'
out_fn = os.path.join(mountpoint, 'input', 'fusexattr')
if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path):
assert xattr.listxattr(out_fn) == ['user.foo', ]
assert no_selinux(xattr.listxattr(out_fn)) == ['user.foo', ]
assert xattr.getxattr(out_fn, 'user.foo') == b'bar'
else:
assert xattr.listxattr(out_fn) == []
@ -1988,6 +1990,12 @@ class ArchiverTestCaseBinary(ArchiverTestCase):
def test_overwrite(self):
pass
def test_fuse(self):
if fakeroot_detected():
unittest.skip('test_fuse with the binary is not compatible with fakeroot')
else:
super().test_fuse()
class ArchiverCheckTestCase(ArchiverTestCaseBase):

10
src/borg/testsuite/helpers.py
View File

@ -84,6 +84,8 @@ class TestLocationWithoutEnv:
"Location(proto='file', user=None, host=None, port=None, path='/some/absolute/path', archive='archive')"
assert repr(Location('/some/absolute/path')) == \
"Location(proto='file', user=None, host=None, port=None, path='/some/absolute/path', archive=None)"
assert repr(Location('ssh://user@host/some/path')) == \
"Location(proto='ssh', user='user', host='host', port=None, path='/some/path', archive=None)"
def test_relpath(self, monkeypatch):
monkeypatch.delenv('BORG_REPO', raising=False)
@ -91,6 +93,12 @@ class TestLocationWithoutEnv:
"Location(proto='file', user=None, host=None, port=None, path='some/relative/path', archive='archive')"
assert repr(Location('some/relative/path')) == \
"Location(proto='file', user=None, host=None, port=None, path='some/relative/path', archive=None)"
assert repr(Location('ssh://user@host/./some/path')) == \
"Location(proto='ssh', user='user', host='host', port=None, path='/./some/path', archive=None)"
assert repr(Location('ssh://user@host/~/some/path')) == \
"Location(proto='ssh', user='user', host='host', port=None, path='/~/some/path', archive=None)"
assert repr(Location('ssh://user@host/~user/some/path')) == \
"Location(proto='ssh', user='user', host='host', port=None, path='/~user/some/path', archive=None)"
def test_with_colons(self, monkeypatch):
monkeypatch.delenv('BORG_REPO', raising=False)
@ -122,7 +130,7 @@ class TestLocationWithoutEnv:
'ssh://user@host:1234/some/path::archive']
for location in locations:
assert Location(location).canonical_path() == \
Location(Location(location).canonical_path()).canonical_path()
Location(Location(location).canonical_path()).canonical_path(), "failed: %s" % location
def test_format_path(self, monkeypatch):
monkeypatch.delenv('BORG_REPO', raising=False)