Merge branch '1.0-maint' into merge-1.0-maint

# Conflicts:
#	MANIFEST.in
#	Vagrantfile
#	docs/changes.rst
#	docs/usage/mount.rst.inc
#	src/borg/archiver.py
#	src/borg/fuse.py
#	src/borg/repository.py
This commit is contained in:
Thomas Waldmann 2017-01-29 05:49:53 +01:00
commit c0dc644ef6
13 changed files with 144 additions and 54 deletions

View File

@ -1,12 +1,8 @@
include README.rst AUTHORS LICENSE CHANGES.rst MANIFEST.in
graft src
recursive-exclude src *.pyc
recursive-exclude src *.pyo
recursive-exclude src *.so
recursive-include docs *
recursive-exclude docs *.pyc
recursive-exclude docs *.pyo
prune docs/_build
exclude .coveragerc .gitattributes .gitignore .travis.yml Vagrantfile
prune .travis
prune .github
exclude .coveragerc .gitattributes .gitignore .travis.yml Vagrantfile
graft src
graft docs
prune docs/_build
global-exclude *.py[co] *.orig *.so *.dll

32
Vagrantfile vendored
View File

@ -63,11 +63,13 @@ end
def packages_darwin
return <<-EOF
# install all the (security and other) updates
sudo softwareupdate --ignore iTunesX
sudo softwareupdate --ignore iTunes
sudo softwareupdate --install --all
# get osxfuse 3.x release code from github:
curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.5.3/osxfuse-3.5.3.dmg >osxfuse.dmg
curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.5.4/osxfuse-3.5.4.dmg >osxfuse.dmg
MOUNTDIR=$(echo `hdiutil mount osxfuse.dmg | tail -1 | awk '{$1="" ; print $0}'` | xargs -0 echo) \
&& sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for macOS 3.5.3.pkg" -target /
&& sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for macOS 3.5.4.pkg" -target /
sudo chown -R vagrant /usr/local # brew must be able to create stuff here
ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
brew update
@ -83,11 +85,13 @@ end
def packages_freebsd
return <<-EOF
# VM has no hostname set
hostname freebsd
# install all the (security and other) updates, base system
freebsd-update --not-running-from-cron fetch install
# for building borgbackup and dependencies:
pkg install -y openssl liblz4 fusefs-libs pkgconf
pkg install -y fakeroot git bash
pkg install -y git bash
# for building python:
pkg install -y sqlite3
# make bash default / work:
@ -227,7 +231,7 @@ def install_pythons(boxname)
pyenv install 3.4.0 # tests
pyenv install 3.5.0 # tests
pyenv install 3.6.0 # tests
pyenv install 3.5.2 # binary build, use latest 3.5.x release
pyenv install 3.5.3 # binary build, use latest 3.5.x release
pyenv rehash
EOF
end
@ -245,8 +249,8 @@ def build_pyenv_venv(boxname)
. ~/.bash_profile
cd /vagrant/borg
# use the latest 3.5 release
pyenv global 3.5.2
pyenv virtualenv 3.5.2 borg-env
pyenv global 3.5.3
pyenv virtualenv 3.5.3 borg-env
ln -s ~/.pyenv/versions/borg-env .
EOF
end
@ -446,6 +450,16 @@ Vagrant.configure(2) do |config|
# OS X
config.vm.define "darwin64" do |b|
b.vm.box = "jhcook/yosemite-clitools"
b.vm.provider :virtualbox do |v|
v.customize ['modifyvm', :id, '--ostype', 'MacOS1010_64']
v.customize ['modifyvm', :id, '--paravirtprovider', 'default']
# Adjust CPU settings according to
# https://github.com/geerlingguy/macos-virtualbox-vm
v.customize ['modifyvm', :id, '--cpuidset',
'00000001', '000306a9', '00020800', '80000201', '178bfbff']
# Disable USB variant requiring Virtualbox proprietary extension pack
v.customize ["modifyvm", :id, '--usbehci', 'off', '--usbxhci', 'off']
end
b.vm.provision "packages darwin", :type => :shell, :privileged => false, :inline => packages_darwin
b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("darwin64")
b.vm.provision "fix pyenv", :type => :shell, :privileged => false, :inline => fix_pyenv_darwin("darwin64")
@ -458,11 +472,11 @@ Vagrant.configure(2) do |config|
end
# BSD
# note: the FreeBSD-10.3-STABLE box needs "vagrant up" twice to start.
# note: the FreeBSD-10.3-RELEASE box needs "vagrant up" twice to start.
config.vm.define "freebsd64" do |b|
b.vm.box = "freebsd/FreeBSD-10.3-STABLE"
b.vm.box = "freebsd/FreeBSD-10.3-RELEASE"
b.vm.provider :virtualbox do |v|
v.memory = 768
v.memory = 1536
end
b.ssh.shell = "sh"
b.vm.provision "install system packages", :type => :shell, :inline => packages_freebsd

View File

@ -5,8 +5,8 @@ This section is used for infos about security and corruption issues.
.. _tam_vuln:
Pre-1.0.9 manifest spoofing vulnerability
-----------------------------------------
Pre-1.0.9 manifest spoofing vulnerability (CVE-2016-10099)
----------------------------------------------------------
A flaw in the cryptographic authentication scheme in Borg allowed an attacker
to spoof the manifest. The attack requires an attacker to be able to
@ -54,7 +54,9 @@ Vulnerability time line:
* 2016-11-14: Vulnerability and fix discovered during review of cryptography by Marian Beermann (@enkore)
* 2016-11-20: First patch
* 2016-12-18: Released fixed versions: 1.0.9, 1.1.0b3
* 2016-12-20: Released fixed version 1.0.9
* 2017-01-02: CVE was assigned
* 2017-01-15: Released fixed version 1.1.0b3 (fix was previously only available from source)
.. _attic013_check_corruption:
@ -207,8 +209,8 @@ Other changes:
- remove all BORG_* env vars from the outer environment
Version 1.0.10rc1 (not released yet)
------------------------------------
Version 1.0.10rc1 (2017-01-29)
------------------------------
Bug fixes:
@ -223,9 +225,16 @@ Bug fixes:
- Fixed change-passphrase crashing with unencrypted repositories, #1978
- Fixed "borg check repo::archive" indicating success if "archive" does not exist, #1997
- borg check: print non-exit-code warning if --last or --prefix aren't fulfilled
- fix bad parsing of wrong repo location syntax
- create: don't create hard link refs to failed files,
mount: handle invalid hard link refs, #2092
- detect mingw byte order, #2073
- creating a new segment: use "xb" mode, #2099
- mount: umount on SIGINT/^C when in foreground, #2082
Other changes:
- binary: use fixed AND freshly compiled pyinstaller bootloader, #2002
- xattr: ignore empty names returned by llistxattr(2) et al
- Enable the fault handler: install handlers for the SIGSEGV, SIGFPE, SIGABRT,
SIGBUS and SIGILL signals to dump the Python traceback.
@ -235,8 +244,11 @@ Other changes:
- tests:
- vagrant / travis / tox: add Python 3.6 based testing
- vagrant: fix openbsd repo, fixes #2042
- vagrant: fix the freebsd64 machine, #2037
- vagrant: fix openbsd repo, #2042
- vagrant: fix the freebsd64 machine, #2037 #2067
- vagrant: use python 3.5.3 to build binaries, #2078
- vagrant: use osxfuse 3.5.4 for tests / to build binaries
vagrant: improve darwin64 VM settings
- travis: fix osxfuse install (fixes OS X testing on Travis CI)
- travis: require succeeding OS X tests, #2028
- travis: use latest pythons for OS X based testing
@ -248,12 +260,18 @@ Other changes:
- language clarification - VM backup FAQ
- borg create: document how to backup stdin, #2013
- borg upgrade: fix incorrect title levels
- add CVE numbers for issues fixed in 1.0.9, #2106
- fix typos (taken from Debian package patch)
- remote: include data hexdump in "unexpected RPC data" error message
- remote: log SSH command line at debug level
- API_VERSION: use numberspaces, #2023
- remove .github from pypi package, #2051
- add pip and setuptools to requirements file, #2030
- SyncFile: fix use of fd object after close (cosmetic)
- Manifest.in: simplify, exclude *.{so,dll,orig}, #2066
- ignore posix_fadvise errors in repository.py, #2095
(works around issues with docker on ARM)
- make LoggedIO.close_segment reentrant, avoid reentrance
Version 1.0.9 (2016-12-20)
@ -264,10 +282,14 @@ Security fixes:
- A flaw in the cryptographic authentication scheme in Borg allowed an attacker
to spoof the manifest. See :ref:`tam_vuln` above for the steps you should
take.
CVE-2016-10099 was assigned to this vulnerability.
- borg check: When rebuilding the manifest (which should only be needed very rarely)
duplicate archive names would be handled on a "first come first serve" basis, allowing
an attacker to apparently replace archives.
CVE-2016-10100 was assigned to this vulnerability.
Bug fixes:
- borg check:

View File

@ -12,18 +12,26 @@
#include <sys/isa_defs.h>
#endif
#if (defined(BYTE_ORDER)&&(BYTE_ORDER == BIG_ENDIAN)) || \
(defined(_BIG_ENDIAN)&&defined(__SVR4)&&defined(__sun))
#define _le32toh(x) __builtin_bswap32(x)
#define _htole32(x) __builtin_bswap32(x)
#elif (defined(BYTE_ORDER)&&(BYTE_ORDER == LITTLE_ENDIAN)) || \
(defined(_LITTLE_ENDIAN)&&defined(__SVR4)&&defined(__sun))
#define _le32toh(x) (x)
#define _htole32(x) (x)
#if (defined(BYTE_ORDER) && (BYTE_ORDER == BIG_ENDIAN)) || \
(defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)) || \
(defined(_BIG_ENDIAN) && defined(__SVR4)&&defined(__sun))
#define BORG_BIG_ENDIAN 1
#elif (defined(BYTE_ORDER) && (BYTE_ORDER == LITTLE_ENDIAN)) || \
(defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) || \
(defined(_LITTLE_ENDIAN) && defined(__SVR4)&&defined(__sun))
#define BORG_BIG_ENDIAN 0
#else
#error Unknown byte order
#endif
#if BORG_BIG_ENDIAN
#define _le32toh(x) __builtin_bswap32(x)
#define _htole32(x) __builtin_bswap32(x)
#else
#define _le32toh(x) (x)
#define _htole32(x) (x)
#endif
#define MAGIC "BORG_IDX"
#define MAGIC_LEN 8

View File

@ -838,8 +838,6 @@ Number of files: {0.stats.nfiles}'''.format(
self.add_item(item)
status = 'h' # regular file, hardlink (to already seen inodes)
return status
else:
self.hard_links[st.st_ino, st.st_dev] = safe_path
is_special_file = is_special(st.st_mode)
if not is_special_file:
path_hash = self.key.id_hash(safe_encode(os.path.join(self.cwd, path)))
@ -890,6 +888,9 @@ Number of files: {0.stats.nfiles}'''.format(
item.mode = stat.S_IFREG | stat.S_IMODE(item.mode)
self.stats.nfiles += 1
self.add_item(item)
if st.st_nlink > 1 and source is None:
# Add the hard link reference *after* the file has been added to the archive.
self.hard_links[st.st_ino, st.st_dev] = safe_path
return status
@staticmethod

View File

@ -2244,6 +2244,13 @@ class Archiver:
to tweak the performance. It sets the number of cached data chunks; additional
memory usage can be up to ~8 MiB times this number. The default is the number
of CPU cores.
When the daemonized process receives a signal or crashes, it does not unmount.
Unmounting in these cases could cause an active rsync or similar process
to unintentionally delete data.
When running in the foreground ^C/SIGINT unmounts cleanly, but other
signals or crashes do not.
""")
subparser = subparsers.add_parser('mount', parents=[common_parser], add_help=False,
description=self.do_mount.__doc__,

View File

@ -5,6 +5,7 @@ import stat
import tempfile
import time
from collections import defaultdict
from signal import SIGINT
from distutils.version import LooseVersion
from zlib import adler32
@ -125,7 +126,8 @@ class FuseOperations(llfuse.Operations):
umount = False
try:
signal = fuse_main()
umount = (signal is None) # no crash and no signal -> umount request
# no crash and no signal (or it's ^C and we're in the foreground) -> umount request
umount = (signal is None or (signal == SIGINT and foreground))
finally:
llfuse.close(umount)
@ -190,6 +192,7 @@ class FuseOperations(llfuse.Operations):
path = os.fsencode(os.path.normpath(item.path))
self.file_versions[path] = version
path = item.path
del item.path # safe some space
if 'source' in item and stat.S_ISREG(item.mode):
# a hardlink, no contents, <source> is the hardlink master
@ -199,7 +202,11 @@ class FuseOperations(llfuse.Operations):
version = self.file_versions[source]
source = make_versioned_name(source, version, add_dir=True)
name = make_versioned_name(name, version)
inode = self._find_inode(source, prefix)
try:
inode = self._find_inode(source, prefix)
except KeyError:
logger.warning('Skipping broken hard link: %s -> %s', path, item.source)
return
item = self.cache.get(inode)
item.nlink = item.get('nlink', 1) + 1
self.items[inode] = item

View File

@ -926,11 +926,17 @@ class Location:
"""
# path must not contain :: (it ends at :: or string end), but may contain single colons.
# to avoid ambiguities with other regexes, it must also not start with ":".
# to avoid ambiguities with other regexes, it must also not start with ":" nor with "//" nor with "ssh://".
path_re = r"""
(?!:) # not starting with ":"
(?!(:|//|ssh://)) # not starting with ":" or // or ssh://
(?P<path>([^:]|(:(?!:)))+) # any chars, but no "::"
"""
# abs_path must not contain :: (it ends at :: or string end), but may contain single colons.
# it must start with a / and that slash is part of the path.
abs_path_re = r"""
(?P<path>(/([^:]|(:(?!:)))+)) # start with /, then any chars, but no "::"
"""
# optional ::archive_name at the end, archive name must not contain "/".
# borg mount's FUSE filesystem creates one level of directories from
# the archive names and of course "/" is not valid in a directory name.
@ -945,7 +951,7 @@ class Location:
(?P<proto>ssh):// # ssh://
""" + optional_user_re + r""" # user@ (optional)
(?P<host>[^:/]+)(?::(?P<port>\d+))? # host or host:port
""" + path_re + optional_archive_re, re.VERBOSE) # path or path::archive
""" + abs_path_re + optional_archive_re, re.VERBOSE) # path or path::archive
file_re = re.compile(r"""
(?P<proto>file):// # file://

View File

@ -8,7 +8,7 @@ Public APIs are documented in platform.base.
from .base import acl_get, acl_set
from .base import set_flags, get_flags
from .base import SaveFile, SyncFile, sync_dir, fdatasync
from .base import SaveFile, SyncFile, sync_dir, fdatasync, safe_fadvise
from .base import swidth, umount, API_VERSION
from .base import process_alive, get_process_id, local_pid_alive

View File

@ -63,6 +63,21 @@ def sync_dir(path):
os.close(fd)
def safe_fadvise(fd, offset, len, advice):
if hasattr(os, 'posix_fadvise'):
try:
os.posix_fadvise(fd, offset, len, advice)
except OSError:
# usually, posix_fadvise can't fail for us, but there seem to
# be failures when running borg under docker on ARM, likely due
# to a bug outside of borg.
# also, there is a python wrapper bug, always giving errno = 0.
# https://github.com/borgbackup/borg/issues/2095
# as this call is not critical for correct function (just to
# optimize cache usage), we ignore these errors.
pass
class SyncFile:
"""
A file class that is supposed to enable write ordering (one way or another) and data durability after close().
@ -103,15 +118,21 @@ class SyncFile:
from .. import platform
self.fd.flush()
platform.fdatasync(self.fileno)
if hasattr(os, 'posix_fadvise'):
os.posix_fadvise(self.fileno, 0, 0, os.POSIX_FADV_DONTNEED)
# tell the OS that it does not need to cache what we just wrote,
# avoids spoiling the cache for the OS and other processes.
safe_fadvise(self.fileno, 0, 0, os.POSIX_FADV_DONTNEED)
def close(self):
"""sync() and close."""
from .. import platform
self.sync()
self.fd.close()
platform.sync_dir(os.path.dirname(self.fd.name))
dirname = None
try:
dirname = os.path.dirname(self.fd.name)
self.sync()
finally:
self.fd.close()
if dirname:
platform.sync_dir(dirname)
class SaveFile:

View File

@ -8,6 +8,7 @@ from ..helpers import posix_acl_use_stored_uid_gid
from ..helpers import user2uid, group2gid
from ..helpers import safe_decode, safe_encode
from .base import SyncFile as BaseSyncFile
from .base import safe_fadvise
from .posix import swidth
from libc cimport errno
@ -216,7 +217,7 @@ cdef _sync_file_range(fd, offset, length, flags):
assert length & PAGE_MASK == 0, "length %d not page-aligned" % length
if sync_file_range(fd, offset, length, flags) != 0:
raise OSError(errno.errno, os.strerror(errno.errno))
os.posix_fadvise(fd, offset, length, os.POSIX_FADV_DONTNEED)
safe_fadvise(fd, offset, length, os.POSIX_FADV_DONTNEED)
cdef unsigned PAGE_MASK = resource.getpagesize() - 1
@ -251,7 +252,9 @@ class SyncFile(BaseSyncFile):
def sync(self):
self.fd.flush()
os.fdatasync(self.fileno)
os.posix_fadvise(self.fileno, 0, 0, os.POSIX_FADV_DONTNEED)
# tell the OS that it does not need to cache what we just wrote,
# avoids spoiling the cache for the OS and other processes.
safe_fadvise(self.fileno, 0, 0, os.POSIX_FADV_DONTNEED)
def umount(mountpoint):

View File

@ -21,7 +21,7 @@ from .helpers import yes
from .locking import Lock, LockError, LockErrorT
from .logger import create_logger
from .lrucache import LRUCache
from .platform import SaveFile, SyncFile, sync_dir
from .platform import SaveFile, SyncFile, sync_dir, safe_fadvise
from .crc32 import crc32
logger = create_logger(__name__)
@ -909,8 +909,7 @@ class LoggedIO:
self.fds = None # Just to make sure we're disabled
def close_fd(self, fd):
if hasattr(os, 'posix_fadvise'): # only on UNIX
os.posix_fadvise(fd.fileno(), 0, 0, os.POSIX_FADV_DONTNEED)
safe_fadvise(fd.fileno(), 0, 0, os.POSIX_FADV_DONTNEED)
fd.close()
def segment_iterator(self, segment=None, reverse=False):
@ -1017,11 +1016,12 @@ class LoggedIO:
return fd
def close_segment(self):
if self._write_fd:
# set self._write_fd to None early to guard against reentry from error handling code pathes:
fd, self._write_fd = self._write_fd, None
if fd is not None:
self.segment += 1
self.offset = 0
self._write_fd.close()
self._write_fd = None
fd.close()
def delete_segment(self, segment):
if segment in self.fds:

View File

@ -137,6 +137,11 @@ class TestLocationWithoutEnv:
location_time2 = Location('/some/path::archive{now:%s}')
assert location_time1.archive != location_time2.archive
def test_bad_syntax(self):
with pytest.raises(ValueError):
# this is invalid due to the 2nd colon, correct: 'ssh://user@host/path'
Location('ssh://user@host:/path')
class TestLocationWithEnv:
def test_ssh(self, monkeypatch):