mirror
/
borg
mirror of https://github.com/borgbackup/borg.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge remote-tracking branch 'upstream/master'

This commit is contained in:
8bit 2017-10-15 17:52:47 -05:00
parent 8db0c770c3 ac0ad9adf0
commit 9340688b4c
17 changed files with 125 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.travis/install.sh
View File

@ -4,8 +4,6 @@ set -e
set -x
if [[ "$(uname -s)" == 'Darwin' ]]; then
brew update || brew update
if [[ "${OPENSSL}" != "0.9.8" ]]; then
brew outdated openssl || brew upgrade openssl
fi

7
README.rst
View File

@ -89,9 +89,12 @@ Main features
Easy to use
~~~~~~~~~~~
Initialize a new backup repository and create a backup archive::
Initialize a new backup repository (see ``borg init --help`` for encryption options)::
$ borg init -e repokey /path/to/repo
Create a backup archive::
$ borg init /path/to/repo
$ borg create /path/to/repo::Saturday1 ~/Documents
Now doing another backup, just to show off the great deduplication::

25
docs/faq.rst
View File

@ -682,6 +682,31 @@ the corruption is caused by a one time event such as a power outage,
running `borg check --repair` will fix most problems.
Why isn't there more progress / ETA information displayed?
----------------------------------------------------------
Some borg runs take quite a bit, so it would be nice to see a progress display,
maybe even including a ETA (expected time of "arrival" [here rather "completion"]).
For some functionality, this can be done: if the total amount of work is more or
less known, we can display progress. So check if there is a ``--progress`` option.
But sometimes, the total amount is unknown (e.g. for ``borg create`` we just do
a single pass over the filesystem, so we do not know the total file count or data
volume before reaching the end). Adding another pass just to determine that would
take additional time and could be incorrect, if the filesystem is changing.
Even if the fs does not change and we knew count and size of all files, we still
could not compute the ``borg create`` ETA as we do not know the amount of changed
chunks, how the bandwidth of source and destination or system performance might
fluctuate.
You see, trying to display ETA would be futile. The borg developers prefer to
rather not implement progress / ETA display than doing futile attempts.
See also: https://xkcd.com/612/
Miscellaneous
#############

6
docs/man_intro.rst
View File

@ -32,8 +32,8 @@ fully trusted targets.
Borg stores a set of files in an *archive*. A *repository* is a collection
of *archives*. The format of repositories is Borg-specific. Borg does not
distinguish archives from each other in a any way other than their name,
it does not matter when or where archives where created (eg. different hosts).
distinguish archives from each other in any way other than their name,
it does not matter when or where archives were created (e.g. different hosts).
EXAMPLES
--------
@ -61,7 +61,7 @@ SEE ALSO
`borg-compression(1)`, `borg-patterns(1)`, `borg-placeholders(1)`
* Main web site https://borgbackup.readthedocs.org/
* Main web site https://www.borgbackup.org/
* Releases https://github.com/borgbackup/borg/releases
* Changelog https://github.com/borgbackup/borg/blob/master/docs/changes.rst
* GitHub https://github.com/borgbackup/borg

8
docs/usage/general.rst
View File

@ -30,3 +30,11 @@ Common options
All Borg commands share these options:
.. include:: common-options.rst.inc
Examples
~~~~~~~~
::
# Create an archive and log: borg version, files list, return code
$ borg create --show-version --list --show-rc /path/to/repo::my-files files

2
docs/usage/list.rst
View File

@ -19,7 +19,7 @@ Examples
-rwxr-xr-x root root 2140 Fri, 2015-03-27 20:24:22 bin/bzdiff
...
$ borg list /path/to/repo::archiveA --list-format="{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}"
$ borg list /path/to/repo::archiveA --format="{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}"
drwxrwxr-x user user 0 Sun, 2015-02-01 11:00:00 .
drwxrwxr-x user user 0 Sun, 2015-02-01 11:00:00 code
drwxrwxr-x user user 0 Sun, 2015-02-01 11:00:00 code/myproject

11
setup.py
View File

@ -12,7 +12,7 @@ from distutils.core import Command
import textwrap
min_python = (3, 4)
min_python = (3, 5)
my_python = sys.version_info
if my_python < min_python:
@ -40,6 +40,8 @@ extras_require = {
# llfuse 0.42 (tested shortly, looks ok), needs FUSE version >= 2.8.0
# llfuse 1.0 (tested shortly, looks ok), needs FUSE version >= 2.8.0
# llfuse 1.1.1 (tested shortly, looks ok), needs FUSE version >= 2.8.0
# llfuse 1.2 (tested shortly, looks ok), needs FUSE version >= 2.8.0
# llfuse 1.3 (tested shortly, looks ok), needs FUSE version >= 2.8.0
# llfuse 2.0 will break API
'fuse': ['llfuse<2.0', ],
}
@ -655,6 +657,13 @@ class build_man(Command):
def gen_man_page(self, name, rst):
from docutils.writers import manpage
from docutils.core import publish_string
from docutils.nodes import inline
from docutils.parsers.rst import roles
def issue(name, rawtext, text, lineno, inliner, options={}, content=[]):
return [inline(rawtext, '#' + text)], []
roles.register_local_role('issue', issue)
# We give the source_path so that docutils can find relative includes
# as-if the document where located in the docs/ directory.
man_page = publish_string(source=rst, source_path='docs/virtmanpage.rst', writer=manpage.Writer())

8
src/borg/archive.py
View File

@ -557,7 +557,7 @@ Utilization of max. archive size: {csize_max:.0%}
original_path = original_path or item.path
dest = self.cwd
if item.path.startswith(('/', '..')):
if item.path.startswith(('/', '../')):
raise Exception('Path should be relative and local')
path = os.path.join(dest, item.path)
# Attempt to remove existing files, ignore errors on failure
@ -965,7 +965,9 @@ class ChunksProcessor:
length = len(item.chunks)
# the item should only have the *additional* chunks we processed after the last partial item:
item.chunks = item.chunks[from_chunk:]
item.get_size(memorize=True)
# for borg recreate, we already have a size member in the source item (giving the total file size),
# but we consider only a part of the file here, thus we must recompute the size from the chunks:
item.get_size(memorize=True, from_chunks=True)
item.path += '.borg_part_%d' % number
item.part = number
number += 1
@ -1813,7 +1815,7 @@ class ArchiveRecreater:
target.save(comment=comment, additional_metadata={
# keep some metadata as in original archive:
'time': archive.metadata.time,
'time_end': archive.metadata.time_end,
'time_end': archive.metadata.get('time_end') or archive.metadata.time,
'cmdline': archive.metadata.cmdline,
# but also remember recreate metadata:
'recreate_cmdline': sys.argv,

13
src/borg/archiver.py
View File

@ -40,7 +40,7 @@ from .archive import FilesystemObjectProcessors, MetadataCollector, ChunksProces
from .cache import Cache, assert_secure
from .constants import * # NOQA
from .compress import CompressionSpec
from .crypto.key import key_creator, tam_required_file, tam_required, RepoKey, PassphraseKey
from .crypto.key import key_creator, key_argument_names, tam_required_file, tam_required, RepoKey, PassphraseKey
from .crypto.keymanager import KeyManager
from .helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR
from .helpers import Error, NoManifestError, set_ec
@ -1326,7 +1326,7 @@ class Archiver:
keep += prune_split(archives, '%Y', args.yearly, keep)
to_delete = (set(archives) | checkpoints) - (set(keep) | set(keep_checkpoints))
stats = Statistics()
with Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) as cache:
with Cache(repository, key, manifest, do_files=False, lock_wait=self.lock_wait) as cache:
list_logger = logging.getLogger('borg.output.list')
if args.output_list:
# set up counters for the progress display
@ -1959,6 +1959,8 @@ class Archiver:
parser.print_help()
return EXIT_SUCCESS
do_maincommand_help = do_subcommand_help
def preprocess_args(self, args):
deprecations = [
# ('--old', '--new' or None, 'Warning: "--old" has been deprecated. Use "--new" instead.'),
@ -2152,8 +2154,6 @@ class Archiver:
help='show/log the borg version')
add_common_option('--show-rc', dest='show_rc', action='store_true',
help='show/log the return code (rc)')
add_common_option('--no-files-cache', dest='cache_files', action='store_false',
help='do not load/update the file metadata cache used to detect unchanged files')
add_common_option('--umask', metavar='M', dest='umask', type=lambda s: int(s, 8), default=UMASK_DEFAULT,
help='set umask to M (local and remote, default: %(default)04o)')
add_common_option('--remote-path', metavar='PATH', dest='remote_path',
@ -2226,6 +2226,7 @@ class Archiver:
parser = argparse.ArgumentParser(prog=self.prog, description='Borg - Deduplicated Backups',
add_help=False)
parser.set_defaults(func=functools.partial(self.do_maincommand_help, parser))
parser.common_options = self.CommonOptions(define_common_options,
suffix_precedence=('_maincommand', '_midcommand', '_subcommand'))
parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + __version__,
@ -2383,7 +2384,7 @@ class Archiver:
type=location_validator(archive=False),
help='repository to create')
subparser.add_argument('-e', '--encryption', metavar='MODE', dest='encryption', required=True,
choices=('none', 'keyfile', 'repokey', 'keyfile-blake2', 'repokey-blake2', 'authenticated'),
choices=key_argument_names(),
help='select encryption key mode **(required)**')
subparser.add_argument('--append-only', dest='append_only', action='store_true',
help='create an append-only mode repository')
@ -2723,6 +2724,8 @@ class Archiver:
help='output stats as JSON. Implies ``--stats``.')
subparser.add_argument('--no-cache-sync', dest='no_cache_sync', action='store_true',
help='experimental: do not synchronize the cache. Implies not using the files cache.')
subparser.add_argument('--no-files-cache', dest='cache_files', action='store_false',
help='do not load/update the file metadata cache used to detect unchanged files')
define_exclusion_group(subparser, tag_files=True)

12
src/borg/cache.py
View File

@ -85,11 +85,11 @@ class SecurityManager:
logger.debug('security: current location %s', current_location)
logger.debug('security: key type %s', str(key.TYPE))
logger.debug('security: manifest timestamp %s', manifest.timestamp)
with open(self.location_file, 'w') as fd:
with SaveFile(self.location_file) as fd:
fd.write(current_location)
with open(self.key_type_file, 'w') as fd:
with SaveFile(self.key_type_file) as fd:
fd.write(str(key.TYPE))
with open(self.manifest_ts_file, 'w') as fd:
with SaveFile(self.manifest_ts_file) as fd:
fd.write(manifest.timestamp)
def assert_location_matches(self, cache_config=None):
@ -119,7 +119,7 @@ class SecurityManager:
raise Cache.RepositoryAccessAborted()
# adapt on-disk config immediately if the new location was accepted
logger.debug('security: updating location stored in cache and security dir')
with open(self.location_file, 'w') as fd:
with SaveFile(self.location_file) as fd:
fd.write(repository_location)
if cache_config:
cache_config.save()
@ -470,7 +470,7 @@ class LocalCache(CacheStatsMixin):
self.cache_config.create()
ChunkIndex().write(os.path.join(self.path, 'chunks'))
os.makedirs(os.path.join(self.path, 'chunks.archive.d'))
with SaveFile(os.path.join(self.path, 'files'), binary=True) as fd:
with SaveFile(os.path.join(self.path, 'files'), binary=True):
pass # empty file
def _do_open(self):
@ -844,7 +844,7 @@ class LocalCache(CacheStatsMixin):
shutil.rmtree(os.path.join(self.path, 'chunks.archive.d'))
os.makedirs(os.path.join(self.path, 'chunks.archive.d'))
self.chunks = ChunkIndex()
with open(os.path.join(self.path, 'files'), 'wb'):
with SaveFile(os.path.join(self.path, 'files'), binary=True):
pass # empty file
self.cache_config.manifest_id = ''
self.cache_config._config.set('cache', 'manifest', '')

21
src/borg/compress.pyx
View File

@ -246,7 +246,7 @@ class Auto(CompressorBase):
lz4_data = self.lz4.compress(data)
ratio = len(lz4_data) / len(data)
if ratio < 0.97:
return self.compressor, None
return self.compressor, lz4_data
elif ratio < 1:
return self.lz4, lz4_data
else:
@ -257,9 +257,24 @@ class Auto(CompressorBase):
def compress(self, data):
compressor, lz4_data = self._decide(data)
if lz4_data is None:
return compressor.compress(data)
if compressor is self.lz4:
# we know that trying to compress with expensive compressor is likely pointless,
# but lz4 managed to at least squeeze the data a bit.
return lz4_data
if compressor is self.none:
# we know that trying to compress with expensive compressor is likely pointless
# and also lz4 did not manage to squeeze the data (not even a bit).
uncompressed_data = compressor.compress(data)
return uncompressed_data
# if we get here, the decider decided to try the expensive compressor.
# we also know that lz4_data is smaller than uncompressed data.
exp_compressed_data = compressor.compress(data)
ratio = len(exp_compressed_data) / len(lz4_data)
if ratio < 0.99:
# the expensive compressor managed to squeeze the data significantly better than lz4.
return exp_compressed_data
else:
# otherwise let's just store the lz4 data, which decompresses extremely fast.
return lz4_data
def decompress(self, data):

5
src/borg/crypto/key.py
View File

@ -103,11 +103,16 @@ class KeyBlobStorage:
def key_creator(repository, args):
for key in AVAILABLE_KEY_TYPES:
if key.ARG_NAME == args.encryption:
assert key.ARG_NAME is not None
return key.create(repository, args)
else:
raise ValueError('Invalid encryption mode "%s"' % args.encryption)
def key_argument_names():
return [key.ARG_NAME for key in AVAILABLE_KEY_TYPES if key.ARG_NAME]
def identify_key(manifest_data):
key_type = manifest_data[0]
if key_type == PassphraseKey.TYPE:

10
src/borg/helpers/fs.py
View File

@ -1,3 +1,4 @@
import errno
import os
import os.path
import re
@ -141,8 +142,13 @@ def truncate_and_unlink(path):
recover. Refer to the "File system interaction" section
in repository.py for further explanations.
"""
with open(path, 'r+b') as fd:
fd.truncate()
try:
with open(path, 'r+b') as fd:
fd.truncate()
except OSError as err:
if err.errno != errno.ENOTSUP:
raise
# don't crash if the above ops are not supported.
os.unlink(path)

2
src/borg/logger.py
View File

@ -80,6 +80,8 @@ def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF', lev
logging.config.fileConfig(f)
configured = True
logger = logging.getLogger(__name__)
borg_logger = logging.getLogger('borg')
borg_logger.json = json
logger.debug('using logging configuration read from "{0}"'.format(conf_fname))
warnings.showwarning = _log_warning
return None

11
src/borg/platform/linux.pyx
View File

@ -72,8 +72,11 @@ BSD_TO_LINUX_FLAGS = {
def set_flags(path, bsd_flags, fd=None):
if fd is None and stat.S_ISLNK(os.lstat(path).st_mode):
return
if fd is None:
st = os.stat(path, follow_symlinks=False)
if stat.S_ISBLK(st.st_mode) or stat.S_ISCHR(st.st_mode) or stat.S_ISLNK(st.st_mode):
# see comment in get_flags()
return
cdef int flags = 0
for bsd_flag, linux_flag in BSD_TO_LINUX_FLAGS.items():
if bsd_flags & bsd_flag:
@ -92,6 +95,10 @@ def set_flags(path, bsd_flags, fd=None):
def get_flags(path, st):
if stat.S_ISBLK(st.st_mode) or stat.S_ISCHR(st.st_mode) or stat.S_ISLNK(st.st_mode):
# avoid opening devices files - trying to open non-present devices can be rather slow.
# avoid opening symlinks, O_NOFOLLOW would make the open() fail anyway.
return 0
cdef int linux_flags
try:
fd = os.open(path, os.O_RDONLY|os.O_NONBLOCK|os.O_NOFOLLOW)

2
src/borg/repository.py
View File

@ -266,7 +266,7 @@ class Repository:
try:
os.link(config_path, old_config_path)
except OSError as e:
if e.errno in (errno.EMLINK, errno.ENOSYS, errno.EPERM):
if e.errno in (errno.EMLINK, errno.ENOSYS, errno.EPERM, errno.ENOTSUP):
logger.warning("Hardlink failed, cannot securely erase old config file")
else:
raise

16
src/borg/testsuite/compress.py
View File

@ -110,12 +110,18 @@ def test_compressor():
def test_auto():
compressor = CompressionSpec('auto,zlib,9').compressor
compressor_auto_zlib = CompressionSpec('auto,zlib,9').compressor
compressor_lz4 = CompressionSpec('lz4').compressor
compressor_zlib = CompressionSpec('zlib,9').compressor
data = bytes(500)
compressed_auto_zlib = compressor_auto_zlib.compress(data)
compressed_lz4 = compressor_lz4.compress(data)
compressed_zlib = compressor_zlib.compress(data)
ratio = len(compressed_zlib) / len(compressed_lz4)
assert Compressor.detect(compressed_auto_zlib) == ZLIB if ratio < 0.99 else LZ4
compressed = compressor.compress(bytes(500))
assert Compressor.detect(compressed) == ZLIB
compressed = compressor.compress(b'\x00\xb8\xa3\xa2-O\xe1i\xb6\x12\x03\xc21\xf3\x8a\xf78\\\x01\xa5b\x07\x95\xbeE\xf8\xa3\x9ahm\xb1~')
data = b'\x00\xb8\xa3\xa2-O\xe1i\xb6\x12\x03\xc21\xf3\x8a\xf78\\\x01\xa5b\x07\x95\xbeE\xf8\xa3\x9ahm\xb1~'
compressed = compressor_auto_zlib.compress(data)
assert Compressor.detect(compressed) == CNONE