mirror of https://github.com/borgbackup/borg.git
Merge remote-tracking branch 'upstream/master'
This commit is contained in:
commit
9340688b4c
|
@ -4,8 +4,6 @@ set -e
|
|||
set -x
|
||||
|
||||
if [[ "$(uname -s)" == 'Darwin' ]]; then
|
||||
brew update || brew update
|
||||
|
||||
if [[ "${OPENSSL}" != "0.9.8" ]]; then
|
||||
brew outdated openssl || brew upgrade openssl
|
||||
fi
|
||||
|
|
|
@ -89,9 +89,12 @@ Main features
|
|||
Easy to use
|
||||
~~~~~~~~~~~
|
||||
|
||||
Initialize a new backup repository and create a backup archive::
|
||||
Initialize a new backup repository (see ``borg init --help`` for encryption options)::
|
||||
|
||||
$ borg init -e repokey /path/to/repo
|
||||
|
||||
Create a backup archive::
|
||||
|
||||
$ borg init /path/to/repo
|
||||
$ borg create /path/to/repo::Saturday1 ~/Documents
|
||||
|
||||
Now doing another backup, just to show off the great deduplication::
|
||||
|
|
25
docs/faq.rst
25
docs/faq.rst
|
@ -682,6 +682,31 @@ the corruption is caused by a one time event such as a power outage,
|
|||
running `borg check --repair` will fix most problems.
|
||||
|
||||
|
||||
Why isn't there more progress / ETA information displayed?
|
||||
----------------------------------------------------------
|
||||
|
||||
Some borg runs take quite a bit, so it would be nice to see a progress display,
|
||||
maybe even including a ETA (expected time of "arrival" [here rather "completion"]).
|
||||
|
||||
For some functionality, this can be done: if the total amount of work is more or
|
||||
less known, we can display progress. So check if there is a ``--progress`` option.
|
||||
|
||||
But sometimes, the total amount is unknown (e.g. for ``borg create`` we just do
|
||||
a single pass over the filesystem, so we do not know the total file count or data
|
||||
volume before reaching the end). Adding another pass just to determine that would
|
||||
take additional time and could be incorrect, if the filesystem is changing.
|
||||
|
||||
Even if the fs does not change and we knew count and size of all files, we still
|
||||
could not compute the ``borg create`` ETA as we do not know the amount of changed
|
||||
chunks, how the bandwidth of source and destination or system performance might
|
||||
fluctuate.
|
||||
|
||||
You see, trying to display ETA would be futile. The borg developers prefer to
|
||||
rather not implement progress / ETA display than doing futile attempts.
|
||||
|
||||
See also: https://xkcd.com/612/
|
||||
|
||||
|
||||
Miscellaneous
|
||||
#############
|
||||
|
||||
|
|
|
@ -32,8 +32,8 @@ fully trusted targets.
|
|||
|
||||
Borg stores a set of files in an *archive*. A *repository* is a collection
|
||||
of *archives*. The format of repositories is Borg-specific. Borg does not
|
||||
distinguish archives from each other in a any way other than their name,
|
||||
it does not matter when or where archives where created (eg. different hosts).
|
||||
distinguish archives from each other in any way other than their name,
|
||||
it does not matter when or where archives were created (e.g. different hosts).
|
||||
|
||||
EXAMPLES
|
||||
--------
|
||||
|
@ -61,7 +61,7 @@ SEE ALSO
|
|||
|
||||
`borg-compression(1)`, `borg-patterns(1)`, `borg-placeholders(1)`
|
||||
|
||||
* Main web site https://borgbackup.readthedocs.org/
|
||||
* Main web site https://www.borgbackup.org/
|
||||
* Releases https://github.com/borgbackup/borg/releases
|
||||
* Changelog https://github.com/borgbackup/borg/blob/master/docs/changes.rst
|
||||
* GitHub https://github.com/borgbackup/borg
|
||||
|
|
|
@ -30,3 +30,11 @@ Common options
|
|||
All Borg commands share these options:
|
||||
|
||||
.. include:: common-options.rst.inc
|
||||
|
||||
Examples
|
||||
~~~~~~~~
|
||||
::
|
||||
|
||||
# Create an archive and log: borg version, files list, return code
|
||||
$ borg create --show-version --list --show-rc /path/to/repo::my-files files
|
||||
|
||||
|
|
|
@ -19,7 +19,7 @@ Examples
|
|||
-rwxr-xr-x root root 2140 Fri, 2015-03-27 20:24:22 bin/bzdiff
|
||||
...
|
||||
|
||||
$ borg list /path/to/repo::archiveA --list-format="{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}"
|
||||
$ borg list /path/to/repo::archiveA --format="{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}"
|
||||
drwxrwxr-x user user 0 Sun, 2015-02-01 11:00:00 .
|
||||
drwxrwxr-x user user 0 Sun, 2015-02-01 11:00:00 code
|
||||
drwxrwxr-x user user 0 Sun, 2015-02-01 11:00:00 code/myproject
|
||||
|
|
11
setup.py
11
setup.py
|
@ -12,7 +12,7 @@ from distutils.core import Command
|
|||
|
||||
import textwrap
|
||||
|
||||
min_python = (3, 4)
|
||||
min_python = (3, 5)
|
||||
my_python = sys.version_info
|
||||
|
||||
if my_python < min_python:
|
||||
|
@ -40,6 +40,8 @@ extras_require = {
|
|||
# llfuse 0.42 (tested shortly, looks ok), needs FUSE version >= 2.8.0
|
||||
# llfuse 1.0 (tested shortly, looks ok), needs FUSE version >= 2.8.0
|
||||
# llfuse 1.1.1 (tested shortly, looks ok), needs FUSE version >= 2.8.0
|
||||
# llfuse 1.2 (tested shortly, looks ok), needs FUSE version >= 2.8.0
|
||||
# llfuse 1.3 (tested shortly, looks ok), needs FUSE version >= 2.8.0
|
||||
# llfuse 2.0 will break API
|
||||
'fuse': ['llfuse<2.0', ],
|
||||
}
|
||||
|
@ -655,6 +657,13 @@ class build_man(Command):
|
|||
def gen_man_page(self, name, rst):
|
||||
from docutils.writers import manpage
|
||||
from docutils.core import publish_string
|
||||
from docutils.nodes import inline
|
||||
from docutils.parsers.rst import roles
|
||||
|
||||
def issue(name, rawtext, text, lineno, inliner, options={}, content=[]):
|
||||
return [inline(rawtext, '#' + text)], []
|
||||
|
||||
roles.register_local_role('issue', issue)
|
||||
# We give the source_path so that docutils can find relative includes
|
||||
# as-if the document where located in the docs/ directory.
|
||||
man_page = publish_string(source=rst, source_path='docs/virtmanpage.rst', writer=manpage.Writer())
|
||||
|
|
|
@ -557,7 +557,7 @@ Utilization of max. archive size: {csize_max:.0%}
|
|||
|
||||
original_path = original_path or item.path
|
||||
dest = self.cwd
|
||||
if item.path.startswith(('/', '..')):
|
||||
if item.path.startswith(('/', '../')):
|
||||
raise Exception('Path should be relative and local')
|
||||
path = os.path.join(dest, item.path)
|
||||
# Attempt to remove existing files, ignore errors on failure
|
||||
|
@ -965,7 +965,9 @@ class ChunksProcessor:
|
|||
length = len(item.chunks)
|
||||
# the item should only have the *additional* chunks we processed after the last partial item:
|
||||
item.chunks = item.chunks[from_chunk:]
|
||||
item.get_size(memorize=True)
|
||||
# for borg recreate, we already have a size member in the source item (giving the total file size),
|
||||
# but we consider only a part of the file here, thus we must recompute the size from the chunks:
|
||||
item.get_size(memorize=True, from_chunks=True)
|
||||
item.path += '.borg_part_%d' % number
|
||||
item.part = number
|
||||
number += 1
|
||||
|
@ -1813,7 +1815,7 @@ class ArchiveRecreater:
|
|||
target.save(comment=comment, additional_metadata={
|
||||
# keep some metadata as in original archive:
|
||||
'time': archive.metadata.time,
|
||||
'time_end': archive.metadata.time_end,
|
||||
'time_end': archive.metadata.get('time_end') or archive.metadata.time,
|
||||
'cmdline': archive.metadata.cmdline,
|
||||
# but also remember recreate metadata:
|
||||
'recreate_cmdline': sys.argv,
|
||||
|
|
|
@ -40,7 +40,7 @@ from .archive import FilesystemObjectProcessors, MetadataCollector, ChunksProces
|
|||
from .cache import Cache, assert_secure
|
||||
from .constants import * # NOQA
|
||||
from .compress import CompressionSpec
|
||||
from .crypto.key import key_creator, tam_required_file, tam_required, RepoKey, PassphraseKey
|
||||
from .crypto.key import key_creator, key_argument_names, tam_required_file, tam_required, RepoKey, PassphraseKey
|
||||
from .crypto.keymanager import KeyManager
|
||||
from .helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR
|
||||
from .helpers import Error, NoManifestError, set_ec
|
||||
|
@ -1326,7 +1326,7 @@ class Archiver:
|
|||
keep += prune_split(archives, '%Y', args.yearly, keep)
|
||||
to_delete = (set(archives) | checkpoints) - (set(keep) | set(keep_checkpoints))
|
||||
stats = Statistics()
|
||||
with Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) as cache:
|
||||
with Cache(repository, key, manifest, do_files=False, lock_wait=self.lock_wait) as cache:
|
||||
list_logger = logging.getLogger('borg.output.list')
|
||||
if args.output_list:
|
||||
# set up counters for the progress display
|
||||
|
@ -1959,6 +1959,8 @@ class Archiver:
|
|||
parser.print_help()
|
||||
return EXIT_SUCCESS
|
||||
|
||||
do_maincommand_help = do_subcommand_help
|
||||
|
||||
def preprocess_args(self, args):
|
||||
deprecations = [
|
||||
# ('--old', '--new' or None, 'Warning: "--old" has been deprecated. Use "--new" instead.'),
|
||||
|
@ -2152,8 +2154,6 @@ class Archiver:
|
|||
help='show/log the borg version')
|
||||
add_common_option('--show-rc', dest='show_rc', action='store_true',
|
||||
help='show/log the return code (rc)')
|
||||
add_common_option('--no-files-cache', dest='cache_files', action='store_false',
|
||||
help='do not load/update the file metadata cache used to detect unchanged files')
|
||||
add_common_option('--umask', metavar='M', dest='umask', type=lambda s: int(s, 8), default=UMASK_DEFAULT,
|
||||
help='set umask to M (local and remote, default: %(default)04o)')
|
||||
add_common_option('--remote-path', metavar='PATH', dest='remote_path',
|
||||
|
@ -2226,6 +2226,7 @@ class Archiver:
|
|||
|
||||
parser = argparse.ArgumentParser(prog=self.prog, description='Borg - Deduplicated Backups',
|
||||
add_help=False)
|
||||
parser.set_defaults(func=functools.partial(self.do_maincommand_help, parser))
|
||||
parser.common_options = self.CommonOptions(define_common_options,
|
||||
suffix_precedence=('_maincommand', '_midcommand', '_subcommand'))
|
||||
parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + __version__,
|
||||
|
@ -2383,7 +2384,7 @@ class Archiver:
|
|||
type=location_validator(archive=False),
|
||||
help='repository to create')
|
||||
subparser.add_argument('-e', '--encryption', metavar='MODE', dest='encryption', required=True,
|
||||
choices=('none', 'keyfile', 'repokey', 'keyfile-blake2', 'repokey-blake2', 'authenticated'),
|
||||
choices=key_argument_names(),
|
||||
help='select encryption key mode **(required)**')
|
||||
subparser.add_argument('--append-only', dest='append_only', action='store_true',
|
||||
help='create an append-only mode repository')
|
||||
|
@ -2723,6 +2724,8 @@ class Archiver:
|
|||
help='output stats as JSON. Implies ``--stats``.')
|
||||
subparser.add_argument('--no-cache-sync', dest='no_cache_sync', action='store_true',
|
||||
help='experimental: do not synchronize the cache. Implies not using the files cache.')
|
||||
subparser.add_argument('--no-files-cache', dest='cache_files', action='store_false',
|
||||
help='do not load/update the file metadata cache used to detect unchanged files')
|
||||
|
||||
define_exclusion_group(subparser, tag_files=True)
|
||||
|
||||
|
|
|
@ -85,11 +85,11 @@ class SecurityManager:
|
|||
logger.debug('security: current location %s', current_location)
|
||||
logger.debug('security: key type %s', str(key.TYPE))
|
||||
logger.debug('security: manifest timestamp %s', manifest.timestamp)
|
||||
with open(self.location_file, 'w') as fd:
|
||||
with SaveFile(self.location_file) as fd:
|
||||
fd.write(current_location)
|
||||
with open(self.key_type_file, 'w') as fd:
|
||||
with SaveFile(self.key_type_file) as fd:
|
||||
fd.write(str(key.TYPE))
|
||||
with open(self.manifest_ts_file, 'w') as fd:
|
||||
with SaveFile(self.manifest_ts_file) as fd:
|
||||
fd.write(manifest.timestamp)
|
||||
|
||||
def assert_location_matches(self, cache_config=None):
|
||||
|
@ -119,7 +119,7 @@ class SecurityManager:
|
|||
raise Cache.RepositoryAccessAborted()
|
||||
# adapt on-disk config immediately if the new location was accepted
|
||||
logger.debug('security: updating location stored in cache and security dir')
|
||||
with open(self.location_file, 'w') as fd:
|
||||
with SaveFile(self.location_file) as fd:
|
||||
fd.write(repository_location)
|
||||
if cache_config:
|
||||
cache_config.save()
|
||||
|
@ -470,7 +470,7 @@ class LocalCache(CacheStatsMixin):
|
|||
self.cache_config.create()
|
||||
ChunkIndex().write(os.path.join(self.path, 'chunks'))
|
||||
os.makedirs(os.path.join(self.path, 'chunks.archive.d'))
|
||||
with SaveFile(os.path.join(self.path, 'files'), binary=True) as fd:
|
||||
with SaveFile(os.path.join(self.path, 'files'), binary=True):
|
||||
pass # empty file
|
||||
|
||||
def _do_open(self):
|
||||
|
@ -844,7 +844,7 @@ class LocalCache(CacheStatsMixin):
|
|||
shutil.rmtree(os.path.join(self.path, 'chunks.archive.d'))
|
||||
os.makedirs(os.path.join(self.path, 'chunks.archive.d'))
|
||||
self.chunks = ChunkIndex()
|
||||
with open(os.path.join(self.path, 'files'), 'wb'):
|
||||
with SaveFile(os.path.join(self.path, 'files'), binary=True):
|
||||
pass # empty file
|
||||
self.cache_config.manifest_id = ''
|
||||
self.cache_config._config.set('cache', 'manifest', '')
|
||||
|
|
|
@ -246,7 +246,7 @@ class Auto(CompressorBase):
|
|||
lz4_data = self.lz4.compress(data)
|
||||
ratio = len(lz4_data) / len(data)
|
||||
if ratio < 0.97:
|
||||
return self.compressor, None
|
||||
return self.compressor, lz4_data
|
||||
elif ratio < 1:
|
||||
return self.lz4, lz4_data
|
||||
else:
|
||||
|
@ -257,9 +257,24 @@ class Auto(CompressorBase):
|
|||
|
||||
def compress(self, data):
|
||||
compressor, lz4_data = self._decide(data)
|
||||
if lz4_data is None:
|
||||
return compressor.compress(data)
|
||||
if compressor is self.lz4:
|
||||
# we know that trying to compress with expensive compressor is likely pointless,
|
||||
# but lz4 managed to at least squeeze the data a bit.
|
||||
return lz4_data
|
||||
if compressor is self.none:
|
||||
# we know that trying to compress with expensive compressor is likely pointless
|
||||
# and also lz4 did not manage to squeeze the data (not even a bit).
|
||||
uncompressed_data = compressor.compress(data)
|
||||
return uncompressed_data
|
||||
# if we get here, the decider decided to try the expensive compressor.
|
||||
# we also know that lz4_data is smaller than uncompressed data.
|
||||
exp_compressed_data = compressor.compress(data)
|
||||
ratio = len(exp_compressed_data) / len(lz4_data)
|
||||
if ratio < 0.99:
|
||||
# the expensive compressor managed to squeeze the data significantly better than lz4.
|
||||
return exp_compressed_data
|
||||
else:
|
||||
# otherwise let's just store the lz4 data, which decompresses extremely fast.
|
||||
return lz4_data
|
||||
|
||||
def decompress(self, data):
|
||||
|
|
|
@ -103,11 +103,16 @@ class KeyBlobStorage:
|
|||
def key_creator(repository, args):
|
||||
for key in AVAILABLE_KEY_TYPES:
|
||||
if key.ARG_NAME == args.encryption:
|
||||
assert key.ARG_NAME is not None
|
||||
return key.create(repository, args)
|
||||
else:
|
||||
raise ValueError('Invalid encryption mode "%s"' % args.encryption)
|
||||
|
||||
|
||||
def key_argument_names():
|
||||
return [key.ARG_NAME for key in AVAILABLE_KEY_TYPES if key.ARG_NAME]
|
||||
|
||||
|
||||
def identify_key(manifest_data):
|
||||
key_type = manifest_data[0]
|
||||
if key_type == PassphraseKey.TYPE:
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
import errno
|
||||
import os
|
||||
import os.path
|
||||
import re
|
||||
|
@ -141,8 +142,13 @@ def truncate_and_unlink(path):
|
|||
recover. Refer to the "File system interaction" section
|
||||
in repository.py for further explanations.
|
||||
"""
|
||||
try:
|
||||
with open(path, 'r+b') as fd:
|
||||
fd.truncate()
|
||||
except OSError as err:
|
||||
if err.errno != errno.ENOTSUP:
|
||||
raise
|
||||
# don't crash if the above ops are not supported.
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
|
|
|
@ -80,6 +80,8 @@ def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF', lev
|
|||
logging.config.fileConfig(f)
|
||||
configured = True
|
||||
logger = logging.getLogger(__name__)
|
||||
borg_logger = logging.getLogger('borg')
|
||||
borg_logger.json = json
|
||||
logger.debug('using logging configuration read from "{0}"'.format(conf_fname))
|
||||
warnings.showwarning = _log_warning
|
||||
return None
|
||||
|
|
|
@ -72,7 +72,10 @@ BSD_TO_LINUX_FLAGS = {
|
|||
|
||||
|
||||
def set_flags(path, bsd_flags, fd=None):
|
||||
if fd is None and stat.S_ISLNK(os.lstat(path).st_mode):
|
||||
if fd is None:
|
||||
st = os.stat(path, follow_symlinks=False)
|
||||
if stat.S_ISBLK(st.st_mode) or stat.S_ISCHR(st.st_mode) or stat.S_ISLNK(st.st_mode):
|
||||
# see comment in get_flags()
|
||||
return
|
||||
cdef int flags = 0
|
||||
for bsd_flag, linux_flag in BSD_TO_LINUX_FLAGS.items():
|
||||
|
@ -92,6 +95,10 @@ def set_flags(path, bsd_flags, fd=None):
|
|||
|
||||
|
||||
def get_flags(path, st):
|
||||
if stat.S_ISBLK(st.st_mode) or stat.S_ISCHR(st.st_mode) or stat.S_ISLNK(st.st_mode):
|
||||
# avoid opening devices files - trying to open non-present devices can be rather slow.
|
||||
# avoid opening symlinks, O_NOFOLLOW would make the open() fail anyway.
|
||||
return 0
|
||||
cdef int linux_flags
|
||||
try:
|
||||
fd = os.open(path, os.O_RDONLY|os.O_NONBLOCK|os.O_NOFOLLOW)
|
||||
|
|
|
@ -266,7 +266,7 @@ class Repository:
|
|||
try:
|
||||
os.link(config_path, old_config_path)
|
||||
except OSError as e:
|
||||
if e.errno in (errno.EMLINK, errno.ENOSYS, errno.EPERM):
|
||||
if e.errno in (errno.EMLINK, errno.ENOSYS, errno.EPERM, errno.ENOTSUP):
|
||||
logger.warning("Hardlink failed, cannot securely erase old config file")
|
||||
else:
|
||||
raise
|
||||
|
|
|
@ -110,12 +110,18 @@ def test_compressor():
|
|||
|
||||
|
||||
def test_auto():
|
||||
compressor = CompressionSpec('auto,zlib,9').compressor
|
||||
compressor_auto_zlib = CompressionSpec('auto,zlib,9').compressor
|
||||
compressor_lz4 = CompressionSpec('lz4').compressor
|
||||
compressor_zlib = CompressionSpec('zlib,9').compressor
|
||||
data = bytes(500)
|
||||
compressed_auto_zlib = compressor_auto_zlib.compress(data)
|
||||
compressed_lz4 = compressor_lz4.compress(data)
|
||||
compressed_zlib = compressor_zlib.compress(data)
|
||||
ratio = len(compressed_zlib) / len(compressed_lz4)
|
||||
assert Compressor.detect(compressed_auto_zlib) == ZLIB if ratio < 0.99 else LZ4
|
||||
|
||||
compressed = compressor.compress(bytes(500))
|
||||
assert Compressor.detect(compressed) == ZLIB
|
||||
|
||||
compressed = compressor.compress(b'\x00\xb8\xa3\xa2-O\xe1i\xb6\x12\x03\xc21\xf3\x8a\xf78\\\x01\xa5b\x07\x95\xbeE\xf8\xa3\x9ahm\xb1~')
|
||||
data = b'\x00\xb8\xa3\xa2-O\xe1i\xb6\x12\x03\xc21\xf3\x8a\xf78\\\x01\xa5b\x07\x95\xbeE\xf8\xa3\x9ahm\xb1~'
|
||||
compressed = compressor_auto_zlib.compress(data)
|
||||
assert Compressor.detect(compressed) == CNONE
|
||||
|
||||
|
||||
|
|
Loading…
Reference in New Issue