mirror/borg
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2025-02-18 20:31:47 +00:00
Code Issues Releases Wiki Activity

Replace verify command with "extract --dry-run"

Browse source 
closes #25
This commit is contained in:
Jonas Borgström 2014-02-18 21:33:06 +01:00
parent 30daa23e42
commit 7b31f23722
8 changed files with 36 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGES
View file

@ -13,6 +13,7 @@ Version 0.11
- Fix exception during "attic create" with repeated files (#39)
- New "--exclude-from" option for attic create/extract/verify.
- Improved archive metadata deduplication.
- Replace the "verify" command with "extract --dry-run" (#25)
Version 0.10
------------

23
attic/archive.py
View file

@ -219,7 +219,13 @@ def add_file_chunks(chunks):
cache.rollback()
return stats
def extract_item(self, item, restore_attrs=True):
def extract_item(self, item, restore_attrs=True, dry_run=False):
if dry_run:
if b'chunks' in item:
for _ in self.pipeline.fetch_many([c[0] for c in item[b'chunks']], is_preloaded=True):
pass
return
dest = self.cwd
if item[b'path'].startswith('/') or item[b'path'].startswith('..'):
raise Exception('Path should be relative and local')
@ -306,21 +312,6 @@ def restore_attrs(self, path, item, symlink=False, fd=None):
elif not symlink:
os.utime(path, (item[b'mtime'] / 10**9, item[b'mtime'] / 10**9))
def verify_file(self, item, start, result):
if not item[b'chunks']:
start(item)
result(item, True)
else:
start(item)
ids = [id for id, size, csize in item[b'chunks']]
try:
for _ in self.pipeline.fetch_many(ids, is_preloaded=True):
pass
except Exception:
result(item, False)
return
result(item, True)
def delete(self, cache):
unpacker = msgpack.Unpacker(use_list=False)
for id_, data in zip(self.metadata[b'items'], self.repository.get_many(self.metadata[b'items'])):

64
attic/archiver.py
View file

@ -188,20 +188,25 @@ def do_extract(self, args):
patterns = adjust_patterns(args.paths, args.excludes)
dirs = []
for item in archive.iter_items(lambda item: not exclude_path(item[b'path'], patterns), preload=True):
while dirs and not item[b'path'].startswith(dirs[-1][b'path']):
archive.extract_item(dirs.pop(-1))
if not args.dry_run:
while dirs and not item[b'path'].startswith(dirs[-1][b'path']):
archive.extract_item(dirs.pop(-1))
self.print_verbose(remove_surrogates(item[b'path']))
try:
if stat.S_ISDIR(item[b'mode']):
dirs.append(item)
archive.extract_item(item, restore_attrs=False)
if args.dry_run:
archive.extract_item(item, dry_run=True)
else:
archive.extract_item(item)
if stat.S_ISDIR(item[b'mode']):
dirs.append(item)
archive.extract_item(item, restore_attrs=False)
else:
archive.extract_item(item)
except IOError as e:
self.print_error('%s: %s', remove_surrogates(item[b'path']), e)
while dirs:
archive.extract_item(dirs.pop(-1))
if not args.dry_run:
while dirs:
archive.extract_item(dirs.pop(-1))
return self.exit_code
def do_delete(self, args):
@ -275,28 +280,6 @@ def do_list(self, args):
print('%-20s %s' % (archive.metadata[b'name'], to_localtime(archive.ts).strftime('%c')))
return self.exit_code
def do_verify(self, args):
"""Verify archive consistency
"""
repository = self.open_repository(args.archive)
manifest, key = Manifest.load(repository)
archive = Archive(repository, key, manifest, args.archive.archive)
patterns = adjust_patterns(args.paths, args.excludes)
def start_cb(item):
self.print_verbose('%s ...', remove_surrogates(item[b'path']), newline=False)
def result_cb(item, success):
if success:
self.print_verbose('OK')
else:
self.print_verbose('ERROR')
self.print_error('%s: verification failed' % remove_surrogates(item[b'path']))
for item in archive.iter_items(lambda item: not exclude_path(item[b'path'], patterns), preload=True):
if stat.S_ISREG(item[b'mode']) and b'chunks' in item:
archive.verify_file(item, start_cb, result_cb)
return self.exit_code
def do_info(self, args):
"""Show archive details such as disk space used
"""
@ -485,6 +468,9 @@ def run(self, args=None):
description=self.do_extract.__doc__,
epilog=extract_epilog)
subparser.set_defaults(func=self.do_extract)
subparser.add_argument('--dry-run', dest='dry_run',
default=False, action='store_true',
help='do not actually change any files')
subparser.add_argument('-e', '--exclude', dest='excludes',
type=ExcludePattern, action='append',
metavar="PATTERN", help='exclude paths matching PATTERN')
@ -526,24 +512,6 @@ def run(self, args=None):
subparser.add_argument('-o', dest='options', type=str,
help='Extra mount options')
verify_epilog = '''See "attic help patterns" for more help on exclude patterns.'''
subparser = subparsers.add_parser('verify', parents=[common_parser],
description=self.do_verify.__doc__,
epilog=verify_epilog)
subparser.set_defaults(func=self.do_verify)
subparser.add_argument('-e', '--exclude', dest='excludes',
type=ExcludePattern, action='append',
metavar="PATTERN", help='exclude paths matching PATTERN')
subparser.add_argument('--exclude-from', dest='exclude_files',
type=argparse.FileType('r'), action='append',
metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line')
subparser.add_argument('archive', metavar='ARCHIVE',
type=location_validator(archive=True),
help='archive to verity integrity of')
subparser.add_argument('paths', metavar='PATH', nargs='*', type=str,
help='paths to verify')
subparser = subparsers.add_parser('info', parents=[common_parser],
description=self.do_info.__doc__)
subparser.set_defaults(func=self.do_info)

5
attic/helpers.py
View file

@ -288,9 +288,8 @@ def format_file_size(v):
return '%d B' % v
class IntegrityError(Exception):
"""
"""
class IntegrityError(Error):
"""Data integrity error"""
def memoize(function):

2
attic/remote.py
View file

@ -135,7 +135,7 @@ def fetch_from_cache(args):
elif error == b'CheckNeeded':
raise Repository.CheckNeeded(self.location.orig)
elif error == b'IntegrityError':
raise IntegrityError
raise IntegrityError(res)
raise self.RPCError(error)
else:
yield res

16
attic/testsuite/archiver.py
View file

@ -11,7 +11,7 @@
from attic import xattr
from attic.archive import Archive
from attic.archiver import Archiver
from attic.helpers import Manifest
from attic.helpers import Manifest, IntegrityError
from attic.repository import Repository
from attic.testsuite import AtticTestCase
from attic.crypto import bytes_to_long, num_aes_blocks
@ -209,10 +209,10 @@ def test_delete(self):
self.attic('init', self.repository_location)
self.attic('create', self.repository_location + '::test', 'input')
self.attic('create', self.repository_location + '::test.2', 'input')
self.attic('verify', self.repository_location + '::test')
self.attic('verify', self.repository_location + '::test.2')
self.attic('extract', '--dry-run', self.repository_location + '::test')
self.attic('extract', '--dry-run', self.repository_location + '::test.2')
self.attic('delete', self.repository_location + '::test')
self.attic('verify', self.repository_location + '::test.2')
self.attic('extract', '--dry-run', self.repository_location + '::test.2')
self.attic('delete', self.repository_location + '::test.2')
# Make sure all data except the manifest has been deleted
repository = Repository(self.repository_path)
@ -221,14 +221,14 @@ def test_delete(self):
def test_corrupted_repository(self):
self.attic('init', self.repository_location)
self.create_src_archive('test')
self.attic('verify', self.repository_location + '::test')
self.attic('extract', '--dry-run', self.repository_location + '::test')
self.attic('check', self.repository_location)
name = sorted(os.listdir(os.path.join(self.tmpdir, 'repository', 'data', '0')), reverse=True)[0]
fd = open(os.path.join(self.tmpdir, 'repository', 'data', '0', name), 'r+')
fd.seek(100)
fd.write('XXXX')
fd.close()
self.attic('verify', self.repository_location + '::test', exit_code=1)
self.assert_raises(IntegrityError, lambda: self.attic('extract', '--dry-run', self.repository_location + '::test'))
self.attic('check', self.repository_location, exit_code=1)
def test_readonly_repository(self):
@ -236,7 +236,7 @@ def test_readonly_repository(self):
self.create_src_archive('test')
os.system('chmod -R ugo-w ' + self.repository_path)
try:
self.attic('verify', self.repository_location + '::test')
self.attic('extract', '--dry-run', self.repository_location + '::test')
finally:
# Restore permissions so shutil.rmtree is able to delete it
os.system('chmod -R u+w ' + self.repository_path)
@ -369,7 +369,7 @@ def test_extra_chunks(self):
self.attic('check', self.repository_location, exit_code=1)
self.attic('check', '--repair', self.repository_location, exit_code=0)
self.attic('check', self.repository_location, exit_code=0)
self.attic('verify', self.repository_location + '::archive1', exit_code=0)
self.attic('extract', '--dry-run', self.repository_location + '::archive1', exit_code=0)
class RemoteArchiverTestCase(ArchiverTestCase):

2
docs/update_usage.sh
View file

@ -2,7 +2,7 @@
if [ ! -d usage ]; then
mkdir usage
fi
for cmd in change-passphrase check create delete extract info init list mount prune verify; do
for cmd in change-passphrase check create delete extract info init list mount prune; do
FILENAME="usage/$cmd.rst.inc"
LINE=`echo -n attic $cmd | tr 'a-z- ' '-'`
echo -e ".. _attic_$cmd:\n" > $FILENAME

9
docs/usage.rst
View file

@ -84,15 +84,6 @@ Examples
# Extract the "src" directory but exclude object files
$ attic extract /data/myrepo::my-files home/USERNAME/src --exclude '*.o'
.. include:: usage/verify.rst.inc
This command is similar to :ref:`attic_extract` but instead of writing any
files to disk the command just verifies that all files are extractable and
not corrupt. |project_name| will not compare the the archived files with the
files on disk.
.. include:: usage/check.rst.inc
The check command verifies the consistency of a repository. Any inconsistencies