mirror
/
borg
mirror of https://github.com/borgbackup/borg.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Replace verify command with "extract --dry-run" 

closes #25
This commit is contained in:
Jonas Borgström 2014-02-18 21:33:06 +01:00
parent 30daa23e42
commit 7b31f23722
8 changed files with 36 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES
View File

@ -13,6 +13,7 @@ Version 0.11
- Fix exception during "attic create" with repeated files (#39)
- New "--exclude-from" option for attic create/extract/verify.
- Improved archive metadata deduplication.
- Replace the "verify" command with "extract --dry-run" (#25)
Version 0.10
------------

23
attic/archive.py
View File

@ -219,7 +219,13 @@ class Archive:
cache.rollback()
return stats
def extract_item(self, item, restore_attrs=True):
def extract_item(self, item, restore_attrs=True, dry_run=False):
if dry_run:
if b'chunks' in item:
for _ in self.pipeline.fetch_many([c[0] for c in item[b'chunks']], is_preloaded=True):
pass
return
dest = self.cwd
if item[b'path'].startswith('/') or item[b'path'].startswith('..'):
raise Exception('Path should be relative and local')
@ -306,21 +312,6 @@ class Archive:
elif not symlink:
os.utime(path, (item[b'mtime'] / 10**9, item[b'mtime'] / 10**9))
def verify_file(self, item, start, result):
if not item[b'chunks']:
start(item)
result(item, True)
else:
start(item)
ids = [id for id, size, csize in item[b'chunks']]
try:
for _ in self.pipeline.fetch_many(ids, is_preloaded=True):
pass
except Exception:
result(item, False)
return
result(item, True)
def delete(self, cache):
unpacker = msgpack.Unpacker(use_list=False)
for id_, data in zip(self.metadata[b'items'], self.repository.get_many(self.metadata[b'items'])):

64
attic/archiver.py
View File

@ -188,20 +188,25 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
patterns = adjust_patterns(args.paths, args.excludes)
dirs = []
for item in archive.iter_items(lambda item: not exclude_path(item[b'path'], patterns), preload=True):
while dirs and not item[b'path'].startswith(dirs[-1][b'path']):
archive.extract_item(dirs.pop(-1))
if not args.dry_run:
while dirs and not item[b'path'].startswith(dirs[-1][b'path']):
archive.extract_item(dirs.pop(-1))
self.print_verbose(remove_surrogates(item[b'path']))
try:
if stat.S_ISDIR(item[b'mode']):
dirs.append(item)
archive.extract_item(item, restore_attrs=False)
if args.dry_run:
archive.extract_item(item, dry_run=True)
else:
archive.extract_item(item)
if stat.S_ISDIR(item[b'mode']):
dirs.append(item)
archive.extract_item(item, restore_attrs=False)
else:
archive.extract_item(item)
except IOError as e:
self.print_error('%s: %s', remove_surrogates(item[b'path']), e)
while dirs:
archive.extract_item(dirs.pop(-1))
if not args.dry_run:
while dirs:
archive.extract_item(dirs.pop(-1))
return self.exit_code
def do_delete(self, args):
@ -275,28 +280,6 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
print('%-20s %s' % (archive.metadata[b'name'], to_localtime(archive.ts).strftime('%c')))
return self.exit_code
def do_verify(self, args):
"""Verify archive consistency
"""
repository = self.open_repository(args.archive)
manifest, key = Manifest.load(repository)
archive = Archive(repository, key, manifest, args.archive.archive)
patterns = adjust_patterns(args.paths, args.excludes)
def start_cb(item):
self.print_verbose('%s ...', remove_surrogates(item[b'path']), newline=False)
def result_cb(item, success):
if success:
self.print_verbose('OK')
else:
self.print_verbose('ERROR')
self.print_error('%s: verification failed' % remove_surrogates(item[b'path']))
for item in archive.iter_items(lambda item: not exclude_path(item[b'path'], patterns), preload=True):
if stat.S_ISREG(item[b'mode']) and b'chunks' in item:
archive.verify_file(item, start_cb, result_cb)
return self.exit_code
def do_info(self, args):
"""Show archive details such as disk space used
"""
@ -485,6 +468,9 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
description=self.do_extract.__doc__,
epilog=extract_epilog)
subparser.set_defaults(func=self.do_extract)
subparser.add_argument('--dry-run', dest='dry_run',
default=False, action='store_true',
help='do not actually change any files')
subparser.add_argument('-e', '--exclude', dest='excludes',
type=ExcludePattern, action='append',
metavar="PATTERN", help='exclude paths matching PATTERN')
@ -526,24 +512,6 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
subparser.add_argument('-o', dest='options', type=str,
help='Extra mount options')
verify_epilog = '''See "attic help patterns" for more help on exclude patterns.'''
subparser = subparsers.add_parser('verify', parents=[common_parser],
description=self.do_verify.__doc__,
epilog=verify_epilog)
subparser.set_defaults(func=self.do_verify)
subparser.add_argument('-e', '--exclude', dest='excludes',
type=ExcludePattern, action='append',
metavar="PATTERN", help='exclude paths matching PATTERN')
subparser.add_argument('--exclude-from', dest='exclude_files',
type=argparse.FileType('r'), action='append',
metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line')
subparser.add_argument('archive', metavar='ARCHIVE',
type=location_validator(archive=True),
help='archive to verity integrity of')
subparser.add_argument('paths', metavar='PATH', nargs='*', type=str,
help='paths to verify')
subparser = subparsers.add_parser('info', parents=[common_parser],
description=self.do_info.__doc__)
subparser.set_defaults(func=self.do_info)

5
attic/helpers.py
View File

@ -288,9 +288,8 @@ def format_file_size(v):
return '%d B' % v
class IntegrityError(Exception):
"""
"""
class IntegrityError(Error):
"""Data integrity error"""
def memoize(function):

2
attic/remote.py
View File

@ -135,7 +135,7 @@ class RemoteRepository(object):
elif error == b'CheckNeeded':
raise Repository.CheckNeeded(self.location.orig)
elif error == b'IntegrityError':
raise IntegrityError
raise IntegrityError(res)
raise self.RPCError(error)
else:
yield res

16
attic/testsuite/archiver.py
View File

@ -11,7 +11,7 @@ from hashlib import sha256
from attic import xattr
from attic.archive import Archive
from attic.archiver import Archiver
from attic.helpers import Manifest
from attic.helpers import Manifest, IntegrityError
from attic.repository import Repository
from attic.testsuite import AtticTestCase
from attic.crypto import bytes_to_long, num_aes_blocks
@ -209,10 +209,10 @@ class ArchiverTestCase(ArchiverTestCaseBase):
self.attic('init', self.repository_location)
self.attic('create', self.repository_location + '::test', 'input')
self.attic('create', self.repository_location + '::test.2', 'input')
self.attic('verify', self.repository_location + '::test')
self.attic('verify', self.repository_location + '::test.2')
self.attic('extract', '--dry-run', self.repository_location + '::test')
self.attic('extract', '--dry-run', self.repository_location + '::test.2')
self.attic('delete', self.repository_location + '::test')
self.attic('verify', self.repository_location + '::test.2')
self.attic('extract', '--dry-run', self.repository_location + '::test.2')
self.attic('delete', self.repository_location + '::test.2')
# Make sure all data except the manifest has been deleted
repository = Repository(self.repository_path)
@ -221,14 +221,14 @@ class ArchiverTestCase(ArchiverTestCaseBase):
def test_corrupted_repository(self):
self.attic('init', self.repository_location)
self.create_src_archive('test')
self.attic('verify', self.repository_location + '::test')
self.attic('extract', '--dry-run', self.repository_location + '::test')
self.attic('check', self.repository_location)
name = sorted(os.listdir(os.path.join(self.tmpdir, 'repository', 'data', '0')), reverse=True)[0]
fd = open(os.path.join(self.tmpdir, 'repository', 'data', '0', name), 'r+')
fd.seek(100)
fd.write('XXXX')
fd.close()
self.attic('verify', self.repository_location + '::test', exit_code=1)
self.assert_raises(IntegrityError, lambda: self.attic('extract', '--dry-run', self.repository_location + '::test'))
self.attic('check', self.repository_location, exit_code=1)
def test_readonly_repository(self):
@ -236,7 +236,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
self.create_src_archive('test')
os.system('chmod -R ugo-w ' + self.repository_path)
try:
self.attic('verify', self.repository_location + '::test')
self.attic('extract', '--dry-run', self.repository_location + '::test')
finally:
# Restore permissions so shutil.rmtree is able to delete it
os.system('chmod -R u+w ' + self.repository_path)
@ -369,7 +369,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
self.attic('check', self.repository_location, exit_code=1)
self.attic('check', '--repair', self.repository_location, exit_code=0)
self.attic('check', self.repository_location, exit_code=0)
self.attic('verify', self.repository_location + '::archive1', exit_code=0)
self.attic('extract', '--dry-run', self.repository_location + '::archive1', exit_code=0)
class RemoteArchiverTestCase(ArchiverTestCase):

2
docs/update_usage.sh
View File

@ -2,7 +2,7 @@
if [ ! -d usage ]; then
mkdir usage
fi
for cmd in change-passphrase check create delete extract info init list mount prune verify; do
for cmd in change-passphrase check create delete extract info init list mount prune; do
FILENAME="usage/$cmd.rst.inc"
LINE=`echo -n attic $cmd | tr 'a-z- ' '-'`
echo -e ".. _attic_$cmd:\n" > $FILENAME

9
docs/usage.rst
View File

@ -84,15 +84,6 @@ Examples
# Extract the "src" directory but exclude object files
$ attic extract /data/myrepo::my-files home/USERNAME/src --exclude '*.o'
.. include:: usage/verify.rst.inc
This command is similar to :ref:`attic_extract` but instead of writing any
files to disk the command just verifies that all files are extractable and
not corrupt. |project_name| will not compare the the archived files with the
files on disk.
.. include:: usage/check.rst.inc
The check command verifies the consistency of a repository. Any inconsistencies