diff --git a/CHANGES b/CHANGES index a7e036355..8b800ce4e 100644 --- a/CHANGES +++ b/CHANGES @@ -13,6 +13,7 @@ Version 0.11 - Fix exception during "attic create" with repeated files (#39) - New "--exclude-from" option for attic create/extract/verify. - Improved archive metadata deduplication. +- Replace the "verify" command with "extract --dry-run" (#25) Version 0.10 ------------ diff --git a/attic/archive.py b/attic/archive.py index d34d256e7..a1e905f41 100644 --- a/attic/archive.py +++ b/attic/archive.py @@ -219,7 +219,13 @@ class Archive: cache.rollback() return stats - def extract_item(self, item, restore_attrs=True): + def extract_item(self, item, restore_attrs=True, dry_run=False): + if dry_run: + if b'chunks' in item: + for _ in self.pipeline.fetch_many([c[0] for c in item[b'chunks']], is_preloaded=True): + pass + return + dest = self.cwd if item[b'path'].startswith('/') or item[b'path'].startswith('..'): raise Exception('Path should be relative and local') @@ -306,21 +312,6 @@ class Archive: elif not symlink: os.utime(path, (item[b'mtime'] / 10**9, item[b'mtime'] / 10**9)) - def verify_file(self, item, start, result): - if not item[b'chunks']: - start(item) - result(item, True) - else: - start(item) - ids = [id for id, size, csize in item[b'chunks']] - try: - for _ in self.pipeline.fetch_many(ids, is_preloaded=True): - pass - except Exception: - result(item, False) - return - result(item, True) - def delete(self, cache): unpacker = msgpack.Unpacker(use_list=False) for id_, data in zip(self.metadata[b'items'], self.repository.get_many(self.metadata[b'items'])): diff --git a/attic/archiver.py b/attic/archiver.py index 536a92900..c2fbe4768 100644 --- a/attic/archiver.py +++ b/attic/archiver.py @@ -188,20 +188,25 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") patterns = adjust_patterns(args.paths, args.excludes) dirs = [] for item in archive.iter_items(lambda item: not exclude_path(item[b'path'], patterns), preload=True): - while dirs and not item[b'path'].startswith(dirs[-1][b'path']): - archive.extract_item(dirs.pop(-1)) + if not args.dry_run: + while dirs and not item[b'path'].startswith(dirs[-1][b'path']): + archive.extract_item(dirs.pop(-1)) self.print_verbose(remove_surrogates(item[b'path'])) try: - if stat.S_ISDIR(item[b'mode']): - dirs.append(item) - archive.extract_item(item, restore_attrs=False) + if args.dry_run: + archive.extract_item(item, dry_run=True) else: - archive.extract_item(item) + if stat.S_ISDIR(item[b'mode']): + dirs.append(item) + archive.extract_item(item, restore_attrs=False) + else: + archive.extract_item(item) except IOError as e: self.print_error('%s: %s', remove_surrogates(item[b'path']), e) - while dirs: - archive.extract_item(dirs.pop(-1)) + if not args.dry_run: + while dirs: + archive.extract_item(dirs.pop(-1)) return self.exit_code def do_delete(self, args): @@ -275,28 +280,6 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") print('%-20s %s' % (archive.metadata[b'name'], to_localtime(archive.ts).strftime('%c'))) return self.exit_code - def do_verify(self, args): - """Verify archive consistency - """ - repository = self.open_repository(args.archive) - manifest, key = Manifest.load(repository) - archive = Archive(repository, key, manifest, args.archive.archive) - patterns = adjust_patterns(args.paths, args.excludes) - - def start_cb(item): - self.print_verbose('%s ...', remove_surrogates(item[b'path']), newline=False) - - def result_cb(item, success): - if success: - self.print_verbose('OK') - else: - self.print_verbose('ERROR') - self.print_error('%s: verification failed' % remove_surrogates(item[b'path'])) - for item in archive.iter_items(lambda item: not exclude_path(item[b'path'], patterns), preload=True): - if stat.S_ISREG(item[b'mode']) and b'chunks' in item: - archive.verify_file(item, start_cb, result_cb) - return self.exit_code - def do_info(self, args): """Show archive details such as disk space used """ @@ -485,6 +468,9 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") description=self.do_extract.__doc__, epilog=extract_epilog) subparser.set_defaults(func=self.do_extract) + subparser.add_argument('--dry-run', dest='dry_run', + default=False, action='store_true', + help='do not actually change any files') subparser.add_argument('-e', '--exclude', dest='excludes', type=ExcludePattern, action='append', metavar="PATTERN", help='exclude paths matching PATTERN') @@ -526,24 +512,6 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") subparser.add_argument('-o', dest='options', type=str, help='Extra mount options') - verify_epilog = '''See "attic help patterns" for more help on exclude patterns.''' - - subparser = subparsers.add_parser('verify', parents=[common_parser], - description=self.do_verify.__doc__, - epilog=verify_epilog) - subparser.set_defaults(func=self.do_verify) - subparser.add_argument('-e', '--exclude', dest='excludes', - type=ExcludePattern, action='append', - metavar="PATTERN", help='exclude paths matching PATTERN') - subparser.add_argument('--exclude-from', dest='exclude_files', - type=argparse.FileType('r'), action='append', - metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line') - subparser.add_argument('archive', metavar='ARCHIVE', - type=location_validator(archive=True), - help='archive to verity integrity of') - subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, - help='paths to verify') - subparser = subparsers.add_parser('info', parents=[common_parser], description=self.do_info.__doc__) subparser.set_defaults(func=self.do_info) diff --git a/attic/helpers.py b/attic/helpers.py index 168a72f52..dbf15ef3b 100644 --- a/attic/helpers.py +++ b/attic/helpers.py @@ -288,9 +288,8 @@ def format_file_size(v): return '%d B' % v -class IntegrityError(Exception): - """ - """ +class IntegrityError(Error): + """Data integrity error""" def memoize(function): diff --git a/attic/remote.py b/attic/remote.py index 033fdba7a..ae61ba310 100644 --- a/attic/remote.py +++ b/attic/remote.py @@ -135,7 +135,7 @@ class RemoteRepository(object): elif error == b'CheckNeeded': raise Repository.CheckNeeded(self.location.orig) elif error == b'IntegrityError': - raise IntegrityError + raise IntegrityError(res) raise self.RPCError(error) else: yield res diff --git a/attic/testsuite/archiver.py b/attic/testsuite/archiver.py index d16e1360d..546baa222 100644 --- a/attic/testsuite/archiver.py +++ b/attic/testsuite/archiver.py @@ -11,7 +11,7 @@ from hashlib import sha256 from attic import xattr from attic.archive import Archive from attic.archiver import Archiver -from attic.helpers import Manifest +from attic.helpers import Manifest, IntegrityError from attic.repository import Repository from attic.testsuite import AtticTestCase from attic.crypto import bytes_to_long, num_aes_blocks @@ -209,10 +209,10 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.attic('init', self.repository_location) self.attic('create', self.repository_location + '::test', 'input') self.attic('create', self.repository_location + '::test.2', 'input') - self.attic('verify', self.repository_location + '::test') - self.attic('verify', self.repository_location + '::test.2') + self.attic('extract', '--dry-run', self.repository_location + '::test') + self.attic('extract', '--dry-run', self.repository_location + '::test.2') self.attic('delete', self.repository_location + '::test') - self.attic('verify', self.repository_location + '::test.2') + self.attic('extract', '--dry-run', self.repository_location + '::test.2') self.attic('delete', self.repository_location + '::test.2') # Make sure all data except the manifest has been deleted repository = Repository(self.repository_path) @@ -221,14 +221,14 @@ class ArchiverTestCase(ArchiverTestCaseBase): def test_corrupted_repository(self): self.attic('init', self.repository_location) self.create_src_archive('test') - self.attic('verify', self.repository_location + '::test') + self.attic('extract', '--dry-run', self.repository_location + '::test') self.attic('check', self.repository_location) name = sorted(os.listdir(os.path.join(self.tmpdir, 'repository', 'data', '0')), reverse=True)[0] fd = open(os.path.join(self.tmpdir, 'repository', 'data', '0', name), 'r+') fd.seek(100) fd.write('XXXX') fd.close() - self.attic('verify', self.repository_location + '::test', exit_code=1) + self.assert_raises(IntegrityError, lambda: self.attic('extract', '--dry-run', self.repository_location + '::test')) self.attic('check', self.repository_location, exit_code=1) def test_readonly_repository(self): @@ -236,7 +236,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.create_src_archive('test') os.system('chmod -R ugo-w ' + self.repository_path) try: - self.attic('verify', self.repository_location + '::test') + self.attic('extract', '--dry-run', self.repository_location + '::test') finally: # Restore permissions so shutil.rmtree is able to delete it os.system('chmod -R u+w ' + self.repository_path) @@ -369,7 +369,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase): self.attic('check', self.repository_location, exit_code=1) self.attic('check', '--repair', self.repository_location, exit_code=0) self.attic('check', self.repository_location, exit_code=0) - self.attic('verify', self.repository_location + '::archive1', exit_code=0) + self.attic('extract', '--dry-run', self.repository_location + '::archive1', exit_code=0) class RemoteArchiverTestCase(ArchiverTestCase): diff --git a/docs/update_usage.sh b/docs/update_usage.sh index 307d5ba79..4962ec825 100755 --- a/docs/update_usage.sh +++ b/docs/update_usage.sh @@ -2,7 +2,7 @@ if [ ! -d usage ]; then mkdir usage fi -for cmd in change-passphrase check create delete extract info init list mount prune verify; do +for cmd in change-passphrase check create delete extract info init list mount prune; do FILENAME="usage/$cmd.rst.inc" LINE=`echo -n attic $cmd | tr 'a-z- ' '-'` echo -e ".. _attic_$cmd:\n" > $FILENAME diff --git a/docs/usage.rst b/docs/usage.rst index 9d5a97446..1491e831e 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -84,15 +84,6 @@ Examples # Extract the "src" directory but exclude object files $ attic extract /data/myrepo::my-files home/USERNAME/src --exclude '*.o' - -.. include:: usage/verify.rst.inc - -This command is similar to :ref:`attic_extract` but instead of writing any -files to disk the command just verifies that all files are extractable and -not corrupt. |project_name| will not compare the the archived files with the -files on disk. - - .. include:: usage/check.rst.inc The check command verifies the consistency of a repository. Any inconsistencies