diff --git a/src/borg/archive.py b/src/borg/archive.py index f375015d4..a0bde175e 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -1124,19 +1124,19 @@ class ArchiveChecker: self.error_found = False self.possibly_superseded = set() - def check(self, repository, repair=False, archive=None, first=0, last=0, sort_by='', prefix='', + def check(self, repository, repair=False, archive=None, first=0, last=0, sort_by='', glob=None, verify_data=False, save_space=False): """Perform a set of checks on 'repository' :param repair: enable repair mode, write updated or corrected data into repository :param archive: only check this archive :param first/last/sort_by: only check this number of first/last archives ordered by sort_by - :param prefix: only check archives with this prefix + :param glob: only check archives matching this glob :param verify_data: integrity verification of data referenced by archives :param save_space: Repository.commit(save_space) """ logger.info('Starting archive consistency check...') - self.check_all = archive is None and not any((first, last, prefix)) + self.check_all = archive is None and not any((first, last, glob)) self.repair = repair self.repository = repository self.init_chunks() @@ -1158,7 +1158,7 @@ class ArchiveChecker: self.error_found = True del self.chunks[Manifest.MANIFEST_ID] self.manifest = self.rebuild_manifest() - self.rebuild_refcounts(archive=archive, first=first, last=last, sort_by=sort_by, prefix=prefix) + self.rebuild_refcounts(archive=archive, first=first, last=last, sort_by=sort_by, glob=glob) self.orphan_chunks_check() self.finish(save_space=save_space) if self.error_found: @@ -1331,7 +1331,7 @@ class ArchiveChecker: logger.info('Manifest rebuild complete.') return manifest - def rebuild_refcounts(self, archive=None, first=0, last=0, sort_by='', prefix=''): + def rebuild_refcounts(self, archive=None, first=0, last=0, sort_by='', glob=None): """Rebuild object reference counts by walking the metadata Missing and/or incorrect data is repaired when detected @@ -1495,10 +1495,10 @@ class ArchiveChecker: if archive is None: sort_by = sort_by.split(',') - if any((first, last, prefix)): - archive_infos = self.manifest.archives.list(sort_by=sort_by, prefix=prefix, first=first, last=last) - if prefix and not archive_infos: - logger.warning('--prefix %s does not match any archives', prefix) + if any((first, last, glob)): + archive_infos = self.manifest.archives.list(sort_by=sort_by, glob=glob, first=first, last=last) + if glob and not archive_infos: + logger.warning('--glob-archives %s does not match any archives', glob) if first and len(archive_infos) < first: logger.warning('--first %d archives: only found %d archives', first, len(archive_infos)) if last and len(archive_infos) < last: diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 23a75dfa4..8c1b5819c 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -33,6 +33,7 @@ import msgpack import borg from . import __version__ from . import helpers +from . import shellpattern from .algorithms.checksums import crc32 from .archive import Archive, ArchiveChecker, ArchiveRecreater, Statistics, is_special from .archive import BackupOSError, backup_io @@ -283,9 +284,11 @@ class Archiver: if not args.archives_only: if not repository.check(repair=args.repair, save_space=args.save_space): return EXIT_WARNING + if args.prefix: + args.glob_archives = args.prefix + '*' if not args.repo_only and not ArchiveChecker().check( repository, repair=args.repair, archive=args.location.archive, - first=args.first, last=args.last, sort_by=args.sort_by or 'ts', prefix=args.prefix, + first=args.first, last=args.last, sort_by=args.sort_by or 'ts', glob=args.glob_archives, verify_data=args.verify_data, save_space=args.save_space): return EXIT_WARNING return EXIT_SUCCESS @@ -1168,7 +1171,7 @@ class Archiver: @with_repository(exclusive=True, manifest=False) def do_delete(self, args, repository): """Delete an existing repository or archives""" - if any((args.location.archive, args.first, args.last, args.prefix)): + if any((args.location.archive, args.first, args.last, args.prefix, args.glob_archives)): return self._delete_archives(args, repository) else: return self._delete_repository(args, repository) @@ -1365,7 +1368,7 @@ class Archiver: @with_repository(cache=True, compatibility=(Manifest.Operation.READ,)) def do_info(self, args, repository, manifest, key, cache): """Show archive details such as disk space used""" - if any((args.location.archive, args.first, args.last, args.prefix)): + if any((args.location.archive, args.first, args.last, args.prefix, args.glob_archives)): return self._info_archives(args, repository, manifest, key, cache) else: return self._info_repository(args, repository, manifest, key, cache) @@ -1463,7 +1466,10 @@ class Archiver: return self.exit_code archives_checkpoints = manifest.archives.list(sort_by=['ts'], reverse=True) # just a ArchiveInfo list if args.prefix: - archives_checkpoints = [arch for arch in archives_checkpoints if arch.name.startswith(args.prefix)] + args.glob_archives = args.prefix + '*' + if args.glob_archives: + regex = re.compile(shellpattern.translate(args.glob_archives)) + archives_checkpoints = [arch for arch in archives_checkpoints if regex.match(arch.name) is not None] is_checkpoint = re.compile(r'\.checkpoint(\.\d+)?$').search checkpoints = [arch for arch in archives_checkpoints if is_checkpoint(arch.name)] # keep the latest checkpoint, if there is no later non-checkpoint archive @@ -3344,8 +3350,7 @@ class Archiver: help='number of monthly archives to keep') subparser.add_argument('-y', '--keep-yearly', dest='yearly', type=int, default=0, help='number of yearly archives to keep') - subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec, - help='only consider archive names starting with this prefix') + self.add_archives_filters_args(subparser, sort_by=False, first_last=False) subparser.add_argument('--save-space', dest='save_space', action='store_true', default=False, help='work slower, but using less space') @@ -3839,21 +3844,28 @@ class Archiver: return parser @staticmethod - def add_archives_filters_args(subparser): + def add_archives_filters_args(subparser, sort_by=True, first_last=True): filters_group = subparser.add_argument_group('filters', 'Archive filters can be applied to repository targets.') - filters_group.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec, default='', - help='only consider archive names starting with this prefix') - - sort_by_default = 'timestamp' - filters_group.add_argument('--sort-by', dest='sort_by', type=SortBySpec, default=sort_by_default, - help='Comma-separated list of sorting keys; valid keys are: {}; default is: {}' - .format(', '.join(HUMAN_SORT_KEYS), sort_by_default)) - group = filters_group.add_mutually_exclusive_group() - group.add_argument('--first', dest='first', metavar='N', default=0, type=int, - help='consider first N archives after other filters were applied') - group.add_argument('--last', dest='last', metavar='N', default=0, type=int, - help='consider last N archives after other filters were applied') + group.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec, default='', + help='only consider archive names starting with this prefix.') + group.add_argument('-a', '--glob-archives', dest='glob_archives', default=None, + help='only consider archive names matching the glob. ' + 'sh: rules apply, see "borg help patterns". ' + '--prefix and --glob-archives are mutually exclusive.') + + if sort_by: + sort_by_default = 'timestamp' + filters_group.add_argument('--sort-by', dest='sort_by', type=SortBySpec, default=sort_by_default, + help='Comma-separated list of sorting keys; valid keys are: {}; default is: {}' + .format(', '.join(HUMAN_SORT_KEYS), sort_by_default)) + + if first_last: + group = filters_group.add_mutually_exclusive_group() + group.add_argument('--first', dest='first', metavar='N', default=0, type=int, + help='consider first N archives after other filters were applied') + group.add_argument('--last', dest='last', metavar='N', default=0, type=int, + help='consider last N archives after other filters were applied') def get_args(self, argv, cmd): """usually, just returns argv, except if we deal with a ssh forced command for borg serve.""" diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 116c9f625..f65aac9bb 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -42,6 +42,7 @@ from . import __version__ as borg_version from . import __version_tuple__ as borg_version_tuple from . import chunker from . import hashindex +from . import shellpattern from .constants import * # NOQA @@ -189,7 +190,7 @@ class Archives(abc.MutableMapping): name = safe_encode(name) del self._archives[name] - def list(self, sort_by=(), reverse=False, prefix='', first=None, last=None): + def list(self, sort_by=(), reverse=False, glob=None, first=None, last=None): """ Inexpensive Archive.list_archives replacement if we just need .name, .id, .ts Returns list of borg.helpers.ArchiveInfo instances. @@ -197,7 +198,8 @@ class Archives(abc.MutableMapping): """ if isinstance(sort_by, (str, bytes)): raise TypeError('sort_by must be a sequence of str') - archives = [x for x in self.values() if x.name.startswith(prefix)] + regex = re.compile(shellpattern.translate(glob or '*')) + archives = [x for x in self.values() if regex.match(x.name) is not None] for sortkey in reversed(sort_by): archives.sort(key=attrgetter(sortkey)) if reverse or last: @@ -207,11 +209,13 @@ class Archives(abc.MutableMapping): def list_considering(self, args): """ - get a list of archives, considering --first/last/prefix/sort cmdline args + get a list of archives, considering --first/last/prefix/glob-archives/sort cmdline args """ if args.location.archive: - raise Error('The options --first, --last and --prefix can only be used on repository targets.') - return self.list(sort_by=args.sort_by.split(','), prefix=args.prefix, first=args.first, last=args.last) + raise Error('The options --first, --last, --prefix and --glob-archives can only be used on repository targets.') + if args.prefix: + args.glob_archives = args.prefix + '*' + return self.list(sort_by=args.sort_by.split(','), glob=args.glob_archives, first=args.first, last=args.last) def set_raw_dict(self, d): """set the dict we get from the msgpack unpacker""" diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index c3de3c73b..6d0b28c4f 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -1708,6 +1708,27 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.assert_in('bar-2015-08-12-10:00', output) self.assert_in('bar-2015-08-12-20:00', output) + def test_prune_repository_glob(self): + self.cmd('init', '--encryption=repokey', self.repository_location) + self.cmd('create', self.repository_location + '::2015-08-12-10:00-foo', src_dir) + self.cmd('create', self.repository_location + '::2015-08-12-20:00-foo', src_dir) + self.cmd('create', self.repository_location + '::2015-08-12-10:00-bar', src_dir) + self.cmd('create', self.repository_location + '::2015-08-12-20:00-bar', src_dir) + output = self.cmd('prune', '--list', '--dry-run', self.repository_location, '--keep-daily=2', '--glob-archives=2015-*-foo') + self.assert_in('Keeping archive: 2015-08-12-20:00-foo', output) + self.assert_in('Would prune: 2015-08-12-10:00-foo', output) + output = self.cmd('list', self.repository_location) + self.assert_in('2015-08-12-10:00-foo', output) + self.assert_in('2015-08-12-20:00-foo', output) + self.assert_in('2015-08-12-10:00-bar', output) + self.assert_in('2015-08-12-20:00-bar', output) + self.cmd('prune', self.repository_location, '--keep-daily=2', '--glob-archives=2015-*-foo') + output = self.cmd('list', self.repository_location) + self.assert_not_in('2015-08-12-10:00-foo', output) + self.assert_in('2015-08-12-20:00-foo', output) + self.assert_in('2015-08-12-10:00-bar', output) + self.assert_in('2015-08-12-20:00-bar', output) + def test_list_prefix(self): self.cmd('init', '--encryption=repokey', self.repository_location) self.cmd('create', self.repository_location + '::test-1', src_dir)