mirror
/
borg
mirror of https://github.com/borgbackup/borg.git
1
0
Fork 0
Code Issues Releases Wiki Activity

implement --glob-archives/-a

This commit is contained in:
Marian Beermann 2017-06-11 12:13:42 +02:00
parent a7fb74dbb1
commit 5f5371f0b1
4 changed files with 70 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/borg/archive.py
View File

@ -1124,19 +1124,19 @@ class ArchiveChecker:
self.error_found = False
self.possibly_superseded = set()
def check(self, repository, repair=False, archive=None, first=0, last=0, sort_by='', prefix='',
def check(self, repository, repair=False, archive=None, first=0, last=0, sort_by='', glob=None,
verify_data=False, save_space=False):
"""Perform a set of checks on 'repository'
:param repair: enable repair mode, write updated or corrected data into repository
:param archive: only check this archive
:param first/last/sort_by: only check this number of first/last archives ordered by sort_by
:param prefix: only check archives with this prefix
:param glob: only check archives matching this glob
:param verify_data: integrity verification of data referenced by archives
:param save_space: Repository.commit(save_space)
"""
logger.info('Starting archive consistency check...')
self.check_all = archive is None and not any((first, last, prefix))
self.check_all = archive is None and not any((first, last, glob))
self.repair = repair
self.repository = repository
self.init_chunks()
@ -1158,7 +1158,7 @@ class ArchiveChecker:
self.error_found = True
del self.chunks[Manifest.MANIFEST_ID]
self.manifest = self.rebuild_manifest()
self.rebuild_refcounts(archive=archive, first=first, last=last, sort_by=sort_by, prefix=prefix)
self.rebuild_refcounts(archive=archive, first=first, last=last, sort_by=sort_by, glob=glob)
self.orphan_chunks_check()
self.finish(save_space=save_space)
if self.error_found:
@ -1331,7 +1331,7 @@ class ArchiveChecker:
logger.info('Manifest rebuild complete.')
return manifest
def rebuild_refcounts(self, archive=None, first=0, last=0, sort_by='', prefix=''):
def rebuild_refcounts(self, archive=None, first=0, last=0, sort_by='', glob=None):
"""Rebuild object reference counts by walking the metadata
Missing and/or incorrect data is repaired when detected
@ -1495,10 +1495,10 @@ class ArchiveChecker:
if archive is None:
sort_by = sort_by.split(',')
if any((first, last, prefix)):
archive_infos = self.manifest.archives.list(sort_by=sort_by, prefix=prefix, first=first, last=last)
if prefix and not archive_infos:
logger.warning('--prefix %s does not match any archives', prefix)
if any((first, last, glob)):
archive_infos = self.manifest.archives.list(sort_by=sort_by, glob=glob, first=first, last=last)
if glob and not archive_infos:
logger.warning('--glob-archives %s does not match any archives', glob)
if first and len(archive_infos) < first:
logger.warning('--first %d archives: only found %d archives', first, len(archive_infos))
if last and len(archive_infos) < last:

50
src/borg/archiver.py
View File

@ -33,6 +33,7 @@ import msgpack
import borg
from . import __version__
from . import helpers
from . import shellpattern
from .algorithms.checksums import crc32
from .archive import Archive, ArchiveChecker, ArchiveRecreater, Statistics, is_special
from .archive import BackupOSError, backup_io
@ -283,9 +284,11 @@ class Archiver:
if not args.archives_only:
if not repository.check(repair=args.repair, save_space=args.save_space):
return EXIT_WARNING
if args.prefix:
args.glob_archives = args.prefix + '*'
if not args.repo_only and not ArchiveChecker().check(
repository, repair=args.repair, archive=args.location.archive,
first=args.first, last=args.last, sort_by=args.sort_by or 'ts', prefix=args.prefix,
first=args.first, last=args.last, sort_by=args.sort_by or 'ts', glob=args.glob_archives,
verify_data=args.verify_data, save_space=args.save_space):
return EXIT_WARNING
return EXIT_SUCCESS
@ -1168,7 +1171,7 @@ class Archiver:
@with_repository(exclusive=True, manifest=False)
def do_delete(self, args, repository):
"""Delete an existing repository or archives"""
if any((args.location.archive, args.first, args.last, args.prefix)):
if any((args.location.archive, args.first, args.last, args.prefix, args.glob_archives)):
return self._delete_archives(args, repository)
else:
return self._delete_repository(args, repository)
@ -1365,7 +1368,7 @@ class Archiver:
@with_repository(cache=True, compatibility=(Manifest.Operation.READ,))
def do_info(self, args, repository, manifest, key, cache):
"""Show archive details such as disk space used"""
if any((args.location.archive, args.first, args.last, args.prefix)):
if any((args.location.archive, args.first, args.last, args.prefix, args.glob_archives)):
return self._info_archives(args, repository, manifest, key, cache)
else:
return self._info_repository(args, repository, manifest, key, cache)
@ -1463,7 +1466,10 @@ class Archiver:
return self.exit_code
archives_checkpoints = manifest.archives.list(sort_by=['ts'], reverse=True) # just a ArchiveInfo list
if args.prefix:
archives_checkpoints = [arch for arch in archives_checkpoints if arch.name.startswith(args.prefix)]
args.glob_archives = args.prefix + '*'
if args.glob_archives:
regex = re.compile(shellpattern.translate(args.glob_archives))
archives_checkpoints = [arch for arch in archives_checkpoints if regex.match(arch.name) is not None]
is_checkpoint = re.compile(r'\.checkpoint(\.\d+)?$').search
checkpoints = [arch for arch in archives_checkpoints if is_checkpoint(arch.name)]
# keep the latest checkpoint, if there is no later non-checkpoint archive
@ -3344,8 +3350,7 @@ class Archiver:
help='number of monthly archives to keep')
subparser.add_argument('-y', '--keep-yearly', dest='yearly', type=int, default=0,
help='number of yearly archives to keep')
subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec,
help='only consider archive names starting with this prefix')
self.add_archives_filters_args(subparser, sort_by=False, first_last=False)
subparser.add_argument('--save-space', dest='save_space', action='store_true',
default=False,
help='work slower, but using less space')
@ -3839,21 +3844,28 @@ class Archiver:
return parser
@staticmethod
def add_archives_filters_args(subparser):
def add_archives_filters_args(subparser, sort_by=True, first_last=True):
filters_group = subparser.add_argument_group('filters', 'Archive filters can be applied to repository targets.')
filters_group.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec, default='',
help='only consider archive names starting with this prefix')
sort_by_default = 'timestamp'
filters_group.add_argument('--sort-by', dest='sort_by', type=SortBySpec, default=sort_by_default,
help='Comma-separated list of sorting keys; valid keys are: {}; default is: {}'
.format(', '.join(HUMAN_SORT_KEYS), sort_by_default))
group = filters_group.add_mutually_exclusive_group()
group.add_argument('--first', dest='first', metavar='N', default=0, type=int,
help='consider first N archives after other filters were applied')
group.add_argument('--last', dest='last', metavar='N', default=0, type=int,
help='consider last N archives after other filters were applied')
group.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec, default='',
help='only consider archive names starting with this prefix.')
group.add_argument('-a', '--glob-archives', dest='glob_archives', default=None,
help='only consider archive names matching the glob. '
'sh: rules apply, see "borg help patterns". '
'--prefix and --glob-archives are mutually exclusive.')
if sort_by:
sort_by_default = 'timestamp'
filters_group.add_argument('--sort-by', dest='sort_by', type=SortBySpec, default=sort_by_default,
help='Comma-separated list of sorting keys; valid keys are: {}; default is: {}'
.format(', '.join(HUMAN_SORT_KEYS), sort_by_default))
if first_last:
group = filters_group.add_mutually_exclusive_group()
group.add_argument('--first', dest='first', metavar='N', default=0, type=int,
help='consider first N archives after other filters were applied')
group.add_argument('--last', dest='last', metavar='N', default=0, type=int,
help='consider last N archives after other filters were applied')
def get_args(self, argv, cmd):
"""usually, just returns argv, except if we deal with a ssh forced command for borg serve."""

14
src/borg/helpers.py
View File

@ -42,6 +42,7 @@ from . import __version__ as borg_version
from . import __version_tuple__ as borg_version_tuple
from . import chunker
from . import hashindex
from . import shellpattern
from .constants import * # NOQA
@ -189,7 +190,7 @@ class Archives(abc.MutableMapping):
name = safe_encode(name)
del self._archives[name]
def list(self, sort_by=(), reverse=False, prefix='', first=None, last=None):
def list(self, sort_by=(), reverse=False, glob=None, first=None, last=None):
"""
Inexpensive Archive.list_archives replacement if we just need .name, .id, .ts
Returns list of borg.helpers.ArchiveInfo instances.
@ -197,7 +198,8 @@ class Archives(abc.MutableMapping):
"""
if isinstance(sort_by, (str, bytes)):
raise TypeError('sort_by must be a sequence of str')
archives = [x for x in self.values() if x.name.startswith(prefix)]
regex = re.compile(shellpattern.translate(glob or '*'))
archives = [x for x in self.values() if regex.match(x.name) is not None]
for sortkey in reversed(sort_by):
archives.sort(key=attrgetter(sortkey))
if reverse or last:
@ -207,11 +209,13 @@ class Archives(abc.MutableMapping):
def list_considering(self, args):
"""
get a list of archives, considering --first/last/prefix/sort cmdline args
get a list of archives, considering --first/last/prefix/glob-archives/sort cmdline args
"""
if args.location.archive:
raise Error('The options --first, --last and --prefix can only be used on repository targets.')
return self.list(sort_by=args.sort_by.split(','), prefix=args.prefix, first=args.first, last=args.last)
raise Error('The options --first, --last, --prefix and --glob-archives can only be used on repository targets.')
if args.prefix:
args.glob_archives = args.prefix + '*'
return self.list(sort_by=args.sort_by.split(','), glob=args.glob_archives, first=args.first, last=args.last)
def set_raw_dict(self, d):
"""set the dict we get from the msgpack unpacker"""

21
src/borg/testsuite/archiver.py
View File

@ -1708,6 +1708,27 @@ class ArchiverTestCase(ArchiverTestCaseBase):
self.assert_in('bar-2015-08-12-10:00', output)
self.assert_in('bar-2015-08-12-20:00', output)
def test_prune_repository_glob(self):
self.cmd('init', '--encryption=repokey', self.repository_location)
self.cmd('create', self.repository_location + '::2015-08-12-10:00-foo', src_dir)
self.cmd('create', self.repository_location + '::2015-08-12-20:00-foo', src_dir)
self.cmd('create', self.repository_location + '::2015-08-12-10:00-bar', src_dir)
self.cmd('create', self.repository_location + '::2015-08-12-20:00-bar', src_dir)
output = self.cmd('prune', '--list', '--dry-run', self.repository_location, '--keep-daily=2', '--glob-archives=2015-*-foo')
self.assert_in('Keeping archive: 2015-08-12-20:00-foo', output)
self.assert_in('Would prune: 2015-08-12-10:00-foo', output)
output = self.cmd('list', self.repository_location)
self.assert_in('2015-08-12-10:00-foo', output)
self.assert_in('2015-08-12-20:00-foo', output)
self.assert_in('2015-08-12-10:00-bar', output)
self.assert_in('2015-08-12-20:00-bar', output)
self.cmd('prune', self.repository_location, '--keep-daily=2', '--glob-archives=2015-*-foo')
output = self.cmd('list', self.repository_location)
self.assert_not_in('2015-08-12-10:00-foo', output)
self.assert_in('2015-08-12-20:00-foo', output)
self.assert_in('2015-08-12-10:00-bar', output)
self.assert_in('2015-08-12-20:00-bar', output)
def test_list_prefix(self):
self.cmd('init', '--encryption=repokey', self.repository_location)
self.cmd('create', self.repository_location + '::test-1', src_dir)