Merge pull request #1554 from funkyfuture/archives_filters

Archives filters

src/borg/archiver.py

@@ -16,6 +16,7 @@
 from binascii import unhexlify
 from datetime import datetime
 from itertools import zip_longest
+from operator import attrgetter
 
 from .logger import create_logger, setup_logging
 logger = create_logger()
@@ -28,7 +29,8 @@
 from .constants import *  # NOQA
 from .helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR
 from .helpers import Error, NoManifestError
-from .helpers import location_validator, archivename_validator, ChunkerParams, CompressionSpec, PrefixSpec
+from .helpers import location_validator, archivename_validator, ChunkerParams, CompressionSpec
+from .helpers import PrefixSpec, SortBySpec, HUMAN_SORT_KEYS
 from .helpers import BaseFormatter, ItemFormatter, ArchiveFormatter, format_time, format_file_size, format_archive
 from .helpers import safe_encode, remove_surrogates, bin_to_hex
 from .helpers import prune_within, prune_split
@@ -768,11 +770,31 @@ def do_rename(self, args, repository, manifest, key, cache, archive):
 
     @with_repository(exclusive=True, manifest=False)
     def do_delete(self, args, repository):
-        """Delete an existing repository or archive"""
+        """Delete an existing repository or archives"""
+        if any((args.location.archive, args.first, args.last, args.prefix)):
+            return self._delete_archives(args, repository)
+        else:
+            return self._delete_repository(args, repository)
+
+    def _delete_archives(self, args, repository):
+        """Delete archives"""
+        manifest, key = Manifest.load(repository)
+
         if args.location.archive:
-            manifest, key = Manifest.load(repository)
-            with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache:
-                archive = Archive(repository, key, manifest, args.location.archive, cache=cache)
+            archive_names = (args.location.archive,)
+        else:
+            archive_names = tuple(x.name for x in self._get_filtered_archives(args, manifest))
+            if not archive_names:
+                return self.exit_code
+
+        stats_logger = logging.getLogger('borg.output.stats')
+        if args.stats:
+            log_multi(DASHES, STATS_HEADER, logger=stats_logger)
+
+        with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache:
+            for i, archive_name in enumerate(archive_names, 1):
+                logger.info('Deleting {} ({}/{}):'.format(archive_name, i, len(archive_names)))
+                archive = Archive(repository, key, manifest, archive_name, cache=cache)
                 stats = Statistics()
                 archive.delete(stats, progress=args.progress, forced=args.forced)
                 manifest.write()
@@ -780,33 +802,41 @@ def do_delete(self, args, repository):
                 cache.commit()
                 logger.info("Archive deleted.")
                 if args.stats:
-                    log_multi(DASHES,
-                              STATS_HEADER,
-                              stats.summary.format(label='Deleted data:', stats=stats),
-                              str(cache),
-                              DASHES, logger=logging.getLogger('borg.output.stats'))
-        else:
-            if not args.cache_only:
-                msg = []
-                try:
-                    manifest, key = Manifest.load(repository)
-                except NoManifestError:
-                    msg.append("You requested to completely DELETE the repository *including* all archives it may contain.")
-                    msg.append("This repository seems to have no manifest, so we can't tell anything about its contents.")
-                else:
-                    msg.append("You requested to completely DELETE the repository *including* all archives it contains:")
-                    for archive_info in manifest.archives.list(sort_by='ts'):
-                        msg.append(format_archive(archive_info))
-                msg.append("Type 'YES' if you understand this and want to continue: ")
-                msg = '\n'.join(msg)
-                if not yes(msg, false_msg="Aborting.", invalid_msg='Invalid answer, aborting.', truish=('YES', ),
-                           retry=False, env_var_override='BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'):
-                    self.exit_code = EXIT_ERROR
-                    return self.exit_code
-                repository.destroy()
-                logger.info("Repository deleted.")
-            Cache.destroy(repository)
-            logger.info("Cache deleted.")
+                    log_multi(stats.summary.format(label='Deleted data:', stats=stats),
+                              DASHES, logger=stats_logger)
+                if not args.forced and self.exit_code:
+                    break
+            if args.stats:
+                stats_logger.info(str(cache))
+
+        return self.exit_code
+
+    def _delete_repository(self, args, repository):
+        """Delete a repository"""
+        if not args.cache_only:
+            msg = []
+            try:
+                manifest, key = Manifest.load(repository)
+            except NoManifestError:
+                msg.append("You requested to completely DELETE the repository *including* all archives it may "
+                           "contain.")
+                msg.append("This repository seems to have no manifest, so we can't tell anything about its "
+                           "contents.")
+            else:
+                msg.append("You requested to completely DELETE the repository *including* all archives it "
+                           "contains:")
+                for archive_info in manifest.archives.list(sort_by='ts'):
+                    msg.append(format_archive(archive_info))
+            msg.append("Type 'YES' if you understand this and want to continue: ")
+            msg = '\n'.join(msg)
+            if not yes(msg, false_msg="Aborting.", invalid_msg='Invalid answer, aborting.', truish=('YES',),
+                       retry=False, env_var_override='BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'):
+                self.exit_code = EXIT_ERROR
+                return self.exit_code
+            repository.destroy()
+            logger.info("Repository deleted.")
+        Cache.destroy(repository)
+        logger.info("Cache deleted.")
         return self.exit_code
 
     @with_repository()
@@ -849,45 +879,62 @@ def write(bytestring):
             write = sys.stdout.buffer.write
 
         if args.location.archive:
-            matcher, _ = self.build_matcher(args.excludes, args.paths)
-            with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache:
-                archive = Archive(repository, key, manifest, args.location.archive, cache=cache,
-                                  consider_part_files=args.consider_part_files)
-
-                if args.format is not None:
-                    format = args.format
-                elif args.short:
-                    format = "{path}{NL}"
-                else:
-                    format = "{mode} {user:6} {group:6} {size:8} {isomtime} {path}{extra}{NL}"
-                formatter = ItemFormatter(archive, format)
-
-                for item in archive.iter_items(lambda item: matcher.match(item.path)):
-                    write(safe_encode(formatter.format_item(item)))
+            return self._list_archive(args, repository, manifest, key, write)
         else:
+            return self._list_repository(args, manifest, write)
+
+    def _list_archive(self, args, repository, manifest, key, write):
+        matcher, _ = self.build_matcher(args.excludes, args.paths)
+        with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache:
+            archive = Archive(repository, key, manifest, args.location.archive, cache=cache,
+                              consider_part_files=args.consider_part_files)
             if args.format is not None:
                 format = args.format
             elif args.short:
-                format = "{archive}{NL}"
+                format = "{path}{NL}"
             else:
-                format = "{archive:<36} {time} [{id}]{NL}"
-            formatter = ArchiveFormatter(format)
+                format = "{mode} {user:6} {group:6} {size:8} {isomtime} {path}{extra}{NL}"
+            formatter = ItemFormatter(archive, format)
 
-            for archive_info in manifest.archives.list(sort_by='ts'):
-                if args.prefix and not archive_info.name.startswith(args.prefix):
-                    continue
-                write(safe_encode(formatter.format_item(archive_info)))
+            for item in archive.iter_items(lambda item: matcher.match(item.path)):
+                write(safe_encode(formatter.format_item(item)))
+        return self.exit_code
+
+    def _list_repository(self, args, manifest, write):
+        if args.format is not None:
+            format = args.format
+        elif args.short:
+            format = "{archive}{NL}"
+        else:
+            format = "{archive:<36} {time} [{id}]{NL}"
+        formatter = ArchiveFormatter(format)
+
+        for archive_info in self._get_filtered_archives(args, manifest):
+            write(safe_encode(formatter.format_item(archive_info)))
 
         return self.exit_code
 
     @with_repository(cache=True)
     def do_info(self, args, repository, manifest, key, cache):
         """Show archive details such as disk space used"""
+        if any((args.location.archive, args.first, args.last, args.prefix)):
+            return self._info_archives(args, repository, manifest, key, cache)
+        else:
+            return self._info_repository(cache)
+
+    def _info_archives(self, args, repository, manifest, key, cache):
         def format_cmdline(cmdline):
             return remove_surrogates(' '.join(shlex.quote(x) for x in cmdline))
 
         if args.location.archive:
-            archive = Archive(repository, key, manifest, args.location.archive, cache=cache,
+            archive_names = (args.location.archive,)
+        else:
+            archive_names = tuple(x.name for x in self._get_filtered_archives(args, manifest))
+            if not archive_names:
+                return self.exit_code
+
+        for i, archive_name in enumerate(archive_names, 1):
+            archive = Archive(repository, key, manifest, archive_name, cache=cache,
                               consider_part_files=args.consider_part_files)
             stats = archive.calc_stats(cache)
             print('Archive name: %s' % archive.name)
@@ -904,9 +951,15 @@ def format_cmdline(cmdline):
             print(STATS_HEADER)
             print(str(stats))
             print(str(cache))
-        else:
-            print(STATS_HEADER)
-            print(str(cache))
+            if self.exit_code:
+                break
+            if len(archive_names) - i:
+                print()
+        return self.exit_code
+
+    def _info_repository(self, cache):
+        print(STATS_HEADER)
+        print(str(cache))
         return self.exit_code
 
     @with_repository(exclusive=True)
@@ -1967,6 +2020,7 @@ def build_parser(self, prog=None):
         subparser.add_argument('location', metavar='TARGET', nargs='?', default='',
                                type=location_validator(),
                                help='archive or repository to delete')
+        self.add_archives_filters_args(subparser)
 
         list_epilog = textwrap.dedent("""
         This command lists the contents of a repository or an archive.
@@ -1993,8 +2047,6 @@ def build_parser(self, prog=None):
         subparser.add_argument('--format', '--list-format', dest='format', type=str,
                                help="""specify format for file listing
                                 (default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NL}")""")
-        subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec,
-                               help='only consider archive names starting with this prefix')
         subparser.add_argument('-e', '--exclude', dest='excludes',
                                type=parse_pattern, action='append',
                                metavar="PATTERN", help='exclude paths matching PATTERN')
@@ -2006,6 +2058,7 @@ def build_parser(self, prog=None):
                                help='repository/archive to list contents of')
         subparser.add_argument('paths', metavar='PATH', nargs='*', type=str,
                                help='paths to list; patterns are supported')
+        self.add_archives_filters_args(subparser)
 
         mount_epilog = textwrap.dedent("""
         This command mounts an archive as a FUSE filesystem. This can be useful for
@@ -2071,6 +2124,7 @@ def build_parser(self, prog=None):
         subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE',
                                type=location_validator(),
                                help='archive or repository to display information about')
+        self.add_archives_filters_args(subparser)
 
         break_lock_epilog = textwrap.dedent("""
         This command breaks the repository and cache locks.
@@ -2549,6 +2603,23 @@ def build_parser(self, prog=None):
 
         return parser
 
+    @staticmethod
+    def add_archives_filters_args(subparser):
+        filters_group = subparser.add_argument_group('filters', 'Archive filters can be applied to repository targets.')
+        filters_group.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec, default='',
+                                   help='only consider archive names starting with this prefix')
+
+        sort_by_default = 'timestamp'
+        filters_group.add_argument('--sort-by', dest='sort_by', type=SortBySpec, default=sort_by_default,
+                                   help='Comma-separated list of sorting keys; valid keys are: {}; default is: {}'
+                                   .format(', '.join(HUMAN_SORT_KEYS), sort_by_default))
+
+        group = filters_group.add_mutually_exclusive_group()
+        group.add_argument('--first', dest='first', metavar='N', default=0, type=int,
+                           help='consider first N archives after other filters were applied')
+        group.add_argument('--last', dest='last', metavar='N', default=0, type=int,
+                           help='consider last N archives after other filters were applied')
+
     def get_args(self, argv, cmd):
         """usually, just returns argv, except if we deal with a ssh forced command for borg serve."""
         result = self.parse_args(argv[1:])
@@ -2611,6 +2682,21 @@ def run(self, args):
             logger.warning("Using a pure-python msgpack! This will result in lower performance.")
         return args.func(args)
 
+    def _get_filtered_archives(self, args, manifest):
+        if args.location.archive:
+            raise Error('The options --first, --last and --prefix can only be used on repository targets.')
+
+        archives = manifest.archives.list(prefix=args.prefix)
+
+        for sortkey in reversed(args.sort_by.split(',')):
+            archives.sort(key=attrgetter(sortkey))
+        if args.last:
+            archives.reverse()
+
+        n = args.first or args.last or len(archives)
+
+        return archives[:n]
+
 
 def sig_info_handler(sig_no, stack):  # pragma: no cover
     """search the stack for infos about the currently processed file and print them"""

src/borg/helpers.py

@@ -142,11 +142,16 @@ def __delitem__(self, name):
         name = safe_encode(name)
         del self._archives[name]
 
-    def list(self, sort_by=None, reverse=False):
-        # inexpensive Archive.list_archives replacement if we just need .name, .id, .ts
-        archives = self.values()  # [self[name] for name in self]
+    def list(self, sort_by=None, reverse=False, prefix=''):
+        """
+        Inexpensive Archive.list_archives replacement if we just need .name, .id, .ts
+        Returns list of borg.helpers.ArchiveInfo instances
+        """
+        archives = [x for x in self.values() if x.name.startswith(prefix)]
         if sort_by is not None:
-            archives = sorted(archives, key=attrgetter(sort_by), reverse=reverse)
+            archives = sorted(archives, key=attrgetter(sort_by))
+        if reverse:
+            archives.reverse()
         return archives
 
     def set_raw_dict(self, d):
@@ -568,10 +573,6 @@ def CompressionSpec(s):
     raise ValueError
 
 
-def PrefixSpec(s):
-    return replace_placeholders(s)
-
-
 def dir_is_cachedir(path):
     """Determines whether the specified path is a cache directory (and
     therefore should potentially be excluded from the backup) according to
@@ -654,6 +655,19 @@ def replace_placeholders(text):
     }
     return format_line(text, data)
 
+PrefixSpec = replace_placeholders
+
+
+HUMAN_SORT_KEYS = ['timestamp'] + list(ArchiveInfo._fields)
+HUMAN_SORT_KEYS.remove('ts')
+
+
+def SortBySpec(text):
+    for token in text.split(','):
+        if token not in HUMAN_SORT_KEYS:
+            raise ValueError('Invalid sort key: %s' % token)
+    return text.replace('timestamp', 'ts')
+
 
 def safe_timestamp(item_timestamp_ns):
     try:

src/borg/testsuite/archiver.py

@@ -59,6 +59,9 @@ def exec_cmd(*args, archiver=None, fork=False, exe=None, **kw):
         except subprocess.CalledProcessError as e:
             output = e.output
             ret = e.returncode
+        except SystemExit as e:  # possibly raised by argparse
+            output = ''
+            ret = e.code
         return ret, os.fsdecode(output)
     else:
         stdin, stdout, stderr = sys.stdin, sys.stdout, sys.stderr
@@ -961,6 +964,8 @@ def test_info(self):
         assert 'All archives:' in info_repo
         info_archive = self.cmd('info', self.repository_location + '::test')
         assert 'Archive name: test\n' in info_archive
+        info_archive = self.cmd('info', '--first', '1', self.repository_location)
+        assert 'Archive name: test\n' in info_archive
 
     def test_comment(self):
         self.create_regular_file('file1', size=1024 * 80)
@@ -987,8 +992,13 @@ def test_delete(self):
         self.cmd('init', self.repository_location)
         self.cmd('create', self.repository_location + '::test', 'input')
         self.cmd('create', self.repository_location + '::test.2', 'input')
+        self.cmd('create', self.repository_location + '::test.3', 'input')
+        self.cmd('create', self.repository_location + '::another_test.1', 'input')
+        self.cmd('create', self.repository_location + '::another_test.2', 'input')
         self.cmd('extract', '--dry-run', self.repository_location + '::test')
         self.cmd('extract', '--dry-run', self.repository_location + '::test.2')
+        self.cmd('delete', '--prefix', 'another_', self.repository_location)
+        self.cmd('delete', '--last', '1', self.repository_location)
         self.cmd('delete', self.repository_location + '::test')
         self.cmd('extract', '--dry-run', self.repository_location + '::test.2')
         output = self.cmd('delete', '--stats', self.repository_location + '::test.2')
@@ -1811,6 +1821,11 @@ def test_recreate_list_output(self):
         self.assert_not_in("input/file1", output)
         self.assert_not_in("x input/file5", output)
 
+    def test_bad_filters(self):
+        self.cmd('init', self.repository_location)
+        self.cmd('create', self.repository_location + '::test', 'input')
+        self.cmd('delete', '--first', '1', '--last', '1', self.repository_location, fork=True, exit_code=2)
+
     def test_key_export_keyfile(self):
         export_file = self.output_path + '/exported'
         self.cmd('init', self.repository_location, '--encryption', 'keyfile')