From dd6b90fe6c2d5369081adf97379118fc903cdbee Mon Sep 17 00:00:00 2001 From: Leo Antunes Date: Sun, 29 Jan 2017 18:13:51 +0100 Subject: [PATCH] change dir_is_tagged to use os.path.exists() Add --keep-exclude-tags option as alias to --keep-tag-files and deprecate the later. Also make tagging accept directories as tags, allowing things like `--exclude-if-present .git`. fixes #1999 --- docs/changes.rst | 12 ++++++++++++ src/borg/archive.py | 12 ++++++------ src/borg/archiver.py | 33 +++++++++++++++++++-------------- src/borg/helpers.py | 8 ++++---- src/borg/testsuite/archiver.py | 17 +++++++++++------ 5 files changed, 52 insertions(+), 30 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 9a21a299e..70bab92f8 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -126,6 +126,18 @@ The best check that everything is ok is to run a dry-run extraction:: Changelog ========= +Version 1.1.0b4 (not released yet) +---------------------------------- + +New features: + +- the --exclude-if-present option now supports tagging a folder with any + filesystem object type (file, folder, etc), instead of expecting only files + as tags, #1999 +- the --keep-tag-files option has been deprecated in favor of the new + --keep-exclude-tags, to account for the change mentioned above. + + Version 1.1.0b3 (2017-01-15) ---------------------------- diff --git a/src/borg/archive.py b/src/borg/archive.py index a08f5fc38..16424196b 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -1439,7 +1439,7 @@ class ArchiveRecreater: return archive_name.endswith('.recreate') def __init__(self, repository, manifest, key, cache, matcher, - exclude_caches=False, exclude_if_present=None, keep_tag_files=False, + exclude_caches=False, exclude_if_present=None, keep_exclude_tags=False, chunker_params=None, compression=None, compression_files=None, always_recompress=False, dry_run=False, stats=False, progress=False, file_status_printer=None, checkpoint_interval=1800): @@ -1451,7 +1451,7 @@ class ArchiveRecreater: self.matcher = matcher self.exclude_caches = exclude_caches self.exclude_if_present = exclude_if_present or [] - self.keep_tag_files = keep_tag_files + self.keep_exclude_tags = keep_exclude_tags self.rechunkify = chunker_params is not None if self.rechunkify: @@ -1591,7 +1591,7 @@ class ArchiveRecreater: def matcher_add_tagged_dirs(self, archive): """Add excludes to the matcher created by exclude_cache and exclude_if_present.""" def exclude(dir, tag_item): - if self.keep_tag_files: + if self.keep_exclude_tags: tag_files.append(PathPrefixPattern(tag_item.path)) tagged_dirs.append(FnmatchPattern(dir + '/')) else: @@ -1607,10 +1607,10 @@ class ArchiveRecreater: filter=lambda item: item.path.endswith(CACHE_TAG_NAME) or matcher.match(item.path)): if item.path.endswith(CACHE_TAG_NAME): cachedir_masters[item.path] = item + dir, tag_file = os.path.split(item.path) + if tag_file in self.exclude_if_present: + exclude(dir, item) if stat.S_ISREG(item.mode): - dir, tag_file = os.path.split(item.path) - if tag_file in self.exclude_if_present: - exclude(dir, item) if self.exclude_caches and tag_file == CACHE_TAG_NAME: if 'chunks' in item: file = open_item(archive, item) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 80ed853cb..a23a61fc6 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -346,7 +346,7 @@ class Archiver: else: restrict_dev = None self._process(archive, cache, matcher, args.exclude_caches, args.exclude_if_present, - args.keep_tag_files, skip_inodes, path, restrict_dev, + args.keep_exclude_tags, skip_inodes, path, restrict_dev, read_special=args.read_special, dry_run=dry_run, st=st) if not dry_run: archive.save(comment=args.comment, timestamp=args.timestamp) @@ -382,7 +382,7 @@ class Archiver: return self.exit_code def _process(self, archive, cache, matcher, exclude_caches, exclude_if_present, - keep_tag_files, skip_inodes, path, restrict_dev, + keep_exclude_tags, skip_inodes, path, restrict_dev, read_special=False, dry_run=False, st=None): if not matcher.match(path): self.print_file_status('x', path) @@ -419,11 +419,11 @@ class Archiver: if recurse: tag_paths = dir_is_tagged(path, exclude_caches, exclude_if_present) if tag_paths: - if keep_tag_files and not dry_run: + if keep_exclude_tags and not dry_run: archive.process_dir(path, st) for tag_path in tag_paths: self._process(archive, cache, matcher, exclude_caches, exclude_if_present, - keep_tag_files, skip_inodes, tag_path, restrict_dev, + keep_exclude_tags, skip_inodes, tag_path, restrict_dev, read_special=read_special, dry_run=dry_run) return if not dry_run: @@ -438,7 +438,7 @@ class Archiver: for dirent in entries: normpath = os.path.normpath(dirent.path) self._process(archive, cache, matcher, exclude_caches, exclude_if_present, - keep_tag_files, skip_inodes, normpath, restrict_dev, + keep_exclude_tags, skip_inodes, normpath, restrict_dev, read_special=read_special, dry_run=dry_run) elif stat.S_ISLNK(st.st_mode): if not dry_run: @@ -1151,7 +1151,7 @@ class Archiver: recreater = ArchiveRecreater(repository, manifest, key, cache, matcher, exclude_caches=args.exclude_caches, exclude_if_present=args.exclude_if_present, - keep_tag_files=args.keep_tag_files, chunker_params=args.chunker_params, + keep_exclude_tags=args.keep_exclude_tags, chunker_params=args.chunker_params, compression=args.compression, compression_files=args.compression_files, always_recompress=args.always_recompress, progress=args.progress, stats=args.stats, @@ -1571,6 +1571,7 @@ class Archiver: deprecations = [ # ('--old', '--new', 'Warning: "--old" has been deprecated. Use "--new" instead.'), ('--list-format', '--format', 'Warning: "--list-format" has been deprecated. Use "--format" instead.'), + ('--keep-tag-files', '--keep-exclude-tags', 'Warning: "--keep-tag-files" has been deprecated. Use "--keep-exclude-tags" instead.'), ] for i, arg in enumerate(args[:]): for old_name, new_name, warning in deprecations: @@ -1974,11 +1975,13 @@ class Archiver: help='exclude directories that contain a CACHEDIR.TAG file (' 'http://www.brynosaurus.com/cachedir/spec.html)') exclude_group.add_argument('--exclude-if-present', dest='exclude_if_present', - metavar='FILENAME', action='append', type=str, - help='exclude directories that contain the specified file') - exclude_group.add_argument('--keep-tag-files', dest='keep_tag_files', + metavar='NAME', action='append', type=str, + help='exclude directories that are tagged by containing a filesystem object with \ + the given NAME') + exclude_group.add_argument('--keep-exclude-tags', '--keep-tag-files', dest='keep_exclude_tags', action='store_true', default=False, - help='keep tag files of excluded caches/directories') + help='keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise \ + excluded caches/directories') fs_group = subparser.add_argument_group('Filesystem options') fs_group.add_argument('-x', '--one-file-system', dest='one_file_system', @@ -2562,11 +2565,13 @@ class Archiver: help='exclude directories that contain a CACHEDIR.TAG file (' 'http://www.brynosaurus.com/cachedir/spec.html)') exclude_group.add_argument('--exclude-if-present', dest='exclude_if_present', - metavar='FILENAME', action='append', type=str, - help='exclude directories that contain the specified file') - exclude_group.add_argument('--keep-tag-files', dest='keep_tag_files', + metavar='NAME', action='append', type=str, + help='exclude directories that are tagged by containing a filesystem object with \ + the given NAME') + exclude_group.add_argument('--keep-exclude-tags', '--keep-tag-files', dest='keep_exclude_tags', action='store_true', default=False, - help='keep tag files of excluded caches/directories') + help='keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise \ + excluded caches/directories') archive_group = subparser.add_argument_group('Archive options') archive_group.add_argument('--target', dest='target', metavar='TARGET', default=None, diff --git a/src/borg/helpers.py b/src/borg/helpers.py index f4338e75c..e61a6d57c 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -633,9 +633,9 @@ def dir_is_cachedir(path): def dir_is_tagged(path, exclude_caches, exclude_if_present): """Determines whether the specified path is excluded by being a cache - directory or containing user-specified tag files. Returns a list of the - paths of the tag files (either CACHEDIR.TAG or the matching - user-specified files). + directory or containing user-specified tag files/directories. Returns a + list of the paths of the tag files/directories (either CACHEDIR.TAG or the + matching user-specified files/directories). """ tag_paths = [] if exclude_caches and dir_is_cachedir(path): @@ -643,7 +643,7 @@ def dir_is_tagged(path, exclude_caches, exclude_if_present): if exclude_if_present is not None: for tag in exclude_if_present: tag_path = os.path.join(path, tag) - if os.path.isfile(tag_path): + if os.path.exists(tag_path): tag_paths.append(tag_path) return tag_paths diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 2d97ce06d..f5b1fe8cb 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -898,12 +898,12 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.create_regular_file('file1', size=1024 * 80) self.create_regular_file('tagged1/.NOBACKUP') self.create_regular_file('tagged2/00-NOBACKUP') - self.create_regular_file('tagged3/.NOBACKUP/file2') + self.create_regular_file('tagged3/.NOBACKUP/file2', size=1024) def _assert_test_tagged(self): with changedir('output'): self.cmd('extract', self.repository_location + '::test') - self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'tagged3']) + self.assert_equal(sorted(os.listdir('output/input')), ['file1']) def test_exclude_tagged(self): self._create_test_tagged() @@ -922,13 +922,13 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.create_regular_file('file0', size=1024) self.create_regular_file('tagged1/.NOBACKUP1') self.create_regular_file('tagged1/file1', size=1024) - self.create_regular_file('tagged2/.NOBACKUP2') + self.create_regular_file('tagged2/.NOBACKUP2/subfile1', size=1024) self.create_regular_file('tagged2/file2', size=1024) self.create_regular_file('tagged3/%s' % CACHE_TAG_NAME, contents=CACHE_TAG_CONTENTS + b' extra stuff') self.create_regular_file('tagged3/file3', size=1024) self.create_regular_file('taggedall/.NOBACKUP1') - self.create_regular_file('taggedall/.NOBACKUP2') + self.create_regular_file('taggedall/.NOBACKUP2/subfile1', size=1024) self.create_regular_file('taggedall/%s' % CACHE_TAG_NAME, contents=CACHE_TAG_CONTENTS + b' extra stuff') self.create_regular_file('taggedall/file4', size=1024) @@ -943,17 +943,22 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.assert_equal(sorted(os.listdir('output/input/taggedall')), ['.NOBACKUP1', '.NOBACKUP2', CACHE_TAG_NAME, ]) + def test_exclude_keep_tagged_deprecation(self): + self.cmd('init', '--encryption=repokey', self.repository_location) + output_warn = self.cmd('create', '--exclude-caches', '--keep-tag-files', self.repository_location + '::test', src_dir) + self.assert_in('--keep-tag-files" has been deprecated.', output_warn) + def test_exclude_keep_tagged(self): self._create_test_keep_tagged() self.cmd('create', '--exclude-if-present', '.NOBACKUP1', '--exclude-if-present', '.NOBACKUP2', - '--exclude-caches', '--keep-tag-files', self.repository_location + '::test', 'input') + '--exclude-caches', '--keep-exclude-tags', self.repository_location + '::test', 'input') self._assert_test_keep_tagged() def test_recreate_exclude_keep_tagged(self): self._create_test_keep_tagged() self.cmd('create', self.repository_location + '::test', 'input') self.cmd('recreate', '--exclude-if-present', '.NOBACKUP1', '--exclude-if-present', '.NOBACKUP2', - '--exclude-caches', '--keep-tag-files', self.repository_location + '::test') + '--exclude-caches', '--keep-exclude-tags', self.repository_location + '::test') self._assert_test_keep_tagged() @pytest.mark.skipif(not xattr.XATTR_FAKEROOT, reason='Linux capabilities test, requires fakeroot >= 1.20.2')