1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2025-03-15 00:21:56 +00:00

Merge pull request #1194 from ThomasWaldmann/more-placeholders

add placeholder support at missing places, add missing help
This commit is contained in:
enkore 2016-06-23 17:38:04 +02:00 committed by GitHub
commit 5b34483310
6 changed files with 117 additions and 58 deletions

View file

@ -20,7 +20,7 @@ from .helpers import Error, location_validator, archivename_validator, format_li
parse_pattern, PathPrefixPattern, to_localtime, timestamp, safe_timestamp, \ parse_pattern, PathPrefixPattern, to_localtime, timestamp, safe_timestamp, \
get_cache_dir, prune_within, prune_split, \ get_cache_dir, prune_within, prune_split, \
Manifest, NoManifestError, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ Manifest, NoManifestError, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \
dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, is_slow_msgpack, yes, sysinfo, \ dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, PrefixSpec, is_slow_msgpack, yes, sysinfo, \
EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, log_multi, PatternMatcher EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, log_multi, PatternMatcher
from .logger import create_logger, setup_logging from .logger import create_logger, setup_logging
logger = create_logger() logger = create_logger()
@ -773,35 +773,68 @@ class Archiver:
whitespace removal paths with whitespace at the beginning or end can only be whitespace removal paths with whitespace at the beginning or end can only be
excluded using regular expressions. excluded using regular expressions.
Examples: Examples::
# Exclude '/home/user/file.o' but not '/home/user/file.odt': # Exclude '/home/user/file.o' but not '/home/user/file.odt':
$ borg create -e '*.o' backup / $ borg create -e '*.o' backup /
# Exclude '/home/user/junk' and '/home/user/subdir/junk' but # Exclude '/home/user/junk' and '/home/user/subdir/junk' but
# not '/home/user/importantjunk' or '/etc/junk': # not '/home/user/importantjunk' or '/etc/junk':
$ borg create -e '/home/*/junk' backup / $ borg create -e '/home/*/junk' backup /
# Exclude the contents of '/home/user/cache' but not the directory itself: # Exclude the contents of '/home/user/cache' but not the directory itself:
$ borg create -e /home/user/cache/ backup / $ borg create -e /home/user/cache/ backup /
# The file '/home/user/cache/important' is *not* backed up: # The file '/home/user/cache/important' is *not* backed up:
$ borg create -e /home/user/cache/ backup / /home/user/cache/important $ borg create -e /home/user/cache/ backup / /home/user/cache/important
# The contents of directories in '/home' are not backed up when their name # The contents of directories in '/home' are not backed up when their name
# ends in '.tmp' # ends in '.tmp'
$ borg create --exclude 're:^/home/[^/]+\.tmp/' backup / $ borg create --exclude 're:^/home/[^/]+\.tmp/' backup /
# Load exclusions from file # Load exclusions from file
$ cat >exclude.txt <<EOF $ cat >exclude.txt <<EOF
# Comment line # Comment line
/home/*/junk /home/*/junk
*.tmp *.tmp
fm:aa:something/* fm:aa:something/*
re:^/home/[^/]\.tmp/ re:^/home/[^/]\.tmp/
sh:/home/*/.thumbnails sh:/home/*/.thumbnails
EOF EOF
$ borg create --exclude-from exclude.txt backup / $ borg create --exclude-from exclude.txt backup /
''')
helptext['placeholders'] = textwrap.dedent('''
Repository (or Archive) URLs and --prefix values support these placeholders:
{hostname}
The (short) hostname of the machine.
{fqdn}
The full name of the machine.
{now}
The current local date and time.
{utcnow}
The current UTC date and time.
{user}
The user name (or UID, if no name is available) of the user running borg.
{pid}
The current process ID.
Examples::
borg create /path/to/repo::{hostname}-{user}-{utcnow} ...
borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S} ...
borg prune --prefix '{hostname}-' ...
''') ''')
def do_help(self, parser, commands, args): def do_help(self, parser, commands, args):
@ -952,7 +985,7 @@ class Archiver:
subparser.add_argument('--last', dest='last', subparser.add_argument('--last', dest='last',
type=int, default=None, metavar='N', type=int, default=None, metavar='N',
help='only check last N archives (Default: all)') help='only check last N archives (Default: all)')
subparser.add_argument('-P', '--prefix', dest='prefix', type=str, subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec,
help='only consider archive names starting with this prefix') help='only consider archive names starting with this prefix')
change_passphrase_epilog = textwrap.dedent(""" change_passphrase_epilog = textwrap.dedent("""
@ -1013,6 +1046,7 @@ class Archiver:
all files on these file systems. all files on these file systems.
See the output of the "borg help patterns" command for more help on exclude patterns. See the output of the "borg help patterns" command for more help on exclude patterns.
See the output of the "borg help placeholders" command for more help on placeholders.
""") """)
subparser = subparsers.add_parser('create', parents=[common_parser], subparser = subparsers.add_parser('create', parents=[common_parser],
@ -1194,7 +1228,7 @@ class Archiver:
help="""specify format for archive file listing help="""specify format for archive file listing
(default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}") (default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}")
Special "{formatkeys}" exists to list available keys""") Special "{formatkeys}" exists to list available keys""")
subparser.add_argument('-P', '--prefix', dest='prefix', type=str, subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec,
help='only consider archive names starting with this prefix') help='only consider archive names starting with this prefix')
subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', nargs='?', default='', subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', nargs='?', default='',
type=location_validator(), type=location_validator(),
@ -1301,7 +1335,7 @@ class Archiver:
help='number of monthly archives to keep') help='number of monthly archives to keep')
subparser.add_argument('-y', '--keep-yearly', dest='yearly', type=int, default=0, subparser.add_argument('-y', '--keep-yearly', dest='yearly', type=int, default=0,
help='number of yearly archives to keep') help='number of yearly archives to keep')
subparser.add_argument('-P', '--prefix', dest='prefix', type=str, subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec,
help='only consider archive names starting with this prefix') help='only consider archive names starting with this prefix')
subparser.add_argument('--save-space', dest='save_space', action='store_true', subparser.add_argument('--save-space', dest='save_space', action='store_true',
default=False, default=False,
@ -1522,7 +1556,15 @@ def main(): # pragma: no cover
setup_signal_handlers() setup_signal_handlers()
archiver = Archiver() archiver = Archiver()
msg = None msg = None
args = archiver.get_args(sys.argv, os.environ.get('SSH_ORIGINAL_COMMAND')) try:
args = archiver.get_args(sys.argv, os.environ.get('SSH_ORIGINAL_COMMAND'))
except Error as e:
msg = e.get_message()
if e.traceback:
msg += "\n%s\n%s" % (traceback.format_exc(), sysinfo())
# we might not have logging setup yet, so get out quickly
print(msg, file=sys.stderr)
sys.exit(e.exit_code)
try: try:
exit_code = archiver.run(args) exit_code = archiver.run(args)
except Error as e: except Error as e:

View file

@ -69,6 +69,10 @@ class NoManifestError(Error):
"""Repository has no manifest.""" """Repository has no manifest."""
class PlaceholderError(Error):
"""Formatting Error: "{}".format({}): {}({})"""
def check_extension_modules(): def check_extension_modules():
from . import platform from . import platform
if hashindex.API_VERSION != 2: if hashindex.API_VERSION != 2:
@ -514,6 +518,10 @@ def CompressionSpec(s):
raise ValueError raise ValueError
def PrefixSpec(s):
return replace_placeholders(s)
def dir_is_cachedir(path): def dir_is_cachedir(path):
"""Determines whether the specified path is a cache directory (and """Determines whether the specified path is a cache directory (and
therefore should potentially be excluded from the backup) according to therefore should potentially be excluded from the backup) according to
@ -552,18 +560,24 @@ def dir_is_tagged(path, exclude_caches, exclude_if_present):
def format_line(format, data): def format_line(format, data):
# TODO: Filter out unwanted properties of str.format(), because "format" is user provided.
try: try:
return format.format(**data) return format.format(**data)
except (KeyError, ValueError) as e:
# this should catch format errors
print('Error in lineformat: "{}" - reason "{}"'.format(format, str(e)))
except Exception as e: except Exception as e:
# something unexpected, print error and raise exception raise PlaceholderError(format, data, e.__class__.__name__, str(e))
print('Error in lineformat: "{}" - reason "{}"'.format(format, str(e)))
raise
return '' def replace_placeholders(text):
"""Replace placeholders in text with their values."""
current_time = datetime.now()
data = {
'pid': os.getpid(),
'fqdn': socket.getfqdn(),
'hostname': socket.gethostname(),
'now': current_time.now(),
'utcnow': current_time.utcnow(),
'user': uid2user(os.getuid(), os.getuid())
}
return format_line(text, data)
def safe_timestamp(item_timestamp_ns): def safe_timestamp(item_timestamp_ns):
@ -720,21 +734,8 @@ class Location:
if not self.parse(self.orig): if not self.parse(self.orig):
raise ValueError raise ValueError
def preformat_text(self, text):
"""Format repository and archive path with common tags"""
current_time = datetime.now()
data = {
'pid': os.getpid(),
'fqdn': socket.getfqdn(),
'hostname': socket.gethostname(),
'now': current_time.now(),
'utcnow': current_time.utcnow(),
'user': uid2user(os.getuid(), os.getuid())
}
return format_line(text, data)
def parse(self, text): def parse(self, text):
text = self.preformat_text(text) text = replace_placeholders(text)
valid = self._parse(text) valid = self._parse(text)
if valid: if valid:
return True return True

View file

@ -946,7 +946,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
self.cmd('create', test_archive, src_dir) self.cmd('create', test_archive, src_dir)
output_1 = self.cmd('list', test_archive) output_1 = self.cmd('list', test_archive)
output_2 = self.cmd('list', '--list-format', '{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}', test_archive) output_2 = self.cmd('list', '--list-format', '{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}', test_archive)
output_3 = self.cmd('list', '--list-format', '{mtime:%s} {path}{NL}', test_archive) output_3 = self.cmd('list', '--list-format', '{mtime:%s} {path}{NEWLINE}', test_archive)
self.assertEqual(output_1, output_2) self.assertEqual(output_1, output_2)
self.assertNotEqual(output_1, output_3) self.assertNotEqual(output_1, output_3)

View file

@ -10,7 +10,7 @@ import msgpack
import msgpack.fallback import msgpack.fallback
import time import time
from ..helpers import Location, format_file_size, format_timedelta, make_path_safe, \ from ..helpers import Location, format_file_size, format_timedelta, format_line, PlaceholderError, make_path_safe, \
prune_within, prune_split, get_cache_dir, get_keys_dir, Statistics, is_slow_msgpack, \ prune_within, prune_split, get_cache_dir, get_keys_dir, Statistics, is_slow_msgpack, \
yes, TRUISH, FALSISH, DEFAULTISH, \ yes, TRUISH, FALSISH, DEFAULTISH, \
StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \ StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \
@ -877,3 +877,18 @@ def test_progress_endless_step(capfd):
pi.show() pi.show()
out, err = capfd.readouterr() out, err = capfd.readouterr()
assert err == '.' assert err == '.'
def test_format_line():
data = dict(foo='bar baz')
assert format_line('', data) == ''
assert format_line('{foo}', data) == 'bar baz'
assert format_line('foo{foo}foo', data) == 'foobar bazfoo'
def test_format_line_erroneous():
data = dict()
with pytest.raises(PlaceholderError):
assert format_line('{invalid}', data)
with pytest.raises(PlaceholderError):
assert format_line('{}', data)

View file

@ -110,7 +110,7 @@ certain number of old archives::
# Backup all of /home and /var/www except a few # Backup all of /home and /var/www except a few
# excluded directories # excluded directories
borg create -v --stats \ borg create -v --stats \
$REPOSITORY::`hostname`-`date +%Y-%m-%d` \ $REPOSITORY::'{hostname}-{now:%Y-%m-%d}' \
/home \ /home \
/var/www \ /var/www \
--exclude '/home/*/.cache' \ --exclude '/home/*/.cache' \
@ -118,10 +118,10 @@ certain number of old archives::
--exclude '*.pyc' --exclude '*.pyc'
# Use the `prune` subcommand to maintain 7 daily, 4 weekly and 6 monthly # Use the `prune` subcommand to maintain 7 daily, 4 weekly and 6 monthly
# archives of THIS machine. --prefix `hostname`- is very important to # archives of THIS machine. The '{hostname}-' prefix is very important to
# limit prune's operation to this machine's archives and not apply to # limit prune's operation to this machine's archives and not apply to
# other machine's archives also. # other machine's archives also.
borg prune -v $REPOSITORY --prefix `hostname`- \ borg prune -v $REPOSITORY --prefix '{hostname}-' \
--keep-daily=7 --keep-weekly=4 --keep-monthly=6 --keep-daily=7 --keep-weekly=4 --keep-monthly=6
.. backup_compression: .. backup_compression:

View file

@ -425,8 +425,9 @@ will see what it would do without it actually doing anything.
# Do a dry-run without actually deleting anything. # Do a dry-run without actually deleting anything.
$ borg prune --dry-run --keep-daily=7 --keep-weekly=4 /path/to/repo $ borg prune --dry-run --keep-daily=7 --keep-weekly=4 /path/to/repo
# Same as above but only apply to archive names starting with "foo": # Same as above but only apply to archive names starting with the hostname
$ borg prune --keep-daily=7 --keep-weekly=4 --prefix=foo /path/to/repo # of the machine followed by a "-" character:
$ borg prune --keep-daily=7 --keep-weekly=4 --prefix='{hostname}-' /path/to/repo
# Keep 7 end of day, 4 additional end of week archives, # Keep 7 end of day, 4 additional end of week archives,
# and an end of month archive for every month: # and an end of month archive for every month: