mirror of
https://github.com/borgbackup/borg.git
synced 2025-03-15 00:21:56 +00:00
Merge pull request #1194 from ThomasWaldmann/more-placeholders
add placeholder support at missing places, add missing help
This commit is contained in:
commit
5b34483310
6 changed files with 117 additions and 58 deletions
|
@ -20,7 +20,7 @@ from .helpers import Error, location_validator, archivename_validator, format_li
|
||||||
parse_pattern, PathPrefixPattern, to_localtime, timestamp, safe_timestamp, \
|
parse_pattern, PathPrefixPattern, to_localtime, timestamp, safe_timestamp, \
|
||||||
get_cache_dir, prune_within, prune_split, \
|
get_cache_dir, prune_within, prune_split, \
|
||||||
Manifest, NoManifestError, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \
|
Manifest, NoManifestError, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \
|
||||||
dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, is_slow_msgpack, yes, sysinfo, \
|
dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, PrefixSpec, is_slow_msgpack, yes, sysinfo, \
|
||||||
EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, log_multi, PatternMatcher
|
EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, log_multi, PatternMatcher
|
||||||
from .logger import create_logger, setup_logging
|
from .logger import create_logger, setup_logging
|
||||||
logger = create_logger()
|
logger = create_logger()
|
||||||
|
@ -773,35 +773,68 @@ class Archiver:
|
||||||
whitespace removal paths with whitespace at the beginning or end can only be
|
whitespace removal paths with whitespace at the beginning or end can only be
|
||||||
excluded using regular expressions.
|
excluded using regular expressions.
|
||||||
|
|
||||||
Examples:
|
Examples::
|
||||||
|
|
||||||
# Exclude '/home/user/file.o' but not '/home/user/file.odt':
|
# Exclude '/home/user/file.o' but not '/home/user/file.odt':
|
||||||
$ borg create -e '*.o' backup /
|
$ borg create -e '*.o' backup /
|
||||||
|
|
||||||
# Exclude '/home/user/junk' and '/home/user/subdir/junk' but
|
# Exclude '/home/user/junk' and '/home/user/subdir/junk' but
|
||||||
# not '/home/user/importantjunk' or '/etc/junk':
|
# not '/home/user/importantjunk' or '/etc/junk':
|
||||||
$ borg create -e '/home/*/junk' backup /
|
$ borg create -e '/home/*/junk' backup /
|
||||||
|
|
||||||
# Exclude the contents of '/home/user/cache' but not the directory itself:
|
# Exclude the contents of '/home/user/cache' but not the directory itself:
|
||||||
$ borg create -e /home/user/cache/ backup /
|
$ borg create -e /home/user/cache/ backup /
|
||||||
|
|
||||||
# The file '/home/user/cache/important' is *not* backed up:
|
# The file '/home/user/cache/important' is *not* backed up:
|
||||||
$ borg create -e /home/user/cache/ backup / /home/user/cache/important
|
$ borg create -e /home/user/cache/ backup / /home/user/cache/important
|
||||||
|
|
||||||
# The contents of directories in '/home' are not backed up when their name
|
# The contents of directories in '/home' are not backed up when their name
|
||||||
# ends in '.tmp'
|
# ends in '.tmp'
|
||||||
$ borg create --exclude 're:^/home/[^/]+\.tmp/' backup /
|
$ borg create --exclude 're:^/home/[^/]+\.tmp/' backup /
|
||||||
|
|
||||||
# Load exclusions from file
|
# Load exclusions from file
|
||||||
$ cat >exclude.txt <<EOF
|
$ cat >exclude.txt <<EOF
|
||||||
# Comment line
|
# Comment line
|
||||||
/home/*/junk
|
/home/*/junk
|
||||||
*.tmp
|
*.tmp
|
||||||
fm:aa:something/*
|
fm:aa:something/*
|
||||||
re:^/home/[^/]\.tmp/
|
re:^/home/[^/]\.tmp/
|
||||||
sh:/home/*/.thumbnails
|
sh:/home/*/.thumbnails
|
||||||
EOF
|
EOF
|
||||||
$ borg create --exclude-from exclude.txt backup /
|
$ borg create --exclude-from exclude.txt backup /
|
||||||
|
''')
|
||||||
|
helptext['placeholders'] = textwrap.dedent('''
|
||||||
|
Repository (or Archive) URLs and --prefix values support these placeholders:
|
||||||
|
|
||||||
|
{hostname}
|
||||||
|
|
||||||
|
The (short) hostname of the machine.
|
||||||
|
|
||||||
|
{fqdn}
|
||||||
|
|
||||||
|
The full name of the machine.
|
||||||
|
|
||||||
|
{now}
|
||||||
|
|
||||||
|
The current local date and time.
|
||||||
|
|
||||||
|
{utcnow}
|
||||||
|
|
||||||
|
The current UTC date and time.
|
||||||
|
|
||||||
|
{user}
|
||||||
|
|
||||||
|
The user name (or UID, if no name is available) of the user running borg.
|
||||||
|
|
||||||
|
{pid}
|
||||||
|
|
||||||
|
The current process ID.
|
||||||
|
|
||||||
|
Examples::
|
||||||
|
|
||||||
|
borg create /path/to/repo::{hostname}-{user}-{utcnow} ...
|
||||||
|
borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S} ...
|
||||||
|
borg prune --prefix '{hostname}-' ...
|
||||||
''')
|
''')
|
||||||
|
|
||||||
def do_help(self, parser, commands, args):
|
def do_help(self, parser, commands, args):
|
||||||
|
@ -952,7 +985,7 @@ class Archiver:
|
||||||
subparser.add_argument('--last', dest='last',
|
subparser.add_argument('--last', dest='last',
|
||||||
type=int, default=None, metavar='N',
|
type=int, default=None, metavar='N',
|
||||||
help='only check last N archives (Default: all)')
|
help='only check last N archives (Default: all)')
|
||||||
subparser.add_argument('-P', '--prefix', dest='prefix', type=str,
|
subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec,
|
||||||
help='only consider archive names starting with this prefix')
|
help='only consider archive names starting with this prefix')
|
||||||
|
|
||||||
change_passphrase_epilog = textwrap.dedent("""
|
change_passphrase_epilog = textwrap.dedent("""
|
||||||
|
@ -1013,6 +1046,7 @@ class Archiver:
|
||||||
all files on these file systems.
|
all files on these file systems.
|
||||||
|
|
||||||
See the output of the "borg help patterns" command for more help on exclude patterns.
|
See the output of the "borg help patterns" command for more help on exclude patterns.
|
||||||
|
See the output of the "borg help placeholders" command for more help on placeholders.
|
||||||
""")
|
""")
|
||||||
|
|
||||||
subparser = subparsers.add_parser('create', parents=[common_parser],
|
subparser = subparsers.add_parser('create', parents=[common_parser],
|
||||||
|
@ -1194,7 +1228,7 @@ class Archiver:
|
||||||
help="""specify format for archive file listing
|
help="""specify format for archive file listing
|
||||||
(default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}")
|
(default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}")
|
||||||
Special "{formatkeys}" exists to list available keys""")
|
Special "{formatkeys}" exists to list available keys""")
|
||||||
subparser.add_argument('-P', '--prefix', dest='prefix', type=str,
|
subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec,
|
||||||
help='only consider archive names starting with this prefix')
|
help='only consider archive names starting with this prefix')
|
||||||
subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', nargs='?', default='',
|
subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', nargs='?', default='',
|
||||||
type=location_validator(),
|
type=location_validator(),
|
||||||
|
@ -1301,7 +1335,7 @@ class Archiver:
|
||||||
help='number of monthly archives to keep')
|
help='number of monthly archives to keep')
|
||||||
subparser.add_argument('-y', '--keep-yearly', dest='yearly', type=int, default=0,
|
subparser.add_argument('-y', '--keep-yearly', dest='yearly', type=int, default=0,
|
||||||
help='number of yearly archives to keep')
|
help='number of yearly archives to keep')
|
||||||
subparser.add_argument('-P', '--prefix', dest='prefix', type=str,
|
subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec,
|
||||||
help='only consider archive names starting with this prefix')
|
help='only consider archive names starting with this prefix')
|
||||||
subparser.add_argument('--save-space', dest='save_space', action='store_true',
|
subparser.add_argument('--save-space', dest='save_space', action='store_true',
|
||||||
default=False,
|
default=False,
|
||||||
|
@ -1522,7 +1556,15 @@ def main(): # pragma: no cover
|
||||||
setup_signal_handlers()
|
setup_signal_handlers()
|
||||||
archiver = Archiver()
|
archiver = Archiver()
|
||||||
msg = None
|
msg = None
|
||||||
args = archiver.get_args(sys.argv, os.environ.get('SSH_ORIGINAL_COMMAND'))
|
try:
|
||||||
|
args = archiver.get_args(sys.argv, os.environ.get('SSH_ORIGINAL_COMMAND'))
|
||||||
|
except Error as e:
|
||||||
|
msg = e.get_message()
|
||||||
|
if e.traceback:
|
||||||
|
msg += "\n%s\n%s" % (traceback.format_exc(), sysinfo())
|
||||||
|
# we might not have logging setup yet, so get out quickly
|
||||||
|
print(msg, file=sys.stderr)
|
||||||
|
sys.exit(e.exit_code)
|
||||||
try:
|
try:
|
||||||
exit_code = archiver.run(args)
|
exit_code = archiver.run(args)
|
||||||
except Error as e:
|
except Error as e:
|
||||||
|
|
|
@ -69,6 +69,10 @@ class NoManifestError(Error):
|
||||||
"""Repository has no manifest."""
|
"""Repository has no manifest."""
|
||||||
|
|
||||||
|
|
||||||
|
class PlaceholderError(Error):
|
||||||
|
"""Formatting Error: "{}".format({}): {}({})"""
|
||||||
|
|
||||||
|
|
||||||
def check_extension_modules():
|
def check_extension_modules():
|
||||||
from . import platform
|
from . import platform
|
||||||
if hashindex.API_VERSION != 2:
|
if hashindex.API_VERSION != 2:
|
||||||
|
@ -514,6 +518,10 @@ def CompressionSpec(s):
|
||||||
raise ValueError
|
raise ValueError
|
||||||
|
|
||||||
|
|
||||||
|
def PrefixSpec(s):
|
||||||
|
return replace_placeholders(s)
|
||||||
|
|
||||||
|
|
||||||
def dir_is_cachedir(path):
|
def dir_is_cachedir(path):
|
||||||
"""Determines whether the specified path is a cache directory (and
|
"""Determines whether the specified path is a cache directory (and
|
||||||
therefore should potentially be excluded from the backup) according to
|
therefore should potentially be excluded from the backup) according to
|
||||||
|
@ -552,18 +560,24 @@ def dir_is_tagged(path, exclude_caches, exclude_if_present):
|
||||||
|
|
||||||
|
|
||||||
def format_line(format, data):
|
def format_line(format, data):
|
||||||
# TODO: Filter out unwanted properties of str.format(), because "format" is user provided.
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
return format.format(**data)
|
return format.format(**data)
|
||||||
except (KeyError, ValueError) as e:
|
|
||||||
# this should catch format errors
|
|
||||||
print('Error in lineformat: "{}" - reason "{}"'.format(format, str(e)))
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
# something unexpected, print error and raise exception
|
raise PlaceholderError(format, data, e.__class__.__name__, str(e))
|
||||||
print('Error in lineformat: "{}" - reason "{}"'.format(format, str(e)))
|
|
||||||
raise
|
|
||||||
return ''
|
def replace_placeholders(text):
|
||||||
|
"""Replace placeholders in text with their values."""
|
||||||
|
current_time = datetime.now()
|
||||||
|
data = {
|
||||||
|
'pid': os.getpid(),
|
||||||
|
'fqdn': socket.getfqdn(),
|
||||||
|
'hostname': socket.gethostname(),
|
||||||
|
'now': current_time.now(),
|
||||||
|
'utcnow': current_time.utcnow(),
|
||||||
|
'user': uid2user(os.getuid(), os.getuid())
|
||||||
|
}
|
||||||
|
return format_line(text, data)
|
||||||
|
|
||||||
|
|
||||||
def safe_timestamp(item_timestamp_ns):
|
def safe_timestamp(item_timestamp_ns):
|
||||||
|
@ -720,21 +734,8 @@ class Location:
|
||||||
if not self.parse(self.orig):
|
if not self.parse(self.orig):
|
||||||
raise ValueError
|
raise ValueError
|
||||||
|
|
||||||
def preformat_text(self, text):
|
|
||||||
"""Format repository and archive path with common tags"""
|
|
||||||
current_time = datetime.now()
|
|
||||||
data = {
|
|
||||||
'pid': os.getpid(),
|
|
||||||
'fqdn': socket.getfqdn(),
|
|
||||||
'hostname': socket.gethostname(),
|
|
||||||
'now': current_time.now(),
|
|
||||||
'utcnow': current_time.utcnow(),
|
|
||||||
'user': uid2user(os.getuid(), os.getuid())
|
|
||||||
}
|
|
||||||
return format_line(text, data)
|
|
||||||
|
|
||||||
def parse(self, text):
|
def parse(self, text):
|
||||||
text = self.preformat_text(text)
|
text = replace_placeholders(text)
|
||||||
valid = self._parse(text)
|
valid = self._parse(text)
|
||||||
if valid:
|
if valid:
|
||||||
return True
|
return True
|
||||||
|
|
|
@ -946,7 +946,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
|
||||||
self.cmd('create', test_archive, src_dir)
|
self.cmd('create', test_archive, src_dir)
|
||||||
output_1 = self.cmd('list', test_archive)
|
output_1 = self.cmd('list', test_archive)
|
||||||
output_2 = self.cmd('list', '--list-format', '{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}', test_archive)
|
output_2 = self.cmd('list', '--list-format', '{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}', test_archive)
|
||||||
output_3 = self.cmd('list', '--list-format', '{mtime:%s} {path}{NL}', test_archive)
|
output_3 = self.cmd('list', '--list-format', '{mtime:%s} {path}{NEWLINE}', test_archive)
|
||||||
self.assertEqual(output_1, output_2)
|
self.assertEqual(output_1, output_2)
|
||||||
self.assertNotEqual(output_1, output_3)
|
self.assertNotEqual(output_1, output_3)
|
||||||
|
|
||||||
|
|
|
@ -10,7 +10,7 @@ import msgpack
|
||||||
import msgpack.fallback
|
import msgpack.fallback
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from ..helpers import Location, format_file_size, format_timedelta, make_path_safe, \
|
from ..helpers import Location, format_file_size, format_timedelta, format_line, PlaceholderError, make_path_safe, \
|
||||||
prune_within, prune_split, get_cache_dir, get_keys_dir, Statistics, is_slow_msgpack, \
|
prune_within, prune_split, get_cache_dir, get_keys_dir, Statistics, is_slow_msgpack, \
|
||||||
yes, TRUISH, FALSISH, DEFAULTISH, \
|
yes, TRUISH, FALSISH, DEFAULTISH, \
|
||||||
StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \
|
StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \
|
||||||
|
@ -877,3 +877,18 @@ def test_progress_endless_step(capfd):
|
||||||
pi.show()
|
pi.show()
|
||||||
out, err = capfd.readouterr()
|
out, err = capfd.readouterr()
|
||||||
assert err == '.'
|
assert err == '.'
|
||||||
|
|
||||||
|
|
||||||
|
def test_format_line():
|
||||||
|
data = dict(foo='bar baz')
|
||||||
|
assert format_line('', data) == ''
|
||||||
|
assert format_line('{foo}', data) == 'bar baz'
|
||||||
|
assert format_line('foo{foo}foo', data) == 'foobar bazfoo'
|
||||||
|
|
||||||
|
|
||||||
|
def test_format_line_erroneous():
|
||||||
|
data = dict()
|
||||||
|
with pytest.raises(PlaceholderError):
|
||||||
|
assert format_line('{invalid}', data)
|
||||||
|
with pytest.raises(PlaceholderError):
|
||||||
|
assert format_line('{}', data)
|
||||||
|
|
|
@ -110,7 +110,7 @@ certain number of old archives::
|
||||||
# Backup all of /home and /var/www except a few
|
# Backup all of /home and /var/www except a few
|
||||||
# excluded directories
|
# excluded directories
|
||||||
borg create -v --stats \
|
borg create -v --stats \
|
||||||
$REPOSITORY::`hostname`-`date +%Y-%m-%d` \
|
$REPOSITORY::'{hostname}-{now:%Y-%m-%d}' \
|
||||||
/home \
|
/home \
|
||||||
/var/www \
|
/var/www \
|
||||||
--exclude '/home/*/.cache' \
|
--exclude '/home/*/.cache' \
|
||||||
|
@ -118,10 +118,10 @@ certain number of old archives::
|
||||||
--exclude '*.pyc'
|
--exclude '*.pyc'
|
||||||
|
|
||||||
# Use the `prune` subcommand to maintain 7 daily, 4 weekly and 6 monthly
|
# Use the `prune` subcommand to maintain 7 daily, 4 weekly and 6 monthly
|
||||||
# archives of THIS machine. --prefix `hostname`- is very important to
|
# archives of THIS machine. The '{hostname}-' prefix is very important to
|
||||||
# limit prune's operation to this machine's archives and not apply to
|
# limit prune's operation to this machine's archives and not apply to
|
||||||
# other machine's archives also.
|
# other machine's archives also.
|
||||||
borg prune -v $REPOSITORY --prefix `hostname`- \
|
borg prune -v $REPOSITORY --prefix '{hostname}-' \
|
||||||
--keep-daily=7 --keep-weekly=4 --keep-monthly=6
|
--keep-daily=7 --keep-weekly=4 --keep-monthly=6
|
||||||
|
|
||||||
.. backup_compression:
|
.. backup_compression:
|
||||||
|
|
|
@ -425,8 +425,9 @@ will see what it would do without it actually doing anything.
|
||||||
# Do a dry-run without actually deleting anything.
|
# Do a dry-run without actually deleting anything.
|
||||||
$ borg prune --dry-run --keep-daily=7 --keep-weekly=4 /path/to/repo
|
$ borg prune --dry-run --keep-daily=7 --keep-weekly=4 /path/to/repo
|
||||||
|
|
||||||
# Same as above but only apply to archive names starting with "foo":
|
# Same as above but only apply to archive names starting with the hostname
|
||||||
$ borg prune --keep-daily=7 --keep-weekly=4 --prefix=foo /path/to/repo
|
# of the machine followed by a "-" character:
|
||||||
|
$ borg prune --keep-daily=7 --keep-weekly=4 --prefix='{hostname}-' /path/to/repo
|
||||||
|
|
||||||
# Keep 7 end of day, 4 additional end of week archives,
|
# Keep 7 end of day, 4 additional end of week archives,
|
||||||
# and an end of month archive for every month:
|
# and an end of month archive for every month:
|
||||||
|
|
Loading…
Add table
Reference in a new issue