mirror/borg
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2025-03-15 00:21:56 +00:00
Code Issues Releases Wiki Activity

Merge pull request #1194 from ThomasWaldmann/more-placeholders

Browse source 
add placeholder support at missing places, add missing help
This commit is contained in:
enkore 2016-06-23 17:38:04 +02:00 • committed by GitHub
commit 5b34483310
6 changed files with 117 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

98
borg/archiver.py
View file

@ -20,7 +20,7 @@ from .helpers import Error, location_validator, archivename_validator, format_li
parse_pattern, PathPrefixPattern, to_localtime, timestamp, safe_timestamp, \
get_cache_dir, prune_within, prune_split, \
Manifest, NoManifestError, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \
dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, is_slow_msgpack, yes, sysinfo, \
dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, PrefixSpec, is_slow_msgpack, yes, sysinfo, \
EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, log_multi, PatternMatcher
from .logger import create_logger, setup_logging
logger = create_logger()
@ -773,35 +773,68 @@ class Archiver:
whitespace removal paths with whitespace at the beginning or end can only be
excluded using regular expressions.
Examples:
Examples::
# Exclude '/home/user/file.o' but not '/home/user/file.odt':
$ borg create -e '*.o' backup /
# Exclude '/home/user/file.o' but not '/home/user/file.odt':
$ borg create -e '*.o' backup /
# Exclude '/home/user/junk' and '/home/user/subdir/junk' but
# not '/home/user/importantjunk' or '/etc/junk':
$ borg create -e '/home/*/junk' backup /
# Exclude '/home/user/junk' and '/home/user/subdir/junk' but
# not '/home/user/importantjunk' or '/etc/junk':
$ borg create -e '/home/*/junk' backup /
# Exclude the contents of '/home/user/cache' but not the directory itself:
$ borg create -e /home/user/cache/ backup /
# Exclude the contents of '/home/user/cache' but not the directory itself:
$ borg create -e /home/user/cache/ backup /
# The file '/home/user/cache/important' is *not* backed up:
$ borg create -e /home/user/cache/ backup / /home/user/cache/important
# The file '/home/user/cache/important' is *not* backed up:
$ borg create -e /home/user/cache/ backup / /home/user/cache/important
# The contents of directories in '/home' are not backed up when their name
# ends in '.tmp'
$ borg create --exclude 're:^/home/[^/]+\.tmp/' backup /
# The contents of directories in '/home' are not backed up when their name
# ends in '.tmp'
$ borg create --exclude 're:^/home/[^/]+\.tmp/' backup /
# Load exclusions from file
$ cat >exclude.txt <<EOF
# Comment line
/home/*/junk
*.tmp
fm:aa:something/*
re:^/home/[^/]\.tmp/
sh:/home/*/.thumbnails
EOF
$ borg create --exclude-from exclude.txt backup /
# Load exclusions from file
$ cat >exclude.txt <<EOF
# Comment line
/home/*/junk
*.tmp
fm:aa:something/*
re:^/home/[^/]\.tmp/
sh:/home/*/.thumbnails
EOF
$ borg create --exclude-from exclude.txt backup /
''')
helptext['placeholders'] = textwrap.dedent('''
Repository (or Archive) URLs and --prefix values support these placeholders:
{hostname}
The (short) hostname of the machine.
{fqdn}
The full name of the machine.
{now}
The current local date and time.
{utcnow}
The current UTC date and time.
{user}
The user name (or UID, if no name is available) of the user running borg.
{pid}
The current process ID.
Examples::
borg create /path/to/repo::{hostname}-{user}-{utcnow} ...
borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S} ...
borg prune --prefix '{hostname}-' ...
''')
def do_help(self, parser, commands, args):
@ -952,7 +985,7 @@ class Archiver:
subparser.add_argument('--last', dest='last',
type=int, default=None, metavar='N',
help='only check last N archives (Default: all)')
subparser.add_argument('-P', '--prefix', dest='prefix', type=str,
subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec,
help='only consider archive names starting with this prefix')
change_passphrase_epilog = textwrap.dedent("""
@ -1013,6 +1046,7 @@ class Archiver:
all files on these file systems.
See the output of the "borg help patterns" command for more help on exclude patterns.
See the output of the "borg help placeholders" command for more help on placeholders.
""")
subparser = subparsers.add_parser('create', parents=[common_parser],
@ -1194,7 +1228,7 @@ class Archiver:
help="""specify format for archive file listing
(default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}")
Special "{formatkeys}" exists to list available keys""")
subparser.add_argument('-P', '--prefix', dest='prefix', type=str,
subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec,
help='only consider archive names starting with this prefix')
subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', nargs='?', default='',
type=location_validator(),
@ -1301,7 +1335,7 @@ class Archiver:
help='number of monthly archives to keep')
subparser.add_argument('-y', '--keep-yearly', dest='yearly', type=int, default=0,
help='number of yearly archives to keep')
subparser.add_argument('-P', '--prefix', dest='prefix', type=str,
subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec,
help='only consider archive names starting with this prefix')
subparser.add_argument('--save-space', dest='save_space', action='store_true',
default=False,
@ -1522,7 +1556,15 @@ def main(): # pragma: no cover
setup_signal_handlers()
archiver = Archiver()
msg = None
args = archiver.get_args(sys.argv, os.environ.get('SSH_ORIGINAL_COMMAND'))
try:
args = archiver.get_args(sys.argv, os.environ.get('SSH_ORIGINAL_COMMAND'))
except Error as e:
msg = e.get_message()
if e.traceback:
msg += "\n%s\n%s" % (traceback.format_exc(), sysinfo())
# we might not have logging setup yet, so get out quickly
print(msg, file=sys.stderr)
sys.exit(e.exit_code)
try:
exit_code = archiver.run(args)
except Error as e:

47
borg/helpers.py
View file

@ -69,6 +69,10 @@ class NoManifestError(Error):
"""Repository has no manifest."""
class PlaceholderError(Error):
"""Formatting Error: "{}".format({}): {}({})"""
def check_extension_modules():
from . import platform
if hashindex.API_VERSION != 2:
@ -514,6 +518,10 @@ def CompressionSpec(s):
raise ValueError
def PrefixSpec(s):
return replace_placeholders(s)
def dir_is_cachedir(path):
"""Determines whether the specified path is a cache directory (and
therefore should potentially be excluded from the backup) according to
@ -552,18 +560,24 @@ def dir_is_tagged(path, exclude_caches, exclude_if_present):
def format_line(format, data):
# TODO: Filter out unwanted properties of str.format(), because "format" is user provided.
try:
return format.format(**data)
except (KeyError, ValueError) as e:
# this should catch format errors
print('Error in lineformat: "{}" - reason "{}"'.format(format, str(e)))
except Exception as e:
# something unexpected, print error and raise exception
print('Error in lineformat: "{}" - reason "{}"'.format(format, str(e)))
raise
return ''
raise PlaceholderError(format, data, e.__class__.__name__, str(e))
def replace_placeholders(text):
"""Replace placeholders in text with their values."""
current_time = datetime.now()
data = {
'pid': os.getpid(),
'fqdn': socket.getfqdn(),
'hostname': socket.gethostname(),
'now': current_time.now(),
'utcnow': current_time.utcnow(),
'user': uid2user(os.getuid(), os.getuid())
}
return format_line(text, data)
def safe_timestamp(item_timestamp_ns):
@ -720,21 +734,8 @@ class Location:
if not self.parse(self.orig):
raise ValueError
def preformat_text(self, text):
"""Format repository and archive path with common tags"""
current_time = datetime.now()
data = {
'pid': os.getpid(),
'fqdn': socket.getfqdn(),
'hostname': socket.gethostname(),
'now': current_time.now(),
'utcnow': current_time.utcnow(),
'user': uid2user(os.getuid(), os.getuid())
}
return format_line(text, data)
def parse(self, text):
text = self.preformat_text(text)
text = replace_placeholders(text)
valid = self._parse(text)
if valid:
return True

2
borg/testsuite/archiver.py
View file

@ -946,7 +946,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
self.cmd('create', test_archive, src_dir)
output_1 = self.cmd('list', test_archive)
output_2 = self.cmd('list', '--list-format', '{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}', test_archive)
output_3 = self.cmd('list', '--list-format', '{mtime:%s} {path}{NL}', test_archive)
output_3 = self.cmd('list', '--list-format', '{mtime:%s} {path}{NEWLINE}', test_archive)
self.assertEqual(output_1, output_2)
self.assertNotEqual(output_1, output_3)

17
borg/testsuite/helpers.py
View file

@ -10,7 +10,7 @@ import msgpack
import msgpack.fallback
import time
from ..helpers import Location, format_file_size, format_timedelta, make_path_safe, \
from ..helpers import Location, format_file_size, format_timedelta, format_line, PlaceholderError, make_path_safe, \
prune_within, prune_split, get_cache_dir, get_keys_dir, Statistics, is_slow_msgpack, \
yes, TRUISH, FALSISH, DEFAULTISH, \
StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \
@ -877,3 +877,18 @@ def test_progress_endless_step(capfd):
pi.show()
out, err = capfd.readouterr()
assert err == '.'
def test_format_line():
data = dict(foo='bar baz')
assert format_line('', data) == ''
assert format_line('{foo}', data) == 'bar baz'
assert format_line('foo{foo}foo', data) == 'foobar bazfoo'
def test_format_line_erroneous():
data = dict()
with pytest.raises(PlaceholderError):
assert format_line('{invalid}', data)
with pytest.raises(PlaceholderError):
assert format_line('{}', data)

6
docs/quickstart.rst
View file

@ -110,7 +110,7 @@ certain number of old archives::
# Backup all of /home and /var/www except a few
# excluded directories
borg create -v --stats \
$REPOSITORY::`hostname`-`date +%Y-%m-%d` \
$REPOSITORY::'{hostname}-{now:%Y-%m-%d}' \
/home \
/var/www \
--exclude '/home/*/.cache' \
@ -118,10 +118,10 @@ certain number of old archives::
--exclude '*.pyc'
# Use the `prune` subcommand to maintain 7 daily, 4 weekly and 6 monthly
# archives of THIS machine. --prefix `hostname`- is very important to
# archives of THIS machine. The '{hostname}-' prefix is very important to
# limit prune's operation to this machine's archives and not apply to
# other machine's archives also.
borg prune -v $REPOSITORY --prefix `hostname`- \
borg prune -v $REPOSITORY --prefix '{hostname}-' \
--keep-daily=7 --keep-weekly=4 --keep-monthly=6
.. backup_compression:

5
docs/usage.rst
View file

@ -425,8 +425,9 @@ will see what it would do without it actually doing anything.
# Do a dry-run without actually deleting anything.
$ borg prune --dry-run --keep-daily=7 --keep-weekly=4 /path/to/repo
# Same as above but only apply to archive names starting with "foo":
$ borg prune --keep-daily=7 --keep-weekly=4 --prefix=foo /path/to/repo
# Same as above but only apply to archive names starting with the hostname
# of the machine followed by a "-" character:
$ borg prune --keep-daily=7 --keep-weekly=4 --prefix='{hostname}-' /path/to/repo
# Keep 7 end of day, 4 additional end of week archives,
# and an end of month archive for every month: