From 72abd12e3cc5b4ec3516b1f7d73d431af75909fd Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 27 Mar 2017 01:35:32 +0200 Subject: [PATCH 1/7] update CHANGES with release date --- docs/changes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index beca3a62f..4548856ab 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -128,8 +128,8 @@ The best check that everything is ok is to run a dry-run extraction:: Changelog ========= -Version 1.1.0b4 (not released yet) ----------------------------------- +Version 1.1.0b4 (2017-03-27) +---------------------------- Compatibility notes: From 5bc17148e127ae21e2591725422fbd04ba80f6ae Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 27 Mar 2017 01:45:04 +0200 Subject: [PATCH 2/7] patterns help: mention path full-match in intro --- src/borg/archiver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 15c6dd9b8..f1bfc67f9 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1561,8 +1561,8 @@ class Archiver: helptext = collections.OrderedDict() helptext['patterns'] = textwrap.dedent(''' - File patterns support four separate styles: fnmatch, shell, regular - expressions and path prefixes. By default, fnmatch is used for + File patterns support these styles: fnmatch, shell, regular expressions, + path prefixes and path full-matches. By default, fnmatch is used for `--exclude` patterns and shell-style is used for `--pattern`. If followed by a colon (':') the first two characters of a pattern are used as a style selector. Explicit style selection is necessary when a From 251983a7411f2da8d4315a26e6b8071c907345ff Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 27 Mar 2017 01:45:45 +0200 Subject: [PATCH 3/7] ran setup.py build_usage --- docs/usage/break-lock.rst.inc | 2 +- docs/usage/change-passphrase.rst.inc | 2 +- docs/usage/check.rst.inc | 2 +- docs/usage/common-options.rst.inc | 4 +- docs/usage/create.rst.inc | 33 ++++++++-- docs/usage/delete.rst.inc | 4 +- docs/usage/diff.rst.inc | 22 +++++-- docs/usage/extract.rst.inc | 6 +- docs/usage/help.rst.inc | 76 +++++++++++++++++++---- docs/usage/info.rst.inc | 6 +- docs/usage/init.rst.inc | 2 +- docs/usage/key_change-passphrase.rst.inc | 2 +- docs/usage/key_export.rst.inc | 4 +- docs/usage/key_import.rst.inc | 2 +- docs/usage/key_migrate-to-repokey.rst.inc | 2 +- docs/usage/list.rst.inc | 28 +++++++-- docs/usage/mount.rst.inc | 2 +- docs/usage/prune.rst.inc | 2 +- docs/usage/recreate.rst.inc | 16 +++-- docs/usage/rename.rst.inc | 2 +- docs/usage/serve.rst.inc | 2 +- docs/usage/umount.rst.inc | 2 +- docs/usage/with-lock.rst.inc | 2 +- 23 files changed, 173 insertions(+), 52 deletions(-) diff --git a/docs/usage/break-lock.rst.inc b/docs/usage/break-lock.rst.inc index 5fa1cda55..756b0d391 100644 --- a/docs/usage/break-lock.rst.inc +++ b/docs/usage/break-lock.rst.inc @@ -20,4 +20,4 @@ Description This command breaks the repository and cache locks. Please use carefully and only while no borg process (on any machine) is -trying to access the Cache or the Repository. +trying to access the Cache or the Repository. \ No newline at end of file diff --git a/docs/usage/change-passphrase.rst.inc b/docs/usage/change-passphrase.rst.inc index 3bb827a4f..92a82c0db 100644 --- a/docs/usage/change-passphrase.rst.inc +++ b/docs/usage/change-passphrase.rst.inc @@ -19,4 +19,4 @@ Description ~~~~~~~~~~~ The key files used for repository encryption are optionally passphrase -protected. This command can be used to change this passphrase. +protected. This command can be used to change this passphrase. \ No newline at end of file diff --git a/docs/usage/check.rst.inc b/docs/usage/check.rst.inc index 7705471b6..9958603ef 100644 --- a/docs/usage/check.rst.inc +++ b/docs/usage/check.rst.inc @@ -86,4 +86,4 @@ repository, decrypting and decompressing it. This is a cryptographic verificatio which will detect (accidental) corruption. For encrypted repositories it is tamper-resistant as well, unless the attacker has access to the keys. -It is also very slow. +It is also very slow. \ No newline at end of file diff --git a/docs/usage/common-options.rst.inc b/docs/usage/common-options.rst.inc index fb235b52a..5dadfd52c 100644 --- a/docs/usage/common-options.rst.inc +++ b/docs/usage/common-options.rst.inc @@ -12,6 +12,8 @@ | enable debug output, work on log level DEBUG ``--debug-topic TOPIC`` | enable TOPIC debugging (can be specified multiple times). The logger path is borg.debug. if TOPIC is not fully qualified. + ``--log-json`` + | Output one JSON object per log line instead of formatted text. ``--lock-wait N`` | wait for the lock, but max. N seconds (default: 1). ``--show-version`` @@ -23,7 +25,7 @@ ``--umask M`` | set umask to M (local and remote, default: 0077) ``--remote-path PATH`` - | set remote path to executable (default: "borg") + | use PATH as borg executable on the remote (default: "borg") ``--remote-ratelimit rate`` | set remote network upload rate limit in kiByte/s (default: 0=unlimited) ``--consider-part-files`` diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc index 6b34278d1..97e053066 100644 --- a/docs/usage/create.rst.inc +++ b/docs/usage/create.rst.inc @@ -25,6 +25,8 @@ optional arguments | output verbose list of items (files, dirs, ...) ``--filter STATUSCHARS`` | only display items with the given status characters + ``--json`` + | output stats as JSON (implies --stats) `Common options`_ | @@ -39,11 +41,15 @@ Exclusion options ``--exclude-if-present NAME`` | exclude directories that are tagged by containing a filesystem object with the given NAME ``--keep-exclude-tags``, ``--keep-tag-files`` - | keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise excluded caches/directories + | if tag objects are specified with --exclude-if-present, don't omit the tag objects themselves from the backup archive + ``--pattern PATTERN`` + | include/exclude paths matching PATTERN + ``--patterns-from PATTERNFILE`` + | read include/exclude patterns from PATTERNFILE, one per line Filesystem options ``-x``, ``--one-file-system`` - | stay in same file system, do not cross mount points + | stay in the same file system and do not store mount points of other file systems ``--numeric-owner`` | only store numeric user and group identifiers ``--noatime`` @@ -73,9 +79,12 @@ Description ~~~~~~~~~~~ This command creates a backup archive containing all files found while recursively -traversing all paths specified. When giving '-' as path, borg will read data -from standard input and create a file 'stdin' in the created archive from that -data. +traversing all paths specified. Paths are added to the archive as they are given, +that means if relative paths are desired, the command has to be run from the correct +directory. + +When giving '-' as path, borg will read data from standard input and create a +file 'stdin' in the created archive from that data. The archive will consume almost no disk space for files or parts of files that have already been stored in other archives. @@ -92,6 +101,11 @@ not provide correct inode information the --ignore-inode flag can be used. This potentially decreases reliability of change detection, while avoiding always reading all files on these file systems. +The mount points of filesystems or filesystem snapshots should be the same for every +creation of a new archive to ensure fast operation. This is because the file cache that +is used to determine changed files quickly uses absolute filenames. +If this is not possible, consider creating a bind mount to a stable location. + See the output of the "borg help patterns" command for more help on exclude patterns. See the output of the "borg help placeholders" command for more help on placeholders. @@ -102,6 +116,15 @@ exclude foo/.bundler/gems. In borg it will not, you need to use --exclude '\*/.bundler/gems' to get the same effect. See ``borg help patterns`` for more information. +In addition to using ``--exclude`` patterns, it is possible to use +``--exclude-if-present`` to specify the name of a filesystem object (e.g. a file +or folder name) which, when contained within another folder, will prevent the +containing folder from being backed up. By default, the containing folder and +all of its contents will be omitted from the backup. If, however, you wish to +only include the objects specified by ``--exclude-if-present`` in your backup, +and not include any other contents of the containing folder, this can be enabled +through using the ``--keep-exclude-tags`` option. + Item flags ++++++++++ diff --git a/docs/usage/delete.rst.inc b/docs/usage/delete.rst.inc index 2a685e885..5c1361a27 100644 --- a/docs/usage/delete.rst.inc +++ b/docs/usage/delete.rst.inc @@ -20,7 +20,7 @@ optional arguments ``-c``, ``--cache-only`` | delete only the local cache for the given repository ``--force`` - | force deletion of corrupted archives + | force deletion of corrupted archives, use --force --force in case --force does not work. ``--save-space`` | work slower, but using less space @@ -42,4 +42,4 @@ Description This command deletes an archive from the repository or the complete repository. Disk space is reclaimed accordingly. If you delete the complete repository, the -local cache for it (if any) is also deleted. +local cache for it (if any) is also deleted. \ No newline at end of file diff --git a/docs/usage/diff.rst.inc b/docs/usage/diff.rst.inc index 1c245cf2a..65d5afe67 100644 --- a/docs/usage/diff.rst.inc +++ b/docs/usage/diff.rst.inc @@ -17,10 +17,6 @@ positional arguments paths of items inside the archives to compare; patterns are supported optional arguments - ``-e PATTERN``, ``--exclude PATTERN`` - | exclude paths matching PATTERN - ``--exclude-from EXCLUDEFILE`` - | read exclude patterns from EXCLUDEFILE, one per line ``--numeric-owner`` | only consider numeric user and group identifiers ``--same-chunker-params`` @@ -31,6 +27,22 @@ optional arguments `Common options`_ | +Exclusion options + ``-e PATTERN``, ``--exclude PATTERN`` + | exclude paths matching PATTERN + ``--exclude-from EXCLUDEFILE`` + | read exclude patterns from EXCLUDEFILE, one per line + ``--exclude-caches`` + | exclude directories that contain a CACHEDIR.TAG file (http://www.brynosaurus.com/cachedir/spec.html) + ``--exclude-if-present NAME`` + | exclude directories that are tagged by containing a filesystem object with the given NAME + ``--keep-exclude-tags``, ``--keep-tag-files`` + | if tag objects are specified with --exclude-if-present, don't omit the tag objects themselves from the backup archive + ``--pattern PATTERN`` + | include/exclude paths matching PATTERN + ``--patterns-from PATTERNFILE`` + | read include/exclude patterns from PATTERNFILE, one per line + Description ~~~~~~~~~~~ @@ -49,4 +61,4 @@ If you did not create the archives with different chunker params, pass --same-chunker-params. Note that the chunker params changed from Borg 0.xx to 1.0. -See the output of the "borg help patterns" command for more help on exclude patterns. +See the output of the "borg help patterns" command for more help on exclude patterns. \ No newline at end of file diff --git a/docs/usage/extract.rst.inc b/docs/usage/extract.rst.inc index 682eaa3a9..704c7c647 100644 --- a/docs/usage/extract.rst.inc +++ b/docs/usage/extract.rst.inc @@ -25,6 +25,10 @@ optional arguments | exclude paths matching PATTERN ``--exclude-from EXCLUDEFILE`` | read exclude patterns from EXCLUDEFILE, one per line + ``--pattern PATTERN`` + | include/exclude paths matching PATTERN + ``--patterns-from PATTERNFILE`` + | read include/exclude patterns from PATTERNFILE, one per line ``--numeric-owner`` | only obey numeric user and group identifiers ``--strip-components NUMBER`` @@ -49,4 +53,4 @@ See the output of the "borg help patterns" command for more help on exclude patt By using ``--dry-run``, you can do all extraction steps except actually writing the output data: reading metadata and data chunks from the repo, checking the hash/hmac, -decrypting, decompressing. +decrypting, decompressing. \ No newline at end of file diff --git a/docs/usage/help.rst.inc b/docs/usage/help.rst.inc index a4a11c4fa..dc2072d61 100644 --- a/docs/usage/help.rst.inc +++ b/docs/usage/help.rst.inc @@ -6,8 +6,9 @@ borg help patterns ~~~~~~~~~~~~~~~~~~ -Exclusion patterns support four separate styles, fnmatch, shell, regular -expressions and path prefixes. By default, fnmatch is used. If followed +File patterns support these styles: fnmatch, shell, regular expressions, +path prefixes and path full-matches. By default, fnmatch is used for +`--exclude` patterns and shell-style is used for `--pattern`. If followed by a colon (':') the first two characters of a pattern are used as a style selector. Explicit style selection is necessary when a non-default style is desired or when the desired pattern starts with @@ -15,12 +16,12 @@ two alphanumeric characters followed by a colon (i.e. `aa:something/*`). `Fnmatch `_, selector `fm:` - This is the default style. These patterns use a variant of shell - pattern syntax, with '*' matching any number of characters, '?' - matching any single character, '[...]' matching any single - character specified, including ranges, and '[!...]' matching any - character not specified. For the purpose of these patterns, the - path separator ('\' for Windows and '/' on other systems) is not + This is the default style for --exclude and --exclude-from. + These patterns use a variant of shell pattern syntax, with '*' matching + any number of characters, '?' matching any single character, '[...]' + matching any single character specified, including ranges, and '[!...]' + matching any character not specified. For the purpose of these patterns, + the path separator ('\' for Windows and '/' on other systems) is not treated specially. Wrap meta-characters in brackets for a literal match (i.e. `[?]` to match the literal character `?`). For a path to match a pattern, it must completely match from start to end, or @@ -31,6 +32,7 @@ two alphanumeric characters followed by a colon (i.e. `aa:something/*`). Shell-style patterns, selector `sh:` + This is the default style for --pattern and --patterns-from. Like fnmatch patterns these are similar to shell patterns. The difference is that the pattern may include `**/` for matching zero or more directory levels, `*` for matching zero or more arbitrary characters with the @@ -47,11 +49,27 @@ Regular expressions, selector `re:` regular expression syntax is described in the `Python documentation for the re module `_. -Prefix path, selector `pp:` +Path prefix, selector `pp:` This pattern style is useful to match whole sub-directories. The pattern `pp:/data/bar` matches `/data/bar` and everything therein. +Path full-match, selector `pf:` + + This pattern style is useful to match whole paths. + This is kind of a pseudo pattern as it can not have any variable or + unspecified parts - the full, precise path must be given. + `pf:/data/foo.txt` matches `/data/foo.txt` only. + + Implementation note: this is implemented via very time-efficient O(1) + hashtable lookups (this means you can have huge amounts of such patterns + without impacting performance much). + Due to that, this kind of pattern does not respect any context or order. + If you use such a pattern to include a file, it will always be included + (if the directory recursion encounters it). + Other include/exclude patterns that would normally match will be ignored. + Same logic applies for exclude. + Exclusions can be passed via the command line option `--exclude`. When used from within a shell the patterns should be quoted to protect them from expansion. @@ -93,6 +111,40 @@ Examples:: EOF $ borg create --exclude-from exclude.txt backup / + +A more general and easier to use way to define filename matching patterns exists +with the `--pattern` and `--patterns-from` options. Using these, you may specify +the backup roots (starting points) and patterns for inclusion/exclusion. A +root path starts with the prefix `R`, followed by a path (a plain path, not a +file pattern). An include rule starts with the prefix +, an exclude rule starts +with the prefix -, both followed by a pattern. +Inclusion patterns are useful to include pathes that are contained in an excluded +path. The first matching pattern is used so if an include pattern matches before +an exclude pattern, the file is backed up. + +Note that the default pattern style for `--pattern` and `--patterns-from` is +shell style (`sh:`), so those patterns behave similar to rsync include/exclude +patterns. The pattern style can be set via the `P` prefix. + +Patterns (`--pattern`) and excludes (`--exclude`) from the command line are +considered first (in the order of appearance). Then patterns from `--patterns-from` +are added. Exclusion patterns from `--exclude-from` files are appended last. + +An example `--patterns-from` file could look like that:: + + # "sh:" pattern style is the default, so the following line is not needed: + P sh + R / + # can be rebuild + - /home/*/.cache + # they're downloads for a reason + - /home/*/Downloads + # susan is a nice person + # include susans home + + /home/susan + # don't backup the other home directories + - /home/* + .. _borg_placeholders: borg help placeholders @@ -156,17 +208,17 @@ borg help compression ~~~~~~~~~~~~~~~~~~~~~ -Compression is off by default, if you want some, you have to specify what you want. +Compression is lz4 by default. If you want something else, you have to specify what you want. Valid compression specifiers are: none - Do not compress. (default) + Do not compress. lz4 - Use lz4 compression. High speed, low compression. + Use lz4 compression. High speed, low compression. (default) zlib[,L] diff --git a/docs/usage/info.rst.inc b/docs/usage/info.rst.inc index be61637eb..4926b6407 100644 --- a/docs/usage/info.rst.inc +++ b/docs/usage/info.rst.inc @@ -12,6 +12,10 @@ positional arguments REPOSITORY_OR_ARCHIVE archive or repository to display information about +optional arguments + ``--json`` + | format output as JSON + `Common options`_ | @@ -37,4 +41,4 @@ are meaning different things: This archive / deduplicated size = amount of data stored ONLY for this archive = unique chunks of this archive. All archives / deduplicated size = amount of data stored in the repo - = all chunks in the repository. + = all chunks in the repository. \ No newline at end of file diff --git a/docs/usage/init.rst.inc b/docs/usage/init.rst.inc index 9fb3264a8..9e858d22c 100644 --- a/docs/usage/init.rst.inc +++ b/docs/usage/init.rst.inc @@ -97,4 +97,4 @@ hardware-accelerated. BLAKE2b is faster than SHA256 on Intel/AMD 64bit CPUs, which makes `authenticated` faster than `none`. On modern ARM CPUs, NEON provides hardware acceleration for SHA256 making it faster -than BLAKE2b-256 there. +than BLAKE2b-256 there. \ No newline at end of file diff --git a/docs/usage/key_change-passphrase.rst.inc b/docs/usage/key_change-passphrase.rst.inc index cbc0d9e6d..c34fd002b 100644 --- a/docs/usage/key_change-passphrase.rst.inc +++ b/docs/usage/key_change-passphrase.rst.inc @@ -19,4 +19,4 @@ Description ~~~~~~~~~~~ The key files used for repository encryption are optionally passphrase -protected. This command can be used to change this passphrase. +protected. This command can be used to change this passphrase. \ No newline at end of file diff --git a/docs/usage/key_export.rst.inc b/docs/usage/key_export.rst.inc index f251d4307..3013b8d2f 100644 --- a/docs/usage/key_export.rst.inc +++ b/docs/usage/key_export.rst.inc @@ -17,6 +17,8 @@ positional arguments optional arguments ``--paper`` | Create an export suitable for printing and later type-in + ``--qr-html`` + | Create an html file suitable for printing and later type-in or qr scan `Common options`_ | @@ -40,4 +42,4 @@ data backup. For repositories using the repokey encryption the key is saved in the repository in the config file. A backup is thus not strictly needed, but guards against the repository becoming inaccessible if the file -is damaged for some reason. +is damaged for some reason. \ No newline at end of file diff --git a/docs/usage/key_import.rst.inc b/docs/usage/key_import.rst.inc index c8c2a0a85..c92bc9cd2 100644 --- a/docs/usage/key_import.rst.inc +++ b/docs/usage/key_import.rst.inc @@ -29,4 +29,4 @@ export command. If the ``--paper`` option is given, the import will be an interactive process in which each line is checked for plausibility before -proceeding to the next line. For this format PATH must not be given. +proceeding to the next line. For this format PATH must not be given. \ No newline at end of file diff --git a/docs/usage/key_migrate-to-repokey.rst.inc b/docs/usage/key_migrate-to-repokey.rst.inc index 0e82f28c2..66629bdbb 100644 --- a/docs/usage/key_migrate-to-repokey.rst.inc +++ b/docs/usage/key_migrate-to-repokey.rst.inc @@ -33,4 +33,4 @@ use, but you may also use a different one. After migrating to repokey mode, you can change the passphrase at any time. But please note: the secrets will always stay the same and they could always -be derived from your (old) passphrase-mode passphrase. +be derived from your (old) passphrase-mode passphrase. \ No newline at end of file diff --git a/docs/usage/list.rst.inc b/docs/usage/list.rst.inc index d3a4485f7..ee14a108c 100644 --- a/docs/usage/list.rst.inc +++ b/docs/usage/list.rst.inc @@ -20,10 +20,8 @@ optional arguments ``--format``, ``--list-format`` | specify format for file listing | (default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NL}") - ``-e PATTERN``, ``--exclude PATTERN`` - | exclude paths matching PATTERN - ``--exclude-from EXCLUDEFILE`` - | read exclude patterns from EXCLUDEFILE, one per line + ``--json`` + | format output as JSON. The form of --format is ignored, but keys used in it are added to the JSON output. Some keys are always present. Note: JSON can only represent text. A "bpath" key is therefore not available. `Common options`_ | @@ -38,6 +36,22 @@ filters ``--last N`` | consider last N archives after other filters were applied +Exclusion options + ``-e PATTERN``, ``--exclude PATTERN`` + | exclude paths matching PATTERN + ``--exclude-from EXCLUDEFILE`` + | read exclude patterns from EXCLUDEFILE, one per line + ``--exclude-caches`` + | exclude directories that contain a CACHEDIR.TAG file (http://www.brynosaurus.com/cachedir/spec.html) + ``--exclude-if-present NAME`` + | exclude directories that are tagged by containing a filesystem object with the given NAME + ``--keep-exclude-tags``, ``--keep-tag-files`` + | if tag objects are specified with --exclude-if-present, don't omit the tag objects themselves from the backup archive + ``--pattern PATTERN`` + | include/exclude paths matching PATTERN + ``--patterns-from PATTERNFILE`` + | read include/exclude patterns from PATTERNFILE, one per line + Description ~~~~~~~~~~~ @@ -45,6 +59,8 @@ This command lists the contents of a repository or an archive. See the "borg help patterns" command for more help on exclude patterns. +.. man NOTES + The following keys are available for --format: - NEWLINE: OS dependent line separator @@ -57,7 +73,7 @@ The following keys are available for --format: Keys for listing repository archives: - - archive: archive name interpreted as text (might be missing non-text characters, see barchive) + - archive, name: archive name interpreted as text (might be missing non-text characters, see barchive) - barchive: verbatim archive name, can contain any character except NUL - time: time of creation of the archive - id: internal ID of the archive @@ -78,6 +94,8 @@ Keys for listing archive files: - size - csize: compressed size + - dsize: deduplicated size + - dcsize: deduplicated compressed size - num_chunks: number of chunks in this file - unique_chunks: number of unique chunks in this file diff --git a/docs/usage/mount.rst.inc b/docs/usage/mount.rst.inc index e15f25afa..e7e60ce27 100644 --- a/docs/usage/mount.rst.inc +++ b/docs/usage/mount.rst.inc @@ -67,4 +67,4 @@ Unmounting in these cases could cause an active rsync or similar process to unintentionally delete data. When running in the foreground ^C/SIGINT unmounts cleanly, but other -signals or crashes do not. +signals or crashes do not. \ No newline at end of file diff --git a/docs/usage/prune.rst.inc b/docs/usage/prune.rst.inc index b6a0052c3..c09b9885d 100644 --- a/docs/usage/prune.rst.inc +++ b/docs/usage/prune.rst.inc @@ -87,4 +87,4 @@ negative number of archives to keep means that there is no limit. The "--keep-last N" option is doing the same as "--keep-secondly N" (and it will keep the last N archives under the assumption that you do not create more than one -backup archive in the same second). +backup archive in the same second). \ No newline at end of file diff --git a/docs/usage/recreate.rst.inc b/docs/usage/recreate.rst.inc index 93f7f414f..0e37696b3 100644 --- a/docs/usage/recreate.rst.inc +++ b/docs/usage/recreate.rst.inc @@ -37,9 +37,13 @@ Exclusion options ``--exclude-caches`` | exclude directories that contain a CACHEDIR.TAG file (http://www.brynosaurus.com/cachedir/spec.html) ``--exclude-if-present NAME`` - | exclude directories that are tagged by containing a filesystem object with the given NAME + | exclude directories that are tagged by containing a filesystem object with the given NAME ``--keep-exclude-tags``, ``--keep-tag-files`` - | keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise excluded caches/directories + | if tag objects are specified with --exclude-if-present, don't omit the tag objects themselves from the backup archive + ``--pattern PATTERN`` + | include/exclude paths matching PATTERN + ``--patterns-from PATTERNFILE`` + | read include/exclude patterns from PATTERNFILE, one per line Archive options ``--target TARGET`` @@ -66,9 +70,9 @@ Recreate the contents of existing archives. This is an *experimental* feature. Do *not* use this on your only backup. ---exclude, --exclude-from and PATH have the exact same semantics -as in "borg create". If PATHs are specified the resulting archive -will only contain files from these PATHs. +--exclude, --exclude-from, --exclude-if-present, --keep-exclude-tags, and PATH +have the exact same semantics as in "borg create". If PATHs are specified the +resulting archive will only contain files from these PATHs. Note that all paths in an archive are relative, therefore absolute patterns/paths will *not* match (--exclude, --exclude-from, --compression-from, PATHs). @@ -97,4 +101,4 @@ With --target the original archive is not replaced, instead a new archive is cre When rechunking space usage can be substantial, expect at least the entire deduplicated size of the archives using the previous chunker params. When recompressing expect approx. (throughput / checkpoint-interval) in space usage, -assuming all chunks are recompressed. +assuming all chunks are recompressed. \ No newline at end of file diff --git a/docs/usage/rename.rst.inc b/docs/usage/rename.rst.inc index 3cff5a8a4..6e53d2450 100644 --- a/docs/usage/rename.rst.inc +++ b/docs/usage/rename.rst.inc @@ -22,4 +22,4 @@ Description This command renames an archive in the repository. -This results in a different archive ID. +This results in a different archive ID. \ No newline at end of file diff --git a/docs/usage/serve.rst.inc b/docs/usage/serve.rst.inc index 351af5e42..628ff399f 100644 --- a/docs/usage/serve.rst.inc +++ b/docs/usage/serve.rst.inc @@ -20,4 +20,4 @@ optional arguments Description ~~~~~~~~~~~ -This command starts a repository server process. This command is usually not used manually. +This command starts a repository server process. This command is usually not used manually. \ No newline at end of file diff --git a/docs/usage/umount.rst.inc b/docs/usage/umount.rst.inc index 28c5f8f0f..f99c1d46a 100644 --- a/docs/usage/umount.rst.inc +++ b/docs/usage/umount.rst.inc @@ -21,4 +21,4 @@ Description This command un-mounts a FUSE filesystem that was mounted with ``borg mount``. This is a convenience wrapper that just calls the platform-specific shell -command - usually this is either umount or fusermount -u. +command - usually this is either umount or fusermount -u. \ No newline at end of file diff --git a/docs/usage/with-lock.rst.inc b/docs/usage/with-lock.rst.inc index c77eb2f60..407bda72b 100644 --- a/docs/usage/with-lock.rst.inc +++ b/docs/usage/with-lock.rst.inc @@ -31,4 +31,4 @@ code as borg's return code. Note: if you copy a repository with the lock held, the lock will be present in the copy, obviously. Thus, before using borg on the copy, you need to - use "borg break-lock" on it. + use "borg break-lock" on it. \ No newline at end of file From acd3da62f4a764721e1c5e2f40e8b2659ed4d989 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 27 Mar 2017 01:57:52 +0200 Subject: [PATCH 4/7] add docstring to do_benchmark_crud --- src/borg/archiver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index f1bfc67f9..e241002eb 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -325,6 +325,7 @@ class Archiver: return EXIT_SUCCESS def do_benchmark_crud(self, args): + """Benchmark Create, Read, Update, Delete for archives.""" def measurement_run(repo, path): archive = repo + '::borg-benchmark-crud' compression = '--compression=none' From 85bfcd439c3d8d3e674b013db8146343c902361d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 27 Mar 2017 01:58:19 +0200 Subject: [PATCH 5/7] ran setup.py build_man --- docs/man/borg-break-lock.1 | 2 +- docs/man/borg-change-passphrase.1 | 2 +- docs/man/borg-check.1 | 2 +- docs/man/borg-common.1 | 7 ++- docs/man/borg-compression.1 | 8 +-- docs/man/borg-create.1 | 45 +++++++++++--- docs/man/borg-delete.1 | 4 +- docs/man/borg-diff.1 | 32 +++++++--- docs/man/borg-extract.1 | 8 ++- docs/man/borg-info.1 | 8 ++- docs/man/borg-init.1 | 2 +- docs/man/borg-key-change-passphrase.1 | 2 +- docs/man/borg-key-export.1 | 5 +- docs/man/borg-key-import.1 | 2 +- docs/man/borg-key-migrate-to-repokey.1 | 2 +- docs/man/borg-key.1 | 2 +- docs/man/borg-list.1 | 55 +++++++++-------- docs/man/borg-mount.1 | 2 +- docs/man/borg-patterns.1 | 82 ++++++++++++++++++++++---- docs/man/borg-placeholders.1 | 2 +- docs/man/borg-prune.1 | 2 +- docs/man/borg-recreate.1 | 18 ++++-- docs/man/borg-rename.1 | 2 +- docs/man/borg-serve.1 | 2 +- docs/man/borg-umount.1 | 2 +- docs/man/borg-upgrade.1 | 2 +- docs/man/borg-with-lock.1 | 2 +- docs/man/borg.1 | 6 +- 28 files changed, 229 insertions(+), 81 deletions(-) diff --git a/docs/man/borg-break-lock.1 b/docs/man/borg-break-lock.1 index b8162436b..6f9f73e0d 100644 --- a/docs/man/borg-break-lock.1 +++ b/docs/man/borg-break-lock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BREAK-LOCK 1 "2017-02-11" "" "borg backup tool" +.TH BORG-BREAK-LOCK 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-break-lock \- Break the repository lock (e.g. in case it was left by a dead borg. . diff --git a/docs/man/borg-change-passphrase.1 b/docs/man/borg-change-passphrase.1 index b74dd336b..2c8f90c58 100644 --- a/docs/man/borg-change-passphrase.1 +++ b/docs/man/borg-change-passphrase.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CHANGE-PASSPHRASE 1 "2017-02-11" "" "borg backup tool" +.TH BORG-CHANGE-PASSPHRASE 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-change-passphrase \- Change repository key file passphrase . diff --git a/docs/man/borg-check.1 b/docs/man/borg-check.1 index 8a0ec36bf..3545a4d60 100644 --- a/docs/man/borg-check.1 +++ b/docs/man/borg-check.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CHECK 1 "2017-02-11" "" "borg backup tool" +.TH BORG-CHECK 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-check \- Check repository consistency . diff --git a/docs/man/borg-common.1 b/docs/man/borg-common.1 index e480c1f8d..a10838f4d 100644 --- a/docs/man/borg-common.1 +++ b/docs/man/borg-common.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-COMMON 1 "2017-02-11" "" "borg backup tool" +.TH BORG-COMMON 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-common \- Common options of Borg commands . @@ -54,6 +54,9 @@ enable debug output, work on log level DEBUG .BI \-\-debug\-topic \ TOPIC enable TOPIC debugging (can be specified multiple times). The logger path is borg.debug. if TOPIC is not fully qualified. .TP +.B \-\-log\-json +Output one JSON object per log line instead of formatted text. +.TP .BI \-\-lock\-wait \ N wait for the lock, but max. N seconds (default: 1). .TP @@ -70,7 +73,7 @@ do not load/update the file metadata cache used to detect unchanged files set umask to M (local and remote, default: 0077) .TP .BI \-\-remote\-path \ PATH -set remote path to executable (default: "borg") +use PATH as borg executable on the remote (default: "borg") .TP .BI \-\-remote\-ratelimit \ rate set remote network upload rate limit in kiByte/s (default: 0=unlimited) diff --git a/docs/man/borg-compression.1 b/docs/man/borg-compression.1 index 51124fea3..e0a98ed12 100644 --- a/docs/man/borg-compression.1 +++ b/docs/man/borg-compression.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-COMPRESSION 1 "2017-02-11" "" "borg backup tool" +.TH BORG-COMPRESSION 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-compression \- Details regarding compression . @@ -32,21 +32,21 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH DESCRIPTION .sp -Compression is off by default, if you want some, you have to specify what you want. +Compression is lz4 by default. If you want something else, you have to specify what you want. .sp Valid compression specifiers are: .sp none .INDENT 0.0 .INDENT 3.5 -Do not compress. (default) +Do not compress. .UNINDENT .UNINDENT .sp lz4 .INDENT 0.0 .INDENT 3.5 -Use lz4 compression. High speed, low compression. +Use lz4 compression. High speed, low compression. (default) .UNINDENT .UNINDENT .sp diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1 index d01d77e27..c0a7ab961 100644 --- a/docs/man/borg-create.1 +++ b/docs/man/borg-create.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CREATE 1 "2017-02-12" "" "borg backup tool" +.TH BORG-CREATE 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-create \- Create new archive . @@ -36,9 +36,12 @@ borg create ARCHIVE PATH .SH DESCRIPTION .sp This command creates a backup archive containing all files found while recursively -traversing all paths specified. When giving \(aq\-\(aq as path, borg will read data -from standard input and create a file \(aqstdin\(aq in the created archive from that -data. +traversing all paths specified. Paths are added to the archive as they are given, +that means if relative paths are desired, the command has to be run from the correct +directory. +.sp +When giving \(aq\-\(aq as path, borg will read data from standard input and create a +file \(aqstdin\(aq in the created archive from that data. .sp The archive will consume almost no disk space for files or parts of files that have already been stored in other archives. @@ -55,6 +58,11 @@ not provide correct inode information the \-\-ignore\-inode flag can be used. Th potentially decreases reliability of change detection, while avoiding always reading all files on these file systems. .sp +The mount points of filesystems or filesystem snapshots should be the same for every +creation of a new archive to ensure fast operation. This is because the file cache that +is used to determine changed files quickly uses absolute filenames. +If this is not possible, consider creating a bind mount to a stable location. +.sp See the output of the "borg help patterns" command for more help on exclude patterns. See the output of the "borg help placeholders" command for more help on placeholders. .SH OPTIONS @@ -86,6 +94,9 @@ output verbose list of items (files, dirs, ...) .TP .BI \-\-filter \ STATUSCHARS only display items with the given status characters +.TP +.B \-\-json +output stats as JSON (implies \-\-stats) .UNINDENT .SS Exclusion options .INDENT 0.0 @@ -103,13 +114,19 @@ exclude directories that contain a CACHEDIR.TAG file (\fI\%http://www.brynosauru exclude directories that are tagged by containing a filesystem object with the given NAME .TP .B \-\-keep\-exclude\-tags\fP,\fB \-\-keep\-tag\-files -keep tag objects (i.e.: arguments to \-\-exclude\-if\-present) in otherwise excluded caches/directories +if tag objects are specified with \-\-exclude\-if\-present, don\(aqt omit the tag objects themselves from the backup archive +.TP +.BI \-\-pattern \ PATTERN +include/exclude paths matching PATTERN +.TP +.BI \-\-patterns\-from \ PATTERNFILE +read include/exclude patterns from PATTERNFILE, one per line .UNINDENT .SS Filesystem options .INDENT 0.0 .TP .B \-x\fP,\fB \-\-one\-file\-system -stay in same file system, do not cross mount points +stay in the same file system and do not store mount points of other file systems .TP .B \-\-numeric\-owner only store numeric user and group identifiers @@ -175,7 +192,7 @@ $ borg create /path/to/repo::my\-files /home \e \-\-exclude \(aqsh:/home/*/.thumbnails\(aq # Backup the root filesystem into an archive named "root\-YYYY\-MM\-DD" -# use zlib compression (good, but slow) \- default is no compression +# use zlib compression (good, but slow) \- default is lz4 (fast, low compression ratio) $ borg create \-C zlib,6 /path/to/repo::root\-{now:%Y\-%m\-%d} / \-\-one\-file\-system # Backup a remote host locally ("pull" style) using sshfs @@ -212,6 +229,11 @@ $ borg create /path/to/repo::{hostname}\-{user}\-{now} ~ $ borg create /path/to/repo::{hostname}\-{user}\-{now:%Y\-%m\-%dT%H:%M:%S} ~ # As above, but add nanoseconds $ borg create /path/to/repo::{hostname}\-{user}\-{now:%Y\-%m\-%dT%H:%M:%S.%f} ~ + +# Backing up relative paths by moving into the correct directory first +$ cd /home/user/Documents +# The root directory of the archive will be "projectA" +$ borg create /path/to/repo::daily\-projectA\-{now:%Y\-%m\-%d} projectA .ft P .fi .UNINDENT @@ -222,6 +244,15 @@ The \-\-exclude patterns are not like tar. In tar \-\-exclude .bundler/gems will exclude foo/.bundler/gems. In borg it will not, you need to use \-\-exclude \(aq*/.bundler/gems\(aq to get the same effect. See \fBborg help patterns\fP for more information. +.sp +In addition to using \fB\-\-exclude\fP patterns, it is possible to use +\fB\-\-exclude\-if\-present\fP to specify the name of a filesystem object (e.g. a file +or folder name) which, when contained within another folder, will prevent the +containing folder from being backed up. By default, the containing folder and +all of its contents will be omitted from the backup. If, however, you wish to +only include the objects specified by \fB\-\-exclude\-if\-present\fP in your backup, +and not include any other contents of the containing folder, this can be enabled +through using the \fB\-\-keep\-exclude\-tags\fP option. .SS Item flags .sp \fB\-\-list\fP outputs a list of all files, directories and other diff --git a/docs/man/borg-delete.1 b/docs/man/borg-delete.1 index 5b0ec5296..01e2167de 100644 --- a/docs/man/borg-delete.1 +++ b/docs/man/borg-delete.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-DELETE 1 "2017-02-11" "" "borg backup tool" +.TH BORG-DELETE 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-delete \- Delete an existing repository or archives . @@ -60,7 +60,7 @@ print statistics for the deleted archive delete only the local cache for the given repository .TP .B \-\-force -force deletion of corrupted archives +force deletion of corrupted archives, use \-\-force \-\-force in case \-\-force does not work. .TP .B \-\-save\-space work slower, but using less space diff --git a/docs/man/borg-diff.1 b/docs/man/borg-diff.1 index f488a72ac..dd0ec88c2 100644 --- a/docs/man/borg-diff.1 +++ b/docs/man/borg-diff.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-DIFF 1 "2017-02-11" "" "borg backup tool" +.TH BORG-DIFF 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-diff \- Diff contents of two archives . @@ -69,12 +69,6 @@ paths of items inside the archives to compare; patterns are supported .SS optional arguments .INDENT 0.0 .TP -.BI \-e \ PATTERN\fP,\fB \ \-\-exclude \ PATTERN -exclude paths matching PATTERN -.TP -.BI \-\-exclude\-from \ EXCLUDEFILE -read exclude patterns from EXCLUDEFILE, one per line -.TP .B \-\-numeric\-owner only consider numeric user and group identifiers .TP @@ -84,6 +78,30 @@ Override check of chunker parameters. .B \-\-sort Sort the output lines by file path. .UNINDENT +.SS Exclusion options +.INDENT 0.0 +.TP +.BI \-e \ PATTERN\fP,\fB \ \-\-exclude \ PATTERN +exclude paths matching PATTERN +.TP +.BI \-\-exclude\-from \ EXCLUDEFILE +read exclude patterns from EXCLUDEFILE, one per line +.TP +.B \-\-exclude\-caches +exclude directories that contain a CACHEDIR.TAG file (\fI\%http://www.brynosaurus.com/cachedir/spec.html\fP) +.TP +.BI \-\-exclude\-if\-present \ NAME +exclude directories that are tagged by containing a filesystem object with the given NAME +.TP +.B \-\-keep\-exclude\-tags\fP,\fB \-\-keep\-tag\-files +if tag objects are specified with \-\-exclude\-if\-present, don\(aqt omit the tag objects themselves from the backup archive +.TP +.BI \-\-pattern \ PATTERN +include/exclude paths matching PATTERN +.TP +.BI \-\-patterns\-from \ PATTERNFILE +read include/exclude patterns from PATTERNFILE, one per line +.UNINDENT .SH EXAMPLES .INDENT 0.0 .INDENT 3.5 diff --git a/docs/man/borg-extract.1 b/docs/man/borg-extract.1 index 426be31e0..089635c8c 100644 --- a/docs/man/borg-extract.1 +++ b/docs/man/borg-extract.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-EXTRACT 1 "2017-02-11" "" "borg backup tool" +.TH BORG-EXTRACT 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-extract \- Extract archive contents . @@ -75,6 +75,12 @@ exclude paths matching PATTERN .BI \-\-exclude\-from \ EXCLUDEFILE read exclude patterns from EXCLUDEFILE, one per line .TP +.BI \-\-pattern \ PATTERN +include/exclude paths matching PATTERN +.TP +.BI \-\-patterns\-from \ PATTERNFILE +read include/exclude patterns from PATTERNFILE, one per line +.TP .B \-\-numeric\-owner only obey numeric user and group identifiers .TP diff --git a/docs/man/borg-info.1 b/docs/man/borg-info.1 index 305f18fad..b75e44ff9 100644 --- a/docs/man/borg-info.1 +++ b/docs/man/borg-info.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-INFO 1 "2017-02-11" "" "borg backup tool" +.TH BORG-INFO 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-info \- Show archive details such as disk space used . @@ -57,6 +57,12 @@ See \fIborg\-common(1)\fP for common options of Borg commands. .B REPOSITORY_OR_ARCHIVE archive or repository to display information about .UNINDENT +.SS optional arguments +.INDENT 0.0 +.TP +.B \-\-json +format output as JSON +.UNINDENT .SS filters .INDENT 0.0 .TP diff --git a/docs/man/borg-init.1 b/docs/man/borg-init.1 index bbe020c43..2b489ff93 100644 --- a/docs/man/borg-init.1 +++ b/docs/man/borg-init.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-INIT 1 "2017-02-11" "" "borg backup tool" +.TH BORG-INIT 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-init \- Initialize an empty repository . diff --git a/docs/man/borg-key-change-passphrase.1 b/docs/man/borg-key-change-passphrase.1 index 86a30ebfa..7b1a71fab 100644 --- a/docs/man/borg-key-change-passphrase.1 +++ b/docs/man/borg-key-change-passphrase.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-02-11" "" "borg backup tool" +.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-key-change-passphrase \- Change repository key file passphrase . diff --git a/docs/man/borg-key-export.1 b/docs/man/borg-key-export.1 index 672023032..8cfe1505a 100644 --- a/docs/man/borg-key-export.1 +++ b/docs/man/borg-key-export.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-EXPORT 1 "2017-02-11" "" "borg backup tool" +.TH BORG-KEY-EXPORT 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-key-export \- Export the repository key for backup . @@ -68,6 +68,9 @@ where to store the backup .TP .B \-\-paper Create an export suitable for printing and later type\-in +.TP +.B \-\-qr\-html +Create an html file suitable for printing and later type\-in or qr scan .UNINDENT .SH SEE ALSO .sp diff --git a/docs/man/borg-key-import.1 b/docs/man/borg-key-import.1 index 91ce569dd..0a5a32453 100644 --- a/docs/man/borg-key-import.1 +++ b/docs/man/borg-key-import.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-IMPORT 1 "2017-02-11" "" "borg backup tool" +.TH BORG-KEY-IMPORT 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-key-import \- Import the repository key from backup . diff --git a/docs/man/borg-key-migrate-to-repokey.1 b/docs/man/borg-key-migrate-to-repokey.1 index 774c9199b..efda66e1d 100644 --- a/docs/man/borg-key-migrate-to-repokey.1 +++ b/docs/man/borg-key-migrate-to-repokey.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-02-11" "" "borg backup tool" +.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-key-migrate-to-repokey \- Migrate passphrase -> repokey . diff --git a/docs/man/borg-key.1 b/docs/man/borg-key.1 index ca1ec2eb3..e0c878d64 100644 --- a/docs/man/borg-key.1 +++ b/docs/man/borg-key.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY 1 "2017-02-12" "" "borg backup tool" +.TH BORG-KEY 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-key \- Manage a keyfile or repokey of a repository . diff --git a/docs/man/borg-list.1 b/docs/man/borg-list.1 index 74e5a2780..be02614e0 100644 --- a/docs/man/borg-list.1 +++ b/docs/man/borg-list.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-LIST 1 "2017-02-12" "" "borg backup tool" +.TH BORG-LIST 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-list \- List archive or repository contents . @@ -60,11 +60,8 @@ only print file/directory names, nothing else specify format for file listing (default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NL}") .TP -.BI \-e \ PATTERN\fP,\fB \ \-\-exclude \ PATTERN -exclude paths matching PATTERN -.TP -.BI \-\-exclude\-from \ EXCLUDEFILE -read exclude patterns from EXCLUDEFILE, one per line +.B \-\-json +format output as JSON. The form of \-\-format is ignored, but keys used in it are added to the JSON output. Some keys are always present. Note: JSON can only represent text. A "bpath" key is therefore not available. .UNINDENT .SS filters .INDENT 0.0 @@ -81,6 +78,30 @@ consider first N archives after other filters were applied .BI \-\-last \ N consider last N archives after other filters were applied .UNINDENT +.SS Exclusion options +.INDENT 0.0 +.TP +.BI \-e \ PATTERN\fP,\fB \ \-\-exclude \ PATTERN +exclude paths matching PATTERN +.TP +.BI \-\-exclude\-from \ EXCLUDEFILE +read exclude patterns from EXCLUDEFILE, one per line +.TP +.B \-\-exclude\-caches +exclude directories that contain a CACHEDIR.TAG file (\fI\%http://www.brynosaurus.com/cachedir/spec.html\fP) +.TP +.BI \-\-exclude\-if\-present \ NAME +exclude directories that are tagged by containing a filesystem object with the given NAME +.TP +.B \-\-keep\-exclude\-tags\fP,\fB \-\-keep\-tag\-files +if tag objects are specified with \-\-exclude\-if\-present, don\(aqt omit the tag objects themselves from the backup archive +.TP +.BI \-\-pattern \ PATTERN +include/exclude paths matching PATTERN +.TP +.BI \-\-patterns\-from \ PATTERNFILE +read include/exclude patterns from PATTERNFILE, one per line +.UNINDENT .SH EXAMPLES .INDENT 0.0 .INDENT 3.5 @@ -141,7 +162,7 @@ Keys for listing repository archives: .INDENT 3.5 .INDENT 0.0 .IP \(bu 2 -archive: archive name interpreted as text (might be missing non\-text characters, see barchive) +archive, name: archive name interpreted as text (might be missing non\-text characters, see barchive) .IP \(bu 2 barchive: verbatim archive name, can contain any character except NUL .IP \(bu 2 @@ -183,6 +204,10 @@ size .IP \(bu 2 csize: compressed size .IP \(bu 2 +dsize: deduplicated size +.IP \(bu 2 +dcsize: deduplicated compressed size +.IP \(bu 2 num_chunks: number of chunks in this file .IP \(bu 2 unique_chunks: number of unique chunks in this file @@ -199,10 +224,6 @@ isoctime .IP \(bu 2 isoatime .IP \(bu 2 -blake2b -.IP \(bu 2 -blake2s -.IP \(bu 2 md5 .IP \(bu 2 sha1 @@ -213,20 +234,8 @@ sha256 .IP \(bu 2 sha384 .IP \(bu 2 -sha3_224 -.IP \(bu 2 -sha3_256 -.IP \(bu 2 -sha3_384 -.IP \(bu 2 -sha3_512 -.IP \(bu 2 sha512 .IP \(bu 2 -shake_128 -.IP \(bu 2 -shake_256 -.IP \(bu 2 archiveid .IP \(bu 2 archivename diff --git a/docs/man/borg-mount.1 b/docs/man/borg-mount.1 index 63a12ceec..cb8ad26ba 100644 --- a/docs/man/borg-mount.1 +++ b/docs/man/borg-mount.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-MOUNT 1 "2017-02-11" "" "borg backup tool" +.TH BORG-MOUNT 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-mount \- Mount archive or an entire repository as a FUSE filesystem . diff --git a/docs/man/borg-patterns.1 b/docs/man/borg-patterns.1 index 646694bfc..c250a4b92 100644 --- a/docs/man/borg-patterns.1 +++ b/docs/man/borg-patterns.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PATTERNS 1 "2017-02-11" "" "borg backup tool" +.TH BORG-PATTERNS 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-patterns \- Details regarding patterns . @@ -32,8 +32,9 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH DESCRIPTION .sp -Exclusion patterns support four separate styles, fnmatch, shell, regular -expressions and path prefixes. By default, fnmatch is used. If followed +File patterns support these styles: fnmatch, shell, regular expressions, +path prefixes and path full\-matches. By default, fnmatch is used for +\fI\-\-exclude\fP patterns and shell\-style is used for \fI\-\-pattern\fP\&. If followed by a colon (\(aq:\(aq) the first two characters of a pattern are used as a style selector. Explicit style selection is necessary when a non\-default style is desired or when the desired pattern starts with @@ -42,12 +43,12 @@ two alphanumeric characters followed by a colon (i.e. \fIaa:something/*\fP). \fI\%Fnmatch\fP, selector \fIfm:\fP .INDENT 0.0 .INDENT 3.5 -This is the default style. These patterns use a variant of shell -pattern syntax, with \(aq*\(aq matching any number of characters, \(aq?\(aq -matching any single character, \(aq[...]\(aq matching any single -character specified, including ranges, and \(aq[!...]\(aq matching any -character not specified. For the purpose of these patterns, the -path separator (\(aq\(aq for Windows and \(aq/\(aq on other systems) is not +This is the default style for \-\-exclude and \-\-exclude\-from. +These patterns use a variant of shell pattern syntax, with \(aq*\(aq matching +any number of characters, \(aq?\(aq matching any single character, \(aq[...]\(aq +matching any single character specified, including ranges, and \(aq[!...]\(aq +matching any character not specified. For the purpose of these patterns, +the path separator (\(aq\(aq for Windows and \(aq/\(aq on other systems) is not treated specially. Wrap meta\-characters in brackets for a literal match (i.e. \fI[?]\fP to match the literal character \fI?\fP). For a path to match a pattern, it must completely match from start to end, or @@ -61,6 +62,7 @@ separator, a \(aq*\(aq is appended before matching is attempted. Shell\-style patterns, selector \fIsh:\fP .INDENT 0.0 .INDENT 3.5 +This is the default style for \-\-pattern and \-\-patterns\-from. Like fnmatch patterns these are similar to shell patterns. The difference is that the pattern may include \fI**/\fP for matching zero or more directory levels, \fI*\fP for matching zero or more arbitrary characters with the @@ -82,7 +84,7 @@ the re module\fP\&. .UNINDENT .UNINDENT .sp -Prefix path, selector \fIpp:\fP +Path prefix, selector \fIpp:\fP .INDENT 0.0 .INDENT 3.5 This pattern style is useful to match whole sub\-directories. The pattern @@ -90,6 +92,25 @@ This pattern style is useful to match whole sub\-directories. The pattern .UNINDENT .UNINDENT .sp +Path full\-match, selector \fIpf:\fP +.INDENT 0.0 +.INDENT 3.5 +This pattern style is useful to match whole paths. +This is kind of a pseudo pattern as it can not have any variable or +unspecified parts \- the full, precise path must be given. +\fIpf:/data/foo.txt\fP matches \fI/data/foo.txt\fP only. +.sp +Implementation note: this is implemented via very time\-efficient O(1) +hashtable lookups (this means you can have huge amounts of such patterns +without impacting performance much). +Due to that, this kind of pattern does not respect any context or order. +If you use such a pattern to include a file, it will always be included +(if the directory recursion encounters it). +Other include/exclude patterns that would normally match will be ignored. +Same logic applies for exclude. +.UNINDENT +.UNINDENT +.sp Exclusions can be passed via the command line option \fI\-\-exclude\fP\&. When used from within a shell the patterns should be quoted to protect them from expansion. @@ -138,6 +159,47 @@ $ borg create \-\-exclude\-from exclude.txt backup / .fi .UNINDENT .UNINDENT +.sp +A more general and easier to use way to define filename matching patterns exists +with the \fI\-\-pattern\fP and \fI\-\-patterns\-from\fP options. Using these, you may specify +the backup roots (starting points) and patterns for inclusion/exclusion. A +root path starts with the prefix \fIR\fP, followed by a path (a plain path, not a +file pattern). An include rule starts with the prefix +, an exclude rule starts +with the prefix \-, both followed by a pattern. +Inclusion patterns are useful to include pathes that are contained in an excluded +path. The first matching pattern is used so if an include pattern matches before +an exclude pattern, the file is backed up. +.sp +Note that the default pattern style for \fI\-\-pattern\fP and \fI\-\-patterns\-from\fP is +shell style (\fIsh:\fP), so those patterns behave similar to rsync include/exclude +patterns. The pattern style can be set via the \fIP\fP prefix. +.sp +Patterns (\fI\-\-pattern\fP) and excludes (\fI\-\-exclude\fP) from the command line are +considered first (in the order of appearance). Then patterns from \fI\-\-patterns\-from\fP +are added. Exclusion patterns from \fI\-\-exclude\-from\fP files are appended last. +.sp +An example \fI\-\-patterns\-from\fP file could look like that: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +# "sh:" pattern style is the default, so the following line is not needed: +P sh +R / +# can be rebuild +\- /home/*/.cache +# they\(aqre downloads for a reason +\- /home/*/Downloads +# susan is a nice person +# include susans home ++ /home/susan +# don\(aqt backup the other home directories +\- /home/* +.ft P +.fi +.UNINDENT +.UNINDENT .SH AUTHOR The Borg Collective .\" Generated by docutils manpage writer. diff --git a/docs/man/borg-placeholders.1 b/docs/man/borg-placeholders.1 index 49cfb6dc7..aa62bb9b6 100644 --- a/docs/man/borg-placeholders.1 +++ b/docs/man/borg-placeholders.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PLACEHOLDERS 1 "2017-02-11" "" "borg backup tool" +.TH BORG-PLACEHOLDERS 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-placeholders \- Details regarding placeholders . diff --git a/docs/man/borg-prune.1 b/docs/man/borg-prune.1 index ff8afa1d5..135415447 100644 --- a/docs/man/borg-prune.1 +++ b/docs/man/borg-prune.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PRUNE 1 "2017-02-11" "" "borg backup tool" +.TH BORG-PRUNE 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-prune \- Prune repository archives according to specified rules . diff --git a/docs/man/borg-recreate.1 b/docs/man/borg-recreate.1 index db822491d..c7e304207 100644 --- a/docs/man/borg-recreate.1 +++ b/docs/man/borg-recreate.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-RECREATE 1 "2017-02-11" "" "borg backup tool" +.TH BORG-RECREATE 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-recreate \- Re-create archives . @@ -39,9 +39,9 @@ Recreate the contents of existing archives. .sp This is an \fIexperimental\fP feature. Do \fInot\fP use this on your only backup. .sp -\-\-exclude, \-\-exclude\-from and PATH have the exact same semantics -as in "borg create". If PATHs are specified the resulting archive -will only contain files from these PATHs. +\-\-exclude, \-\-exclude\-from, \-\-exclude\-if\-present, \-\-keep\-exclude\-tags, and PATH +have the exact same semantics as in "borg create". If PATHs are specified the +resulting archive will only contain files from these PATHs. .sp Note that all paths in an archive are relative, therefore absolute patterns/paths will \fInot\fP match (\-\-exclude, \-\-exclude\-from, \-\-compression\-from, PATHs). @@ -114,10 +114,16 @@ read exclude patterns from EXCLUDEFILE, one per line exclude directories that contain a CACHEDIR.TAG file (\fI\%http://www.brynosaurus.com/cachedir/spec.html\fP) .TP .BI \-\-exclude\-if\-present \ NAME -exclude directories that are tagged by containing a filesystem object with the given NAME +exclude directories that are tagged by containing a filesystem object with the given NAME .TP .B \-\-keep\-exclude\-tags\fP,\fB \-\-keep\-tag\-files -keep tag objects (i.e.: arguments to \-\-exclude\-if\-present) in otherwise excluded caches/directories +if tag objects are specified with \-\-exclude\-if\-present, don\(aqt omit the tag objects themselves from the backup archive +.TP +.BI \-\-pattern \ PATTERN +include/exclude paths matching PATTERN +.TP +.BI \-\-patterns\-from \ PATTERNFILE +read include/exclude patterns from PATTERNFILE, one per line .UNINDENT .SS Archive options .INDENT 0.0 diff --git a/docs/man/borg-rename.1 b/docs/man/borg-rename.1 index e3c1ee773..764ec91e0 100644 --- a/docs/man/borg-rename.1 +++ b/docs/man/borg-rename.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-RENAME 1 "2017-02-11" "" "borg backup tool" +.TH BORG-RENAME 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-rename \- Rename an existing archive . diff --git a/docs/man/borg-serve.1 b/docs/man/borg-serve.1 index ee62ee7f8..d6a6fbcd8 100644 --- a/docs/man/borg-serve.1 +++ b/docs/man/borg-serve.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-SERVE 1 "2017-02-11" "" "borg backup tool" +.TH BORG-SERVE 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-serve \- Start in server mode. This command is usually not used manually. . diff --git a/docs/man/borg-umount.1 b/docs/man/borg-umount.1 index 7d8af5878..c8cea6f45 100644 --- a/docs/man/borg-umount.1 +++ b/docs/man/borg-umount.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-UMOUNT 1 "2017-02-11" "" "borg backup tool" +.TH BORG-UMOUNT 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-umount \- un-mount the FUSE filesystem . diff --git a/docs/man/borg-upgrade.1 b/docs/man/borg-upgrade.1 index 0c0df0d59..c8111a4eb 100644 --- a/docs/man/borg-upgrade.1 +++ b/docs/man/borg-upgrade.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-UPGRADE 1 "2017-02-11" "" "borg backup tool" +.TH BORG-UPGRADE 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-upgrade \- upgrade a repository from a previous version . diff --git a/docs/man/borg-with-lock.1 b/docs/man/borg-with-lock.1 index 2db9c3fc8..7ed9d53e3 100644 --- a/docs/man/borg-with-lock.1 +++ b/docs/man/borg-with-lock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-WITH-LOCK 1 "2017-02-11" "" "borg backup tool" +.TH BORG-WITH-LOCK 1 "2017-03-26" "" "borg backup tool" .SH NAME borg-with-lock \- run a user specified command with the repository lock held . diff --git a/docs/man/borg.1 b/docs/man/borg.1 index b720d71c2..7dffe320b 100644 --- a/docs/man/borg.1 +++ b/docs/man/borg.1 @@ -336,6 +336,10 @@ Main usecase for this is to fully automate \fBborg change\-passphrase\fP\&. .B BORG_DISPLAY_PASSPHRASE When set, use the value to answer the "display the passphrase for verification" question when defining a new passphrase for encrypted repositories. .TP +.B BORG_HOSTNAME_IS_UNIQUE=yes +Use this to assert that your hostname is unique. +Borg will then automatically remove locks that it could determine to be stale. +.TP .B BORG_LOGGING_CONF When set, use the given filename as \fI\%INI\fP\-style logging configuration. .TP @@ -344,7 +348,7 @@ When set, use this command instead of \fBssh\fP\&. This can be used to specify s a custom identity file \fBssh \-i /path/to/private/key\fP\&. See \fBman ssh\fP for other options. .TP .B BORG_REMOTE_PATH -When set, use the given path/filename as remote path (default is "borg"). +When set, use the given path as borg executable on the remote (defaults to "borg" if unset). Using \fB\-\-remote\-path PATH\fP commandline option overrides the environment variable. .TP .B BORG_FILES_CACHE_TTL From 38860b3f53407b7202f0b8464573c833eaa015db Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 27 Mar 2017 12:08:54 +0200 Subject: [PATCH 6/7] lz4 compress: lower max. buffer size, exception handling on the wheezy32 test machine, a test testing with corrupted data crashed with a MemoryError when it tried to get a ~800MB large buffer. MemoryError is now transformed to DecompressionError, so it gets handled better. Also, the bound for giving up is now much lower: 1GiB -> 128MiB. --- src/borg/compress.pyx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/borg/compress.pyx b/src/borg/compress.pyx index 916167252..786e19fd1 100644 --- a/src/borg/compress.pyx +++ b/src/borg/compress.pyx @@ -104,13 +104,16 @@ class LZ4(CompressorBase): # allocate more if isize * 3 is already bigger, to avoid having to resize often. osize = max(int(1.1 * 2**23), isize * 3) while True: - buf = buffer.get(osize) + try: + buf = buffer.get(osize) + except MemoryError: + raise DecompressionError('MemoryError') dest = buf with nogil: rsize = LZ4_decompress_safe(source, dest, isize, osize) if rsize >= 0: break - if osize > 2 ** 30: + if osize > 2 ** 27: # 128MiB (should be enough, considering max. repo obj size and very good compression) # this is insane, get out of here raise DecompressionError('lz4 decompress failed') # likely the buffer was too small, get a bigger one: From 963949812d2e82e34d2de8ec2cab47ea8e8c3f72 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 27 Mar 2017 12:38:19 +0200 Subject: [PATCH 7/7] vagrant: increase memory for parallel testing considering that we run pytest-xdist with 4 parallel processes, we need a bit more RAM to avoid the OOM killer and MemoryError. so, 1GiB for 32bit, 1.5GiB for 64bit VMs. --- Vagrantfile | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index babbe432d..0a3e4af9d 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -355,7 +355,7 @@ Vagrant.configure(2) do |config| config.vm.define "centos7_64" do |b| b.vm.box = "centos/7" b.vm.provider :virtualbox do |v| - v.memory = 768 + v.memory = 1536 end b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos7_64") @@ -367,6 +367,9 @@ Vagrant.configure(2) do |config| config.vm.define "centos6_32" do |b| b.vm.box = "centos6-32" + b.vm.provider :virtualbox do |v| + v.memory = 1024 + end b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos6_32") b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos6_32") @@ -378,7 +381,7 @@ Vagrant.configure(2) do |config| config.vm.define "centos6_64" do |b| b.vm.box = "centos6-64" b.vm.provider :virtualbox do |v| - v.memory = 768 + v.memory = 1536 end b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos6_64") @@ -391,7 +394,7 @@ Vagrant.configure(2) do |config| config.vm.define "xenial64" do |b| b.vm.box = "ubuntu/xenial64" b.vm.provider :virtualbox do |v| - v.memory = 768 + v.memory = 1536 end b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("xenial64") @@ -402,7 +405,7 @@ Vagrant.configure(2) do |config| config.vm.define "trusty64" do |b| b.vm.box = "ubuntu/trusty64" b.vm.provider :virtualbox do |v| - v.memory = 768 + v.memory = 1536 end b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("trusty64") @@ -413,7 +416,7 @@ Vagrant.configure(2) do |config| config.vm.define "jessie64" do |b| b.vm.box = "debian/jessie64" b.vm.provider :virtualbox do |v| - v.memory = 768 + v.memory = 1536 end b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("jessie64") @@ -423,6 +426,9 @@ Vagrant.configure(2) do |config| config.vm.define "wheezy32" do |b| b.vm.box = "boxcutter/debian7-i386" + b.vm.provider :virtualbox do |v| + v.memory = 1024 + end b.vm.provision "packages prepare wheezy", :type => :shell, :inline => packages_prepare_wheezy b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("wheezy32") @@ -492,7 +498,7 @@ Vagrant.configure(2) do |config| config.vm.define "openbsd64" do |b| b.vm.box = "openbsd60-64" # note: basic openbsd install for vagrant WITH sudo and rsync pre-installed b.vm.provider :virtualbox do |v| - v.memory = 768 + v.memory = 1536 end b.vm.provision "packages openbsd", :type => :shell, :inline => packages_openbsd b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openbsd64") @@ -503,7 +509,7 @@ Vagrant.configure(2) do |config| config.vm.define "netbsd64" do |b| b.vm.box = "netbsd70-64" b.vm.provider :virtualbox do |v| - v.memory = 768 + v.memory = 1536 end b.vm.provision "packages netbsd", :type => :shell, :inline => packages_netbsd b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("netbsd64")