diff --git a/README.rst b/README.rst index 2417b24d5..a9d338151 100644 --- a/README.rst +++ b/README.rst @@ -113,13 +113,14 @@ Links ===== * `Main Web Site `_ -* `Releases `_ -* `PyPI packages `_ -* `ChangeLog `_ -* `GitHub `_ -* `Issue Tracker `_ -* `Bounties & Fundraisers `_ -* `Mailing List `_ +* `Releases `_, + `PyPI packages `_ and + `ChangeLog `_ +* `GitHub `_, + `Issue Tracker `_ and + `Bounties & Fundraisers `_ +* `Web-Chat (IRC) `_ and + `Mailing List `_ * `License `_ Notes diff --git a/docs/changes.rst b/docs/changes.rst index 341b6eb48..05089a025 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,6 +1,55 @@ Changelog ========= +Important note about pre-1.0.4 potential repo corruption +-------------------------------------------------------- + +Some external errors (like network or disk I/O errors) could lead to +corruption of the backup repository due to issue #1138. + +A sign that this happened is if "E" status was reported for a file that can +not be explained by problems with the source file. If you still have logs from +"borg create -v --list", you can check for "E" status. + +Here is what could cause corruption and what you can do now: + +1) I/O errors (e.g. repo disk errors) while writing data to repo. + +This could lead to corrupted segment files. + +Fix:: + + # check for corrupt chunks / segments: + borg check -v --repository-only REPO + + # repair the repo: + borg check -v --repository-only --repair REPO + + # make sure everything is fixed: + borg check -v --repository-only REPO + +2) Unreliable network / unreliable connection to the repo. + +This could lead to archive metadata corruption. + +Fix:: + + # check for corrupt archives: + borg check -v --archives-only REPO + + # delete the corrupt archives: + borg delete --force REPO::CORRUPT_ARCHIVE + + # make sure everything is fixed: + borg check -v --archives-only REPO + +3) In case you want to do more intensive checking. + +The best check that everything is ok is to run a dry-run extraction:: + + borg extract -v --dry-run REPO::ARCHIVE + + Version 1.1.0 (not released yet) -------------------------------- @@ -73,6 +122,100 @@ Other changes: - vagrant: add ubuntu/xenial 64bit - this box has still some issues - ChunkBuffer: add test for leaving partial chunk in buffer, fixes #945 +Version 1.0.4 (not released yet) +-------------------------------- + +New features: + +- borg serve --append-only, #1168 + This was included because it was a simple change (append-only functionality + was already present via repository config file) and makes better security now + practically usable. +- BORG_REMOTE_PATH environment variable, #1258 + This was included because it was a simple change (--remote-path cli option + was already present) and makes borg much easier to use if you need it. +- Repository: cleanup incomplete transaction on "no space left" condition. + In many cases, this can avoid a 100% full repo filesystem (which is very + problematic as borg always needs free space - even to delete archives). + +Bug fixes: + +- Fix wrong handling and reporting of OSErrors in borg create, #1138. + This was a serious issue: in the context of "borg create", errors like + repository I/O errors (e.g. disk I/O errors, ssh repo connection errors) + were handled badly and did not lead to a crash (which would be good for this + case, because the repo transaction would be incomplete and trigger a + transaction rollback to clean up). + Now, error handling for source files is cleanly separated from every other + error handling, so only problematic input files are logged and skipped. +- Implement fail-safe error handling for borg extract. + Note that this isn't nearly as critical as the borg create error handling + bug, since nothing is written to the repo. So this was "merely" misleading + error reporting. +- Add missing error handler in directory attr restore loop. +- repo: make sure write data hits disk before the commit tag (#1236) and also + sync the containing directory. +- fixes for --read-special mode: + + - ignore known files cache, #1241 + - fake regular file mode, #1214 + - improve symlinks handling, #1215 +- remove passphrase from subprocess environment, #1105 +- Ignore empty index file (will trigger index rebuild), #1195 +- add missing placeholder support for --prefix, #1027 +- improve exception handling for placeholder replacement +- catch and format exceptions in arg parsing +- helpers: fix "undefined name 'e'" in exception handler +- better error handling for missing repo manifest, #1043 +- borg delete: + + - make it possible to delete a repo without manifest + - borg delete --forced allows to delete corrupted archives, #1139 +- borg check: + + - make borg check work for empty repo + - fix resync and msgpacked item qualifier, #1135 + - rebuild_manifest: fix crash if 'name' or 'time' key were missing. + - better validation of item metadata dicts, #1130 + - better validation of archive metadata dicts +- close the repo on exit - even if rollback did not work, #1197. + This is rather cosmetic, it avoids repo closing in the destructor. + +- tests: + + - fix sparse file test, #1170 + - flake8: ignore new F405, #1185 + - catch "invalid argument" on cygwin, #257 + - fix sparseness assertion in test prep, #1264 + +Other changes: + +- make borg build/work on OpenSSL 1.0 and 1.1, #1187 +- docs / help: + + - fix / clarify prune help, #1143 + - fix "patterns" help formatting + - add missing docs / help about placeholders + - resources: rename atticmatic to borgmatic + - document sshd settings, #545 + - more details about checkpoints, add split trick, #1171 + - support docs: add freenode web chat link, #1175 + - add prune visualization / example, #723 + - add note that Fnmatch is default, #1247 + - make clear that lzma levels > 6 are a waste of cpu cycles + - add a "do not edit" note to auto-generated files, #1250 +- repository interoperability with borg master (1.1dev) branch: + + - borg check: read item metadata keys from manifest, #1147 + - read v2 hints files, #1235 + - fix hints file "unknown version" error handling bug +- tests: add tests for format_line +- llfuse: update version requirement for freebsd +- Vagrantfile: use openbsd 5.9, #716 +- use Python 3.5.2 to build the binaries +- glibc compatibility checker: scripts/glibc_check.py +- add .eggs to .gitignore + Version 1.0.3 ------------- diff --git a/docs/faq.rst b/docs/faq.rst index bae9a2e45..a7e79e35d 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -267,7 +267,7 @@ Once your backup has finished successfully, you can delete all ``.checkpoint`` archives. If you run ``borg prune``, it will also care for deleting unneeded checkpoints. -How can I backup huge file(s) over a instable connection? +How can I backup huge file(s) over a unstable connection? --------------------------------------------------------- You can use this "split trick" as a workaround for the in-between-files-only diff --git a/docs/internals.rst b/docs/internals.rst index 9d1bbd84c..1bd14bb1e 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -280,6 +280,7 @@ emptied to 25%, its size is shrinked. So operations on it have a variable complexity between constant and linear with low factor, and memory overhead varies between 33% and 300%. +.. _cache-memory-usage: Indexes / Caches memory usage ----------------------------- diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 5b50398a7..fbf98da00 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -11,11 +11,15 @@ The next section continues by showing how backups can be automated. Important note about free space ------------------------------- -Before you start creating backups, please make sure that there is **always** +Before you start creating backups, please make sure that there is *always* a good amount of free space on the filesystem that has your backup repository -(and also on ~/.cache). It is hard to tell how much, maybe 1-5%. +(and also on ~/.cache). A few GB should suffice for most hard-drive sized +repositories. See also :ref:`cache-memory-usage`. -If you run out of disk space, it can be hard or impossible to free space, +If |project_name| runs out of disk space, it tries to free as much space as it +can while aborting the current operation safely, which allows to free more space +by deleting/pruning archives. This mechanism is not bullet-proof though. +If you *really* run out of disk space, it can be hard or impossible to free space, because |project_name| needs free space to operate - even to delete backup archives. There is a ``--save-space`` option for some commands, but even with that |project_name| will need free space to operate. diff --git a/docs/support.rst b/docs/support.rst index 7cd1890d3..9d64621fc 100644 --- a/docs/support.rst +++ b/docs/support.rst @@ -16,15 +16,15 @@ ticket on the project's `issue tracker`_. For more general questions or discussions, IRC or mailing list are preferred. -IRC ---- +Chat (IRC) +---------- Join us on channel #borgbackup on chat.freenode.net. As usual on IRC, just ask or tell directly and then patiently wait for replies. Stay connected. You could use the following link (after connecting, you can change the random -nickname you got by typing "/nick mydesirednickname"): +nickname you get by typing "/nick mydesirednickname"): http://webchat.freenode.net/?randomnick=1&channels=%23borgbackup&uio=MTY9dHJ1ZSY5PXRydWUa8 diff --git a/docs/usage.rst b/docs/usage.rst index 0fab01d26..bbdafbbc6 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -79,6 +79,9 @@ General: BORG_RSH When set, use this command instead of ``ssh``. This can be used to specify ssh options, such as a custom identity file ``ssh -i /path/to/private/key``. See ``man ssh`` for other options. + BORG_REMOTE_PATH + When set, use the given path/filename as remote path (default is "borg"). + Using ``--remote-path PATH`` commandline option overrides the environment variable. TMPDIR where temporary files are stored (might need a lot of temporary space for some operations) diff --git a/docs/usage/break-lock.rst.inc b/docs/usage/break-lock.rst.inc index 15afaa0b9..5fa1cda55 100644 --- a/docs/usage/break-lock.rst.inc +++ b/docs/usage/break-lock.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_break-lock: borg break-lock diff --git a/docs/usage/change-passphrase.rst.inc b/docs/usage/change-passphrase.rst.inc index 16ca9608c..3bb827a4f 100644 --- a/docs/usage/change-passphrase.rst.inc +++ b/docs/usage/change-passphrase.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_change-passphrase: borg change-passphrase diff --git a/docs/usage/check.rst.inc b/docs/usage/check.rst.inc index 4dd5b05fb..187172ffd 100644 --- a/docs/usage/check.rst.inc +++ b/docs/usage/check.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_check: borg check @@ -15,6 +17,8 @@ optional arguments | only perform repository checks ``--archives-only`` | only perform archives checks + ``--verify-data`` + | perform cryptographic archive data integrity verification (conflicts with --repository-only) ``--repair`` | attempt to repair any inconsistencies found ``--save-space`` @@ -23,6 +27,8 @@ optional arguments | only check last N archives (Default: all) ``-P``, ``--prefix`` | only consider archive names starting with this prefix + ``-p``, ``--progress`` + | show progress display while checking `Common options`_ | @@ -64,3 +70,15 @@ Second, the consistency and correctness of the archive metadata is verified: required). - The archive checks can be time consuming, they can be skipped using the --repository-only option. + +The --verify-data option will perform a full integrity verification (as opposed to +checking the CRC32 of the segment) of data, which means reading the data from the +repository, decrypting and decompressing it. This is a cryptographic verification, +which will detect (accidental) corruption. For encrypted repositories it is +tamper-resistant as well, unless the attacker has access to the keys. + +It is also very slow. + +--verify-data only verifies data used by the archives specified with --last, +--prefix or an explicitly named archive. If none of these are passed, +all data in the repository is verified. diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc index 7b49f109f..6e176a52e 100644 --- a/docs/usage/create.rst.inc +++ b/docs/usage/create.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_create: borg create @@ -47,7 +49,7 @@ Filesystem options ``--ignore-inode`` | ignore inode data in the file metadata cache used to detect unchanged files. ``--read-special`` - | open and read special files as if they were regular files + | open and read block and char device files as well as FIFOs as if they were regular files. Also follows symlinks pointing to these kinds of files. Archive options ``--comment COMMENT`` @@ -55,17 +57,21 @@ Archive options ``--timestamp yyyy-mm-ddThh:mm:ss`` | manually specify the archive creation date/time (UTC). alternatively, give a reference file/directory. ``-c SECONDS``, ``--checkpoint-interval SECONDS`` - | write checkpoint every SECONDS seconds (Default: 300) + | write checkpoint every SECONDS seconds (Default: 1800) ``--chunker-params CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE`` | specify the chunker parameters. default: 19,23,21,4095 ``-C COMPRESSION``, ``--compression COMPRESSION`` | select compression algorithm (and level): | none == no compression (default), + | auto,C[,L] == built-in heuristic decides between none or C[,L] - with C[,L] + | being any valid compression algorithm (and optional level), | lz4 == lz4, | zlib == zlib (default level 6), | zlib,0 .. zlib,9 == zlib (with level 0..9), | lzma == lzma (default level 6), | lzma,0 .. lzma,9 == lzma (with level 0..9). + ``--compression-from COMPRESSIONCONFIG`` + | read compression patterns from COMPRESSIONCONFIG, one per line Description ~~~~~~~~~~~ diff --git a/docs/usage/debug-delete-obj.rst.inc b/docs/usage/debug-delete-obj.rst.inc index 8e9f6e667..4fcfb48f9 100644 --- a/docs/usage/debug-delete-obj.rst.inc +++ b/docs/usage/debug-delete-obj.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_debug-delete-obj: borg debug-delete-obj diff --git a/docs/usage/debug-dump-archive-items.rst.inc b/docs/usage/debug-dump-archive-items.rst.inc index ec35c80a1..63c395460 100644 --- a/docs/usage/debug-dump-archive-items.rst.inc +++ b/docs/usage/debug-dump-archive-items.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_debug-dump-archive-items: borg debug-dump-archive-items diff --git a/docs/usage/debug-get-obj.rst.inc b/docs/usage/debug-get-obj.rst.inc index 986b2c4f5..a0b3f4577 100644 --- a/docs/usage/debug-get-obj.rst.inc +++ b/docs/usage/debug-get-obj.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_debug-get-obj: borg debug-get-obj diff --git a/docs/usage/debug-put-obj.rst.inc b/docs/usage/debug-put-obj.rst.inc index 8a4c3987d..d03ace849 100644 --- a/docs/usage/debug-put-obj.rst.inc +++ b/docs/usage/debug-put-obj.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_debug-put-obj: borg debug-put-obj diff --git a/docs/usage/delete.rst.inc b/docs/usage/delete.rst.inc index e07069d3d..87451ec50 100644 --- a/docs/usage/delete.rst.inc +++ b/docs/usage/delete.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_delete: borg delete @@ -17,6 +19,8 @@ optional arguments | print statistics for the deleted archive ``-c``, ``--cache-only`` | delete only the local cache for the given repository + ``--force`` + | force deletion of corrupted archives ``--save-space`` | work slower, but using less space diff --git a/docs/usage/diff.rst.inc b/docs/usage/diff.rst.inc index a24be1446..9836af579 100644 --- a/docs/usage/diff.rst.inc +++ b/docs/usage/diff.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_diff: borg diff diff --git a/docs/usage/extract.rst.inc b/docs/usage/extract.rst.inc index f18458c30..c68eaa768 100644 --- a/docs/usage/extract.rst.inc +++ b/docs/usage/extract.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_extract: borg extract @@ -42,3 +44,7 @@ by passing a list of ``PATHs`` as arguments. The file selection can further be restricted by using the ``--exclude`` option. See the output of the "borg help patterns" command for more help on exclude patterns. + +By using ``--dry-run``, you can do all extraction steps except actually writing the +output data: reading metadata and data chunks from the repo, checking the hash/hmac, +decrypting, decompressing. diff --git a/docs/usage/help.rst.inc b/docs/usage/help.rst.inc index 9b8d23d66..c83dfb0aa 100644 --- a/docs/usage/help.rst.inc +++ b/docs/usage/help.rst.inc @@ -1,41 +1,5 @@ -.. _borg_placeholders: +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! -borg help placeholders -~~~~~~~~~~~~~~~~~~~~~~ -:: - - -Repository (or Archive) URLs and --prefix values support these placeholders: - -{hostname} - - The (short) hostname of the machine. - -{fqdn} - - The full name of the machine. - -{now} - - The current local date and time. - -{utcnow} - - The current UTC date and time. - -{user} - - The user name (or UID, if no name is available) of the user running borg. - -{pid} - - The current process ID. - -Examples:: - - borg create /path/to/repo::{hostname}-{user}-{utcnow} ... - borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S} ... - borg prune --prefix '{hostname}-' ... .. _borg_patterns: borg help patterns @@ -129,3 +93,41 @@ Examples:: sh:/home/*/.thumbnails EOF $ borg create --exclude-from exclude.txt backup / +.. _borg_placeholders: + +borg help placeholders +~~~~~~~~~~~~~~~~~~~~~~ +:: + + +Repository (or Archive) URLs and --prefix values support these placeholders: + +{hostname} + + The (short) hostname of the machine. + +{fqdn} + + The full name of the machine. + +{now} + + The current local date and time. + +{utcnow} + + The current UTC date and time. + +{user} + + The user name (or UID, if no name is available) of the user running borg. + +{pid} + + The current process ID. + +Examples:: + + borg create /path/to/repo::{hostname}-{user}-{utcnow} ... + borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S} ... + borg prune --prefix '{hostname}-' ... diff --git a/docs/usage/info.rst.inc b/docs/usage/info.rst.inc index f5e61f48f..e9e5f8935 100644 --- a/docs/usage/info.rst.inc +++ b/docs/usage/info.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_info: borg info @@ -17,3 +19,7 @@ Description ~~~~~~~~~~~ This command displays some detailed information about the specified archive. + +The "This archive" line refers exclusively to this archive: +"Deduplicated size" is the size of the unique chunks stored only for this +archive. Non-unique / common chunks show up under "All archives". diff --git a/docs/usage/init.rst.inc b/docs/usage/init.rst.inc index ff85f73b2..b2c84131c 100644 --- a/docs/usage/init.rst.inc +++ b/docs/usage/init.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_init: borg init @@ -22,4 +24,45 @@ Description This command initializes an empty repository. A repository is a filesystem directory containing the deduplicated data from zero or more archives. -Encryption can be enabled at repository init time. + +Encryption can be enabled at repository init time (the default). + +It is not recommended to disable encryption. Repository encryption protects you +e.g. against the case that an attacker has access to your backup repository. + +But be careful with the key / the passphrase: + +If you want "passphrase-only" security, use the repokey mode. The key will +be stored inside the repository (in its "config" file). In above mentioned +attack scenario, the attacker will have the key (but not the passphrase). + +If you want "passphrase and having-the-key" security, use the keyfile mode. +The key will be stored in your home directory (in .config/borg/keys). In +the attack scenario, the attacker who has just access to your repo won't have +the key (and also not the passphrase). + +Make a backup copy of the key file (keyfile mode) or repo config file +(repokey mode) and keep it at a safe place, so you still have the key in +case it gets corrupted or lost. Also keep the passphrase at a safe place. +The backup that is encrypted with that key won't help you with that, of course. + +Make sure you use a good passphrase. Not too short, not too simple. The real +encryption / decryption key is encrypted with / locked by your passphrase. +If an attacker gets your key, he can't unlock and use it without knowing the +passphrase. + +Be careful with special or non-ascii characters in your passphrase: + +- Borg processes the passphrase as unicode (and encodes it as utf-8), + so it does not have problems dealing with even the strangest characters. +- BUT: that does not necessarily apply to your OS / VM / keyboard configuration. + +So better use a long passphrase made from simple ascii chars than one that +includes non-ascii stuff or characters that are hard/impossible to enter on +a different keyboard layout. + +You can change your passphrase for existing repos at any time, it won't affect +the encryption/decryption key or other secrets. + +When encrypting, AES-CTR-256 is used for encryption, and HMAC-SHA256 for +authentication. Hardware acceleration will be used automatically. diff --git a/docs/usage/list.rst.inc b/docs/usage/list.rst.inc index 4917c269f..8e32df6ab 100644 --- a/docs/usage/list.rst.inc +++ b/docs/usage/list.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_list: borg list diff --git a/docs/usage/migrate-to-repokey.rst.inc b/docs/usage/migrate-to-repokey.rst.inc index eb05cc047..ec5f1a52b 100644 --- a/docs/usage/migrate-to-repokey.rst.inc +++ b/docs/usage/migrate-to-repokey.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_migrate-to-repokey: borg migrate-to-repokey diff --git a/docs/usage/mount.rst.inc b/docs/usage/mount.rst.inc index 98efc629c..83310d8b5 100644 --- a/docs/usage/mount.rst.inc +++ b/docs/usage/mount.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_mount: borg mount @@ -35,3 +37,8 @@ used in fstab entries: To allow a regular user to use fstab entries, add the ``user`` option: ``/path/to/repo /mnt/point fuse.borgfs defaults,noauto,user 0 0`` + +The BORG_MOUNT_DATA_CACHE_ENTRIES environment variable is meant for advanced users +to tweak the performance. It sets the number of cached data chunks; additional +memory usage can be up to ~8 MiB times this number. The default is the number +of CPU cores. diff --git a/docs/usage/prune.rst.inc b/docs/usage/prune.rst.inc index 37e9f7b9f..5c63d44ff 100644 --- a/docs/usage/prune.rst.inc +++ b/docs/usage/prune.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_prune: borg prune @@ -13,12 +15,18 @@ positional arguments optional arguments ``-n``, ``--dry-run`` | do not change repository + ``--force`` + | force pruning of corrupted archives ``-s``, ``--stats`` | print statistics for the deleted archive ``--list`` | output verbose list of archives it keeps/prunes ``--keep-within WITHIN`` | keep all archives within this time interval + ``--keep-last``, ``--keep-secondly`` + | number of secondly archives to keep + ``--keep-minutely`` + | number of minutely archives to keep ``-H``, ``--keep-hourly`` | number of hourly archives to keep ``-d``, ``--keep-daily`` @@ -44,13 +52,19 @@ The prune command prunes a repository by deleting all archives not matching any of the specified retention options. This command is normally used by automated backup scripts wanting to keep a certain number of historic backups. -As an example, "-d 7" means to keep the latest backup on each day, up to 7 -most recent days with backups (days without backups do not count). -The rules are applied from hourly to yearly, and backups selected by previous -rules do not count towards those of later rules. The time that each backup -starts is used for pruning purposes. Dates and times are interpreted in -the local timezone, and weeks go from Monday to Sunday. Specifying a -negative number of archives to keep means that there is no limit. +Also, prune automatically removes checkpoint archives (incomplete archives left +behind by interrupted backup runs) except if the checkpoint is the latest +archive (and thus still needed). Checkpoint archives are not considered when +comparing archive counts against the retention limits (--keep-*). + +If a prefix is set with -P, then only archives that start with the prefix are +considered for deletion and only those archives count towards the totals +specified by the rules. +Otherwise, *all* archives in the repository are candidates for deletion! + +If you have multiple sequences of archives with different data sets (e.g. +from different machines) in one shared repository, use one prune call per +data set that matches only the respective archives using the -P option. The "--keep-within" option takes an argument of the form "", where char is "H", "d", "w", "m", "y". For example, "--keep-within 2d" means @@ -58,7 +72,15 @@ to keep all archives that were created within the past 48 hours. "1m" is taken to mean "31d". The archives kept with this option do not count towards the totals specified by any other options. -If a prefix is set with -P, then only archives that start with the prefix are -considered for deletion and only those archives count towards the totals -specified by the rules. -Otherwise, *all* archives in the repository are candidates for deletion! +A good procedure is to thin out more and more the older your backups get. +As an example, "--keep-daily 7" means to keep the latest backup on each day, +up to 7 most recent days with backups (days without backups do not count). +The rules are applied from secondly to yearly, and backups selected by previous +rules do not count towards those of later rules. The time that each backup +starts is used for pruning purposes. Dates and times are interpreted in +the local timezone, and weeks go from Monday to Sunday. Specifying a +negative number of archives to keep means that there is no limit. + +The "--keep-last N" option is doing the same as "--keep-secondly N" (and it will +keep the last N archives under the assumption that you do not create more than one +backup archive in the same second). diff --git a/docs/usage/recreate.rst.inc b/docs/usage/recreate.rst.inc index 2eb3b4fcc..f4134b0a8 100644 --- a/docs/usage/recreate.rst.inc +++ b/docs/usage/recreate.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_recreate: borg recreate @@ -47,11 +49,15 @@ Archive options ``-C COMPRESSION``, ``--compression COMPRESSION`` | select compression algorithm (and level): | none == no compression (default), + | auto,C[,L] == built-in heuristic decides between none or C[,L] - with C[,L] + | being any valid compression algorithm (and optional level), | lz4 == lz4, | zlib == zlib (default level 6), | zlib,0 .. zlib,9 == zlib (with level 0..9), | lzma == lzma (default level 6), | lzma,0 .. lzma,9 == lzma (with level 0..9). + ``--compression-from COMPRESSIONCONFIG`` + | read compression patterns from COMPRESSIONCONFIG, one per line ``--chunker-params CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE`` | specify the chunker parameters (or "default"). diff --git a/docs/usage/rename.rst.inc b/docs/usage/rename.rst.inc index 971b0a634..3cff5a8a4 100644 --- a/docs/usage/rename.rst.inc +++ b/docs/usage/rename.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_rename: borg rename diff --git a/docs/usage/serve.rst.inc b/docs/usage/serve.rst.inc index bbab518fc..933f72b9f 100644 --- a/docs/usage/serve.rst.inc +++ b/docs/usage/serve.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_serve: borg serve @@ -9,6 +11,8 @@ borg serve optional arguments ``--restrict-to-path PATH`` | restrict repository access to PATH + ``--append-only`` + | only allow appending to repository segment files `Common options`_ | diff --git a/docs/usage/upgrade.rst.inc b/docs/usage/upgrade.rst.inc index 3f9523d45..525c5ebde 100644 --- a/docs/usage/upgrade.rst.inc +++ b/docs/usage/upgrade.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_upgrade: borg upgrade diff --git a/docs/usage/with-lock.rst.inc b/docs/usage/with-lock.rst.inc index 3037ee809..c77eb2f60 100644 --- a/docs/usage/with-lock.rst.inc +++ b/docs/usage/with-lock.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_with-lock: borg with-lock diff --git a/scripts/glibc_check.py b/scripts/glibc_check.py new file mode 100644 index 000000000..a400bbd1d --- /dev/null +++ b/scripts/glibc_check.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +""" +Check if all given binaries work with the given glibc version. + +check_glibc.py 2.11 bin [bin ...] +""" + +import re +import subprocess +import sys + +verbose = True +objdump = "objdump -T %s" +glibc_re = re.compile(r'GLIBC_([0-9]\.[0-9]+)') + + +def parse_version(v): + major, minor = v.split('.') + return int(major), int(minor) + + +def format_version(version): + return "%d.%d" % version + + +def main(): + given = parse_version(sys.argv[1]) + filenames = sys.argv[2:] + + overall_versions = set() + for filename in filenames: + try: + output = subprocess.check_output(objdump % filename, shell=True, + stderr=subprocess.STDOUT) + output = output.decode('utf-8') + versions = set(parse_version(match.group(1)) + for match in glibc_re.finditer(output)) + requires_glibc = max(versions) + overall_versions.add(requires_glibc) + if verbose: + print("%s %s" % (filename, format_version(requires_glibc))) + except subprocess.CalledProcessError as e: + if verbose: + print("%s errored." % filename) + + wanted = max(overall_versions) + ok = given >= wanted + + if verbose: + if ok: + print("The binaries work with the given glibc %s." % + format_version(given)) + else: + print("The binaries do not work with the given glibc %s. " + "Minimum is: %s" % (format_version(given), format_version(wanted))) + return ok + + +if __name__ == '__main__': + ok = main() + sys.exit(0 if ok else 1) diff --git a/src/borg/hash_sizes.py b/scripts/hash_sizes.py similarity index 100% rename from src/borg/hash_sizes.py rename to scripts/hash_sizes.py diff --git a/setup.py b/setup.py index 0079d5098..3e916eea0 100644 --- a/setup.py +++ b/setup.py @@ -181,6 +181,7 @@ class build_usage(Command): for command, parser in choices.items(): print('generating help for %s' % command) with open('docs/usage/%s.rst.inc' % command, 'w') as doc: + doc.write(".. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!\n\n") if command == 'help': for topic in Archiver.helptext: params = {"topic": topic, diff --git a/src/borg/archive.py b/src/borg/archive.py index f7883bab7..d107af92b 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -588,25 +588,61 @@ Number of files: {0.stats.nfiles}'''.format( self.set_meta(b'name', name) del self.manifest.archives[oldname] - def delete(self, stats, progress=False): - unpacker = msgpack.Unpacker(use_list=False) - items_ids = self.metadata[b'items'] - pi = ProgressIndicatorPercent(total=len(items_ids), msg="Decrementing references %3.0f%%", same_line=True) - for (i, (items_id, data)) in enumerate(zip(items_ids, self.repository.get_many(items_ids))): + def delete(self, stats, progress=False, forced=False): + class ChunksIndexError(Error): + """Chunk ID {} missing from chunks index, corrupted chunks index - aborting transaction.""" + + def chunk_decref(id, stats): + nonlocal error + try: + self.cache.chunk_decref(id, stats) + except KeyError: + cid = hexlify(id).decode('ascii') + raise ChunksIndexError(cid) + except Repository.ObjectNotFound as e: + # object not in repo - strange, but we wanted to delete it anyway. + if not forced: + raise + error = True + + error = False + try: + unpacker = msgpack.Unpacker(use_list=False) + items_ids = self.metadata[b'items'] + pi = ProgressIndicatorPercent(total=len(items_ids), msg="Decrementing references %3.0f%%", same_line=True) + for (i, (items_id, data)) in enumerate(zip(items_ids, self.repository.get_many(items_ids))): + if progress: + pi.show(i) + _, data = self.key.decrypt(items_id, data) + unpacker.feed(data) + chunk_decref(items_id, stats) + try: + for item in unpacker: + item = Item(internal_dict=item) + if 'chunks' in item: + for chunk_id, size, csize in item.chunks: + chunk_decref(chunk_id, stats) + except (TypeError, ValueError): + # if items metadata spans multiple chunks and one chunk got dropped somehow, + # it could be that unpacker yields bad types + if not forced: + raise + error = True if progress: - pi.show(i) - _, data = self.key.decrypt(items_id, data) - unpacker.feed(data) - self.cache.chunk_decref(items_id, stats) - for item in unpacker: - item = Item(internal_dict=item) - if 'chunks' in item: - for chunk_id, size, csize in item.chunks: - self.cache.chunk_decref(chunk_id, stats) - if progress: - pi.finish() - self.cache.chunk_decref(self.id, stats) + pi.finish() + except (msgpack.UnpackException, Repository.ObjectNotFound): + # items metadata corrupted + if not forced: + raise + error = True + # in forced delete mode, we try hard to delete at least the manifest entry, + # if possible also the archive superblock, even if processing the items raises + # some harmless exception. + chunk_decref(self.id, stats) del self.manifest.archives[self.name] + if error: + logger.warning('forced deletion succeeded, but the deleted archive was corrupted.') + logger.warning('borg check --repair is required to free all space.') def stat_attrs(self, st, path): attrs = dict( diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 04329c83b..a01409805 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -696,7 +696,7 @@ class Archiver: with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache: archive = Archive(repository, key, manifest, args.location.archive, cache=cache) stats = Statistics() - archive.delete(stats, progress=args.progress) + archive.delete(stats, progress=args.progress, forced=args.forced) manifest.write() repository.commit(save_space=args.save_space) cache.commit() @@ -874,7 +874,7 @@ class Archiver: else: if args.output_list: list_logger.info('Pruning archive: %s' % format_archive(archive)) - Archive(repository, key, manifest, archive.name, cache).delete(stats) + Archive(repository, key, manifest, archive.name, cache).delete(stats, forced=args.forced) else: if args.output_list: list_logger.info('Keeping archive: %s' % format_archive(archive)) @@ -1225,8 +1225,8 @@ class Archiver: help='do not load/update the file metadata cache used to detect unchanged files') common_group.add_argument('--umask', dest='umask', type=lambda s: int(s, 8), default=UMASK_DEFAULT, metavar='M', help='set umask to M (local and remote, default: %(default)04o)') - common_group.add_argument('--remote-path', dest='remote_path', default='borg', metavar='PATH', - help='set remote path to executable (default: "%(default)s")') + common_group.add_argument('--remote-path', dest='remote_path', metavar='PATH', + help='set remote path to executable (default: "borg")') parser = argparse.ArgumentParser(prog=prog, description='Borg - Deduplicated Backups') parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + __version__, @@ -1673,6 +1673,9 @@ class Archiver: subparser.add_argument('-c', '--cache-only', dest='cache_only', action='store_true', default=False, help='delete only the local cache for the given repository') + subparser.add_argument('--force', dest='forced', + action='store_true', default=False, + help='force deletion of corrupted archives') subparser.add_argument('--save-space', dest='save_space', action='store_true', default=False, help='work slower, but using less space') @@ -1832,6 +1835,9 @@ class Archiver: subparser.add_argument('-n', '--dry-run', dest='dry_run', default=False, action='store_true', help='do not change repository') + subparser.add_argument('--force', dest='forced', + action='store_true', default=False, + help='force pruning of corrupted archives') subparser.add_argument('-s', '--stats', dest='stats', action='store_true', default=False, help='print statistics for the deleted archive') diff --git a/src/borg/remote.py b/src/borg/remote.py index 39b325fd4..1a55b9eeb 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -228,7 +228,8 @@ class RemoteRepository: if testing: return [sys.executable, '-m', 'borg.archiver', 'serve'] + opts + self.extra_test_args else: # pragma: no cover - return [args.remote_path, 'serve'] + opts + remote_path = args.remote_path or os.environ.get('BORG_REMOTE_PATH', 'borg') + return [remote_path, 'serve'] + opts def ssh_cmd(self, location): """return a ssh command line that can be prefixed to a borg command line""" diff --git a/src/borg/repository.py b/src/borg/repository.py index 526db9e4e..fa9287f7f 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -126,6 +126,12 @@ class Repository: def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is not None: + no_space_left_on_device = exc_type is OSError and exc_val.errno == errno.ENOSPC + # The ENOSPC could have originated somewhere else besides the Repository. The cleanup is always safe, unless + # EIO or FS corruption ensues, which is why we specifically check for ENOSPC. + if self._active_txn and no_space_left_on_device: + logger.warning('No space left on device, cleaning up partial transaction to free space.') + self.io.cleanup(self.io.get_segments_transaction_id()) self.rollback() self.close() diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 5ec7eab5c..3a2dc4f07 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -290,10 +290,14 @@ class ArchiverTestCaseBase(BaseTestCase): # File mode os.chmod('input/dir2', 0o555) # if we take away write perms, we need root to remove contents # File owner - os.chown('input/file1', 100, 200) + os.chown('input/file1', 100, 200) # raises OSError invalid argument on cygwin have_root = True # we have (fake)root except PermissionError: have_root = False + except OSError as e: + if e.errno != errno.EINVAL: + raise + have_root = False return have_root @@ -426,7 +430,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): st = os.stat(filename) self.assert_equal(st.st_size, total_len) if sparse_support and hasattr(st, 'st_blocks'): - self.assert_true(st.st_blocks * 512 < total_len / 9) # is input sparse? + self.assert_true(st.st_blocks * 512 < total_len) # is input sparse? self.cmd('init', self.repository_location) self.cmd('create', self.repository_location + '::test', 'input') with changedir('output'): @@ -443,7 +447,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): if sparse_support: if hasattr(st, 'st_blocks'): # do only check if it is less, do NOT check if it is much less - # as that causes troubles on xfs and zfs: + # as that causes troubles on xfs, zfs, ntfs: self.assert_true(st.st_blocks * 512 < total_len) if hasattr(os, 'SEEK_HOLE') and hasattr(os, 'SEEK_DATA'): with open(filename, 'rb') as fd: