mirror of https://github.com/borgbackup/borg.git
Merge branch 'master' into borg2
strange conflicts, automated patches seemed to not have applied correctly. also had to fix some stuff manually, tests were failing.
This commit is contained in:
commit
e0c64629d1
|
@ -57,6 +57,9 @@ jobs:
|
|||
- os: ubuntu-20.04
|
||||
python-version: '3.10'
|
||||
toxenv: py310-fuse3
|
||||
- os: ubuntu-20.04
|
||||
python-version: '3.11-dev'
|
||||
toxenv: py311-fuse2
|
||||
- os: macos-10.15 # macos-latest is macos 11.6.2 and hanging at test_fuse, #6099
|
||||
python-version: '3.9'
|
||||
toxenv: py39-fuse2
|
||||
|
|
|
@ -163,7 +163,7 @@ def install_pythons(boxname)
|
|||
return <<-EOF
|
||||
. ~/.bash_profile
|
||||
pyenv install 3.10.0 # tests, version supporting openssl 1.1
|
||||
pyenv install 3.9.12 # tests, version supporting openssl 1.1, binary build
|
||||
pyenv install 3.9.13 # tests, version supporting openssl 1.1, binary build
|
||||
pyenv rehash
|
||||
EOF
|
||||
end
|
||||
|
@ -181,8 +181,8 @@ def build_pyenv_venv(boxname)
|
|||
. ~/.bash_profile
|
||||
cd /vagrant/borg
|
||||
# use the latest 3.9 release
|
||||
pyenv global 3.9.12
|
||||
pyenv virtualenv 3.9.12 borg-env
|
||||
pyenv global 3.9.13
|
||||
pyenv virtualenv 3.9.13 borg-env
|
||||
ln -s ~/.pyenv/versions/borg-env .
|
||||
EOF
|
||||
end
|
||||
|
@ -206,10 +206,7 @@ def install_pyinstaller()
|
|||
. ~/.bash_profile
|
||||
cd /vagrant/borg
|
||||
. borg-env/bin/activate
|
||||
git clone https://github.com/thomaswaldmann/pyinstaller.git
|
||||
cd pyinstaller
|
||||
git checkout v4.7-maint
|
||||
python setup.py install
|
||||
pip install 'pyinstaller==4.10'
|
||||
EOF
|
||||
end
|
||||
|
||||
|
@ -232,8 +229,8 @@ def run_tests(boxname, skip_env)
|
|||
. ../borg-env/bin/activate
|
||||
if which pyenv 2> /dev/null; then
|
||||
# for testing, use the earliest point releases of the supported python versions:
|
||||
pyenv global 3.9.12 3.10.0
|
||||
pyenv local 3.9.12 3.10.0
|
||||
pyenv global 3.9.13 3.10.0
|
||||
pyenv local 3.9.13 3.10.0
|
||||
fi
|
||||
# otherwise: just use the system python
|
||||
# some OSes can only run specific test envs, e.g. because they miss FUSE support:
|
||||
|
|
150
docs/changes.rst
150
docs/changes.rst
|
@ -305,6 +305,10 @@ and maybe just were not noticed.
|
|||
|
||||
Compatibility notes:
|
||||
|
||||
- matching of path patterns has been aligned with borg storing relative paths.
|
||||
Borg archives file paths without leading slashes. Previously, include/exclude
|
||||
patterns could contain leading slashes. You should check your patterns and
|
||||
remove leading slashes.
|
||||
- dropped support / testing for older Pythons, minimum requirement is 3.8.
|
||||
In case your OS does not provide Python >= 3.8, consider using our binary,
|
||||
which does not need an external Python interpreter. Or continue using
|
||||
|
@ -1080,6 +1084,152 @@ Other changes:
|
|||
- vagrant: new VMs for linux/bsd/darwin, most with OpenSSL 1.1 and py36
|
||||
|
||||
|
||||
Version 1.1.18 (2022-06-05)
|
||||
---------------------------
|
||||
|
||||
Compatibility notes:
|
||||
|
||||
- When upgrading from borg 1.0.x to 1.1.x, please note:
|
||||
|
||||
- read all the compatibility notes for 1.1.0*, starting from 1.1.0b1.
|
||||
- borg upgrade: you do not need to and you also should not run it.
|
||||
- borg might ask some security-related questions once after upgrading.
|
||||
You can answer them either manually or via environment variable.
|
||||
One known case is if you use unencrypted repositories, then it will ask
|
||||
about a unknown unencrypted repository one time.
|
||||
- your first backup with 1.1.x might be significantly slower (it might
|
||||
completely read, chunk, hash a lot files) - this is due to the
|
||||
--files-cache mode change (and happens every time you change mode).
|
||||
You can avoid the one-time slowdown by using the pre-1.1.0rc4-compatible
|
||||
mode (but that is less safe for detecting changed files than the default).
|
||||
See the --files-cache docs for details.
|
||||
- 1.1.11 removes WSL autodetection (Windows 10 Subsystem for Linux).
|
||||
If WSL still has a problem with sync_file_range, you need to set
|
||||
BORG_WORKAROUNDS=basesyncfile in the borg process environment to
|
||||
work around the WSL issue.
|
||||
- 1.1.14 changes return codes due to a bug fix:
|
||||
In case you have scripts expecting rc == 2 for a signal exit, you need to
|
||||
update them to check for >= 128 (as documented since long).
|
||||
- 1.1.15 drops python 3.4 support, minimum requirement is 3.5 now.
|
||||
- 1.1.17 install_requires the "packaging" pypi package now.
|
||||
|
||||
New features:
|
||||
|
||||
- check --repair: significantly speed up search for next valid object in segment, #6022
|
||||
- create: add retry_erofs workaround for O_NOATIME issue on volume shadow copies in WSL1, #6024
|
||||
- key export: display key if path is '-' or not given, #6092
|
||||
- list --format: add command_line to format keys, #6108
|
||||
|
||||
Fixes:
|
||||
|
||||
- check: improve error handling for corrupt archive metadata block,
|
||||
make robust_iterator more robust, #4777
|
||||
- diff: support presence change for blkdev, chrdev and fifo items, #6483
|
||||
- diff: reduce memory consumption, fix is_hardlink_master
|
||||
- init: disallow overwriting of existing keyfiles
|
||||
- info: fix authenticated mode repo to show "Encrypted: No", #6462
|
||||
- info: emit repo info even if repo has 0 archives, #6120
|
||||
- list: remove placeholders for shake_* hashes, #6082
|
||||
- mount -o versions: give clear error msg instead of crashing
|
||||
- show_progress: add finished=true/false to archive_progress json, #6570
|
||||
- fix hardlinkable file type check, #6037
|
||||
- do not show archive name in error msgs referring to the repository, #6023
|
||||
- prettier error msg (no stacktrace) if exclude file is missing, #5734
|
||||
- do not require BORG_CONFIG_DIR if BORG_{SECURITY,KEYS}_DIR are set, #5979
|
||||
- atomically create the CACHE_TAG file, #6028
|
||||
- deal with the SaveFile/SyncFile race, docs, see #6176 5c5b59bc9
|
||||
- avoid expanding path into LHS of formatting operation + tests, #6064 #6063
|
||||
- repository: quota / compactable computation fixes, #6119.
|
||||
This is mainly to keep the repo code in sync with borg 1.2. As borg 1.1
|
||||
compacts immediately, there was not really an issue with this in 1.1.
|
||||
- fix transaction rollback: use files cache filename as found in txn.active, #6353
|
||||
- do not load files cache for commands not using it, fixes #5673
|
||||
- fix scp repo url parsing for ip v6 addrs, #6526
|
||||
- repo::archive location placeholder expansion fixes, #5826, #5998
|
||||
|
||||
- use expanded location for log output
|
||||
- support placeholder expansion for BORG_REPO env var
|
||||
- respect umask for created directory and file modes, #6400
|
||||
- safer truncate_and_unlink implementation
|
||||
|
||||
Other changes:
|
||||
|
||||
- upgrade bundled xxhash code to 0.8.1
|
||||
- fix xxh64 related build (setup.py and post-0.8.1 patch for static_assert).
|
||||
The patch was required to build the bundled xxhash code on FreeBSD, see
|
||||
https://github.com/Cyan4973/xxHash/pull/670
|
||||
- msgpack build: remove endianness macro, #6105
|
||||
- update and fix shell completions
|
||||
- fuse: remove unneeded version check and compat code
|
||||
- delete --force: do not ask when deleting a repo, #5941
|
||||
- delete: don't commit if nothing was deleted, avoid cache sync, #6060
|
||||
- delete: add repository id and location to prompt
|
||||
- compact segments: improve freeable / freed space log output, #5679
|
||||
- if ensure_dir() fails, give more informative error message, #5952
|
||||
- load_key: no key is same as empty key, #6441
|
||||
- better error msg for defect or unsupported repo configs, #6566
|
||||
- use hmac.compare_digest instead of ==, #6470
|
||||
- implement more standard hashindex.setdefault behaviour
|
||||
- remove stray punctuation from secure-erase message
|
||||
- add development.lock.txt, use a real python 3.5 to generate frozen reqs
|
||||
- setuptools 60.7.0 breaks pyinstaller, #6246
|
||||
- setup.py clean2 was added to work around some setuptools customizability limitation.
|
||||
- allow extra compiler flags for every extension build
|
||||
- C code: make switch fallthrough explicit
|
||||
- Cython code: fix "useless trailing comma" cython warnings
|
||||
- requirements.lock.txt: use the latest cython 0.29.30
|
||||
- fix compilation warnings: ‘PyUnicode_AsUnicode’ is deprecated
|
||||
- docs:
|
||||
|
||||
- ~/.config/borg/keys is not used for repokey keys, #6107
|
||||
- excluded parent dir's metadata can't restore, #6062
|
||||
- permissions note rewritten to make it less confusing, #5490
|
||||
- add note about grandfather-father-son backup retention policy / rotation scheme
|
||||
- clarify who starts the remote agent (borg serve)
|
||||
- test/improve pull backup docs, #5903
|
||||
- document the socat pull mode described in #900 #515ß
|
||||
- borg serve: improve ssh forced commands docs, #6083
|
||||
- improve docs for borg list --format, #6080
|
||||
- fix the broken link to .nix file
|
||||
- clarify pattern usage with commands, #5176
|
||||
- clarify user_id vs uid for fuse, #5723
|
||||
- fix binary build freebsd/macOS version, #5942
|
||||
- FAQ: fix manifest-timestamp path, #6016
|
||||
- remove duplicate faq entries, #5926
|
||||
- fix sphinx warnings, #5919
|
||||
- virtualisation speed tips
|
||||
- fix values of TAG bytes, #6515
|
||||
- recommend umask for passphrase file perms
|
||||
- update link to ubuntu packages, #6485
|
||||
- clarify on-disk order and size of log entry fields, #6357
|
||||
- do not transform --/--- to unicode dashes
|
||||
- improve linking inside docs, link to borg_placeholders, link to borg_patterns
|
||||
- use same phrasing in misc. help texts
|
||||
- borg init: explain the encryption modes better
|
||||
- explain the difference between a path that ends with or without a slash, #6297
|
||||
- clarify usage of patternfile roots, #6242
|
||||
- borg key export: add examples
|
||||
- updates about features not experimental any more: FUSE "versions" view, --pattern*, #6134
|
||||
- fix/update cygwin package requirements
|
||||
- impact of deleting path/to/repo/nonce, #5858
|
||||
- warn about tampered server nonce
|
||||
- mention BORG_FILES_CACHE_SUFFIX as alternative to BORG_FILES_CACHE_TTL, #5602
|
||||
- add a troubleshooting note about "is not a valid repository" to the FAQ
|
||||
- vagrant / CI / testing:
|
||||
|
||||
- misc. fixes and updates, new python versions
|
||||
- macOS on github: re-enable fuse2 testing by downgrading to older macOS, #6099
|
||||
- fix OpenBSD symlink mode test failure, #2055
|
||||
- use the generic/openbsd6 box
|
||||
- strengthen the test: we can read data w/o nonces
|
||||
- add tests for path/to/repo/nonce deletion
|
||||
- darwin64: backport some tunings from master
|
||||
- darwin64: remove fakeroot, #6314
|
||||
- darwin64: fix vagrant scp, #5921
|
||||
- darwin64: use macfuse instead of osxfuse
|
||||
- add ubuntu "jammy" 22.04 LTS VM
|
||||
- adapt memory for openindiana64 and darwin64
|
||||
|
||||
|
||||
Version 1.1.17 (2021-07-12)
|
||||
---------------------------
|
||||
|
|
|
@ -136,8 +136,8 @@ modify it to suit your needs (e.g. more backup sets, dumping databases etc.).
|
|||
|
||||
# This is just an example, change it however you see fit
|
||||
borg create $BORG_OPTS \
|
||||
--exclude /root/.cache \
|
||||
--exclude /var/lib/docker/devicemapper \
|
||||
--exclude root/.cache \
|
||||
--exclude var/lib/docker/devicemapper \
|
||||
$TARGET::$DATE-$$-system \
|
||||
/ /boot
|
||||
|
||||
|
@ -145,7 +145,7 @@ modify it to suit your needs (e.g. more backup sets, dumping databases etc.).
|
|||
# Even if it isn't (add --exclude /home above), it probably makes sense
|
||||
# to have /home in a separate archive.
|
||||
borg create $BORG_OPTS \
|
||||
--exclude 'sh:/home/*/.cache' \
|
||||
--exclude 'sh:home/*/.cache' \
|
||||
$TARGET::$DATE-$$-home \
|
||||
/home/
|
||||
|
||||
|
|
|
@ -81,7 +81,7 @@ The options which are added to the key will perform the following:
|
|||
Due to the ``cd`` command we use, the server automatically changes the current
|
||||
working directory. Then client doesn't need to have knowledge of the absolute
|
||||
or relative remote repository path and can directly access the repositories at
|
||||
``ssh://<user>@<host>/./<repo>``.
|
||||
``<user>@<host>:<repo>``.
|
||||
|
||||
.. note:: The setup above ignores all client given commandline parameters
|
||||
which are normally appended to the `borg serve` command.
|
||||
|
@ -93,21 +93,21 @@ The client needs to initialize the `pictures` repository like this:
|
|||
|
||||
::
|
||||
|
||||
borg init ssh://backup@backup01.srv.local/./pictures
|
||||
borg init backup@backup01.srv.local:pictures
|
||||
|
||||
Or with the full path (should actually never be used, as only for demonstrational purposes).
|
||||
The server should automatically change the current working directory to the `<client fqdn>` folder.
|
||||
|
||||
::
|
||||
|
||||
borg init ssh://backup@backup01.srv.local/home/backup/repos/johndoe.clnt.local/pictures
|
||||
borg init backup@backup01.srv.local:/home/backup/repos/johndoe.clnt.local/pictures
|
||||
|
||||
When `johndoe.clnt.local` tries to access a not restricted path the following error is raised.
|
||||
John Doe tries to backup into the Web 01 path:
|
||||
|
||||
::
|
||||
|
||||
borg init ssh://backup@backup01.srv.local/home/backup/repos/web01.srv.local/pictures
|
||||
borg init backup@backup01.srv.local:/home/backup/repos/web01.srv.local/pictures
|
||||
|
||||
::
|
||||
|
||||
|
|
|
@ -98,9 +98,9 @@ create the backup, retaining the original paths, excluding the repository:
|
|||
|
||||
::
|
||||
|
||||
borg create --exclude /borgrepo --files-cache ctime,size /borgrepo::archive /
|
||||
borg create --exclude borgrepo --files-cache ctime,size /borgrepo::archive /
|
||||
|
||||
For the sake of simplicity only ``/borgrepo`` is excluded here. You may want to
|
||||
For the sake of simplicity only ``borgrepo`` is excluded here. You may want to
|
||||
set up an exclude file with additional files and folders to be excluded. Also
|
||||
note that we have to modify Borg's file change detection behaviour – SSHFS
|
||||
cannot guarantee stable inode numbers, so we have to supply the
|
||||
|
|
|
@ -45,6 +45,12 @@ repository is only modified from one place. Also keep in mind that
|
|||
Borg will keep an exclusive lock on the repository while creating
|
||||
or deleting archives, which may make *simultaneous* backups fail.
|
||||
|
||||
Can I back up to multiple, swapped backup targets?
|
||||
--------------------------------------------------
|
||||
|
||||
It is possible to swap your backup disks if each backup medium is assigned its
|
||||
own repository by creating a new one with :ref:`borg_init`.
|
||||
|
||||
Can I copy or synchronize my repo to another location?
|
||||
------------------------------------------------------
|
||||
|
||||
|
@ -430,7 +436,7 @@ Say you want to prune ``/var/log`` faster than the rest of
|
|||
archive *names* and then implement different prune policies for
|
||||
different prefixes. For example, you could have a script that does::
|
||||
|
||||
borg create --exclude /var/log $REPOSITORY:main-$(date +%Y-%m-%d) /
|
||||
borg create --exclude var/log $REPOSITORY:main-$(date +%Y-%m-%d) /
|
||||
borg create $REPOSITORY:logs-$(date +%Y-%m-%d) /var/log
|
||||
|
||||
Then you would have two different prune calls with different policies::
|
||||
|
|
|
@ -371,7 +371,7 @@ While we try not to break master, there are no guarantees on anything.
|
|||
git clone https://github.com/borgbackup/borg.git
|
||||
|
||||
# create a virtual environment
|
||||
virtualenv --python=${which python3} borg-env
|
||||
virtualenv --python=$(which python3) borg-env
|
||||
source borg-env/bin/activate # always before using!
|
||||
|
||||
# install borg + dependencies into virtualenv
|
||||
|
|
|
@ -182,14 +182,14 @@ backed up and that the ``prune`` command is keeping and deleting the correct bac
|
|||
--show-rc \
|
||||
--compression lz4 \
|
||||
--exclude-caches \
|
||||
--exclude '/home/*/.cache/*' \
|
||||
--exclude '/var/tmp/*' \
|
||||
--exclude 'home/*/.cache/*' \
|
||||
--exclude 'var/tmp/*' \
|
||||
\
|
||||
::'{hostname}-{now}' \
|
||||
/etc \
|
||||
/home \
|
||||
/root \
|
||||
/var \
|
||||
/var
|
||||
|
||||
backup_exit=$?
|
||||
|
||||
|
@ -206,7 +206,7 @@ backed up and that the ``prune`` command is keeping and deleting the correct bac
|
|||
--show-rc \
|
||||
--keep-daily 7 \
|
||||
--keep-weekly 4 \
|
||||
--keep-monthly 6 \
|
||||
--keep-monthly 6
|
||||
|
||||
prune_exit=$?
|
||||
|
||||
|
@ -405,7 +405,7 @@ Borg can initialize and access repositories on remote hosts if the
|
|||
host is accessible using SSH. This is fastest and easiest when Borg
|
||||
is installed on the remote host, in which case the following syntax is used::
|
||||
|
||||
$ borg init ssh://user@hostname/path/to/repo
|
||||
$ borg init user@hostname:/path/to/repo
|
||||
|
||||
Note: please see the usage chapter for a full documentation of repo URLs.
|
||||
|
||||
|
|
|
@ -19,7 +19,7 @@ Examples
|
|||
# Backup home directories excluding image thumbnails (i.e. only
|
||||
# /home/<one directory>/.thumbnails is excluded, not /home/*/*/.thumbnails etc.)
|
||||
$ borg create /path/to/repo::my-files /home \
|
||||
--exclude 'sh:/home/*/.thumbnails'
|
||||
--exclude 'sh:home/*/.thumbnails'
|
||||
|
||||
# Backup the root filesystem into an archive named "root-YYYY-MM-DD"
|
||||
# use zlib compression (good, but slow) - default is lz4 (fast, low compression ratio)
|
||||
|
|
|
@ -14,16 +14,32 @@ Note: you may also prepend a ``file://`` to a filesystem path to get URL style.
|
|||
|
||||
**Remote repositories** accessed via ssh user@host:
|
||||
|
||||
``ssh://user@host:port/path/to/repo`` - remote repo, absolute path
|
||||
``user@host:/path/to/repo`` - remote repo, absolute path
|
||||
|
||||
``ssh://user@host:port/path/to/repo`` - same, alternative syntax, port can be given
|
||||
|
||||
|
||||
**Remote repositories with relative paths** can be given using this syntax:
|
||||
|
||||
``user@host:path/to/repo`` - path relative to current directory
|
||||
|
||||
``user@host:~/path/to/repo`` - path relative to user's home directory
|
||||
|
||||
``user@host:~other/path/to/repo`` - path relative to other's home directory
|
||||
|
||||
Note: giving ``user@host:/./path/to/repo`` or ``user@host:/~/path/to/repo`` or
|
||||
``user@host:/~other/path/to/repo`` is also supported, but not required here.
|
||||
|
||||
|
||||
**Remote repositories with relative paths, alternative syntax with port**:
|
||||
|
||||
``ssh://user@host:port/./path/to/repo`` - path relative to current directory
|
||||
|
||||
``ssh://user@host:port/~/path/to/repo`` - path relative to user's home directory
|
||||
|
||||
``ssh://user@host:port/~other/path/to/repo`` - path relative to other's home directory
|
||||
|
||||
|
||||
If you frequently need the same repo URL, it is a good idea to set the
|
||||
``BORG_REPO`` environment variable to set a default for the repo URL:
|
||||
|
||||
|
@ -36,31 +52,3 @@ to use the default - it will be read from BORG_REPO then.
|
|||
|
||||
Use ``::`` syntax to give the repo URL when syntax requires giving a positional
|
||||
argument for the repo (e.g. ``borg mount :: /mnt``).
|
||||
|
||||
Converting from scp-style repo URLs
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Borg does not support scp-style repo URLs any more.
|
||||
|
||||
Here is how you can convert to URL style:
|
||||
|
||||
::
|
||||
|
||||
user@host:path/to/repo # relative to cwd
|
||||
-->
|
||||
ssh://user@host:22/./path/to/repo # relative to cwd
|
||||
or (usually the cwd is the home dir after ssh login)
|
||||
ssh://user@host:22/~/path/to/repo # relative to home dir
|
||||
|
||||
user@host:/path/to/repo # absolute repo path
|
||||
-->
|
||||
ssh://user@host:22/path/to/repo # absolute repo path
|
||||
|
||||
Notes:
|
||||
|
||||
Port 22 is the default, so you can omit the ``:22`` if you like.
|
||||
|
||||
If you used some hack to use a non-standard port (which was not directly
|
||||
supported by the scp-style repo specification), you can now do it that way
|
||||
with the ``ssh:`` URL and remove the hack (usually in ssh configuration or
|
||||
given via ``--rsh`` borg option).
|
||||
|
|
|
@ -15,8 +15,8 @@ Examples
|
|||
|
||||
# Remote repository (accesses a remote borg via ssh)
|
||||
# repokey: stores the (encrypted) key into <REPO_DIR>/config
|
||||
$ borg init --encryption=repokey-aes-ocb ssh://user@hostname/./backup
|
||||
$ borg init --encryption=repokey-aes-ocb user@hostname:backup
|
||||
|
||||
# Remote repository (accesses a remote borg via ssh)
|
||||
# keyfile: stores the (encrypted) key into ~/.config/borg/keys/
|
||||
$ borg init --encryption=keyfile-aes-ocb ssh://user@hostname/./backup
|
||||
$ borg init --encryption=keyfile-aes-ocb user@hostname:backup
|
||||
|
|
|
@ -1,13 +1,13 @@
|
|||
setuptools==60.7.1
|
||||
setuptools==62.1.0
|
||||
setuptools-scm==6.4.2
|
||||
pip==22.0.3
|
||||
virtualenv==20.13.0
|
||||
pip==22.1.2
|
||||
virtualenv==20.14.1
|
||||
pkgconfig==1.5.5
|
||||
tox==3.24.5
|
||||
pytest==7.0.0
|
||||
tox==3.25.0
|
||||
pytest==7.0.1
|
||||
pytest-xdist==2.5.0
|
||||
pytest-cov==3.0.0
|
||||
pytest-benchmark==3.4.1
|
||||
Cython==0.29.27
|
||||
Cython==0.29.30
|
||||
twine==3.8.0
|
||||
python-dateutil==2.8.2
|
||||
|
|
|
@ -20,11 +20,12 @@ classifiers =
|
|||
Programming Language :: Python :: 3
|
||||
Programming Language :: Python :: 3.9
|
||||
Programming Language :: Python :: 3.10
|
||||
Programming Language :: Python :: 3.11
|
||||
Topic :: Security :: Cryptography
|
||||
Topic :: System :: Archiving :: Backup
|
||||
platforms = Linux, MacOS X, FreeBSD, OpenBSD, NetBSD
|
||||
license = BSD
|
||||
license_file = LICENSE
|
||||
license_files = LICENSE
|
||||
project_urls =
|
||||
Bug Tracker = https://github.com/borgbackup/borg/issues
|
||||
Documentation = https://borgbackup.readthedocs.io
|
||||
|
@ -38,7 +39,7 @@ python_requires = >=3.9
|
|||
setup_requires =
|
||||
setuptools_scm[toml] >= 6.2
|
||||
install_requires =
|
||||
msgpack >=1.0.3, <=1.0.3
|
||||
msgpack >=1.0.3, <=1.0.4
|
||||
packaging
|
||||
argon2-cffi
|
||||
tests_require =
|
||||
|
|
|
@ -199,7 +199,7 @@ def with_repository(fake=False, invert_fake=False, create=False, lock=True,
|
|||
if cache:
|
||||
with Cache(repository, kwargs['key'], kwargs['manifest'],
|
||||
progress=getattr(args, 'progress', False), lock_wait=self.lock_wait,
|
||||
cache_mode=getattr(args, 'files_cache_mode', DEFAULT_FILES_CACHE_MODE),
|
||||
cache_mode=getattr(args, 'files_cache_mode', FILES_CACHE_MODE_DISABLED),
|
||||
consider_part_files=getattr(args, 'consider_part_files', False),
|
||||
iec=getattr(args, 'iec', False)) as cache_:
|
||||
return method(self, args, repository=repository, cache=cache_, **kwargs)
|
||||
|
@ -243,7 +243,7 @@ def with_other_repository(manifest=False, key=False, cache=False, compatibility=
|
|||
if cache:
|
||||
with Cache(repository, key_, manifest_,
|
||||
progress=False, lock_wait=self.lock_wait,
|
||||
cache_mode=getattr(args, 'files_cache_mode', DEFAULT_FILES_CACHE_MODE),
|
||||
cache_mode=getattr(args, 'files_cache_mode', FILES_CACHE_MODE_DISABLED),
|
||||
consider_part_files=getattr(args, 'consider_part_files', False),
|
||||
iec=getattr(args, 'iec', False)) as cache_:
|
||||
kwargs['other_cache'] = cache_
|
||||
|
@ -757,7 +757,7 @@ class Archiver:
|
|||
key_96 = os.urandom(12)
|
||||
|
||||
import io
|
||||
from borg.chunker import get_chunker
|
||||
from .chunker import get_chunker
|
||||
print("Chunkers =======================================================")
|
||||
size = "1GB"
|
||||
|
||||
|
@ -774,7 +774,7 @@ class Archiver:
|
|||
print(f"{spec:<24} {size:<10} {timeit(func, number=100):.3f}s")
|
||||
|
||||
import zlib
|
||||
from borg.checksums import crc32, deflate_crc32, xxh64
|
||||
from .checksums import crc32, deflate_crc32, xxh64
|
||||
print("Non-cryptographic checksums / hashes ===========================")
|
||||
size = "1GB"
|
||||
tests = [
|
||||
|
@ -791,7 +791,7 @@ class Archiver:
|
|||
for spec, func in tests:
|
||||
print(f"{spec:<24} {size:<10} {timeit(func, number=100):.3f}s")
|
||||
|
||||
from borg.crypto.low_level import hmac_sha256, blake2b_256
|
||||
from .crypto.low_level import hmac_sha256, blake2b_256
|
||||
print("Cryptographic hashes / MACs ====================================")
|
||||
size = "1GB"
|
||||
for spec, func in [
|
||||
|
@ -800,8 +800,8 @@ class Archiver:
|
|||
]:
|
||||
print(f"{spec:<24} {size:<10} {timeit(func, number=100):.3f}s")
|
||||
|
||||
from borg.crypto.low_level import AES256_CTR_BLAKE2b, AES256_CTR_HMAC_SHA256
|
||||
from borg.crypto.low_level import AES256_OCB, CHACHA20_POLY1305
|
||||
from .crypto.low_level import AES256_CTR_BLAKE2b, AES256_CTR_HMAC_SHA256
|
||||
from .crypto.low_level import AES256_OCB, CHACHA20_POLY1305
|
||||
print("Encryption =====================================================")
|
||||
size = "1GB"
|
||||
|
||||
|
@ -826,7 +826,7 @@ class Archiver:
|
|||
]:
|
||||
print(f"{spec:<24} {count:<10} {timeit(func, number=count):.3f}s")
|
||||
|
||||
from borg.compress import CompressionSpec
|
||||
from .compress import CompressionSpec
|
||||
print("Compression ====================================================")
|
||||
for spec in [
|
||||
'lz4',
|
||||
|
@ -914,6 +914,7 @@ class Archiver:
|
|||
pipe_bin = sys.stdin.buffer
|
||||
pipe = TextIOWrapper(pipe_bin, errors='surrogateescape')
|
||||
for path in iter_separated(pipe, paths_sep):
|
||||
path = os.path.normpath(path)
|
||||
try:
|
||||
with backup_io('stat'):
|
||||
st = os_stat(path=path, parent_fd=None, name=None, follow_symlinks=False)
|
||||
|
@ -2351,7 +2352,7 @@ class Archiver:
|
|||
key = key_factory(repository, cdata)
|
||||
break
|
||||
i = 0
|
||||
for id, cdata, tag, segment, offset in repository.scan_low_level():
|
||||
for id, cdata, tag, segment, offset in repository.scan_low_level(segment=args.segment, offset=args.offset):
|
||||
if tag == TAG_PUT:
|
||||
decrypt_dump(i, id, cdata, tag='put', segment=segment, offset=offset)
|
||||
elif tag == TAG_DELETE:
|
||||
|
@ -2546,42 +2547,35 @@ class Archiver:
|
|||
|
||||
helptext = collections.OrderedDict()
|
||||
helptext['patterns'] = textwrap.dedent('''
|
||||
The path/filenames used as input for the pattern matching start from the
|
||||
currently active recursion root. You usually give the recursion root(s)
|
||||
when invoking borg and these can be either relative or absolute paths.
|
||||
When specifying one or more file paths in a Borg command that supports
|
||||
patterns for the respective option or argument, you can apply the
|
||||
patterns described here to include only desired files and/or exclude
|
||||
unwanted ones. Patterns can be used
|
||||
|
||||
So, when you give `relative/` as root, the paths going into the matcher
|
||||
will look like `relative/.../file.ext`. When you give `/absolute/` as
|
||||
root, they will look like `/absolute/.../file.ext`.
|
||||
- for ``--exclude`` option,
|
||||
- in the file given with ``--exclude-from`` option,
|
||||
- for ``--pattern`` option,
|
||||
- in the file given with ``--patterns-from`` option and
|
||||
- for ``PATH`` arguments that explicitly support them.
|
||||
|
||||
File paths in Borg archives are always stored normalized and relative.
|
||||
This means that e.g. ``borg create /path/to/repo ../some/path`` will
|
||||
store all files as `some/path/.../file.ext` and ``borg create
|
||||
/path/to/repo /home/user`` will store all files as
|
||||
`home/user/.../file.ext`.
|
||||
Borg always stores all file paths normalized and relative to the
|
||||
current recursion root. The recursion root is also named ``PATH`` in
|
||||
Borg commands like `borg create` that do a file discovery, so do not
|
||||
confuse the root with the ``PATH`` argument of e.g. `borg extract`.
|
||||
|
||||
A directory exclusion pattern can end either with or without a slash ('/').
|
||||
If it ends with a slash, such as `some/path/`, the directory will be
|
||||
included but not its content. If it does not end with a slash, such as
|
||||
`some/path`, both the directory and content will be excluded.
|
||||
Starting with Borg 1.2, paths that are matched against patterns always
|
||||
appear relative. If you give ``/absolute/`` as root, the paths going
|
||||
into the matcher will look relative like ``absolute/.../file.ext``.
|
||||
If you give ``../some/path`` as root, the paths will look like
|
||||
``some/path/.../file.ext``.
|
||||
|
||||
File patterns support these styles: fnmatch, shell, regular expressions,
|
||||
path prefixes and path full-matches. By default, fnmatch is used for
|
||||
``--exclude`` patterns and shell-style is used for the ``--pattern``
|
||||
option. For commands that support patterns in their ``PATH`` argument
|
||||
like (``borg list``), the default pattern is path prefix.
|
||||
File patterns support five different styles. If followed by a colon ':',
|
||||
the first two characters of a pattern are used as a style selector.
|
||||
Explicit style selection is necessary if a non-default style is desired
|
||||
or when the desired pattern starts with two alphanumeric characters
|
||||
followed by a colon (i.e. ``aa:something/*``).
|
||||
|
||||
Starting with Borg 1.2, for all but regular expression pattern matching
|
||||
styles, all paths are treated as relative, meaning that a leading path
|
||||
separator is removed after normalizing and before matching. This allows
|
||||
you to use absolute or relative patterns arbitrarily.
|
||||
|
||||
If followed by a colon (':') the first two characters of a pattern are
|
||||
used as a style selector. Explicit style selection is necessary when a
|
||||
non-default style is desired or when the desired pattern starts with
|
||||
two alphanumeric characters followed by a colon (i.e. `aa:something/*`).
|
||||
|
||||
`Fnmatch <https://docs.python.org/3/library/fnmatch.html>`_, selector `fm:`
|
||||
`Fnmatch <https://docs.python.org/3/library/fnmatch.html>`_, selector ``fm:``
|
||||
This is the default style for ``--exclude`` and ``--exclude-from``.
|
||||
These patterns use a variant of shell pattern syntax, with '\\*' matching
|
||||
any number of characters, '?' matching any single character, '[...]'
|
||||
|
@ -2589,7 +2583,7 @@ class Archiver:
|
|||
matching any character not specified. For the purpose of these patterns,
|
||||
the path separator (backslash for Windows and '/' on other systems) is not
|
||||
treated specially. Wrap meta-characters in brackets for a literal
|
||||
match (i.e. `[?]` to match the literal character `?`). For a path
|
||||
match (i.e. ``[?]`` to match the literal character '?'). For a path
|
||||
to match a pattern, the full path must match, or it must match
|
||||
from the start of the full path to just before a path separator. Except
|
||||
for the root path, paths will never end in the path separator when
|
||||
|
@ -2597,33 +2591,31 @@ class Archiver:
|
|||
separator, a '\\*' is appended before matching is attempted. A leading
|
||||
path separator is always removed.
|
||||
|
||||
Shell-style patterns, selector `sh:`
|
||||
Shell-style patterns, selector ``sh:``
|
||||
This is the default style for ``--pattern`` and ``--patterns-from``.
|
||||
Like fnmatch patterns these are similar to shell patterns. The difference
|
||||
is that the pattern may include `**/` for matching zero or more directory
|
||||
levels, `*` for matching zero or more arbitrary characters with the
|
||||
is that the pattern may include ``**/`` for matching zero or more directory
|
||||
levels, ``*`` for matching zero or more arbitrary characters with the
|
||||
exception of any path separator. A leading path separator is always removed.
|
||||
|
||||
Regular expressions, selector `re:`
|
||||
Regular expressions similar to those found in Perl are supported. Unlike
|
||||
shell patterns regular expressions are not required to match the full
|
||||
`Regular expressions <https://docs.python.org/3/library/re.html>`_, selector ``re:``
|
||||
Unlike shell patterns, regular expressions are not required to match the full
|
||||
path and any substring match is sufficient. It is strongly recommended to
|
||||
anchor patterns to the start ('^'), to the end ('$') or both. Path
|
||||
separators (backslash for Windows and '/' on other systems) in paths are
|
||||
always normalized to a forward slash ('/') before applying a pattern. The
|
||||
regular expression syntax is described in the `Python documentation for
|
||||
the re module <https://docs.python.org/3/library/re.html>`_.
|
||||
always normalized to a forward slash '/' before applying a pattern.
|
||||
|
||||
Path prefix, selector `pp:`
|
||||
Path prefix, selector ``pp:``
|
||||
This pattern style is useful to match whole sub-directories. The pattern
|
||||
`pp:root/somedir` matches `root/somedir` and everything therein. A leading
|
||||
path separator is always removed.
|
||||
``pp:root/somedir`` matches ``root/somedir`` and everything therein.
|
||||
A leading path separator is always removed.
|
||||
|
||||
Path full-match, selector `pf:`
|
||||
Path full-match, selector ``pf:``
|
||||
This pattern style is (only) useful to match full paths.
|
||||
This is kind of a pseudo pattern as it can not have any variable or
|
||||
unspecified parts - the full path must be given. `pf:root/file.ext` matches
|
||||
`root/file.ext` only. A leading path separator is always removed.
|
||||
unspecified parts - the full path must be given. ``pf:root/file.ext``
|
||||
matches ``root/file.ext`` only. A leading path separator is always
|
||||
removed.
|
||||
|
||||
Implementation note: this is implemented via very time-efficient O(1)
|
||||
hashtable lookups (this means you can have huge amounts of such patterns
|
||||
|
@ -2636,20 +2628,20 @@ class Archiver:
|
|||
|
||||
.. note::
|
||||
|
||||
`re:`, `sh:` and `fm:` patterns are all implemented on top of the Python SRE
|
||||
engine. It is very easy to formulate patterns for each of these types which
|
||||
requires an inordinate amount of time to match paths. If untrusted users
|
||||
are able to supply patterns, ensure they cannot supply `re:` patterns.
|
||||
Further, ensure that `sh:` and `fm:` patterns only contain a handful of
|
||||
wildcards at most.
|
||||
``re:``, ``sh:`` and ``fm:`` patterns are all implemented on top of
|
||||
the Python SRE engine. It is very easy to formulate patterns for each
|
||||
of these types which requires an inordinate amount of time to match
|
||||
paths. If untrusted users are able to supply patterns, ensure they
|
||||
cannot supply ``re:`` patterns. Further, ensure that ``sh:`` and
|
||||
``fm:`` patterns only contain a handful of wildcards at most.
|
||||
|
||||
Exclusions can be passed via the command line option ``--exclude``. When used
|
||||
from within a shell, the patterns should be quoted to protect them from
|
||||
expansion.
|
||||
|
||||
The ``--exclude-from`` option permits loading exclusion patterns from a text
|
||||
file with one pattern per line. Lines empty or starting with the number sign
|
||||
('#') after removing whitespace on both ends are ignored. The optional style
|
||||
file with one pattern per line. Lines empty or starting with the hash sign
|
||||
'#' after removing whitespace on both ends are ignored. The optional style
|
||||
selector prefix is also supported for patterns loaded from a file. Due to
|
||||
whitespace removal, paths with whitespace at the beginning or end can only be
|
||||
excluded using regular expressions.
|
||||
|
@ -2660,63 +2652,83 @@ class Archiver:
|
|||
Examples::
|
||||
|
||||
# Exclude '/home/user/file.o' but not '/home/user/file.odt':
|
||||
$ borg create -e '*.o' backup /
|
||||
$ borg create -e '*.o' /path/to/repo::archive /
|
||||
|
||||
# Exclude '/home/user/junk' and '/home/user/subdir/junk' but
|
||||
# not '/home/user/importantjunk' or '/etc/junk':
|
||||
$ borg create -e '/home/*/junk' backup /
|
||||
$ borg create -e 'home/*/junk' /path/to/repo::archive /
|
||||
|
||||
# Exclude the contents of '/home/user/cache' but not the directory itself:
|
||||
$ borg create -e home/user/cache/ backup /
|
||||
$ borg create -e home/user/cache/ /path/to/repo::archive /
|
||||
|
||||
# The file '/home/user/cache/important' is *not* backed up:
|
||||
$ borg create -e /home/user/cache/ backup / /home/user/cache/important
|
||||
$ borg create -e home/user/cache/ /path/to/repo::archive / /home/user/cache/important
|
||||
|
||||
# The contents of directories in '/home' are not backed up when their name
|
||||
# ends in '.tmp'
|
||||
$ borg create --exclude 're:^/home/[^/]+\\.tmp/' backup /
|
||||
$ borg create --exclude 're:^home/[^/]+\\.tmp/' /path/to/repo::archive /
|
||||
|
||||
# Load exclusions from file
|
||||
$ cat >exclude.txt <<EOF
|
||||
# Comment line
|
||||
/home/*/junk
|
||||
home/*/junk
|
||||
*.tmp
|
||||
fm:aa:something/*
|
||||
re:^/home/[^/]+\\.tmp/
|
||||
sh:/home/*/.thumbnails
|
||||
re:^home/[^/]+\\.tmp/
|
||||
sh:home/*/.thumbnails
|
||||
# Example with spaces, no need to escape as it is processed by borg
|
||||
some file with spaces.txt
|
||||
EOF
|
||||
$ borg create --exclude-from exclude.txt backup /
|
||||
$ borg create --exclude-from exclude.txt /path/to/repo::archive /
|
||||
|
||||
A more general and easier to use way to define filename matching patterns exists
|
||||
with the ``--pattern`` and ``--patterns-from`` options. Using these, you may
|
||||
specify the backup roots (starting points) and patterns for inclusion/exclusion.
|
||||
A root path starts with the prefix `R`, followed by a path (a plain path, not a
|
||||
file pattern). An include rule starts with the prefix +, an exclude rule starts
|
||||
with the prefix -, an exclude-norecurse rule starts with !, all followed by a pattern.
|
||||
A more general and easier to use way to define filename matching patterns
|
||||
exists with the ``--pattern`` and ``--patterns-from`` options. Using
|
||||
these, you may specify the backup roots, default pattern styles and
|
||||
patterns for inclusion and exclusion.
|
||||
|
||||
Root path prefix ``R``
|
||||
A recursion root path starts with the prefix ``R``, followed by a path
|
||||
(a plain path, not a file pattern). Use this prefix to have the root
|
||||
paths in the patterns file rather than as command line arguments.
|
||||
|
||||
Pattern style prefix ``P``
|
||||
To change the default pattern style, use the ``P`` prefix, followed by
|
||||
the pattern style abbreviation (``fm``, ``pf``, ``pp``, ``re``, ``sh``).
|
||||
All patterns following this line will use this style until another style
|
||||
is specified.
|
||||
|
||||
Exclude pattern prefix ``-``
|
||||
Use the prefix ``-``, followed by a pattern, to define an exclusion.
|
||||
This has the same effect as the ``--exclude`` option.
|
||||
|
||||
Exclude no-recurse pattern prefix ``!``
|
||||
Use the prefix ``!``, followed by a pattern, to define an exclusion
|
||||
that does not recurse into subdirectories. This saves time, but
|
||||
prevents include patterns to match any files in subdirectories.
|
||||
|
||||
Include pattern prefix ``+``
|
||||
Use the prefix ``+``, followed by a pattern, to define inclusions.
|
||||
This is useful to include paths that are covered in an exclude
|
||||
pattern and would otherwise not be backed up.
|
||||
|
||||
The first matching pattern is used, so if an include pattern matches
|
||||
before an exclude pattern, the file is backed up. Note that a no-recurse
|
||||
exclude stops examination of subdirectories so that potential includes
|
||||
will not match - use normal exludes for such use cases.
|
||||
|
||||
**Tip: You can easily test your patterns with --dry-run and --list**::
|
||||
|
||||
$ borg create --dry-run --list --patterns-from patterns.txt /path/to/repo::archive
|
||||
|
||||
This will list the considered files one per line, prefixed with a
|
||||
character that indicates the action (e.g. 'x' for excluding, see
|
||||
**Item flags** in `borg create` usage docs).
|
||||
|
||||
.. note::
|
||||
|
||||
Via ``--pattern`` or ``--patterns-from`` you can define BOTH inclusion and exclusion
|
||||
of files using pattern prefixes ``+`` and ``-``. With ``--exclude`` and
|
||||
``--exclude-from`` ONLY excludes are defined.
|
||||
|
||||
Inclusion patterns are useful to include paths that are contained in an excluded
|
||||
path. The first matching pattern is used so if an include pattern matches before
|
||||
an exclude pattern, the file is backed up. If an exclude-norecurse pattern matches
|
||||
a directory, it won't recurse into it and won't discover any potential matches for
|
||||
include rules below that directory.
|
||||
|
||||
.. note::
|
||||
|
||||
It's possible that a sub-directory/file is matched while parent directories are not.
|
||||
In that case, parent directories are not backed up thus their user, group, permission,
|
||||
etc. can not be restored.
|
||||
|
||||
Note that the default pattern style for ``--pattern`` and ``--patterns-from`` is
|
||||
shell style (`sh:`), so those patterns behave similar to rsync include/exclude
|
||||
patterns. The pattern style can be set via the `P` prefix.
|
||||
It's possible that a sub-directory/file is matched while parent
|
||||
directories are not. In that case, parent directories are not backed
|
||||
up and thus their user, group, permission, etc. cannot be restored.
|
||||
|
||||
Patterns (``--pattern``) and excludes (``--exclude``) from the command line are
|
||||
considered first (in the order of appearance). Then patterns from ``--patterns-from``
|
||||
|
@ -2726,45 +2738,45 @@ class Archiver:
|
|||
|
||||
# backup pics, but not the ones from 2018, except the good ones:
|
||||
# note: using = is essential to avoid cmdline argument parsing issues.
|
||||
borg create --pattern=+pics/2018/good --pattern=-pics/2018 repo::arch pics
|
||||
borg create --pattern=+pics/2018/good --pattern=-pics/2018 /path/to/repo::archive pics
|
||||
|
||||
# use a file with patterns:
|
||||
borg create --patterns-from patterns.lst repo::arch
|
||||
# backup only JPG/JPEG files (case insensitive) in all home directories:
|
||||
borg create --pattern '+ re:\\.jpe?g(?i)$' /path/to/repo::archive /home
|
||||
|
||||
# backup homes, but exclude big downloads (like .ISO files) or hidden files:
|
||||
borg create --exclude 're:\\.iso(?i)$' --exclude 'sh:home/**/.*' /path/to/repo::archive /home
|
||||
|
||||
# use a file with patterns (recursion root '/' via command line):
|
||||
borg create --patterns-from patterns.lst /path/to/repo::archive /
|
||||
|
||||
The patterns.lst file could look like that::
|
||||
|
||||
# "sh:" pattern style is the default, so the following line is not needed:
|
||||
P sh
|
||||
R /
|
||||
# can be rebuild
|
||||
- /home/*/.cache
|
||||
# they're downloads for a reason
|
||||
- /home/*/Downloads
|
||||
# susan is a nice person
|
||||
# "sh:" pattern style is the default
|
||||
# exclude caches
|
||||
- home/*/.cache
|
||||
# include susans home
|
||||
+ /home/susan
|
||||
+ home/susan
|
||||
# also back up this exact file
|
||||
+ pf:/home/bobby/specialfile.txt
|
||||
+ pf:home/bobby/specialfile.txt
|
||||
# don't backup the other home directories
|
||||
- /home/*
|
||||
# don't even look in /proc
|
||||
! /proc
|
||||
- home/*
|
||||
# don't even look in /dev, /proc, /run, /sys, /tmp (note: would exclude files like /device, too)
|
||||
! re:^(dev|proc|run|sys|tmp)
|
||||
|
||||
You can specify recursion roots either on the command line or in a patternfile::
|
||||
|
||||
# these two commands do the same thing
|
||||
borg create --exclude /home/bobby/junk repo::arch /home/bobby /home/susan
|
||||
borg create --patterns-from patternfile.lst repo::arch
|
||||
borg create --exclude home/bobby/junk /path/to/repo::archive /home/bobby /home/susan
|
||||
borg create --patterns-from patternfile.lst /path/to/repo::archive
|
||||
|
||||
The patternfile::
|
||||
patternfile.lst::
|
||||
|
||||
# note that excludes use fm: by default and patternfiles use sh: by default.
|
||||
# therefore, we need to specify fm: to have the same exact behavior.
|
||||
P fm
|
||||
R /home/bobby
|
||||
R /home/susan
|
||||
|
||||
- /home/bobby/junk
|
||||
- home/bobby/junk
|
||||
|
||||
This allows you to share the same patterns between multiple repositories
|
||||
without needing to specify them on the command line.\n\n''')
|
||||
|
@ -3849,8 +3861,8 @@ class Archiver:
|
|||
fs_group.add_argument('--sparse', dest='sparse', action='store_true',
|
||||
help='detect sparse holes in input (supported only by fixed chunker)')
|
||||
fs_group.add_argument('--files-cache', metavar='MODE', dest='files_cache_mode', action=Highlander,
|
||||
type=FilesCacheMode, default=DEFAULT_FILES_CACHE_MODE_UI,
|
||||
help='operate files cache in MODE. default: %s' % DEFAULT_FILES_CACHE_MODE_UI)
|
||||
type=FilesCacheMode, default=FILES_CACHE_MODE_UI_DEFAULT,
|
||||
help='operate files cache in MODE. default: %s' % FILES_CACHE_MODE_UI_DEFAULT)
|
||||
fs_group.add_argument('--read-special', dest='read_special', action='store_true',
|
||||
help='open and read block and char device files as well as FIFOs as if they were '
|
||||
'regular files. Also follows symlinks pointing to these kinds of files.')
|
||||
|
@ -3958,6 +3970,10 @@ class Archiver:
|
|||
subparser.set_defaults(func=self.do_debug_dump_repo_objs)
|
||||
subparser.add_argument('--ghost', dest='ghost', action='store_true',
|
||||
help='dump all segment file contents, including deleted/uncommitted objects and commits.')
|
||||
subparser.add_argument('--segment', metavar='SEG', dest='segment', default=None, type=positive_int_validator,
|
||||
help='used together with --ghost: limit processing to given segment.')
|
||||
subparser.add_argument('--offset', metavar='OFFS', dest='offset', default=None, type=positive_int_validator,
|
||||
help='used together with --ghost: limit processing to given offset.')
|
||||
|
||||
debug_search_repo_objs_epilog = process_epilog("""
|
||||
This command searches raw (but decrypted and decompressed) repo objects for a specific bytes sequence.
|
||||
|
|
|
@ -12,7 +12,7 @@ logger = create_logger()
|
|||
|
||||
files_cache_logger = create_logger('borg.debug.files_cache')
|
||||
|
||||
from .constants import CACHE_README, DEFAULT_FILES_CACHE_MODE
|
||||
from .constants import CACHE_README, FILES_CACHE_MODE_DISABLED
|
||||
from .hashindex import ChunkIndex, ChunkIndexEntry, CacheSynchronizer
|
||||
from .helpers import Location
|
||||
from .helpers import Error
|
||||
|
@ -371,7 +371,7 @@ class Cache:
|
|||
shutil.rmtree(path)
|
||||
|
||||
def __new__(cls, repository, key, manifest, path=None, sync=True, warn_if_unencrypted=True,
|
||||
progress=False, lock_wait=None, permit_adhoc_cache=False, cache_mode=DEFAULT_FILES_CACHE_MODE,
|
||||
progress=False, lock_wait=None, permit_adhoc_cache=False, cache_mode=FILES_CACHE_MODE_DISABLED,
|
||||
consider_part_files=False, iec=False):
|
||||
|
||||
def local():
|
||||
|
@ -449,7 +449,7 @@ class LocalCache(CacheStatsMixin):
|
|||
"""
|
||||
|
||||
def __init__(self, repository, key, manifest, path=None, sync=True, warn_if_unencrypted=True,
|
||||
progress=False, lock_wait=None, cache_mode=DEFAULT_FILES_CACHE_MODE, consider_part_files=False,
|
||||
progress=False, lock_wait=None, cache_mode=FILES_CACHE_MODE_DISABLED, consider_part_files=False,
|
||||
iec=False):
|
||||
"""
|
||||
:param warn_if_unencrypted: print warning if accessing unknown unencrypted repository
|
||||
|
|
|
@ -84,8 +84,8 @@ ITEMS_CHUNKER_PARAMS = (CH_BUZHASH, 15, 19, 17, HASH_WINDOW_SIZE)
|
|||
CH_DATA, CH_ALLOC, CH_HOLE = 0, 1, 2
|
||||
|
||||
# operating mode of the files cache (for fast skipping of unchanged files)
|
||||
DEFAULT_FILES_CACHE_MODE_UI = 'ctime,size,inode' # default for "borg create" command (CLI UI)
|
||||
DEFAULT_FILES_CACHE_MODE = 'd' # most borg commands do not use the files cache at all (disable)
|
||||
FILES_CACHE_MODE_UI_DEFAULT = 'ctime,size,inode' # default for "borg create" command (CLI UI)
|
||||
FILES_CACHE_MODE_DISABLED = 'd' # most borg commands do not use the files cache at all (disable)
|
||||
|
||||
# return codes returned by borg command
|
||||
# when borg is killed by signal N, rc = 128 + N
|
||||
|
|
|
@ -23,15 +23,14 @@ class ExtensionModuleError(Error):
|
|||
|
||||
|
||||
def check_extension_modules():
|
||||
import borg.crypto.low_level
|
||||
from .. import platform, compress, item, chunker, hashindex
|
||||
from .. import platform, compress, crypto, item, chunker, hashindex
|
||||
if hashindex.API_VERSION != '1.2_01':
|
||||
raise ExtensionModuleError
|
||||
if chunker.API_VERSION != '1.2_01':
|
||||
raise ExtensionModuleError
|
||||
if compress.API_VERSION != '1.2_02':
|
||||
raise ExtensionModuleError
|
||||
if borg.crypto.low_level.API_VERSION != '1.3_01':
|
||||
if crypto.low_level.API_VERSION != '1.3_01':
|
||||
raise ExtensionModuleError
|
||||
if item.API_VERSION != '1.2_01':
|
||||
raise ExtensionModuleError
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
from ..constants import * # NOQA
|
||||
|
||||
import borg.crypto.low_level
|
||||
from ..crypto.low_level import IntegrityError as IntegrityErrorBase
|
||||
|
||||
|
||||
class Error(Exception):
|
||||
|
@ -30,7 +30,7 @@ class ErrorWithTraceback(Error):
|
|||
traceback = True
|
||||
|
||||
|
||||
class IntegrityError(ErrorWithTraceback, borg.crypto.low_level.IntegrityError):
|
||||
class IntegrityError(ErrorWithTraceback, IntegrityErrorBase):
|
||||
"""Data integrity error: {}"""
|
||||
|
||||
|
||||
|
|
|
@ -257,13 +257,23 @@ def scandir_inorder(*, path, fd=None):
|
|||
return sorted(os.scandir(arg), key=scandir_keyfunc)
|
||||
|
||||
|
||||
def secure_erase(path):
|
||||
"""Attempt to securely erase a file by writing random data over it before deleting it."""
|
||||
def secure_erase(path, *, avoid_collateral_damage):
|
||||
"""Attempt to securely erase a file by writing random data over it before deleting it.
|
||||
|
||||
If avoid_collateral_damage is True, we only secure erase if the total link count is 1,
|
||||
otherwise we just do a normal "delete" (unlink) without first overwriting it with random.
|
||||
This avoids other hardlinks pointing to same inode as <path> getting damaged, but might be less secure.
|
||||
A typical scenario where this is useful are quick "hardlink copies" of bigger directories.
|
||||
|
||||
If avoid_collateral_damage is False, we always secure erase.
|
||||
If there are hardlinks pointing to the same inode as <path>, they will contain random garbage afterwards.
|
||||
"""
|
||||
with open(path, 'r+b') as fd:
|
||||
length = os.stat(fd.fileno()).st_size
|
||||
fd.write(os.urandom(length))
|
||||
fd.flush()
|
||||
os.fsync(fd.fileno())
|
||||
st = os.stat(fd.fileno())
|
||||
if not (st.st_nlink > 1 and avoid_collateral_damage):
|
||||
fd.write(os.urandom(st.st_size))
|
||||
fd.flush()
|
||||
os.fsync(fd.fileno())
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
|
|
|
@ -180,7 +180,7 @@ def is_slow_msgpack():
|
|||
def is_supported_msgpack():
|
||||
# DO NOT CHANGE OR REMOVE! See also requirements and comments in setup.py.
|
||||
import msgpack
|
||||
return (1, 0, 3) <= msgpack.version <= (1, 0, 3) and \
|
||||
return (1, 0, 3) <= msgpack.version <= (1, 0, 4) and \
|
||||
msgpack.version not in [] # < add bad releases here to deny list
|
||||
|
||||
|
||||
|
|
|
@ -4,9 +4,9 @@ import socket
|
|||
import tempfile
|
||||
import uuid
|
||||
|
||||
from borg.constants import UMASK_DEFAULT
|
||||
from borg.helpers import safe_unlink
|
||||
from borg.platformflags import is_win32
|
||||
from ..constants import UMASK_DEFAULT
|
||||
from ..helpers import safe_unlink
|
||||
from ..platformflags import is_win32
|
||||
|
||||
"""
|
||||
platform base module
|
||||
|
|
|
@ -318,7 +318,7 @@ class Repository:
|
|||
|
||||
if os.path.isfile(old_config_path):
|
||||
logger.warning("Old config file not securely erased on previous config update")
|
||||
secure_erase(old_config_path)
|
||||
secure_erase(old_config_path, avoid_collateral_damage=True)
|
||||
|
||||
if os.path.isfile(config_path):
|
||||
link_error_msg = ("Failed to securely erase old repository config file (hardlinks not supported). "
|
||||
|
@ -345,7 +345,7 @@ class Repository:
|
|||
"read-only repositories." % (e.strerror, e.filename))
|
||||
|
||||
if os.path.isfile(old_config_path):
|
||||
secure_erase(old_config_path)
|
||||
secure_erase(old_config_path, avoid_collateral_damage=True)
|
||||
|
||||
def save_key(self, keydata):
|
||||
assert self.config
|
||||
|
@ -1115,7 +1115,7 @@ class Repository:
|
|||
logger.info('Finished %s repository check, no problems found.', mode)
|
||||
return not error_found or repair
|
||||
|
||||
def scan_low_level(self):
|
||||
def scan_low_level(self, segment=None, offset=None):
|
||||
"""Very low level scan over all segment file entries.
|
||||
|
||||
It does NOT care about what's committed and what not.
|
||||
|
@ -1124,13 +1124,21 @@ class Repository:
|
|||
|
||||
This is intended as a last-resort way to get access to all repo contents of damaged repos,
|
||||
when there is uncommitted, but valuable data in there...
|
||||
|
||||
When segment or segment+offset is given, limit processing to this location only.
|
||||
"""
|
||||
for segment, filename in self.io.segment_iterator():
|
||||
for current_segment, filename in self.io.segment_iterator(segment=segment):
|
||||
if segment is not None and current_segment > segment:
|
||||
break
|
||||
try:
|
||||
for tag, key, offset, data in self.io.iter_objects(segment, include_data=True):
|
||||
yield key, data, tag, segment, offset
|
||||
for tag, key, current_offset, data in self.io.iter_objects(segment=current_segment,
|
||||
offset=offset or 0, include_data=True):
|
||||
if offset is not None and current_offset > offset:
|
||||
break
|
||||
yield key, data, tag, current_segment, current_offset
|
||||
except IntegrityError as err:
|
||||
logger.error('Segment %d (%s) has IntegrityError(s) [%s] - skipping.' % (segment, filename, str(err)))
|
||||
logger.error('Segment %d (%s) has IntegrityError(s) [%s] - skipping.' % (
|
||||
current_segment, filename, str(err)))
|
||||
|
||||
def _rollback(self, *, cleanup):
|
||||
"""
|
||||
|
|
|
@ -188,11 +188,13 @@ def test_obfuscate():
|
|||
data = bytes(1000)
|
||||
compressed = compressor.compress(data)
|
||||
# N blocks + 2 id bytes obfuscator. 4 length bytes
|
||||
assert 1000 + 6 <= len(compressed) <= 1000 + 6 + 1024
|
||||
# The 'none' compressor also adds 2 id bytes
|
||||
assert 6 + 2 + 1000 <= len(compressed) <= 6 + 2 + 1000 + 1024
|
||||
data = bytes(1100)
|
||||
compressed = compressor.compress(data)
|
||||
# N blocks + 2 id bytes obfuscator. 4 length bytes
|
||||
assert 1100 + 6 <= len(compressed) <= 1100 + 6 + 1024
|
||||
# The 'none' compressor also adds 2 id bytes
|
||||
assert 6 + 2 + 1100 <= len(compressed) <= 6 + 2 + 1100 + 1024
|
||||
|
||||
|
||||
def test_compression_specs():
|
||||
|
|
|
@ -155,11 +155,6 @@ class TestLocationWithoutEnv:
|
|||
"Location(proto='file', user=None, host=None, port=None, path='/abs/path:with:colons')"
|
||||
assert Location('/abs/path:with:colons').to_key_filename() == keys_dir + 'abs_path_with_colons'
|
||||
|
||||
def test_user_parsing(self):
|
||||
# see issue #1930
|
||||
assert repr(Location('ssh://host/path')) == \
|
||||
"Location(proto='ssh', user=None, host='host', port=None, path='/path')"
|
||||
|
||||
def test_canonical_path(self, monkeypatch):
|
||||
monkeypatch.delenv('BORG_REPO', raising=False)
|
||||
locations = ['some/path', 'file://some/path', 'host:some/path',
|
||||
|
|
Loading…
Reference in New Issue