mirror
/
borg
mirror of https://github.com/borgbackup/borg.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #3240 from Anakonda/updateWindows 

Update windows branch
This commit is contained in:
TW 2017-11-01 15:03:39 +01:00 committed by GitHub
parent 7b37b16d3b f252f6f6d7
commit 3c68040c50
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
273 changed files with 54604 additions and 15116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
.coafile Normal file
View File

@ -0,0 +1,38 @@
[all]
# note: put developer specific settings into ~/.coarc (e.g. editor = ...)
max_line_length = 255
use_spaces = True
[all.general]
files = src/borg/**/*.(py|pyx|c)
ignore = src/borg/(chunker.c|compress.c|hashindex.c|item.c),
src/borg/algorithms/(checksums.c|crc32.c),
src/borg/algorithms/blake2/*,
src/borg/algorithms/xxh64/*,
src/borg/crypto/low_level.c,
src/borg/platform/*.c
bears = SpaceConsistencyBear, FilenameBear, InvalidLinkBear, LineLengthBear
file_naming_convention = snake
[all.python]
files = src/borg/**/*.py
bears = PEP8Bear, PyDocStyleBear, PyLintBear
pep_ignore = E123,E125,E126,E127,E128,E226,E301,E309,E402,F401,F405,F811,W690
pylint_disable = C0103, C0111, C0112, C0123, C0301, C0302, C0325, C0330, C0411, C0412, C0413, C1801,
W0102, W0104, W0106, W0108, W0120, W0201, W0212, W0221, W0231, W0401, W0404,
W0511, W0603, W0611, W0612, W0613, W0614, W0621, W0622, W0702, W0703,
W1201, W1202, W1401,
R0101, R0201, R0204, R0901, R0902, R0903, R0904, R0911, R0912, R0913, R0914, R0915, R0916, R1701, R1704, R1705, R1706,
E0102, E0202, E0401, E0601, E0611, E0702, E1101, E1102, E1120, E1129, E1130
pydocstyle_ignore = D100, D101, D102, D103, D104, D105, D200, D201, D202, D203, D204, D205, D209, D210,
D212, D213, D300, D301, D400, D401, D402, D403, D404
[all.c]
files = src/borg/**/*.c
bears = CPPCheckBear
[all.html]
files = src/borg/**/*.html
bears = HTMLLintBear
htmllint_ignore = *

11
.editorconfig Normal file
View File

@ -0,0 +1,11 @@
# EditorConfig is awesome: http://EditorConfig.org
root = true
[*]
end_of_line = lf
charset = utf-8
indent_style = space
indent_size = 4
insert_final_newline = true
trim_trailing_whitespace = true

4
.gitattributes vendored
View File

@ -1 +1,5 @@
borg/_version.py export-subst
*.py diff=python
docs/usage/*.rst.inc merge=ours
docs/man/* merge=ours

8
.github/PULL_REQUEST_TEMPLATE vendored Normal file
View File

@ -0,0 +1,8 @@
Thank you for contributing code to Borg, your help is appreciated!
Please, before you submit a pull request, make sure it complies with the
guidelines given in our documentation:
https://borgbackup.readthedocs.io/en/latest/development.html#contributions
**Please remove all above text before submitting your pull request.**

24
.gitignore vendored
View File

@ -4,26 +4,28 @@ build
dist
borg-env
.tox
hashindex.c
chunker.c
compress.c
crypto.c
platform_darwin.c
platform_freebsd.c
platform_linux.c
platform_windows.c
platform_posix.c
src/borg/compress.c
src/borg/crypto/low_level.c
src/borg/hashindex.c
src/borg/item.c
src/borg/chunker.c
src/borg/algorithms/checksums.c
src/borg/platform/darwin.c
src/borg/platform/freebsd.c
src/borg/platform/linux.c
src/borg/platform/posix.c
src/borg/platform/windows.c
src/borg/_version.py
*.egg-info
*.pyc
*.pyo
*.so
.idea/
.cache/
src/borg/_version.py
borg.build/
borg.dist/
borg.exe
*.dll
.coverage
.coverage.*
.vagrant
.eggs

14
.mailmap Normal file
View File

@ -0,0 +1,14 @@
Abdel-Rahman <abodyxplay1@gmail.com>
Brian Johnson <brian@sherbang.com>
Carlo Teubner <carlo.teubner@gmail.com>
Mark Edgington <edgimar@gmail.com>
Leo Famulari <leo@famulari.name>
Marian Beermann <public@enkore.de>
Thomas Waldmann <tw@waldmann-edv.de>
Dan Christensen <jdc@uwo.ca> <jdc+github@uwo.ca>
Antoine Beaupré <anarcat@koumbit.org> <anarcat@debian.org> <anarcat@users.noreply.github.com>
Hartmut Goebel <h.goebel@crazy-compilers.com> <htgoebel@users.noreply.github.com>
Michael Gajda <michaelg@speciesm.net> <michael.gajda@tu-dortmund.de>
Milkey Mouse <milkeymouse@meme.institute> <milkey-mouse@users.noreply.github.com>
Ronny Pfannschmidt <opensource@ronnypfannschmidt.de> <ronny.pfannschmidt@redhat.com>
Stefan Tatschner <rumpelsepp@sevenbyte.org> <stefan@sevenbyte.org>

36
.travis.yml
View File

@ -6,37 +6,45 @@ cache:
directories:
- $HOME/.cache/pip
# note: use py 3.5.2, it has lzma support. 3.5(.0) on travis.org/trusty does not.
matrix:
include:
- python: 3.4
os: linux
dist: trusty
env: TOXENV=py34
- python: 3.5.2
- python: 3.5
os: linux
dist: trusty
env: TOXENV=py35
- python: nightly
- python: 3.6
os: linux
dist: trusty
env: TOXENV=py36
- python: 3.4
- python: 3.5
os: linux
dist: trusty
env: TOXENV=flake8
- language: generic
os: osx
osx_image: xcode6.4
env: TOXENV=py34
- python: "3.6-dev"
os: linux
dist: trusty
env: TOXENV=py36
- language: generic
os: osx
osx_image: xcode6.4
env: TOXENV=py35
allow_failures:
- python: nightly
- language: generic
os: osx
osx_image: xcode6.4
env: TOXENV=py36
before_install:
- |
test $TRAVIS_EVENT_TYPE != "pull_request" || {
echo Checking whether $TRAVIS_COMMIT_RANGE changed only docs
git diff --name-only $TRAVIS_COMMIT_RANGE | grep --quiet --invert-match --extended-regexp '(AUTHORS|README\.rst|^(docs)/)' || {
echo "Only docs were updated, stopping build process."
exit
}
}
install:
- git fetch --unshallow --tags
- ./.travis/install.sh
script:

26
.travis/install.sh
View File

@ -4,8 +4,6 @@ set -e
set -x
if [[ "$(uname -s)" == 'Darwin' ]]; then
brew update || brew update
if [[ "${OPENSSL}" != "0.9.8" ]]; then
brew outdated openssl || brew upgrade openssl
fi
@ -17,27 +15,33 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then
brew install lz4
brew install xz # required for python lzma module
brew outdated pyenv || brew upgrade pyenv
brew install pkg-config
brew install Caskroom/cask/osxfuse
case "${TOXENV}" in
py34)
pyenv install 3.4.3
pyenv global 3.4.3
;;
py35)
pyenv install 3.5.1
pyenv global 3.5.1
pyenv install 3.5.2
pyenv global 3.5.2
;;
py36)
pyenv install 3.6.0
pyenv global 3.6.0
;;
esac
pyenv rehash
python -m pip install --user 'virtualenv<14.0'
python -m pip install --user virtualenv
else
pip install 'virtualenv<14.0'
pip install virtualenv
sudo apt-get update
sudo apt-get install -y fakeroot
sudo apt-get install -y liblz4-dev
sudo apt-get install -y libacl1-dev
sudo apt-get install -y libfuse-dev fuse pkg-config # optional, for FUSE support
fi
python -m virtualenv ~/.venv
source ~/.venv/bin/activate
pip install -r requirements.d/development.txt
pip install codecov
pip install -e .
python setup.py --version
pip install -e .[fuse]

3
.travis/upload_coverage.sh
View File

@ -4,9 +4,8 @@ set -e
set -x
NO_COVERAGE_TOXENVS=(pep8)
if ! [[ "${NO_COVERAGE_TOXENVS[*]}" =~ "${TOXENV}" ]]; then
if ! [[ "${NO_COVERAGE_TOXENVS[*]}" =~ ${TOXENV} ]]; then
source ~/.venv/bin/activate
ln .tox/.coverage .coverage
# on osx, tests run as root, need access to .coverage
sudo chmod 666 .coverage
codecov -e TRAVIS_OS_NAME TOXENV

30
AUTHORS
View File

@ -1,5 +1,5 @@
Borg Contributors ("The Borg Collective")
=========================================
Borg authors ("The Borg Collective")
------------------------------------
- Thomas Waldmann <tw@waldmann-edv.de>
- Antoine Beaupré <anarcat@debian.org>
@ -8,8 +8,10 @@ Borg Contributors ("The Borg Collective")
- Michael Hanselmann <public@hansmi.ch>
- Teemu Toivanen <public@profnetti.fi>
- Marian Beermann <public@enkore.de>
- Martin Hostettler <textshell@uchuujin.de>
- Daniel Reichelt <hacking@nachtgeist.net>
- Lauri Niskanen <ape@ape3000.com>
- Abdel-Rahman A. (Abogical)
Borg is a fork of Attic.
@ -31,3 +33,27 @@ Attic Patches and Suggestions
- Johann Klähn
- Petros Moisiadis
- Thomas Waldmann
BLAKE2
------
Borg includes BLAKE2: Copyright 2012, Samuel Neves <sneves@dei.uc.pt>, licensed under the terms
of the CC0, the OpenSSL Licence, or the Apache Public License 2.0.
Slicing CRC32
-------------
Borg includes a fast slice-by-8 implementation of CRC32, Copyright 2011-2015 Stephan Brumme,
licensed under the terms of a zlib license. See http://create.stephan-brumme.com/crc32/
Folding CRC32
-------------
Borg includes an extremely fast folding implementation of CRC32, Copyright 2013 Intel Corporation,
licensed under the terms of the zlib license.
xxHash
------
XXH64, a fast non-cryptographic hash algorithm. Copyright 2012-2016 Yann Collet,
licensed under a BSD 2-clause license.

24
LICENSE
View File

@ -1,4 +1,4 @@
Copyright (C) 2015-2016 The Borg Collective (see AUTHORS file)
Copyright (C) 2015-2017 The Borg Collective (see AUTHORS file)
Copyright (C) 2010-2014 Jonas Borgström <jonas@borgstrom.se>
All rights reserved.
@ -16,14 +16,14 @@ are met:
products derived from this software without specific prior
written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

14
MANIFEST.in
View File

@ -1,10 +1,8 @@
include README.rst AUTHORS LICENSE CHANGES.rst MANIFEST.in
graft src
recursive-exclude src *.pyc
recursive-exclude src *.pyo
recursive-include docs *
recursive-exclude docs *.pyc
recursive-exclude docs *.pyo
prune docs/_build
exclude .coafile .coveragerc .gitattributes .gitignore .travis.yml Vagrantfile
prune .travis
exclude .coveragerc .gitattributes .gitignore .travis.yml Vagrantfile
prune .github
graft src
graft docs
prune docs/_build
global-exclude *.pyc *.orig *.so *.dll

117
README.rst
View File

@ -1,9 +1,9 @@
|screencast|
|screencast_basic|
.. highlight:: bash
More screencasts: `installation`_, `advanced usage`_
What is BorgBackup?
===================
-------------------
BorgBackup (short: Borg) is a deduplicating backup program.
Optionally, it supports compression and authenticated encryption.
@ -20,27 +20,31 @@ downloaded Borg, ``docs/installation.rst`` to get started with Borg.
.. _installation manual: https://borgbackup.readthedocs.org/en/stable/installation.html
Main features
-------------
~~~~~~~~~~~~~
**Space efficient storage**
Deduplication based on content-defined chunking is used to reduce the number
of bytes stored: each file is split into a number of variable length chunks
and only chunks that have never been seen before are added to the repository.
A chunk is considered duplicate if its id_hash value is identical.
A cryptographically strong hash or MAC function is used as id_hash, e.g.
(hmac-)sha256.
To deduplicate, all the chunks in the same repository are considered, no
matter whether they come from different machines, from previous backups,
from the same backup or even from the same single file.
Compared to other deduplication approaches, this method does NOT depend on:
* file/directory names staying the same: So you can move your stuff around
* file/directory names staying the same: So you can move your stuff around
without killing the deduplication, even between machines sharing a repo.
* complete files or time stamps staying the same: If a big file changes a
little, only a few new chunks need to be stored - this is great for VMs or
* complete files or time stamps staying the same: If a big file changes a
little, only a few new chunks need to be stored - this is great for VMs or
raw disks.
* The absolute position of a data chunk inside a file: Stuff may get shifted
* The absolute position of a data chunk inside a file: Stuff may get shifted
and will still be found by the deduplication algorithm.
**Speed**
@ -75,24 +79,25 @@ Main features
* FreeBSD
* OpenBSD and NetBSD (no xattrs/ACLs support or binaries yet)
* Cygwin (not supported, no binaries yet)
* Linux Subsystem of Windows 10 (not supported)
**Free and Open Source Software**
* security and functionality can be audited independently
* licensed under the BSD (3-clause) license
* licensed under the BSD (3-clause) license, see `License`_ for the
complete license
Easy to use
-----------
~~~~~~~~~~~
Initialize a new backup repository and create a backup archive::
Initialize a new backup repository (see ``borg init --help`` for encryption options)::
$ borg init -e repokey /path/to/repo
Create a backup archive::
$ borg init /path/to/repo
$ borg create /path/to/repo::Saturday1 ~/Documents
Now doing another backup, just to show off the great deduplication:
.. code-block:: none
:emphasize-lines: 11
Now doing another backup, just to show off the great deduplication::
$ borg create -v --stats /path/to/repo::Saturday2 ~/Documents
-----------------------------------------------------------------------------
@ -114,51 +119,32 @@ Now doing another backup, just to show off the great deduplication:
For a graphical frontend refer to our complementary project `BorgWeb <https://borgweb.readthedocs.io/>`_.
Helping, Donations and Bounties
-------------------------------
Your help is always welcome!
Spread the word, give feedback, help with documentation, testing or development.
You can also give monetary support to the project, see there for details:
https://borgbackup.readthedocs.io/en/stable/support.html#bounties-and-fundraisers
Links
=====
-----
* `Main Web Site <https://borgbackup.readthedocs.org/>`_
* `Releases <https://github.com/borgbackup/borg/releases>`_,
`PyPI packages <https://pypi.python.org/pypi/borgbackup>`_ and
`ChangeLog <https://github.com/borgbackup/borg/blob/master/docs/changes.rst>`_
* `GitHub <https://github.com/borgbackup/borg>`_,
`Issue Tracker <https://github.com/borgbackup/borg/issues>`_ and
`Bounties & Fundraisers <https://www.bountysource.com/teams/borgbackup>`_
* `GitHub <https://github.com/borgbackup/borg>`_ and
`Issue Tracker <https://github.com/borgbackup/borg/issues>`_.
* `Web-Chat (IRC) <http://webchat.freenode.net/?randomnick=1&channels=%23borgbackup&uio=MTY9dHJ1ZSY5PXRydWUa8>`_ and
`Mailing List <https://mail.python.org/mailman/listinfo/borgbackup>`_
* `License <https://borgbackup.readthedocs.org/en/stable/authors.html#license>`_
* `Security contact <https://borgbackup.readthedocs.io/en/latest/support.html#security-contact>`_
Notes
-----
Borg is a fork of `Attic`_ and maintained by "`The Borg collective`_".
.. _Attic: https://github.com/jborg/attic
.. _The Borg collective: https://borgbackup.readthedocs.org/en/latest/authors.html
Differences between Attic and Borg
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Here's a (incomplete) list of some major changes:
* more open, faster paced development (see `issue #1 <https://github.com/borgbackup/borg/issues/1>`_)
* lots of attic issues fixed (see `issue #5 <https://github.com/borgbackup/borg/issues/5>`_)
* less chunk management overhead (less memory and disk usage for chunks index)
* faster remote cache resync (useful when backing up multiple machines into same repo)
* compression: no, lz4, zlib or lzma compression, adjustable compression levels
* repokey replaces problematic passphrase mode (you can't change the passphrase nor the pbkdf2 iteration count in "passphrase" mode)
* simple sparse file support, great for virtual machine disk files
* can read special files (e.g. block devices) or from stdin, write to stdout
* mkdir-based locking is more compatible than attic's posix locking
* uses fadvise to not spoil / blow up the fs cache
* better error messages / exception handling
* better logging, screen output, progress indication
* tested on misc. Linux systems, 32 and 64bit, FreeBSD, OpenBSD, NetBSD, Mac OS X
Please read the `ChangeLog`_ (or ``docs/changes.rst`` in the source distribution) for more
information.
BORG IS NOT COMPATIBLE WITH ORIGINAL ATTIC (but there is a one-way conversion).
Compatibility notes
-------------------
EXPECT THAT WE WILL BREAK COMPATIBILITY REPEATEDLY WHEN MAJOR RELEASE NUMBER
CHANGES (like when going from 0.x.y to 1.0.0 or from 1.x.y to 2.0.0).
@ -167,9 +153,16 @@ NOT RELEASED DEVELOPMENT VERSIONS HAVE UNKNOWN COMPATIBILITY PROPERTIES.
THIS IS SOFTWARE IN DEVELOPMENT, DECIDE YOURSELF WHETHER IT FITS YOUR NEEDS.
Borg is distributed under a 3-clause BSD license, see `License`_ for the complete license.
Security issues should be reported to the `Security contact`_ (or
see ``docs/suppport.rst`` in the source distribution).
|doc| |build| |coverage|
.. start-badges
|doc| |build| |coverage| |bestpractices| |bounties|
.. |bounties| image:: https://api.bountysource.com/badge/team?team_id=78284&style=bounties_posted
:alt: Bounty Source
:target: https://www.bountysource.com/teams/borgbackup
.. |doc| image:: https://readthedocs.org/projects/borgbackup/badge/?version=stable
:alt: Documentation
@ -183,6 +176,16 @@ Borg is distributed under a 3-clause BSD license, see `License`_ for the complet
:alt: Test Coverage
:target: https://codecov.io/github/borgbackup/borg?branch=master
.. |screencast| image:: https://asciinema.org/a/28691.png
:alt: BorgBackup Installation and Basic Usage
:target: https://asciinema.org/a/28691?autoplay=1&speed=2
.. |screencast_basic| image:: https://asciinema.org/a/133292.png
:alt: BorgBackup Basic Usage
:target: https://asciinema.org/a/133292?autoplay=1&speed=1
.. _installation: https://asciinema.org/a/133291?autoplay=1&speed=1
.. _advanced usage: https://asciinema.org/a/133293?autoplay=1&speed=1
.. |bestpractices| image:: https://bestpractices.coreinfrastructure.org/projects/271/badge
:alt: Best Practices Score
:target: https://bestpractices.coreinfrastructure.org/projects/271
.. end-badges

475
Vagrantfile vendored
View File

@ -3,154 +3,43 @@
# Automated creation of testing environments / binaries on misc. platforms
def packages_prepare_wheezy
return <<-EOF
# debian 7 wheezy does not have lz4, but it is available from wheezy-backports:
echo "deb http://http.debian.net/debian wheezy-backports main" > /etc/apt/sources.list.d/wheezy-backports.list
EOF
end
$cpus = Integer(ENV.fetch('VMCPUS', '4')) # create VMs with that many cpus
$xdistn = Integer(ENV.fetch('XDISTN', '4')) # dispatch tests to that many pytest workers
$wmem = $xdistn * 256 # give the VM additional memory for workers [MB]
def packages_debianoid
def packages_debianoid(user)
return <<-EOF
apt-get update
# install all the (security and other) updates
apt-get dist-upgrade -y
# for building borgbackup and dependencies:
apt-get install -y libssl-dev libacl1-dev liblz4-dev libfuse-dev fuse pkg-config
usermod -a -G fuse vagrant
usermod -a -G fuse #{user}
chgrp fuse /dev/fuse
chmod 666 /dev/fuse
apt-get install -y fakeroot build-essential git
apt-get install -y python3-dev python3-setuptools
# for building python:
apt-get install -y zlib1g-dev libbz2-dev libncurses5-dev libreadline-dev liblzma-dev libsqlite3-dev
# this way it works on older dists (like ubuntu 12.04) also:
# for python 3.2 on ubuntu 12.04 we need pip<8 and virtualenv<14 as
# newer versions are not compatible with py 3.2 any more.
easy_install3 'pip<8.0'
pip3 install 'virtualenv<14.0'
touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
easy_install3 'pip'
pip3 install 'virtualenv'
EOF
end
def packages_redhatted
def packages_arch
return <<-EOF
yum install -y epel-release
yum update -y
# for building borgbackup and dependencies:
yum install -y openssl-devel openssl libacl-devel libacl lz4-devel fuse-devel fuse pkgconfig
usermod -a -G fuse vagrant
yum install -y fakeroot gcc git patch
# needed to compile msgpack-python (otherwise it will use slow fallback code):
yum install -y gcc-c++
# for building python:
yum install -y zlib-devel bzip2-devel ncurses-devel readline-devel xz xz-devel sqlite-devel
#yum install -y python-pip
#pip install virtualenv
touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
EOF
end
def packages_darwin
return <<-EOF
# install all the (security and other) updates
sudo softwareupdate --install --all
# get osxfuse 3.x pre-release code from github:
curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.3.3/osxfuse-3.3.3.dmg >osxfuse.dmg
MOUNTDIR=$(echo `hdiutil mount osxfuse.dmg | tail -1 | awk '{$1="" ; print $0}'` | xargs -0 echo) \
&& sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for OS X 3.3.3.pkg" -target /
sudo chown -R vagrant /usr/local # brew must be able to create stuff here
ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
brew update
brew install openssl
brew install lz4
brew install xz # required for python lzma module
brew install fakeroot
brew install git
brew install pkgconfig
touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
EOF
end
def packages_freebsd
return <<-EOF
# install all the (security and other) updates, base system
freebsd-update --not-running-from-cron fetch install
# for building borgbackup and dependencies:
pkg install -y openssl liblz4 fusefs-libs pkgconf
pkg install -y fakeroot git bash
# for building python:
pkg install -y sqlite3
# make bash default / work:
chsh -s bash vagrant
mount -t fdescfs fdesc /dev/fd
echo 'fdesc /dev/fd fdescfs rw 0 0' >> /etc/fstab
# make FUSE work
echo 'fuse_load="YES"' >> /boot/loader.conf
echo 'vfs.usermount=1' >> /etc/sysctl.conf
kldload fuse
sysctl vfs.usermount=1
pw groupmod operator -M vagrant
touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
# install all the (security and other) updates, packages
pkg update
yes | pkg upgrade
EOF
end
def packages_openbsd
return <<-EOF
. ~/.profile
mkdir -p /home/vagrant/borg
rsync -aH /vagrant/borg/ /home/vagrant/borg/
rm -rf /vagrant/borg
ln -sf /home/vagrant/borg /vagrant/
pkg_add bash
chsh -s /usr/local/bin/bash vagrant
pkg_add openssl
pkg_add lz4
# pkg_add fuse # does not install, sdl dependency missing
pkg_add git # no fakeroot
pkg_add py3-setuptools
ln -sf /usr/local/bin/python3.4 /usr/local/bin/python3
ln -sf /usr/local/bin/python3.4 /usr/local/bin/python
easy_install-3.4 pip
pip3 install virtualenv
touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
EOF
end
def packages_netbsd
return <<-EOF
hostname netbsd # the box we use has an invalid hostname
PKG_PATH="ftp://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/amd64/6.1.5/All/"
export PKG_PATH
pkg_add mozilla-rootcerts lz4 git bash
chsh -s bash vagrant
mkdir -p /usr/local/opt/lz4/include
mkdir -p /usr/local/opt/lz4/lib
ln -s /usr/pkg/include/lz4*.h /usr/local/opt/lz4/include/
ln -s /usr/pkg/lib/liblz4* /usr/local/opt/lz4/lib/
touch /etc/openssl/openssl.cnf # avoids a flood of "can't open ..."
mozilla-rootcerts install
pkg_add pkg-config # avoids some "pkg-config missing" error msg, even without fuse
# pkg_add fuse # llfuse supports netbsd, but is still buggy.
# https://bitbucket.org/nikratio/python-llfuse/issues/70/perfuse_open-setsockopt-no-buffer-space
pkg_add python34 py34-setuptools
ln -s /usr/pkg/bin/python3.4 /usr/pkg/bin/python
ln -s /usr/pkg/bin/python3.4 /usr/pkg/bin/python3
easy_install-3.4 pip
pip install virtualenv
touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
chown vagrant.vagrant /vagrant
pacman --sync --noconfirm python-virtualenv python-pip
EOF
end
def install_pyenv(boxname)
return <<-EOF
curl -s -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash
echo 'export PATH="$HOME/.pyenv/bin:/vagrant/borg:$PATH"' >> ~/.bash_profile
echo 'export PATH="$HOME/.pyenv/bin:$PATH"' >> ~/.bash_profile
echo 'eval "$(pyenv init -)"' >> ~/.bash_profile
echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bash_profile
echo 'export PYTHON_CONFIGURE_OPTS="--enable-shared"' >> ~/.bash_profile
echo 'export LANG=en_US.UTF-8' >> ~/.bash_profile
EOF
end
@ -163,9 +52,9 @@ end
def install_pythons(boxname)
return <<-EOF
. ~/.bash_profile
pyenv install 3.4.0 # tests
pyenv install 3.5.0 # tests
pyenv install 3.5.2 # binary build, use latest 3.5.x release
pyenv install 3.6.0 # tests
pyenv install 3.6.2 # binary build, use latest 3.6.x release
pyenv rehash
EOF
end
@ -182,71 +71,50 @@ def build_pyenv_venv(boxname)
return <<-EOF
. ~/.bash_profile
cd /vagrant/borg
# use the latest 3.5 release
pyenv global 3.5.2
pyenv virtualenv 3.5.2 borg-env
# use the latest 3.6 release
pyenv global 3.6.2
pyenv virtualenv 3.6.2 borg-env
ln -s ~/.pyenv/versions/borg-env .
EOF
end
def install_borg(boxname)
return <<-EOF
def install_borg(fuse)
script = <<-EOF
. ~/.bash_profile
cd /vagrant/borg
. borg-env/bin/activate
pip install -U wheel # upgrade wheel, too old for 3.5
cd borg
# clean up (wrong/outdated) stuff we likely got via rsync:
rm -f borg/*.so borg/*.cpy*
rm -f borg/{chunker,crypto,compress,hashindex,platform_linux}.c
rm -rf borg/__pycache__ borg/support/__pycache__ borg/testsuite/__pycache__
rm -rf __pycache__
find src -name '__pycache__' -exec rm -rf {} \\;
pip install -r requirements.d/development.txt
# by using [fuse], setup.py can handle different fuse requirements:
pip install -e .[fuse]
python setup.py clean
EOF
if fuse
script += <<-EOF
# by using [fuse], setup.py can handle different FUSE requirements:
pip install -e .[fuse]
EOF
else
script += <<-EOF
pip install -e .
# do not install llfuse into the virtualenvs built by tox:
sed -i.bak '/fuse.txt/d' tox.ini
EOF
end
return script
end
def install_borg_no_fuse(boxname)
def install_pyinstaller()
return <<-EOF
. ~/.bash_profile
cd /vagrant/borg
. borg-env/bin/activate
pip install -U wheel # upgrade wheel, too old for 3.5
cd borg
# clean up (wrong/outdated) stuff we likely got via rsync:
rm -f borg/*.so borg/*.cpy*
rm -f borg/{chunker,crypto,compress,hashindex,platform_linux}.c
rm -rf borg/__pycache__ borg/support/__pycache__ borg/testsuite/__pycache__
pip install -r requirements.d/development.txt
pip install -e .
EOF
end
def install_pyinstaller(boxname)
return <<-EOF
. ~/.bash_profile
cd /vagrant/borg
. borg-env/bin/activate
git clone https://github.com/pyinstaller/pyinstaller.git
git clone https://github.com/thomaswaldmann/pyinstaller.git
cd pyinstaller
git checkout v3.1.1
pip install -e .
EOF
end
def install_pyinstaller_bootloader(boxname)
return <<-EOF
. ~/.bash_profile
cd /vagrant/borg
. borg-env/bin/activate
git clone https://github.com/pyinstaller/pyinstaller.git
cd pyinstaller
git checkout v3.1.1
# build bootloader, if it is not included
cd bootloader
python ./waf all
cd ..
pip install -e .
git checkout v3.3-fixed
python setup.py install
EOF
end
@ -256,7 +124,8 @@ def build_binary_with_pyinstaller(boxname)
cd /vagrant/borg
. borg-env/bin/activate
cd borg
pyinstaller -F -n borg.exe --distpath=/vagrant/borg --clean src/borg/__main__.py --hidden-import=borg.platform.posix
pyinstaller --clean --distpath=/vagrant/borg scripts/borg.exe.spec
echo 'export PATH="/vagrant/borg:$PATH"' >> ~/.bash_profile
EOF
end
@ -265,12 +134,13 @@ def run_tests(boxname)
. ~/.bash_profile
cd /vagrant/borg/borg
. ../borg-env/bin/activate
if which pyenv > /dev/null; then
if which pyenv 2> /dev/null; then
# for testing, use the earliest point releases of the supported python versions:
pyenv global 3.4.0 3.5.0
pyenv global 3.5.0 3.6.0
pyenv local 3.5.0 3.6.0
fi
# otherwise: just use the system python
if which fakeroot > /dev/null; then
if which fakeroot 2> /dev/null; then
echo "Running tox WITH fakeroot -u"
fakeroot -u tox --skip-missing-interpreters
else
@ -280,242 +150,65 @@ def run_tests(boxname)
EOF
end
def fix_perms
def fs_init(user)
return <<-EOF
# . ~/.profile
chown -R vagrant /vagrant/borg
EOF
end
def windows_requirements
return <<-EOF
$url = "https://eternallybored.org/misc/wget/releases/wget-1.18-win64.zip"
(New-Object System.Net.WebClient).DownloadFile($url, "./wget.zip")
& { Add-Type -A 'System.IO.Compression.FileSystem'; [IO.Compression.ZipFile]::ExtractToDirectory('wget.zip', './wget'); }
wget\\wget.exe "http://gnuwin32.sourceforge.net/downlinks/tar-bin-zip.php" -O tar-bin.zip
wget\\wget.exe "http://gnuwin32.sourceforge.net/downlinks/tar-dep-zip.php" -O tar-dep.zip
& { Add-Type -A 'System.IO.Compression.FileSystem'; [IO.Compression.ZipFile]::ExtractToDirectory('tar-bin.zip', './tar'); }
& { Add-Type -A 'System.IO.Compression.FileSystem'; [IO.Compression.ZipFile]::ExtractToDirectory('tar-dep.zip', './tar'); }
wget\\wget.exe http://tukaani.org/xz/xz-5.2.1-windows.zip -O xz.zip
& { Add-Type -A 'System.IO.Compression.FileSystem'; [IO.Compression.ZipFile]::ExtractToDirectory('xz.zip', './xz'); }
wget\\wget.exe -Axz "https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20160205.tar.xz/download" -O msys.tar.xz
xz\\bin_x86-64\\xz -d msys.tar.xz
tar\\bin\\tar -xf msys.tar
msys64\\msys2.exe -c | more
Start-Sleep -s 10
msys64\\usr\\bin\\pacman --noconfirm -S mingw-w64-x86_64-python3 git mingw-w64-x86_64-lz4 mingw-w64-x86_64-python3-pip mingw-w64-x86_64-cython mingw-w64-x86_64-gcc mingw-w64-x86_64-ntldd-git
python3 -m pip install --upgrade pip
EOF
end
def windows_build
return <<-EOF
$msyspath = $(get-location).path + "\\msys64"
$Env:Path = $Env:Path + ";" + $msyspath + "\\mingw64\\bin;" + $msyspath + "\\usr\\bin"
net use x: \\\\vboxsvr\\vagrant
x:
pip3 install -r requirements.d/development.txt
pip3 install -e .
python3 deployment/windows/buildwin32.py
chown -R #{user} /vagrant/borg
touch ~#{user}/.bash_profile ; chown #{user} ~#{user}/.bash_profile
echo 'export LANG=en_US.UTF-8' >> ~#{user}/.bash_profile
echo 'export LC_CTYPE=en_US.UTF-8' >> ~#{user}/.bash_profile
echo 'export XDISTN=#{$xdistn}' >> ~#{user}/.bash_profile
EOF
end
Vagrant.configure(2) do |config|
# use rsync to copy content to the folder
config.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"], :rsync__chown => false
# do not let the VM access . on the host machine via the default shared folder!
config.vm.synced_folder ".", "/vagrant", disabled: true
config.vm.provider :virtualbox do |v|
#v.gui = true
v.cpus = 1
end
# Linux
config.vm.define "centos7_64" do |b|
b.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"]
b.vm.synced_folder ".", "/vagrant", disabled: true
b.vm.provision "fix perms", :type => :shell, :inline => fix_perms
b.vm.box = "centos/7"
b.vm.provider :virtualbox do |v|
v.memory = 768
end
b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted
b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos7_64")
b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos7_64")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("centos7_64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("centos7_64")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("centos7_64")
end
config.vm.define "centos6_32" do |b|
b.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"]
b.vm.synced_folder ".", "/vagrant", disabled: true
b.vm.provision "fix perms", :type => :shell, :inline => fix_perms
b.vm.box = "centos6-32"
b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted
b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos6_32")
b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos6_32")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("centos6_32")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg_no_fuse("centos6_32")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("centos6_32")
end
config.vm.define "centos6_64" do |b|
b.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"]
b.vm.synced_folder ".", "/vagrant", disabled: true
b.vm.provision "fix perms", :type => :shell, :inline => fix_perms
b.vm.box = "centos6-64"
b.vm.provider :virtualbox do |v|
v.memory = 768
end
b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted
b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos6_64")
b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos6_64")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("centos6_64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg_no_fuse("centos6_64")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("centos6_64")
v.cpus = $cpus
end
config.vm.define "xenial64" do |b|
b.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"]
b.vm.synced_folder ".", "/vagrant", disabled: true
b.vm.provision "fix perms", :type => :shell, :inline => fix_perms
b.vm.box = "ubuntu/xenial64"
b.vm.provider :virtualbox do |v|
v.memory = 768
v.memory = 1024 + $wmem
end
b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("trusty64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("trusty64")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("trusty64")
b.vm.provision "fs init", :type => :shell, :inline => fs_init("ubuntu")
b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("ubuntu")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("xenial64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("xenial64")
end
config.vm.define "trusty64" do |b|
b.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"]
b.vm.synced_folder ".", "/vagrant", disabled: true
b.vm.provision "fix perms", :type => :shell, :inline => fix_perms
b.vm.box = "ubuntu/trusty64"
config.vm.define "stretch64" do |b|
b.vm.box = "debian/stretch64"
b.vm.provider :virtualbox do |v|
v.memory = 768
v.memory = 1024 + $wmem
end
b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("trusty64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("trusty64")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("trusty64")
b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("stretch64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller()
b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("stretch64")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("stretch64")
end
config.vm.define "jessie64" do |b|
b.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"]
b.vm.synced_folder ".", "/vagrant", disabled: true
b.vm.provision "fix perms", :type => :shell, :inline => fix_perms
b.vm.box = "debian/jessie64"
config.vm.define "arch64" do |b|
b.vm.box = "terrywang/archlinux"
b.vm.provider :virtualbox do |v|
v.memory = 768
v.memory = 1024 + $wmem
end
b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("jessie64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("jessie64")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("jessie64")
b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
b.vm.provision "packages arch", :type => :shell, :privileged => true, :inline => packages_arch
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("arch64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("arch64")
end
config.vm.define "wheezy32" do |b|
b.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"]
b.vm.synced_folder ".", "/vagrant", disabled: true
b.vm.provision "fix perms", :type => :shell, :inline => fix_perms
b.vm.box = "boxcutter/debian79-i386"
b.vm.provision "packages prepare wheezy", :type => :shell, :inline => packages_prepare_wheezy
b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid
b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("wheezy32")
b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("wheezy32")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("wheezy32")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("wheezy32")
b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller("wheezy32")
b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("wheezy32")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("wheezy32")
end
config.vm.define "wheezy64" do |b|
b.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"]
b.vm.synced_folder ".", "/vagrant", disabled: true
b.vm.provision "fix perms", :type => :shell, :inline => fix_perms
b.vm.box = "boxcutter/debian79"
b.vm.provision "packages prepare wheezy", :type => :shell, :inline => packages_prepare_wheezy
b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid
b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("wheezy64")
b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("wheezy64")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("wheezy64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("wheezy64")
b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller("wheezy64")
b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("wheezy64")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("wheezy64")
end
# OS X
config.vm.define "darwin64" do |b|
b.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"]
b.vm.synced_folder ".", "/vagrant", disabled: true
b.vm.provision "fix perms", :type => :shell, :inline => fix_perms
b.vm.box = "jhcook/yosemite-clitools"
b.vm.provision "packages darwin", :type => :shell, :privileged => false, :inline => packages_darwin
b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("darwin64")
b.vm.provision "fix pyenv", :type => :shell, :privileged => false, :inline => fix_pyenv_darwin("darwin64")
b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("darwin64")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("darwin64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("darwin64")
b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller("darwin64")
b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("darwin64")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("darwin64")
end
# BSD
config.vm.define "freebsd64" do |b|
b.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"]
b.vm.synced_folder ".", "/vagrant", disabled: true
b.vm.provision "fix perms", :type => :shell, :inline => fix_perms
b.vm.box = "geoffgarside/freebsd-10.2"
b.vm.provider :virtualbox do |v|
v.memory = 768
end
b.vm.provision "install system packages", :type => :shell, :inline => packages_freebsd
b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("freebsd")
b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("freebsd")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("freebsd")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("freebsd")
b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller_bootloader("freebsd")
b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("freebsd")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("freebsd")
end
config.vm.define "openbsd64" do |b|
b.vm.box = "kaorimatz/openbsd-5.9-amd64"
b.vm.provider :virtualbox do |v|
v.memory = 768
end
b.vm.provision "packages openbsd", :type => :shell, :inline => packages_openbsd
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openbsd64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg_no_fuse("openbsd64")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openbsd64")
end
config.vm.define "netbsd64" do |b|
b.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"]
b.vm.synced_folder ".", "/vagrant", disabled: true
b.vm.provision "fix perms", :type => :shell, :inline => fix_perms
b.vm.box = "alex-skimlinks/netbsd-6.1.5-amd64"
b.vm.provider :virtualbox do |v|
v.memory = 768
end
b.vm.provision "packages netbsd", :type => :shell, :inline => packages_netbsd
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("netbsd64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg_no_fuse("netbsd64")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("netbsd64")
end
config.vm.define "windows" do |b|
b.vm.communicator = "winrm"
# Defaults for vagrant box from Microsoft
# b.winrm.username = "IEUser"
# b.winrm.password = "Passw0rd!"
# b.ssh.username = "IEUser"
# b.ssh.password = "Passw0rd!"
b.vm.box = "joeapearson/win10-x64"
b.vm.provision "install msys2", :type => :shell, :privileged => false, :inline => windows_requirements
b.vm.provision "build", :type => :shell, :privileged => false, :inline => windows_build
end
# TODO: create more VMs with python 3.5+ and openssl 1.1.
# See branch 1.1-maint for a better equipped Vagrantfile (but still on py34 and openssl 1.0).
end

5
appveyor.yml
View File

@ -1,8 +1,5 @@
version: '{build}'
init:
- ps: Update-AppveyorBuild -Version "1.0.3-$($env:appveyor_repo_commit.substring(0,7))"
environment:
matrix:
- compiler: msys2
@ -16,6 +13,8 @@ install:
- 'pip3 install -r requirements.d/development.txt'
- 'pip3 install setuptools_scm'
build_script:
- ps: $version = git describe --long
- ps: Update-AppveyorBuild -Version $version
- 'pip3 install -e .'
- 'python3 deployment/windows/buildwin32.py'
test_script:

82
conftest.py
View File

@ -1,23 +1,79 @@
import os
import pytest
# IMPORTANT keep this above all other borg imports to avoid inconsistent values
# for `from borg.constants import PBKDF2_ITERATIONS` (or star import) usages before
# this is executed
from borg import constants
# no fixture-based monkey-patching since star-imports are used for the constants module
constants.PBKDF2_ITERATIONS = 1
# needed to get pretty assertion failures in unit tests:
if hasattr(pytest, 'register_assert_rewrite'):
pytest.register_assert_rewrite('borg.testsuite')
import borg.cache
from borg.logger import setup_logging
# Ensure that the loggers exist for all tests
setup_logging()
from borg.testsuite import has_lchflags, no_lchlfags_because, has_llfuse
from borg.testsuite.platform import fakeroot_detected
from borg import xattr, constants
from borg.testsuite import has_lchflags, has_llfuse
from borg.testsuite import are_symlinks_supported, are_hardlinks_supported, is_utime_fully_supported
from borg.testsuite.platform import fakeroot_detected, are_acls_working
from borg import xattr
def pytest_configure(config):
# no fixture-based monkey-patching since star-imports are used for the constants module
constants.PBKDF2_ITERATIONS = 1
@pytest.fixture(autouse=True)
def clean_env(tmpdir_factory, monkeypatch):
# avoid that we access / modify the user's normal .config / .cache directory:
monkeypatch.setenv('XDG_CONFIG_HOME', tmpdir_factory.mktemp('xdg-config-home'))
monkeypatch.setenv('XDG_CACHE_HOME', tmpdir_factory.mktemp('xdg-cache-home'))
# also avoid to use anything from the outside environment:
keys = [key for key in os.environ if key.startswith('BORG_')]
for key in keys:
monkeypatch.delenv(key, raising=False)
def pytest_report_header(config, startdir):
yesno = ['no', 'yes']
flags = 'Testing BSD-style flags: %s %s' % (yesno[has_lchflags], no_lchlfags_because)
fakeroot = 'fakeroot: %s (>=1.20.2: %s)' % (
yesno[fakeroot_detected()],
yesno[xattr.XATTR_FAKEROOT])
llfuse = 'Testing fuse: %s' % yesno[has_llfuse]
return '\n'.join((flags, llfuse, fakeroot))
tests = {
"BSD flags": has_lchflags,
"fuse": has_llfuse,
"root": not fakeroot_detected(),
"symlinks": are_symlinks_supported(),
"hardlinks": are_hardlinks_supported(),
"atime/mtime": is_utime_fully_supported(),
"modes": "BORG_TESTS_IGNORE_MODES" not in os.environ
}
enabled = []
disabled = []
for test in tests:
if tests[test]:
enabled.append(test)
else:
disabled.append(test)
output = "Tests enabled: " + ", ".join(enabled) + "\n"
output += "Tests disabled: " + ", ".join(disabled)
return output
class DefaultPatches:
def __init__(self, request):
self.org_cache_wipe_cache = borg.cache.LocalCache.wipe_cache
def wipe_should_not_be_called(*a, **kw):
raise AssertionError("Cache wipe was triggered, if this is part of the test add @pytest.mark.allow_cache_wipe")
if 'allow_cache_wipe' not in request.keywords:
borg.cache.LocalCache.wipe_cache = wipe_should_not_be_called
request.addfinalizer(self.undo)
def undo(self):
borg.cache.LocalCache.wipe_cache = self.org_cache_wipe_cache
@pytest.fixture(autouse=True)
def default_patches(request):
return DefaultPatches(request)

23
deployment/windows/buildwin32.py
View File

@ -10,6 +10,8 @@ import zipfile
builddir = 'win32exe'
pythonversion = str(sys.version_info[0]) + '.' + str(sys.version_info[1])
if os.path.exists(builddir):
shutil.rmtree(builddir)
os.mkdir(builddir)
@ -30,7 +32,7 @@ if gccpath == '':
source = open('wrapper.c', 'w')
source.write(
"""
#include <python3.5m/python.h>
#include <python""" + pythonversion + """m/python.h>
#include <windows.h>
#include <wchar.h>
#include <string>
@ -59,12 +61,12 @@ int wmain(int argc , wchar_t *argv[] )
}
""")
source.close()
subprocess.check_call('g++ wrapper.c -lpython3.5m -lshlwapi -municode -o ' + builddir + '/borg.exe')
subprocess.check_call('g++ wrapper.c -lpython' + pythonversion + 'm -lshlwapi -municode -o ' + builddir + '/borg.exe')
os.remove('wrapper.c')
print('Searching modules')
modulepath = os.path.abspath(os.path.join(gccpath, '../lib/python3.5/'))
modulepath = os.path.abspath(os.path.join(gccpath, '../lib/python' + pythonversion + '/'))
# Bundle all encodings - In theory user may use any encoding in command prompt
for file in os.listdir(os.path.join(modulepath, 'encodings')):
@ -76,7 +78,8 @@ finder.run_script('src/borg/__main__.py')
# For some reason modulefinder does not find these, add them manually
extramodules = [os.path.join(modulepath, 'site.py'), os.path.join(modulepath, 'encodings/idna.py'),
os.path.join(modulepath, 'stringprep.py')]
os.path.join(modulepath, 'stringprep.py'), os.path.join(modulepath, 'ctypes/wintypes.py'),
os.path.join(modulepath, 'lib-dynload/_sysconfigdata_m_win32_.py')]
for module in extramodules:
finder.run_script(module)
@ -101,6 +104,7 @@ def finddlls(exe):
re.append(dll)
return re
items = finder.modules.items()
for name, mod in items:
file = mod.__file__
@ -113,7 +117,7 @@ for name, mod in items:
os.makedirs(os.path.join(builddir, 'bin', os.path.split(relpath)[0]), exist_ok=True)
shutil.copyfile(file, os.path.join(builddir, 'bin', relpath))
else:
relativepath = file[file.find('lib')+len('lib/python3.5/'):]
relativepath = file[file.find('lib')+len('lib/python' + pythonversion + '/'):]
if 'encodings' in file:
continue
if relativepath not in library.namelist():
@ -130,10 +134,13 @@ for dll in finddlls(os.path.join(builddir, "borg.exe")):
shutil.copyfile(dll, os.path.join(builddir, os.path.split(dll)[1]))
shutil.copyfile(os.path.join('src', 'borg', '__main__.py'), os.path.join(builddir, 'bin', 'borg', '__main__.py'))
library.write(os.path.join(modulepath, 'site.py'), 'site.py')
library.write(os.path.join(modulepath, 'lib-dynload/_sysconfigdata_m_win32_.py'), '_sysconfigdata_m_win32_.py')
library.write(os.path.join(modulepath, 'ctypes/wintypes.py'), 'ctypes/wintypes.py')
for extmodule in ['src/borg/chunker-cpython-35m.dll', 'src/borg/compress-cpython-35m.dll',
'src/borg/crypto-cpython-35m.dll', 'src/borg/hashindex-cpython-35m.dll']:
for extmodule in ['src/borg/chunker-cpython-' + str(sys.version_info[0]) + str(sys.version_info[1]) + 'm.dll',
'src/borg/compress-cpython-' + str(sys.version_info[0]) + str(sys.version_info[1]) + 'm.dll',
'src/borg/item-cpython-' + str(sys.version_info[0]) + str(sys.version_info[1]) + 'm.dll',
'src/borg/hashindex-cpython-' + str(sys.version_info[0]) + str(sys.version_info[1]) + 'm.dll']:
for dll in finddlls(extmodule):
if builddir not in dll:
shutil.copyfile(dll, os.path.join(builddir, os.path.split(dll)[1]))

11
docs/_static/Makefile vendored Normal file
View File

@ -0,0 +1,11 @@
all: logo.pdf logo.png
logo.pdf: logo.svg
inkscape logo.svg --export-pdf=logo.pdf
logo.png: logo.svg
inkscape logo.svg --export-png=logo.png --export-dpi=72,72
clean:
rm -f logo.pdf logo.png

72
docs/_static/logo.pdf vendored Normal file
View File

@ -0,0 +1,72 @@
%PDF-1.4
%µí®û
3 0 obj
<< /Length 4 0 R
/Filter /FlateDecode
>>
stream
xœ}TAr!¼û
_@PäyBÙì!{Hòÿªàè¨5›Êžì°iØù
µ
'‰¿?âËÆ<>ŸvI0&tö=ÞŽÈZü§=N*ˆQ,¤J|DÎPê ?Ô •‹ÃœA“-\
$´‘ŒkÃæ’Ä@…Ö½dÑ…Y€3Í‚kž<6B>oÇ6T_±¸,3¼Ã0±«^Ñ'¼‡´wé˜äÕåÀ‡<C380>´4=åÝBJ5Ï~î»AÆ‘`ºêŒ3³WhF%$ ·¨acܒļR®xÎê|k]MX)÷`xLp¼ûˆ–<CB86>RZ:>4V'}8ÇY(¯ „“˜¥Gð||ºâ¥¾ ÿhs?ýóìÙýõ˜­Ì$ Jéâ}£<>( VRØ%ïÂŽe3Ý鯵"ý4 Ÿ5)¥æ†Ë`•”¾ñOĹ){ö-ú­êÖ—“·Y<˹uPm³“èØŽm¶ƒ‰mèV·)ÖjVƒ”tÃn y‚€"‡<>©<EFBFBD>9µCç$î.}_T—~Ô«wíXËø¿ÔÙxóùèª/QXÄj»^}5ÿRx­'b.ìŠná5ü}ßS
endstream
endobj
4 0 obj
430
endobj
2 0 obj
<<
/ExtGState <<
/a0 << /CA 1 /ca 1 >>
>>
>>
endobj
5 0 obj
<< /Type /Page
/Parent 1 0 R
/MediaBox [ 0 0 240 100 ]
/Contents 3 0 R
/Group <<
/Type /Group
/S /Transparency
/I true
/CS /DeviceRGB
>>
/Resources 2 0 R
>>
endobj
1 0 obj
<< /Type /Pages
/Kids [ 5 0 R ]
/Count 1
>>
endobj
6 0 obj
<< /Creator (cairo 1.14.8 (http://cairographics.org))
/Producer (cairo 1.14.8 (http://cairographics.org))
>>
endobj
7 0 obj
<< /Type /Catalog
/Pages 1 0 R
>>
endobj
xref
0 8
0000000000 65535 f
0000000830 00000 n
0000000544 00000 n
0000000015 00000 n
0000000522 00000 n
0000000616 00000 n
0000000895 00000 n
0000001022 00000 n
trailer
<< /Size 8
/Root 7 0 R
/Info 6 0 R
>>
startxref
1074
%%EOF

BIN
docs/_static/logo.png vendored
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1.6 KiB

After

Width:  |  Height:  |  Size: 1.7 KiB

10
docs/_static/logo.svg vendored Normal file
View File

@ -0,0 +1,10 @@
<?xml version="1.0" encoding="utf-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN" "http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
<!-- Created using Karbon, part of Calligra: http://www.calligra.org/karbon -->
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="240pt" height="100pt">
<rect width="320" height="133.333" fill="#000200"/>
<path id="p1" transform="translate(20.9086, 32.2192)" fill="#00dd00" d="M43.75 13.8021L26.6667 13.8021L26.6667 0L53.3854 0L67.2396 13.8021L67.2396 27.8646L60.3125 34.7917L67.2396 41.7187L67.2396 55.3125L53.3854 69.1146L26.6667 69.1146L26.6667 55.3125L43.75 55.3125L43.75 40.5729L26.6667 40.5729L26.6667 28.5417L43.75 28.5417ZM0 0L23.0208 0L23.0208 69.1146L0 69.1146Z"/>
<path id="p2" transform="translate(97.6794, 46.0213)" fill="#00dd00" d="M62.1354 41.5104L48.3333 55.3125L32.9167 55.3125L32.9167 42.3958L38.6458 42.3958L38.6458 13.8021L32.9167 13.8021L32.9167 0L48.3333 0L62.1354 13.8021ZM23.2813 42.3958L29.2708 42.3958L29.2708 55.3125L13.8021 55.3125L0 41.5104L0 13.8021L13.8021 0L29.2708 0L29.2708 13.8021L23.2813 13.8021Z"/>
<path id="p3" transform="translate(170.231, 46.0213)" fill="#00dd00" d="M36.5104 13.8021L26.7187 13.8021L26.7187 7.76042L34.4271 0L48.3854 0L59.5833 12.9167L59.5833 27.2396L36.5104 27.2396ZM0 55.3125L0 7.10543e-15L23.0208 7.10543e-15L23.0208 55.3125Z"/>
<path id="p4" transform="translate(236.429, 46.0213)" fill="#00dd00" d="M36.875 13.8021L26.6667 13.8021L26.6667 7.10543e-15L46.0937 7.10543e-15L59.8958 13.8021L59.8958 60.7812L46.0937 74.6875L15.7292 74.6875L8.80208 67.7083L8.80208 62.6042L36.875 62.6042ZM33.2292 42.3958L33.2292 48.4896L26.3542 55.3125L13.8021 55.3125L0 41.5104L0 13.8021L13.8021 0L23.0208 0L23.0208 42.3958Z"/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.7 KiB

20
docs/_templates/globaltoc.html vendored Normal file
View File

@ -0,0 +1,20 @@
<div class="sidebar-block">
<div class="sidebar-toc">
{# Restrict the sidebar toc depth to two levels while generating command usage pages.
This avoids superfluous entries for each "Description" and "Examples" heading. #}
{% if pagename.startswith("usage/") and pagename not in (
"usage/general", "usage/help", "usage/debug", "usage/notes",
) %}
{% set maxdepth = 2 %}
{% else %}
{% set maxdepth = 3 %}
{% endif %}
{% set toctree = toctree(maxdepth=maxdepth, collapse=True) %}
{% if toctree %}
{{ toctree }}
{% else %}
{{ toc }}
{% endif %}
</div>
</div>

5
docs/_templates/logo-text.html vendored Normal file
View File

@ -0,0 +1,5 @@
<a href="{{ homepage() }}" class="text-logo">
<img src='{{ pathto('_static/logo.svg', 1) }}' width='100%'>
{{ theme_project_nav_name or shorttitle }}
</a>

80
docs/api.rst
View File

@ -1,80 +0,0 @@
.. highlight:: python
API Documentation
=================
.. automodule:: borg.archiver
:members:
:undoc-members:
.. automodule:: borg.archive
:members:
:undoc-members:
.. automodule:: borg.repository
:members:
:undoc-members:
.. automodule:: borg.remote
:members:
:undoc-members:
.. automodule:: borg.cache
:members:
:undoc-members:
.. automodule:: borg.key
:members:
:undoc-members:
.. automodule:: borg.logger
:members:
:undoc-members:
.. automodule:: borg.helpers
:members:
:undoc-members:
.. automodule:: borg.locking
:members:
:undoc-members:
.. automodule:: borg.shellpattern
:members:
:undoc-members:
.. automodule:: borg.lrucache
:members:
:undoc-members:
.. automodule:: borg.fuse
:members:
:undoc-members:
.. automodule:: borg.xattr
:members:
:undoc-members:
.. automodule:: borg.platform
:members:
:undoc-members:
.. automodule:: borg.platform_linux
:members:
:undoc-members:
.. automodule:: borg.hashindex
:members:
:undoc-members:
.. automodule:: borg.compress
:members: get_compressor, Compressor, CompressorBase
:undoc-members:
.. automodule:: borg.chunker
:members:
:undoc-members:
.. automodule:: borg.crypto
:members:
:undoc-members:

3
docs/authors.rst
View File

@ -1,5 +1,8 @@
.. include:: global.rst.inc
Authors
=======
.. include:: ../AUTHORS
License

19
docs/book.rst Normal file
View File

@ -0,0 +1,19 @@
.. include:: global.rst.inc
Borg documentation
==================
.. when you add an element here, do not forget to add it to index.rst
.. Note: Some things are in appendices (see latex_appendices in conf.py)
.. toctree::
:maxdepth: 2
introduction
installation
quickstart
usage
deployment
faq
internals
development

195
docs/borg_theme/css/borg.css
View File

@ -1,22 +1,181 @@
@import url("theme.css");
/* The Return of the Borg.
*
* Have a bit green and grey and darkness (and if only in the upper left corner).
*/
.wy-side-nav-search {
background-color: #000000 !important;
}
.wy-side-nav-search > a {
color: rgba(255, 255, 255, 0.5);
}
.wy-side-nav-search > div.version {
color: rgba(255, 255, 255, 0.5);
}
#usage dt code {
dt code {
font-weight: normal;
}
#internals .toctree-wrapper > ul {
column-count: 3;
-webkit-column-count: 3;
}
#internals .toctree-wrapper > ul > li {
display: inline-block;
font-weight: bold;
}
#internals .toctree-wrapper > ul > li > ul {
font-weight: normal;
}
/* bootstrap has a .container class which clashes with docutils' container class. */
.docutils.container {
width: auto;
margin: 0;
padding: 0;
}
/* the default (38px) produces a jumpy baseline in Firefox on Linux. */
h1 {
font-size: 36px;
}
.text-logo {
background-color: #000200;
color: #00dd00;
}
.text-logo:hover,
.text-logo:active,
.text-logo:focus {
color: #5afe57;
}
/* by default the top and bottom margins are unequal which looks a bit unbalanced. */
.sidebar-block {
padding: 0;
margin: 14px 0 24px 0;
}
#borg-documentation h1 + p .external img {
width: 100%;
}
.container.experimental,
#debugging-facilities,
#borg-recreate {
/* don't change text dimensions */
margin: 0 -30px; /* padding below + border width */
padding: 0 10px; /* 10 px visual margin between edge of text and the border */
/* fallback for browsers that don't have repeating-linear-gradient: thick, red lines */
border-left: 20px solid red;
border-right: 20px solid red;
/* fancy red stripes */
border-image: repeating-linear-gradient(
-45deg,rgba(255,0,0,0.1) 0,rgba(255,0,0,0.75) 10px,rgba(0,0,0,0) 10px,rgba(0,0,0,0) 20px,rgba(255,0,0,0.75) 20px) 0 20 repeat;
}
.topic {
margin: 0 1em;
padding: 0 1em;
/* #4e4a4a = background of the ToC sidebar */
border-left: 2px solid #4e4a4a;;
border-right: 2px solid #4e4a4a;;
}
table.docutils:not(.footnote) td,
table.docutils:not(.footnote) th {
padding: .2em;
}
table.docutils:not(.footnote) {
border-collapse: collapse;
border: none;
}
table.docutils:not(.footnote) td,
table.docutils:not(.footnote) th {
border: 1px solid #ddd;
}
table.docutils:not(.footnote) tr:first-child th,
table.docutils:not(.footnote) tr:first-child td {
border-top: 0;
}
table.docutils:not(.footnote) tr:last-child td {
border-bottom: 0;
}
table.docutils:not(.footnote) tr td:first-child,
table.docutils:not(.footnote) tr th:first-child {
border-left: 0;
}
table.docutils:not(.footnote) tr td:last-child,
table.docutils:not(.footnote) tr th:last-child,
table.docutils.borg-options-table tr td {
border-right: 0;
}
table.docutils.option-list tr td,
table.docutils.borg-options-table tr td {
border-left: 0;
border-right: 0;
}
table.docutils.borg-options-table tr td:first-child:not([colspan="3"]) {
border-top: 0;
border-bottom: 0;
}
.borg-options-table td[colspan="3"] p {
margin: 0;
}
.borg-options-table {
width: 100%;
}
kbd, /* used in usage pages for options */
code,
.rst-content tt.literal,
.rst-content tt.literal,
.rst-content code.literal,
.rst-content tt,
.rst-content code,
p .literal,
p .literal span {
border: none;
padding: 0;
color: black; /* slight contrast with #404040 of regular text */
background: none;
}
kbd {
box-shadow: none;
line-height: 23px;
word-wrap: normal;
font-size: 15px;
font-family: Consolas, monospace;
}
.borg-options-table tr td:nth-child(2) .pre {
white-space: nowrap;
}
.borg-options-table tr td:first-child {
width: 2em;
}
cite {
white-space: nowrap;
color: black; /* slight contrast with #404040 of regular text */
font-family: Consolas, "Andale Mono WT", "Andale Mono", "Lucida Console", "Lucida Sans Typewriter",
"DejaVu Sans Mono", "Bitstream Vera Sans Mono", "Liberation Mono", "Nimbus Mono L", Monaco, "Courier New", Courier, monospace;
font-style: normal;
text-decoration: underline;
}
.borg-common-opt-ref {
font-weight: bold;
}
.sidebar-toc ul li.toctree-l2 a,
.sidebar-toc ul li.toctree-l3 a {
padding-right: 25px;
}
#common-options .option {
white-space: nowrap;
}

1303
docs/changes.rst
View File

File diff suppressed because it is too large Load Diff

90
docs/conf.py
View File

@ -19,8 +19,6 @@ sys.path.insert(0, os.path.abspath('../src'))
from borg import __version__ as sw_version
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
# -- General configuration -----------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
@ -44,14 +42,15 @@ master_doc = 'index'
# General information about the project.
project = 'Borg - Deduplicating Archiver'
copyright = '2010-2014 Jonas Borgström, 2015-2016 The Borg Collective (see AUTHORS file)'
copyright = '2010-2014 Jonas Borgström, 2015-2017 The Borg Collective (see AUTHORS file)'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = sw_version.split('-')[0]
split_char = '+' if '+' in sw_version else '-'
version = sw_version.split(split_char)[0]
# The full version, including alpha/beta/rc tags.
release = version
@ -74,6 +73,10 @@ exclude_patterns = ['_build']
# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None
# The Borg docs contain no or very little Python docs.
# Thus, the primary domain is rst.
primary_domain = 'rst'
# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True
@ -96,25 +99,29 @@ pygments_style = 'sphinx'
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#html_theme = ''
if not on_rtd: # only import and set the theme if we're building docs locally
import sphinx_rtd_theme
html_theme = 'sphinx_rtd_theme'
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
html_style = 'css/borg.css'
else:
html_context = {
'css_files': [
'https://media.readthedocs.org/css/sphinx_rtd_theme.css',
'https://media.readthedocs.org/css/readthedocs-doc-embed.css',
'_static/css/borg.css',
],
}
import guzzle_sphinx_theme
html_theme_path = guzzle_sphinx_theme.html_theme_path()
html_theme = 'guzzle_sphinx_theme'
def set_rst_settings(app):
app.env.settings.update({
'field_name_limit': 0,
'option_limit': 0,
})
def setup(app):
app.add_stylesheet('css/borg.css')
app.connect('builder-inited', set_rst_settings)
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}
html_theme_options = {
'project_nav_name': 'Borg %s' % version,
}
# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = ['_themes']
@ -128,7 +135,7 @@ else:
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
html_logo = '_static/logo.png'
html_logo = '_static/logo.svg'
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
@ -140,19 +147,21 @@ html_favicon = '_static/favicon.ico'
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['borg_theme']
html_extra_path = ['../src/borg/paperkey.html']
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
html_last_updated_fmt = '%Y-%m-%d'
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True
html_use_smartypants = True
# Custom sidebar templates, maps document names to template names.
html_sidebars = {
'index': ['sidebarlogo.html', 'sidebarusefullinks.html', 'searchbox.html'],
'**': ['sidebarlogo.html', 'relations.html', 'searchbox.html', 'localtoc.html', 'sidebarusefullinks.html']
'**': ['logo-text.html', 'searchbox.html', 'globaltoc.html'],
}
# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}
@ -189,22 +198,22 @@ htmlhelp_basename = 'borgdoc'
# -- Options for LaTeX output --------------------------------------------------
# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'
# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
('index', 'Borg.tex', 'Borg Documentation',
'see "AUTHORS" file', 'manual'),
('book', 'Borg.tex', 'Borg Documentation',
'The Borg Collective', 'manual'),
]
# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None
latex_logo = '_static/logo.pdf'
latex_elements = {
'papersize': 'a4paper',
'pointsize': '10pt',
'figure_align': 'H',
}
# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
@ -214,13 +223,18 @@ latex_documents = [
#latex_show_pagerefs = False
# If true, show URL addresses after external links.
#latex_show_urls = False
latex_show_urls = 'footnote'
# Additional stuff for the LaTeX preamble.
#latex_preamble = ''
# Documents to append as an appendix to all manuals.
#latex_appendices = []
latex_appendices = [
'support',
'resources',
'changes',
'authors',
]
# If false, no module index is generated.
#latex_domain_indices = True
@ -237,7 +251,13 @@ man_pages = [
1),
]
extensions = ['sphinx.ext.extlinks', 'sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode']
extensions = [
'sphinx.ext.extlinks',
'sphinx.ext.autodoc',
'sphinx.ext.todo',
'sphinx.ext.coverage',
'sphinx.ext.viewcode',
]
extlinks = {
'issue': ('https://github.com/borgbackup/borg/issues/%s', '#'),

219
docs/deployment.rst
View File

@ -1,221 +1,14 @@
.. include:: global.rst.inc
.. highlight:: none
.. _deployment:
Deployment
==========
This chapter will give an example how to setup a borg repository server for multiple
clients.
This chapter details deployment strategies for the following scenarios.
Machines
--------
.. toctree::
:titlesonly:
There are multiple machines used in this chapter and will further be named by their
respective fully qualified domain name (fqdn).
* The backup server: `backup01.srv.local`
* The clients:
- John Doe's desktop: `johndoe.clnt.local`
- Webserver 01: `web01.srv.local`
- Application server 01: `app01.srv.local`
User and group
--------------
The repository server needs to have only one UNIX user for all the clients.
Recommended user and group with additional settings:
* User: `backup`
* Group: `backup`
* Shell: `/bin/bash` (or other capable to run the `borg serve` command)
* Home: `/home/backup`
Most clients shall initiate a backup from the root user to catch all
users, groups and permissions (e.g. when backing up `/home`).
Folders
-------
The following folder tree layout is suggested on the repository server:
* User home directory, /home/backup
* Repositories path (storage pool): /home/backup/repos
* Clients restricted paths (`/home/backup/repos/<client fqdn>`):
- johndoe.clnt.local: `/home/backup/repos/johndoe.clnt.local`
- web01.srv.local: `/home/backup/repos/web01.srv.local`
- app01.srv.local: `/home/backup/repos/app01.srv.local`
Restrictions
------------
Borg is instructed to restrict clients into their own paths:
``borg serve --restrict-to-path /home/backup/repos/<client fqdn>``
There is only one ssh key per client allowed. Keys are added for ``johndoe.clnt.local``, ``web01.srv.local`` and
``app01.srv.local``. But they will access the backup under only one UNIX user account as:
``backup@backup01.srv.local``. Every key in ``$HOME/.ssh/authorized_keys`` has a
forced command and restrictions applied as shown below:
::
command="cd /home/backup/repos/<client fqdn>;
borg serve --restrict-to-path /home/backup/repos/<client fqdn>",
no-port-forwarding,no-X11-forwarding,no-pty,
no-agent-forwarding,no-user-rc <keytype> <key> <host>
.. note:: The text shown above needs to be written on a single line!
The options which are added to the key will perform the following:
1. Change working directory
2. Run ``borg serve`` restricted to the client base path
3. Restrict ssh and do not allow stuff which imposes a security risk
Due to the ``cd`` command we use, the server automatically changes the current
working directory. Then client doesn't need to have knowledge of the absolute
or relative remote repository path and can directly access the repositories at
``<user>@<host>:<repo>``.
.. note:: The setup above ignores all client given commandline parameters
which are normally appended to the `borg serve` command.
Client
------
The client needs to initialize the `pictures` repository like this:
borg init backup@backup01.srv.local:pictures
Or with the full path (should actually never be used, as only for demonstrational purposes).
The server should automatically change the current working directory to the `<client fqdn>` folder.
borg init backup@backup01.srv.local:/home/backup/repos/johndoe.clnt.local/pictures
When `johndoe.clnt.local` tries to access a not restricted path the following error is raised.
John Doe tries to backup into the Web 01 path:
borg init backup@backup01.srv.local:/home/backup/repos/web01.srv.local/pictures
::
~~~ SNIP ~~~
Remote: borg.remote.PathNotAllowed: /home/backup/repos/web01.srv.local/pictures
~~~ SNIP ~~~
Repository path not allowed
Ansible
-------
Ansible takes care of all the system-specific commands to add the user, create the
folder. Even when the configuration is changed the repository server configuration is
satisfied and reproducible.
Automate setting up an repository server with the user, group, folders and
permissions a Ansible playbook could be used. Keep in mind the playbook
uses the Arch Linux `pacman <https://www.archlinux.org/pacman/pacman.8.html>`_
package manager to install and keep borg up-to-date.
::
- hosts: backup01.srv.local
vars:
user: backup
group: backup
home: /home/backup
pool: "{{ home }}/repos"
auth_users:
- host: johndoe.clnt.local
key: "{{ lookup('file', '/path/to/keys/johndoe.clnt.local.pub') }}"
- host: web01.clnt.local
key: "{{ lookup('file', '/path/to/keys/web01.clnt.local.pub') }}"
- host: app01.clnt.local
key: "{{ lookup('file', '/path/to/keys/app01.clnt.local.pub') }}"
tasks:
- pacman: name=borg state=latest update_cache=yes
- group: name="{{ group }}" state=present
- user: name="{{ user }}" shell=/bin/bash home="{{ home }}" createhome=yes group="{{ group }}" groups= state=present
- file: path="{{ home }}" owner="{{ user }}" group="{{ group }}" mode=0700 state=directory
- file: path="{{ home }}/.ssh" owner="{{ user }}" group="{{ group }}" mode=0700 state=directory
- file: path="{{ pool }}" owner="{{ user }}" group="{{ group }}" mode=0700 state=directory
- authorized_key: user="{{ user }}"
key="{{ item.key }}"
key_options='command="cd {{ pool }}/{{ item.host }};borg serve --restrict-to-path {{ pool }}/{{ item.host }}",no-port-forwarding,no-X11-forwarding,no-pty,no-agent-forwarding,no-user-rc'
with_items: auth_users
- file: path="{{ home }}/.ssh/authorized_keys" owner="{{ user }}" group="{{ group }}" mode=0600 state=file
- file: path="{{ pool }}/{{ item.host }}" owner="{{ user }}" group="{{ group }}" mode=0700 state=directory
with_items: auth_users
Salt
----
This is a configuration similar to the one above, configured to be deployed with
Salt running on a Debian system.
::
Install borg backup from pip:
pkg.installed:
- pkgs:
- python3
- python3-dev
- python3-pip
- python-virtualenv
- libssl-dev
- openssl
- libacl1-dev
- libacl1
- liblz4-dev
- liblz4-1
- build-essential
- libfuse-dev
- fuse
- pkg-config
pip.installed:
- pkgs: ["borgbackup"]
- bin_env: /usr/bin/pip3
Setup backup user:
user.present:
- name: backup
- fullname: Backup User
- home: /home/backup
- shell: /bin/bash
# CAUTION!
# If you change the ssh command= option below, it won't necessarily get pushed to the backup
# server correctly unless you delete the ~/.ssh/authorized_keys file and re-create it!
{% for host in backupclients %}
Give backup access to {{host}}:
ssh_auth.present:
- user: backup
- source: salt://conf/ssh-pubkeys/{{host}}-backup.id_ecdsa.pub
- options:
- command="cd /home/backup/repos/{{host}}; borg serve --restrict-to-path /home/backup/repos/{{host}}"
- no-port-forwarding
- no-X11-forwarding
- no-pty
- no-agent-forwarding
- no-user-rc
{% endfor %}
Enhancements
------------
As this chapter only describes a simple and effective setup it could be further
enhanced when supporting (a limited set) of client supplied commands. A wrapper
for starting `borg serve` could be written. Or borg itself could be enhanced to
autodetect it runs under SSH by checking the `SSH_ORIGINAL_COMMAND` environment
variable. This is left open for future improvements.
When extending ssh autodetection in borg no external wrapper script is necessary
and no other interpreter or application has to be deployed.
See also
--------
* `SSH Daemon manpage <http://www.openbsd.org/cgi-bin/man.cgi/OpenBSD-current/man8/sshd.8>`_
* `Ansible <https://docs.ansible.com>`_
* `Salt <https://docs.saltstack.com/>`_
deployment/central-backup-server
deployment/hosting-repositories
deployment/automated-local

231
docs/deployment/automated-local.rst Normal file
View File

@ -0,0 +1,231 @@
.. include:: ../global.rst.inc
.. highlight:: none
Automated backups to a local hard drive
=======================================
This guide shows how to automate backups to a hard drive directly connected
to your computer. If a backup hard drive is connected, backups are automatically
started, and the drive shut-down and disconnected when they are done.
This guide is written for a Linux-based operating system and makes use of
systemd and udev.
Overview
--------
An udev rule is created to trigger on the addition of block devices. The rule contains a tag
that triggers systemd to start a oneshot service. The oneshot service executes a script in
the standard systemd service environment, which automatically captures stdout/stderr and
logs it to the journal.
The script mounts the added block device, if it is a registered backup drive, and creates
backups on it. When done, it optionally unmounts the file system and spins the drive down,
so that it may be physically disconnected.
Configuring the system
----------------------
First, create the ``/etc/backups`` directory (as root).
All configuration goes into this directory.
Then, create ``etc/backups/40-backup.rules`` with the following content (all on one line)::
ACTION=="add", SUBSYSTEM=="bdi", DEVPATH=="/devices/virtual/bdi/*",
TAG+="systemd", ENV{SYSTEMD_WANTS}="automatic-backup.service"
.. topic:: Finding a more precise udev rule
If you always connect the drive(s) to the same physical hardware path, e.g. the same
eSATA port, then you can make a more precise udev rule.
Execute ``udevadm monitor`` and connect a drive to the port you intend to use.
You should see a flurry of events, find those regarding the `block` subsystem.
Pick the event whose device path ends in something similar to a device file name,
typically`sdX/sdXY`. Use the event's device path and replace `sdX/sdXY` after the
`/block/` part in the path with a star (\*). For example:
`DEVPATH=="/devices/pci0000:00/0000:00:11.0/ata3/host2/target2:0:0/2:0:0:0/block/*"`.
Reboot a few times to ensure that the hardware path does not change: on some motherboards
components of it can be random. In these cases you cannot use a more accurate rule,
or need to insert additional stars for matching the path.
The "systemd" tag in conjunction with the SYSTEMD_WANTS environment variable has systemd
launch the "automatic-backup" service, which we will create next, as the
``/etc/backups/automatic-backup.service`` file:
.. code-block:: ini
[Service]
Type=oneshot
ExecStart=/etc/backups/run.sh
Now, create the main backup script, ``/etc/backups/run.sh``. Below is a template,
modify it to suit your needs (e.g. more backup sets, dumping databases etc.).
.. code-block:: bash
#!/bin/bash -ue
# The udev rule is not terribly accurate and may trigger our service before
# the kernel has finished probing partitions. Sleep for a bit to ensure
# the kernel is done.
#
# This can be avoided by using a more precise udev rule, e.g. matching
# a specific hardware path and partition.
sleep 5
#
# Script configuration
#
# The backup partition is mounted there
MOUNTPOINT=/mnt/backup
# This is the location of the Borg repository
TARGET=$MOUNTPOINT/borg-backups/backup.borg
# Archive name schema
DATE=$(date --iso-8601)-$(hostname)
# This is the file that will later contain UUIDs of registered backup drives
DISKS=/etc/backups/backup.disks
# Find whether the connected block device is a backup drive
for uuid in $(lsblk --noheadings --list --output uuid)
do
if grep --quiet --fixed-strings $uuid $DISKS; then
break
fi
uuid=
done
if [ ! $uuid ]; then
echo "No backup disk found, exiting"
exit 0
fi
echo "Disk $uuid is a backup disk"
partition_path=/dev/disk/by-uuid/$uuid
# Mount file system if not already done. This assumes that if something is already
# mounted at $MOUNTPOINT, it is the backup drive. It won't find the drive if
# it was mounted somewhere else.
(mount | grep $MOUNTPOINT) || mount $partition_path $MOUNTPOINT
drive=$(lsblk --inverse --noheadings --list --paths --output name $partition_path | head --lines 1)
echo "Drive path: $drive"
#
# Create backups
#
# Options for borg create
BORG_OPTS="--stats --one-file-system --compression lz4 --checkpoint-interval 86400"
# Set BORG_PASSPHRASE or BORG_PASSCOMMAND somewhere around here, using export,
# if encryption is used.
# No one can answer if Borg asks these questions, it is better to just fail quickly
# instead of hanging.
export BORG_RELOCATED_REPO_ACCESS_IS_OK=no
export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no
# Log Borg version
borg --version
echo "Starting backup for $DATE"
# This is just an example, change it however you see fit
borg create $BORG_OPTS \
--exclude /root/.cache \
--exclude /var/cache \
--exclude /var/lib/docker/devicemapper \
$TARGET::$DATE-$$-system \
/ /boot
# /home is often a separate partition / file system.
# Even if it isn't (add --exclude /home above), it probably makes sense
# to have /home in a separate archive.
borg create $BORG_OPTS \
--exclude 'sh:/home/*/.cache' \
$TARGET::$DATE-$$-home \
/home/
echo "Completed backup for $DATE"
# Just to be completely paranoid
sync
if [ -f /etc/backups/autoeject ]; then
umount $MOUNTPOINT
hdparm -Y $drive
fi
if [ -f /etc/backups/backup-suspend ]; then
systemctl suspend
fi
Create the ``/etc/backups/autoeject`` file to have the script automatically eject the drive
after creating the backup. Rename the file to something else (e.g. ``/etc/backup/autoeject-no``)
when you want to do something with the drive after creating backups (e.g running check).
Create the ``/etc/backups/backup-suspend`` file if the machine should suspend after completing
the backup. Don't forget to physically disconnect the device before resuming,
otherwise you'll enter a cycle. You can also add an option to power down instead.
Create an empty ``/etc/backups/backup.disks`` file, you'll register your backup drives
there.
The last part is to actually enable the udev rules and services:
.. code-block:: bash
ln -s /etc/backups/40-backup.rules /etc/udev/rules.d/40-backup.rules
ln -s /etc/backups/automatic-backup.service /etc/systemd/system/automatic-backup.service
systemctl daemon-reload
udevadm control --reload
Adding backup hard drives
-------------------------
Connect your backup hard drive. Format it, if not done already.
Find the UUID of the file system that backups should be stored on::
lsblk -o+uuid,label
Note the UUID into the ``/etc/backup/backup.disks`` file.
Mount the drive to /mnt/backup.
Initialize a Borg repository at the location indicated by ``TARGET``::
borg init --encryption ... /mnt/backup/borg-backups/backup.borg
Unmount and reconnect the drive, or manually start the ``automatic-backup`` service
to start the first backup::
systemctl start --no-block automatic-backup
See backup logs using journalctl::
journalctl -fu automatic-backup [-n number-of-lines]
Security considerations
-----------------------
The script as shown above will mount any file system with an UUID listed in
``/etc/backup/backup.disks``. The UUID check is a safety / annoyance-reduction
mechanism to keep the script from blowing up whenever a random USB thumb drive is connected.
It is not meant as a security mechanism. Mounting file systems and reading repository
data exposes additional attack surfaces (kernel file system drivers,
possibly user space services and Borg itself). On the other hand, someone
standing right next to your computer can attempt a lot of attacks, most of which
are easier to do than e.g. exploiting file systems (installing a physical key logger,
DMA attacks, stealing the machine, ...).
Borg ensures that backups are not created on random drives that "just happen"
to contain a Borg repository. If an unknown unencrypted repository is encountered,
then the script aborts (BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no).
Backups are only created on hard drives that contain a Borg repository that is
either known (by ID) to your machine or you are using encryption and the
passphrase of the repository has to match the passphrase supplied to Borg.

221
docs/deployment/central-backup-server.rst Normal file
View File

@ -0,0 +1,221 @@
.. include:: ../global.rst.inc
.. highlight:: none
Central repository server with Ansible or Salt
==============================================
This section will give an example how to setup a borg repository server for multiple
clients.
Machines
--------
There are multiple machines used in this section and will further be named by their
respective fully qualified domain name (fqdn).
* The backup server: `backup01.srv.local`
* The clients:
- John Doe's desktop: `johndoe.clnt.local`
- Webserver 01: `web01.srv.local`
- Application server 01: `app01.srv.local`
User and group
--------------
The repository server needs to have only one UNIX user for all the clients.
Recommended user and group with additional settings:
* User: `backup`
* Group: `backup`
* Shell: `/bin/bash` (or other capable to run the `borg serve` command)
* Home: `/home/backup`
Most clients shall initiate a backup from the root user to catch all
users, groups and permissions (e.g. when backing up `/home`).
Folders
-------
The following folder tree layout is suggested on the repository server:
* User home directory, /home/backup
* Repositories path (storage pool): /home/backup/repos
* Clients restricted paths (`/home/backup/repos/<client fqdn>`):
- johndoe.clnt.local: `/home/backup/repos/johndoe.clnt.local`
- web01.srv.local: `/home/backup/repos/web01.srv.local`
- app01.srv.local: `/home/backup/repos/app01.srv.local`
Restrictions
------------
Borg is instructed to restrict clients into their own paths:
``borg serve --restrict-to-path /home/backup/repos/<client fqdn>``
The client will be able to access any file or subdirectory inside of ``/home/backup/repos/<client fqdn>``
but no other directories. You can allow a client to access several separate directories by passing multiple
``--restrict-to-path`` flags, for instance: ``borg serve --restrict-to-path /home/backup/repos/<client fqdn> --restrict-to-path /home/backup/repos/<other client fqdn>``,
which could make sense if multiple machines belong to one person which should then have access to all the
backups of their machines.
There is only one ssh key per client allowed. Keys are added for ``johndoe.clnt.local``, ``web01.srv.local`` and
``app01.srv.local``. But they will access the backup under only one UNIX user account as:
``backup@backup01.srv.local``. Every key in ``$HOME/.ssh/authorized_keys`` has a
forced command and restrictions applied as shown below:
::
command="cd /home/backup/repos/<client fqdn>;
borg serve --restrict-to-path /home/backup/repos/<client fqdn>",
restrict <keytype> <key> <host>
.. note:: The text shown above needs to be written on a single line!
The options which are added to the key will perform the following:
1. Change working directory
2. Run ``borg serve`` restricted to the client base path
3. Restrict ssh and do not allow stuff which imposes a security risk
Due to the ``cd`` command we use, the server automatically changes the current
working directory. Then client doesn't need to have knowledge of the absolute
or relative remote repository path and can directly access the repositories at
``<user>@<host>:<repo>``.
.. note:: The setup above ignores all client given commandline parameters
which are normally appended to the `borg serve` command.
Client
------
The client needs to initialize the `pictures` repository like this:
borg init backup@backup01.srv.local:pictures
Or with the full path (should actually never be used, as only for demonstrational purposes).
The server should automatically change the current working directory to the `<client fqdn>` folder.
borg init backup@backup01.srv.local:/home/backup/repos/johndoe.clnt.local/pictures
When `johndoe.clnt.local` tries to access a not restricted path the following error is raised.
John Doe tries to backup into the Web 01 path:
borg init backup@backup01.srv.local:/home/backup/repos/web01.srv.local/pictures
::
~~~ SNIP ~~~
Remote: borg.remote.PathNotAllowed: /home/backup/repos/web01.srv.local/pictures
~~~ SNIP ~~~
Repository path not allowed
Ansible
-------
Ansible takes care of all the system-specific commands to add the user, create the
folder. Even when the configuration is changed the repository server configuration is
satisfied and reproducible.
Automate setting up an repository server with the user, group, folders and
permissions a Ansible playbook could be used. Keep in mind the playbook
uses the Arch Linux `pacman <https://www.archlinux.org/pacman/pacman.8.html>`_
package manager to install and keep borg up-to-date.
::
- hosts: backup01.srv.local
vars:
user: backup
group: backup
home: /home/backup
pool: "{{ home }}/repos"
auth_users:
- host: johndoe.clnt.local
key: "{{ lookup('file', '/path/to/keys/johndoe.clnt.local.pub') }}"
- host: web01.clnt.local
key: "{{ lookup('file', '/path/to/keys/web01.clnt.local.pub') }}"
- host: app01.clnt.local
key: "{{ lookup('file', '/path/to/keys/app01.clnt.local.pub') }}"
tasks:
- pacman: name=borg state=latest update_cache=yes
- group: name="{{ group }}" state=present
- user: name="{{ user }}" shell=/bin/bash home="{{ home }}" createhome=yes group="{{ group }}" groups= state=present
- file: path="{{ home }}" owner="{{ user }}" group="{{ group }}" mode=0700 state=directory
- file: path="{{ home }}/.ssh" owner="{{ user }}" group="{{ group }}" mode=0700 state=directory
- file: path="{{ pool }}" owner="{{ user }}" group="{{ group }}" mode=0700 state=directory
- authorized_key: user="{{ user }}"
key="{{ item.key }}"
key_options='command="cd {{ pool }}/{{ item.host }};borg serve --restrict-to-path {{ pool }}/{{ item.host }}",restrict'
with_items: "{{ auth_users }}"
- file: path="{{ home }}/.ssh/authorized_keys" owner="{{ user }}" group="{{ group }}" mode=0600 state=file
- file: path="{{ pool }}/{{ item.host }}" owner="{{ user }}" group="{{ group }}" mode=0700 state=directory
with_items: "{{ auth_users }}"
Salt
----
This is a configuration similar to the one above, configured to be deployed with
Salt running on a Debian system.
::
Install borg backup from pip:
pkg.installed:
- pkgs:
- python3
- python3-dev
- python3-pip
- python-virtualenv
- libssl-dev
- openssl
- libacl1-dev
- libacl1
- liblz4-dev
- liblz4-1
- build-essential
- libfuse-dev
- fuse
- pkg-config
pip.installed:
- pkgs: ["borgbackup"]
- bin_env: /usr/bin/pip3
Setup backup user:
user.present:
- name: backup
- fullname: Backup User
- home: /home/backup
- shell: /bin/bash
# CAUTION!
# If you change the ssh command= option below, it won't necessarily get pushed to the backup
# server correctly unless you delete the ~/.ssh/authorized_keys file and re-create it!
{% for host in backupclients %}
Give backup access to {{host}}:
ssh_auth.present:
- user: backup
- source: salt://conf/ssh-pubkeys/{{host}}-backup.id_ecdsa.pub
- options:
- command="cd /home/backup/repos/{{host}}; borg serve --restrict-to-path /home/backup/repos/{{host}}"
- restrict
{% endfor %}
Enhancements
------------
As this section only describes a simple and effective setup it could be further
enhanced when supporting (a limited set) of client supplied commands. A wrapper
for starting `borg serve` could be written. Or borg itself could be enhanced to
autodetect it runs under SSH by checking the `SSH_ORIGINAL_COMMAND` environment
variable. This is left open for future improvements.
When extending ssh autodetection in borg no external wrapper script is necessary
and no other interpreter or application has to be deployed.
See also
--------
* `SSH Daemon manpage <http://www.openbsd.org/cgi-bin/man.cgi/OpenBSD-current/man8/sshd.8>`_
* `Ansible <https://docs.ansible.com>`_
* `Salt <https://docs.saltstack.com/>`_

73
docs/deployment/hosting-repositories.rst Normal file
View File

@ -0,0 +1,73 @@
.. include:: ../global.rst.inc
.. highlight:: none
Hosting repositories
====================
This sections shows how to securely provide repository storage for users.
Optionally, each user can have a storage quota.
Repositories are accessed through SSH. Each user of the service should
have her own login which is only able to access the user's files.
Technically it would be possible to have multiple users share one login,
however, separating them is better. Separate logins increase isolation
and are thus an additional layer of security and safety for both the
provider and the users.
For example, if a user manages to breach ``borg serve`` then she can
only damage her own data (assuming that the system does not have further
vulnerabilities).
Use the standard directory structure of the operating system. Each user
is assigned a home directory and repositories of the user reside in her
home directory.
The following ``~user/.ssh/authorized_keys`` file is the most important
piece for a correct deployment. It allows the user to login via
their public key (which must be provided by the user), and restricts
SSH access to safe operations only.
::
command="borg serve --restrict-to-repository /home/<user>/repository",restrict
<key type> <key> <key host>
.. note:: The text shown above needs to be written on a **single** line!
.. warning::
If this file should be automatically updated (e.g. by a web console),
pay **utmost attention** to sanitizing user input. Strip all whitespace
around the user-supplied key, ensure that it **only** contains ASCII
with no control characters and that it consists of three parts separated
by a single space. Ensure that no newlines are contained within the key.
The `restrict` keyword enables all restrictions, i.e. disables port, agent
and X11 forwarding, as well as disabling PTY allocation and execution of ~/.ssh/rc.
If any future restriction capabilities are added to authorized_keys
files they will be included in this set.
The `command` keyword forces execution of the specified command line
upon login. This must be ``borg serve``. The `--restrict-to-repository`
option permits access to exactly **one** repository. It can be given
multiple times to permit access to more than one repository.
The repository may not exist yet; it can be initialized by the user,
which allows for encryption.
**Storage quotas** can be enabled by adding the ``--storage-quota`` option
to the ``borg serve`` command line::
restrict,command="borg serve --storage-quota 20G ..." ...
The storage quotas of repositories are completely independent. If a
client is able to access multiple repositories, each repository
can be filled to the specified quota.
If storage quotas are used, ensure that all deployed Borg releases
support storage quotas.
Refer to :ref:`internals_storage_quota` for more details on storage quotas.
Refer to the `sshd(8) <http://www.openbsd.org/cgi-bin/man.cgi/OpenBSD-current/man8/sshd.8>`_
man page for more details on SSH options.

196
docs/development.rst
View File

@ -10,6 +10,115 @@ This chapter will get you started with |project_name| development.
|project_name| is written in Python (with a little bit of Cython and C for
the performance critical parts).
Contributions
-------------
... are welcome!
Some guidance for contributors:
- discuss changes on the GitHub issue tracker, on IRC or on the mailing list
- make your PRs on the ``master`` branch (see `Branching Model`_ for details)
- do clean changesets:
- focus on some topic, resist changing anything else.
- do not do style changes mixed with functional changes.
- try to avoid refactorings mixed with functional changes.
- if you need to fix something after commit/push:
- if there are ongoing reviews: do a fixup commit you can
squash into the bad commit later.
- if there are no ongoing reviews or you did not push the
bad commit yet: amend the commit to include your fix or
merge the fixup commit before pushing.
- have a nice, clear, typo-free commit comment
- if you fixed an issue, refer to it in your commit comment
- follow the style guide (see below)
- if you write new code, please add tests and docs for it
- run the tests, fix any issues that come up
- make a pull request on GitHub
- wait for review by other developers
Branching model
---------------
Borg development happens on the ``master`` branch and uses GitHub pull
requests (if you don't have GitHub or don't want to use it you can
send smaller patches via the borgbackup :ref:`mailing_list` to the maintainers).
Stable releases are maintained on maintenance branches named ``x.y-maint``, eg.
the maintenance branch of the 1.0.x series is ``1.0-maint``.
Most PRs should be filed against the ``master`` branch. Only if an
issue affects **only** a particular maintenance branch a PR should be
filed against it directly.
While discussing / reviewing a PR it will be decided whether the
change should be applied to maintenance branches. Each maintenance
branch has a corresponding *backport/x.y-maint* label, which will then
be applied.
Changes that are typically considered for backporting:
- Data loss, corruption and inaccessibility fixes
- Security fixes
- Forward-compatibility improvements
- Documentation corrections
.. rubric:: Maintainer part
From time to time a maintainer will backport the changes for a
maintenance branch, typically before a release or if enough changes
were collected:
1. Notify others that you're doing this to avoid duplicate work.
2. Branch a backporting branch off the maintenance branch.
3. Cherry pick and backport the changes from each labelled PR, remove
the label for each PR you've backported.
To preserve authorship metadata, do not follow the ``git cherry-pick``
instructions to use ``git commit`` after resolving conflicts. Instead,
stage conflict resolutions and run ``git cherry-pick --continue``,
much like using ``git rebase``.
To avoid merge issues (a cherry pick is a form of merge), use
these options (similar to the ``git merge`` options used previously,
the ``-x`` option adds a reference to the original commit)::
git cherry-pick --strategy recursive -X rename-threshold=5% -x
4. Make a PR of the backporting branch against the maintenance branch
for backport review. Mention the backported PRs in this PR, e.g.:
Includes changes from #2055 #2057 #2381
This way GitHub will automatically show in these PRs where they
were backported.
.. rubric:: Historic model
Previously (until release 1.0.10) Borg used a `"merge upwards"
<https://git-scm.com/docs/gitworkflows#_merging_upwards>`_ model where
most minor changes and fixes where committed to a maintenance branch
(eg. 1.0-maint), and the maintenance branch(es) were regularly merged
back into the main development branch. This became more and more
troublesome due to merges growing more conflict-heavy and error-prone.
Code and issues
---------------
Code is stored on GitHub, in the `Borgbackup organization
<https://github.com/borgbackup/borg/>`_. `Issues
<https://github.com/borgbackup/borg/issues>`_ and `pull requests
<https://github.com/borgbackup/borg/pulls>`_ should be sent there as
well. See also the :ref:`support` section for more details.
Style guide
-----------
@ -19,6 +128,17 @@ instead of 79. We do *not* use form-feed (``^L``) characters to
separate sections either. Compliance is tested automatically when
you run the tests.
Continuous Integration
----------------------
All pull requests go through Travis-CI_, which runs the tests on Linux
and Mac OS X as well as the flake8 style checker. Windows builds run on AppVeyor_,
while additional Unix-like platforms are tested on Golem_.
.. _AppVeyor: https://ci.appveyor.com/project/borgbackup/borg/
.. _Golem: https://golem.enkore.de/view/Borg/
.. _Travis-CI: https://travis-ci.org/borgbackup/borg
Output and Logging
------------------
When writing logger calls, always use correct log level (debug only for
@ -76,7 +196,7 @@ Some more advanced examples::
# verify a changed tox.ini (run this after any change to tox.ini):
fakeroot -u tox --recreate
fakeroot -u tox -e py34 # run all tests, but only on python 3.4
fakeroot -u tox -e py35 # run all tests, but only on python 3.5
fakeroot -u tox borg.testsuite.locking # only run 1 test module
@ -92,31 +212,52 @@ As tox doesn't run on Windows you have to manually run command::
py.test --cov=borg --cov-config=.coveragerc --benchmark-skip --pyargs borg/testsuite
Regenerate usage files
----------------------
Running more checks using coala
-------------------------------
Usage and API documentation is currently committed directly to git,
although those files are generated automatically from the source
tree.
First install coala and some checkers ("bears"):
When a new module is added, the ``docs/api.rst`` file needs to be
regenerated::
pip install -r requirements.d/coala.txt
./setup.py build_api
You can now run coala from the toplevel directory; it will read its settings
from ``.coafile`` there:
When a command is added, a commandline flag changed, added or removed,
coala
Some bears have additional requirements and they usually tell you about
them in case they are missing.
Documentation
-------------
Generated files
~~~~~~~~~~~~~~~
Usage documentation (found in ``docs/usage/``) and man pages
(``docs/man/``) are generated automatically from the command line
parsers declared in the program and their documentation, which is
embedded in the program (see archiver.py). These are committed to git
for easier use by packagers downstream.
When a command is added, a command line flag changed, added or removed,
the usage docs need to be rebuilt as well::
./setup.py build_usage
python setup.py build_usage
python setup.py build_man
However, we prefer to do this as part of our :ref:`releasing`
preparations, so it is generally not necessary to update these when
submitting patches that change something about the command line.
Building the docs with Sphinx
-----------------------------
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The documentation (in reStructuredText format, .rst) is in docs/.
To build the html version of it, you need to have sphinx installed::
To build the html version of it, you need to have Sphinx installed
(in your Borg virtualenv with Python 3)::
pip3 install sphinx # important: this will install sphinx with Python 3
pip install -r requirements.d/docs.txt
Now run::
@ -125,7 +266,7 @@ Now run::
Then point a web browser at docs/_build/html/index.html.
The website is updated automatically through Github web hooks on the
The website is updated automatically by ReadTheDocs through GitHub web hooks on the
main repository.
Using Vagrant
@ -137,10 +278,14 @@ standalone binaries for various platforms.
For better security, there is no automatic sync in the VM to host direction.
The plugin `vagrant-scp` is useful to copy stuff from the VMs to the host.
The "windows10" box requires the `reload` plugin (``vagrant plugin install vagrant-reload``).
Usage::
# To create and provision the VM:
vagrant up OS
# same, but use 6 VM cpus and 12 workers for pytest:
VMCPUS=6 XDISTN=12 vagrant up OS
# To create an ssh session to the VM:
vagrant ssh OS
# To execute a command via ssh in the VM:
@ -175,6 +320,8 @@ On Windows use `python buildwin32.py` to build standalone executable in `win32ex
with all necessary files to run.
.. _releasing:
Creating a new release
----------------------
@ -187,25 +334,36 @@ Checklist:
- update ``CHANGES.rst``, based on ``git log $PREVIOUS_RELEASE..``
- check version number of upcoming release in ``CHANGES.rst``
- verify that ``MANIFEST.in`` and ``setup.py`` are complete
- ``python setup.py build_api ; python setup.py build_usage`` and commit
- ``python setup.py build_usage ; python setup.py build_man`` and
commit (be sure to build with Python 3.5 as Python 3.6 added `more
guaranteed hashing algorithms
<https://github.com/borgbackup/borg/issues/2123>`_)
- tag the release::
git tag -s -m "tagged/signed release X.Y.Z" X.Y.Z
- create a clean repo and use it for the following steps::
git clone borg borg-clean
This makes sure no uncommitted files get into the release archive.
It will also reveal uncommitted required files.
Moreover, it makes sure the vagrant machines only get committed files and
do a fresh start based on that.
- run tox and/or binary builds on all supported platforms via vagrant,
check for test failures
- create a release on PyPi::
python setup.py register sdist upload --identity="Thomas Waldmann" --sign
- close release milestone on Github
- close the release milestone on GitHub
- announce on:
- Mailing list
- Twitter (follow @ThomasJWaldmann for these tweets)
- Twitter
- IRC channel (change ``/topic``)
- create a Github release, include:
- create a GitHub release, include:
* standalone binaries (see above for how to create them)

649
docs/faq.rst
View File

@ -5,6 +5,9 @@
Frequently asked questions
==========================
Usage & Limitations
###################
Can I backup VM disk images?
----------------------------
@ -12,6 +15,39 @@ Yes, the `deduplication`_ technique used by
|project_name| makes sure only the modified parts of the file are stored.
Also, we have optional simple sparse file support for extract.
If you use non-snapshotting backup tools like Borg to back up virtual machines,
then the VMs should be turned off for the duration of the backup. Backing up live VMs can (and will)
result in corrupted or inconsistent backup contents: a VM image is just a regular file to
Borg with the same issues as regular files when it comes to concurrent reading and writing from
the same file.
For backing up live VMs use file system snapshots on the VM host, which establishes
crash-consistency for the VM images. This means that with most file systems
(that are journaling) the FS will always be fine in the backup (but may need a
journal replay to become accessible).
Usually this does not mean that file *contents* on the VM are consistent, since file
contents are normally not journaled. Notable exceptions are ext4 in data=journal mode,
ZFS and btrfs (unless nodatacow is used).
Applications designed with crash-consistency in mind (most relational databases
like PostgreSQL, SQLite etc. but also for example Borg repositories) should always
be able to recover to a consistent state from a backup created with
crash-consistent snapshots (even on ext4 with data=writeback or XFS).
Hypervisor snapshots capturing most of the VM's state can also be used for backups
and can be a better alternative to pure file system based snapshots of the VM's disk,
since no state is lost. Depending on the application this can be the easiest and most
reliable way to create application-consistent backups.
Other applications may require a lot of work to reach application-consistency:
It's a broad and complex issue that cannot be explained in entirety here.
Borg doesn't intend to address these issues due to their huge complexity
and platform/software dependency. Combining Borg with the mechanisms provided
by the platform (snapshots, hypervisor features) will be the best approach
to start tackling them.
Can I backup from multiple servers into a single repository?
------------------------------------------------------------
@ -31,7 +67,7 @@ Can I copy or synchronize my repo to another location?
------------------------------------------------------
Yes, you could just copy all the files. Make sure you do that while no
backup is running. So what you get here is this:
backup is running (use `borg with-lock ...`). So what you get here is this:
- client machine ---borg create---> repo1
- repo1 ---copy---> repo2
@ -47,25 +83,6 @@ If you want to have 2 independent backups, it is better to do it like this:
- client machine ---borg create---> repo1
- client machine ---borg create---> repo2
Which file types, attributes, etc. are preserved?
-------------------------------------------------
* Directories
* Regular files
* Hardlinks (considering all files in the same archive)
* Symlinks (stored as symlink, the symlink is not followed)
* Character and block device files
* FIFOs ("named pipes")
* Name
* Contents
* Timestamps in nanosecond precision: mtime, atime, ctime
* IDs of owning user and owning group
* Names of owning user and owning group (if the IDs can be resolved)
* Unix Mode/Permissions (u/g/o permissions, suid, sgid, sticky)
* Extended Attributes (xattrs) on Linux, OS X and FreeBSD
* Access Control Lists (ACL_) on Linux, OS X and FreeBSD
* BSD flags on OS X and FreeBSD
Which file types, attributes, etc. are *not* preserved?
-------------------------------------------------------
@ -79,34 +96,215 @@ Which file types, attributes, etc. are *not* preserved?
backed up as (deduplicated and compressed) runs of zero bytes.
Archive extraction has optional support to extract all-zero chunks as
holes in a sparse file.
* filesystem specific attributes, like ext4 immutable bit, see :issue:`618`.
* Some filesystem specific attributes, like btrfs NOCOW, see :ref:`platforms`.
* For hardlinked symlinks, the hardlinking can not be archived (and thus,
the hardlinking will not be done at extraction time). The symlinks will
be archived and extracted as non-hardlinked symlinks, see :issue:`2379`.
Why is my backup bigger than with attic? Why doesn't |project_name| do compression by default?
----------------------------------------------------------------------------------------------
Are there other known limitations?
----------------------------------
Attic was rather unflexible when it comes to compression, it always
compressed using zlib level 6 (no way to switch compression off or
adjust the level or algorithm).
- A single archive can only reference a limited volume of file/dir metadata,
usually corresponding to tens or hundreds of millions of files/dirs.
When trying to go beyond that limit, you will get a fatal IntegrityError
exception telling that the (archive) object is too big.
An easy workaround is to create multiple archives with less items each.
See also the :ref:`archive_limitation` and :issue:`1452`.
|project_name| offers a lot of different compression algorithms and
levels. Which of them is the best for you pretty much depends on your
use case, your data, your hardware -- so you need to do an informed
decision about whether you want to use compression, which algorithm
and which level you want to use. This is why compression defaults to
none.
:ref:`borg_info` shows how large (relative to the maximum size) existing
archives are.
.. _checkpoints_parts:
If a backup stops mid-way, does the already-backed-up data stay there?
----------------------------------------------------------------------
Yes, |project_name| supports resuming backups.
During a backup a special checkpoint archive named ``<archive-name>.checkpoint``
is saved every checkpoint interval (the default value for this is 30
minutes) containing all the data backed-up until that point.
This checkpoint archive is a valid archive,
but it is only a partial backup (not all files that you wanted to backup are
contained in it). Having it in the repo until a successful, full backup is
completed is useful because it references all the transmitted chunks up
to the checkpoint. This means that in case of an interruption, you only need to
retransfer the data since the last checkpoint.
If a backup was interrupted, you do not need to do any special considerations,
just invoke ``borg create`` as you always do. You may use the same archive name
as in previous attempt or a different one (e.g. if you always include the current
datetime), it does not matter.
|project_name| always does full single-pass backups, so it will start again
from the beginning - but it will be much faster, because some of the data was
already stored into the repo (and is still referenced by the checkpoint
archive), so it does not need to get transmitted and stored again.
Once your backup has finished successfully, you can delete all
``<archive-name>.checkpoint`` archives. If you run ``borg prune``, it will
also care for deleting unneeded checkpoints.
Note: the checkpointing mechanism creates hidden, partial files in an archive,
so that checkpoints even work while a big file is being processed.
They are named ``<filename>.borg_part_<N>`` and all operations usually ignore
these files, but you can make them considered by giving the option
``--consider-part-files``. You usually only need that option if you are
really desperate (e.g. if you have no completed backup of that file and you'ld
rather get a partial file extracted than nothing). You do **not** want to give
that option under any normal circumstances.
How can I backup huge file(s) over a unstable connection?
---------------------------------------------------------
This is not a problem anymore.
For more details, see :ref:`checkpoints_parts`.
How can I restore huge file(s) over an unstable connection?
-----------------------------------------------------------
If you cannot manage to extract the whole big file in one go, you can extract
all the part files and manually concatenate them together.
For more details, see :ref:`checkpoints_parts`.
Can |project_name| add redundancy to the backup data to deal with hardware malfunction?
---------------------------------------------------------------------------------------
No, it can't. While that at first sounds like a good idea to defend against
some defect HDD sectors or SSD flash blocks, dealing with this in a
reliable way needs a lot of low-level storage layout information and
control which we do not have (and also can't get, even if we wanted).
So, if you need that, consider RAID or a filesystem that offers redundant
storage or just make backups to different locations / different hardware.
See also :issue:`225`.
Can |project_name| verify data integrity of a backup archive?
-------------------------------------------------------------
Yes, if you want to detect accidental data damage (like bit rot), use the
``check`` operation. It will notice corruption using CRCs and hashes.
If you want to be able to detect malicious tampering also, use an encrypted
repo. It will then be able to check using CRCs and HMACs.
Can I use Borg on SMR hard drives?
----------------------------------
SMR (shingled magnetic recording) hard drives are very different from
regular hard drives. Applications have to behave in certain ways or
performance will be heavily degraded.
Borg 1.1 ships with default settings suitable for SMR drives,
and has been successfully tested on *Seagate Archive v2* drives
using the ext4 file system.
Some Linux kernel versions between 3.19 and 4.5 had various bugs
handling device-managed SMR drives, leading to IO errors, unresponsive
drives and unreliable operation in general.
For more details, refer to :issue:`2252`.
.. _faq-integrityerror:
I get an IntegrityError or similar - what now?
----------------------------------------------
A single error does not necessarily indicate bad hardware or a Borg
bug. All hardware exhibits a bit error rate (BER). Hard drives are typically
specified as exhibiting less than one error every 12 to 120 TB
(one bit error in 10e14 to 10e15 bits). The specification is often called
*unrecoverable read error rate* (URE rate).
Apart from these very rare errors there are two main causes of errors:
(i) Defective hardware: described below.
(ii) Bugs in software (Borg, operating system, libraries):
Ensure software is up to date.
Check whether the issue is caused by any fixed bugs described in :ref:`important_notes`.
.. rubric:: Finding defective hardware
.. note::
Hardware diagnostics are operating system dependent and do not
apply universally. The commands shown apply for popular Unix-like
systems. Refer to your operating system's manual.
Checking hard drives
Find the drive containing the repository and use *findmnt*, *mount* or *lsblk*
to learn the device path (typically */dev/...*) of the drive.
Then, smartmontools can retrieve self-diagnostics of the drive in question::
# smartctl -a /dev/sdSomething
The *Offline_Uncorrectable*, *Current_Pending_Sector* and *Reported_Uncorrect*
attributes indicate data corruption. A high *UDMA_CRC_Error_Count* usually
indicates a bad cable.
I/O errors logged by the system (refer to the system journal or
dmesg) can point to issues as well. I/O errors only affecting the
file system easily go unnoticed, since they are not reported to
applications (e.g. Borg), while these errors can still corrupt data.
Drives can corrupt some sectors in one event, while remaining
reliable otherwise. Conversely, drives can fail completely with no
advance warning. If in doubt, copy all data from the drive in
question to another drive -- just in case it fails completely.
If any of these are suspicious, a self-test is recommended::
# smartctl -t long /dev/sdSomething
Running ``fsck`` if not done already might yield further insights.
Checking memory
Intermittent issues, such as ``borg check`` finding errors
inconsistently between runs, are frequently caused by bad memory.
Run memtest86+ (or an equivalent memory tester) to verify that
the memory subsystem is operating correctly.
Checking processors
Processors rarely cause errors. If they do, they are usually overclocked
or otherwise operated outside their specifications. We do not recommend to
operate hardware outside its specifications for productive use.
Tools to verify correct processor operation include Prime95 (mprime), linpack,
and the `Intel Processor Diagnostic Tool
<https://downloadcenter.intel.com/download/19792/Intel-Processor-Diagnostic-Tool>`_
(applies only to Intel processors).
.. rubric:: Repairing a damaged repository
With any defective hardware found and replaced, the damage done to the repository
needs to be ascertained and fixed.
:ref:`borg_check` provides diagnostics and ``--repair`` options for repositories with
issues. We recommend to first run without ``--repair`` to assess the situation.
If the found issues and proposed repairs seem right, re-run "check" with ``--repair`` enabled.
Security
########
How can I specify the encryption passphrase programmatically?
-------------------------------------------------------------
The encryption passphrase can be specified programmatically using the
`BORG_PASSPHRASE` environment variable. This is convenient when setting up
automated encrypted backups. Another option is to use
key file based encryption with a blank passphrase. See
:ref:`encrypted_repos` for more details.
There are several ways to specify a passphrase without human intervention:
.. _password_env:
.. note:: Be careful how you set the environment; using the ``env``
Setting ``BORG_PASSPHRASE``
The passphrase can be specified using the ``BORG_PASSPHRASE`` enviroment variable.
This is often the simplest option, but can be insecure if the script that sets it
is world-readable.
.. _password_env:
.. note:: Be careful how you set the environment; using the ``env``
command, a ``system()`` call or using inline shell scripts
(e.g. ``BORG_PASSPHRASE=hunter2 borg ...``)
might expose the credentials in the process list directly
and they will be readable to all users on a system. Using
``export`` in a shell script file should be safe, however, as
@ -114,6 +312,84 @@ key file based encryption with a blank passphrase. See
user
<https://security.stackexchange.com/questions/14000/environment-variable-accessibility-in-linux/14009#14009>`_.
Using ``BORG_PASSCOMMAND`` with a properly permissioned file
Another option is to create a file with a password in it in your home
directory and use permissions to keep anyone else from reading it. For
example, first create a key::
head -c 1024 /dev/urandom | base64 > ~/.borg-passphrase
chmod 400 ~/.borg-passphrase
Then in an automated script one can put::
export BORG_PASSCOMMAND="cat ~/.borg-passphrase"
and Borg will automatically use that passphrase.
Using keyfile-based encryption with a blank passphrase
It is possible to encrypt your repository in ``keyfile`` mode instead of the default
``repokey`` mode and use a blank passphrase for the key file (simply press Enter twice
when ``borg init`` asks for the password). See :ref:`encrypted_repos`
for more details.
Using ``BORG_PASSCOMMAND`` with macOS Keychain
macOS has a native manager for secrets (such as passphrases) which is safer
than just using a file as it is encrypted at rest and unlocked manually
(fortunately, the login keyring automatically unlocks when you login). With
the built-in ``security`` command, you can access it from the command line,
making it useful for ``BORG_PASSCOMMAND``.
First generate a passphrase and use ``security`` to save it to your login
(default) keychain::
security add-generic-password -D secret -U -a $USER -s borg-passphrase -w $(head -c 1024 /dev/urandom | base64)
In your backup script retrieve it in the ``BORG_PASSCOMMAND``::
export BORG_PASSCOMMAND="security find-generic-password -a $USER -s borg-passphrase -w"
Using ``BORG_PASSCOMMAND`` with GNOME Keyring
GNOME also has a keyring daemon that can be used to store a Borg passphrase.
First ensure ``libsecret-tools``, ``gnome-keyring`` and ``libpam-gnome-keyring``
are installed. If ``libpam-gnome-keyring`` wasn't already installed, ensure it
runs on login::
sudo sh -c "echo session optional pam_gnome_keyring.so auto_start >> /etc/pam.d/login"
sudo sh -c "echo password optional pam_gnome_keyring.so >> /etc/pam.d/passwd"
# you may need to relogin afterwards to activate the login keyring
Then add a secret to the login keyring::
head -c 1024 /dev/urandom | base64 | secret-tool store borg-repository repo-name --label="Borg Passphrase"
If a dialog box pops up prompting you to pick a password for a new keychain, use your
login password. If there is a checkbox for automatically unlocking on login, check it
to allow backups without any user intervention whatsoever.
Once the secret is saved, retrieve it in a backup script using ``BORG_PASSCOMMAND``::
export BORG_PASSCOMMAND="secret-tool lookup borg-repository repo-name"
.. note:: For this to automatically unlock the keychain it must be run
in the ``dbus`` session of an unlocked terminal; for example, running a backup
script as a ``cron`` job might not work unless you also ``export DISPLAY=:0``
so ``secret-tool`` can pick up your open session. `It gets even more complicated`__
when you are running the tool as a different user (e.g. running a backup as root
with the password stored in the user keyring).
__ https://github.com/borgbackup/borg/pull/2837#discussion_r127641330
Using ``BORG_PASSCOMMAND`` with KWallet
KDE also has a keychain feature in the form of KWallet. The command-line tool
``kwalletcli`` can be used to store and retrieve secrets. Ensure ``kwalletcli``
is installed, generate a passphrase, and store it in your "wallet"::
head -c 1024 /dev/urandom | base64 | kwalletcli -Pe borg-passphrase -f Passwords
Once the secret is saved, retrieve it in a backup script using ``BORG_PASSCOMMAND``::
export BORG_PASSCOMMAND="kwalletcli -e borg-passphrase -f Passwords"
When backing up to remote encrypted repos, is encryption done locally?
----------------------------------------------------------------------
@ -178,6 +454,16 @@ Thus:
- have media at another place
- have a relatively recent backup on your media
How do I report a security issue with Borg?
-------------------------------------------
Send a private email to the :ref:`security contact <security-contact>`
if you think you have discovered a security issue.
Please disclose security issues responsibly.
Common issues
#############
Why do I get "connection closed by remote" after a while?
---------------------------------------------------------
@ -201,10 +487,12 @@ yet noticed on the server. Try these settings:
ClientAliveCountMax 3
If you have multiple borg create ... ; borg create ... commands in a already
serialized way in a single script, you need to give them --lock-wait N (with N
serialized way in a single script, you need to give them ``--lock-wait N`` (with N
being a bit more than the time the server needs to terminate broken down
connections and release the lock).
.. _disable_archive_chunks:
The borg cache eats way too much disk space, what can I do?
-----------------------------------------------------------
@ -235,60 +523,13 @@ This has some pros and cons, though:
The long term plan to improve this is called "borgception", see :issue:`474`.
If a backup stops mid-way, does the already-backed-up data stay there?
----------------------------------------------------------------------
Can I backup my root partition (/) with Borg?
---------------------------------------------
Yes, |project_name| supports resuming backups.
During a backup a special checkpoint archive named ``<archive-name>.checkpoint``
is saved every checkpoint interval (the default value for this is 5
minutes) containing all the data backed-up until that point.
Checkpoints only happen between files (so they don't help for interruptions
happening while a very large file is being processed).
This checkpoint archive is a valid archive (all files in it are valid and complete),
but it is only a partial backup (not all files that you wanted to backup are
contained in it). Having it in the repo until a successful, full backup is
completed is useful because it references all the transmitted chunks up
to the checkpoint. This means that in case of an interruption, you only need to
retransfer the data since the last checkpoint.
If a backup was interrupted, you do not need to do any special considerations,
just invoke ``borg create`` as you always do. You may use the same archive name
as in previous attempt or a different one (e.g. if you always include the current
datetime), it does not matter.
|project_name| always does full single-pass backups, so it will start again
from the beginning - but it will be much faster, because some of the data was
already stored into the repo (and is still referenced by the checkpoint
archive), so it does not need to get transmitted and stored again.
Once your backup has finished successfully, you can delete all
``<archive-name>.checkpoint`` archives. If you run ``borg prune``, it will
also care for deleting unneeded checkpoints.
How can I backup huge file(s) over a unstable connection?
---------------------------------------------------------
You can use this "split trick" as a workaround for the in-between-files-only
checkpoints (see above), huge files and a instable connection to the repository:
Split the huge file(s) into parts of manageable size (e.g. 100MB) and create
a temporary archive of them. Borg will create checkpoints now more frequently
than if you try to backup the files in their original form (e.g. 100GB).
After that, you can remove the parts again and backup the huge file(s) in
their original form. This will now work a lot faster as a lot of content chunks
are already in the repository.
After you have successfully backed up the huge original file(s), you can remove
the temporary archive you made from the parts.
We realize that this is just a better-than-nothing workaround, see :issue:`1198`
for a potential solution.
Please note that this workaround only helps you for backup, not for restore.
Backing up your entire root partition works just fine, but remember to
exclude directories that make no sense to backup, such as /dev, /proc,
/sys, /tmp and /run, and to use ``--one-file-system`` if you only want to
backup the root partition (and not any mounted devices e.g.).
If it crashes with a UnicodeError, what can I do?
-------------------------------------------------
@ -309,39 +550,15 @@ If you run into that, try this:
the parent directory (or even everything)
- mount the repo using FUSE and use some file manager
Can |project_name| add redundancy to the backup data to deal with hardware malfunction?
---------------------------------------------------------------------------------------
No, it can't. While that at first sounds like a good idea to defend against
some defect HDD sectors or SSD flash blocks, dealing with this in a
reliable way needs a lot of low-level storage layout information and
control which we do not have (and also can't get, even if we wanted).
So, if you need that, consider RAID or a filesystem that offers redundant
storage or just make backups to different locations / different hardware.
See also :issue:`225`.
Can |project_name| verify data integrity of a backup archive?
-------------------------------------------------------------
Yes, if you want to detect accidental data damage (like bit rot), use the
``check`` operation. It will notice corruption using CRCs and hashes.
If you want to be able to detect malicious tampering also, use an encrypted
repo. It will then be able to check using CRCs and HMACs.
.. _a_status_oddity:
I am seeing 'A' (added) status for a unchanged file!?
-----------------------------------------------------
I am seeing 'A' (added) status for an unchanged file!?
------------------------------------------------------
The files cache is used to determine whether |project_name| already
"knows" / has backed up a file and if so, to skip the file from
chunking. It does intentionally *not* contain files that:
- have >= 10 as "entry age" (|project_name| has not seen this file for a while)
- have a modification time (mtime) same as the newest mtime in the created
archive
chunking. It does intentionally *not* contain files that have a modification
time (mtime) same as the newest mtime in the created archive.
So, if you see an 'A' status for unchanged file(s), they are likely the files
with the most recent mtime in that archive.
@ -368,23 +585,58 @@ those files are reported as being added when, really, chunks are
already used.
.. _always_chunking:
It always chunks all my files, even unchanged ones!
---------------------------------------------------
|project_name| maintains a files cache where it remembers the mtime, size and
inode of files. When |project_name| does a new backup and starts processing a
file, it first looks whether the file has changed (compared to the values
stored in the files cache). If the values are the same, the file is assumed
unchanged and thus its contents won't get chunked (again).
|project_name| can't keep an infinite history of files of course, thus entries
in the files cache have a "maximum time to live" which is set via the
environment variable BORG_FILES_CACHE_TTL (and defaults to 20).
Every time you do a backup (on the same machine, using the same user), the
cache entries' ttl values of files that were not "seen" are incremented by 1
and if they reach BORG_FILES_CACHE_TTL, the entry is removed from the cache.
So, for example, if you do daily backups of 26 different data sets A, B,
C, ..., Z on one machine (using the default TTL), the files from A will be
already forgotten when you repeat the same backups on the next day and it
will be slow because it would chunk all the files each time. If you set
BORG_FILES_CACHE_TTL to at least 26 (or maybe even a small multiple of that),
it would be much faster.
Another possible reason is that files don't always have the same path, for
example if you mount a filesystem without stable mount points for each backup or if you are running the backup from a filesystem snapshot whose name is not stable.
If the directory where you mount a filesystem is different every time,
|project_name| assume they are different files.
Is there a way to limit bandwidth with |project_name|?
------------------------------------------------------
There is no command line option to limit bandwidth with |project_name|, but
bandwidth limiting can be accomplished with pipeviewer_:
To limit upload (i.e. :ref:`borg_create`) bandwidth, use the
``--remote-ratelimit`` option.
There is no built-in way to limit *download*
(i.e. :ref:`borg_extract`) bandwidth, but limiting download bandwidth
can be accomplished with pipeviewer_:
Create a wrapper script: /usr/local/bin/pv-wrapper ::
#!/bin/bash
#!/bin/sh
## -q, --quiet do not output any transfer information at all
## -L, --rate-limit RATE limit transfer to RATE bytes per second
export RATE=307200
RATE=307200
pv -q -L $RATE | "$@"
Add BORG_RSH environment variable to use pipeviewer wrapper script with ssh. ::
export BORG_RSH='/usr/local/bin/pv-wrapper.sh ssh'
export BORG_RSH='/usr/local/bin/pv-wrapper ssh'
Now |project_name| will be bandwidth limited. Nice thing about pv is that you can change rate-limit on the fly: ::
@ -413,6 +665,74 @@ If you can reproduce the issue with the proven filesystem, please file an
issue in the |project_name| issue tracker about that.
Why does running 'borg check --repair' warn about data loss?
------------------------------------------------------------
Repair usually works for recovering data in a corrupted archive. However,
it's impossible to predict all modes of corruption. In some very rare
instances, such as malfunctioning storage hardware, additional repo
corruption may occur. If you can't afford to lose the repo, it's strongly
recommended that you perform repair on a copy of the repo.
In other words, the warning is there to emphasize that |project_name|:
- Will perform automated routines that modify your backup repository
- Might not actually fix the problem you are experiencing
- Might, in very rare cases, further corrupt your repository
In the case of malfunctioning hardware, such as a drive or USB hub
corrupting data when read or written, it's best to diagnose and fix the
cause of the initial corruption before attempting to repair the repo. If
the corruption is caused by a one time event such as a power outage,
running `borg check --repair` will fix most problems.
Why isn't there more progress / ETA information displayed?
----------------------------------------------------------
Some borg runs take quite a bit, so it would be nice to see a progress display,
maybe even including a ETA (expected time of "arrival" [here rather "completion"]).
For some functionality, this can be done: if the total amount of work is more or
less known, we can display progress. So check if there is a ``--progress`` option.
But sometimes, the total amount is unknown (e.g. for ``borg create`` we just do
a single pass over the filesystem, so we do not know the total file count or data
volume before reaching the end). Adding another pass just to determine that would
take additional time and could be incorrect, if the filesystem is changing.
Even if the fs does not change and we knew count and size of all files, we still
could not compute the ``borg create`` ETA as we do not know the amount of changed
chunks, how the bandwidth of source and destination or system performance might
fluctuate.
You see, trying to display ETA would be futile. The borg developers prefer to
rather not implement progress / ETA display than doing futile attempts.
See also: https://xkcd.com/612/
Miscellaneous
#############
Requirements for the borg single-file binary, esp. (g)libc?
-----------------------------------------------------------
We try to build the binary on old, but still supported systems - to keep the
minimum requirement for the (g)libc low. The (g)libc can't be bundled into
the binary as it needs to fit your kernel and OS, but Python and all other
required libraries will be bundled into the binary.
If your system fulfills the minimum (g)libc requirement (see the README that
is released with the binary), there should be no problem. If you are slightly
below the required version, maybe just try. Due to the dynamic loading (or not
loading) of some shared libraries, it might still work depending on what
libraries are actually loaded and used.
In the borg git repository, there is scripts/glibc_check.py that can determine
(based on the symbols' versions they want to link to) whether a set of given
(Linux) binaries works with a given glibc version.
Why was Borg forked from Attic?
-------------------------------
@ -441,3 +761,80 @@ Borg intends to be:
* do not break compatibility accidentally, without a good reason
or without warning. allow compatibility breaking for other cases.
* if major version number changes, it may have incompatible changes
Migrating from Attic
####################
What are the differences between Attic and Borg?
------------------------------------------------
Borg is a fork of `Attic`_ and maintained by "`The Borg collective`_".
.. _Attic: https://github.com/jborg/attic
.. _The Borg collective: https://borgbackup.readthedocs.org/en/latest/authors.html
Here's a (incomplete) list of some major changes:
* lots of attic issues fixed (see `issue #5 <https://github.com/borgbackup/borg/issues/5>`_),
including critical data corruption bugs and security issues.
* more open, faster paced development (see `issue #1 <https://github.com/borgbackup/borg/issues/1>`_)
* less chunk management overhead (less memory and disk usage for chunks index)
* faster remote cache resync (useful when backing up multiple machines into same repo)
* compression: no, lz4, zlib or lzma compression, adjustable compression levels
* repokey replaces problematic passphrase mode (you can't change the passphrase nor the pbkdf2 iteration count in "passphrase" mode)
* simple sparse file support, great for virtual machine disk files
* can read special files (e.g. block devices) or from stdin, write to stdout
* mkdir-based locking is more compatible than attic's posix locking
* uses fadvise to not spoil / blow up the fs cache
* better error messages / exception handling
* better logging, screen output, progress indication
* tested on misc. Linux systems, 32 and 64bit, FreeBSD, OpenBSD, NetBSD, macOS
Please read the :ref:`changelog` (or ``docs/changes.rst`` in the source distribution) for more
information.
Borg is not compatible with original Attic (but there is a one-way conversion).
How do I migrate from Attic to Borg?
------------------------------------
Use :ref:`borg_upgrade`. This is a one-way process that cannot be reversed.
There are some caveats:
- The upgrade can only be performed on local repositories.
It cannot be performed on remote repositories.
- If the repository is in "keyfile" encryption mode, the keyfile must
exist locally or it must be manually moved after performing the upgrade:
1. Locate the repository ID, contained in the ``config`` file in the repository.
2. Locate the attic key file at ``~/.attic/keys/``. The correct key for the
repository starts with the line ``ATTIC_KEY <repository id>``.
3. Copy the attic key file to ``~/.config/borg/keys/``
4. Change the first line from ``ATTIC_KEY ...`` to ``BORG_KEY ...``.
5. Verify that the repository is now accessible (e.g. ``borg list <repository>``).
- Attic and Borg use different :ref:`"chunker params" <chunker-params>`.
This means that data added by Borg won't deduplicate with the existing data
stored by Attic. The effect is lessened if the files cache is used with Borg.
- Repositories in "passphrase" mode *must* be migrated to "repokey" mode using
:ref:`borg_key_migrate-to-repokey`. Borg does not support the "passphrase" mode
any other way.
Why is my backup bigger than with attic?
----------------------------------------
Attic was rather unflexible when it comes to compression, it always
compressed using zlib level 6 (no way to switch compression off or
adjust the level or algorithm).
The default in Borg is lz4, which is fast enough to not use significant CPU time
in most cases, but can only achieve modest compression. It still compresses
easily compressed data fairly well.
zlib compression with all levels (1-9) as well as LZMA (1-6) are available
as well, for cases where they are worth it.
Which choice is the best option depends on a number of factors, like
bandwidth to the repository, how well the data compresses, available CPU
power and so on.

2
docs/global.rst.inc
View File

@ -15,6 +15,8 @@
.. _libacl: https://savannah.nongnu.org/projects/acl/
.. _libattr: https://savannah.nongnu.org/projects/attr/
.. _liblz4: https://github.com/Cyan4973/lz4
.. _libb2: https://github.com/BLAKE2/libb2
.. _ZeroMQ: http://zeromq.org/
.. _OpenSSL: https://www.openssl.org/
.. _`Python 3`: https://www.python.org/
.. _Buzhash: https://en.wikipedia.org/wiki/Buzhash

4
docs/index.rst
View File

@ -1,11 +1,13 @@
.. include:: global.rst.inc
.. highlight:: none
Borg Documentation
==================
.. include:: ../README.rst
.. when you add an element here, do not forget to add it to book.rst
.. toctree::
:maxdepth: 2

94
docs/installation.rst
View File

@ -42,28 +42,25 @@ package which can be installed with the package manager.
Distribution Source Command
============ ============================================= =======
Arch Linux `[community]`_ ``pacman -S borg``
Debian `jessie-backports`_, `stretch`_, `sid`_ ``apt install borgbackup``
Debian `Debian packages`_ ``apt install borgbackup``
Gentoo `ebuild`_ ``emerge borgbackup``
GNU Guix `GNU Guix`_ ``guix package --install borg``
Fedora/RHEL `Fedora official repository`_, `EPEL`_ ``dnf install borgbackup``
Fedora/RHEL `Fedora official repository`_ ``dnf install borgbackup``
FreeBSD `FreeBSD ports`_ ``cd /usr/ports/archivers/py-borgbackup && make install clean``
Mageia `cauldron`_ ``urpmi borgbackup``
NetBSD `pkgsrc`_ ``pkg_add py-borgbackup``
NixOS `.nix file`_ N/A
NixOS `.nix file`_ ``nix-env -i borgbackup``
OpenBSD `OpenBSD ports`_ ``pkg_add borgbackup``
OpenIndiana `OpenIndiana hipster repository`_ ``pkg install borg``
openSUSE `openSUSE official repository`_ ``zypper in python3-borgbackup``
openSUSE `openSUSE official repository`_ ``zypper in borgbackup``
OS X `Brew cask`_ ``brew cask install borgbackup``
Raspbian `Raspbian testing`_ ``apt install borgbackup``
Ubuntu `16.04`_, backports (PPA): `15.10`_, `14.04`_ ``apt install borgbackup``
Ubuntu `Ubuntu packages`_, `Ubuntu PPA`_ ``apt install borgbackup``
============ ============================================= =======
.. _[community]: https://www.archlinux.org/packages/?name=borg
.. _jessie-backports: https://packages.debian.org/jessie-backports/borgbackup
.. _stretch: https://packages.debian.org/stretch/borgbackup
.. _sid: https://packages.debian.org/sid/borgbackup
.. _Debian packages: https://packages.debian.org/search?keywords=borgbackup&searchon=names&exact=1&suite=all&section=all
.. _Fedora official repository: https://apps.fedoraproject.org/packages/borgbackup
.. _EPEL: https://admin.fedoraproject.org/pkgdb/package/rpms/borgbackup/
.. _FreeBSD ports: http://www.freshports.org/archivers/py-borgbackup/
.. _ebuild: https://packages.gentoo.org/packages/app-backup/borgbackup
.. _GNU Guix: https://www.gnu.org/software/guix/package-list.html#borg
@ -73,11 +70,10 @@ Ubuntu `16.04`_, backports (PPA): `15.10`_, `14.04`_ ``apt install borgbac
.. _OpenBSD ports: http://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/sysutils/borgbackup/
.. _OpenIndiana hipster repository: http://pkg.openindiana.org/hipster/en/search.shtml?token=borg&action=Search
.. _openSUSE official repository: http://software.opensuse.org/package/borgbackup
.. _Brew cask: http://caskroom.io/
.. _Brew cask: https://caskroom.github.io/
.. _Raspbian testing: http://archive.raspbian.org/raspbian/pool/main/b/borgbackup/
.. _16.04: https://launchpad.net/ubuntu/xenial/+source/borgbackup
.. _15.10: https://launchpad.net/~costamagnagianfranco/+archive/ubuntu/borgbackup
.. _14.04: https://launchpad.net/~costamagnagianfranco/+archive/ubuntu/borgbackup
.. _Ubuntu packages: http://packages.ubuntu.com/xenial/borgbackup
.. _Ubuntu PPA: https://launchpad.net/~costamagnagianfranco/+archive/ubuntu/borgbackup
Please ask package maintainers to build a package or, if you can package /
submit it yourself, please help us with that! See :issue:`105` on
@ -88,11 +84,14 @@ github to followup on packaging efforts.
Standalone Binary
-----------------
.. note:: Releases are signed with an OpenPGP key, see
:ref:`security-contact` for more instructions.
|project_name| binaries (generated with `pyinstaller`_) are available
on the releases_ page for the following platforms:
* **Linux**: glibc >= 2.13 (ok for most supported Linux releases). Maybe older
glibc versions also work, if they are compatible to 2.13.
* **Linux**: glibc >= 2.13 (ok for most supported Linux releases).
Older glibc releases are untested and may not work.
* **Mac OS X**: 10.10 (does not work with older OS X releases)
* **FreeBSD**: 10.2 (unknown whether it works for older releases)
@ -129,24 +128,35 @@ To install on Windows just extract the zip anywhere and add the bin directory to
From Source
-----------
.. note::
Some older Linux systems (like RHEL/CentOS 5) and Python interpreter binaries
compiled to be able to run on such systems (like Python installed via Anaconda)
might miss functions required by Borg.
This issue will be detected early and Borg will abort with a fatal error.
Dependencies
~~~~~~~~~~~~
To install |project_name| from a source package (including pip), you have to install the
following dependencies first:
* `Python 3`_ >= 3.4.0, plus development headers. Even though Python 3 is not
* `Python 3`_ >= 3.5.0, plus development headers. Even though Python 3 is not
the default Python version on most systems, it is usually available as an
optional install.
* OpenSSL_ >= 1.0.0, plus development headers.
* libacl_ (that pulls in libattr_ also), both plus development headers.
* libacl_ (which depends on libattr_), both plus development headers.
* liblz4_, plus development headers.
* ZeroMQ_ >= 4.0.0, plus development headers.
* some Python dependencies, pip will automatically install them for you
* optionally, the llfuse_ Python package is required if you wish to mount an
archive as a FUSE filesystem. See setup.py about the version requirements.
* optionally libb2_. If it is not found a bundled implementation is used instead.
If you have troubles finding the right package names, have a look at the
distribution specific sections below and also at the Vagrantfile in our repo.
distribution specific sections below or the Vagrantfile in the git repository,
which contains installation scripts for a number of operating systems.
In the following, the steps needed to install the dependencies are listed for a
selection of platforms. If your distribution is not covered by these
@ -166,6 +176,7 @@ Install the dependencies with development headers::
libssl-dev openssl \
libacl1-dev libacl1 \
liblz4-dev liblz4-1 \
libzmq3-dev libzmq3 \
build-essential
sudo apt-get install libfuse-dev fuse pkg-config # optional, for FUSE support
@ -176,6 +187,8 @@ group, log out and log in again.
Fedora / Korora
+++++++++++++++
.. todo:: Add zeromq, use python 3.5 or 3.6
Install the dependencies with development headers::
sudo dnf install python3 python3-devel python3-pip python3-virtualenv
@ -186,14 +199,33 @@ Install the dependencies with development headers::
sudo dnf install redhat-rpm-config # not needed in Korora
sudo dnf install fuse-devel fuse pkgconfig # optional, for FUSE support
openSUSE Tumbleweed / Leap
++++++++++++++++++++++++++
.. todo:: Add zeromq, use python 3.5 or 3.6
Install the dependencies automatically using zypper::
sudo zypper source-install --build-deps-only borgbackup
Alternatively, you can enumerate all build dependencies in the command line::
sudo zypper install python3 python3-devel \
libacl-devel liblz4-devel openssl-devel \
python3-Cython python3-Sphinx python3-msgpack-python \
python3-pytest python3-setuptools python3-setuptools_scm \
python3-sphinx_rtd_theme python3-llfuse gcc gcc-c++
Mac OS X
++++++++
.. todo:: Add zeromq, use python 3.5 or 3.6
Assuming you have installed homebrew_, the following steps will install all the
dependencies::
brew install python3 lz4 openssl
brew install pkg-config # optional, for FUSE support
pip3 install virtualenv
For FUSE support to mount the backup archives, you need at least version 3.0 of
@ -204,8 +236,11 @@ FUSE for OS X, which is available as a pre-release_.
FreeBSD
++++++++
Listed below are packages you will need to install |project_name|, its dependencies,
and commands to make fuse work for using the mount command.
.. todo:: Add zeromq, use python 3.5 or 3.6
Listed below are packages you will need to install Borg, its dependencies,
and commands to make FUSE work for using the mount command.
::
@ -217,7 +252,16 @@ and commands to make fuse work for using the mount command.
echo 'vfs.usermount=1' >> /etc/sysctl.conf
kldload fuse
sysctl vfs.usermount=1
Windows 10's Linux Subsystem
++++++++++++++++++++++++++++
.. note::
Running under Windows 10's Linux Subsystem is experimental and has not been tested much yet.
Just follow the Ubuntu Linux installation steps. You can omit the FUSE stuff, it won't work anyway.
Windows
+++++++
@ -232,11 +276,13 @@ Cygwin
.. note::
Running under Cygwin is experimental and has only been tested with Cygwin
(x86-64) v2.5.2.
(x86-64) v2.5.2. Remote repositories are known broken, local repositories should work.
.. todo:: Add zeromq, use python 3.5 or 3.6
Use the Cygwin installer to install the dependencies::
python3 python3-setuptools
python3 python3-devel python3-setuptools
binutils gcc-g++
libopenssl openssl-devel
liblz4_1 liblz4-devel
@ -293,9 +339,9 @@ While we try not to break master, there are no guarantees on anything. ::
source borg-env/bin/activate # always before using!
# install borg + dependencies into virtualenv
pip install sphinx # optional, to build the docs
cd borg
pip install -r requirements.d/development.txt
pip install -r requirements.d/docs.txt # optional, to build the docs
pip install -r requirements.d/fuse.txt # optional, for FUSE support
pip install -e . # in-place editable mode

484
docs/internals.rst
View File

@ -1,451 +1,45 @@
.. include:: global.rst.inc
.. highlight:: none
.. _internals:
Internals
=========
This page documents the internal data structures and storage
mechanisms of |project_name|. It is partly based on `mailing list
discussion about internals`_ and also on static code analysis.
Repository and Archives
-----------------------
|project_name| stores its data in a `Repository`. Each repository can
hold multiple `Archives`, which represent individual backups that
contain a full archive of the files specified when the backup was
performed. Deduplication is performed across multiple backups, both on
data and metadata, using `Chunks` created by the chunker using the Buzhash_
algorithm.
Each repository has the following file structure:
README
simple text file telling that this is a |project_name| repository
config
repository configuration
data/
directory where the actual data is stored
hints.%d
hints for repository compaction
index.%d
repository index
lock.roster and lock.exclusive/*
used by the locking system to manage shared and exclusive locks
Lock files
----------
|project_name| uses locks to get (exclusive or shared) access to the cache and
the repository.
The locking system is based on creating a directory `lock.exclusive` (for
exclusive locks). Inside the lock directory, there is a file indication
hostname, process id and thread id of the lock holder.
There is also a json file `lock.roster` that keeps a directory of all shared
and exclusive lockers.
If the process can create the `lock.exclusive` directory for a resource, it has
the lock for it. If creation fails (because the directory has already been
created by some other process), lock acquisition fails.
The cache lock is usually in `~/.cache/borg/REPOID/lock.*`.
The repository lock is in `repository/lock.*`.
In case you run into troubles with the locks, you can use the ``borg break-lock``
command after you first have made sure that no |project_name| process is
running on any machine that accesses this resource. Be very careful, the cache
or repository might get damaged if multiple processes use it at the same time.
Config file
-----------
Each repository has a ``config`` file which which is a ``INI``-style file
and looks like this::
[repository]
version = 1
segments_per_dir = 10000
max_segment_size = 5242880
id = 57d6c1d52ce76a836b532b0e42e677dec6af9fca3673db511279358828a21ed6
This is where the ``repository.id`` is stored. It is a unique
identifier for repositories. It will not change if you move the
repository around so you can make a local transfer then decide to move
the repository to another (even remote) location at a later time.
Keys
----
The key to address the key/value store is usually computed like this:
key = id = id_hash(unencrypted_data)
The id_hash function is:
* sha256 (no encryption keys available)
* hmac-sha256 (encryption keys available)
Segments and archives
---------------------
A |project_name| repository is a filesystem based transactional key/value
store. It makes extensive use of msgpack_ to store data and, unless
otherwise noted, data is stored in msgpack_ encoded files.
Objects referenced by a key are stored inline in files (`segments`) of approx.
5MB size in numbered subdirectories of ``repo/data``.
They contain:
* header size
* crc
* size
* tag
* key
* data
Segments are built locally, and then uploaded. Those files are
strictly append-only and modified only once.
Tag is either ``PUT``, ``DELETE``, or ``COMMIT``. A segment file is
basically a transaction log where each repository operation is
appended to the file. So if an object is written to the repository a
``PUT`` tag is written to the file followed by the object id and
data. If an object is deleted a ``DELETE`` tag is appended
followed by the object id. A ``COMMIT`` tag is written when a
repository transaction is committed. When a repository is opened any
``PUT`` or ``DELETE`` operations not followed by a ``COMMIT`` tag are
discarded since they are part of a partial/uncommitted transaction.
The manifest
------------
The manifest is an object with an all-zero key that references all the
archives.
It contains:
* version
* list of archive infos
* timestamp
* config
Each archive info contains:
* name
* id
* time
It is the last object stored, in the last segment, and is replaced
each time.
The Archive
-----------
The archive metadata does not contain the file items directly. Only
references to other objects that contain that data. An archive is an
object that contains:
* version
* name
* list of chunks containing item metadata
* cmdline
* hostname
* username
* time
The Item
--------
Each item represents a file, directory or other fs item and is stored as an
``item`` dictionary that contains:
* path
* list of data chunks
* user
* group
* uid
* gid
* mode (item type + permissions)
* source (for links)
* rdev (for devices)
* mtime, atime, ctime in nanoseconds
* xattrs
* acl
* bsdfiles
All items are serialized using msgpack and the resulting byte stream
is fed into the same chunker algorithm as used for regular file data
and turned into deduplicated chunks. The reference to these chunks is then added
to the archive metadata. To achieve a finer granularity on this metadata
stream, we use different chunker params for this chunker, which result in
smaller chunks.
A chunk is stored as an object as well, of course.
.. _chunker_details:
Chunks
------
The |project_name| chunker uses a rolling hash computed by the Buzhash_ algorithm.
It triggers (chunks) when the last HASH_MASK_BITS bits of the hash are zero,
producing chunks of 2^HASH_MASK_BITS Bytes on average.
``borg create --chunker-params CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE``
can be used to tune the chunker parameters, the default is:
- CHUNK_MIN_EXP = 19 (minimum chunk size = 2^19 B = 512 kiB)
- CHUNK_MAX_EXP = 23 (maximum chunk size = 2^23 B = 8 MiB)
- HASH_MASK_BITS = 21 (statistical medium chunk size ~= 2^21 B = 2 MiB)
- HASH_WINDOW_SIZE = 4095 [B] (`0xFFF`)
The buzhash table is altered by XORing it with a seed randomly generated once
for the archive, and stored encrypted in the keyfile. This is to prevent chunk
size based fingerprinting attacks on your encrypted repo contents (to guess
what files you have based on a specific set of chunk sizes).
For some more general usage hints see also ``--chunker-params``.
Indexes / Caches
----------------
The **files cache** is stored in ``cache/files`` and is indexed on the
``file path hash``. At backup time, it is used to quickly determine whether we
need to chunk a given file (or whether it is unchanged and we already have all
its pieces).
It contains:
* age
* file inode number
* file size
* file mtime_ns
* file content chunk hashes
The inode number is stored to make sure we distinguish between
different files, as a single path may not be unique across different
archives in different setups.
The files cache is stored as a python associative array storing
python objects, which generates a lot of overhead.
The **chunks cache** is stored in ``cache/chunks`` and is indexed on the
``chunk id_hash``. It is used to determine whether we already have a specific
chunk, to count references to it and also for statistics.
It contains:
* reference count
* size
* encrypted/compressed size
The **repository index** is stored in ``repo/index.%d`` and is indexed on the
``chunk id_hash``. It is used to determine a chunk's location in the repository.
It contains:
* segment (that contains the chunk)
* offset (where the chunk is located in the segment)
The repository index file is random access.
Hints are stored in a file (``repo/hints.%d``).
It contains:
* version
* list of segments
* compact
hints and index can be recreated if damaged or lost using ``check --repair``.
The chunks cache and the repository index are stored as hash tables, with
only one slot per bucket, but that spreads the collisions to the following
buckets. As a consequence the hash is just a start position for a linear
search, and if the element is not in the table the index is linearly crossed
until an empty bucket is found.
When the hash table is filled to 75%, its size is grown. When it's
emptied to 25%, its size is shrinked. So operations on it have a variable
complexity between constant and linear with low factor, and memory overhead
varies between 33% and 300%.
.. _cache-memory-usage:
Indexes / Caches memory usage
-----------------------------
Here is the estimated memory usage of |project_name|:
chunk_count ~= total_file_size / 2 ^ HASH_MASK_BITS
repo_index_usage = chunk_count * 40
chunks_cache_usage = chunk_count * 44
files_cache_usage = total_file_count * 240 + chunk_count * 80
mem_usage ~= repo_index_usage + chunks_cache_usage + files_cache_usage
= chunk_count * 164 + total_file_count * 240
All units are Bytes.
It is assuming every chunk is referenced exactly once (if you have a lot of
duplicate chunks, you will have less chunks than estimated above).
It is also assuming that typical chunk size is 2^HASH_MASK_BITS (if you have
a lot of files smaller than this statistical medium chunk size, you will have
more chunks than estimated above, because 1 file is at least 1 chunk).
If a remote repository is used the repo index will be allocated on the remote side.
E.g. backing up a total count of 1 Mi (IEC binary prefix e.g. 2^20) files with a total size of 1TiB.
a) with ``create --chunker-params 10,23,16,4095`` (custom, like borg < 1.0 or attic):
mem_usage = 2.8GiB
b) with ``create --chunker-params 19,23,21,4095`` (default):
mem_usage = 0.31GiB
.. note:: There is also the ``--no-files-cache`` option to switch off the files cache.
You'll save some memory, but it will need to read / chunk all the files as
it can not skip unmodified files then.
Encryption
----------
AES_-256 is used in CTR mode (so no need for padding). A 64bit initialization
vector is used, a `HMAC-SHA256`_ is computed on the encrypted chunk with a
random 64bit nonce and both are stored in the chunk.
The header of each chunk is: ``TYPE(1)`` + ``HMAC(32)`` + ``NONCE(8)`` + ``CIPHERTEXT``.
Encryption and HMAC use two different keys.
In AES CTR mode you can think of the IV as the start value for the counter.
The counter itself is incremented by one after each 16 byte block.
The IV/counter is not required to be random but it must NEVER be reused.
So to accomplish this |project_name| initializes the encryption counter to be
higher than any previously used counter value before encrypting new data.
To reduce payload size, only 8 bytes of the 16 bytes nonce is saved in the
payload, the first 8 bytes are always zeros. This does not affect security but
limits the maximum repository capacity to only 295 exabytes (2**64 * 16 bytes).
Encryption keys (and other secrets) are kept either in a key file on the client
('keyfile' mode) or in the repository config on the server ('repokey' mode).
In both cases, the secrets are generated from random and then encrypted by a
key derived from your passphrase (this happens on the client before the key
is stored into the keyfile or as repokey).
The passphrase is passed through the ``BORG_PASSPHRASE`` environment variable
or prompted for interactive usage.
Key files
---------
When initialized with the ``init -e keyfile`` command, |project_name|
needs an associated file in ``$HOME/.config/borg/keys`` to read and write
the repository. The format is based on msgpack_, base64 encoding and
PBKDF2_ SHA256 hashing, which is then encoded again in a msgpack_.
The internal data structure is as follows:
version
currently always an integer, 1
repository_id
the ``id`` field in the ``config`` ``INI`` file of the repository.
enc_key
the key used to encrypt data with AES (256 bits)
enc_hmac_key
the key used to HMAC the encrypted data (256 bits)
id_key
the key used to HMAC the plaintext chunk data to compute the chunk's id
chunk_seed
the seed for the buzhash chunking table (signed 32 bit integer)
Those fields are processed using msgpack_. The utf-8 encoded passphrase
is processed with PBKDF2_ (SHA256_, 100000 iterations, random 256 bit salt)
to give us a derived key. The derived key is 256 bits long.
A `HMAC-SHA256`_ checksum of the above fields is generated with the derived
key, then the derived key is also used to encrypt the above pack of fields.
Then the result is stored in a another msgpack_ formatted as follows:
version
currently always an integer, 1
salt
random 256 bits salt used to process the passphrase
iterations
number of iterations used to process the passphrase (currently 100000)
algorithm
the hashing algorithm used to process the passphrase and do the HMAC
checksum (currently the string ``sha256``)
hash
the HMAC of the encrypted derived key
data
the derived key, encrypted with AES over a PBKDF2_ SHA256 key
described above
The resulting msgpack_ is then encoded using base64 and written to the
key file, wrapped using the standard ``textwrap`` module with a header.
The header is a single line with a MAGIC string, a space and a hexadecimal
representation of the repository id.
Compression
-----------
|project_name| supports the following compression methods:
- none (no compression, pass through data 1:1)
- lz4 (low compression, but super fast)
- zlib (level 0-9, level 0 is no compression [but still adding zlib overhead],
level 1 is low, level 9 is high compression)
- lzma (level 0-9, level 0 is low, level 9 is high compression).
Speed: none > lz4 > zlib > lzma
Compression: lzma > zlib > lz4 > none
Be careful, higher zlib and especially lzma compression levels might take a
lot of resources (CPU and memory).
The overall speed of course also depends on the speed of your target storage.
If that is slow, using a higher compression level might yield better overall
performance. You need to experiment a bit. Maybe just watch your CPU load, if
that is relatively low, increase compression until 1 core is 70-100% loaded.
Even if your target storage is rather fast, you might see interesting effects:
while doing no compression at all (none) is a operation that takes no time, it
likely will need to store more data to the storage compared to using lz4.
The time needed to transfer and store the additional data might be much more
than if you had used lz4 (which is super fast, but still might compress your
data about 2:1). This is assuming your data is compressible (if you backup
already compressed data, trying to compress them at backup time is usually
pointless).
Compression is applied after deduplication, thus using different compression
methods in one repo does not influence deduplication.
See ``borg create --help`` about how to specify the compression level and its default.
The internals chapter describes and analyses most of the inner workings
of Borg.
Borg uses a low-level, key-value store, the :ref:`repository`, and
implements a more complex data structure on top of it, which is made
up of the :ref:`manifest <manifest>`, :ref:`archives <archive>`,
:ref:`items <item>` and data :ref:`chunks`.
Each repository can hold multiple :ref:`archives <archive>`, which
represent individual backups that contain a full archive of the files
specified when the backup was performed.
Deduplication is performed globally across all data in the repository
(multiple backups and even multiple hosts), both on data and file
metadata, using :ref:`chunks` created by the chunker using the
Buzhash_ algorithm.
To actually perform the repository-wide deduplication, a hash of each
chunk is checked against the :ref:`chunks cache <cache>`, which is a
hash-table of all chunks that already exist.
.. figure:: internals/structure.png
:figwidth: 100%
:width: 100%
Layers in Borg. On the very top commands are implemented, using
a data access layer provided by the Archive and Item classes.
The "key" object provides both compression and authenticated
encryption used by the data access layer. The "key" object represents
the sole trust boundary in Borg.
The lowest layer is the repository, either accessed directly
(Repository) or remotely (RemoteRepository).
.. toctree::
:caption: Internals contents
internals/security
internals/data-structures
internals/frontends

BIN
docs/internals/compaction.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 757 KiB

BIN
docs/internals/compaction.vsd Normal file
View File

Binary file not shown.

1153
docs/internals/data-structures.rst Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
docs/internals/encryption.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 66 KiB

BIN
docs/internals/encryption.vsd Normal file
View File

Binary file not shown.

551
docs/internals/frontends.rst Normal file
View File

@ -0,0 +1,551 @@
.. include:: ../global.rst.inc
.. highlight:: json
.. _json_output:
All about JSON: How to develop frontends
========================================
Borg does not have a public API on the Python level. That does not keep you from writing :code:`import borg`,
but does mean that there are no release-to-release guarantees on what you might find in that package, not
even for point releases (1.1.x), and there is no documentation beyond the code and the internals documents.
Borg does on the other hand provide an API on a command-line level. In other words, a frontend should to
(for example) create a backup archive just invoke :ref:`borg_create`, give commandline parameters/options
as needed and parse JSON output from borg.
Important: JSON output is expected to be UTF-8, but currently borg depends on the locale being configured
for that (must be a UTF-8 locale and *not* "C" or "ascii"), so that Python will choose to encode to UTF-8.
The same applies to any inputs read by borg, they are expected to be UTF-8 encoded also.
We consider this a bug (see :issue:`2273`) and might fix it later, so borg will use UTF-8 independent of
the locale.
On POSIX systems, you can usually set environment vars to choose a UTF-8 locale:
::
export LANG=en_US.UTF-8
export LC_CTYPE=en_US.UTF-8
Logging
-------
Especially for graphical frontends it is important to be able to convey and reformat progress information
in meaningful ways. The ``--log-json`` option turns the stderr stream of Borg into a stream of JSON lines,
where each line is a JSON object. The *type* key of the object determines its other contents.
Since JSON can only encode text, any string representing a file system path may miss non-text parts.
The following types are in use. Progress information is governed by the usual rules for progress information,
it is not produced unless ``--progress`` is specified.
archive_progress
Output during operations creating archives (:ref:`borg_create` and :ref:`borg_recreate`).
The following keys exist, each represents the current progress.
original_size
Original size of data processed so far (before compression and deduplication)
compressed_size
Compressed size
deduplicated_size
Deduplicated size
nfiles
Number of (regular) files processed so far
path
Current path
time
Unix timestamp (float)
progress_message
A message-based progress information with no concrete progress information, just a message
saying what is currently being worked on.
operation
unique, opaque integer ID of the operation
:ref:`msgid <msgid>`
Message ID of the operation (may be *null*)
finished
boolean indicating whether the operation has finished, only the last object for an *operation*
can have this property set to *true*.
message
current progress message (may be empty/absent)
time
Unix timestamp (float)
progress_percent
Absolute progress information with defined end/total and current value.
operation
unique, opaque integer ID of the operation
:ref:`msgid <msgid>`
Message ID of the operation (may be *null*)
finished
boolean indicating whether the operation has finished, only the last object for an *operation*
can have this property set to *true*.
message
A formatted progress message, this will include the percentage and perhaps other information
current
Current value (always less-or-equal to *total*)
info
Array that describes the current item, may be *null*, contents depend on *msgid*
total
Total value
time
Unix timestamp (float)
file_status
This is only output by :ref:`borg_create` and :ref:`borg_recreate` if ``--list`` is specified. The usual
rules for the file listing applies, including the ``--filter`` option.
status
Single-character status as for regular list output
path
Path of the file system object
log_message
Any regular log output invokes this type. Regular log options and filtering applies to these as well.
time
Unix timestamp (float)
levelname
Upper-case log level name (also called severity). Defined levels are: DEBUG, INFO, WARNING, ERROR, CRITICAL
name
Name of the emitting entity
message
Formatted log message
:ref:`msgid <msgid>`
Message ID, may be *null* or absent
See Prompts_ for the types used by prompts.
.. rubric:: Examples (reformatted, each object would be on exactly one line)
:ref:`borg_extract` progress::
{"message": "100.0% Extracting: src/borgbackup.egg-info/entry_points.txt",
"current": 13000228, "total": 13004993, "info": ["src/borgbackup.egg-info/entry_points.txt"],
"operation": 1, "msgid": "extract", "type": "progress_percent", "finished": false}
{"message": "100.0% Extracting: src/borgbackup.egg-info/SOURCES.txt",
"current": 13004993, "total": 13004993, "info": ["src/borgbackup.egg-info/SOURCES.txt"],
"operation": 1, "msgid": "extract", "type": "progress_percent", "finished": false}
{"operation": 1, "msgid": "extract", "type": "progress_percent", "finished": true}
:ref:`borg_create` file listing with progress::
{"original_size": 0, "compressed_size": 0, "deduplicated_size": 0, "nfiles": 0, "type": "archive_progress", "path": "src"}
{"type": "file_status", "status": "U", "path": "src/borgbackup.egg-info/entry_points.txt"}
{"type": "file_status", "status": "U", "path": "src/borgbackup.egg-info/SOURCES.txt"}
{"type": "file_status", "status": "d", "path": "src/borgbackup.egg-info"}
{"type": "file_status", "status": "d", "path": "src"}
{"original_size": 13176040, "compressed_size": 11386863, "deduplicated_size": 503, "nfiles": 277, "type": "archive_progress", "path": ""}
Internal transaction progress::
{"message": "Saving files cache", "operation": 2, "msgid": "cache.commit", "type": "progress_message", "finished": false}
{"message": "Saving cache config", "operation": 2, "msgid": "cache.commit", "type": "progress_message", "finished": false}
{"message": "Saving chunks cache", "operation": 2, "msgid": "cache.commit", "type": "progress_message", "finished": false}
{"operation": 2, "msgid": "cache.commit", "type": "progress_message", "finished": true}
A debug log message::
{"message": "35 self tests completed in 0.08 seconds",
"type": "log_message", "created": 1488278449.5575905, "levelname": "DEBUG", "name": "borg.archiver"}
Prompts
-------
Prompts assume a JSON form as well when the ``--log-json`` option is specified. Responses
are still read verbatim from *stdin*, while prompts are JSON messages printed to *stderr*,
just like log messages.
Prompts use the *question_prompt* and *question_prompt_retry* types for the prompt itself,
and *question_invalid_answer*, *question_accepted_default*, *question_accepted_true*,
*question_accepted_false* and *question_env_answer* types for information about
prompt processing.
The *message* property contains the same string displayed regularly in the same situation,
while the *msgid* property may contain a msgid_, typically the name of the
environment variable that can be used to override the prompt. It is the same for all JSON
messages pertaining to the same prompt.
.. rubric:: Examples (reformatted, each object would be on exactly one line)
Providing an invalid answer::
{"type": "question_prompt", "msgid": "BORG_CHECK_I_KNOW_WHAT_I_AM_DOING",
"message": "... Type 'YES' if you understand this and want to continue: "}
incorrect answer # input on stdin
{"type": "question_invalid_answer", "msgid": "BORG_CHECK_I_KNOW_WHAT_I_AM_DOING", "is_prompt": false,
"message": "Invalid answer, aborting."}
Providing a false (negative) answer::
{"type": "question_prompt", "msgid": "BORG_CHECK_I_KNOW_WHAT_I_AM_DOING",
"message": "... Type 'YES' if you understand this and want to continue: "}
NO # input on stdin
{"type": "question_accepted_false", "msgid": "BORG_CHECK_I_KNOW_WHAT_I_AM_DOING",
"message": "Aborting.", "is_prompt": false}
Providing a true (affirmative) answer::
{"type": "question_prompt", "msgid": "BORG_CHECK_I_KNOW_WHAT_I_AM_DOING",
"message": "... Type 'YES' if you understand this and want to continue: "}
YES # input on stdin
# no further output, just like the prompt without --log-json
Passphrase prompts
------------------
Passphrase prompts should be handled differently. Use the environment variables *BORG_PASSPHRASE*
and *BORG_NEW_PASSPHRASE* (see :ref:`env_vars` for reference) to pass passphrases to Borg, don't
use the interactive passphrase prompts.
When setting a new passphrase (:ref:`borg_init`, :ref:`borg_key_change-passphrase`) normally
Borg prompts whether it should display the passphrase. This can be suppressed by setting
the environment variable *BORG_DISPLAY_PASSPHRASE* to *no*.
When "confronted" with an unknown repository, where the application does not know whether
the repository is encrypted, the following algorithm can be followed to detect encryption:
1. Set *BORG_PASSPHRASE* to gibberish (for example a freshly generated UUID4, which cannot
possibly be the passphrase)
2. Invoke ``borg list repository ...``
3. If this fails, due the repository being encrypted and the passphrase obviously being
wrong, you'll get an error with the *PassphraseWrong* msgid.
The repository is encrypted, for further access the application will need the passphrase.
4. If this does not fail, then the repository is not encrypted.
Standard output
---------------
*stdout* is different and more command-dependent than logging. Commands like :ref:`borg_info`, :ref:`borg_create`
and :ref:`borg_list` implement a ``--json`` option which turns their regular output into a single JSON object.
Dates are formatted according to ISO 8601 in local time. No explicit time zone is specified *at this time*
(subject to change). The equivalent strftime format string is '%Y-%m-%dT%H:%M:%S.%f',
e.g. ``2017-08-07T12:27:20.123456``.
The root object at least contains a *repository* key with an object containing:
id
The ID of the repository, normally 64 hex characters
location
Canonicalized repository path, thus this may be different from what is specified on the command line
last_modified
Date when the repository was last modified by the Borg client
The *encryption* key, if present, contains:
mode
Textual encryption mode name (same as :ref:`borg_init` ``--encryption`` names)
keyfile
Path to the local key file used for access. Depending on *mode* this key may be absent.
The *cache* key, if present, contains:
path
Path to the local repository cache
stats
Object containing cache stats:
total_chunks
Number of chunks
total_unique_chunks
Number of unique chunks
total_size
Total uncompressed size of all chunks multiplied with their reference counts
total_csize
Total compressed and encrypted size of all chunks multiplied with their reference counts
unique_size
Uncompressed size of all chunks
unique_csize
Compressed and encrypted size of all chunks
Example *borg info* output::
{
"cache": {
"path": "/home/user/.cache/borg/0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23",
"stats": {
"total_chunks": 511533,
"total_csize": 17948017540,
"total_size": 22635749792,
"total_unique_chunks": 54892,
"unique_csize": 1920405405,
"unique_size": 2449675468
}
},
"encryption": {
"mode": "repokey"
},
"repository": {
"id": "0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23",
"last_modified": "2017-08-07T12:27:20.789123",
"location": "/home/user/testrepo"
},
"security_dir": "/home/user/.config/borg/security/0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23",
"archives": []
}
Archive formats
+++++++++++++++
:ref:`borg_info` uses an extended format for archives, which is more expensive to retrieve, while
:ref:`borg_list` uses a simpler format that is faster to retrieve. Either return archives in an
array under the *archives* key, while :ref:`borg_create` returns a single archive object under the
*archive* key.
Both formats contain a *name* key with the archive name, the *id* key with the hexadecimal archive ID,
and the *start* key with the start timestamp.
*borg info* and *borg create* further have:
end
End timestamp
duration
Duration in seconds between start and end in seconds (float)
stats
Archive statistics (freshly calculated, this is what makes "info" more expensive)
original_size
Size of files and metadata before compression
compressed_size
Size after compression
deduplicated_size
Deduplicated size (against the current repository, not when the archive was created)
nfiles
Number of regular files in the archive
limits
Object describing the utilization of Borg limits
max_archive_size
Float between 0 and 1 describing how large this archive is relative to the maximum size allowed by Borg
command_line
Array of strings of the command line that created the archive
The note about paths from above applies here as well.
:ref:`borg_info` further has:
hostname
Hostname of the creating host
username
Name of the creating user
comment
Archive comment, if any
Example of a simple archive listing (``borg list --last 1 --json``)::
{
"archives": [
{
"id": "80cd07219ad725b3c5f665c1dcf119435c4dee1647a560ecac30f8d40221a46a",
"name": "host-system-backup-2017-02-27",
"start": "2017-08-07T12:27:20.789123"
}
],
"encryption": {
"mode": "repokey"
},
"repository": {
"id": "0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23",
"last_modified": "2017-08-07T12:27:20.789123",
"location": "/home/user/repository"
}
}
The same archive with more information (``borg info --last 1 --json``)::
{
"archives": [
{
"command_line": [
"/home/user/.local/bin/borg",
"create",
"/home/user/repository",
"..."
],
"comment": "",
"duration": 5.641542,
"end": "2017-02-27T12:27:20.789123",
"hostname": "host",
"id": "80cd07219ad725b3c5f665c1dcf119435c4dee1647a560ecac30f8d40221a46a",
"limits": {
"max_archive_size": 0.0001330855110409714
},
"name": "host-system-backup-2017-02-27",
"start": "2017-02-27T12:27:20.789123",
"stats": {
"compressed_size": 1880961894,
"deduplicated_size": 2791,
"nfiles": 53669,
"original_size": 2400471280
},
"username": "user"
}
],
"cache": {
"path": "/home/user/.cache/borg/0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23",
"stats": {
"total_chunks": 511533,
"total_csize": 17948017540,
"total_size": 22635749792,
"total_unique_chunks": 54892,
"unique_csize": 1920405405,
"unique_size": 2449675468
}
},
"encryption": {
"mode": "repokey"
},
"repository": {
"id": "0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23",
"last_modified": "2017-08-07T12:27:20.789123",
"location": "/home/user/repository"
}
}
File listings
+++++++++++++
Listing the contents of an archive can produce *a lot* of JSON. Since many JSON implementations
don't support a streaming mode of operation, which is pretty much required to deal with this amount of
JSON, output is generated in the `JSON lines <http://jsonlines.org/>`_ format, which is simply
a number of JSON objects separated by new lines.
Each item (file, directory, ...) is described by one object in the :ref:`borg_list` output.
Refer to the *borg list* documentation for the available keys and their meaning.
Example (excerpt) of ``borg list --json-lines``::
{"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux", "healthy": true, "source": "", "linktarget": "", "flags": null, "mtime": "2017-02-27T12:27:20.023407", "size": 0}
{"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux/baz", "healthy": true, "source": "", "linktarget": "", "flags": null, "mtime": "2017-02-27T12:27:20.585407", "size": 0}
.. _msgid:
Message IDs
-----------
Message IDs are strings that essentially give a log message or operation a name, without actually using the
full text, since texts change more frequently. Message IDs are unambiguous and reduce the need to parse
log messages.
Assigned message IDs are:
.. See scripts/errorlist.py; this is slightly edited.
Errors
Archive.AlreadyExists
Archive {} already exists
Archive.DoesNotExist
Archive {} does not exist
Archive.IncompatibleFilesystemEncodingError
Failed to encode filename "{}" into file system encoding "{}". Consider configuring the LANG environment variable.
Cache.CacheInitAbortedError
Cache initialization aborted
Cache.EncryptionMethodMismatch
Repository encryption method changed since last access, refusing to continue
Cache.RepositoryAccessAborted
Repository access aborted
Cache.RepositoryIDNotUnique
Cache is newer than repository - do you have multiple, independently updated repos with same ID?
Cache.RepositoryReplay
Cache is newer than repository - this is either an attack or unsafe (multiple repos with same ID)
Buffer.MemoryLimitExceeded
Requested buffer size {} is above the limit of {}.
ExtensionModuleError
The Borg binary extension modules do not seem to be properly installed
IntegrityError
Data integrity error: {}
NoManifestError
Repository has no manifest.
PlaceholderError
Formatting Error: "{}".format({}): {}({})
KeyfileInvalidError
Invalid key file for repository {} found in {}.
KeyfileMismatchError
Mismatch between repository {} and key file {}.
KeyfileNotFoundError
No key file for repository {} found in {}.
PassphraseWrong
passphrase supplied in BORG_PASSPHRASE is incorrect
PasswordRetriesExceeded
exceeded the maximum password retries
RepoKeyNotFoundError
No key entry found in the config of repository {}.
UnsupportedManifestError
Unsupported manifest envelope. A newer version is required to access this repository.
UnsupportedPayloadError
Unsupported payload type {}. A newer version is required to access this repository.
NotABorgKeyFile
This file is not a borg key backup, aborting.
RepoIdMismatch
This key backup seems to be for a different backup repository, aborting.
UnencryptedRepo
Keymanagement not available for unencrypted repositories.
UnknownKeyType
Keytype {0} is unknown.
LockError
Failed to acquire the lock {}.
LockErrorT
Failed to acquire the lock {}.
ConnectionClosed
Connection closed by remote host
InvalidRPCMethod
RPC method {} is not valid
PathNotAllowed
Repository path not allowed
RemoteRepository.RPCServerOutdated
Borg server is too old for {}. Required version {}
UnexpectedRPCDataFormatFromClient
Borg {}: Got unexpected RPC data format from client.
UnexpectedRPCDataFormatFromServer
Got unexpected RPC data format from server:
{}
Repository.AlreadyExists
Repository {} already exists.
Repository.CheckNeeded
Inconsistency detected. Please run "borg check {}".
Repository.DoesNotExist
Repository {} does not exist.
Repository.InsufficientFreeSpaceError
Insufficient free space to complete transaction (required: {}, available: {}).
Repository.InvalidRepository
{} is not a valid repository. Check repo config.
Repository.AtticRepository
Attic repository detected. Please run "borg upgrade {}".
Repository.ObjectNotFound
Object with key {} not found in repository {}.
Operations
- cache.begin_transaction
- cache.download_chunks, appears with ``borg create --no-cache-sync``
- cache.commit
- cache.sync
*info* is one string element, the name of the archive currently synced.
- repository.compact_segments
- repository.replay_segments
- repository.check_segments
- check.verify_data
- extract
*info* is one string element, the name of the path currently extracted.
- extract.permissions
- archive.delete
Prompts
BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK
For "Warning: Attempting to access a previously unknown unencrypted repository"
BORG_RELOCATED_REPO_ACCESS_IS_OK
For "Warning: The repository at location ... was previously located at ..."
BORG_CHECK_I_KNOW_WHAT_I_AM_DOING
For "Warning: 'check --repair' is an experimental feature that might result in data loss."
BORG_DELETE_I_KNOW_WHAT_I_AM_DOING
For "You requested to completely DELETE the repository *including* all archives it contains:"
BORG_RECREATE_I_KNOW_WHAT_I_AM_DOING
For "recreate is an experimental feature."

BIN
docs/internals/object-graph.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 313 KiB

BIN
docs/internals/object-graph.vsd Normal file
View File

Binary file not shown.

371
docs/internals/security.rst Normal file
View File

@ -0,0 +1,371 @@
.. somewhat surprisingly the "bash" highlighter gives nice results with
the pseudo-code notation used in the "Encryption" section.
.. highlight:: bash
========
Security
========
.. _borgcrypto:
Cryptography in Borg
====================
Attack model
------------
The attack model of Borg is that the environment of the client process
(e.g. ``borg create``) is trusted and the repository (server) is not. The
attacker has any and all access to the repository, including interactive
manipulation (man-in-the-middle) for remote repositories.
Furthermore the client environment is assumed to be persistent across
attacks (practically this means that the security database cannot be
deleted between attacks).
Under these circumstances Borg guarantees that the attacker cannot
1. modify the data of any archive without the client detecting the change
2. rename, remove or add an archive without the client detecting the change
3. recover plain-text data
4. recover definite (heuristics based on access patterns are possible)
structural information such as the object graph (which archives
refer to what chunks)
The attacker can always impose a denial of service per definition (he could
forbid connections to the repository, or delete it entirely).
.. _security_structural_auth:
Structural Authentication
-------------------------
Borg is fundamentally based on an object graph structure (see :ref:`internals`),
where the root object is called the manifest.
Borg follows the `Horton principle`_, which states that
not only the message must be authenticated, but also its meaning (often
expressed through context), because every object used is referenced by a
parent object through its object ID up to the manifest. The object ID in
Borg is a MAC of the object's plaintext, therefore this ensures that
an attacker cannot change the context of an object without forging the MAC.
In other words, the object ID itself only authenticates the plaintext of the
object and not its context or meaning. The latter is established by a different
object referring to an object ID, thereby assigning a particular meaning to
an object. For example, an archive item contains a list of object IDs that
represent packed file metadata. On their own it's not clear that these objects
would represent what they do, but by the archive item referring to them
in a particular part of its own data structure assigns this meaning.
This results in a directed acyclic graph of authentication from the manifest
to the data chunks of individual files.
.. _tam_description:
.. rubric:: Authenticating the manifest
Since the manifest has a fixed ID (000...000) the aforementioned authentication
does not apply to it, indeed, cannot apply to it; it is impossible to authenticate
the root node of a DAG through its edges, since the root node has no incoming edges.
With the scheme as described so far an attacker could easily replace the manifest,
therefore Borg includes a tertiary authentication mechanism (TAM) that is applied
to the manifest since version 1.0.9 (see :ref:`tam_vuln`).
TAM works by deriving a separate key through HKDF_ from the other encryption and
authentication keys and calculating the HMAC of the metadata to authenticate [#]_::
# RANDOM(n) returns n random bytes
salt = RANDOM(64)
ikm = id_key || enc_key || enc_hmac_key
# *context* depends on the operation, for manifest authentication it is
# the ASCII string "borg-metadata-authentication-manifest".
tam_key = HKDF-SHA-512(ikm, salt, context)
# *data* is a dict-like structure
data[hmac] = zeroes
packed = pack(data)
data[hmac] = HMAC(tam_key, packed)
packed_authenticated = pack(data)
Since an attacker cannot gain access to this key and also cannot make the
client authenticate arbitrary data using this mechanism, the attacker is unable
to forge the authentication.
This effectively 'anchors' the manifest to the key, which is controlled by the
client, thereby anchoring the entire DAG, making it impossible for an attacker
to add, remove or modify any part of the DAG without Borg being able to detect
the tampering.
Note that when using BORG_PASSPHRASE the attacker cannot swap the *entire*
repository against a new repository with e.g. repokey mode and no passphrase,
because Borg will abort access when BORG_PASSPRHASE is incorrect.
However, interactively a user might not notice this kind of attack
immediately, if she assumes that the reason for the absent passphrase
prompt is a set BORG_PASSPHRASE. See issue :issue:`2169` for details.
.. [#] The reason why the authentication tag is stored in the packed
data itself is that older Borg versions can still read the
manifest this way, while a changed layout would have broken
compatibility.
Encryption
----------
Encryption is currently based on the Encrypt-then-MAC construction,
which is generally seen as the most robust way to create an authenticated
encryption scheme from encryption and message authentication primitives.
Every operation (encryption, MAC / authentication, chunk ID derivation)
uses independent, random keys generated by `os.urandom`_ [#]_.
Borg does not support unauthenticated encryption -- only authenticated encryption
schemes are supported. No unauthenticated encryption schemes will be added
in the future.
Depending on the chosen mode (see :ref:`borg_init`) different primitives are used:
- The actual encryption is currently always AES-256 in CTR mode. The
counter is added in plaintext, since it is needed for decryption,
and is also tracked locally on the client to avoid counter reuse.
- The authentication primitive is either HMAC-SHA-256 or BLAKE2b-256
in a keyed mode. HMAC-SHA-256 uses 256 bit keys, while BLAKE2b-256
uses 512 bit keys.
The latter is secure not only because BLAKE2b itself is not
susceptible to `length extension`_, but also since it truncates the
hash output from 512 bits to 256 bits, which would make the
construction safe even if BLAKE2b were broken regarding length
extension or similar attacks.
- The primitive used for authentication is always the same primitive
that is used for deriving the chunk ID, but they are always
used with independent keys.
Encryption::
id = AUTHENTICATOR(id_key, data)
compressed = compress(data)
iv = reserve_iv()
encrypted = AES-256-CTR(enc_key, 8-null-bytes || iv, compressed)
authenticated = type-byte || AUTHENTICATOR(enc_hmac_key, encrypted) || iv || encrypted
Decryption::
# Given: input *authenticated* data, possibly a *chunk-id* to assert
type-byte, mac, iv, encrypted = SPLIT(authenticated)
ASSERT(type-byte is correct)
ASSERT( CONSTANT-TIME-COMPARISON( mac, AUTHENTICATOR(enc_hmac_key, encrypted) ) )
decrypted = AES-256-CTR(enc_key, 8-null-bytes || iv, encrypted)
decompressed = decompress(decrypted)
ASSERT( CONSTANT-TIME-COMPARISON( chunk-id, AUTHENTICATOR(id_key, decompressed) ) )
The client needs to track which counter values have been used, since
encrypting a chunk requires a starting counter value and no two chunks
may have overlapping counter ranges (otherwise the bitwise XOR of the
overlapping plaintexts is revealed).
The client does not directly track the counter value, because it
changes often (with each encrypted chunk), instead it commits a
"reservation" to the security database and the repository by taking
the current counter value and adding 4 GiB / 16 bytes (the block size)
to the counter. Thus the client only needs to commit a new reservation
every few gigabytes of encrypted data.
This mechanism also avoids reusing counter values in case the client
crashes or the connection to the repository is severed, since any
reservation would have been committed to both the security database
and the repository before any data is encrypted. Borg uses its
standard mechanism (SaveFile) to ensure that reservations are durable
(on most hardware / storage systems), therefore a crash of the
client's host would not impact tracking of reservations.
However, this design is not infallible, and requires synchronization
between clients, which is handled through the repository. Therefore in
a multiple-client scenario a repository can trick a client into
reusing counter values by ignoring counter reservations and replaying
the manifest (which will fail if the client has seen a more recent
manifest or has a more recent nonce reservation). If the repository is
untrusted, but a trusted synchronization channel exists between
clients, the security database could be synchronized between them over
said trusted channel. This is not part of Borgs functionality.
.. [#] Using the :ref:`borg key migrate-to-repokey <borg_key_migrate-to-repokey>`
command a user can convert repositories created using Attic in "passphrase"
mode to "repokey" mode. In this case the keys were directly derived from
the user's passphrase at some point using PBKDF2.
Borg does not support "passphrase" mode otherwise any more.
.. _key_encryption:
Offline key security
--------------------
Borg cannot secure the key material while it is running, because the keys
are needed in plain to decrypt/encrypt repository objects.
For offline storage of the encryption keys they are encrypted with a
user-chosen passphrase.
A 256 bit key encryption key (KEK) is derived from the passphrase
using PBKDF2-HMAC-SHA256 with a random 256 bit salt which is then used
to Encrypt-*and*-MAC (unlike the Encrypt-*then*-MAC approach used
otherwise) a packed representation of the keys with AES-256-CTR with a
constant initialization vector of 0. A HMAC-SHA256 of the plaintext is
generated using the same KEK and is stored alongside the ciphertext,
which is converted to base64 in its entirety.
This base64 blob (commonly referred to as *keyblob*) is then stored in
the key file or in the repository config (keyfile and repokey modes
respectively).
This scheme, and specifically the use of a constant IV with the CTR
mode, is secure because an identical passphrase will result in a
different derived KEK for every key encryption due to the salt.
The use of Encrypt-and-MAC instead of Encrypt-then-MAC is seen as
uncritical (but not ideal) here, since it is combined with AES-CTR mode,
which is not vulnerable to padding attacks.
.. seealso::
Refer to the :ref:`key_files` section for details on the format.
Refer to issue :issue:`747` for suggested improvements of the encryption
scheme and password-based key derivation.
Implementations used
--------------------
We do not implement cryptographic primitives ourselves, but rely
on widely used libraries providing them:
- AES-CTR and HMAC-SHA-256 from OpenSSL 1.0 / 1.1 are used,
which is also linked into the static binaries we provide.
We think this is not an additional risk, since we don't ever
use OpenSSL's networking, TLS or X.509 code, but only their
primitives implemented in libcrypto.
- SHA-256 and SHA-512 from Python's hashlib_ standard library module are used.
Borg requires a Python built with OpenSSL support (due to PBKDF2), therefore
these functions are delegated to OpenSSL by Python.
- HMAC, PBKDF2 and a constant-time comparison from Python's hmac_ standard
library module is used. While the HMAC implementation is written in Python,
the PBKDF2 implementation is provided by OpenSSL. The constant-time comparison
(``compare_digest``) is written in C and part of Python.
- BLAKE2b is either provided by the system's libb2, an official implementation,
or a bundled copy of the BLAKE2 reference implementation (written in C).
Implemented cryptographic constructions are:
- Encrypt-then-MAC based on AES-256-CTR and either HMAC-SHA-256
or keyed BLAKE2b256 as described above under Encryption_.
- Encrypt-and-MAC based on AES-256-CTR and HMAC-SHA-256
as described above under `Offline key security`_.
- HKDF_-SHA-512
.. _Horton principle: https://en.wikipedia.org/wiki/Horton_Principle
.. _HKDF: https://tools.ietf.org/html/rfc5869
.. _length extension: https://en.wikipedia.org/wiki/Length_extension_attack
.. _hashlib: https://docs.python.org/3/library/hashlib.html
.. _hmac: https://docs.python.org/3/library/hmac.html
.. _os.urandom: https://docs.python.org/3/library/os.html#os.urandom
Remote RPC protocol security
============================
.. note:: This section could be further expanded / detailed.
The RPC protocol is fundamentally based on msgpack'd messages exchanged
over an encrypted SSH channel (the system's SSH client is used for this
by piping data from/to it).
This means that the authorization and transport security properties
are inherited from SSH and the configuration of the SSH client and the
SSH server -- Borg RPC does not contain *any* networking
code. Networking is done by the SSH client running in a separate
process, Borg only communicates over the standard pipes (stdout,
stderr and stdin) with this process. This also means that Borg doesn't
have to directly use a SSH client (or SSH at all). For example,
``sudo`` or ``qrexec`` could be used as an intermediary.
By using the system's SSH client and not implementing a
(cryptographic) network protocol Borg sidesteps many security issues
that would normally impact distributing statically linked / standalone
binaries.
The remainder of this section will focus on the security of the RPC
protocol within Borg.
The assumed worst-case a server can inflict to a client is a
denial of repository service.
The situation were a server can create a general DoS on the client
should be avoided, but might be possible by e.g. forcing the client to
allocate large amounts of memory to decode large messages (or messages
that merely indicate a large amount of data follows). The RPC protocol
code uses a limited msgpack Unpacker to prohibit this.
We believe that other kinds of attacks, especially critical vulnerabilities
like remote code execution are inhibited by the design of the protocol:
1. The server cannot send requests to the client on its own accord,
it only can send responses. This avoids "unexpected inversion of control"
issues.
2. msgpack serialization does not allow embedding or referencing code that
is automatically executed. Incoming messages are unpacked by the msgpack
unpacker into native Python data structures (like tuples and dictionaries),
which are then passed to the rest of the program.
Additional verification of the correct form of the responses could be implemented.
3. Remote errors are presented in two forms:
1. A simple plain-text *stderr* channel. A prefix string indicates the kind of message
(e.g. WARNING, INFO, ERROR), which is used to suppress it according to the
log level selected in the client.
A server can send arbitrary log messages, which may confuse a user. However,
log messages are only processed when server requests are in progress, therefore
the server cannot interfere / confuse with security critical dialogue like
the password prompt.
2. Server-side exceptions passed over the main data channel. These follow the
general pattern of server-sent responses and are sent instead of response data
for a request.
The msgpack implementation used (msgpack-python) has a good security track record,
a large test suite and no issues found by fuzzing. It is based on the msgpack-c implementation,
sharing the unpacking engine and some support code. msgpack-c has a good track record as well.
Some issues [#]_ in the past were located in code not included in msgpack-python.
Borg does not use msgpack-c.
.. [#] - `MessagePack fuzzing <https://blog.gypsyengineer.com/fun/msgpack-fuzzing.html>`_
- `Fixed integer overflow and EXT size problem <https://github.com/msgpack/msgpack-c/pull/547>`_
- `Fixed array and map size overflow <https://github.com/msgpack/msgpack-c/pull/550>`_
Using OpenSSL
=============
Borg uses the OpenSSL library for most cryptography (see `Implementations used`_ above).
OpenSSL is bundled with static releases, thus the bundled copy is not updated with system
updates.
OpenSSL is a large and complex piece of software and has had its share of vulnerabilities,
however, it is important to note that Borg links against ``libcrypto`` **not** ``libssl``.
libcrypto is the low-level cryptography part of OpenSSL,
while libssl implements TLS and related protocols.
The latter is not used by Borg (cf. `Remote RPC protocol security`_, Borg itself does not implement
any network access) and historically contained most vulnerabilities, especially critical ones.
The static binaries released by the project contain neither libssl nor the Python ssl/_ssl modules.

BIN
docs/internals/structure.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 197 KiB

BIN
docs/internals/structure.vsd Normal file
View File

Binary file not shown.

8
docs/introduction.rst Normal file
View File

@ -0,0 +1,8 @@
Introduction
============
.. this shim is here to fix the structure in the PDF
rendering. without this stub, the elements in the toctree of
index.rst show up a level below the README file included
.. include:: ../README.rst

101
docs/man/borg-benchmark-crud.1 Normal file
View File

@ -0,0 +1,101 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-BENCHMARK-CRUD 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-benchmark-crud \- Benchmark Create, Read, Update, Delete for archives.
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] benchmark crud <options> REPO PATH
.SH DESCRIPTION
.sp
This command benchmarks borg CRUD (create, read, update, delete) operations.
.sp
It creates input data below the given PATH and backups this data into the given REPO.
The REPO must already exist (it could be a fresh empty repo or an existing repo, the
command will create / read / update / delete some archives named borg\-test\-data* there.
.sp
Make sure you have free space there, you\(aqll need about 1GB each (+ overhead).
.sp
If your repository is encrypted and borg needs a passphrase to unlock the key, use:
.sp
BORG_PASSPHRASE=mysecret borg benchmark crud REPO PATH
.sp
Measurements are done with different input file sizes and counts.
The file contents are very artificial (either all zero or all random),
thus the measurement results do not necessarily reflect performance with real data.
Also, due to the kind of content used, no compression is used in these benchmarks.
.INDENT 0.0
.TP
.B C\- == borg create (1st archive creation, no compression, do not use files cache)
C\-Z\- == all\-zero files. full dedup, this is primarily measuring reader/chunker/hasher.
C\-R\- == random files. no dedup, measuring throughput through all processing stages.
.TP
.B R\- == borg extract (extract archive, dry\-run, do everything, but do not write files to disk)
R\-Z\- == all zero files. Measuring heavily duplicated files.
R\-R\- == random files. No duplication here, measuring throughput through all processing
.IP "System Message: ERROR/3 (docs/virtmanpage.rst:, line 56)"
Unexpected indentation.
.INDENT 7.0
.INDENT 3.5
stages, except writing to disk.
.UNINDENT
.UNINDENT
.TP
.B U\- == borg create (2nd archive creation of unchanged input files, measure files cache speed)
The throughput value is kind of virtual here, it does not actually read the file.
U\-Z\- == needs to check the 2 all\-zero chunks\(aq existence in the repo.
U\-R\- == needs to check existence of a lot of different chunks in the repo.
.TP
.B D\- == borg delete archive (delete last remaining archive, measure deletion + compaction)
D\-Z\- == few chunks to delete / few segments to compact/remove.
D\-R\- == many chunks to delete / many segments to compact/remove.
.UNINDENT
.sp
Please note that there might be quite some variance in these measurements.
Try multiple measurements and having a otherwise idle machine (and network, if you use it).
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.INDENT 0.0
.TP
.B REPO
repo to use for benchmark (must exist)
.TP
.B PATH
path were to create benchmark input data
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

47
docs/man/borg-benchmark.1 Normal file
View File

@ -0,0 +1,47 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-BENCHMARK 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-benchmark \- benchmark command
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.nf
borg [common options] benchmark crud ...
.fi
.sp
.SH DESCRIPTION
.sp
These commands do various benchmarks.
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP, \fIborg\-benchmark\-crud(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

56
docs/man/borg-break-lock.1 Normal file
View File

@ -0,0 +1,56 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-BREAK-LOCK 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-break-lock \- Break the repository lock (e.g. in case it was left by a dead borg.
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] break\-lock <options> REPOSITORY
.SH DESCRIPTION
.sp
This command breaks the repository and cache locks.
Please use carefully and only while no borg process (on any machine) is
trying to access the Cache or the Repository.
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.INDENT 0.0
.TP
.B REPOSITORY
repository for which to break the locks
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

52
docs/man/borg-change-passphrase.1 Normal file
View File

@ -0,0 +1,52 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-CHANGE-PASSPHRASE 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-change-passphrase \- Change repository key file passphrase
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] change\-passphrase <options> REPOSITORY
.SH DESCRIPTION
.sp
The key files used for repository encryption are optionally passphrase
protected. This command can be used to change this passphrase.
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.sp
REPOSITORY
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

148
docs/man/borg-check.1 Normal file
View File

@ -0,0 +1,148 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-CHECK 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-check \- Check repository consistency
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] check <options> REPOSITORY_OR_ARCHIVE
.SH DESCRIPTION
.sp
The check command verifies the consistency of a repository and the corresponding archives.
.sp
First, the underlying repository data files are checked:
.INDENT 0.0
.IP \(bu 2
For all segments the segment magic (header) is checked
.IP \(bu 2
For all objects stored in the segments, all metadata (e.g. crc and size) and
all data is read. The read data is checked by size and CRC. Bit rot and other
types of accidental damage can be detected this way.
.IP \(bu 2
If we are in repair mode and a integrity error is detected for a segment,
we try to recover as many objects from the segment as possible.
.IP \(bu 2
In repair mode, it makes sure that the index is consistent with the data
stored in the segments.
.IP \(bu 2
If you use a remote repo server via ssh:, the repo check is executed on the
repo server without causing significant network traffic.
.IP \(bu 2
The repository check can be skipped using the \fB\-\-archives\-only\fP option.
.UNINDENT
.sp
Second, the consistency and correctness of the archive metadata is verified:
.INDENT 0.0
.IP \(bu 2
Is the repo manifest present? If not, it is rebuilt from archive metadata
chunks (this requires reading and decrypting of all metadata and data).
.IP \(bu 2
Check if archive metadata chunk is present. if not, remove archive from
manifest.
.IP \(bu 2
For all files (items) in the archive, for all chunks referenced by these
files, check if chunk is present.
If a chunk is not present and we are in repair mode, replace it with a same\-size
replacement chunk of zeros.
If a previously lost chunk reappears (e.g. via a later backup) and we are in
repair mode, the all\-zero replacement chunk will be replaced by the correct chunk.
This requires reading of archive and file metadata, but not data.
.IP \(bu 2
If we are in repair mode and we checked all the archives: delete orphaned
chunks from the repo.
.IP \(bu 2
if you use a remote repo server via ssh:, the archive check is executed on
the client machine (because if encryption is enabled, the checks will require
decryption and this is always done client\-side, because key access will be
required).
.IP \(bu 2
The archive checks can be time consuming, they can be skipped using the
\fB\-\-repository\-only\fP option.
.UNINDENT
.sp
The \fB\-\-verify\-data\fP option will perform a full integrity verification (as opposed to
checking the CRC32 of the segment) of data, which means reading the data from the
repository, decrypting and decompressing it. This is a cryptographic verification,
which will detect (accidental) corruption. For encrypted repositories it is
tamper\-resistant as well, unless the attacker has access to the keys.
.sp
It is also very slow.
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.INDENT 0.0
.TP
.B REPOSITORY_OR_ARCHIVE
repository or archive to check consistency of
.UNINDENT
.SS optional arguments
.INDENT 0.0
.TP
.B \-\-repository\-only
only perform repository checks
.TP
.B \-\-archives\-only
only perform archives checks
.TP
.B \-\-verify\-data
perform cryptographic archive data integrity verification (conflicts with \fB\-\-repository\-only\fP)
.TP
.B \-\-repair
attempt to repair any inconsistencies found
.TP
.B \-\-save\-space
work slower, but using less space
.UNINDENT
.SS filters
.INDENT 0.0
.TP
.B \-P\fP,\fB \-\-prefix
only consider archive names starting with this prefix.
.TP
.B \-a\fP,\fB \-\-glob\-archives
only consider archive names matching the glob. sh: rules apply, see "borg help patterns". \fB\-\-prefix\fP and \fB\-\-glob\-archives\fP are mutually exclusive.
.TP
.B \-\-sort\-by
Comma\-separated list of sorting keys; valid keys are: timestamp, name, id; default is: timestamp
.TP
.BI \-\-first \ N
consider first N archives after other filters were applied
.TP
.BI \-\-last \ N
consider last N archives after other filters were applied
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

96
docs/man/borg-common.1 Normal file
View File

@ -0,0 +1,96 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-COMMON 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-common \- Common options of Borg commands
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.INDENT 0.0
.TP
.B \-h\fP,\fB \-\-help
show this help message and exit
.TP
.B \-\-critical
work on log level CRITICAL
.TP
.B \-\-error
work on log level ERROR
.TP
.B \-\-warning
work on log level WARNING (default)
.TP
.B \-\-info\fP,\fB \-v\fP,\fB \-\-verbose
work on log level INFO
.TP
.B \-\-debug
enable debug output, work on log level DEBUG
.TP
.BI \-\-debug\-topic \ TOPIC
enable TOPIC debugging (can be specified multiple times). The logger path is borg.debug.<TOPIC> if TOPIC is not fully qualified.
.TP
.B \-p\fP,\fB \-\-progress
show progress information
.TP
.B \-\-log\-json
Output one JSON object per log line instead of formatted text.
.TP
.BI \-\-lock\-wait \ N
wait for the lock, but max. N seconds (default: 1).
.TP
.B \-\-show\-version
show/log the borg version
.TP
.B \-\-show\-rc
show/log the return code (rc)
.TP
.B \-\-no\-files\-cache
do not load/update the file metadata cache used to detect unchanged files
.TP
.BI \-\-umask \ M
set umask to M (local and remote, default: 0077)
.TP
.BI \-\-remote\-path \ PATH
use PATH as borg executable on the remote (default: "borg")
.TP
.BI \-\-remote\-ratelimit \ rate
set remote network upload rate limit in kiByte/s (default: 0=unlimited)
.TP
.B \-\-consider\-part\-files
treat part files like normal files (e.g. to list/extract them)
.TP
.BI \-\-debug\-profile \ FILE
Write execution profile in Borg format into FILE. For local use a Python\-compatible file can be generated by suffixing FILE with ".pyprof".
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

96
docs/man/borg-compression.1 Normal file
View File

@ -0,0 +1,96 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-COMPRESSION 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-compression \- Details regarding compression
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH DESCRIPTION
.sp
It is no problem to mix different compression methods in one repo,
deduplication is done on the source data chunks (not on the compressed
or encrypted data).
.sp
If some specific chunk was once compressed and stored into the repo, creating
another backup that also uses this chunk will not change the stored chunk.
So if you use different compression specs for the backups, whichever stores a
chunk first determines its compression. See also borg recreate.
.sp
Compression is lz4 by default. If you want something else, you have to specify what you want.
.sp
Valid compression specifiers are:
.INDENT 0.0
.TP
.B none
Do not compress.
.TP
.B lz4
Use lz4 compression. High speed, low compression. (default)
.TP
.B zlib[,L]
Use zlib ("gz") compression. Medium speed, medium compression.
If you do not explicitely give the compression level L (ranging from 0
to 9), it will use level 6.
Giving level 0 (means "no compression", but still has zlib protocol
overhead) is usually pointless, you better use "none" compression.
.TP
.B lzma[,L]
Use lzma ("xz") compression. Low speed, high compression.
If you do not explicitely give the compression level L (ranging from 0
to 9), it will use level 6.
Giving levels above 6 is pointless and counterproductive because it does
not compress better due to the buffer size used by borg \- but it wastes
lots of CPU cycles and RAM.
.TP
.B auto,C[,L]
Use a built\-in heuristic to decide per chunk whether to compress or not.
The heuristic tries with lz4 whether the data is compressible.
For incompressible data, it will not use compression (uses "none").
For compressible data, it uses the given C[,L] compression \- with C[,L]
being any valid compression specifier.
.UNINDENT
.sp
Examples:
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
borg create \-\-compression lz4 REPO::ARCHIVE data
borg create \-\-compression zlib REPO::ARCHIVE data
borg create \-\-compression zlib,1 REPO::ARCHIVE data
borg create \-\-compression auto,lzma,6 REPO::ARCHIVE data
borg create \-\-compression auto,lzma ...
.ft P
.fi
.UNINDENT
.UNINDENT
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

313
docs/man/borg-create.1 Normal file
View File

@ -0,0 +1,313 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-CREATE 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-create \- Create new archive
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] create <options> ARCHIVE PATH
.SH DESCRIPTION
.sp
This command creates a backup archive containing all files found while recursively
traversing all paths specified. Paths are added to the archive as they are given,
that means if relative paths are desired, the command has to be run from the correct
directory.
.sp
When giving \(aq\-\(aq as path, borg will read data from standard input and create a
file \(aqstdin\(aq in the created archive from that data.
.sp
The archive will consume almost no disk space for files or parts of files that
have already been stored in other archives.
.sp
The archive name needs to be unique. It must not end in \(aq.checkpoint\(aq or
\(aq.checkpoint.N\(aq (with N being a number), because these names are used for
checkpoints and treated in special ways.
.sp
In the archive name, you may use the following placeholders:
{now}, {utcnow}, {fqdn}, {hostname}, {user} and some others.
.sp
To speed up pulling backups over sshfs and similar network file systems which do
not provide correct inode information the \fB\-\-ignore\-inode\fP flag can be used. This
potentially decreases reliability of change detection, while avoiding always reading
all files on these file systems.
.sp
The mount points of filesystems or filesystem snapshots should be the same for every
creation of a new archive to ensure fast operation. This is because the file cache that
is used to determine changed files quickly uses absolute filenames.
If this is not possible, consider creating a bind mount to a stable location.
.sp
The \fB\-\-progress\fP option shows (from left to right) Original, Compressed and Deduplicated
(O, C and D, respectively), then the Number of files (N) processed so far, followed by
the currently processed path.
.sp
See the output of the "borg help patterns" command for more help on exclude patterns.
See the output of the "borg help placeholders" command for more help on placeholders.
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.INDENT 0.0
.TP
.B ARCHIVE
name of archive to create (must be also a valid directory name)
.TP
.B PATH
paths to archive
.UNINDENT
.SS optional arguments
.INDENT 0.0
.TP
.B \-n\fP,\fB \-\-dry\-run
do not create a backup archive
.TP
.B \-s\fP,\fB \-\-stats
print statistics for the created archive
.TP
.B \-\-list
output verbose list of items (files, dirs, ...)
.TP
.BI \-\-filter \ STATUSCHARS
only display items with the given status characters
.TP
.B \-\-json
output stats as JSON (implies \-\-stats)
.TP
.B \-\-no\-cache\-sync
experimental: do not synchronize the cache. Implies \-\-no\-files\-cache.
.UNINDENT
.SS Exclusion options
.INDENT 0.0
.TP
.BI \-e \ PATTERN\fP,\fB \ \-\-exclude \ PATTERN
exclude paths matching PATTERN
.TP
.BI \-\-exclude\-from \ EXCLUDEFILE
read exclude patterns from EXCLUDEFILE, one per line
.TP
.B \-\-exclude\-caches
exclude directories that contain a CACHEDIR.TAG file (\fI\%http://www.brynosaurus.com/cachedir/spec.html\fP)
.TP
.BI \-\-exclude\-if\-present \ NAME
exclude directories that are tagged by containing a filesystem object with the given NAME
.TP
.B \-\-keep\-exclude\-tags\fP,\fB \-\-keep\-tag\-files
if tag objects are specified with \-\-exclude\-if\-present, don\(aqt omit the tag objects themselves from the backup archive
.TP
.BI \-\-pattern \ PATTERN
experimental: include/exclude paths matching PATTERN
.TP
.BI \-\-patterns\-from \ PATTERNFILE
experimental: read include/exclude patterns from PATTERNFILE, one per line
.UNINDENT
.SS Filesystem options
.INDENT 0.0
.TP
.B \-x\fP,\fB \-\-one\-file\-system
stay in the same file system and do not store mount points of other file systems
.TP
.B \-\-numeric\-owner
only store numeric user and group identifiers
.TP
.B \-\-noatime
do not store atime into archive
.TP
.B \-\-noctime
do not store ctime into archive
.TP
.B \-\-ignore\-inode
ignore inode data in the file metadata cache used to detect unchanged files.
.TP
.B \-\-read\-special
open and read block and char device files as well as FIFOs as if they were regular files. Also follows symlinks pointing to these kinds of files.
.UNINDENT
.SS Archive options
.INDENT 0.0
.TP
.BI \-\-comment \ COMMENT
add a comment text to the archive
.TP
.BI \-\-timestamp \ TIMESTAMP
manually specify the archive creation date/time (UTC, yyyy\-mm\-ddThh:mm:ss format). alternatively, give a reference file/directory.
.TP
.BI \-c \ SECONDS\fP,\fB \ \-\-checkpoint\-interval \ SECONDS
write checkpoint every SECONDS seconds (Default: 1800)
.TP
.BI \-\-chunker\-params \ PARAMS
specify the chunker parameters (CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE). default: 19,23,21,4095
.TP
.BI \-C \ COMPRESSION\fP,\fB \ \-\-compression \ COMPRESSION
select compression algorithm, see the output of the "borg help compression" command for details.
.UNINDENT
.SH EXAMPLES
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
# Backup ~/Documents into an archive named "my\-documents"
$ borg create /path/to/repo::my\-documents ~/Documents
# same, but list all files as we process them
$ borg create \-\-list /path/to/repo::my\-documents ~/Documents
# Backup ~/Documents and ~/src but exclude pyc files
$ borg create /path/to/repo::my\-files \e
~/Documents \e
~/src \e
\-\-exclude \(aq*.pyc\(aq
# Backup home directories excluding image thumbnails (i.e. only
# /home/<one directory>/.thumbnails is excluded, not /home/*/*/.thumbnails etc.)
$ borg create /path/to/repo::my\-files /home \e
\-\-exclude \(aqsh:/home/*/.thumbnails\(aq
# Backup the root filesystem into an archive named "root\-YYYY\-MM\-DD"
# use zlib compression (good, but slow) \- default is lz4 (fast, low compression ratio)
$ borg create \-C zlib,6 /path/to/repo::root\-{now:%Y\-%m\-%d} / \-\-one\-file\-system
# Backup a remote host locally ("pull" style) using sshfs
$ mkdir sshfs\-mount
$ sshfs root@example.com:/ sshfs\-mount
$ cd sshfs\-mount
$ borg create /path/to/repo::example.com\-root\-{now:%Y\-%m\-%d} .
$ cd ..
$ fusermount \-u sshfs\-mount
# Make a big effort in fine granular deduplication (big chunk management
# overhead, needs a lot of RAM and disk space, see formula in internals
# docs \- same parameters as borg < 1.0 or attic):
$ borg create \-\-chunker\-params 10,23,16,4095 /path/to/repo::small /smallstuff
# Backup a raw device (must not be active/in use/mounted at that time)
$ dd if=/dev/sdx bs=10M | borg create /path/to/repo::my\-sdx \-
# No compression (default)
$ borg create /path/to/repo::arch ~
# Super fast, low compression
$ borg create \-\-compression lz4 /path/to/repo::arch ~
# Less fast, higher compression (N = 0..9)
$ borg create \-\-compression zlib,N /path/to/repo::arch ~
# Even slower, even higher compression (N = 0..9)
$ borg create \-\-compression lzma,N /path/to/repo::arch ~
# Use short hostname, user name and current time in archive name
$ borg create /path/to/repo::{hostname}\-{user}\-{now} ~
# Similar, use the same datetime format as borg 1.1 will have as default
$ borg create /path/to/repo::{hostname}\-{user}\-{now:%Y\-%m\-%dT%H:%M:%S} ~
# As above, but add nanoseconds
$ borg create /path/to/repo::{hostname}\-{user}\-{now:%Y\-%m\-%dT%H:%M:%S.%f} ~
# Backing up relative paths by moving into the correct directory first
$ cd /home/user/Documents
# The root directory of the archive will be "projectA"
$ borg create /path/to/repo::daily\-projectA\-{now:%Y\-%m\-%d} projectA
.ft P
.fi
.UNINDENT
.UNINDENT
.SH NOTES
.sp
The \fB\-\-exclude\fP patterns are not like tar. In tar \fB\-\-exclude\fP .bundler/gems will
exclude foo/.bundler/gems. In borg it will not, you need to use \fB\-\-exclude\fP
\(aq*/.bundler/gems\(aq to get the same effect. See \fBborg help patterns\fP for
more information.
.sp
In addition to using \fB\-\-exclude\fP patterns, it is possible to use
\fB\-\-exclude\-if\-present\fP to specify the name of a filesystem object (e.g. a file
or folder name) which, when contained within another folder, will prevent the
containing folder from being backed up. By default, the containing folder and
all of its contents will be omitted from the backup. If, however, you wish to
only include the objects specified by \fB\-\-exclude\-if\-present\fP in your backup,
and not include any other contents of the containing folder, this can be enabled
through using the \fB\-\-keep\-exclude\-tags\fP option.
.SS Item flags
.sp
\fB\-\-list\fP outputs a list of all files, directories and other
file system items it considered (no matter whether they had content changes
or not). For each item, it prefixes a single\-letter flag that indicates type
and/or status of the item.
.sp
If you are interested only in a subset of that output, you can give e.g.
\fB\-\-filter=AME\fP and it will only show regular files with A, M or E status (see
below).
.sp
A uppercase character represents the status of a regular file relative to the
"files" cache (not relative to the repo \-\- this is an issue if the files cache
is not used). Metadata is stored in any case and for \(aqA\(aq and \(aqM\(aq also new data
chunks are stored. For \(aqU\(aq all data chunks refer to already existing chunks.
.INDENT 0.0
.IP \(bu 2
\(aqA\(aq = regular file, added (see also \fIa_status_oddity\fP in the FAQ)
.IP \(bu 2
\(aqM\(aq = regular file, modified
.IP \(bu 2
\(aqU\(aq = regular file, unchanged
.IP \(bu 2
\(aqE\(aq = regular file, an error happened while accessing/reading \fIthis\fP file
.UNINDENT
.sp
A lowercase character means a file type other than a regular file,
borg usually just stores their metadata:
.INDENT 0.0
.IP \(bu 2
\(aqd\(aq = directory
.IP \(bu 2
\(aqb\(aq = block device
.IP \(bu 2
\(aqc\(aq = char device
.IP \(bu 2
\(aqh\(aq = regular file, hardlink (to already seen inodes)
.IP \(bu 2
\(aqs\(aq = symlink
.IP \(bu 2
\(aqf\(aq = fifo
.UNINDENT
.sp
Other flags used include:
.INDENT 0.0
.IP \(bu 2
\(aqi\(aq = backup data was read from standard input (stdin)
.IP \(bu 2
\(aq\-\(aq = dry run, item was \fInot\fP backed up
.IP \(bu 2
\(aqx\(aq = excluded, item was \fInot\fP backed up
.IP \(bu 2
\(aq?\(aq = missing status code (if you see this, please file a bug report!)
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP, \fIborg\-delete(1)\fP, \fIborg\-prune(1)\fP, \fIborg\-check(1)\fP, \fIborg\-patterns(1)\fP, \fIborg\-placeholders(1)\fP, \fIborg\-compression(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

109
docs/man/borg-delete.1 Normal file
View File

@ -0,0 +1,109 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-DELETE 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-delete \- Delete an existing repository or archives
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] delete <options> TARGET
.SH DESCRIPTION
.sp
This command deletes an archive from the repository or the complete repository.
Disk space is reclaimed accordingly. If you delete the complete repository, the
local cache for it (if any) is also deleted.
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.INDENT 0.0
.TP
.B TARGET
archive or repository to delete
.UNINDENT
.SS optional arguments
.INDENT 0.0
.TP
.B \-s\fP,\fB \-\-stats
print statistics for the deleted archive
.TP
.B \-c\fP,\fB \-\-cache\-only
delete only the local cache for the given repository
.TP
.B \-\-force
force deletion of corrupted archives, use \-\-force \-\-force in case \-\-force does not work.
.TP
.B \-\-save\-space
work slower, but using less space
.UNINDENT
.SS filters
.INDENT 0.0
.TP
.B \-P\fP,\fB \-\-prefix
only consider archive names starting with this prefix.
.TP
.B \-a\fP,\fB \-\-glob\-archives
only consider archive names matching the glob. sh: rules apply, see "borg help patterns". \fB\-\-prefix\fP and \fB\-\-glob\-archives\fP are mutually exclusive.
.TP
.B \-\-sort\-by
Comma\-separated list of sorting keys; valid keys are: timestamp, name, id; default is: timestamp
.TP
.BI \-\-first \ N
consider first N archives after other filters were applied
.TP
.BI \-\-last \ N
consider last N archives after other filters were applied
.UNINDENT
.SH EXAMPLES
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
# delete a single backup archive:
$ borg delete /path/to/repo::Monday
# delete the whole repository and the related local cache:
$ borg delete /path/to/repo
You requested to completely DELETE the repository *including* all archives it contains:
repo Mon, 2016\-02\-15 19:26:54
root\-2016\-02\-15 Mon, 2016\-02\-15 19:36:29
newname Mon, 2016\-02\-15 19:50:19
Type \(aqYES\(aq if you understand this and want to continue: YES
.ft P
.fi
.UNINDENT
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

151
docs/man/borg-diff.1 Normal file
View File

@ -0,0 +1,151 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-DIFF 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-diff \- Diff contents of two archives
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] diff <options> REPO_ARCHIVE1 ARCHIVE2 PATH
.SH DESCRIPTION
.sp
This command finds differences (file contents, user/group/mode) between archives.
.sp
A repository location and an archive name must be specified for REPO_ARCHIVE1.
ARCHIVE2 is just another archive name in same repository (no repository location
allowed).
.sp
For archives created with Borg 1.1 or newer diff automatically detects whether
the archives are created with the same chunker params. If so, only chunk IDs
are compared, which is very fast.
.sp
For archives prior to Borg 1.1 chunk contents are compared by default.
If you did not create the archives with different chunker params,
pass \fB\-\-same\-chunker\-params\fP\&.
Note that the chunker params changed from Borg 0.xx to 1.0.
.sp
See the output of the "borg help patterns" command for more help on exclude patterns.
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.INDENT 0.0
.TP
.B REPO_ARCHIVE1
repository location and ARCHIVE1 name
.TP
.B ARCHIVE2
ARCHIVE2 name (no repository location allowed)
.TP
.B PATH
paths of items inside the archives to compare; patterns are supported
.UNINDENT
.SS optional arguments
.INDENT 0.0
.TP
.B \-\-numeric\-owner
only consider numeric user and group identifiers
.TP
.B \-\-same\-chunker\-params
Override check of chunker parameters.
.TP
.B \-\-sort
Sort the output lines by file path.
.UNINDENT
.SS Exclusion options
.INDENT 0.0
.TP
.BI \-e \ PATTERN\fP,\fB \ \-\-exclude \ PATTERN
exclude paths matching PATTERN
.TP
.BI \-\-exclude\-from \ EXCLUDEFILE
read exclude patterns from EXCLUDEFILE, one per line
.TP
.B \-\-exclude\-caches
exclude directories that contain a CACHEDIR.TAG file (\fI\%http://www.brynosaurus.com/cachedir/spec.html\fP)
.TP
.BI \-\-exclude\-if\-present \ NAME
exclude directories that are tagged by containing a filesystem object with the given NAME
.TP
.B \-\-keep\-exclude\-tags\fP,\fB \-\-keep\-tag\-files
if tag objects are specified with \-\-exclude\-if\-present, don\(aqt omit the tag objects themselves from the backup archive
.TP
.BI \-\-pattern \ PATTERN
experimental: include/exclude paths matching PATTERN
.TP
.BI \-\-patterns\-from \ PATTERNFILE
experimental: read include/exclude patterns from PATTERNFILE, one per line
.UNINDENT
.SH EXAMPLES
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
$ borg init \-e=none testrepo
$ mkdir testdir
$ cd testdir
$ echo asdf > file1
$ dd if=/dev/urandom bs=1M count=4 > file2
$ touch file3
$ borg create ../testrepo::archive1 .
$ chmod a+x file1
$ echo "something" >> file2
$ borg create ../testrepo::archive2 .
$ rm file3
$ touch file4
$ borg create ../testrepo::archive3 .
$ cd ..
$ borg diff testrepo::archive1 archive2
[\-rw\-r\-\-r\-\- \-> \-rwxr\-xr\-x] file1
+135 B \-252 B file2
$ borg diff testrepo::archive2 archive3
added 0 B file4
removed 0 B file3
$ borg diff testrepo::archive1 archive3
[\-rw\-r\-\-r\-\- \-> \-rwxr\-xr\-x] file1
+135 B \-252 B file2
added 0 B file4
removed 0 B file3
.ft P
.fi
.UNINDENT
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

142
docs/man/borg-export-tar.1 Normal file
View File

@ -0,0 +1,142 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-EXPORT-TAR 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-export-tar \- Export archive contents as a tarball
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] export\-tar <options> ARCHIVE FILE PATH
.SH DESCRIPTION
.sp
This command creates a tarball from an archive.
.sp
When giving \(aq\-\(aq as the output FILE, Borg will write a tar stream to standard output.
.sp
By default (\fB\-\-tar\-filter=auto\fP) Borg will detect whether the FILE should be compressed
based on its file extension and pipe the tarball through an appropriate filter
before writing it to FILE:
.INDENT 0.0
.IP \(bu 2
\&.tar.gz: gzip
.IP \(bu 2
\&.tar.bz2: bzip2
.IP \(bu 2
\&.tar.xz: xz
.UNINDENT
.sp
Alternatively a \fB\-\-tar\-filter\fP program may be explicitly specified. It should
read the uncompressed tar stream from stdin and write a compressed/filtered
tar stream to stdout.
.sp
The generated tarball uses the GNU tar format.
.sp
export\-tar is a lossy conversion:
BSD flags, ACLs, extended attributes (xattrs), atime and ctime are not exported.
Timestamp resolution is limited to whole seconds, not the nanosecond resolution
otherwise supported by Borg.
.sp
A \fB\-\-sparse\fP option (as found in borg extract) is not supported.
.sp
By default the entire archive is extracted but a subset of files and directories
can be selected by passing a list of \fBPATHs\fP as arguments.
The file selection can further be restricted by using the \fB\-\-exclude\fP option.
.sp
See the output of the "borg help patterns" command for more help on exclude patterns.
.sp
\fB\-\-progress\fP can be slower than no progress display, since it makes one additional
pass over the archive metadata.
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.INDENT 0.0
.TP
.B ARCHIVE
archive to export
.TP
.B FILE
output tar file. "\-" to write to stdout instead.
.TP
.B PATH
paths to extract; patterns are supported
.UNINDENT
.SS optional arguments
.INDENT 0.0
.TP
.B \-\-tar\-filter
filter program to pipe data through
.TP
.B \-\-list
output verbose list of items (files, dirs, ...)
.TP
.BI \-e \ PATTERN\fP,\fB \ \-\-exclude \ PATTERN
exclude paths matching PATTERN
.TP
.BI \-\-exclude\-from \ EXCLUDEFILE
read exclude patterns from EXCLUDEFILE, one per line
.TP
.BI \-\-pattern \ PATTERN
experimental: include/exclude paths matching PATTERN
.TP
.BI \-\-patterns\-from \ PATTERNFILE
experimental: read include/exclude patterns from PATTERNFILE, one per line
.TP
.BI \-\-strip\-components \ NUMBER
Remove the specified number of leading path elements. Pathnames with fewer elements will be silently skipped.
.UNINDENT
.SH EXAMPLES
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
# export as uncompressed tar
$ borg export\-tar /path/to/repo::Monday Monday.tar
# exclude some types, compress using gzip
$ borg export\-tar /path/to/repo::Monday Monday.tar.gz \-\-exclude \(aq*.so\(aq
# use higher compression level with gzip
$ borg export\-tar testrepo::linux \-\-tar\-filter="gzip \-9" Monday.tar.gz
# export a gzipped tar, but instead of storing it on disk,
# upload it to a remote site using curl.
$ borg export\-tar ... \-\-tar\-filter="gzip" \- | curl \-\-data\-binary @\- https://somewhere/to/POST
.ft P
.fi
.UNINDENT
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

137
docs/man/borg-extract.1 Normal file
View File

@ -0,0 +1,137 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-EXTRACT 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-extract \- Extract archive contents
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] extract <options> ARCHIVE PATH
.SH DESCRIPTION
.sp
This command extracts the contents of an archive. By default the entire
archive is extracted but a subset of files and directories can be selected
by passing a list of \fBPATHs\fP as arguments. The file selection can further
be restricted by using the \fB\-\-exclude\fP option.
.sp
See the output of the "borg help patterns" command for more help on exclude patterns.
.sp
By using \fB\-\-dry\-run\fP, you can do all extraction steps except actually writing the
output data: reading metadata and data chunks from the repo, checking the hash/hmac,
decrypting, decompressing.
.sp
\fB\-\-progress\fP can be slower than no progress display, since it makes one additional
pass over the archive metadata.
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.INDENT 0.0
.TP
.B ARCHIVE
archive to extract
.TP
.B PATH
paths to extract; patterns are supported
.UNINDENT
.SS optional arguments
.INDENT 0.0
.TP
.B \-\-list
output verbose list of items (files, dirs, ...)
.TP
.B \-n\fP,\fB \-\-dry\-run
do not actually change any files
.TP
.BI \-e \ PATTERN\fP,\fB \ \-\-exclude \ PATTERN
exclude paths matching PATTERN
.TP
.BI \-\-exclude\-from \ EXCLUDEFILE
read exclude patterns from EXCLUDEFILE, one per line
.TP
.BI \-\-pattern \ PATTERN
experimental: include/exclude paths matching PATTERN
.TP
.BI \-\-patterns\-from \ PATTERNFILE
experimental: read include/exclude patterns from PATTERNFILE, one per line
.TP
.B \-\-numeric\-owner
only obey numeric user and group identifiers
.TP
.BI \-\-strip\-components \ NUMBER
Remove the specified number of leading path elements. Pathnames with fewer elements will be silently skipped.
.TP
.B \-\-stdout
write all extracted data to stdout
.TP
.B \-\-sparse
create holes in output sparse file from all\-zero chunks
.UNINDENT
.SH EXAMPLES
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
# Extract entire archive
$ borg extract /path/to/repo::my\-files
# Extract entire archive and list files while processing
$ borg extract \-\-list /path/to/repo::my\-files
# Verify whether an archive could be successfully extracted, but do not write files to disk
$ borg extract \-\-dry\-run /path/to/repo::my\-files
# Extract the "src" directory
$ borg extract /path/to/repo::my\-files home/USERNAME/src
# Extract the "src" directory but exclude object files
$ borg extract /path/to/repo::my\-files home/USERNAME/src \-\-exclude \(aq*.o\(aq
# Restore a raw device (must not be active/in use/mounted at that time)
$ borg extract \-\-stdout /path/to/repo::my\-sdx | dd of=/dev/sdx bs=10M
.ft P
.fi
.UNINDENT
.UNINDENT
.sp
\fBNOTE:\fP
.INDENT 0.0
.INDENT 3.5
Currently, extract always writes into the current working directory ("."),
so make sure you \fBcd\fP to the right place before calling \fBborg extract\fP\&.
.UNINDENT
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP, \fIborg\-mount(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

116
docs/man/borg-info.1 Normal file
View File

@ -0,0 +1,116 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-INFO 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-info \- Show archive details such as disk space used
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] info <options> REPOSITORY_OR_ARCHIVE
.SH DESCRIPTION
.sp
This command displays detailed information about the specified archive or repository.
.sp
Please note that the deduplicated sizes of the individual archives do not add
up to the deduplicated size of the repository ("all archives"), because the two
are meaning different things:
.INDENT 0.0
.TP
.B This archive / deduplicated size = amount of data stored ONLY for this archive
= unique chunks of this archive.
.TP
.B All archives / deduplicated size = amount of data stored in the repo
= all chunks in the repository.
.UNINDENT
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.INDENT 0.0
.TP
.B REPOSITORY_OR_ARCHIVE
archive or repository to display information about
.UNINDENT
.SS optional arguments
.INDENT 0.0
.TP
.B \-\-json
format output as JSON
.UNINDENT
.SS filters
.INDENT 0.0
.TP
.B \-P\fP,\fB \-\-prefix
only consider archive names starting with this prefix.
.TP
.B \-a\fP,\fB \-\-glob\-archives
only consider archive names matching the glob. sh: rules apply, see "borg help patterns". \fB\-\-prefix\fP and \fB\-\-glob\-archives\fP are mutually exclusive.
.TP
.B \-\-sort\-by
Comma\-separated list of sorting keys; valid keys are: timestamp, name, id; default is: timestamp
.TP
.BI \-\-first \ N
consider first N archives after other filters were applied
.TP
.BI \-\-last \ N
consider last N archives after other filters were applied
.UNINDENT
.SH EXAMPLES
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
$ borg info /path/to/repo::root\-2016\-02\-15
Name: root\-2016\-02\-15
Fingerprint: 57c827621f21b000a8d363c1e163cc55983822b3afff3a96df595077a660be50
Hostname: myhostname
Username: root
Time (start): Mon, 2016\-02\-15 19:36:29
Time (end): Mon, 2016\-02\-15 19:39:26
Command line: /usr/local/bin/borg create \-\-list \-C zlib,6 /path/to/repo::root\-2016\-02\-15 / \-\-one\-file\-system
Number of files: 38100
Original size Compressed size Deduplicated size
This archive: 1.33 GB 613.25 MB 571.64 MB
All archives: 1.63 GB 853.66 MB 584.12 MB
Unique chunks Total chunks
Chunk index: 36858 48844
.ft P
.fi
.UNINDENT
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP, \fIborg\-list(1)\fP, \fIborg\-diff(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

217
docs/man/borg-init.1 Normal file
View File

@ -0,0 +1,217 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-INIT 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-init \- Initialize an empty repository
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] init <options> REPOSITORY
.SH DESCRIPTION
.sp
This command initializes an empty repository. A repository is a filesystem
directory containing the deduplicated data from zero or more archives.
.sp
Encryption can be enabled at repository init time. It cannot be changed later.
.sp
It is not recommended to work without encryption. Repository encryption protects
you e.g. against the case that an attacker has access to your backup repository.
.sp
But be careful with the key / the passphrase:
.sp
If you want "passphrase\-only" security, use one of the repokey modes. The
key will be stored inside the repository (in its "config" file). In above
mentioned attack scenario, the attacker will have the key (but not the
passphrase).
.sp
If you want "passphrase and having\-the\-key" security, use one of the keyfile
modes. The key will be stored in your home directory (in .config/borg/keys).
In the attack scenario, the attacker who has just access to your repo won\(aqt
have the key (and also not the passphrase).
.sp
Make a backup copy of the key file (keyfile mode) or repo config file
(repokey mode) and keep it at a safe place, so you still have the key in
case it gets corrupted or lost. Also keep the passphrase at a safe place.
The backup that is encrypted with that key won\(aqt help you with that, of course.
.sp
Make sure you use a good passphrase. Not too short, not too simple. The real
encryption / decryption key is encrypted with / locked by your passphrase.
If an attacker gets your key, he can\(aqt unlock and use it without knowing the
passphrase.
.sp
Be careful with special or non\-ascii characters in your passphrase:
.INDENT 0.0
.IP \(bu 2
Borg processes the passphrase as unicode (and encodes it as utf\-8),
so it does not have problems dealing with even the strangest characters.
.IP \(bu 2
BUT: that does not necessarily apply to your OS / VM / keyboard configuration.
.UNINDENT
.sp
So better use a long passphrase made from simple ascii chars than one that
includes non\-ascii stuff or characters that are hard/impossible to enter on
a different keyboard layout.
.sp
You can change your passphrase for existing repos at any time, it won\(aqt affect
the encryption/decryption key or other secrets.
.SS Encryption modes
.\" nanorst: inline-fill
.
.TS
center;
|l|l|l|l|.
_
T{
Hash/MAC
T} T{
Not encrypted
no auth
T} T{
Not encrypted,
but authenticated
T} T{
Encrypted (AEAD w/ AES)
and authenticated
T}
_
T{
SHA\-256
T} T{
none
T} T{
\fIauthenticated\fP
T} T{
repokey
keyfile
T}
_
T{
BLAKE2b
T} T{
n/a
T} T{
\fIauthenticated\-blake2\fP
T} T{
\fIrepokey\-blake2\fP
\fIkeyfile\-blake2\fP
T}
_
.TE
.\" nanorst: inline-replace
.
.sp
\fIMarked modes\fP are new in Borg 1.1 and are not backwards\-compatible with Borg 1.0.x.
.sp
On modern Intel/AMD CPUs (except very cheap ones), AES is usually
hardware\-accelerated.
BLAKE2b is faster than SHA256 on Intel/AMD 64\-bit CPUs
(except AMD Ryzen and future CPUs with SHA extensions),
which makes \fIauthenticated\-blake2\fP faster than \fInone\fP and \fIauthenticated\fP\&.
.sp
On modern ARM CPUs, NEON provides hardware acceleration for SHA256 making it faster
than BLAKE2b\-256 there. NEON accelerates AES as well.
.sp
Hardware acceleration is always used automatically when available.
.sp
\fIrepokey\fP and \fIkeyfile\fP use AES\-CTR\-256 for encryption and HMAC\-SHA256 for
authentication in an encrypt\-then\-MAC (EtM) construction. The chunk ID hash
is HMAC\-SHA256 as well (with a separate key).
These modes are compatible with Borg 1.0.x.
.sp
\fIrepokey\-blake2\fP and \fIkeyfile\-blake2\fP are also authenticated encryption modes,
but use BLAKE2b\-256 instead of HMAC\-SHA256 for authentication. The chunk ID
hash is a keyed BLAKE2b\-256 hash.
These modes are new and \fInot\fP compatible with Borg 1.0.x.
.sp
\fIauthenticated\fP mode uses no encryption, but authenticates repository contents
through the same HMAC\-SHA256 hash as the \fIrepokey\fP and \fIkeyfile\fP modes (it uses it
as the chunk ID hash). The key is stored like \fIrepokey\fP\&.
This mode is new and \fInot\fP compatible with Borg 1.0.x.
.sp
\fIauthenticated\-blake2\fP is like \fIauthenticated\fP, but uses the keyed BLAKE2b\-256 hash
from the other blake2 modes.
This mode is new and \fInot\fP compatible with Borg 1.0.x.
.sp
\fInone\fP mode uses no encryption and no authentication. It uses SHA256 as chunk
ID hash. Not recommended, rather consider using an authenticated or
authenticated/encrypted mode. This mode has possible denial\-of\-service issues
when running \fBborg create\fP on contents controlled by an attacker.
Use it only for new repositories where no encryption is wanted \fBand\fP when compatibility
with 1.0.x is important. If compatibility with 1.0.x is not important, use
\fIauthenticated\-blake2\fP or \fIauthenticated\fP instead.
This mode is compatible with Borg 1.0.x.
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.INDENT 0.0
.TP
.B REPOSITORY
repository to create
.UNINDENT
.SS optional arguments
.INDENT 0.0
.TP
.B \-e\fP,\fB \-\-encryption
select encryption key mode \fB(required)\fP
.TP
.B \-\-append\-only
create an append\-only mode repository
.TP
.B \-\-storage\-quota
Set storage quota of the new repository (e.g. 5G, 1.5T). Default: no quota.
.UNINDENT
.SH EXAMPLES
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
# Local repository, repokey encryption, BLAKE2b (often faster, since Borg 1.1)
$ borg init \-\-encryption=repokey\-blake2 /path/to/repo
# Local repository (no encryption)
$ borg init \-\-encryption=none /path/to/repo
# Remote repository (accesses a remote borg via ssh)
$ borg init \-\-encryption=repokey\-blake2 user@hostname:backup
# Remote repository (store the key your home dir)
$ borg init \-\-encryption=keyfile user@hostname:backup
.ft P
.fi
.UNINDENT
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP, \fIborg\-create(1)\fP, \fIborg\-delete(1)\fP, \fIborg\-check(1)\fP, \fIborg\-list(1)\fP, \fIborg\-key\-import(1)\fP, \fIborg\-key\-export(1)\fP, \fIborg\-key\-change\-passphrase(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

96
docs/man/borg-key-change-passphrase.1 Normal file
View File

@ -0,0 +1,96 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-key-change-passphrase \- Change repository key file passphrase
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] key change\-passphrase <options> REPOSITORY
.SH DESCRIPTION
.sp
The key files used for repository encryption are optionally passphrase
protected. This command can be used to change this passphrase.
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.sp
REPOSITORY
.SH EXAMPLES
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
# Create a key file protected repository
$ borg init \-\-encryption=keyfile \-v /path/to/repo
Initializing repository at "/path/to/repo"
Enter new passphrase:
Enter same passphrase again:
Remember your passphrase. Your data will be inaccessible without it.
Key in "/root/.config/borg/keys/mnt_backup" created.
Keep this key safe. Your data will be inaccessible without it.
Synchronizing chunks cache...
Archives: 0, w/ cached Idx: 0, w/ outdated Idx: 0, w/o cached Idx: 0.
Done.
# Change key file passphrase
$ borg key change\-passphrase \-v /path/to/repo
Enter passphrase for key /root/.config/borg/keys/mnt_backup:
Enter new passphrase:
Enter same passphrase again:
Remember your passphrase. Your data will be inaccessible without it.
Key updated
.ft P
.fi
.UNINDENT
.UNINDENT
.sp
Fully automated using environment variables:
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
$ BORG_NEW_PASSPHRASE=old borg init \-e=repokey repo
# now "old" is the current passphrase.
$ BORG_PASSPHRASE=old BORG_NEW_PASSPHRASE=new borg key change\-passphrase repo
# now "new" is the current passphrase.
.ft P
.fi
.UNINDENT
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

81
docs/man/borg-key-export.1 Normal file
View File

@ -0,0 +1,81 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-KEY-EXPORT 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-key-export \- Export the repository key for backup
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] key export <options> REPOSITORY PATH
.SH DESCRIPTION
.sp
If repository encryption is used, the repository is inaccessible
without the key. This command allows to backup this essential key.
.sp
There are two backup formats. The normal backup format is suitable for
digital storage as a file. The \fB\-\-paper\fP backup format is optimized
for printing and typing in while importing, with per line checks to
reduce problems with manual input.
.sp
For repositories using keyfile encryption the key is saved locally
on the system that is capable of doing backups. To guard against loss
of this key, the key needs to be backed up independently of the main
data backup.
.sp
For repositories using the repokey encryption the key is saved in the
repository in the config file. A backup is thus not strictly needed,
but guards against the repository becoming inaccessible if the file
is damaged for some reason.
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.sp
REPOSITORY
.INDENT 0.0
.TP
.B PATH
where to store the backup
.UNINDENT
.SS optional arguments
.INDENT 0.0
.TP
.B \-\-paper
Create an export suitable for printing and later type\-in
.TP
.B \-\-qr\-html
Create an html file suitable for printing and later type\-in or qr scan
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP, \fIborg\-key\-import(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

67
docs/man/borg-key-import.1 Normal file
View File

@ -0,0 +1,67 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-KEY-IMPORT 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-key-import \- Import the repository key from backup
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] key import <options> REPOSITORY PATH
.SH DESCRIPTION
.sp
This command allows to restore a key previously backed up with the
export command.
.sp
If the \fB\-\-paper\fP option is given, the import will be an interactive
process in which each line is checked for plausibility before
proceeding to the next line. For this format PATH must not be given.
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.sp
REPOSITORY
.INDENT 0.0
.TP
.B PATH
path to the backup
.UNINDENT
.SS optional arguments
.INDENT 0.0
.TP
.B \-\-paper
interactively import from a backup done with \fB\-\-paper\fP
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP, \fIborg\-key\-export(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

66
docs/man/borg-key-migrate-to-repokey.1 Normal file
View File

@ -0,0 +1,66 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-key-migrate-to-repokey \- Migrate passphrase -> repokey
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] key migrate\-to\-repokey <options> REPOSITORY
.SH DESCRIPTION
.sp
This command migrates a repository from passphrase mode (removed in Borg 1.0)
to repokey mode.
.sp
You will be first asked for the repository passphrase (to open it in passphrase
mode). This is the same passphrase as you used to use for this repo before 1.0.
.sp
It will then derive the different secrets from this passphrase.
.sp
Then you will be asked for a new passphrase (twice, for safety). This
passphrase will be used to protect the repokey (which contains these same
secrets in encrypted form). You may use the same passphrase as you used to
use, but you may also use a different one.
.sp
After migrating to repokey mode, you can change the passphrase at any time.
But please note: the secrets will always stay the same and they could always
be derived from your (old) passphrase\-mode passphrase.
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.sp
REPOSITORY
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

47
docs/man/borg-key.1 Normal file
View File

@ -0,0 +1,47 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-KEY 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-key \- Manage a keyfile or repokey of a repository
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.nf
borg [common options] key export ...
borg [common options] key import ...
borg [common options] key change\-passphrase ...
borg [common options] key migrate\-to\-repokey ...
.fi
.sp
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP, \fIborg\-key\-export(1)\fP, \fIborg\-key\-import(1)\fP, \fIborg\-key\-change\-passphrase(1)\fP, \fIborg\-key\-migrate\-to\-repokey(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

249
docs/man/borg-list.1 Normal file
View File

@ -0,0 +1,249 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-LIST 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-list \- List archive or repository contents
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] list <options> REPOSITORY_OR_ARCHIVE PATH
.SH DESCRIPTION
.sp
This command lists the contents of a repository or an archive.
.sp
See the "borg help patterns" command for more help on exclude patterns.
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.INDENT 0.0
.TP
.B REPOSITORY_OR_ARCHIVE
repository/archive to list contents of
.TP
.B PATH
paths to list; patterns are supported
.UNINDENT
.SS optional arguments
.INDENT 0.0
.TP
.B \-\-short
only print file/directory names, nothing else
.TP
.B \-\-format\fP,\fB \-\-list\-format
specify format for file listing
(default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NL}")
.TP
.B \-\-json
Only valid for listing repository contents. Format output as JSON. The form of \fB\-\-format\fP is ignored, but keys used in it are added to the JSON output. Some keys are always present. Note: JSON can only represent text. A "barchive" key is therefore not available.
.TP
.B \-\-json\-lines
Only valid for listing archive contents. Format output as JSON Lines. The form of \fB\-\-format\fP is ignored, but keys used in it are added to the JSON output. Some keys are always present. Note: JSON can only represent text. A "bpath" key is therefore not available.
.UNINDENT
.SS filters
.INDENT 0.0
.TP
.B \-P\fP,\fB \-\-prefix
only consider archive names starting with this prefix.
.TP
.B \-a\fP,\fB \-\-glob\-archives
only consider archive names matching the glob. sh: rules apply, see "borg help patterns". \fB\-\-prefix\fP and \fB\-\-glob\-archives\fP are mutually exclusive.
.TP
.B \-\-sort\-by
Comma\-separated list of sorting keys; valid keys are: timestamp, name, id; default is: timestamp
.TP
.BI \-\-first \ N
consider first N archives after other filters were applied
.TP
.BI \-\-last \ N
consider last N archives after other filters were applied
.UNINDENT
.SS Exclusion options
.INDENT 0.0
.TP
.BI \-e \ PATTERN\fP,\fB \ \-\-exclude \ PATTERN
exclude paths matching PATTERN
.TP
.BI \-\-exclude\-from \ EXCLUDEFILE
read exclude patterns from EXCLUDEFILE, one per line
.TP
.B \-\-exclude\-caches
exclude directories that contain a CACHEDIR.TAG file (\fI\%http://www.brynosaurus.com/cachedir/spec.html\fP)
.TP
.BI \-\-exclude\-if\-present \ NAME
exclude directories that are tagged by containing a filesystem object with the given NAME
.TP
.B \-\-keep\-exclude\-tags\fP,\fB \-\-keep\-tag\-files
if tag objects are specified with \-\-exclude\-if\-present, don\(aqt omit the tag objects themselves from the backup archive
.TP
.BI \-\-pattern \ PATTERN
experimental: include/exclude paths matching PATTERN
.TP
.BI \-\-patterns\-from \ PATTERNFILE
experimental: read include/exclude patterns from PATTERNFILE, one per line
.UNINDENT
.SH EXAMPLES
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
$ borg list /path/to/repo
Monday Mon, 2016\-02\-15 19:15:11
repo Mon, 2016\-02\-15 19:26:54
root\-2016\-02\-15 Mon, 2016\-02\-15 19:36:29
newname Mon, 2016\-02\-15 19:50:19
\&...
$ borg list /path/to/repo::root\-2016\-02\-15
drwxr\-xr\-x root root 0 Mon, 2016\-02\-15 17:44:27 .
drwxrwxr\-x root root 0 Mon, 2016\-02\-15 19:04:49 bin
\-rwxr\-xr\-x root root 1029624 Thu, 2014\-11\-13 00:08:51 bin/bash
lrwxrwxrwx root root 0 Fri, 2015\-03\-27 20:24:26 bin/bzcmp \-> bzdiff
\-rwxr\-xr\-x root root 2140 Fri, 2015\-03\-27 20:24:22 bin/bzdiff
\&...
$ borg list /path/to/repo::archiveA \-\-list\-format="{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}"
drwxrwxr\-x user user 0 Sun, 2015\-02\-01 11:00:00 .
drwxrwxr\-x user user 0 Sun, 2015\-02\-01 11:00:00 code
drwxrwxr\-x user user 0 Sun, 2015\-02\-01 11:00:00 code/myproject
\-rw\-rw\-r\-\- user user 1416192 Sun, 2015\-02\-01 11:00:00 code/myproject/file.ext
\&...
.ft P
.fi
.UNINDENT
.UNINDENT
.SH NOTES
.sp
The following keys are available for \fB\-\-format\fP:
.INDENT 0.0
.IP \(bu 2
NEWLINE: OS dependent line separator
.IP \(bu 2
NL: alias of NEWLINE
.IP \(bu 2
NUL: NUL character for creating print0 / xargs \-0 like output, see barchive/bpath
.IP \(bu 2
SPACE
.IP \(bu 2
TAB
.IP \(bu 2
CR
.IP \(bu 2
LF
.UNINDENT
.sp
Keys for listing repository archives:
.INDENT 0.0
.IP \(bu 2
archive, name: archive name interpreted as text (might be missing non\-text characters, see barchive)
.IP \(bu 2
barchive: verbatim archive name, can contain any character except NUL
.IP \(bu 2
time: time of creation of the archive
.IP \(bu 2
id: internal ID of the archive
.UNINDENT
.sp
Keys for listing archive files:
.INDENT 0.0
.IP \(bu 2
type
.IP \(bu 2
mode
.IP \(bu 2
uid
.IP \(bu 2
gid
.IP \(bu 2
user
.IP \(bu 2
group
.IP \(bu 2
path: path interpreted as text (might be missing non\-text characters, see bpath)
.IP \(bu 2
bpath: verbatim POSIX path, can contain any character except NUL
.IP \(bu 2
source: link target for links (identical to linktarget)
.IP \(bu 2
linktarget
.IP \(bu 2
flags
.IP \(bu 2
size
.IP \(bu 2
csize: compressed size
.IP \(bu 2
dsize: deduplicated size
.IP \(bu 2
dcsize: deduplicated compressed size
.IP \(bu 2
num_chunks: number of chunks in this file
.IP \(bu 2
unique_chunks: number of unique chunks in this file
.IP \(bu 2
mtime
.IP \(bu 2
ctime
.IP \(bu 2
atime
.IP \(bu 2
isomtime
.IP \(bu 2
isoctime
.IP \(bu 2
isoatime
.IP \(bu 2
md5
.IP \(bu 2
sha1
.IP \(bu 2
sha224
.IP \(bu 2
sha256
.IP \(bu 2
sha384
.IP \(bu 2
sha512
.IP \(bu 2
archiveid
.IP \(bu 2
archivename
.IP \(bu 2
extra: prepends {source} with " \-> " for soft links and " link to " for hard links
.IP \(bu 2
health: either "healthy" (file ok) or "broken" (if file has all\-zero replacement chunks)
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP, \fIborg\-info(1)\fP, \fIborg\-diff(1)\fP, \fIborg\-prune(1)\fP, \fIborg\-patterns(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

118
docs/man/borg-mount.1 Normal file
View File

@ -0,0 +1,118 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-MOUNT 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-mount \- Mount archive or an entire repository as a FUSE filesystem
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] mount <options> REPOSITORY_OR_ARCHIVE MOUNTPOINT
.SH DESCRIPTION
.sp
This command mounts an archive as a FUSE filesystem. This can be useful for
browsing an archive or restoring individual files. Unless the \fB\-\-foreground\fP
option is given the command will run in the background until the filesystem
is \fBumounted\fP\&.
.sp
The command \fBborgfs\fP provides a wrapper for \fBborg mount\fP\&. This can also be
used in fstab entries:
\fB/path/to/repo /mnt/point fuse.borgfs defaults,noauto 0 0\fP
.sp
To allow a regular user to use fstab entries, add the \fBuser\fP option:
\fB/path/to/repo /mnt/point fuse.borgfs defaults,noauto,user 0 0\fP
.sp
For mount options, see the fuse(8) manual page. Additional mount options
supported by borg:
.INDENT 0.0
.IP \(bu 2
versions: when used with a repository mount, this gives a merged, versioned
view of the files in the archives. EXPERIMENTAL, layout may change in future.
.IP \(bu 2
allow_damaged_files: by default damaged files (where missing chunks were
replaced with runs of zeros by borg check \fB\-\-repair\fP) are not readable and
return EIO (I/O error). Set this option to read such files.
.UNINDENT
.sp
The BORG_MOUNT_DATA_CACHE_ENTRIES environment variable is meant for advanced users
to tweak the performance. It sets the number of cached data chunks; additional
memory usage can be up to ~8 MiB times this number. The default is the number
of CPU cores.
.sp
When the daemonized process receives a signal or crashes, it does not unmount.
Unmounting in these cases could cause an active rsync or similar process
to unintentionally delete data.
.sp
When running in the foreground ^C/SIGINT unmounts cleanly, but other
signals or crashes do not.
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.INDENT 0.0
.TP
.B REPOSITORY_OR_ARCHIVE
repository/archive to mount
.TP
.B MOUNTPOINT
where to mount filesystem
.UNINDENT
.SS optional arguments
.INDENT 0.0
.TP
.B \-f\fP,\fB \-\-foreground
stay in foreground, do not daemonize
.TP
.B \-o
Extra mount options
.UNINDENT
.SS filters
.INDENT 0.0
.TP
.B \-P\fP,\fB \-\-prefix
only consider archive names starting with this prefix.
.TP
.B \-a\fP,\fB \-\-glob\-archives
only consider archive names matching the glob. sh: rules apply, see "borg help patterns". \fB\-\-prefix\fP and \fB\-\-glob\-archives\fP are mutually exclusive.
.TP
.B \-\-sort\-by
Comma\-separated list of sorting keys; valid keys are: timestamp, name, id; default is: timestamp
.TP
.BI \-\-first \ N
consider first N archives after other filters were applied
.TP
.BI \-\-last \ N
consider last N archives after other filters were applied
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP, \fIborg\-umount(1)\fP, \fIborg\-extract(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

201
docs/man/borg-patterns.1 Normal file
View File

@ -0,0 +1,201 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-PATTERNS 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-patterns \- Details regarding patterns
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH DESCRIPTION
.sp
File patterns support these styles: fnmatch, shell, regular expressions,
path prefixes and path full\-matches. By default, fnmatch is used for
\fB\-\-exclude\fP patterns and shell\-style is used for the experimental \fB\-\-pattern\fP
option.
.sp
If followed by a colon (\(aq:\(aq) the first two characters of a pattern are used as a
style selector. Explicit style selection is necessary when a
non\-default style is desired or when the desired pattern starts with
two alphanumeric characters followed by a colon (i.e. \fIaa:something/*\fP).
.INDENT 0.0
.TP
.B \fI\%Fnmatch\fP, selector \fIfm:\fP
This is the default style for \fB\-\-exclude\fP and \fB\-\-exclude\-from\fP\&.
These patterns use a variant of shell pattern syntax, with \(aq*\(aq matching
any number of characters, \(aq?\(aq matching any single character, \(aq[...]\(aq
matching any single character specified, including ranges, and \(aq[!...]\(aq
matching any character not specified. For the purpose of these patterns,
the path separator (\(aq\(aq for Windows and \(aq/\(aq on other systems) is not
treated specially. Wrap meta\-characters in brackets for a literal
match (i.e. \fI[?]\fP to match the literal character \fI?\fP). For a path
to match a pattern, it must completely match from start to end, or
must match from the start to just before a path separator. Except
for the root path, paths will never end in the path separator when
matching is attempted. Thus, if a given pattern ends in a path
separator, a \(aq*\(aq is appended before matching is attempted.
.TP
.B Shell\-style patterns, selector \fIsh:\fP
This is the default style for \-\-pattern and \-\-patterns\-from.
Like fnmatch patterns these are similar to shell patterns. The difference
is that the pattern may include \fI**/\fP for matching zero or more directory
levels, \fI*\fP for matching zero or more arbitrary characters with the
exception of any path separator.
.TP
.B Regular expressions, selector \fIre:\fP
Regular expressions similar to those found in Perl are supported. Unlike
shell patterns regular expressions are not required to match the complete
path and any substring match is sufficient. It is strongly recommended to
anchor patterns to the start (\(aq^\(aq), to the end (\(aq$\(aq) or both. Path
separators (\(aq\(aq for Windows and \(aq/\(aq on other systems) in paths are
always normalized to a forward slash (\(aq/\(aq) before applying a pattern. The
regular expression syntax is described in the \fI\%Python documentation for
the re module\fP\&.
.TP
.B Path prefix, selector \fIpp:\fP
This pattern style is useful to match whole sub\-directories. The pattern
\fIpp:/data/bar\fP matches \fI/data/bar\fP and everything therein.
.TP
.B Path full\-match, selector \fIpf:\fP
This pattern style is useful to match whole paths.
This is kind of a pseudo pattern as it can not have any variable or
unspecified parts \- the full, precise path must be given.
\fIpf:/data/foo.txt\fP matches \fI/data/foo.txt\fP only.
.sp
Implementation note: this is implemented via very time\-efficient O(1)
hashtable lookups (this means you can have huge amounts of such patterns
without impacting performance much).
Due to that, this kind of pattern does not respect any context or order.
If you use such a pattern to include a file, it will always be included
(if the directory recursion encounters it).
Other include/exclude patterns that would normally match will be ignored.
Same logic applies for exclude.
.UNINDENT
.sp
\fBNOTE:\fP
.INDENT 0.0
.INDENT 3.5
\fIre:\fP, \fIsh:\fP and \fIfm:\fP patterns are all implemented on top of the Python SRE
engine. It is very easy to formulate patterns for each of these types which
requires an inordinate amount of time to match paths. If untrusted users
are able to supply patterns, ensure they cannot supply \fIre:\fP patterns.
Further, ensure that \fIsh:\fP and \fIfm:\fP patterns only contain a handful of
wildcards at most.
.UNINDENT
.UNINDENT
.sp
Exclusions can be passed via the command line option \fB\-\-exclude\fP\&. When used
from within a shell the patterns should be quoted to protect them from
expansion.
.sp
The \fB\-\-exclude\-from\fP option permits loading exclusion patterns from a text
file with one pattern per line. Lines empty or starting with the number sign
(\(aq#\(aq) after removing whitespace on both ends are ignored. The optional style
selector prefix is also supported for patterns loaded from a file. Due to
whitespace removal paths with whitespace at the beginning or end can only be
excluded using regular expressions.
.sp
Examples:
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
# Exclude \(aq/home/user/file.o\(aq but not \(aq/home/user/file.odt\(aq:
$ borg create \-e \(aq*.o\(aq backup /
# Exclude \(aq/home/user/junk\(aq and \(aq/home/user/subdir/junk\(aq but
# not \(aq/home/user/importantjunk\(aq or \(aq/etc/junk\(aq:
$ borg create \-e \(aq/home/*/junk\(aq backup /
# Exclude the contents of \(aq/home/user/cache\(aq but not the directory itself:
$ borg create \-e /home/user/cache/ backup /
# The file \(aq/home/user/cache/important\(aq is *not* backed up:
$ borg create \-e /home/user/cache/ backup / /home/user/cache/important
# The contents of directories in \(aq/home\(aq are not backed up when their name
# ends in \(aq.tmp\(aq
$ borg create \-\-exclude \(aqre:^/home/[^/]+\e.tmp/\(aq backup /
# Load exclusions from file
$ cat >exclude.txt <<EOF
# Comment line
/home/*/junk
*.tmp
fm:aa:something/*
re:^/home/[^/]\e.tmp/
sh:/home/*/.thumbnails
EOF
$ borg create \-\-exclude\-from exclude.txt backup /
.ft P
.fi
.UNINDENT
.UNINDENT
A more general and easier to use way to define filename matching patterns exists
with the experimental \fB\-\-pattern\fP and \fB\-\-patterns\-from\fP options. Using these, you
may specify the backup roots (starting points) and patterns for inclusion/exclusion.
A root path starts with the prefix \fIR\fP, followed by a path (a plain path, not a
file pattern). An include rule starts with the prefix +, an exclude rule starts
with the prefix \-, both followed by a pattern.
Inclusion patterns are useful to include paths that are contained in an excluded
path. The first matching pattern is used so if an include pattern matches before
an exclude pattern, the file is backed up.
.sp
Note that the default pattern style for \fB\-\-pattern\fP and \fB\-\-patterns\-from\fP is
shell style (\fIsh:\fP), so those patterns behave similar to rsync include/exclude
patterns. The pattern style can be set via the \fIP\fP prefix.
.sp
Patterns (\fB\-\-pattern\fP) and excludes (\fB\-\-exclude\fP) from the command line are
considered first (in the order of appearance). Then patterns from \fB\-\-patterns\-from\fP
are added. Exclusion patterns from \fB\-\-exclude\-from\fP files are appended last.
.sp
An example \fB\-\-patterns\-from\fP file could look like that:
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
# "sh:" pattern style is the default, so the following line is not needed:
P sh
R /
# can be rebuild
\- /home/*/.cache
# they\(aqre downloads for a reason
\- /home/*/Downloads
# susan is a nice person
# include susans home
+ /home/susan
# don\(aqt backup the other home directories
\- /home/*
.ft P
.fi
.UNINDENT
.UNINDENT
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

114
docs/man/borg-placeholders.1 Normal file
View File

@ -0,0 +1,114 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-PLACEHOLDERS 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-placeholders \- Details regarding placeholders
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH DESCRIPTION
.sp
Repository (or Archive) URLs, \fB\-\-prefix\fP and \fB\-\-remote\-path\fP values support these
placeholders:
.INDENT 0.0
.TP
.B {hostname}
The (short) hostname of the machine.
.TP
.B {fqdn}
The full name of the machine.
.TP
.B {now}
The current local date and time, by default in ISO\-8601 format.
You can also supply your own \fI\%format string\fP, e.g. {now:%Y\-%m\-%d_%H:%M:%S}
.TP
.B {utcnow}
The current UTC date and time, by default in ISO\-8601 format.
You can also supply your own \fI\%format string\fP, e.g. {utcnow:%Y\-%m\-%d_%H:%M:%S}
.TP
.B {user}
The user name (or UID, if no name is available) of the user running borg.
.TP
.B {pid}
The current process ID.
.TP
.B {borgversion}
The version of borg, e.g.: 1.0.8rc1
.TP
.B {borgmajor}
The version of borg, only the major version, e.g.: 1
.TP
.B {borgminor}
The version of borg, only major and minor version, e.g.: 1.0
.TP
.B {borgpatch}
The version of borg, only major, minor and patch version, e.g.: 1.0.8
.UNINDENT
.sp
If literal curly braces need to be used, double them for escaping:
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
borg create /path/to/repo::{{literal_text}}
.ft P
.fi
.UNINDENT
.UNINDENT
.sp
Examples:
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
borg create /path/to/repo::{hostname}\-{user}\-{utcnow} ...
borg create /path/to/repo::{hostname}\-{now:%Y\-%m\-%d_%H:%M:%S} ...
borg prune \-\-prefix \(aq{hostname}\-\(aq ...
.ft P
.fi
.UNINDENT
.UNINDENT
.sp
\fBNOTE:\fP
.INDENT 0.0
.INDENT 3.5
systemd uses a difficult, non\-standard syntax for command lines in unit files (refer to
the \fIsystemd.unit(5)\fP manual page).
.sp
When invoking borg from unit files, pay particular attention to escaping,
especially when using the now/utcnow placeholders, since systemd performs its own
%\-based variable replacement even in quoted text. To avoid interference from systemd,
double all percent signs (\fB{hostname}\-{now:%Y\-%m\-%d_%H:%M:%S}\fP
becomes \fB{hostname}\-{now:%%Y\-%%m\-%%d_%%H:%%M:%%S}\fP).
.UNINDENT
.UNINDENT
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

180
docs/man/borg-prune.1 Normal file
View File

@ -0,0 +1,180 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-PRUNE 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-prune \- Prune repository archives according to specified rules
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] prune <options> REPOSITORY
.SH DESCRIPTION
.sp
The prune command prunes a repository by deleting all archives not matching
any of the specified retention options. This command is normally used by
automated backup scripts wanting to keep a certain number of historic backups.
.sp
Also, prune automatically removes checkpoint archives (incomplete archives left
behind by interrupted backup runs) except if the checkpoint is the latest
archive (and thus still needed). Checkpoint archives are not considered when
comparing archive counts against the retention limits (\fB\-\-keep\-X\fP).
.sp
If a prefix is set with \-P, then only archives that start with the prefix are
considered for deletion and only those archives count towards the totals
specified by the rules.
Otherwise, \fIall\fP archives in the repository are candidates for deletion!
There is no automatic distinction between archives representing different
contents. These need to be distinguished by specifying matching prefixes.
.sp
If you have multiple sequences of archives with different data sets (e.g.
from different machines) in one shared repository, use one prune call per
data set that matches only the respective archives using the \-P option.
.sp
The \fB\-\-keep\-within\fP option takes an argument of the form "<int><char>",
where char is "H", "d", "w", "m", "y". For example, \fB\-\-keep\-within 2d\fP means
to keep all archives that were created within the past 48 hours.
"1m" is taken to mean "31d". The archives kept with this option do not
count towards the totals specified by any other options.
.sp
A good procedure is to thin out more and more the older your backups get.
As an example, \fB\-\-keep\-daily 7\fP means to keep the latest backup on each day,
up to 7 most recent days with backups (days without backups do not count).
The rules are applied from secondly to yearly, and backups selected by previous
rules do not count towards those of later rules. The time that each backup
starts is used for pruning purposes. Dates and times are interpreted in
the local timezone, and weeks go from Monday to Sunday. Specifying a
negative number of archives to keep means that there is no limit.
.sp
The \fB\-\-keep\-last N\fP option is doing the same as \fB\-\-keep\-secondly N\fP (and it will
keep the last N archives under the assumption that you do not create more than one
backup archive in the same second).
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.INDENT 0.0
.TP
.B REPOSITORY
repository to prune
.UNINDENT
.SS optional arguments
.INDENT 0.0
.TP
.B \-n\fP,\fB \-\-dry\-run
do not change repository
.TP
.B \-\-force
force pruning of corrupted archives
.TP
.B \-s\fP,\fB \-\-stats
print statistics for the deleted archive
.TP
.B \-\-list
output verbose list of archives it keeps/prunes
.TP
.BI \-\-keep\-within \ WITHIN
keep all archives within this time interval
.TP
.B \-\-keep\-last\fP,\fB \-\-keep\-secondly
number of secondly archives to keep
.TP
.B \-\-keep\-minutely
number of minutely archives to keep
.TP
.B \-H\fP,\fB \-\-keep\-hourly
number of hourly archives to keep
.TP
.B \-d\fP,\fB \-\-keep\-daily
number of daily archives to keep
.TP
.B \-w\fP,\fB \-\-keep\-weekly
number of weekly archives to keep
.TP
.B \-m\fP,\fB \-\-keep\-monthly
number of monthly archives to keep
.TP
.B \-y\fP,\fB \-\-keep\-yearly
number of yearly archives to keep
.TP
.B \-\-save\-space
work slower, but using less space
.UNINDENT
.SS filters
.INDENT 0.0
.TP
.B \-P\fP,\fB \-\-prefix
only consider archive names starting with this prefix.
.TP
.B \-a\fP,\fB \-\-glob\-archives
only consider archive names matching the glob. sh: rules apply, see "borg help patterns". \fB\-\-prefix\fP and \fB\-\-glob\-archives\fP are mutually exclusive.
.UNINDENT
.SH EXAMPLES
.sp
Be careful, prune is a potentially dangerous command, it will remove backup
archives.
.sp
The default of prune is to apply to \fBall archives in the repository\fP unless
you restrict its operation to a subset of the archives using \fB\-\-prefix\fP\&.
When using \fB\-\-prefix\fP, be careful to choose a good prefix \- e.g. do not use a
prefix "foo" if you do not also want to match "foobar".
.sp
It is strongly recommended to always run \fBprune \-v \-\-list \-\-dry\-run ...\fP
first so you will see what it would do without it actually doing anything.
.sp
There is also a visualized prune example in \fBdocs/misc/prune\-example.txt\fP\&.
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
# Keep 7 end of day and 4 additional end of week archives.
# Do a dry\-run without actually deleting anything.
$ borg prune \-v \-\-list \-\-dry\-run \-\-keep\-daily=7 \-\-keep\-weekly=4 /path/to/repo
# Same as above but only apply to archive names starting with the hostname
# of the machine followed by a "\-" character:
$ borg prune \-v \-\-list \-\-keep\-daily=7 \-\-keep\-weekly=4 \-\-prefix=\(aq{hostname}\-\(aq /path/to/repo
# Keep 7 end of day, 4 additional end of week archives,
# and an end of month archive for every month:
$ borg prune \-v \-\-list \-\-keep\-daily=7 \-\-keep\-weekly=4 \-\-keep\-monthly=\-1 /path/to/repo
# Keep all backups in the last 10 days, 4 additional end of week archives,
# and an end of month archive for every month:
$ borg prune \-v \-\-list \-\-keep\-within=10d \-\-keep\-weekly=4 \-\-keep\-monthly=\-1 /path/to/repo
.ft P
.fi
.UNINDENT
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

192
docs/man/borg-recreate.1 Normal file
View File

@ -0,0 +1,192 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-RECREATE 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-recreate \- Re-create archives
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] recreate <options> REPOSITORY_OR_ARCHIVE PATH
.SH DESCRIPTION
.sp
Recreate the contents of existing archives.
.sp
This is an \fIexperimental\fP feature. Do \fInot\fP use this on your only backup.
.sp
\fB\-\-exclude\fP, \fB\-\-exclude\-from\fP, \fB\-\-exclude\-if\-present\fP, \fB\-\-keep\-exclude\-tags\fP, and PATH
have the exact same semantics as in "borg create". If PATHs are specified the
resulting archive will only contain files from these PATHs.
.sp
Note that all paths in an archive are relative, therefore absolute patterns/paths
will \fInot\fP match (\fB\-\-exclude\fP, \fB\-\-exclude\-from\fP, PATHs).
.sp
\fB\-\-recompress\fP allows to change the compression of existing data in archives.
Due to how Borg stores compressed size information this might display
incorrect information for archives that were not recreated at the same time.
There is no risk of data loss by this.
.sp
\fB\-\-chunker\-params\fP will re\-chunk all files in the archive, this can be
used to have upgraded Borg 0.xx or Attic archives deduplicate with
Borg 1.x archives.
.sp
\fBUSE WITH CAUTION.\fP
Depending on the PATHs and patterns given, recreate can be used to permanently
delete files from archives.
When in doubt, use \fB\-\-dry\-run \-\-verbose \-\-list\fP to see how patterns/PATHS are
interpreted.
.sp
The archive being recreated is only removed after the operation completes. The
archive that is built during the operation exists at the same time at
"<ARCHIVE>.recreate". The new archive will have a different archive ID.
.sp
With \fB\-\-target\fP the original archive is not replaced, instead a new archive is created.
.sp
When rechunking space usage can be substantial, expect at least the entire
deduplicated size of the archives using the previous chunker params.
When recompressing expect approx. (throughput / checkpoint\-interval) in space usage,
assuming all chunks are recompressed.
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.INDENT 0.0
.TP
.B REPOSITORY_OR_ARCHIVE
repository/archive to recreate
.TP
.B PATH
paths to recreate; patterns are supported
.UNINDENT
.SS optional arguments
.INDENT 0.0
.TP
.B \-\-list
output verbose list of items (files, dirs, ...)
.TP
.BI \-\-filter \ STATUSCHARS
only display items with the given status characters
.TP
.B \-n\fP,\fB \-\-dry\-run
do not change anything
.TP
.B \-s\fP,\fB \-\-stats
print statistics at end
.UNINDENT
.SS Exclusion options
.INDENT 0.0
.TP
.BI \-e \ PATTERN\fP,\fB \ \-\-exclude \ PATTERN
exclude paths matching PATTERN
.TP
.BI \-\-exclude\-from \ EXCLUDEFILE
read exclude patterns from EXCLUDEFILE, one per line
.TP
.B \-\-exclude\-caches
exclude directories that contain a CACHEDIR.TAG file (\fI\%http://www.brynosaurus.com/cachedir/spec.html\fP)
.TP
.BI \-\-exclude\-if\-present \ NAME
exclude directories that are tagged by containing a filesystem object with the given NAME
.TP
.B \-\-keep\-exclude\-tags\fP,\fB \-\-keep\-tag\-files
if tag objects are specified with \fB\-\-exclude\-if\-present\fP, don\(aqt omit the tag objects themselves from the backup archive
.TP
.BI \-\-pattern \ PATTERN
experimental: include/exclude paths matching PATTERN
.TP
.BI \-\-patterns\-from \ PATTERNFILE
experimental: read include/exclude patterns from PATTERNFILE, one per line
.UNINDENT
.SS Archive options
.INDENT 0.0
.TP
.BI \-\-target \ TARGET
create a new archive with the name ARCHIVE, do not replace existing archive (only applies for a single archive)
.TP
.BI \-c \ SECONDS\fP,\fB \ \-\-checkpoint\-interval \ SECONDS
write checkpoint every SECONDS seconds (Default: 1800)
.TP
.BI \-\-comment \ COMMENT
add a comment text to the archive
.TP
.BI \-\-timestamp \ TIMESTAMP
manually specify the archive creation date/time (UTC, yyyy\-mm\-ddThh:mm:ss format). alternatively, give a reference file/directory.
.TP
.BI \-C \ COMPRESSION\fP,\fB \ \-\-compression \ COMPRESSION
select compression algorithm, see the output of the "borg help compression" command for details.
.TP
.B \-\-recompress
recompress data chunks according to \fB\-\-compression\fP if \fIif\-different\fP\&. When \fIalways\fP, chunks that are already compressed that way are not skipped, but compressed again. Only the algorithm is considered for \fIif\-different\fP, not the compression level (if any).
.TP
.BI \-\-chunker\-params \ PARAMS
specify the chunker parameters (CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE) or \fIdefault\fP to use the current defaults. default: 19,23,21,4095
.UNINDENT
.SH EXAMPLES
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
# Make old (Attic / Borg 0.xx) archives deduplicate with Borg 1.x archives
# Archives created with Borg 1.1+ and the default chunker params are skipped (archive ID stays the same)
$ borg recreate /mnt/backup \-\-chunker\-params default \-\-progress
# Create a backup with little but fast compression
$ borg create /mnt/backup::archive /some/files \-\-compression lz4
# Then compress it \- this might take longer, but the backup has already completed, so no inconsistencies
# from a long\-running backup job.
$ borg recreate /mnt/backup::archive \-\-recompress \-\-compression zlib,9
# Remove unwanted files from all archives in a repository
$ borg recreate /mnt/backup \-e /home/icke/Pictures/drunk_photos
# Change archive comment
$ borg create \-\-comment "This is a comment" /mnt/backup::archivename ~
$ borg info /mnt/backup::archivename
Name: archivename
Fingerprint: ...
Comment: This is a comment
\&...
$ borg recreate \-\-comment "This is a better comment" /mnt/backup::archivename
$ borg info /mnt/backup::archivename
Name: archivename
Fingerprint: ...
Comment: This is a better comment
\&...
.ft P
.fi
.UNINDENT
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP, \fIborg\-patterns(1)\fP, \fIborg\-placeholders(1)\fP, \fIborg\-compression(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

76
docs/man/borg-rename.1 Normal file
View File

@ -0,0 +1,76 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-RENAME 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-rename \- Rename an existing archive
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] rename <options> ARCHIVE NEWNAME
.SH DESCRIPTION
.sp
This command renames an archive in the repository.
.sp
This results in a different archive ID.
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.INDENT 0.0
.TP
.B ARCHIVE
archive to rename
.TP
.B NEWNAME
the new archive name to use
.UNINDENT
.SH EXAMPLES
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
$ borg create /path/to/repo::archivename ~
$ borg list /path/to/repo
archivename Mon, 2016\-02\-15 19:50:19
$ borg rename /path/to/repo::archivename newname
$ borg list /path/to/repo
newname Mon, 2016\-02\-15 19:50:19
.ft P
.fi
.UNINDENT
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

96
docs/man/borg-serve.1 Normal file
View File

@ -0,0 +1,96 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-SERVE 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-serve \- Start in server mode. This command is usually not used manually.
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] serve <options>
.SH DESCRIPTION
.sp
This command starts a repository server process. This command is usually not used manually.
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS optional arguments
.INDENT 0.0
.TP
.BI \-\-restrict\-to\-path \ PATH
restrict repository access to PATH. Can be specified multiple times to allow the client access to several directories. Access to all sub\-directories is granted implicitly; PATH doesn\(aqt need to directly point to a repository.
.TP
.BI \-\-restrict\-to\-repository \ PATH
restrict repository access. Only the repository located at PATH (no sub\-directories are considered) is accessible. Can be specified multiple times to allow the client access to several repositories. Unlike \-\-restrict\-to\-path sub\-directories are not accessible; PATH needs to directly point at a repository location. PATH may be an empty directory or the last element of PATH may not exist, in which case the client may initialize a repository there.
.TP
.B \-\-append\-only
only allow appending to repository segment files
.TP
.B \-\-storage\-quota
Override storage quota of the repository (e.g. 5G, 1.5T). When a new repository is initialized, sets the storage quota on the new repository as well. Default: no quota.
.UNINDENT
.SH EXAMPLES
.sp
borg serve has special support for ssh forced commands (see \fBauthorized_keys\fP
example below): it will detect that you use such a forced command and extract
the value of the \fB\-\-restrict\-to\-path\fP option(s).
.sp
It will then parse the original command that came from the client, makes sure
that it is also \fBborg serve\fP and enforce path restriction(s) as given by the
forced command. That way, other options given by the client (like \fB\-\-info\fP or
\fB\-\-umask\fP) are preserved (and are not fixed by the forced command).
.sp
Environment variables (such as BORG_HOSTNAME_IS_UNIQUE) contained in the original
command sent by the client are \fInot\fP interpreted, but ignored. If BORG_XXX environment
variables should be set on the \fBborg serve\fP side, then these must be set in system\-specific
locations like \fB/etc/environment\fP or in the forced command itself (example below).
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
# Allow an SSH keypair to only run borg, and only have access to /path/to/repo.
# Use key options to disable unneeded and potentially dangerous SSH functionality.
# This will help to secure an automated remote backup system.
$ cat ~/.ssh/authorized_keys
command="borg serve \-\-restrict\-to\-path /path/to/repo",no\-pty,no\-agent\-forwarding,no\-port\-forwarding,no\-X11\-forwarding,no\-user\-rc ssh\-rsa AAAAB3[...]
# Set a BORG_XXX environment variable on the "borg serve" side
$ cat ~/.ssh/authorized_keys
command="export BORG_XXX=value; borg serve [...]",restrict ssh\-rsa [...]
.ft P
.fi
.UNINDENT
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

115
docs/man/borg-umount.1 Normal file
View File

@ -0,0 +1,115 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-UMOUNT 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-umount \- un-mount the FUSE filesystem
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] umount <options> MOUNTPOINT
.SH DESCRIPTION
.sp
This command un\-mounts a FUSE filesystem that was mounted with \fBborg mount\fP\&.
.sp
This is a convenience wrapper that just calls the platform\-specific shell
command \- usually this is either umount or fusermount \-u.
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.INDENT 0.0
.TP
.B MOUNTPOINT
mountpoint of the filesystem to umount
.UNINDENT
.SH EXAMPLES
.SS borg mount
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
$ borg mount /path/to/repo::root\-2016\-02\-15 /tmp/mymountpoint
$ ls /tmp/mymountpoint
bin boot etc home lib lib64 lost+found media mnt opt root sbin srv tmp usr var
$ borg umount /tmp/mymountpoint
.ft P
.fi
.UNINDENT
.UNINDENT
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
$ borg mount \-o versions /path/to/repo /tmp/mymountpoint
$ ls \-l /tmp/mymountpoint/home/user/doc.txt/
total 24
\-rw\-rw\-r\-\- 1 user group 12357 Aug 26 21:19 doc.txt.cda00bc9
\-rw\-rw\-r\-\- 1 user group 12204 Aug 26 21:04 doc.txt.fa760f28
$ fusermount \-u /tmp/mymountpoint
.ft P
.fi
.UNINDENT
.UNINDENT
.SS borgfs
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
$ echo \(aq/mnt/backup /tmp/myrepo fuse.borgfs defaults,noauto 0 0\(aq >> /etc/fstab
$ echo \(aq/mnt/backup::root\-2016\-02\-15 /tmp/myarchive fuse.borgfs defaults,noauto 0 0\(aq >> /etc/fstab
$ mount /tmp/myrepo
$ mount /tmp/myarchive
$ ls /tmp/myrepo
root\-2016\-02\-01 root\-2016\-02\-2015
$ ls /tmp/myarchive
bin boot etc home lib lib64 lost+found media mnt opt root sbin srv tmp usr var
.ft P
.fi
.UNINDENT
.UNINDENT
.sp
\fBNOTE:\fP
.INDENT 0.0
.INDENT 3.5
\fBborgfs\fP will be automatically provided if you used a distribution
package, \fBpip\fP or \fBsetup.py\fP to install Borg. Users of the
standalone binary will have to manually create a symlink (see
\fIpyinstaller\-binary\fP).
.UNINDENT
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP, \fIborg\-mount(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

187
docs/man/borg-upgrade.1 Normal file
View File

@ -0,0 +1,187 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-UPGRADE 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-upgrade \- upgrade a repository from a previous version
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] upgrade <options> REPOSITORY
.SH DESCRIPTION
.sp
Upgrade an existing, local Borg repository.
.SS When you do not need borg upgrade
.sp
Not every change requires that you run \fBborg upgrade\fP\&.
.sp
You do \fBnot\fP need to run it when:
.INDENT 0.0
.IP \(bu 2
moving your repository to a different place
.IP \(bu 2
upgrading to another point release (like 1.0.x to 1.0.y),
except when noted otherwise in the changelog
.IP \(bu 2
upgrading from 1.0.x to 1.1.x,
except when noted otherwise in the changelog
.UNINDENT
.SS Borg 1.x.y upgrades
.sp
Use \fBborg upgrade \-\-tam REPO\fP to require manifest authentication
introduced with Borg 1.0.9 to address security issues. This means
that modifying the repository after doing this with a version prior
to 1.0.9 will raise a validation error, so only perform this upgrade
after updating all clients using the repository to 1.0.9 or newer.
.sp
This upgrade should be done on each client for safety reasons.
.sp
If a repository is accidentally modified with a pre\-1.0.9 client after
this upgrade, use \fBborg upgrade \-\-tam \-\-force REPO\fP to remedy it.
.sp
If you routinely do this you might not want to enable this upgrade
(which will leave you exposed to the security issue). You can
reverse the upgrade by issuing \fBborg upgrade \-\-disable\-tam REPO\fP\&.
.sp
See
\fI\%https://borgbackup.readthedocs.io/en/stable/changes.html#pre\-1\-0\-9\-manifest\-spoofing\-vulnerability\fP
for details.
.SS Attic and Borg 0.xx to Borg 1.x
.sp
This currently supports converting an Attic repository to Borg and also
helps with converting Borg 0.xx to 1.0.
.sp
Currently, only LOCAL repositories can be upgraded (issue #465).
.sp
Please note that \fBborg create\fP (since 1.0.0) uses bigger chunks by
default than old borg or attic did, so the new chunks won\(aqt deduplicate
with the old chunks in the upgraded repository.
See \fB\-\-chunker\-params\fP option of \fBborg create\fP and \fBborg recreate\fP\&.
.sp
\fBborg upgrade\fP will change the magic strings in the repository\(aqs
segments to match the new Borg magic strings. The keyfiles found in
$ATTIC_KEYS_DIR or ~/.attic/keys/ will also be converted and
copied to $BORG_KEYS_DIR or ~/.config/borg/keys.
.sp
The cache files are converted, from $ATTIC_CACHE_DIR or
~/.cache/attic to $BORG_CACHE_DIR or ~/.cache/borg, but the
cache layout between Borg and Attic changed, so it is possible
the first backup after the conversion takes longer than expected
due to the cache resync.
.sp
Upgrade should be able to resume if interrupted, although it
will still iterate over all segments. If you want to start
from scratch, use \fIborg delete\fP over the copied repository to
make sure the cache files are also removed:
.INDENT 0.0
.INDENT 3.5
borg delete borg
.UNINDENT
.UNINDENT
.sp
Unless \fB\-\-inplace\fP is specified, the upgrade process first
creates a backup copy of the repository, in
REPOSITORY.upgrade\-DATETIME, using hardlinks. This takes
longer than in place upgrades, but is much safer and gives
progress information (as opposed to \fBcp \-al\fP). Once you are
satisfied with the conversion, you can safely destroy the
backup copy.
.sp
WARNING: Running the upgrade in place will make the current
copy unusable with older version, with no way of going back
to previous versions. This can PERMANENTLY DAMAGE YOUR
REPOSITORY! Attic CAN NOT READ BORG REPOSITORIES, as the
magic strings have changed. You have been warned.
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.INDENT 0.0
.TP
.B REPOSITORY
path to the repository to be upgraded
.UNINDENT
.SS optional arguments
.INDENT 0.0
.TP
.B \-n\fP,\fB \-\-dry\-run
do not change repository
.TP
.B \-\-inplace
rewrite repository in place, with no chance of going back to older
versions of the repository.
.TP
.B \-\-force
Force upgrade
.TP
.B \-\-tam
Enable manifest authentication (in key and cache) (Borg 1.0.9 and later)
.TP
.B \-\-disable\-tam
Disable manifest authentication (in key and cache)
.UNINDENT
.SH EXAMPLES
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
# Upgrade the borg repository to the most recent version.
$ borg upgrade \-v /path/to/repo
making a hardlink copy in /path/to/repo.upgrade\-2016\-02\-15\-20:51:55
opening attic repository with borg and converting
no key file found for repository
converting repo index /path/to/repo/index.0
converting 1 segments...
converting borg 0.xx to borg current
no key file found for repository
.ft P
.fi
.UNINDENT
.UNINDENT
.SS Upgrading a passphrase encrypted attic repo
.sp
attic offered a "passphrase" encryption mode, but this was removed in borg 1.0
and replaced by the "repokey" mode (which stores the passphrase\-protected
encryption key into the repository config).
.sp
Thus, to upgrade a "passphrase" attic repo to a "repokey" borg repo, 2 steps
are needed, in this order:
.INDENT 0.0
.IP \(bu 2
borg upgrade repo
.IP \(bu 2
borg key migrate\-to\-repokey repo
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

71
docs/man/borg-with-lock.1 Normal file
View File

@ -0,0 +1,71 @@
.\" Man page generated from reStructuredText.
.
.TH BORG-WITH-LOCK 1 "2017-06-18" "" "borg backup tool"
.SH NAME
borg-with-lock \- run a user specified command with the repository lock held
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] with\-lock <options> REPOSITORY COMMAND ARGS
.SH DESCRIPTION
.sp
This command runs a user\-specified command while the repository lock is held.
.sp
It will first try to acquire the lock (make sure that no other operation is
running in the repo), then execute the given command as a subprocess and wait
for its termination, release the lock and return the user command\(aqs return
code as borg\(aqs return code.
.INDENT 0.0
.TP
.B Note: if you copy a repository with the lock held, the lock will be present in
the copy, obviously. Thus, before using borg on the copy, you need to
use "borg break\-lock" on it.
.UNINDENT
.SH OPTIONS
.sp
See \fIborg\-common(1)\fP for common options of Borg commands.
.SS arguments
.INDENT 0.0
.TP
.B REPOSITORY
repository to lock
.TP
.B COMMAND
command to run
.TP
.B ARGS
command arguments
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

603
docs/man/borg.1 Normal file
View File

@ -0,0 +1,603 @@
.\" Man page generated from reStructuredText.
.
.TH BORG 1 "2017-02-05" "" "borg backup tool"
.SH NAME
borg \- deduplicating and encrypting backup tool
.
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.SH SYNOPSIS
.sp
borg [common options] <command> [options] [arguments]
.SH DESCRIPTION
.\" we don't include the README.rst here since we want to keep this terse.
.
.sp
BorgBackup (short: Borg) is a deduplicating backup program.
Optionally, it supports compression and authenticated encryption.
.sp
The main goal of Borg is to provide an efficient and secure way to backup data.
The data deduplication technique used makes Borg suitable for daily backups
since only changes are stored.
The authenticated encryption technique makes it suitable for backups to not
fully trusted targets.
.sp
Borg stores a set of files in an \fIarchive\fP\&. A \fIrepository\fP is a collection
of \fIarchives\fP\&. The format of repositories is Borg\-specific. Borg does not
distinguish archives from each other in a any way other than their name,
it does not matter when or where archives where created (eg. different hosts).
.SH EXAMPLES
.SS A step\-by\-step example
.INDENT 0.0
.IP 1. 3
Before a backup can be made a repository has to be initialized:
.INDENT 3.0
.INDENT 3.5
.sp
.nf
.ft C
$ borg init \-\-encryption=repokey /path/to/repo
.ft P
.fi
.UNINDENT
.UNINDENT
.IP 2. 3
Backup the \fB~/src\fP and \fB~/Documents\fP directories into an archive called
\fIMonday\fP:
.INDENT 3.0
.INDENT 3.5
.sp
.nf
.ft C
$ borg create /path/to/repo::Monday ~/src ~/Documents
.ft P
.fi
.UNINDENT
.UNINDENT
.IP 3. 3
The next day create a new archive called \fITuesday\fP:
.INDENT 3.0
.INDENT 3.5
.sp
.nf
.ft C
$ borg create \-\-stats /path/to/repo::Tuesday ~/src ~/Documents
.ft P
.fi
.UNINDENT
.UNINDENT
.sp
This backup will be a lot quicker and a lot smaller since only new never
before seen data is stored. The \fB\-\-stats\fP option causes Borg to
output statistics about the newly created archive such as the amount of unique
data (not shared with other archives):
.INDENT 3.0
.INDENT 3.5
.sp
.nf
.ft C
\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-
Archive name: Tuesday
Archive fingerprint: bd31004d58f51ea06ff735d2e5ac49376901b21d58035f8fb05dbf866566e3c2
Time (start): Tue, 2016\-02\-16 18:15:11
Time (end): Tue, 2016\-02\-16 18:15:11
Duration: 0.19 seconds
Number of files: 127
\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-
Original size Compressed size Deduplicated size
This archive: 4.16 MB 4.17 MB 26.78 kB
All archives: 8.33 MB 8.34 MB 4.19 MB
Unique chunks Total chunks
Chunk index: 132 261
\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-
.ft P
.fi
.UNINDENT
.UNINDENT
.IP 4. 3
List all archives in the repository:
.INDENT 3.0
.INDENT 3.5
.sp
.nf
.ft C
$ borg list /path/to/repo
Monday Mon, 2016\-02\-15 19:14:44
Tuesday Tue, 2016\-02\-16 19:15:11
.ft P
.fi
.UNINDENT
.UNINDENT
.IP 5. 3
List the contents of the \fIMonday\fP archive:
.INDENT 3.0
.INDENT 3.5
.sp
.nf
.ft C
$ borg list /path/to/repo::Monday
drwxr\-xr\-x user group 0 Mon, 2016\-02\-15 18:22:30 home/user/Documents
\-rw\-r\-\-r\-\- user group 7961 Mon, 2016\-02\-15 18:22:30 home/user/Documents/Important.doc
\&...
.ft P
.fi
.UNINDENT
.UNINDENT
.IP 6. 3
Restore the \fIMonday\fP archive by extracting the files relative to the current directory:
.INDENT 3.0
.INDENT 3.5
.sp
.nf
.ft C
$ borg extract /path/to/repo::Monday
.ft P
.fi
.UNINDENT
.UNINDENT
.IP 7. 3
Recover disk space by manually deleting the \fIMonday\fP archive:
.INDENT 3.0
.INDENT 3.5
.sp
.nf
.ft C
$ borg delete /path/to/repo::Monday
.ft P
.fi
.UNINDENT
.UNINDENT
.UNINDENT
.sp
\fBNOTE:\fP
.INDENT 0.0
.INDENT 3.5
Borg is quiet by default (it works on WARNING log level).
You can use options like \fB\-\-progress\fP or \fB\-\-list\fP to get specific
reports during command execution. You can also add the \fB\-v\fP (or
\fB\-\-verbose\fP or \fB\-\-info\fP) option to adjust the log level to INFO to
get other informational messages.
.UNINDENT
.UNINDENT
.SH NOTES
.SS Repository URLs
.sp
\fBLocal filesystem\fP (or locally mounted network filesystem):
.sp
\fB/path/to/repo\fP \- filesystem path to repo directory, absolute path
.sp
\fBpath/to/repo\fP \- filesystem path to repo directory, relative path
.sp
Also, stuff like \fB~/path/to/repo\fP or \fB~other/path/to/repo\fP works (this is
expanded by your shell).
.sp
Note: you may also prepend a \fBfile://\fP to a filesystem path to get URL style.
.sp
\fBRemote repositories\fP accessed via ssh \fI\%user@host\fP:
.sp
\fBuser@host:/path/to/repo\fP \- remote repo, absolute path
.sp
\fBssh://user@host:port/path/to/repo\fP \- same, alternative syntax, port can be given
.sp
\fBRemote repositories with relative paths\fP can be given using this syntax:
.sp
\fBuser@host:path/to/repo\fP \- path relative to current directory
.sp
\fBuser@host:~/path/to/repo\fP \- path relative to user\(aqs home directory
.sp
\fBuser@host:~other/path/to/repo\fP \- path relative to other\(aqs home directory
.sp
Note: giving \fBuser@host:/./path/to/repo\fP or \fBuser@host:/~/path/to/repo\fP or
\fBuser@host:/~other/path/to/repo\fP is also supported, but not required here.
.sp
\fBRemote repositories with relative paths, alternative syntax with port\fP:
.sp
\fBssh://user@host:port/./path/to/repo\fP \- path relative to current directory
.sp
\fBssh://user@host:port/~/path/to/repo\fP \- path relative to user\(aqs home directory
.sp
\fBssh://user@host:port/~other/path/to/repo\fP \- path relative to other\(aqs home directory
.sp
If you frequently need the same repo URL, it is a good idea to set the
\fBBORG_REPO\fP environment variable to set a default for the repo URL:
.INDENT 0.0
.INDENT 3.5
.sp
.nf
.ft C
export BORG_REPO=\(aqssh://user@host:port/path/to/repo\(aq
.ft P
.fi
.UNINDENT
.UNINDENT
.sp
Then just leave away the repo URL if only a repo URL is needed and you want
to use the default \- it will be read from BORG_REPO then.
.sp
Use \fB::\fP syntax to give the repo URL when syntax requires giving a positional
argument for the repo (e.g. \fBborg mount :: /mnt\fP).
.SS Repository / Archive Locations
.sp
Many commands want either a repository (just give the repo URL, see above) or
an archive location, which is a repo URL followed by \fB::archive_name\fP\&.
.sp
Archive names must not contain the \fB/\fP (slash) character. For simplicity,
maybe also avoid blanks or other characters that have special meaning on the
shell or in a filesystem (borg mount will use the archive name as directory
name).
.sp
If you have set BORG_REPO (see above) and an archive location is needed, use
\fB::archive_name\fP \- the repo URL part is then read from BORG_REPO.
.SS Type of log output
.sp
The log level of the builtin logging configuration defaults to WARNING.
This is because we want Borg to be mostly silent and only output
warnings, errors and critical messages, unless output has been requested
by supplying an option that implies output (e.g. \fB\-\-list\fP or \fB\-\-progress\fP).
.sp
Log levels: DEBUG < INFO < WARNING < ERROR < CRITICAL
.sp
Use \fB\-\-debug\fP to set DEBUG log level \-
to get debug, info, warning, error and critical level output.
.sp
Use \fB\-\-info\fP (or \fB\-v\fP or \fB\-\-verbose\fP) to set INFO log level \-
to get info, warning, error and critical level output.
.sp
Use \fB\-\-warning\fP (default) to set WARNING log level \-
to get warning, error and critical level output.
.sp
Use \fB\-\-error\fP to set ERROR log level \-
to get error and critical level output.
.sp
Use \fB\-\-critical\fP to set CRITICAL log level \-
to get critical level output.
.sp
While you can set misc. log levels, do not expect that every command will
give different output on different log levels \- it\(aqs just a possibility.
.sp
\fBWARNING:\fP
.INDENT 0.0
.INDENT 3.5
Options \fB\-\-critical\fP and \fB\-\-error\fP are provided for completeness,
their usage is not recommended as you might miss important information.
.UNINDENT
.UNINDENT
.SS Return codes
.sp
Borg can exit with the following return codes (rc):
.TS
center;
|l|l|.
_
T{
Return code
T} T{
Meaning
T}
_
T{
0
T} T{
success (logged as INFO)
T}
_
T{
1
T} T{
warning (operation reached its normal end, but there were warnings \-\-
you should check the log, logged as WARNING)
T}
_
T{
2
T} T{
error (like a fatal error, a local or remote exception, the operation
did not reach its normal end, logged as ERROR)
T}
_
T{
128+N
T} T{
killed by signal N (e.g. 137 == kill \-9)
T}
_
.TE
.sp
If you use \fB\-\-show\-rc\fP, the return code is also logged at the indicated
level as the last log entry.
.SS Environment Variables
.sp
Borg uses some environment variables for automation:
.INDENT 0.0
.TP
.B General:
.INDENT 7.0
.TP
.B BORG_REPO
When set, use the value to give the default repository location. If a command needs an archive
parameter, you can abbreviate as \fB::archive\fP\&. If a command needs a repository parameter, you
can either leave it away or abbreviate as \fB::\fP, if a positional parameter is required.
.TP
.B BORG_PASSPHRASE
When set, use the value to answer the passphrase question for encrypted repositories.
It is used when a passphrase is needed to access an encrypted repo as well as when a new
passphrase should be initially set when initializing an encrypted repo.
See also BORG_NEW_PASSPHRASE.
.TP
.B BORG_PASSCOMMAND
When set, use the standard output of the command (trailing newlines are stripped) to answer the
passphrase question for encrypted repositories.
It is used when a passphrase is needed to access an encrypted repo as well as when a new
passphrase should be initially set when initializing an encrypted repo.
If BORG_PASSPHRASE is also set, it takes precedence.
See also BORG_NEW_PASSPHRASE.
.TP
.B BORG_NEW_PASSPHRASE
When set, use the value to answer the passphrase question when a \fBnew\fP passphrase is asked for.
This variable is checked first. If it is not set, BORG_PASSPHRASE and BORG_PASSCOMMAND will also
be checked.
Main usecase for this is to fully automate \fBborg change\-passphrase\fP\&.
.TP
.B BORG_DISPLAY_PASSPHRASE
When set, use the value to answer the "display the passphrase for verification" question when defining a new passphrase for encrypted repositories.
.TP
.B BORG_HOSTNAME_IS_UNIQUE=no
Borg assumes that it can derive a unique hostname / identity (see \fBborg debug info\fP).
If this is not the case or you do not want Borg to automatically remove stale locks,
set this to \fIno\fP\&.
.TP
.B BORG_LOGGING_CONF
When set, use the given filename as \fI\%INI\fP\-style logging configuration.
.TP
.B BORG_RSH
When set, use this command instead of \fBssh\fP\&. This can be used to specify ssh options, such as
a custom identity file \fBssh \-i /path/to/private/key\fP\&. See \fBman ssh\fP for other options.
.TP
.B BORG_REMOTE_PATH
When set, use the given path as borg executable on the remote (defaults to "borg" if unset).
Using \fB\-\-remote\-path PATH\fP commandline option overrides the environment variable.
.TP
.B BORG_FILES_CACHE_TTL
When set to a numeric value, this determines the maximum "time to live" for the files cache
entries (default: 20). The files cache is used to quickly determine whether a file is unchanged.
The FAQ explains this more detailed in: \fIalways_chunking\fP
.TP
.B TMPDIR
where temporary files are stored (might need a lot of temporary space for some operations)
.UNINDENT
.TP
.B Some automatic "answerers" (if set, they automatically answer confirmation questions):
.INDENT 7.0
.TP
.B BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no (or =yes)
For "Warning: Attempting to access a previously unknown unencrypted repository"
.TP
.B BORG_RELOCATED_REPO_ACCESS_IS_OK=no (or =yes)
For "Warning: The repository at location ... was previously located at ..."
.TP
.B BORG_CHECK_I_KNOW_WHAT_I_AM_DOING=NO (or =YES)
For "Warning: \(aqcheck \-\-repair\(aq is an experimental feature that might result in data loss."
.TP
.B BORG_DELETE_I_KNOW_WHAT_I_AM_DOING=NO (or =YES)
For "You requested to completely DELETE the repository \fIincluding\fP all archives it contains:"
.TP
.B BORG_RECREATE_I_KNOW_WHAT_I_AM_DOING=NO (or =YES)
For "recreate is an experimental feature."
.UNINDENT
.sp
Note: answers are case sensitive. setting an invalid answer value might either give the default
answer or ask you interactively, depending on whether retries are allowed (they by default are
allowed). So please test your scripts interactively before making them a non\-interactive script.
.TP
.B Directories and files:
.INDENT 7.0
.TP
.B BORG_KEYS_DIR
Default to \(aq~/.config/borg/keys\(aq. This directory contains keys for encrypted repositories.
.TP
.B BORG_KEY_FILE
When set, use the given filename as repository key file.
.TP
.B BORG_SECURITY_DIR
Default to \(aq~/.config/borg/security\(aq. This directory contains information borg uses to
track its usage of NONCES ("numbers used once" \- usually in encryption context) and other
security relevant data.
.TP
.B BORG_CACHE_DIR
Default to \(aq~/.cache/borg\(aq. This directory contains the local cache and might need a lot
of space for dealing with big repositories).
.UNINDENT
.TP
.B Building:
.INDENT 7.0
.TP
.B BORG_OPENSSL_PREFIX
Adds given OpenSSL header file directory to the default locations (setup.py).
.TP
.B BORG_LZ4_PREFIX
Adds given LZ4 header file directory to the default locations (setup.py).
.TP
.B BORG_LIBB2_PREFIX
Adds given prefix directory to the default locations. If a \(aqinclude/blake2.h\(aq is found Borg
will be linked against the system libb2 instead of a bundled implementation. (setup.py)
.UNINDENT
.UNINDENT
.sp
Please note:
.INDENT 0.0
.IP \(bu 2
be very careful when using the "yes" sayers, the warnings with prompt exist for your / your data\(aqs security/safety
.IP \(bu 2
also be very careful when putting your passphrase into a script, make sure it has appropriate file permissions
(e.g. mode 600, root:root).
.UNINDENT
.SS File systems
.sp
We strongly recommend against using Borg (or any other database\-like
software) on non\-journaling file systems like FAT, since it is not
possible to assume any consistency in case of power failures (or a
sudden disconnect of an external drive or similar failures).
.sp
While Borg uses a data store that is resilient against these failures
when used on journaling file systems, it is not possible to guarantee
this with some hardware \-\- independent of the software used. We don\(aqt
know a list of affected hardware.
.sp
If you are suspicious whether your Borg repository is still consistent
and readable after one of the failures mentioned above occurred, run
\fBborg check \-\-verify\-data\fP to make sure it is consistent.
.SS Units
.sp
To display quantities, Borg takes care of respecting the
usual conventions of scale. Disk sizes are displayed in \fI\%decimal\fP, using powers of ten (so
\fBkB\fP means 1000 bytes). For memory usage, \fI\%binary prefixes\fP are used, and are
indicated using the \fI\%IEC binary prefixes\fP,
using powers of two (so \fBKiB\fP means 1024 bytes).
.SS Date and Time
.sp
We format date and time conforming to ISO\-8601, that is: YYYY\-MM\-DD and
HH:MM:SS (24h clock).
.sp
For more information about that, see: \fI\%https://xkcd.com/1179/\fP
.sp
Unless otherwise noted, we display local date and time.
Internally, we store and process date and time as UTC.
.SS Resource Usage
.sp
Borg might use a lot of resources depending on the size of the data set it is dealing with.
.sp
If one uses Borg in a client/server way (with a ssh: repository),
the resource usage occurs in part on the client and in another part on the
server.
.sp
If one uses Borg as a single process (with a filesystem repo),
all the resource usage occurs in that one process, so just add up client +
server to get the approximate resource usage.
.INDENT 0.0
.TP
.B CPU client:
borg create: does chunking, hashing, compression, crypto (high CPU usage)
chunks cache sync: quite heavy on CPU, doing lots of hashtable operations.
borg extract: crypto, decompression (medium to high CPU usage)
borg check: similar to extract, but depends on options given.
borg prune / borg delete archive: low to medium CPU usage
borg delete repo: done on the server
It won\(aqt go beyond 100% of 1 core as the code is currently single\-threaded.
Especially higher zlib and lzma compression levels use significant amounts
of CPU cycles. Crypto might be cheap on the CPU (if hardware accelerated) or
expensive (if not).
.TP
.B CPU server:
It usually doesn\(aqt need much CPU, it just deals with the key/value store
(repository) and uses the repository index for that.
.sp
borg check: the repository check computes the checksums of all chunks
(medium CPU usage)
borg delete repo: low CPU usage
.TP
.B CPU (only for client/server operation):
When using borg in a client/server way with a \fI\%ssh:\-type\fP repo, the ssh
processes used for the transport layer will need some CPU on the client and
on the server due to the crypto they are doing \- esp. if you are pumping
big amounts of data.
.TP
.B Memory (RAM) client:
The chunks index and the files index are read into memory for performance
reasons. Might need big amounts of memory (see below).
Compression, esp. lzma compression with high levels might need substantial
amounts of memory.
.TP
.B Memory (RAM) server:
The server process will load the repository index into memory. Might need
considerable amounts of memory, but less than on the client (see below).
.TP
.B Chunks index (client only):
Proportional to the amount of data chunks in your repo. Lots of chunks
in your repo imply a big chunks index.
It is possible to tweak the chunker params (see create options).
.TP
.B Files index (client only):
Proportional to the amount of files in your last backups. Can be switched
off (see create options), but next backup might be much slower if you do.
The speed benefit of using the files cache is proportional to file size.
.TP
.B Repository index (server only):
Proportional to the amount of data chunks in your repo. Lots of chunks
in your repo imply a big repository index.
It is possible to tweak the chunker params (see create options) to
influence the amount of chunks being created.
.TP
.B Temporary files (client):
Reading data and metadata from a FUSE mounted repository will consume up to
the size of all deduplicated, small chunks in the repository. Big chunks
won\(aqt be locally cached.
.TP
.B Temporary files (server):
None.
.TP
.B Cache files (client only):
Contains the chunks index and files index (plus a collection of single\-
archive chunk indexes which might need huge amounts of disk space,
depending on archive count and size \- see FAQ about how to reduce).
.TP
.B Network (only for client/server operation):
If your repository is remote, all deduplicated (and optionally compressed/
encrypted) data of course has to go over the connection (\fBssh://\fP repo url).
If you use a locally mounted network filesystem, additionally some copy
operations used for transaction support also go over the connection. If
you backup multiple sources to one target repository, additional traffic
happens for cache resynchronization.
.UNINDENT
.SH SEE ALSO
.sp
\fIborg\-common(1)\fP for common command line options
.sp
\fIborg\-init(1)\fP,
\fIborg\-create(1)\fP, \fIborg\-mount(1)\fP, \fIborg\-extract(1)\fP,
\fIborg\-list(1)\fP, \fIborg\-info(1)\fP,
\fIborg\-delete(1)\fP, \fIborg\-prune(1)\fP,
\fIborg\-recreate(1)\fP
.sp
\fIborg\-compression(1)\fP, \fIborg\-patterns(1)\fP, \fIborg\-placeholders(1)\fP
.INDENT 0.0
.IP \(bu 2
Main web site \fI\%https://borgbackup.readthedocs.org/\fP
.IP \(bu 2
Releases \fI\%https://github.com/borgbackup/borg/releases\fP
.IP \(bu 2
Changelog \fI\%https://github.com/borgbackup/borg/blob/master/docs/changes.rst\fP
.IP \(bu 2
GitHub \fI\%https://github.com/borgbackup/borg\fP
.IP \(bu 2
Security contact \fI\%https://borgbackup.readthedocs.io/en/latest/support.html#security\-contact\fP
.UNINDENT
.SH AUTHOR
The Borg Collective
.\" Generated by docutils manpage writer.
.

68
docs/man_intro.rst Normal file
View File

@ -0,0 +1,68 @@
====
borg
====
----------------------------------------
deduplicating and encrypting backup tool
----------------------------------------
:Author: The Borg Collective
:Date: 2017-02-05
:Manual section: 1
:Manual group: borg backup tool
SYNOPSIS
--------
borg [common options] <command> [options] [arguments]
DESCRIPTION
-----------
.. we don't include the README.rst here since we want to keep this terse.
BorgBackup (short: Borg) is a deduplicating backup program.
Optionally, it supports compression and authenticated encryption.
The main goal of Borg is to provide an efficient and secure way to backup data.
The data deduplication technique used makes Borg suitable for daily backups
since only changes are stored.
The authenticated encryption technique makes it suitable for backups to not
fully trusted targets.
Borg stores a set of files in an *archive*. A *repository* is a collection
of *archives*. The format of repositories is Borg-specific. Borg does not
distinguish archives from each other in any way other than their name,
it does not matter when or where archives were created (e.g. different hosts).
EXAMPLES
--------
A step-by-step example
~~~~~~~~~~~~~~~~~~~~~~
.. include:: quickstart_example.rst.inc
NOTES
-----
.. include:: usage_general.rst.inc
SEE ALSO
--------
`borg-common(1)` for common command line options
`borg-init(1)`,
`borg-create(1)`, `borg-mount(1)`, `borg-extract(1)`,
`borg-list(1)`, `borg-info(1)`,
`borg-delete(1)`, `borg-prune(1)`,
`borg-recreate(1)`
`borg-compression(1)`, `borg-patterns(1)`, `borg-placeholders(1)`
* Main web site https://www.borgbackup.org/
* Releases https://github.com/borgbackup/borg/releases
* Changelog https://github.com/borgbackup/borg/blob/master/docs/changes.rst
* GitHub https://github.com/borgbackup/borg
* Security contact https://borgbackup.readthedocs.io/en/latest/support.html#security-contact

7733
docs/misc/asciinema/advanced.json Normal file
View File

File diff suppressed because it is too large Load Diff

65
docs/misc/asciinema/advanced.sh Normal file
View File

@ -0,0 +1,65 @@
# For the pro users, here are some advanced features of borg, so you can impress your friends. ;)
# Note: This screencast was made with borg version 1.1.0 – older or newer borg versions may behave differently.
# First of all, we can use several environment variables for borg.
# E.g. we do not want to type in our repo path and password again and again…
export BORG_REPO='/media/backup/borgdemo'
export BORG_PASSPHRASE='1234'
# Problem solved, borg will use this automatically… :)
# We'll use this right away…
## ADVANCED CREATION ##
# We can also use some placeholders in our archive name…
borg create --stats --progress --compression lz4 ::{user}-{now} Wallpaper
# Notice the backup name.
# And we can put completely different data, with different backup settings, in our backup. It will be deduplicated, anyway:
borg create --stats --progress --compression zlib,6 --exclude ~/Downloads/big ::{user}-{now} ~/Downloads
# Or let's backup a device via STDIN.
sudo dd if=/dev/loop0 bs=10M | borg create --progress --stats ::specialbackup -
# Let's continue with some simple things:
## USEFUL COMMANDS ##
# You can show some information about an archive. You can even do it without needing to specify the archive name:
borg info :: --last 1
# So let's rename our last archive:
borg rename ::specialbackup backup-block-device
<up>
borg info :: --last 1
# A very important step if you choose keyfile mode (where the keyfile is only saved locally) is to export your keyfile and possibly print it, etc.
borg key export :: --qr-code file.html # this creates a nice HTML, but when you want something simpler…
< remove comment >
< let there: borg check > --paper # this is a "manual input"-only backup (but it is also included in the --qr-code option)
## MAINTENANCE ##
# Sometimes backups get broken or we want a regular "checkup" that everything is okay…
borg check -v ::
# Next problem: Usually you do not have infinite disk space. So you may need to prune your archive…
# You can tune this in every detail. See the docs for details. Here only a simple example:
borg prune --list --keep-last 1 --dry-run
# When actually executing it in a script, you have to use it without the --dry-run option, of course.
## RESTORE ##
# When you want to see the diff between two archives use this command.
# E.g. what happened between the first two backups?
borg diff ::backup1 backup2
# Ah, we added a file, right…
# There are also other ways to extract the data.
# E.g. as a tar archive.
borg export-tar --progress ::backup2 backup.tar.gz
ls -l
# You can mount an archive or even the whole repository:
mkdir /tmp/mount
borg mount :: /tmp/mount
ls -la /tmp/mount
borg umount /tmp/mount
# That's it, but of course there is more to explore, so have a look at the docs.

4862
docs/misc/asciinema/basic.json Normal file
View File

File diff suppressed because it is too large Load Diff

53
docs/misc/asciinema/basic.sh Normal file
View File

@ -0,0 +1,53 @@
# Here you'll see some basic commands to start working with borg.
# Note: This teaser screencast was made with borg version 1.1.0 – older or newer borg versions may behave differently.
# But let's start.
# First of all, you can always get help:
borg help
# These are a lot of commands, so better we start with a few:
# Let's create a repo on an external drive…
borg init --encryption=repokey /media/backup/borgdemo
# This uses the repokey encryption. You may look at "borg help init" or the online doc at https://borgbackup.readthedocs.io/ for other modes.
# So now, let's create our first (compressed) backup.
borg create --stats --progress --compression lz4 /media/backup/borgdemo::backup1 Wallpaper
# That's nice, so far.
# So let's add a new file…
echo "new nice file" > Wallpaper/newfile.txt
<up>
borg create --stats --progress --compression lz4 /media/backup/borgdemo::backup2 Wallpaper
# Wow, this was a lot faster!
# Notice the "Deduplicated size" for "This archive"!
# Borg recognized that most files did not change and deduplicated them.
# But what happens, when we move a dir and create a new backup?
mv …
borg create --stats --progress --compression lz4 /media/backup/borgdemo::backup3 Wallpaper
# Still quite fast…
# But when you look at the "deduplicated file size" again, you see that borg also recognized that only the dir and not the files changed in this backup.
# Now lets look into a repo.
borg list /media/backup/borgdemo
# You'll see a list of all backups.
# You can also use the same command to look into an archive. But we better filter the output here:
borg list /media/backup/borgdemo::backup3 | grep 'deer.jpg'
# Oh, we found our picture. Now extract it…
mv Wallpaper Wallpaper.orig
borg extract /media/backup/borgdemo::backup3 <copy>
# And check that it's the same:
diff -s Wallpaper/deer.jpg Wallpaper.orig/deer.jpg
# And, of course, we can also create remote repos via ssh when borg is setup there. This command creates a new remote repo in a subdirectory called "demo":
borg init --encryption=repokey borgdemo@remoteserver.example:./demo
# Easy, isn't it? That's all you need to know for basic usage.
# If you want to see more, have a look at the screencast showing the "advanced usage".
# In any case, enjoy using borg!

1354
docs/misc/asciinema/install.json Normal file
View File

File diff suppressed because it is too large Load Diff

21
docs/misc/asciinema/install.sh Normal file
View File

@ -0,0 +1,21 @@
# This asciinema will show you the installation of borg as a standalone binary. Usually you only need this if you want to have an up-to-date version of borg or no package is available for your distro/OS.
# First, we need to download the version, we'd like to install…
wget -q --show-progress https://github.com/borgbackup/borg/releases/download/1.1.0b6/borg-linux64
# and do not forget the GPG signature…!
wget -q --show-progress https://github.com/borgbackup/borg/releases/download/1.1.0b6/borg-linux64.asc
# In this case, we have already imported the public key of a borg developer. So we only need to verify it:
gpg --verify borg-linux64.asc
# Okay, the binary is valid!
# Now install it:
sudo cp borg-linux64 /usr/local/bin/borg
sudo chown root:root /usr/local/bin/borg
# and make it executable…
sudo chmod 755 /usr/local/bin/borg
# Now check it: (possibly needs a terminal restart)
borg -V
# That's it! Check out the other screencasts to see how to actually use borgbackup.

5550
docs/misc/asciinema/install_and_basics.json
View File

File diff suppressed because it is too large Load Diff

51
docs/misc/asciinema/install_and_basics.txt
View File

@ -1,51 +0,0 @@
# borgbackup - installation and basic usage
# I have already downloaded the binary release from github:
ls -l
# binary file + GPG signature
# verifying whether the binary is valid:
gpg --verify borg-linux64.asc borg-linux64
# install it as "borg":
cp borg-linux64 ~/bin/borg
# making it executable:
chmod +x ~/bin/borg
# yay, installation done! let's make backups!
# creating a repository:
borg init repo
# creating our first backup with stuff from "data" directory:
borg create --stats --progress --compression lz4 repo::backup1 data
# changing the data slightly:
echo "some more data" > data/one_file_more
# creating another backup:
borg create --stats --progress repo::backup2 data
# that was much faster! it recognized/deduplicated unchanged files.
# see the "Deduplicated size" column for "This archive"! :)
# extracting a backup archive:
mv data data.orig
borg extract repo::backup2
# checking if restored data differs from original data:
diff -r data.orig data
# no, it doesn't! :)
# listing the repo contents:
borg list repo
# listing the backup2 archive contents (shortened):
borg list repo::backup2 | tail
# easy, isn't it?
# if you like #borgbackup, spread the word!

64
docs/misc/benchmark-crud.txt Normal file
View File

@ -0,0 +1,64 @@
borg benchmark crud
===================
Here is some example of borg benchmark crud output.
I ran it on my laptop, Core i5-4200u, 8GB RAM, SATA SSD, Linux, ext4 fs.
"src" as well as repo is local, on this SSD.
$ BORG_PASSPHRASE=secret borg init --encryption repokey-blake2 repo
$ BORG_PASSPHRASE=secret borg benchmark crud repo src
C-Z-BIG 116.06 MB/s (10 * 100.00 MB all-zero files: 8.62s)
R-Z-BIG 197.00 MB/s (10 * 100.00 MB all-zero files: 5.08s)
U-Z-BIG 418.07 MB/s (10 * 100.00 MB all-zero files: 2.39s)
D-Z-BIG 724.94 MB/s (10 * 100.00 MB all-zero files: 1.38s)
C-R-BIG 42.21 MB/s (10 * 100.00 MB random files: 23.69s)
R-R-BIG 134.45 MB/s (10 * 100.00 MB random files: 7.44s)
U-R-BIG 316.83 MB/s (10 * 100.00 MB random files: 3.16s)
D-R-BIG 251.10 MB/s (10 * 100.00 MB random files: 3.98s)
C-Z-MEDIUM 118.53 MB/s (1000 * 1.00 MB all-zero files: 8.44s)
R-Z-MEDIUM 218.49 MB/s (1000 * 1.00 MB all-zero files: 4.58s)
U-Z-MEDIUM 591.59 MB/s (1000 * 1.00 MB all-zero files: 1.69s)
D-Z-MEDIUM 730.04 MB/s (1000 * 1.00 MB all-zero files: 1.37s)
C-R-MEDIUM 31.46 MB/s (1000 * 1.00 MB random files: 31.79s)
R-R-MEDIUM 129.64 MB/s (1000 * 1.00 MB random files: 7.71s)
U-R-MEDIUM 621.86 MB/s (1000 * 1.00 MB random files: 1.61s)
D-R-MEDIUM 234.82 MB/s (1000 * 1.00 MB random files: 4.26s)
C-Z-SMALL 19.81 MB/s (10000 * 10.00 kB all-zero files: 5.05s)
R-Z-SMALL 97.69 MB/s (10000 * 10.00 kB all-zero files: 1.02s)
U-Z-SMALL 36.35 MB/s (10000 * 10.00 kB all-zero files: 2.75s)
D-Z-SMALL 57.04 MB/s (10000 * 10.00 kB all-zero files: 1.75s)
C-R-SMALL 9.81 MB/s (10000 * 10.00 kB random files: 10.19s)
R-R-SMALL 92.21 MB/s (10000 * 10.00 kB random files: 1.08s)
U-R-SMALL 64.62 MB/s (10000 * 10.00 kB random files: 1.55s)
D-R-SMALL 51.62 MB/s (10000 * 10.00 kB random files: 1.94s)
A second run some time later gave:
C-Z-BIG 115.22 MB/s (10 * 100.00 MB all-zero files: 8.68s)
R-Z-BIG 196.06 MB/s (10 * 100.00 MB all-zero files: 5.10s)
U-Z-BIG 439.50 MB/s (10 * 100.00 MB all-zero files: 2.28s)
D-Z-BIG 671.11 MB/s (10 * 100.00 MB all-zero files: 1.49s)
C-R-BIG 43.40 MB/s (10 * 100.00 MB random files: 23.04s)
R-R-BIG 133.17 MB/s (10 * 100.00 MB random files: 7.51s)
U-R-BIG 464.50 MB/s (10 * 100.00 MB random files: 2.15s)
D-R-BIG 245.19 MB/s (10 * 100.00 MB random files: 4.08s)
C-Z-MEDIUM 110.82 MB/s (1000 * 1.00 MB all-zero files: 9.02s)
R-Z-MEDIUM 217.96 MB/s (1000 * 1.00 MB all-zero files: 4.59s)
U-Z-MEDIUM 601.54 MB/s (1000 * 1.00 MB all-zero files: 1.66s)
D-Z-MEDIUM 686.99 MB/s (1000 * 1.00 MB all-zero files: 1.46s)
C-R-MEDIUM 39.91 MB/s (1000 * 1.00 MB random files: 25.06s)
R-R-MEDIUM 128.91 MB/s (1000 * 1.00 MB random files: 7.76s)
U-R-MEDIUM 599.00 MB/s (1000 * 1.00 MB random files: 1.67s)
D-R-MEDIUM 230.69 MB/s (1000 * 1.00 MB random files: 4.33s)
C-Z-SMALL 14.78 MB/s (10000 * 10.00 kB all-zero files: 6.76s)
R-Z-SMALL 96.86 MB/s (10000 * 10.00 kB all-zero files: 1.03s)
U-Z-SMALL 35.22 MB/s (10000 * 10.00 kB all-zero files: 2.84s)
D-Z-SMALL 64.93 MB/s (10000 * 10.00 kB all-zero files: 1.54s)
C-R-SMALL 11.08 MB/s (10000 * 10.00 kB random files: 9.02s)
R-R-SMALL 92.34 MB/s (10000 * 10.00 kB random files: 1.08s)
U-R-SMALL 64.49 MB/s (10000 * 10.00 kB random files: 1.55s)
D-R-SMALL 46.96 MB/s (10000 * 10.00 kB random files: 2.13s)

BIN
docs/misc/borg-data-flow.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 98 KiB

BIN
docs/misc/borg-data-flow.vsd Normal file
View File

Binary file not shown.

56
docs/misc/compression.conf
View File

@ -1,56 +0,0 @@
# example config file for --compression-from option
#
# Format of non-comment / non-empty lines:
# <compression-spec>:<path/filename pattern>
# compression-spec is same format as for --compression option
# path/filename pattern is same format as for --exclude option
# archives / files:
none:*.gz
none:*.tgz
none:*.bz2
none:*.tbz2
none:*.xz
none:*.txz
none:*.lzma
none:*.lzo
none:*.zip
none:*.rar
none:*.7z
# audio:
none:*.mp3
none:*.ogg
none:*.oga
none:*.flac
none:*.aac
none:*.m4a
# video:
none:*.mp4
none:*.mkv
none:*.m4v
none:*.avi
none:*.mpg
none:*.mpeg
none:*.webm
none:*.vob
none:*.ts
none:*.ogv
none:*.mov
none:*.flv
none:*.ogm
# pictures/images
none:*.jpg
none:*.jpeg
none:*.png
none:*.gif
# disk images
none:*.dmg
# software archives
none:*.rpm
none:*.deb
none:*.msi

26
docs/misc/internals-picture.txt
View File

@ -11,22 +11,22 @@ BorgBackup from 10.000m
| | |
+------+-------+ |
| | | |
/chunk\/chunk\/chunk\... /maybe different chunks lists\
/chunk\/chunk\/chunk\... /maybe different chunks lists\
+-----------------------------------------------------------------+
|item list |
+-----------------------------------------------------------------+
|
+-------------------------------------+--------------+
| | |
| | |
+-------------+ +-------------+ |
|item0 | |item1 | |
| - owner | | - owner | |
| - size | | - size | ...
| - ... | | - ... |
| - chunks | | - chunks |
+----+--------+ +-----+-------+
| |
|
+-------------------------------------+--------------+
| | |
| | |
+-------------+ +-------------+ |
|item0 | |item1 | |
| - owner | | - owner | |
| - size | | - size | ...
| - ... | | - ... |
| - chunks | | - chunks |
+----+--------+ +-----+-------+
| |
| +-----+----------------------------+-----------------+
| | | |
+-o-----o------------+ |

72
docs/misc/prune-example.txt
View File

@ -2,7 +2,7 @@ borg prune visualized
=====================
Assume it is 2016-01-01, today's backup has not yet been made and you have
created at least one backup on each day in 2015 except on 2015-12-20 (no
created at least one backup on each day in 2015 except on 2015-12-19 (no
backup made on that day).
This is what borg prune --keep-daily 14 --keep-monthly 6 would keep.
@ -14,41 +14,41 @@ Calendar view
-------------
2015
January February March
Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su
1 2 3 4 1 1
5 6 7 8 9 10 11 2 3 4 5 6 7 8 2 3 4 5 6 7 8
12 13 14 15 16 17 18 9 10 11 12 13 14 15 9 10 11 12 13 14 15
19 20 21 22 23 24 25 16 17 18 19 20 21 22 16 17 18 19 20 21 22
26 27 28 29 30 31 23 24 25 26 27 28 23 24 25 26 27 28 29
30 31
January February March
Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su
1 2 3 4 1 1
5 6 7 8 9 10 11 2 3 4 5 6 7 8 2 3 4 5 6 7 8
12 13 14 15 16 17 18 9 10 11 12 13 14 15 9 10 11 12 13 14 15
19 20 21 22 23 24 25 16 17 18 19 20 21 22 16 17 18 19 20 21 22
26 27 28 29 30 31 23 24 25 26 27 28 23 24 25 26 27 28 29
30 31
April May June
Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su
1 2 3 4 5 1 2 3 1 2 3 4 5 6 7
6 7 8 9 10 11 12 4 5 6 7 8 9 10 8 9 10 11 12 13 14
13 14 15 16 17 18 19 11 12 13 14 15 16 17 15 16 17 18 19 20 21
20 21 22 23 24 25 26 18 19 20 21 22 23 24 22 23 24 25 26 27 28
27 28 29 30 25 26 27 28 29 30 31 29 30m
April May June
Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su
1 2 3 4 5 1 2 3 1 2 3 4 5 6 7
6 7 8 9 10 11 12 4 5 6 7 8 9 10 8 9 10 11 12 13 14
13 14 15 16 17 18 19 11 12 13 14 15 16 17 15 16 17 18 19 20 21
20 21 22 23 24 25 26 18 19 20 21 22 23 24 22 23 24 25 26 27 28
27 28 29 30 25 26 27 28 29 30 31 29 30m
July August September
Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su
1 2 3 4 5 1 2 1 2 3 4 5 6
6 7 8 9 10 11 12 3 4 5 6 7 8 9 7 8 9 10 11 12 13
13 14 15 16 17 18 19 10 11 12 13 14 15 16 14 15 16 17 18 19 20
20 21 22 23 24 25 26 17 18 19 20 21 22 23 21 22 23 24 25 26 27
27 28 29 30 31m 24 25 26 27 28 29 30 28 29 30m
31m
October November December
Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su
1 2 3 4 1 1 2 3 4 5 6
5 6 7 8 9 10 11 2 3 4 5 6 7 8 7 8 9 10 11 12 13
12 13 14 15 16 17 18 9 10 11 12 13 14 15 14 15 16 17d18d19d20
19 20 21 22 23 24 25 16 17 18 19 20 21 22 21d22d23d24d25d26d27d
26 27 28 29 30 31m 23 24 25 26 27 28 29 28d29d30d31d
30m
July August September
Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su
1 2 3 4 5 1 2 1 2 3 4 5 6
6 7 8 9 10 11 12 3 4 5 6 7 8 9 7 8 9 10 11 12 13
13 14 15 16 17 18 19 10 11 12 13 14 15 16 14 15 16 17 18 19 20
20 21 22 23 24 25 26 17 18 19 20 21 22 23 21 22 23 24 25 26 27
27 28 29 30 31m 24 25 26 27 28 29 30 28 29 30m
31m
October November December
Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su
1 2 3 4 1 1 2 3 4 5 6
5 6 7 8 9 10 11 2 3 4 5 6 7 8 7 8 9 10 11 12 13
12 13 14 15 16 17 18 9 10 11 12 13 14 15 14 15 16 17d18d19 20d
19 20 21 22 23 24 25 16 17 18 19 20 21 22 21d22d23d24d25d26d27d
26 27 28 29 30 31m 23 24 25 26 27 28 29 28d29d30d31d
30m
List view
---------
@ -66,8 +66,8 @@ List view
9. 2015-12-23
10. 2015-12-22
11. 2015-12-21
(no backup made on 2015-12-20)
12. 2015-12-19
12. 2015-12-20
(no backup made on 2015-12-19)
13. 2015-12-18
14. 2015-12-17
@ -83,7 +83,7 @@ Jun. December is not considered for this rule, because that backup was already
kept because of the daily rule.
2015-12-17 is kept to satisfy the --keep-daily 14 rule - because no backup was
made on 2015-12-20. If a backup had been made on that day, it would not keep
made on 2015-12-19. If a backup had been made on that day, it would not keep
the one from 2015-12-17.
We did not include yearly, weekly, hourly, minutely or secondly rules to keep

252
docs/quickstart.rst
View File

@ -5,9 +5,13 @@
Quick Start
===========
This chapter will get you started with |project_name|. The first section
presents a simple step by step example that uses |project_name| to backup data.
The next section continues by showing how backups can be automated.
This chapter will get you started with |project_name| and covers
various use cases.
A step by step example
----------------------
.. include:: quickstart_example.rst.inc
Important note about free space
-------------------------------
@ -17,13 +21,20 @@ a good amount of free space on the filesystem that has your backup repository
(and also on ~/.cache). A few GB should suffice for most hard-drive sized
repositories. See also :ref:`cache-memory-usage`.
|project_name| doesn't use space reserved for root on repository disks (even when run as root),
on file systems which do not support this mechanism (e.g. XFS) we recommend to
reserve some space in |project_name| itself just to be safe by adjusting the
``additional_free_space`` setting in the ``[repository]`` section of a repositories
``config`` file. A good starting point is ``2G``.
If |project_name| runs out of disk space, it tries to free as much space as it
can while aborting the current operation safely, which allows to free more space
by deleting/pruning archives. This mechanism is not bullet-proof though.
can while aborting the current operation safely, which allows the user to free more space
by deleting/pruning archives. This mechanism is not bullet-proof in some
circumstances [1]_.
If you *really* run out of disk space, it can be hard or impossible to free space,
because |project_name| needs free space to operate - even to delete backup
archives. There is a ``--save-space`` option for some commands, but even with
that |project_name| will need free space to operate.
archives.
You can use some monitoring process or just include the free space information
in your backup log files (you check them regularly anyway, right?).
@ -36,116 +47,147 @@ Also helpful:
- consider using quotas
- use `prune` regularly
.. [1] This failsafe can fail in these circumstances:
A step by step example
----------------------
1. Before a backup can be made a repository has to be initialized::
$ borg init /path/to/repo
2. Backup the ``~/src`` and ``~/Documents`` directories into an archive called
*Monday*::
$ borg create /path/to/repo::Monday ~/src ~/Documents
3. The next day create a new archive called *Tuesday*::
$ borg create --stats /path/to/repo::Tuesday ~/src ~/Documents
This backup will be a lot quicker and a lot smaller since only new never
before seen data is stored. The ``--stats`` option causes |project_name| to
output statistics about the newly created archive such as the amount of unique
data (not shared with other archives)::
------------------------------------------------------------------------------
Archive name: Tuesday
Archive fingerprint: bd31004d58f51ea06ff735d2e5ac49376901b21d58035f8fb05dbf866566e3c2
Time (start): Tue, 2016-02-16 18:15:11
Time (end): Tue, 2016-02-16 18:15:11
Duration: 0.19 seconds
Number of files: 127
------------------------------------------------------------------------------
Original size Compressed size Deduplicated size
This archive: 4.16 MB 4.17 MB 26.78 kB
All archives: 8.33 MB 8.34 MB 4.19 MB
Unique chunks Total chunks
Chunk index: 132 261
------------------------------------------------------------------------------
4. List all archives in the repository::
$ borg list /path/to/repo
Monday Mon, 2016-02-15 19:14:44
Tuesday Tue, 2016-02-16 19:15:11
5. List the contents of the *Monday* archive::
$ borg list /path/to/repo::Monday
drwxr-xr-x user group 0 Mon, 2016-02-15 18:22:30 home/user/Documents
-rw-r--r-- user group 7961 Mon, 2016-02-15 18:22:30 home/user/Documents/Important.doc
...
6. Restore the *Monday* archive::
$ borg extract /path/to/repo::Monday
7. Recover disk space by manually deleting the *Monday* archive::
$ borg delete /path/to/repo::Monday
.. Note::
Borg is quiet by default (it works on WARNING log level).
You can use options like ``--progress`` or ``--list`` to get specific
reports during command execution. You can also add the ``-v`` (or
``--verbose`` or ``--info``) option to adjust the log level to INFO to
get other informational messages.
- The underlying file system doesn't support statvfs(2), or returns incorrect
data, or the repository doesn't reside on a single file system
- Other tasks fill the disk simultaneously
- Hard quotas (which may not be reflected in statvfs(2))
Automating backups
------------------
The following example script backs up ``/home`` and ``/var/www`` to a remote
server. The script also uses the :ref:`borg_prune` subcommand to maintain a
certain number of old archives:
The following example script is meant to be run daily by the ``root`` user on
different local machines. It backs up a machine's important files (but not the
complete operating system) to a repository ``~/backup/main`` on a remote server.
Some files which aren't necessarily needed in this backup are excluded. See
:ref:`borg_patterns` on how to add more exclude options.
After the backup this script also uses the :ref:`borg_prune` subcommand to keep
only a certain number of old archives and deletes the others in order to preserve
disk space.
Before running, make sure that the repository is initialized as documented in
:ref:`remote_repos` and that the script has the correct permissions to be executable
by the root user, but not executable or readable by anyone else, i.e. root:root 0700.
You can use this script as a starting point and modify it where it's necessary to fit
your setup.
Do not forget to test your created backups to make sure everything you need is being
backed up and that the ``prune`` command is keeping and deleting the correct backups.
::
#!/bin/sh
# setting this, so the repo does not need to be given on the commandline:
export BORG_REPO=username@remoteserver.com:backup
# setting this, so you won't be asked for your passphrase - make sure the
# script has appropriate owner/group and mode, e.g. root.root 600:
export BORG_PASSPHRASE=mysecret
# Setting this, so the repo does not need to be given on the commandline:
export BORG_REPO=ssh://username@example.com:2022/~/backup/main
# Backup most important stuff:
borg create --stats -C lz4 ::'{hostname}-{now:%Y-%m-%d}' \
/etc \
/home \
/var \
--exclude '/home/*/.cache' \
--exclude '*.pyc'
# Setting this, so you won't be asked for your repository passphrase:
export BORG_PASSPHRASE='XYZl0ngandsecurepa_55_phrasea&&123'
# or this to ask an external program to supply the passphrase:
export BORG_PASSCOMMAND='pass show backup'
# some helpers and error handling:
info() { printf "\n%s %s\n\n" "$( date )" "$*" >&2; }
trap 'echo $( date ) Backup interrupted >&2; exit 2' INT TERM
info "Starting backup"
# Backup the most important directories into an archive named after
# the machine this script is currently running on:
borg create \
--verbose \
--filter AME \
--list \
--stats \
--show-rc \
--compression lz4 \
--exclude-caches \
--exclude '/home/*/.cache/*' \
--exclude '/var/cache/*' \
--exclude '/var/tmp/*' \
\
::'{hostname}-{now}' \
/etc \
/home \
/root \
/var \
backup_exit=$?
info "Pruning repository"
# Use the `prune` subcommand to maintain 7 daily, 4 weekly and 6 monthly
# archives of THIS machine. The '{hostname}-' prefix is very important to
# limit prune's operation to this machine's archives and not apply to
# other machine's archives also.
borg prune -v --prefix '{hostname}-' \
--keep-daily=7 --keep-weekly=4 --keep-monthly=6
# other machines' archives also:
borg prune \
--list \
--prefix '{hostname}-' \
--show-rc \
--keep-daily 7 \
--keep-weekly 4 \
--keep-monthly 6 \
prune_exit=$?
# use highest exit code as global exit code
global_exit=$(( backup_exit > prune_exit ? backup_exit : prune_exit ))
if [ ${global_exit} -eq 1 ];
then
info "Backup and/or Prune finished with a warning"
fi
if [ ${global_exit} -gt 1 ];
then
info "Backup and/or Prune finished with an error"
fi
exit ${global_exit}
Pitfalls with shell variables and environment variables
-------------------------------------------------------
This applies to all environment variables you want |project_name| to see, not just
``BORG_PASSPHRASE``. The short explanation is: always ``export`` your variable,
and use single quotes if you're unsure of the details of your shell's expansion
behavior. E.g.::
export BORG_PASSPHRASE='complicated & long'
This is because ``export`` exposes variables to subprocesses, which |project_name| may be
one of. More on ``export`` can be found in the "ENVIRONMENT" section of the
bash(1) man page.
Beware of how ``sudo`` interacts with environment variables. For example, you
may be surprised that the following ``export`` has no effect on your command::
export BORG_PASSPHRASE='complicated & long'
sudo ./yourborgwrapper.sh # still prompts for password
For more information, refer to the sudo(8) man page and ``env_keep`` in
the sudoers(5) man page.
.. Tip::
To debug what your borg process is actually seeing, find its PID
(``ps aux|grep borg``) and then look into ``/proc/<PID>/environ``.
.. backup_compression:
Backup compression
------------------
Default is no compression, but we support different methods with high speed
or high compression:
The default is lz4 (very fast, but low compression ratio), but other methods are
supported for different situations.
If you have a fast repo storage and you want some compression: ::
If you have a fast repo storage and you want minimum CPU usage, no compression::
$ borg create --compression lz4 /path/to/repo::arch ~
$ borg create --compression none /path/to/repo::arch ~
If you have a less fast repo storage and you want a bit more compression (N=0..9,
0 means no compression, 9 means high compression): ::
@ -207,8 +249,16 @@ For automated backups the passphrase can be specified using the
the key in case it gets corrupted or lost. Also keep your passphrase
at a safe place.
The backup that is encrypted with that key/passphrase won't help you
with that, of course.
You can make backups using :ref:`borg_key_export` subcommand.
If you want to print a backup of your key to paper use the ``--paper``
option of this command and print the result, or this print `template`_
if you need a version with QR-Code.
A backup inside of the backup that is encrypted with that key/passphrase
won't help you with that, of course.
.. _template: paperkey.html
.. _remote_repos:
@ -221,16 +271,14 @@ is installed on the remote host, in which case the following syntax is used::
$ borg init user@hostname:/path/to/repo
or::
$ borg init ssh://user@hostname:port//path/to/repo
Note: please see the usage chapter for a full documentation of repo URLs.
Remote operations over SSH can be automated with SSH keys. You can restrict the
use of the SSH keypair by prepending a forced command to the SSH public key in
the remote server's `authorized_keys` file. This example will start |project_name|
in server mode and limit it to a specific filesystem path::
command="borg serve --restrict-to-path /path/to/repo",no-pty,no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-user-rc ssh-rsa AAAAB3[...]
command="borg serve --restrict-to-path /path/to/repo",restrict ssh-rsa AAAAB3[...]
If it is not possible to install |project_name| on the remote host,
it is still possible to use the remote host to store a repository by
@ -239,3 +287,7 @@ mounting the remote filesystem, for example, using sshfs::
$ sshfs user@hostname:/path/to /path/to
$ borg init /path/to/repo
$ fusermount -u /path/to
You can also use other remote filesystems in a similar way. Just be careful,
not all filesystems out there are really stable and working good enough to
be acceptable for backup usage.

62
docs/quickstart_example.rst.inc Normal file
View File

@ -0,0 +1,62 @@
1. Before a backup can be made a repository has to be initialized::
$ borg init --encryption=repokey /path/to/repo
2. Backup the ``~/src`` and ``~/Documents`` directories into an archive called
*Monday*::
$ borg create /path/to/repo::Monday ~/src ~/Documents
3. The next day create a new archive called *Tuesday*::
$ borg create --stats /path/to/repo::Tuesday ~/src ~/Documents
This backup will be a lot quicker and a lot smaller since only new never
before seen data is stored. The ``--stats`` option causes Borg to
output statistics about the newly created archive such as the amount of unique
data (not shared with other archives)::
------------------------------------------------------------------------------
Archive name: Tuesday
Archive fingerprint: bd31004d58f51ea06ff735d2e5ac49376901b21d58035f8fb05dbf866566e3c2
Time (start): Tue, 2016-02-16 18:15:11
Time (end): Tue, 2016-02-16 18:15:11
Duration: 0.19 seconds
Number of files: 127
------------------------------------------------------------------------------
Original size Compressed size Deduplicated size
This archive: 4.16 MB 4.17 MB 26.78 kB
All archives: 8.33 MB 8.34 MB 4.19 MB
Unique chunks Total chunks
Chunk index: 132 261
------------------------------------------------------------------------------
4. List all archives in the repository::
$ borg list /path/to/repo
Monday Mon, 2016-02-15 19:14:44
Tuesday Tue, 2016-02-16 19:15:11
5. List the contents of the *Monday* archive::
$ borg list /path/to/repo::Monday
drwxr-xr-x user group 0 Mon, 2016-02-15 18:22:30 home/user/Documents
-rw-r--r-- user group 7961 Mon, 2016-02-15 18:22:30 home/user/Documents/Important.doc
...
6. Restore the *Monday* archive by extracting the files relative to the current directory::
$ borg extract /path/to/repo::Monday
7. Recover disk space by manually deleting the *Monday* archive::
$ borg delete /path/to/repo::Monday
.. Note::
Borg is quiet by default (it works on WARNING log level).
You can use options like ``--progress`` or ``--list`` to get specific
reports during command execution. You can also add the ``-v`` (or
``--verbose`` or ``--info``) option to adjust the log level to INFO to
get other informational messages.

28
docs/support.rst
View File

@ -28,6 +28,7 @@ nickname you get by typing "/nick mydesirednickname"):
http://webchat.freenode.net/?randomnick=1&channels=%23borgbackup&uio=MTY9dHJ1ZSY5PXRydWUa8
.. _mailing_list:
Mailing list
------------
@ -37,6 +38,15 @@ unsubscribe and where you can find the archives of the list, see the
`mailing list homepage
<https://mail.python.org/mailman/listinfo/borgbackup>`_.
Twitter
-------
Follow @borgbackup for announcements. You can also add @borgbackup if you
would like to get retweeted for a borg related tweet.
Please understand that Twitter is not suitable for longer / more complex
discussions - use one of the other channels for that.
Bounties and Fundraisers
------------------------
@ -56,3 +66,21 @@ As a developer, you can become a Bounty Hunter and win bounties (earn money) by
contributing to |project_name|, a free and open source software project.
We might also use BountySource to fund raise for some bigger goals.
.. _security-contact:
Security
--------
In case you discover a security issue, please use this contact for reporting it privately
and please, if possible, use encrypted E-Mail:
Thomas Waldmann <tw@waldmann-edv.de>
GPG Key Fingerprint: 6D5B EF9A DD20 7580 5747 B70F 9F88 FB52 FAF7 B393
The public key can be fetched from any GPG keyserver, but be careful: you must
use the **full fingerprint** to check that you got the correct key.
`Releases <https://github.com/borgbackup/borg/releases>`_ are signed with this GPG key,
please use GPG to verify their authenticity.

Some files were not shown because too many files have changed in this diff Show More