2016-04-17 01:15:19 +00:00
|
|
|
|
# this set must be kept complete, otherwise the RobustUnpacker might malfunction:
|
2022-03-02 03:19:33 +00:00
|
|
|
|
# fmt: off
|
2023-01-16 19:08:52 +00:00
|
|
|
|
ITEM_KEYS = frozenset(['path', 'source', 'target', 'rdev', 'chunks', 'chunks_healthy', 'hardlink_master', 'hlid',
|
2017-11-13 13:55:10 +00:00
|
|
|
|
'mode', 'user', 'group', 'uid', 'gid', 'mtime', 'atime', 'ctime', 'birthtime', 'size',
|
2016-06-26 16:07:01 +00:00
|
|
|
|
'xattrs', 'bsdflags', 'acl_nfs4', 'acl_access', 'acl_default', 'acl_extended',
|
2016-07-21 22:19:56 +00:00
|
|
|
|
'part'])
|
2022-03-02 03:19:33 +00:00
|
|
|
|
# fmt: on
|
2016-06-12 21:36:56 +00:00
|
|
|
|
|
|
|
|
|
# this is the set of keys that are always present in items:
|
|
|
|
|
REQUIRED_ITEM_KEYS = frozenset(["path", "mtime"])
|
|
|
|
|
|
|
|
|
|
# this set must be kept complete, otherwise rebuild_manifest might malfunction:
|
2022-03-02 03:19:33 +00:00
|
|
|
|
# fmt: off
|
2023-01-19 22:57:43 +00:00
|
|
|
|
ARCHIVE_KEYS = frozenset(['version', 'name', 'hostname', 'username', 'time', 'time_end',
|
2022-08-05 20:06:08 +00:00
|
|
|
|
'items', # legacy v1 archives
|
|
|
|
|
'item_ptrs', # v2+ archives
|
2016-06-12 21:36:56 +00:00
|
|
|
|
'comment', 'chunker_params',
|
2023-01-19 22:57:43 +00:00
|
|
|
|
'command_line', 'recreate_command_line', # v2+ archives
|
|
|
|
|
'cmdline', 'recreate_cmdline', # legacy
|
2019-02-23 09:49:24 +00:00
|
|
|
|
'recreate_source_id', 'recreate_args', 'recreate_partial_chunks', # used in 1.1.0b1 .. b2
|
2023-01-31 20:05:12 +00:00
|
|
|
|
'size', 'nfiles',
|
|
|
|
|
'size_parts', 'nfiles_parts', # legacy v1 archives
|
|
|
|
|
])
|
2022-03-02 03:19:33 +00:00
|
|
|
|
# fmt: on
|
2016-06-12 21:36:56 +00:00
|
|
|
|
|
|
|
|
|
# this is the set of keys that are always present in archives:
|
2023-01-19 22:57:43 +00:00
|
|
|
|
REQUIRED_ARCHIVE_KEYS = frozenset(["version", "name", "item_ptrs", "command_line", "time"])
|
2016-04-17 01:15:19 +00:00
|
|
|
|
|
2017-06-09 14:49:30 +00:00
|
|
|
|
# default umask, overridden by --umask, defaults to read/write only for owner
|
2016-04-17 01:15:19 +00:00
|
|
|
|
UMASK_DEFAULT = 0o077
|
|
|
|
|
|
2020-11-01 17:45:56 +00:00
|
|
|
|
# default file mode to store stdin data, defaults to read/write for owner and group
|
|
|
|
|
# forcing to 0o100XXX later
|
|
|
|
|
STDIN_MODE_DEFAULT = 0o660
|
|
|
|
|
|
2023-09-15 20:19:29 +00:00
|
|
|
|
# RepoObj types
|
|
|
|
|
ROBJ_MANIFEST = "M" # Manifest (directory of archives, other metadata) object
|
|
|
|
|
ROBJ_ARCHIVE_META = "A" # main archive metadata object
|
|
|
|
|
ROBJ_ARCHIVE_CHUNKIDS = "C" # objects with a list of archive metadata stream chunkids
|
|
|
|
|
ROBJ_ARCHIVE_STREAM = "S" # archive metadata stream chunk (containing items)
|
|
|
|
|
ROBJ_FILE_STREAM = "F" # file content stream chunk (containing user data)
|
|
|
|
|
ROBJ_DONTCARE = "*" # used to parse without type assertion (= accept any type)
|
|
|
|
|
|
2022-03-31 16:21:46 +00:00
|
|
|
|
# in borg < 1.3, this has been defined like this:
|
|
|
|
|
# 20 MiB minus 41 bytes for a PUT header (because the "size" field in the Repository includes
|
|
|
|
|
# the header, and the total size was set to precisely 20 MiB for borg < 1.3).
|
2017-02-22 23:34:40 +00:00
|
|
|
|
MAX_DATA_SIZE = 20971479
|
2017-02-22 15:53:03 +00:00
|
|
|
|
|
2022-03-31 16:21:46 +00:00
|
|
|
|
# MAX_OBJECT_SIZE = MAX_DATA_SIZE + len(PUT2 header)
|
|
|
|
|
# note: for borg >= 1.3, this makes the MAX_OBJECT_SIZE grow slightly over the precise 20MiB used by
|
|
|
|
|
# borg < 1.3, but this is not expected to cause any issues.
|
|
|
|
|
MAX_OBJECT_SIZE = MAX_DATA_SIZE + 41 + 8 # see assertion at end of repository module
|
2017-06-24 16:31:34 +00:00
|
|
|
|
|
2022-08-11 15:36:57 +00:00
|
|
|
|
# How many segment files borg puts into a single directory by default.
|
|
|
|
|
DEFAULT_SEGMENTS_PER_DIR = 1000
|
|
|
|
|
|
|
|
|
|
# A large, but not unreasonably large segment size. Always less than 2 GiB (for legacy file systems). We choose
|
|
|
|
|
# 500 MiB which means that no indirection from the inode is needed for typical Linux file systems.
|
|
|
|
|
# Note that this is a soft-limit and can be exceeded (worst case) by a full maximum chunk size and some metadata
|
|
|
|
|
# bytes. That's why it's 500 MiB instead of 512 MiB.
|
|
|
|
|
DEFAULT_MAX_SEGMENT_SIZE = 500 * 1024 * 1024
|
2022-08-05 20:06:08 +00:00
|
|
|
|
|
2018-03-02 23:17:40 +00:00
|
|
|
|
# repo config max_segment_size value must be below this limit to stay within uint32 offsets:
|
|
|
|
|
MAX_SEGMENT_SIZE_LIMIT = 2**32 - MAX_OBJECT_SIZE
|
|
|
|
|
|
2022-08-11 15:36:57 +00:00
|
|
|
|
# how many metadata stream chunk ids do we store into a "pointer chunk" of the ArchiveItem.item_ptrs list?
|
|
|
|
|
IDS_PER_CHUNK = 3 # MAX_DATA_SIZE // 40
|
|
|
|
|
|
2021-01-14 19:02:18 +00:00
|
|
|
|
# have one all-zero bytes object
|
|
|
|
|
# we use it at all places where we need to detect or create all-zero buffers
|
|
|
|
|
zeros = bytes(MAX_DATA_SIZE)
|
|
|
|
|
|
2017-06-24 16:31:34 +00:00
|
|
|
|
# borg.remote read() buffer size
|
|
|
|
|
BUFSIZE = 10 * 1024 * 1024
|
|
|
|
|
|
2017-06-23 03:56:41 +00:00
|
|
|
|
# to use a safe, limited unpacker, we need to set a upper limit to the archive count in the manifest.
|
|
|
|
|
# this does not mean that you can always really reach that number, because it also needs to be less than
|
|
|
|
|
# MAX_DATA_SIZE or it will trigger the check for that.
|
|
|
|
|
MAX_ARCHIVES = 400000
|
|
|
|
|
|
2017-06-24 16:31:34 +00:00
|
|
|
|
# repo.list() / .scan() result count limit the borg client uses
|
|
|
|
|
LIST_SCAN_LIMIT = 100000
|
|
|
|
|
|
2018-06-13 00:43:34 +00:00
|
|
|
|
FD_MAX_AGE = 4 * 60 # 4 minutes
|
|
|
|
|
|
2023-05-27 11:23:31 +00:00
|
|
|
|
# Some bounds on segment / segment_dir indexes
|
|
|
|
|
MIN_SEGMENT_INDEX = 0
|
|
|
|
|
MAX_SEGMENT_INDEX = 2**32 - 1
|
|
|
|
|
MIN_SEGMENT_DIR_INDEX = 0
|
|
|
|
|
MAX_SEGMENT_DIR_INDEX = 2**32 - 1
|
|
|
|
|
|
2022-08-11 15:36:57 +00:00
|
|
|
|
# chunker algorithms
|
|
|
|
|
CH_BUZHASH = "buzhash"
|
|
|
|
|
CH_FIXED = "fixed"
|
2023-02-12 23:41:01 +00:00
|
|
|
|
CH_FAIL = "fail"
|
2022-08-11 15:36:57 +00:00
|
|
|
|
|
|
|
|
|
# buzhash chunker params
|
2016-04-17 01:15:19 +00:00
|
|
|
|
CHUNK_MIN_EXP = 19 # 2**19 == 512kiB
|
|
|
|
|
CHUNK_MAX_EXP = 23 # 2**23 == 8MiB
|
|
|
|
|
HASH_WINDOW_SIZE = 0xFFF # 4095B
|
|
|
|
|
HASH_MASK_BITS = 21 # results in ~2MiB chunks statistically
|
|
|
|
|
|
|
|
|
|
# defaults, use --chunker-params to override
|
2019-02-13 03:36:09 +00:00
|
|
|
|
CHUNKER_PARAMS = (CH_BUZHASH, CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE)
|
2016-04-17 01:15:19 +00:00
|
|
|
|
|
|
|
|
|
# chunker params for the items metadata stream, finer granularity
|
2019-02-13 03:36:09 +00:00
|
|
|
|
ITEMS_CHUNKER_PARAMS = (CH_BUZHASH, 15, 19, 17, HASH_WINDOW_SIZE)
|
2016-04-17 01:15:19 +00:00
|
|
|
|
|
2020-12-14 22:46:04 +00:00
|
|
|
|
# normal on-disk data, allocated (but not written, all zeros), not allocated hole (all zeros)
|
|
|
|
|
CH_DATA, CH_ALLOC, CH_HOLE = 0, 1, 2
|
|
|
|
|
|
implement files cache mode control, fixes #911
You can now control the files cache mode using this option:
--files-cache={ctime,mtime,size,inode,rechunk,disabled}*
(only some combinations are supported)
Previously, only these modes were supported:
- mtime,size,inode (default of borg < 1.1.0rc4)
- mtime,size (by using --ignore-inode)
- disabled (by using --no-files-cache)
Now, you additionally get:
- ctime alternatively to mtime (more safe), e.g.:
ctime,size,inode (this is the new default of borg >= 1.1.0rc4)
- rechunk (consider all files as changed, rechunk them)
Deprecated:
- --ignore-inodes (use modes without "inode")
- --no-files-cache (use "disabled" mode)
The tests needed some changes:
- previously, we use os.utime() to set a files mtime (atime) to specific
values, but that does not work for ctime.
- now use time.sleep() to create the "latest file" that usually does
not end up in the files cache (see FAQ)
2017-09-11 00:54:52 +00:00
|
|
|
|
# operating mode of the files cache (for fast skipping of unchanged files)
|
2022-05-30 12:01:19 +00:00
|
|
|
|
FILES_CACHE_MODE_UI_DEFAULT = "ctime,size,inode" # default for "borg create" command (CLI UI)
|
|
|
|
|
FILES_CACHE_MODE_DISABLED = "d" # most borg commands do not use the files cache at all (disable)
|
implement files cache mode control, fixes #911
You can now control the files cache mode using this option:
--files-cache={ctime,mtime,size,inode,rechunk,disabled}*
(only some combinations are supported)
Previously, only these modes were supported:
- mtime,size,inode (default of borg < 1.1.0rc4)
- mtime,size (by using --ignore-inode)
- disabled (by using --no-files-cache)
Now, you additionally get:
- ctime alternatively to mtime (more safe), e.g.:
ctime,size,inode (this is the new default of borg >= 1.1.0rc4)
- rechunk (consider all files as changed, rechunk them)
Deprecated:
- --ignore-inodes (use modes without "inode")
- --no-files-cache (use "disabled" mode)
The tests needed some changes:
- previously, we use os.utime() to set a files mtime (atime) to specific
values, but that does not work for ctime.
- now use time.sleep() to create the "latest file" that usually does
not end up in the files cache (see FAQ)
2017-09-11 00:54:52 +00:00
|
|
|
|
|
2016-04-17 01:15:19 +00:00
|
|
|
|
# return codes returned by borg command
|
|
|
|
|
EXIT_SUCCESS = 0 # everything done, no problems
|
2023-11-09 05:00:13 +00:00
|
|
|
|
EXIT_WARNING = 1 # reached normal end of operation, but there were issues (generic warning)
|
|
|
|
|
EXIT_ERROR = 2 # terminated abruptly, did not reach end of operation (generic error)
|
|
|
|
|
EXIT_ERROR_BASE = 3 # specific error codes are 3..99 (enabled by BORG_EXIT_CODES=modern)
|
|
|
|
|
EXIT_WARNING_BASE = 100 # specific warning codes are 100..127 (enabled by BORG_EXIT_CODES=modern)
|
2020-06-29 22:11:12 +00:00
|
|
|
|
EXIT_SIGNAL_BASE = 128 # terminated due to signal, rc = 128 + sig_no
|
2016-04-17 01:15:19 +00:00
|
|
|
|
|
2017-09-05 02:44:38 +00:00
|
|
|
|
ISO_FORMAT_NO_USECS = "%Y-%m-%dT%H:%M:%S"
|
|
|
|
|
ISO_FORMAT = ISO_FORMAT_NO_USECS + ".%f"
|
|
|
|
|
|
2016-04-17 01:15:19 +00:00
|
|
|
|
DASHES = "-" * 78
|
|
|
|
|
|
|
|
|
|
PBKDF2_ITERATIONS = 100000
|
2016-11-11 20:24:16 +00:00
|
|
|
|
|
2022-04-07 14:22:34 +00:00
|
|
|
|
# https://www.rfc-editor.org/rfc/rfc9106.html#section-4-6.2
|
|
|
|
|
ARGON2_ARGS = {"time_cost": 3, "memory_cost": 2**16, "parallelism": 4, "type": "id"}
|
|
|
|
|
ARGON2_SALT_BYTES = 16
|
|
|
|
|
|
|
|
|
|
# Maps the CLI argument to our internal identifier for the format
|
|
|
|
|
KEY_ALGORITHMS = {
|
|
|
|
|
# encrypt-and-MAC, kdf: PBKDF2(HMAC−SHA256), encryption: AES256-CTR, authentication: HMAC-SHA256
|
|
|
|
|
"pbkdf2": "sha256",
|
2022-04-16 09:32:59 +00:00
|
|
|
|
# encrypt-then-MAC, kdf: argon2, encryption: chacha20, authentication: poly1305
|
|
|
|
|
"argon2": "argon2 chacha20-poly1305",
|
2022-04-07 14:22:34 +00:00
|
|
|
|
}
|
|
|
|
|
|
2016-11-11 20:24:16 +00:00
|
|
|
|
|
2022-03-11 20:20:26 +00:00
|
|
|
|
class KeyBlobStorage:
|
|
|
|
|
NO_STORAGE = "no_storage"
|
|
|
|
|
KEYFILE = "keyfile"
|
|
|
|
|
REPO = "repository"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class KeyType:
|
2022-03-18 00:46:56 +00:00
|
|
|
|
# legacy crypto
|
|
|
|
|
# upper 4 bits are ciphersuite, 0 == legacy AES-CTR
|
2022-03-11 20:20:26 +00:00
|
|
|
|
KEYFILE = 0x00
|
|
|
|
|
# repos with PASSPHRASE mode could not be created any more since borg 1.0, see #97.
|
2022-06-09 16:13:40 +00:00
|
|
|
|
# in borg 2. all of its code and also the "borg key migrate-to-repokey" command was removed.
|
2022-03-11 20:20:26 +00:00
|
|
|
|
# if you still need to, you can use "borg key migrate-to-repokey" with borg 1.0, 1.1 and 1.2.
|
|
|
|
|
# Nowadays, we just dispatch this to RepoKey and assume the passphrase was migrated to a repokey.
|
2022-07-13 14:55:29 +00:00
|
|
|
|
PASSPHRASE = 0x01 # legacy, borg < 1.0
|
2022-03-11 20:20:26 +00:00
|
|
|
|
PLAINTEXT = 0x02
|
|
|
|
|
REPO = 0x03
|
|
|
|
|
BLAKE2KEYFILE = 0x04
|
|
|
|
|
BLAKE2REPO = 0x05
|
|
|
|
|
BLAKE2AUTHENTICATED = 0x06
|
|
|
|
|
AUTHENTICATED = 0x07
|
2022-03-18 00:46:56 +00:00
|
|
|
|
# new crypto
|
|
|
|
|
# upper 4 bits are ciphersuite, lower 4 bits are keytype
|
|
|
|
|
AESOCBKEYFILE = 0x10
|
|
|
|
|
AESOCBREPO = 0x11
|
|
|
|
|
CHPOKEYFILE = 0x20
|
|
|
|
|
CHPOREPO = 0x21
|
|
|
|
|
BLAKE2AESOCBKEYFILE = 0x30
|
|
|
|
|
BLAKE2AESOCBREPO = 0x31
|
|
|
|
|
BLAKE2CHPOKEYFILE = 0x40
|
|
|
|
|
BLAKE2CHPOREPO = 0x41
|
2022-03-11 20:20:26 +00:00
|
|
|
|
|
|
|
|
|
|
2022-08-11 15:36:57 +00:00
|
|
|
|
CACHE_TAG_NAME = "CACHEDIR.TAG"
|
|
|
|
|
CACHE_TAG_CONTENTS = b"Signature: 8a477f597d28d172789f06886806bc55"
|
|
|
|
|
|
2016-11-11 20:24:16 +00:00
|
|
|
|
REPOSITORY_README = """This is a Borg Backup repository.
|
|
|
|
|
See https://borgbackup.readthedocs.io/
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
CACHE_README = """This is a Borg Backup cache.
|
|
|
|
|
See https://borgbackup.readthedocs.io/
|
|
|
|
|
"""
|