Merge pull request #1637 from ThomasWaldmann/merge-1.0-maint

Merge 1.0 maint

docs/faq.rst

@@ -399,7 +399,7 @@ Create a wrapper script:  /usr/local/bin/pv-wrapper  ::
 
 Add BORG_RSH environment variable to use pipeviewer wrapper script with ssh. ::
 
-    export BORG_RSH='/usr/local/bin/pv-wrapper.sh ssh'
+    export BORG_RSH='/usr/local/bin/pv-wrapper ssh'
 
 Now |project_name| will be bandwidth limited. Nice thing about pv is that you can change rate-limit on the fly: ::
 

src/borg/archive.py

@@ -89,9 +89,13 @@ class Statistics:
                 msg = '{0.osize_fmt} O {0.csize_fmt} C {0.usize_fmt} D {0.nfiles} N '.format(self)
                 path = remove_surrogates(item.path) if item else ''
                 space = columns - swidth(msg)
-                if space < swidth('...') + swidth(path):
-                    path = '%s...%s' % (path[:(space // 2) - swidth('...')], path[-space // 2:])
-                msg += "{0:<{space}}".format(path, space=space)
+                if space < 12:
+                    msg = ''
+                    space = columns - swidth(msg)
+                if space >= 8:
+                    if space < swidth('...') + swidth(path):
+                        path = '%s...%s' % (path[:(space // 2) - swidth('...')], path[-space // 2:])
+                    msg += "{0:<{space}}".format(path, space=space)
             else:
                 msg = ' ' * columns
             print(msg, file=stream or sys.stderr, end="\r", flush=True)
@@ -798,7 +802,7 @@ Number of files: {0.stats.nfiles}'''.format(
         # Is it a hard link?
         if st.st_nlink > 1:
             source = self.hard_links.get((st.st_ino, st.st_dev))
-            if (st.st_ino, st.st_dev) in self.hard_links:
+            if source is not None:
                 item = Item(path=safe_path, source=source)
                 item.update(self.stat_attrs(st, path))
                 self.add_item(item)

src/borg/archiver.py

@@ -44,6 +44,7 @@ from .helpers import ErrorIgnoringTextIOWrapper
 from .helpers import ProgressIndicatorPercent
 from .item import Item
 from .key import key_creator, RepoKey, PassphraseKey
+from .keymanager import KeyManager
 from .platform import get_flags
 from .remote import RepositoryServer, RemoteRepository, cache_if_remote
 from .repository import Repository
@@ -221,6 +222,39 @@ class Archiver:
         key.change_passphrase()
         return EXIT_SUCCESS
 
+    @with_repository(lock=False, exclusive=False, manifest=False, cache=False)
+    def do_key_export(self, args, repository):
+        """Export the repository key for backup"""
+        manager = KeyManager(repository)
+        manager.load_keyblob()
+        if args.paper:
+            manager.export_paperkey(args.path)
+        else:
+            if not args.path:
+                self.print_error("output file to export key to expected")
+                return EXIT_ERROR
+            manager.export(args.path)
+        return EXIT_SUCCESS
+
+    @with_repository(lock=False, exclusive=False, manifest=False, cache=False)
+    def do_key_import(self, args, repository):
+        """Import the repository key from backup"""
+        manager = KeyManager(repository)
+        if args.paper:
+            if args.path:
+                self.print_error("with --paper import from file is not supported")
+                return EXIT_ERROR
+            manager.import_paperkey(args)
+        else:
+            if not args.path:
+                self.print_error("input file to import key from expected")
+                return EXIT_ERROR
+            if not os.path.exists(args.path):
+                self.print_error("input file does not exist: " + args.path)
+                return EXIT_ERROR
+            manager.import_keyfile(args)
+        return EXIT_SUCCESS
+
     @with_repository(manifest=False)
     def do_migrate_to_repokey(self, args, repository):
         """Migrate passphrase -> repokey"""
@@ -1501,6 +1535,34 @@ class Archiver:
         subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
                                type=location_validator(archive=False))
 
+        subparser = subparsers.add_parser('key-export', parents=[common_parser], add_help=False,
+                                          description=self.do_key_export.__doc__,
+                                          epilog="",
+                                          formatter_class=argparse.RawDescriptionHelpFormatter,
+                                          help='export repository key for backup')
+        subparser.set_defaults(func=self.do_key_export)
+        subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
+                               type=location_validator(archive=False))
+        subparser.add_argument('path', metavar='PATH', nargs='?', type=str,
+                               help='where to store the backup')
+        subparser.add_argument('--paper', dest='paper', action='store_true',
+                               default=False,
+                               help='Create an export suitable for printing and later type-in')
+
+        subparser = subparsers.add_parser('key-import', parents=[common_parser], add_help=False,
+                                          description=self.do_key_import.__doc__,
+                                          epilog="",
+                                          formatter_class=argparse.RawDescriptionHelpFormatter,
+                                          help='import repository key from backup')
+        subparser.set_defaults(func=self.do_key_import)
+        subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
+                               type=location_validator(archive=False))
+        subparser.add_argument('path', metavar='PATH', nargs='?', type=str,
+                               help='path to the backup')
+        subparser.add_argument('--paper', dest='paper', action='store_true',
+                               default=False,
+                               help='interactively import from a backup done with --paper')
+
         migrate_to_repokey_epilog = textwrap.dedent("""
         This command migrates a repository from passphrase mode (not supported any
         more) to repokey mode.

src/borg/keymanager.py

@@ -0,0 +1,213 @@
+from binascii import hexlify, unhexlify, a2b_base64, b2a_base64
+import binascii
+import textwrap
+from hashlib import sha256
+
+from .key import KeyfileKey, RepoKey, PassphraseKey, KeyfileNotFoundError, PlaintextKey
+from .helpers import Manifest, NoManifestError, Error, yes
+from .repository import Repository
+
+
+class UnencryptedRepo(Error):
+    """Keymanagement not available for unencrypted repositories."""
+
+
+class UnknownKeyType(Error):
+    """Keytype {0} is unknown."""
+
+
+class RepoIdMismatch(Error):
+    """This key backup seems to be for a different backup repository, aborting."""
+
+
+class NotABorgKeyFile(Error):
+    """This file is not a borg key backup, aborting."""
+
+
+def sha256_truncated(data, num):
+    h = sha256()
+    h.update(data)
+    return h.hexdigest()[:num]
+
+
+KEYBLOB_LOCAL = 'local'
+KEYBLOB_REPO = 'repo'
+
+
+class KeyManager:
+    def __init__(self, repository):
+        self.repository = repository
+        self.keyblob = None
+        self.keyblob_storage = None
+
+        try:
+            cdata = self.repository.get(Manifest.MANIFEST_ID)
+        except Repository.ObjectNotFound:
+            raise NoManifestError
+
+        key_type = cdata[0]
+        if key_type == KeyfileKey.TYPE:
+            self.keyblob_storage = KEYBLOB_LOCAL
+        elif key_type == RepoKey.TYPE or key_type == PassphraseKey.TYPE:
+            self.keyblob_storage = KEYBLOB_REPO
+        elif key_type == PlaintextKey.TYPE:
+            raise UnencryptedRepo()
+        else:
+            raise UnknownKeyType(key_type)
+
+    def load_keyblob(self):
+        if self.keyblob_storage == KEYBLOB_LOCAL:
+            k = KeyfileKey(self.repository)
+            target = k.find_key()
+            with open(target, 'r') as fd:
+                self.keyblob = ''.join(fd.readlines()[1:])
+
+        elif self.keyblob_storage == KEYBLOB_REPO:
+            self.keyblob = self.repository.load_key().decode()
+
+    def store_keyblob(self, args):
+        if self.keyblob_storage == KEYBLOB_LOCAL:
+            k = KeyfileKey(self.repository)
+            try:
+                target = k.find_key()
+            except KeyfileNotFoundError:
+                target = k.get_new_target(args)
+
+            self.store_keyfile(target)
+        elif self.keyblob_storage == KEYBLOB_REPO:
+            self.repository.save_key(self.keyblob.encode('utf-8'))
+
+    def store_keyfile(self, target):
+        with open(target, 'w') as fd:
+            fd.write('%s %s\n' % (KeyfileKey.FILE_ID, hexlify(self.repository.id).decode('ascii')))
+            fd.write(self.keyblob)
+            if not self.keyblob.endswith('\n'):
+                fd.write('\n')
+
+    def export(self, path):
+        self.store_keyfile(path)
+
+    def export_paperkey(self, path):
+        def grouped(s):
+            ret = ''
+            i = 0
+            for ch in s:
+                if i and i % 6 == 0:
+                    ret += ' '
+                ret += ch
+                i += 1
+            return ret
+
+        export = 'To restore key use borg key-import --paper /path/to/repo\n\n'
+
+        binary = a2b_base64(self.keyblob)
+        export += 'BORG PAPER KEY v1\n'
+        lines = (len(binary) + 17) // 18
+        repoid = hexlify(self.repository.id).decode('ascii')[:18]
+        complete_checksum = sha256_truncated(binary, 12)
+        export += 'id: {0:d} / {1} / {2} - {3}\n'.format(lines,
+                                       grouped(repoid),
+                                       grouped(complete_checksum),
+                                       sha256_truncated((str(lines) + '/' + repoid + '/' + complete_checksum).encode('ascii'), 2))
+        idx = 0
+        while len(binary):
+            idx += 1
+            binline = binary[:18]
+            checksum = sha256_truncated(idx.to_bytes(2, byteorder='big') + binline, 2)
+            export += '{0:2d}: {1} - {2}\n'.format(idx, grouped(hexlify(binline).decode('ascii')), checksum)
+            binary = binary[18:]
+
+        if path:
+            with open(path, 'w') as fd:
+                fd.write(export)
+        else:
+            print(export)
+
+    def import_keyfile(self, args):
+        file_id = KeyfileKey.FILE_ID
+        first_line = file_id + ' ' + hexlify(self.repository.id).decode('ascii') + '\n'
+        with open(args.path, 'r') as fd:
+            file_first_line = fd.read(len(first_line))
+            if file_first_line != first_line:
+                if not file_first_line.startswith(file_id):
+                    raise NotABorgKeyFile()
+                else:
+                    raise RepoIdMismatch()
+            self.keyblob = fd.read()
+
+        self.store_keyblob(args)
+
+    def import_paperkey(self, args):
+        # imported here because it has global side effects
+        import readline
+
+        repoid = hexlify(self.repository.id).decode('ascii')[:18]
+        try:
+            while True:  # used for repeating on overall checksum mismatch
+                # id line input
+                while True:
+                    idline = input('id: ').replace(' ', '')
+                    if idline == "":
+                        if yes("Abort import? [yN]:"):
+                            raise EOFError()
+
+                    try:
+                        (data, checksum) = idline.split('-')
+                    except ValueError:
+                        print("each line must contain exactly one '-', try again")
+                        continue
+                    try:
+                        (id_lines, id_repoid, id_complete_checksum) = data.split('/')
+                    except ValueError:
+                        print("the id line must contain exactly three '/', try again")
+                    if sha256_truncated(data.lower().encode('ascii'), 2) != checksum:
+                        print('line checksum did not match, try same line again')
+                        continue
+                    try:
+                        lines = int(id_lines)
+                    except ValueError:
+                        print('internal error while parsing length')
+
+                    break
+
+                if repoid != id_repoid:
+                    raise RepoIdMismatch()
+
+                result = b''
+                idx = 1
+                # body line input
+                while True:
+                    inline = input('{0:2d}: '.format(idx))
+                    inline = inline.replace(' ', '')
+                    if inline == "":
+                        if yes("Abort import? [yN]:"):
+                            raise EOFError()
+                    try:
+                        (data, checksum) = inline.split('-')
+                    except ValueError:
+                        print("each line must contain exactly one '-', try again")
+                        continue
+                    try:
+                        part = unhexlify(data)
+                    except binascii.Error:
+                        print("only characters 0-9 and a-f and '-' are valid, try again")
+                        continue
+                    if sha256_truncated(idx.to_bytes(2, byteorder='big') + part, 2) != checksum:
+                        print('line checksum did not match, try line {0} again'.format(idx))
+                        continue
+                    result += part
+                    if idx == lines:
+                        break
+                    idx += 1
+
+                if sha256_truncated(result, 12) != id_complete_checksum:
+                    print('The overall checksum did not match, retry or enter a blank line to abort.')
+                    continue
+
+                self.keyblob = '\n'.join(textwrap.wrap(b2a_base64(result).decode('ascii'))) + '\n'
+                self.store_keyblob(args)
+                break
+
+        except EOFError:
+            print('\n - aborted')
+            return

src/borg/testsuite/archiver.py

@@ -1,3 +1,4 @@
+from binascii import hexlify, unhexlify, b2a_base64
 from configparser import ConfigParser
 import errno
 import os
@@ -33,7 +34,8 @@ from ..helpers import Chunk, Manifest
 from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR
 from ..helpers import bin_to_hex
 from ..item import Item
-from ..key import KeyfileKeyBase
+from ..key import KeyfileKeyBase, RepoKey, KeyfileKey, Passphrase
+from ..keymanager import RepoIdMismatch, NotABorgKeyFile
 from ..remote import RemoteRepository, PathNotAllowed
 from ..repository import Repository
 from . import has_lchflags, has_llfuse
@@ -1809,6 +1811,106 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         self.assert_not_in("input/file1", output)
         self.assert_not_in("x input/file5", output)
 
+    def test_key_export_keyfile(self):
+        export_file = self.output_path + '/exported'
+        self.cmd('init', self.repository_location, '--encryption', 'keyfile')
+        repo_id = self._extract_repository_id(self.repository_path)
+        self.cmd('key-export', self.repository_location, export_file)
+
+        with open(export_file, 'r') as fd:
+            export_contents = fd.read()
+
+        assert export_contents.startswith('BORG_KEY ' + hexlify(repo_id).decode() + '\n')
+
+        key_file = self.keys_path + '/' + os.listdir(self.keys_path)[0]
+
+        with open(key_file, 'r') as fd:
+            key_contents = fd.read()
+
+        assert key_contents == export_contents
+
+        os.unlink(key_file)
+
+        self.cmd('key-import', self.repository_location, export_file)
+
+        with open(key_file, 'r') as fd:
+            key_contents2 = fd.read()
+
+        assert key_contents2 == key_contents
+
+    def test_key_export_repokey(self):
+        export_file = self.output_path + '/exported'
+        self.cmd('init', self.repository_location, '--encryption', 'repokey')
+        repo_id = self._extract_repository_id(self.repository_path)
+        self.cmd('key-export', self.repository_location, export_file)
+
+        with open(export_file, 'r') as fd:
+            export_contents = fd.read()
+
+        assert export_contents.startswith('BORG_KEY ' + hexlify(repo_id).decode() + '\n')
+
+        with Repository(self.repository_path) as repository:
+            repo_key = RepoKey(repository)
+            repo_key.load(None, Passphrase.env_passphrase())
+
+        backup_key = KeyfileKey(None)
+        backup_key.load(export_file, Passphrase.env_passphrase())
+
+        assert repo_key.enc_key == backup_key.enc_key
+
+        with Repository(self.repository_path) as repository:
+            repository.save_key(b'')
+
+        self.cmd('key-import', self.repository_location, export_file)
+
+        with Repository(self.repository_path) as repository:
+            repo_key2 = RepoKey(repository)
+            repo_key2.load(None, Passphrase.env_passphrase())
+
+        assert repo_key2.enc_key == repo_key2.enc_key
+
+    def test_key_import_errors(self):
+        export_file = self.output_path + '/exported'
+        self.cmd('init', self.repository_location, '--encryption', 'keyfile')
+
+        self.cmd('key-import', self.repository_location, export_file, exit_code=EXIT_ERROR)
+
+        with open(export_file, 'w') as fd:
+            fd.write('something not a key\n')
+
+        self.assert_raises(NotABorgKeyFile, lambda: self.cmd('key-import', self.repository_location, export_file))
+
+        with open(export_file, 'w') as fd:
+            fd.write('BORG_KEY a0a0a0\n')
+
+        self.assert_raises(RepoIdMismatch, lambda: self.cmd('key-import', self.repository_location, export_file))
+
+    def test_key_export_paperkey(self):
+        repo_id = 'e294423506da4e1ea76e8dcdf1a3919624ae3ae496fddf905610c351d3f09239'
+
+        export_file = self.output_path + '/exported'
+        self.cmd('init', self.repository_location, '--encryption', 'keyfile')
+        self._set_repository_id(self.repository_path, unhexlify(repo_id))
+
+        key_file = self.keys_path + '/' + os.listdir(self.keys_path)[0]
+
+        with open(key_file, 'w') as fd:
+            fd.write(KeyfileKey.FILE_ID + ' ' + repo_id + '\n')
+            fd.write(b2a_base64(b'abcdefghijklmnopqrstu').decode())
+
+        self.cmd('key-export', '--paper', self.repository_location, export_file)
+
+        with open(export_file, 'r') as fd:
+            export_contents = fd.read()
+
+        assert export_contents == """To restore key use borg key-import --paper /path/to/repo
+
+BORG PAPER KEY v1
+id: 2 / e29442 3506da 4e1ea7 / 25f62a 5a3d41 - 02
+ 1: 616263 646566 676869 6a6b6c 6d6e6f 707172 - 6d
+ 2: 737475 - 88
+"""
+
 
 @unittest.skipUnless('binary' in BORG_EXES, 'no borg.exe available')
 class ArchiverTestCaseBinary(ArchiverTestCase):