From e9ab11be497f1195d39503226925e9535bd5d312 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 10 Dec 2015 10:51:56 +0100 Subject: [PATCH 1/4] borg upgrade - fix README contents --- borg/upgrader.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/borg/upgrader.py b/borg/upgrader.py index 819814983..b7e6bf4cb 100644 --- a/borg/upgrader.py +++ b/borg/upgrader.py @@ -50,11 +50,18 @@ class AtticRepositoryUpgrader(Repository): try: self.convert_cache(dryrun) self.convert_segments(segments, dryrun=dryrun, inplace=inplace) + self.borg_readme() finally: self.lock.release() self.lock = None return backup + def borg_readme(self): + readme = os.path.join(self.path, 'README') + os.remove(readme) + with open(readme, 'w') as fd: + fd.write('This is a Borg repository\n') + @staticmethod def convert_segments(segments, dryrun=True, inplace=False): """convert repository segments from attic to borg From 7acda553ffd6ef4383af15c24cb2ddcea5418b6e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 10 Dec 2015 11:11:06 +0100 Subject: [PATCH 2/4] borg upgrade - fix locking because Repository.__init__ normally opens and locks the repo, and the upgrader just inherited from (borg) Repository, it created a lock file there before the "backup copy" was made. No big problem, but a bit unclean. Fixed it to not lock at the beginning, then make the copy, then lock. --- borg/upgrader.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/borg/upgrader.py b/borg/upgrader.py index b7e6bf4cb..d4c628e6e 100644 --- a/borg/upgrader.py +++ b/borg/upgrader.py @@ -16,6 +16,10 @@ ATTIC_MAGIC = b'ATTICSEG' class AtticRepositoryUpgrader(Repository): + def __init__(self, *args, **kw): + kw['lock'] = False # do not create borg lock files (now) in attic repo + super().__init__(*args, **kw) + def upgrade(self, dryrun=True, inplace=False): """convert an attic repository to a borg repository @@ -34,8 +38,8 @@ class AtticRepositoryUpgrader(Repository): if not dryrun: shutil.copytree(self.path, backup, copy_function=os.link) logger.info("opening attic repository with borg and converting") - # we need to open the repo to load configuration, keyfiles and segments - self.open(self.path, exclusive=False) + # now lock the repo, after we have made the copy + self.lock = UpgradableLock(os.path.join(self.path, 'lock'), exclusive=True, timeout=1.0).acquire() segments = [filename for i, filename in self.io.segment_iterator()] try: keyfile = self.find_attic_keyfile() From cd804e72b703cda47dd0eadfbe986d49dab85a72 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 10 Dec 2015 11:35:48 +0100 Subject: [PATCH 3/4] borg upgrade - move some code to convert_repo_index it was a bit confusing to have repo-related code in a method called "convert_cache". also fixed a typo "index index". --- borg/upgrader.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/borg/upgrader.py b/borg/upgrader.py index d4c628e6e..4c14c8567 100644 --- a/borg/upgrader.py +++ b/borg/upgrader.py @@ -52,6 +52,7 @@ class AtticRepositoryUpgrader(Repository): self.lock = UpgradableLock(os.path.join(self.path, 'lock'), exclusive=True).acquire() try: + self.convert_repo_index(dryrun) self.convert_cache(dryrun) self.convert_segments(segments, dryrun=dryrun, inplace=inplace) self.borg_readme() @@ -150,8 +151,8 @@ class AtticRepositoryUpgrader(Repository): with open(keyfile, 'w') as f: f.write(data) - def convert_cache(self, dryrun): - """convert caches from attic to borg + def convert_repo_index(self, dryrun): + """convert some repo files those are all hash indexes, so we need to `s/ATTICIDX/BORG_IDX/` in a few locations: @@ -161,6 +162,21 @@ class AtticRepositoryUpgrader(Repository): should probably update, with a lock, see `Repository.open()`, which i'm not sure we should use because it may write data on `Repository.close()`... + """ + transaction_id = self.get_index_transaction_id() + if transaction_id is None: + logger.warning('no index file found for repository %s' % self.path) + else: + index = os.path.join(self.path, 'index.%d' % transaction_id).encode('utf-8') + logger.info("converting repo index %s" % index) + if not dryrun: + AtticRepositoryUpgrader.header_replace(index, b'ATTICIDX', b'BORG_IDX') + + def convert_cache(self, dryrun): + """convert caches from attic to borg + + those are all hash indexes, so we need to + `s/ATTICIDX/BORG_IDX/` in a few locations: * the `files` and `chunks` cache (in `$ATTIC_CACHE_DIR` or `$HOME/.cache/attic//`), which we could just drop, @@ -168,15 +184,6 @@ class AtticRepositoryUpgrader(Repository): `Cache.open()`, edit in place and then `Cache.close()` to make sure we have locking right """ - transaction_id = self.get_index_transaction_id() - if transaction_id is None: - logger.warning('no index file found for repository %s' % self.path) - else: - index = os.path.join(self.path, 'index.%d' % transaction_id).encode('utf-8') - logger.info("converting index index %s" % index) - if not dryrun: - AtticRepositoryUpgrader.header_replace(index, b'ATTICIDX', b'BORG_IDX') - # copy of attic's get_cache_dir() attic_cache_dir = os.environ.get('ATTIC_CACHE_DIR', os.path.join(os.path.expanduser('~'), From 60babd14c3df852870eefeed59776c7b3c40b4f4 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 10 Dec 2015 11:59:13 +0100 Subject: [PATCH 4/4] borg upgrade - do not overwrite backup_repo/index.*, fixes #466 the code obviously wrote to both index.* files as they were hardlinked. now we make a normal copy of index (and also hints) to avoid this. --- borg/upgrader.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/borg/upgrader.py b/borg/upgrader.py index 4c14c8567..57412b5c9 100644 --- a/borg/upgrader.py +++ b/borg/upgrader.py @@ -37,6 +37,16 @@ class AtticRepositoryUpgrader(Repository): logger.info('making a hardlink copy in %s', backup) if not dryrun: shutil.copytree(self.path, backup, copy_function=os.link) + # we need to create a real copy (not hardlink copy) of index.* and hints.* + # otherwise the backup copy will get modified. + transaction_id = self.get_index_transaction_id() + for name in ['index', 'hints']: + fname = "%s.%d" % (name, transaction_id) + fname_orig = os.path.join(self.path, fname) + fname_backup = os.path.join(backup, fname) + os.remove(fname_backup) + shutil.copy(fname_orig, fname_backup) + logger.info("opening attic repository with borg and converting") # now lock the repo, after we have made the copy self.lock = UpgradableLock(os.path.join(self.path, 'lock'), exclusive=True, timeout=1.0).acquire()