From 6a97e936ac64d8cc22e48fabb6e47a4820ca421c Mon Sep 17 00:00:00 2001 From: Rayyan Ansari Date: Thu, 10 Nov 2022 17:56:45 +0000 Subject: [PATCH 1/5] repository: use os.replace instead of os.rename On Windows, os.rename raises an exception if the destination file already exists, unlike os.replace which replaces the destination file. Docs: https://docs.python.org/3/library/os.html#os.rename https://docs.python.org/3/library/os.html#os.replace --- src/borg/repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/repository.py b/src/borg/repository.py index 56c846480..cbcb14ef1 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -613,7 +613,7 @@ def flush_and_sync(fd): os.fsync(fd.fileno()) def rename_tmp(file): - os.rename(file + '.tmp', file) + os.replace(file + ".tmp", file) hints = { b'version': 2, From 05505de807b8a40030031fa27d812a612b61a753 Mon Sep 17 00:00:00 2001 From: Rayyan Ansari Date: Thu, 10 Nov 2022 17:59:16 +0000 Subject: [PATCH 2/5] repository: cleanup(): close segment before unlinking On Windows, trying to delete a file that is already open raises an exception. Docs: https://docs.python.org/3/library/os.html#os.remove --- src/borg/repository.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/borg/repository.py b/src/borg/repository.py index cbcb14ef1..61905b394 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -1358,13 +1358,12 @@ def get_segments_transaction_id(self): def cleanup(self, transaction_id): """Delete segment files left by aborted transactions """ + self.close_segment() self.segment = transaction_id + 1 count = 0 for segment, filename in self.segment_iterator(reverse=True): if segment > transaction_id: - if segment in self.fds: - del self.fds[segment] - safe_unlink(filename) + self.delete_segment(segment) count += 1 else: break From 9cff9d96fa88ea207f76234855c207e7db4cbe8a Mon Sep 17 00:00:00 2001 From: Rayyan Ansari Date: Tue, 15 Nov 2022 17:19:57 +0000 Subject: [PATCH 3/5] testsuite: repository: close fd before deleting segment See last commit. --- src/borg/testsuite/repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/testsuite/repository.py b/src/borg/testsuite/repository.py index c52820e33..d7997d072 100644 --- a/src/borg/testsuite/repository.py +++ b/src/borg/testsuite/repository.py @@ -720,7 +720,7 @@ def corrupt_object(self, id_): fd.write(b'BOOM') def delete_segment(self, segment): - os.unlink(os.path.join(self.tmppath, 'repository', 'data', '0', str(segment))) + self.repository.io.delete_segment(segment) def delete_index(self): os.unlink(os.path.join(self.tmppath, 'repository', f'index.{self.get_head()}')) From 82f6adf8e064198902fa67c481558ddf215834e5 Mon Sep 17 00:00:00 2001 From: Rayyan Ansari Date: Tue, 15 Nov 2022 17:20:50 +0000 Subject: [PATCH 4/5] file_integrity.py: make sure file_fd is always closed on exit --- src/borg/crypto/file_integrity.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/borg/crypto/file_integrity.py b/src/borg/crypto/file_integrity.py index 87518a2a7..76503057f 100644 --- a/src/borg/crypto/file_integrity.py +++ b/src/borg/crypto/file_integrity.py @@ -127,6 +127,7 @@ def __init__(self, path, write, filename=None, override_fd=None, integrity_data= self.writing = write mode = 'wb' if write else 'rb' self.file_fd = override_fd or open(path, mode) + self.file_opened = override_fd is None self.digests = {} hash_cls = XXH64FileHashingWrapper @@ -189,9 +190,13 @@ def hash_part(self, partname, is_final=False): def __exit__(self, exc_type, exc_val, exc_tb): exception = exc_type is not None - if not exception: - self.hash_part('final', is_final=True) - self.hasher.__exit__(exc_type, exc_val, exc_tb) + try: + if not exception: + self.hash_part("final", is_final=True) + self.hasher.__exit__(exc_type, exc_val, exc_tb) + finally: + if self.file_opened: + self.file_fd.close() if exception: return if self.writing: From d0b09dfc958dfa5b0d827f9b727d88c7b8832107 Mon Sep 17 00:00:00 2001 From: Rayyan Ansari Date: Tue, 15 Nov 2022 19:12:13 +0000 Subject: [PATCH 5/5] testsuite: repository: skip some tests for RemoteRepository --- src/borg/testsuite/repository.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/borg/testsuite/repository.py b/src/borg/testsuite/repository.py index d7997d072..c51574920 100644 --- a/src/borg/testsuite/repository.py +++ b/src/borg/testsuite/repository.py @@ -1002,6 +1002,14 @@ def test_crash_before_compact(self): # skip this test, we can't mock-patch a Repository class in another process! pass + def test_repair_missing_commit_segment(self): + # skip this test, files in RemoteRepository cannot be deleted + pass + + def test_repair_missing_segment(self): + # skip this test, files in RemoteRepository cannot be deleted + pass + class RemoteLoggerTestCase(BaseTestCase): def setUp(self):