diff --git a/src/borg/cache.py b/src/borg/cache.py index 4dc4c2181..473f156b2 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -19,6 +19,7 @@ from .item import Item from .key import PlaintextKey from .locking import UpgradableLock +from .platform import SaveFile from .remote import cache_if_remote ChunkListEntry = namedtuple('ChunkListEntry', 'id size csize') @@ -141,11 +142,11 @@ def create(self): config.set('cache', 'version', '1') config.set('cache', 'repository', self.repository.id_str) config.set('cache', 'manifest', '') - with open(os.path.join(self.path, 'config'), 'w') as fd: + with SaveFile(os.path.join(self.path, 'config')) as fd: config.write(fd) ChunkIndex().write(os.path.join(self.path, 'chunks').encode('utf-8')) os.makedirs(os.path.join(self.path, 'chunks.archive.d')) - with open(os.path.join(self.path, 'files'), 'wb') as fd: + with SaveFile(os.path.join(self.path, 'files'), binary=True) as fd: pass # empty file def _do_open(self): @@ -212,7 +213,7 @@ def commit(self): if not self.txn_active: return if self.files is not None: - with open(os.path.join(self.path, 'files'), 'wb') as fd: + with SaveFile(os.path.join(self.path, 'files'), binary=True) as fd: for path_hash, item in self.files.items(): # Discard cached files with the newest mtime to avoid # issues with filesystem snapshots and mtime precision @@ -223,7 +224,7 @@ def commit(self): self.config.set('cache', 'timestamp', self.manifest.timestamp) self.config.set('cache', 'key_type', str(self.key.TYPE)) self.config.set('cache', 'previous_location', self.repository._location.canonical_path()) - with open(os.path.join(self.path, 'config'), 'w') as fd: + with SaveFile(os.path.join(self.path, 'config')) as fd: self.config.write(fd) self.chunks.write(os.path.join(self.path, 'chunks').encode('utf-8')) os.rename(os.path.join(self.path, 'txn.active'), diff --git a/src/borg/key.py b/src/borg/key.py index 6965ae737..b122b638c 100644 --- a/src/borg/key.py +++ b/src/borg/key.py @@ -22,6 +22,7 @@ from .helpers import bin_to_hex from .helpers import CompressionDecider2, CompressionSpec from .item import Key, EncryptedKey +from .platform import SaveFile PREFIX = b'\0' * 8 @@ -470,7 +471,7 @@ def load(self, target, passphrase): def save(self, target, passphrase): key_data = self._save(passphrase) - with open(target, 'w') as fd: + with SaveFile(target) as fd: fd.write('%s %s\n' % (self.FILE_ID, bin_to_hex(self.repository_id))) fd.write(key_data) fd.write('\n') diff --git a/src/borg/platform/__init__.py b/src/borg/platform/__init__.py index 29a97fc43..e1772c5e1 100644 --- a/src/borg/platform/__init__.py +++ b/src/borg/platform/__init__.py @@ -8,7 +8,7 @@ from .base import acl_get, acl_set from .base import set_flags, get_flags -from .base import SyncFile, sync_dir, fdatasync +from .base import SaveFile, SyncFile, sync_dir, fdatasync from .base import swidth, API_VERSION if sys.platform.startswith('linux'): # pragma: linux only diff --git a/src/borg/platform/base.py b/src/borg/platform/base.py index ef8853e31..da8d3bc0c 100644 --- a/src/borg/platform/base.py +++ b/src/borg/platform/base.py @@ -80,8 +80,9 @@ class SyncFile: TODO: A Windows implementation should use CreateFile with FILE_FLAG_WRITE_THROUGH. """ - def __init__(self, path): - self.fd = open(path, 'xb') + def __init__(self, path, binary=False): + mode = 'xb' if binary else 'x' + self.fd = open(path, mode) self.fileno = self.fd.fileno() def __enter__(self): @@ -112,6 +113,43 @@ def close(self): platform.sync_dir(os.path.dirname(self.fd.name)) +class SaveFile: + """ + Update file contents atomically. + + Must be used as a context manager (defining the scope of the transaction). + + On a journaling file system the file contents are always updated + atomically and won't become corrupted, even on power failures or + crashes (for caveats see SyncFile). + """ + + SUFFIX = '.tmp' + + def __init__(self, path, binary=False): + self.binary = binary + self.path = path + self.tmppath = self.path + self.SUFFIX + + def __enter__(self): + from .. import platform + try: + os.unlink(self.tmppath) + except FileNotFoundError: + pass + self.fd = platform.SyncFile(self.tmppath, self.binary) + return self.fd + + def __exit__(self, exc_type, exc_val, exc_tb): + from .. import platform + self.fd.close() + if exc_type is not None: + os.unlink(self.tmppath) + return + os.replace(self.tmppath, self.path) + platform.sync_dir(os.path.dirname(self.path)) + + def swidth(s): """terminal output width of string diff --git a/src/borg/platform/linux.pyx b/src/borg/platform/linux.pyx index 4bbdcc356..d35b28ac2 100644 --- a/src/borg/platform/linux.pyx +++ b/src/borg/platform/linux.pyx @@ -228,8 +228,8 @@ class SyncFile(BaseSyncFile): disk in the immediate future. """ - def __init__(self, path): - super().__init__(path) + def __init__(self, path, binary=False): + super().__init__(path, binary) self.offset = 0 self.write_window = (16 * 1024 ** 2) & ~PAGE_MASK self.last_sync = 0 diff --git a/src/borg/repository.py b/src/borg/repository.py index 468a2efa5..72906dc24 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -23,7 +23,7 @@ from .helpers import bin_to_hex from .locking import UpgradableLock, LockError, LockErrorT from .lrucache import LRUCache -from .platform import SyncFile, sync_dir +from .platform import SaveFile, SyncFile, sync_dir MAX_OBJECT_SIZE = 20 * 1024 * 1024 MAGIC = b'BORG_SEG' @@ -171,7 +171,7 @@ def create(self, path): def save_config(self, path, config): config_path = os.path.join(path, 'config') - with open(config_path, 'w') as fd: + with SaveFile(config_path) as fd: config.write(fd) def save_key(self, keydata): @@ -784,7 +784,7 @@ def get_write_fd(self, no_new=False, raise_full=False): if not os.path.exists(dirname): os.mkdir(dirname) sync_dir(os.path.join(self.path, 'data')) - self._write_fd = SyncFile(self.segment_filename(self.segment)) + self._write_fd = SyncFile(self.segment_filename(self.segment), binary=True) self._write_fd.write(MAGIC) self.offset = MAGIC_LEN return self._write_fd