1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2024-12-25 09:19:31 +00:00

Merge pull request #1377 from enkore/issue/1060

Use atomic file writing and updating for configuration and key files
This commit is contained in:
TW 2016-07-30 16:06:01 +02:00 committed by GitHub
commit 02e229b255
6 changed files with 53 additions and 13 deletions

View file

@ -19,6 +19,7 @@
from .item import Item
from .key import PlaintextKey
from .locking import UpgradableLock
from .platform import SaveFile
from .remote import cache_if_remote
ChunkListEntry = namedtuple('ChunkListEntry', 'id size csize')
@ -141,11 +142,11 @@ def create(self):
config.set('cache', 'version', '1')
config.set('cache', 'repository', self.repository.id_str)
config.set('cache', 'manifest', '')
with open(os.path.join(self.path, 'config'), 'w') as fd:
with SaveFile(os.path.join(self.path, 'config')) as fd:
config.write(fd)
ChunkIndex().write(os.path.join(self.path, 'chunks').encode('utf-8'))
os.makedirs(os.path.join(self.path, 'chunks.archive.d'))
with open(os.path.join(self.path, 'files'), 'wb') as fd:
with SaveFile(os.path.join(self.path, 'files'), binary=True) as fd:
pass # empty file
def _do_open(self):
@ -212,7 +213,7 @@ def commit(self):
if not self.txn_active:
return
if self.files is not None:
with open(os.path.join(self.path, 'files'), 'wb') as fd:
with SaveFile(os.path.join(self.path, 'files'), binary=True) as fd:
for path_hash, item in self.files.items():
# Discard cached files with the newest mtime to avoid
# issues with filesystem snapshots and mtime precision
@ -223,7 +224,7 @@ def commit(self):
self.config.set('cache', 'timestamp', self.manifest.timestamp)
self.config.set('cache', 'key_type', str(self.key.TYPE))
self.config.set('cache', 'previous_location', self.repository._location.canonical_path())
with open(os.path.join(self.path, 'config'), 'w') as fd:
with SaveFile(os.path.join(self.path, 'config')) as fd:
self.config.write(fd)
self.chunks.write(os.path.join(self.path, 'chunks').encode('utf-8'))
os.rename(os.path.join(self.path, 'txn.active'),

View file

@ -22,6 +22,7 @@
from .helpers import bin_to_hex
from .helpers import CompressionDecider2, CompressionSpec
from .item import Key, EncryptedKey
from .platform import SaveFile
PREFIX = b'\0' * 8
@ -470,7 +471,7 @@ def load(self, target, passphrase):
def save(self, target, passphrase):
key_data = self._save(passphrase)
with open(target, 'w') as fd:
with SaveFile(target) as fd:
fd.write('%s %s\n' % (self.FILE_ID, bin_to_hex(self.repository_id)))
fd.write(key_data)
fd.write('\n')

View file

@ -8,7 +8,7 @@
from .base import acl_get, acl_set
from .base import set_flags, get_flags
from .base import SyncFile, sync_dir, fdatasync
from .base import SaveFile, SyncFile, sync_dir, fdatasync
from .base import swidth, API_VERSION
if sys.platform.startswith('linux'): # pragma: linux only

View file

@ -80,8 +80,9 @@ class SyncFile:
TODO: A Windows implementation should use CreateFile with FILE_FLAG_WRITE_THROUGH.
"""
def __init__(self, path):
self.fd = open(path, 'xb')
def __init__(self, path, binary=False):
mode = 'xb' if binary else 'x'
self.fd = open(path, mode)
self.fileno = self.fd.fileno()
def __enter__(self):
@ -112,6 +113,43 @@ def close(self):
platform.sync_dir(os.path.dirname(self.fd.name))
class SaveFile:
"""
Update file contents atomically.
Must be used as a context manager (defining the scope of the transaction).
On a journaling file system the file contents are always updated
atomically and won't become corrupted, even on power failures or
crashes (for caveats see SyncFile).
"""
SUFFIX = '.tmp'
def __init__(self, path, binary=False):
self.binary = binary
self.path = path
self.tmppath = self.path + self.SUFFIX
def __enter__(self):
from .. import platform
try:
os.unlink(self.tmppath)
except FileNotFoundError:
pass
self.fd = platform.SyncFile(self.tmppath, self.binary)
return self.fd
def __exit__(self, exc_type, exc_val, exc_tb):
from .. import platform
self.fd.close()
if exc_type is not None:
os.unlink(self.tmppath)
return
os.replace(self.tmppath, self.path)
platform.sync_dir(os.path.dirname(self.path))
def swidth(s):
"""terminal output width of string <s>

View file

@ -228,8 +228,8 @@ class SyncFile(BaseSyncFile):
disk in the immediate future.
"""
def __init__(self, path):
super().__init__(path)
def __init__(self, path, binary=False):
super().__init__(path, binary)
self.offset = 0
self.write_window = (16 * 1024 ** 2) & ~PAGE_MASK
self.last_sync = 0

View file

@ -23,7 +23,7 @@
from .helpers import bin_to_hex
from .locking import UpgradableLock, LockError, LockErrorT
from .lrucache import LRUCache
from .platform import SyncFile, sync_dir
from .platform import SaveFile, SyncFile, sync_dir
MAX_OBJECT_SIZE = 20 * 1024 * 1024
MAGIC = b'BORG_SEG'
@ -171,7 +171,7 @@ def create(self, path):
def save_config(self, path, config):
config_path = os.path.join(path, 'config')
with open(config_path, 'w') as fd:
with SaveFile(config_path) as fd:
config.write(fd)
def save_key(self, keydata):
@ -784,7 +784,7 @@ def get_write_fd(self, no_new=False, raise_full=False):
if not os.path.exists(dirname):
os.mkdir(dirname)
sync_dir(os.path.join(self.path, 'data'))
self._write_fd = SyncFile(self.segment_filename(self.segment))
self._write_fd = SyncFile(self.segment_filename(self.segment), binary=True)
self._write_fd.write(MAGIC)
self.offset = MAGIC_LEN
return self._write_fd