1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2024-12-25 01:06:50 +00:00
borg/darc/archive.py

418 lines
15 KiB
Python
Raw Normal View History

from datetime import datetime, timedelta
2010-10-24 20:07:54 +00:00
from getpass import getuser
2013-06-03 11:45:48 +00:00
from itertools import zip_longest
2010-10-23 19:38:42 +00:00
import msgpack
2010-10-20 17:59:15 +00:00
import os
import socket
2010-10-20 17:59:15 +00:00
import stat
2010-10-20 19:08:46 +00:00
import sys
2011-09-10 15:19:02 +00:00
import time
2013-06-03 11:45:48 +00:00
from io import BytesIO
import xattr
2010-10-20 17:59:15 +00:00
2013-05-28 12:35:55 +00:00
from .chunker import chunkify
from .helpers import uid2user, user2uid, gid2group, group2gid, \
2013-06-15 18:56:27 +00:00
Statistics, decode_dict, st_mtime_ns
2010-10-20 17:59:15 +00:00
ITEMS_BUFFER = 1024 * 1024
CHUNK_MIN = 1024
WINDOW_SIZE = 0xfff
CHUNK_MASK = 0xffff
2010-10-20 17:59:15 +00:00
2013-06-03 11:45:48 +00:00
utime_supports_fd = os.utime in getattr(os, 'supports_fd', {})
2013-06-15 18:56:27 +00:00
has_mtime_ns = sys.version >= '3.3'
2013-06-03 11:45:48 +00:00
has_lchmod = hasattr(os, 'lchmod')
2010-10-27 17:30:21 +00:00
2010-10-20 17:59:15 +00:00
class ItemIter(object):
def __init__(self, unpacker, filter):
self.unpacker = iter(unpacker)
self.filter = filter
self.stack = []
self.peeks = 0
self._peek = None
self._peek_iter = None
global foo
foo = self
def __iter__(self):
return self
2013-06-03 11:45:48 +00:00
def __next__(self):
if self.stack:
item = self.stack.pop(0)
else:
self._peek = None
item = self.get_next()
2013-06-03 11:45:48 +00:00
self.peeks = max(0, self.peeks - len(item.get(b'chunks', [])))
return item
def get_next(self):
2013-06-22 11:33:21 +00:00
while True:
2013-06-03 11:45:48 +00:00
n = next(self.unpacker)
2013-06-22 11:33:21 +00:00
decode_dict(n, (b'path', b'source', b'user', b'group'))
if not self.filter or self.filter(n):
return n
def peek(self):
while True:
while not self._peek or not self._peek_iter:
if self.peeks > 100:
raise StopIteration
self._peek = self.get_next()
self.stack.append(self._peek)
2013-06-03 11:45:48 +00:00
if b'chunks' in self._peek:
self._peek_iter = iter(self._peek[b'chunks'])
else:
self._peek_iter = None
try:
2013-06-03 11:45:48 +00:00
item = next(self._peek_iter)
self.peeks += 1
return item
except StopIteration:
self._peek = None
2010-10-20 17:59:15 +00:00
class Archive(object):
2010-10-30 11:44:25 +00:00
class DoesNotExist(Exception):
pass
2011-09-10 15:19:02 +00:00
class AlreadyExists(Exception):
pass
2013-06-20 10:44:58 +00:00
def __init__(self, repository, key, manifest, name, cache=None, create=False,
2012-02-29 22:59:17 +00:00
checkpoint_interval=300, numeric_owner=False):
2013-06-03 11:45:48 +00:00
self.cwd = os.getcwd()
self.key = key
2013-06-20 10:44:58 +00:00
self.repository = repository
self.cache = cache
self.manifest = manifest
2013-06-03 11:45:48 +00:00
self.items = BytesIO()
self.items_ids = []
self.hard_links = {}
self.stats = Statistics()
2011-09-10 15:19:02 +00:00
self.name = name
self.checkpoint_interval = checkpoint_interval
2012-02-29 22:59:17 +00:00
self.numeric_owner = numeric_owner
2011-09-10 15:19:02 +00:00
if create:
if name in manifest.archives:
2012-12-09 22:06:33 +00:00
raise self.AlreadyExists(name)
2011-09-10 15:19:02 +00:00
self.last_checkpoint = time.time()
i = 0
while True:
self.checkpoint_name = '%s.checkpoint%s' % (name, i and ('.%d' % i) or '')
if not self.checkpoint_name in manifest.archives:
break
i += 1
else:
if name not in self.manifest.archives:
2012-12-09 22:06:33 +00:00
raise self.DoesNotExist(name)
info = self.manifest.archives[name]
2013-06-03 11:45:48 +00:00
self.load(info[b'id'])
def load(self, id):
self.id = id
2013-06-20 10:44:58 +00:00
data = self.key.decrypt(self.id, self.repository.get(self.id))
self.metadata = msgpack.unpackb(data)
2013-06-03 11:45:48 +00:00
if self.metadata[b'version'] != 1:
raise Exception('Unknown archive metadata version')
2013-06-03 11:45:48 +00:00
decode_dict(self.metadata, (b'name', b'hostname', b'username', b'time'))
self.metadata[b'cmdline'] = [arg.decode('utf-8', 'surrogateescape') for arg in self.metadata[b'cmdline']]
self.name = self.metadata[b'name']
@property
def ts(self):
"""Timestamp of archive creation in UTC"""
2013-06-03 11:45:48 +00:00
t, f = self.metadata[b'time'].split('.', 1)
2011-07-01 20:12:21 +00:00
return datetime.strptime(t, '%Y-%m-%dT%H:%M:%S') + timedelta(seconds=float('.' + f))
2011-08-11 19:18:13 +00:00
def __repr__(self):
return 'Archive(%r)' % self.name
def iter_items(self, filter=None):
unpacker = msgpack.Unpacker(use_list=False)
i = 0
n = 20
while True:
2013-06-03 11:45:48 +00:00
items = self.metadata[b'items'][i:i + n]
i += n
if not items:
break
2013-06-20 10:44:58 +00:00
for id, chunk in [(id, chunk) for id, chunk in zip_longest(items, self.repository.get_many(items))]:
unpacker.feed(self.key.decrypt(id, chunk))
iter = ItemIter(unpacker, filter)
for item in iter:
yield item, iter.peek
def add_item(self, item):
2013-06-03 11:45:48 +00:00
self.items.write(msgpack.packb(item, unicode_errors='surrogateescape'))
2011-09-10 15:19:02 +00:00
now = time.time()
if now - self.last_checkpoint > self.checkpoint_interval:
self.last_checkpoint = now
self.write_checkpoint()
if self.items.tell() > ITEMS_BUFFER:
self.flush_items()
def flush_items(self, flush=False):
if self.items.tell() == 0:
return
self.items.seek(0)
2013-06-03 11:45:48 +00:00
chunks = list(bytes(s) for s in chunkify(self.items, WINDOW_SIZE, CHUNK_MASK, CHUNK_MIN, self.key.chunk_seed))
self.items.seek(0)
self.items.truncate()
for chunk in chunks[:-1]:
id, _, _ = self.cache.add_chunk(self.key.id_hash(chunk), chunk, self.stats)
self.items_ids.append(id)
if flush or len(chunks) == 1:
id, _, _ = self.cache.add_chunk(self.key.id_hash(chunks[-1]), chunks[-1], self.stats)
self.items_ids.append(id)
else:
self.items.write(chunks[-1])
2011-09-10 15:19:02 +00:00
def write_checkpoint(self):
self.save(self.checkpoint_name)
del self.manifest.archives[self.checkpoint_name]
self.cache.chunk_decref(self.id)
def save(self, name=None):
name = name or self.name
if name in self.manifest.archives:
2011-09-10 15:19:02 +00:00
raise self.AlreadyExists(name)
self.flush_items(flush=True)
metadata = {
2010-10-20 19:08:46 +00:00
'version': 1,
2010-10-20 17:59:15 +00:00
'name': name,
'items': self.items_ids,
'cmdline': sys.argv,
'hostname': socket.gethostname(),
2010-10-24 20:07:54 +00:00
'username': getuser(),
'time': datetime.utcnow().isoformat(),
2010-10-20 17:59:15 +00:00
}
2013-06-03 11:45:48 +00:00
data = msgpack.packb(metadata, unicode_errors='surrogateescape')
self.id = self.key.id_hash(data)
2011-09-10 15:19:02 +00:00
self.cache.add_chunk(self.id, data, self.stats)
self.manifest.archives[name] = {'id': self.id, 'time': metadata['time']}
self.manifest.write()
2013-06-20 10:44:58 +00:00
self.repository.commit()
2011-09-10 15:19:02 +00:00
self.cache.commit()
2010-10-20 17:59:15 +00:00
def calc_stats(self, cache):
def add(id):
count, size, csize = self.cache.chunks[id]
stats.update(size, csize, count == 1)
self.cache.chunks[id] = count - 1, size, csize
# This function is a bit evil since it abuses the cache to calculate
# the stats. The cache transaction must be rolled back afterwards
unpacker = msgpack.Unpacker(use_list=False)
cache.begin_txn()
stats = Statistics()
add(self.id)
2013-06-20 10:44:58 +00:00
for id, chunk in zip_longest(self.metadata[b'items'], self.repository.get_many(self.metadata[b'items'])):
2012-11-29 19:38:03 +00:00
add(id)
unpacker.feed(self.key.decrypt(id, chunk))
for item in unpacker:
try:
2013-06-03 11:45:48 +00:00
for id, size, csize in item[b'chunks']:
add(id)
stats.nfiles += 1
except KeyError:
pass
cache.rollback()
return stats
2010-10-20 17:59:15 +00:00
def extract_item(self, item, dest=None, restore_attrs=True, peek=None):
dest = dest or self.cwd
2013-06-03 11:45:48 +00:00
assert item[b'path'][:1] not in ('/', '\\', ':')
path = os.path.join(dest, item[b'path'])
# Attempt to remove existing files, ignore errors on failure
try:
st = os.lstat(path)
if stat.S_ISDIR(st.st_mode):
os.rmdir(path)
else:
os.unlink(path)
except OSError:
pass
2013-06-03 11:45:48 +00:00
mode = item[b'mode']
2010-10-30 11:44:25 +00:00
if stat.S_ISDIR(mode):
if not os.path.exists(path):
os.makedirs(path)
if restore_attrs:
self.restore_attrs(path, item)
2010-10-30 11:44:25 +00:00
elif stat.S_ISREG(mode):
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
# Hard link?
2013-06-03 11:45:48 +00:00
if b'source' in item:
source = os.path.join(dest, item[b'source'])
if os.path.exists(path):
os.unlink(path)
os.link(source, path)
2010-10-30 11:44:25 +00:00
else:
2013-06-03 11:45:48 +00:00
with open(path, 'wb') as fd:
ids = [id for id, size, csize in item[b'chunks']]
2013-06-20 10:44:58 +00:00
for id, chunk in zip_longest(ids, self.repository.get_many(ids, peek)):
data = self.key.decrypt(id, chunk)
fd.write(data)
2013-06-03 11:45:48 +00:00
self.restore_attrs(path, item, fd=fd.fileno())
elif stat.S_ISFIFO(mode):
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
os.mkfifo(path)
self.restore_attrs(path, item)
elif stat.S_ISLNK(mode):
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
2013-06-03 11:45:48 +00:00
source = item[b'source']
if os.path.exists(path):
os.unlink(path)
os.symlink(source, path)
self.restore_attrs(path, item, symlink=True)
elif stat.S_ISCHR(mode) or stat.S_ISBLK(mode):
2013-06-03 11:45:48 +00:00
os.mknod(path, item[b'mode'], item[b'rdev'])
self.restore_attrs(path, item)
2010-10-30 11:44:25 +00:00
else:
2013-06-03 11:45:48 +00:00
raise Exception('Unknown archive item type %r' % item[b'mode'])
2013-06-03 11:45:48 +00:00
def restore_attrs(self, path, item, symlink=False, fd=None):
xattrs = item.get(b'xattrs')
2010-10-31 20:55:09 +00:00
if xattrs:
2010-11-01 22:06:56 +00:00
for k, v in xattrs.items():
try:
2013-06-03 11:45:48 +00:00
xattr.set(fd or path, k, v)
except (EnvironmentError):
2010-11-01 22:06:56 +00:00
pass
2012-02-29 22:59:17 +00:00
uid = gid = None
if not self.numeric_owner:
2013-06-03 11:45:48 +00:00
uid = user2uid(item[b'user'])
gid = group2gid(item[b'group'])
uid = uid or item[b'uid']
gid = gid or item[b'gid']
# This code is a bit of a mess due to os specific differences
try:
2013-06-03 11:45:48 +00:00
if fd:
os.fchown(fd, uid, gid)
else:
os.lchown(path, uid, gid)
except OSError:
pass
2013-06-03 11:45:48 +00:00
if fd:
os.fchmod(fd, item[b'mode'])
elif not symlink:
os.chmod(path, item[b'mode'])
elif has_lchmod: # Not available on Linux
os.lchmod(path, item[b'mode'])
if fd and utime_supports_fd: # Python >= 3.3
2013-06-15 18:56:27 +00:00
os.utime(fd, None, ns=(item[b'mtime'], item[b'mtime']))
2013-06-03 11:45:48 +00:00
elif utime_supports_fd: # Python >= 3.3
2013-06-15 18:56:27 +00:00
os.utime(path, None, ns=(item[b'mtime'], item[b'mtime']), follow_symlinks=False)
elif not symlink:
2013-06-15 18:56:27 +00:00
os.utime(path, (item[b'mtime'] / 10**9, item[b'mtime'] / 10**9))
2010-10-20 17:59:15 +00:00
def verify_file(self, item, start, result, peek=None):
2013-06-03 11:45:48 +00:00
if not item[b'chunks']:
2011-07-17 21:53:23 +00:00
start(item)
result(item, True)
else:
start(item)
2013-06-03 11:45:48 +00:00
ids = [id for id, size, csize in item[b'chunks']]
try:
2013-06-20 10:44:58 +00:00
for id, chunk in zip_longest(ids, self.repository.get_many(ids, peek)):
self.key.decrypt(id, chunk)
2012-11-30 20:47:35 +00:00
except Exception:
result(item, False)
return
result(item, True)
2010-10-20 17:59:15 +00:00
def delete(self, cache):
unpacker = msgpack.Unpacker(use_list=False)
2013-06-03 11:45:48 +00:00
for id in self.metadata[b'items']:
2013-06-20 10:44:58 +00:00
unpacker.feed(self.key.decrypt(id, self.repository.get(id)))
for item in unpacker:
try:
2013-06-03 11:45:48 +00:00
for chunk_id, size, csize in item[b'chunks']:
self.cache.chunk_decref(chunk_id)
except KeyError:
pass
self.cache.chunk_decref(id)
self.cache.chunk_decref(self.id)
del self.manifest.archives[self.name]
self.manifest.write()
2013-06-20 10:44:58 +00:00
self.repository.commit()
2010-12-21 20:29:09 +00:00
cache.commit()
2010-10-20 17:59:15 +00:00
2010-10-31 19:31:56 +00:00
def stat_attrs(self, st, path):
2010-10-31 20:55:09 +00:00
item = {
2013-06-03 11:45:48 +00:00
b'mode': st.st_mode,
b'uid': st.st_uid, b'user': uid2user(st.st_uid),
b'gid': st.st_gid, b'group': gid2group(st.st_gid),
2013-06-15 18:56:27 +00:00
b'mtime': st_mtime_ns(st),
2010-10-30 11:44:25 +00:00
}
2012-02-29 22:59:17 +00:00
if self.numeric_owner:
2013-06-03 11:45:48 +00:00
item[b'user'] = item[b'group'] = None
2010-10-31 20:55:09 +00:00
try:
2013-06-03 11:45:48 +00:00
xattrs = xattr.get_all(path, True)
2010-11-01 22:06:56 +00:00
if xattrs:
2013-06-03 11:45:48 +00:00
item[b'xattrs'] = dict(xattrs)
except EnvironmentError:
2010-10-31 20:55:09 +00:00
pass
return item
2010-10-30 11:44:25 +00:00
def process_item(self, path, st):
2013-06-03 11:45:48 +00:00
item = {b'path': path.lstrip('/\\:')}
2010-10-31 19:31:56 +00:00
item.update(self.stat_attrs(st, path))
self.add_item(item)
2010-10-31 19:12:32 +00:00
2012-03-03 13:02:22 +00:00
def process_dev(self, path, st):
2013-06-03 11:45:48 +00:00
item = {b'path': path.lstrip('/\\:'), b'rdev': st.st_rdev}
2012-03-03 13:02:22 +00:00
item.update(self.stat_attrs(st, path))
self.add_item(item)
def process_symlink(self, path, st):
2010-10-20 17:59:15 +00:00
source = os.readlink(path)
2013-06-03 11:45:48 +00:00
item = {b'path': path.lstrip('/\\:'), b'source': source}
2010-10-31 19:31:56 +00:00
item.update(self.stat_attrs(st, path))
self.add_item(item)
2010-10-30 11:44:25 +00:00
def process_file(self, path, st, cache):
safe_path = path.lstrip('/\\:')
# Is it a hard link?
if st.st_nlink > 1:
source = self.hard_links.get((st.st_ino, st.st_dev))
if (st.st_ino, st.st_dev) in self.hard_links:
item = self.stat_attrs(st, path)
2013-06-03 11:45:48 +00:00
item.update({b'path': safe_path, b'source': source})
self.add_item(item)
return
else:
self.hard_links[st.st_ino, st.st_dev] = safe_path
path_hash = self.key.id_hash(os.path.join(self.cwd, path).encode('utf-8', 'surrogateescape'))
ids = cache.file_known_and_unchanged(path_hash, st)
chunks = None
if ids is not None:
# Make sure all ids are available
for id in ids:
if not cache.seen_chunk(id):
break
else:
chunks = [cache.chunk_incref(id, self.stats) for id in ids]
# Only chunkify the file if needed
if chunks is None:
2010-10-30 11:44:25 +00:00
with open(path, 'rb') as fd:
chunks = []
for chunk in chunkify(fd, WINDOW_SIZE, CHUNK_MASK, CHUNK_MIN, self.key.chunk_seed):
chunks.append(cache.add_chunk(self.key.id_hash(chunk), chunk, self.stats))
ids = [id for id, _, _ in chunks]
2010-12-21 20:29:09 +00:00
cache.memorize_file(path_hash, st, ids)
2013-06-03 11:45:48 +00:00
item = {b'path': safe_path, b'chunks': chunks}
2010-10-31 19:31:56 +00:00
item.update(self.stat_attrs(st, path))
self.stats.nfiles += 1
self.add_item(item)
2010-10-20 17:59:15 +00:00
@staticmethod
2013-06-20 10:44:58 +00:00
def list_archives(repository, key, manifest, cache=None):
for name, info in manifest.archives.items():
2013-06-20 10:44:58 +00:00
yield Archive(repository, key, manifest, name, cache=cache)