1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2024-12-25 01:06:50 +00:00
borg/darc/helpers.py

357 lines
9.7 KiB
Python
Raw Normal View History

2010-10-15 18:46:17 +00:00
import argparse
from datetime import datetime, timedelta
2010-11-02 21:47:39 +00:00
from fnmatch import fnmatchcase
2011-08-11 19:18:13 +00:00
from operator import attrgetter
import grp
import msgpack
2010-10-31 19:12:32 +00:00
import os
import pwd
import re
2010-10-31 19:12:32 +00:00
import stat
import sys
import time
import urllib
2011-10-27 20:17:47 +00:00
class Manifest(object):
2013-06-03 11:45:48 +00:00
MANIFEST_ID = b'\0' * 32
def __init__(self):
self.archives = {}
self.config = {}
@classmethod
2013-06-20 10:44:58 +00:00
def load(cls, repository):
from .key import key_factory
manifest = cls()
2013-06-20 10:44:58 +00:00
manifest.repository = repository
cdata = repository.get(manifest.MANIFEST_ID)
manifest.key = key = key_factory(repository, cdata)
data = key.decrypt(None, cdata)
manifest.id = key.id_hash(data)
m = msgpack.unpackb(data)
2013-06-03 11:45:48 +00:00
if not m.get(b'version') == 1:
raise ValueError('Invalid manifest version')
2013-06-03 11:45:48 +00:00
manifest.archives = dict((k.decode('utf-8'), v) for k,v in m[b'archives'].items())
manifest.config = m[b'config']
return manifest, key
def write(self):
data = msgpack.packb({
'version': 1,
'archives': self.archives,
'config': self.config,
})
self.id = self.key.id_hash(data)
2013-06-20 10:44:58 +00:00
self.repository.put(self.MANIFEST_ID, self.key.encrypt(data))
2011-11-22 20:47:17 +00:00
def prune_split(archives, pattern, n, skip=[]):
2011-08-12 06:49:01 +00:00
items = {}
keep = []
for a in archives:
key = to_localtime(a.ts).strftime(pattern)
items.setdefault(key, [])
items[key].append(a)
for key, values in sorted(items.items(), reverse=True):
if n and values[0] not in skip:
values.sort(key=attrgetter('ts'), reverse=True)
2011-08-12 06:49:01 +00:00
keep.append(values[0])
n -= 1
return keep
2011-08-11 19:18:13 +00:00
class Statistics(object):
def __init__(self):
self.osize = self.csize = self.usize = self.nfiles = 0
def update(self, size, csize, unique):
self.osize += size
self.csize += csize
if unique:
self.usize += csize
def print_(self):
2013-06-03 11:45:48 +00:00
print('Number of files: %d' % self.nfiles)
print('Original size: %d (%s)' % (self.osize, format_file_size(self.osize)))
print('Compressed size: %s (%s)' % (self.csize, format_file_size(self.csize)))
print('Unique data: %d (%s)' % (self.usize, format_file_size(self.usize)))
2010-12-19 11:46:42 +00:00
2011-08-06 11:01:58 +00:00
def get_keys_dir():
2013-06-20 10:44:58 +00:00
"""Determine where to repository keys and cache"""
2011-08-06 11:01:58 +00:00
return os.environ.get('DARC_KEYS_DIR',
os.path.join(os.path.expanduser('~'), '.darc', 'keys'))
2011-08-06 11:01:58 +00:00
def get_cache_dir():
2013-06-20 10:44:58 +00:00
"""Determine where to repository keys and cache"""
2011-08-06 11:01:58 +00:00
return os.environ.get('DARC_CACHE_DIR',
2013-06-26 11:55:41 +00:00
os.path.join(os.path.expanduser('~'), '.cache', 'darc'))
2011-08-06 11:01:58 +00:00
def to_localtime(ts):
"""Convert datetime object from UTC to local time zone"""
return ts - timedelta(seconds=time.altzone)
2010-12-19 11:46:42 +00:00
def adjust_patterns(patterns):
2012-10-17 10:27:45 +00:00
if patterns and not isinstance(patterns[-1], ExcludePattern):
patterns.append(ExcludePattern('*'))
2010-11-02 21:47:39 +00:00
def exclude_path(path, patterns):
"""Used by create and extract sub-commands to determine
if an item should be processed or not
"""
for pattern in (patterns or []):
if pattern.match(path):
return isinstance(pattern, ExcludePattern)
return False
class IncludePattern(object):
"""--include PATTERN
"""
def __init__(self, pattern):
self.pattern = self.dirpattern = pattern
if not pattern.endswith(os.path.sep):
self.dirpattern += os.path.sep
def match(self, path):
dir, name = os.path.split(path)
2010-11-23 11:41:04 +00:00
return (path == self.pattern
or (dir + os.path.sep).startswith(self.dirpattern)
or fnmatchcase(name, self.pattern))
2010-11-02 21:47:39 +00:00
def __repr__(self):
return '%s(%s)' % (type(self), self.pattern)
class ExcludePattern(IncludePattern):
"""
"""
def walk_path(path, skip_inodes=None):
2010-10-31 19:12:32 +00:00
st = os.lstat(path)
if skip_inodes and (st.st_ino, st.st_dev) in skip_inodes:
return
2010-10-31 19:12:32 +00:00
yield path, st
if stat.S_ISDIR(st.st_mode):
for f in os.listdir(path):
for x in walk_path(os.path.join(path, f), skip_inodes):
2010-10-31 19:12:32 +00:00
yield x
2010-10-15 18:46:17 +00:00
2010-10-27 17:30:21 +00:00
def format_time(t):
"""Format datetime suitable for fixed length list output
"""
if (datetime.now() - t).days < 365:
return t.strftime('%b %d %H:%M')
else:
return t.strftime('%b %d %Y')
2010-10-27 17:30:21 +00:00
def format_timedelta(td):
"""Format timedelta in a human friendly format
"""
# Since td.total_seconds() requires python 2.7
2011-10-31 20:18:28 +00:00
ts = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / float(10 ** 6)
s = ts % 60
m = int(ts / 60) % 60
h = int(ts / 3600) % 24
txt = '%.2f seconds' % s
if m:
txt = '%d minutes %s' % (m, txt)
if h:
txt = '%d hours %s' % (h, txt)
if td.days:
txt = '%d days %s' % (td.days, txt)
return txt
2010-10-27 17:30:21 +00:00
def format_file_mode(mod):
"""Format file mode bits for list output
2010-10-23 19:38:42 +00:00
"""
2010-10-27 17:30:21 +00:00
def x(v):
return ''.join(v & m and s or '-'
for m, s in ((4, 'r'), (2, 'w'), (1, 'x')))
2013-06-03 11:45:48 +00:00
return '%s%s%s' % (x(mod // 64), x(mod // 8), x(mod))
2010-10-27 17:30:21 +00:00
2011-10-31 20:18:28 +00:00
2010-10-27 17:30:21 +00:00
def format_file_size(v):
"""Format file size into a human friendly format
2010-10-23 19:38:42 +00:00
"""
2010-10-27 17:30:21 +00:00
if v > 1024 * 1024 * 1024:
return '%.2f GB' % (v / 1024. / 1024. / 1024.)
elif v > 1024 * 1024:
return '%.2f MB' % (v / 1024. / 1024.)
elif v > 1024:
return '%.2f kB' % (v / 1024.)
else:
return '%d B' % v
2010-10-23 19:38:42 +00:00
2011-10-31 20:18:28 +00:00
2010-10-27 17:30:21 +00:00
class IntegrityError(Exception):
"""
"""
2010-10-23 19:38:42 +00:00
2011-10-31 20:18:28 +00:00
def memoize(function):
cache = {}
2011-10-31 20:18:28 +00:00
def decorated_function(*args):
try:
return cache[args]
except KeyError:
val = function(*args)
cache[args] = val
return val
return decorated_function
2010-10-15 18:46:17 +00:00
2011-10-31 20:18:28 +00:00
@memoize
def uid2user(uid):
try:
return pwd.getpwuid(uid).pw_name
except KeyError:
return None
2011-10-31 20:18:28 +00:00
@memoize
def user2uid(user):
try:
2012-03-03 13:02:22 +00:00
return user and pwd.getpwnam(user).pw_uid
except KeyError:
return None
2011-10-31 20:18:28 +00:00
@memoize
def gid2group(gid):
try:
return grp.getgrgid(gid).gr_name
except KeyError:
return None
2010-10-19 19:12:12 +00:00
2011-10-31 20:18:28 +00:00
@memoize
def group2gid(group):
try:
2012-03-03 13:02:22 +00:00
return group and grp.getgrnam(group).gr_gid
except KeyError:
return None
2010-10-27 17:30:21 +00:00
class Location:
2013-06-20 10:44:58 +00:00
"""Object representing a repository / archive location
"""
proto = user = host = port = path = archive = None
ssh_re = re.compile(r'(?P<proto>ssh)://(?:(?P<user>[^@]+)@)?'
r'(?P<host>[^:/#]+)(?::(?P<port>\d+))?'
r'(?P<path>[^:]+)(?:::(?P<archive>.+))?')
file_re = re.compile(r'(?P<proto>file)://'
r'(?P<path>[^:]+)(?:::(?P<archive>.+))?')
scp_re = re.compile(r'((?:(?P<user>[^@]+)@)?(?P<host>[^:/]+):)?'
r'(?P<path>[^:]+)(?:::(?P<archive>.+))?')
2010-10-15 18:46:17 +00:00
def __init__(self, text):
2011-10-27 20:17:47 +00:00
self.orig = text
if not self.parse(text):
2010-10-15 18:46:17 +00:00
raise ValueError
def parse(self, text):
m = self.ssh_re.match(text)
if m:
self.proto = m.group('proto')
self.user = m.group('user')
self.host = m.group('host')
self.port = m.group('port') and int(m.group('port')) or 22
self.path = m.group('path')
self.archive = m.group('archive')
return True
m = self.file_re.match(text)
if m:
self.proto = m.group('proto')
self.path = m.group('path')
self.archive = m.group('archive')
return True
m = self.scp_re.match(text)
if m:
self.user = m.group('user')
self.host = m.group('host')
self.path = m.group('path')
self.archive = m.group('archive')
self.proto = self.host and 'ssh' or 'file'
if self.proto == 'ssh':
self.port = 22
return True
return False
2010-10-15 18:46:17 +00:00
def __str__(self):
items = []
items.append('proto=%r' % self.proto)
items.append('user=%r' % self.user)
items.append('host=%r' % self.host)
items.append('port=%r' % self.port)
2012-07-31 12:43:32 +00:00
items.append('path=%r' % self.path)
items.append('archive=%r' % self.archive)
return ', '.join(items)
2010-10-15 18:46:17 +00:00
2011-08-04 13:27:52 +00:00
def to_key_filename(self):
name = re.sub('[^\w]', '_', self.path).strip('_')
if self.proto != 'file':
name = self.host + '__' + name
2011-08-06 11:01:58 +00:00
return os.path.join(get_keys_dir(), name)
2011-08-04 13:27:52 +00:00
2010-10-15 18:46:17 +00:00
def __repr__(self):
return "Location(%s)" % self
2010-10-15 18:46:17 +00:00
def location_validator(archive=None):
def validator(text):
try:
loc = Location(text)
except ValueError:
raise argparse.ArgumentTypeError('Invalid location format: "%s"' % text)
if archive is True and not loc.archive:
raise argparse.ArgumentTypeError('"%s": No archive specified' % text)
elif archive is False and loc.archive:
raise argparse.ArgumentTypeError('"%s" No archive can be specified' % text)
return loc
return validator
def read_msgpack(filename):
with open(filename, 'rb') as fd:
return msgpack.unpack(fd)
2012-07-31 12:43:32 +00:00
def write_msgpack(filename, d):
2012-07-31 12:43:32 +00:00
with open(filename + '.tmp', 'wb') as fd:
msgpack.pack(d, fd)
fd.flush()
os.fsync(fd)
2012-07-31 12:43:32 +00:00
os.rename(filename + '.tmp', filename)
2013-06-03 11:45:48 +00:00
def decode_dict(d, keys, encoding='utf-8', errors='surrogateescape'):
for key in keys:
if isinstance(d.get(key), bytes):
d[key] = d[key].decode(encoding, errors)
return d
def remove_surrogates(s, errors='replace'):
2013-06-15 18:56:27 +00:00
return s.encode('utf-8', errors).decode('utf-8')
if sys.version < '3.3':
def st_mtime_ns(st):
return int(st.st_mtime * 10**9)
else:
def st_mtime_ns(st):
return st.st_mtime_ns