mirror of https://github.com/borgbackup/borg.git
416 lines
18 KiB
Python
416 lines
18 KiB
Python
import argparse
|
|
from datetime import datetime
|
|
from operator import attrgetter
|
|
import os
|
|
import stat
|
|
import sys
|
|
|
|
from .archive import Archive
|
|
from .store import Store
|
|
from .cache import Cache
|
|
from .key import key_creator
|
|
from .helpers import location_validator, format_time, \
|
|
format_file_mode, IncludePattern, ExcludePattern, exclude_path, adjust_patterns, to_localtime, \
|
|
get_cache_dir, format_timedelta, prune_split, Manifest, Location
|
|
from .remote import StoreServer, RemoteStore
|
|
|
|
|
|
class Archiver(object):
|
|
|
|
def __init__(self):
|
|
self.exit_code = 0
|
|
|
|
def open_store(self, location, create=False):
|
|
if location.proto == 'ssh':
|
|
store = RemoteStore(location, create=create)
|
|
else:
|
|
store = Store(location.path, create=create)
|
|
store._location = location
|
|
return store
|
|
|
|
def print_error(self, msg, *args):
|
|
msg = args and msg % args or msg
|
|
self.exit_code = 1
|
|
print >> sys.stderr, msg
|
|
|
|
def print_verbose(self, msg, *args, **kw):
|
|
if self.verbose:
|
|
msg = args and msg % args or msg
|
|
if kw.get('newline', True):
|
|
print msg
|
|
else:
|
|
print msg,
|
|
|
|
def do_serve(self, args):
|
|
return StoreServer().serve()
|
|
|
|
def do_init(self, args):
|
|
print 'Initializing store "%s"' % args.store.orig
|
|
store = self.open_store(args.store, create=True)
|
|
key = key_creator(store, args)
|
|
manifest = Manifest()
|
|
manifest.store = store
|
|
manifest.key = key
|
|
manifest.write()
|
|
store.commit()
|
|
return self.exit_code
|
|
|
|
def do_change_passphrase(self, args):
|
|
store = self.open_store(Location(args.store))
|
|
manifest, key = Manifest.load(store)
|
|
key.change_passphrase()
|
|
return self.exit_code
|
|
|
|
def do_create(self, args):
|
|
t0 = datetime.now()
|
|
store = self.open_store(args.archive)
|
|
manifest, key = Manifest.load(store)
|
|
cache = Cache(store, key, manifest)
|
|
archive = Archive(store, key, manifest, args.archive.archive, cache=cache,
|
|
create=True, checkpoint_interval=args.checkpoint_interval,
|
|
numeric_owner=args.numeric_owner)
|
|
# Add darc cache dir to inode_skip list
|
|
skip_inodes = set()
|
|
try:
|
|
st = os.stat(get_cache_dir())
|
|
skip_inodes.add((st.st_ino, st.st_dev))
|
|
except IOError:
|
|
pass
|
|
# Add local store dir to inode_skip list
|
|
if not args.archive.host:
|
|
try:
|
|
st = os.stat(args.archive.path)
|
|
skip_inodes.add((st.st_ino, st.st_dev))
|
|
except IOError:
|
|
pass
|
|
for path in args.paths:
|
|
if args.dontcross:
|
|
try:
|
|
restrict_dev = os.lstat(path).st_dev
|
|
except OSError, e:
|
|
self.print_error('%s: %s', path, e)
|
|
continue
|
|
else:
|
|
restrict_dev = None
|
|
self._process(archive, cache, args.patterns, skip_inodes, path, restrict_dev)
|
|
archive.save()
|
|
if args.stats:
|
|
t = datetime.now()
|
|
diff = t - t0
|
|
print '-' * 40
|
|
print 'Archive name: %s' % args.archive.archive
|
|
print 'Archive fingerprint: %s' % archive.id.encode('hex')
|
|
print 'Start time: %s' % t0.strftime('%c')
|
|
print 'End time: %s' % t.strftime('%c')
|
|
print 'Duration: %s' % format_timedelta(diff)
|
|
archive.stats.print_()
|
|
print '-' * 40
|
|
return self.exit_code
|
|
|
|
def _process(self, archive, cache, patterns, skip_inodes, path, restrict_dev):
|
|
if exclude_path(path, patterns):
|
|
return
|
|
try:
|
|
st = os.lstat(path)
|
|
except OSError, e:
|
|
self.print_error('%s: %s', path, e)
|
|
return
|
|
if (st.st_ino, st.st_dev) in skip_inodes:
|
|
return
|
|
# Entering a new filesystem?
|
|
if restrict_dev and st.st_dev != restrict_dev:
|
|
return
|
|
# Ignore unix sockets
|
|
if stat.S_ISSOCK(st.st_mode):
|
|
return
|
|
self.print_verbose(path)
|
|
if stat.S_ISREG(st.st_mode):
|
|
try:
|
|
archive.process_file(path, st, cache)
|
|
except IOError, e:
|
|
self.print_error('%s: %s', path, e)
|
|
elif stat.S_ISDIR(st.st_mode):
|
|
archive.process_item(path, st)
|
|
try:
|
|
entries = os.listdir(path)
|
|
except OSError, e:
|
|
self.print_error('%s: %s', path, e)
|
|
else:
|
|
for filename in sorted(entries):
|
|
self._process(archive, cache, patterns, skip_inodes,
|
|
os.path.join(path, filename), restrict_dev)
|
|
elif stat.S_ISLNK(st.st_mode):
|
|
archive.process_symlink(path, st)
|
|
elif stat.S_ISFIFO(st.st_mode):
|
|
archive.process_item(path, st)
|
|
elif stat.S_ISCHR(st.st_mode) or stat.S_ISBLK(st.st_mode):
|
|
archive.process_dev(path, st)
|
|
else:
|
|
self.print_error('Unknown file type: %s', path)
|
|
|
|
def do_extract(self, args):
|
|
def start_cb(item):
|
|
self.print_verbose(item['path'])
|
|
|
|
store = self.open_store(args.archive)
|
|
manifest, key = Manifest.load(store)
|
|
archive = Archive(store, key, manifest, args.archive.archive,
|
|
numeric_owner=args.numeric_owner)
|
|
dirs = []
|
|
for item, peek in archive.iter_items(lambda item: not exclude_path(item['path'], args.patterns)):
|
|
if stat.S_ISDIR(item['mode']):
|
|
dirs.append(item)
|
|
archive.extract_item(item, args.dest, start_cb, restore_attrs=False)
|
|
else:
|
|
archive.extract_item(item, args.dest, start_cb, peek=peek)
|
|
if dirs and not item['path'].startswith(dirs[-1]['path']):
|
|
archive.extract_item(dirs.pop(-1), args.dest)
|
|
while dirs:
|
|
archive.extract_item(dirs.pop(-1), args.dest)
|
|
return self.exit_code
|
|
|
|
def do_delete(self, args):
|
|
store = self.open_store(args.archive)
|
|
manifest, key = Manifest.load(store)
|
|
cache = Cache(store, key, manifest)
|
|
archive = Archive(store, key, manifest, args.archive.archive, cache=cache)
|
|
archive.delete(cache)
|
|
return self.exit_code
|
|
|
|
def do_list(self, args):
|
|
store = self.open_store(args.src)
|
|
manifest, key = Manifest.load(store)
|
|
if args.src.archive:
|
|
tmap = {1: 'p', 2: 'c', 4: 'd', 6: 'b', 010: '-', 012: 'l', 014: 's'}
|
|
archive = Archive(store, key, manifest, args.src.archive)
|
|
for item, _ in archive.iter_items():
|
|
type = tmap.get(item['mode'] / 4096, '?')
|
|
mode = format_file_mode(item['mode'])
|
|
size = 0
|
|
if type == '-':
|
|
try:
|
|
size = sum(size for _, size, _ in item['chunks'])
|
|
except KeyError:
|
|
pass
|
|
mtime = format_time(datetime.fromtimestamp(item['mtime']))
|
|
if 'source' in item:
|
|
if type == 'l':
|
|
extra = ' -> %s' % item['source']
|
|
else:
|
|
type = 'h'
|
|
extra = ' link to %s' % item['source']
|
|
else:
|
|
extra = ''
|
|
print '%s%s %-6s %-6s %8d %s %s%s' % (type, mode, item['user'] or item['uid'],
|
|
item['group'] or item['gid'], size, mtime,
|
|
item['path'], extra)
|
|
else:
|
|
for archive in sorted(Archive.list_archives(store, key, manifest), key=attrgetter('ts')):
|
|
print '%-20s %s' % (archive.metadata['name'], to_localtime(archive.ts).strftime('%c'))
|
|
return self.exit_code
|
|
|
|
def do_verify(self, args):
|
|
store = self.open_store(args.archive)
|
|
manifest, key = Manifest.load(store)
|
|
archive = Archive(store, key, manifest, args.archive.archive)
|
|
|
|
def start_cb(item):
|
|
self.print_verbose('%s ...', item['path'], newline=False)
|
|
|
|
def result_cb(item, success):
|
|
if success:
|
|
self.print_verbose('OK')
|
|
else:
|
|
self.print_verbose('ERROR')
|
|
self.print_error('%s: verification failed' % item['path'])
|
|
for item, peek in archive.iter_items(lambda item: not exclude_path(item['path'], args.patterns)):
|
|
if stat.S_ISREG(item['mode']) and 'chunks' in item:
|
|
archive.verify_file(item, start_cb, result_cb, peek=peek)
|
|
return self.exit_code
|
|
|
|
def do_info(self, args):
|
|
store = self.open_store(args.archive)
|
|
manifest, key = Manifest.load(store)
|
|
cache = Cache(store, key, manifest)
|
|
archive = Archive(store, key, manifest, args.archive.archive, cache=cache)
|
|
stats = archive.calc_stats(cache)
|
|
print 'Name:', archive.name
|
|
print 'Fingerprint: %s' % archive.id.encode('hex')
|
|
print 'Hostname:', archive.metadata['hostname']
|
|
print 'Username:', archive.metadata['username']
|
|
print 'Time:', to_localtime(archive.ts).strftime('%c')
|
|
print 'Command line:', ' '.join(archive.metadata['cmdline'])
|
|
stats.print_()
|
|
return self.exit_code
|
|
|
|
def do_prune(self, args):
|
|
store = self.open_store(args.store)
|
|
manifest, key = Manifest.load(store)
|
|
cache = Cache(store, key, manifest)
|
|
archives = list(sorted(Archive.list_archives(store, key, manifest, cache),
|
|
key=attrgetter('ts'), reverse=True))
|
|
if args.hourly + args.daily + args.weekly + args.monthly + args.yearly == 0:
|
|
self.print_error('At least one of the "hourly", "daily", "weekly", "monthly" or "yearly" '
|
|
'settings must be specified')
|
|
return 1
|
|
if args.prefix:
|
|
archives = [archive for archive in archives if archive.name.startswith(args.prefix)]
|
|
keep = []
|
|
if args.hourly:
|
|
keep += prune_split(archives, '%Y-%m-%d %H', args.hourly)
|
|
if args.daily:
|
|
keep += prune_split(archives, '%Y-%m-%d', args.daily, keep)
|
|
if args.weekly:
|
|
keep += prune_split(archives, '%Y-%V', args.weekly, keep)
|
|
if args.monthly:
|
|
keep += prune_split(archives, '%Y-%m', args.monthly, keep)
|
|
if args.yearly:
|
|
keep += prune_split(archives, '%Y', args.yearly, keep)
|
|
|
|
keep.sort(key=attrgetter('ts'), reverse=True)
|
|
to_delete = [a for a in archives if a not in keep]
|
|
|
|
for archive in keep:
|
|
self.print_verbose('Keeping archive "%s"' % archive.name)
|
|
for archive in to_delete:
|
|
self.print_verbose('Pruning archive "%s"', archive.name)
|
|
archive.delete(cache)
|
|
return self.exit_code
|
|
|
|
def run(self, args=None):
|
|
dot_path = os.path.join(os.path.expanduser('~'), '.darc')
|
|
if not os.path.exists(dot_path):
|
|
os.mkdir(dot_path)
|
|
os.mkdir(os.path.join(dot_path, 'keys'))
|
|
os.mkdir(os.path.join(dot_path, 'cache'))
|
|
common_parser = argparse.ArgumentParser(add_help=False)
|
|
common_parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
|
|
default=False,
|
|
help='Verbose output')
|
|
|
|
parser = argparse.ArgumentParser(description='DARC - Deduplicating Archiver')
|
|
subparsers = parser.add_subparsers(title='Available subcommands')
|
|
|
|
subparser = subparsers.add_parser('serve', parents=[common_parser])
|
|
subparser.set_defaults(func=self.do_serve)
|
|
|
|
subparser = subparsers.add_parser('init', parents=[common_parser])
|
|
subparser.set_defaults(func=self.do_init)
|
|
subparser.add_argument('store',
|
|
type=location_validator(archive=False),
|
|
help='Store to create')
|
|
subparser.add_argument('--key-file', dest='keyfile',
|
|
action='store_true', default=False,
|
|
help='Encrypt data using key file')
|
|
subparser.add_argument('--passphrase', dest='passphrase',
|
|
action='store_true', default=False,
|
|
help='Encrypt data using passphrase derived key')
|
|
|
|
subparser = subparsers.add_parser('change-passphrase', parents=[common_parser])
|
|
subparser.set_defaults(func=self.do_change_passphrase)
|
|
subparser.add_argument('store', type=location_validator(archive=False))
|
|
|
|
subparser = subparsers.add_parser('create', parents=[common_parser])
|
|
subparser.set_defaults(func=self.do_create)
|
|
subparser.add_argument('-s', '--stats', dest='stats',
|
|
action='store_true', default=False,
|
|
help='Print statistics for the created archive')
|
|
subparser.add_argument('-i', '--include', dest='patterns',
|
|
type=IncludePattern, action='append',
|
|
help='Include condition')
|
|
subparser.add_argument('-e', '--exclude', dest='patterns',
|
|
type=ExcludePattern, action='append',
|
|
help='Include condition')
|
|
subparser.add_argument('-c', '--checkpoint-interval', dest='checkpoint_interval',
|
|
type=int, default=300, metavar='SECONDS',
|
|
help='Write checkpointe ever SECONDS seconds (Default: 300)')
|
|
subparser.add_argument('--do-not-cross-mountpoints', dest='dontcross',
|
|
action='store_true', default=False,
|
|
help='Do not cross mount points')
|
|
subparser.add_argument('--numeric-owner', dest='numeric_owner',
|
|
action='store_true', default=False,
|
|
help='Only store numeric user and group identifiers')
|
|
subparser.add_argument('archive', metavar='ARCHIVE',
|
|
type=location_validator(archive=True),
|
|
help='Archive to create')
|
|
subparser.add_argument('paths', metavar='PATH', nargs='*', type=str,
|
|
default=['.'], help='Paths to add to archive')
|
|
|
|
subparser = subparsers.add_parser('extract', parents=[common_parser])
|
|
subparser.set_defaults(func=self.do_extract)
|
|
subparser.add_argument('-i', '--include', dest='patterns',
|
|
type=IncludePattern, action='append',
|
|
help='Include condition')
|
|
subparser.add_argument('-e', '--exclude', dest='patterns',
|
|
type=ExcludePattern, action='append',
|
|
help='Include condition')
|
|
subparser.add_argument('--numeric-owner', dest='numeric_owner',
|
|
action='store_true', default=False,
|
|
help='Only obey numeric user and group identifiers')
|
|
subparser.add_argument('archive', metavar='ARCHIVE',
|
|
type=location_validator(archive=True),
|
|
help='Archive to create')
|
|
subparser.add_argument('dest', metavar='DEST', type=str, nargs='?',
|
|
help='Where to extract files')
|
|
|
|
subparser = subparsers.add_parser('delete', parents=[common_parser])
|
|
subparser.set_defaults(func=self.do_delete)
|
|
subparser.add_argument('archive', metavar='ARCHIVE',
|
|
type=location_validator(archive=True),
|
|
help='Archive to delete')
|
|
|
|
subparser = subparsers.add_parser('list', parents=[common_parser])
|
|
subparser.set_defaults(func=self.do_list)
|
|
subparser.add_argument('src', metavar='SRC', type=location_validator(),
|
|
help='Store/Archive to list contents of')
|
|
|
|
subparser = subparsers.add_parser('verify', parents=[common_parser])
|
|
subparser.set_defaults(func=self.do_verify)
|
|
subparser.add_argument('-i', '--include', dest='patterns',
|
|
type=IncludePattern, action='append',
|
|
help='Include condition')
|
|
subparser.add_argument('-e', '--exclude', dest='patterns',
|
|
type=ExcludePattern, action='append',
|
|
help='Include condition')
|
|
subparser.add_argument('archive', metavar='ARCHIVE',
|
|
type=location_validator(archive=True),
|
|
help='Archive to verity integrity of')
|
|
|
|
subparser = subparsers.add_parser('info', parents=[common_parser])
|
|
subparser.set_defaults(func=self.do_info)
|
|
subparser.add_argument('archive', metavar='ARCHIVE',
|
|
type=location_validator(archive=True),
|
|
help='Archive to display information about')
|
|
|
|
subparser = subparsers.add_parser('prune', parents=[common_parser])
|
|
subparser.set_defaults(func=self.do_prune)
|
|
subparser.add_argument('-H', '--hourly', dest='hourly', type=int, default=0,
|
|
help='Number of hourly archives to keep')
|
|
subparser.add_argument('-d', '--daily', dest='daily', type=int, default=0,
|
|
help='Number of daily archives to keep')
|
|
subparser.add_argument('-w', '--weekly', dest='weekly', type=int, default=0,
|
|
help='Number of daily archives to keep')
|
|
subparser.add_argument('-m', '--monthly', dest='monthly', type=int, default=0,
|
|
help='Number of monthly archives to keep')
|
|
subparser.add_argument('-y', '--yearly', dest='yearly', type=int, default=0,
|
|
help='Number of yearly archives to keep')
|
|
subparser.add_argument('-p', '--prefix', dest='prefix', type=str,
|
|
help='Only consider archive names starting with this prefix')
|
|
subparser.add_argument('store', metavar='STORE',
|
|
type=location_validator(archive=False),
|
|
help='Store to prune')
|
|
|
|
args = parser.parse_args(args)
|
|
if getattr(args, 'patterns', None):
|
|
adjust_patterns(args.patterns)
|
|
self.verbose = args.verbose
|
|
return args.func(args)
|
|
|
|
|
|
def main():
|
|
archiver = Archiver()
|
|
sys.exit(archiver.run())
|
|
|
|
if __name__ == '__main__':
|
|
main()
|