diff --git a/scripts/msgpack2marshal.py b/scripts/msgpack2marshal.py new file mode 100755 index 000000000..890da1ddb --- /dev/null +++ b/scripts/msgpack2marshal.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +import marshal +import sys + +import msgpack + +if len(sys.argv) not in (2, 3): + print('Synopsis:', sys.argv[0], '', '[marshal output]', file=sys.stderr) + sys.exit(1) + +if len(sys.argv) == 2: + outfile = sys.stdout +else: + outfile = open(sys.argv[2], 'wb') + +with outfile: + with open(sys.argv[1], 'rb') as infile: + marshal.dump(msgpack.unpack(infile, use_list=False, encoding='utf-8'), outfile) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index e5c606d15..87bc1e255 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -2050,7 +2050,8 @@ def define_common_options(add_common_option): action='store_true', default=False, help='treat part files like normal files (e.g. to list/extract them)') add_common_option('--debug-profile', dest='debug_profile', default=None, metavar='FILE', - help='Store a Python profile at FILE') + help='Write Python profile in msgpack format into FILE. For local use a cProfile-' + 'compatible file can be generated by suffixing FILE with ".pyprof".') parser = argparse.ArgumentParser(prog=self.prog, description='Borg - Deduplicated Backups', add_help=False) @@ -3545,7 +3546,7 @@ def run(self, args): if is_slow_msgpack(): logger.warning("Using a pure-python msgpack! This will result in lower performance.") if args.debug_profile: - # Import these only when needed - avoids a further increase in startup time + # Import only when needed - avoids a further increase in startup time import cProfile import marshal logger.debug('Writing execution profile to %s', args.debug_profile) @@ -3554,10 +3555,24 @@ def run(self, args): with open(args.debug_profile, 'wb') as fd: profiler = cProfile.Profile() variables = dict(locals()) - profiler.runctx('rc = set_ec(func(args))', globals(), variables) - profiler.snapshot_stats() - marshal.dump(profiler.stats, fd) - return variables['rc'] + profiler.enable() + try: + return set_ec(func(args)) + finally: + profiler.disable() + profiler.snapshot_stats() + if args.debug_profile.endswith('.pyprof'): + marshal.dump(profiler.stats, fd) + else: + # We use msgpack here instead of the marshal module used by cProfile itself, + # because the latter is insecure. Since these files may be shared over the + # internet we don't want a format that is impossible to interpret outside + # an insecure implementation. + # See scripts/msgpack2marshal.py for a small script that turns a msgpack file + # into a marshal file that can be read by e.g. pyprof2calltree. + # For local use it's unnecessary hassle, though, that's why .pyprof makes + # it compatible (see above). + msgpack.pack(profiler.stats, fd, use_bin_type=True) else: return set_ec(func(args)) diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index e2c2be5c7..a3e224285 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -1686,10 +1686,16 @@ def test_debug_profile(self): self.create_test_files() self.cmd('init', '--encryption=repokey', self.repository_location) self.cmd('create', self.repository_location + '::test', 'input', '--debug-profile=create.prof') - stats = pstats.Stats('create.prof') + stats = pstats.Stats() + with open('create.prof', 'rb') as fd: + stats.stats = msgpack.unpack(fd, use_list=False, encoding='utf-8') + stats.strip_dirs() + stats.sort_stats('cumtime') + + self.cmd('create', self.repository_location + '::test2', 'input', '--debug-profile=create.pyprof') + stats = pstats.Stats('create.pyprof') # Only do this on trusted data! stats.strip_dirs() stats.sort_stats('cumtime') - # Ok, stats can be loaded, good enough. def test_common_options(self): self.create_test_files()