borg/setup.py

646 lines
25 KiB
Python

# -*- encoding: utf-8 *-*
import os
import io
import re
import sys
from collections import OrderedDict
from datetime import datetime
from glob import glob
from distutils.command.build import build
from distutils.core import Command
import textwrap
min_python = (3, 4)
my_python = sys.version_info
if my_python < min_python:
print("Borg requires Python %d.%d or later" % min_python)
sys.exit(1)
# Are we building on ReadTheDocs?
on_rtd = os.environ.get('READTHEDOCS')
# msgpack pure python data corruption was fixed in 0.4.6.
# Also, we might use some rather recent API features.
install_requires = ['msgpack-python>=0.4.6', ]
# note for package maintainers: if you package borgbackup for distribution,
# please add llfuse as a *requirement* on all platforms that have a working
# llfuse package. "borg mount" needs llfuse to work.
# if you do not have llfuse, do not require it, most of borgbackup will work.
extras_require = {
# llfuse 0.40 (tested, proven, ok), needs FUSE version >= 2.8.0
# llfuse 0.41 (tested shortly, looks ok), needs FUSE version >= 2.8.0
# llfuse 0.41.1 (tested shortly, looks ok), needs FUSE version >= 2.8.0
# llfuse 0.42 (tested shortly, looks ok), needs FUSE version >= 2.8.0
# llfuse 1.0 (tested shortly, looks ok), needs FUSE version >= 2.8.0
# llfuse 1.1.1 (tested shortly, looks ok), needs FUSE version >= 2.8.0
# llfuse 2.0 will break API
'fuse': ['llfuse<2.0', ],
}
if sys.platform.startswith('freebsd'):
# llfuse was frequently broken / did not build on freebsd
# llfuse 0.41.1, 1.1 are ok
extras_require['fuse'] = ['llfuse <2.0, !=0.42.*, !=0.43, !=1.0', ]
from setuptools import setup, find_packages, Extension
from setuptools.command.sdist import sdist
compress_source = 'src/borg/compress.pyx'
crypto_source = 'src/borg/crypto.pyx'
chunker_source = 'src/borg/algorithms/chunker.pyx'
hashindex_source = 'src/borg/hashindex.pyx'
item_source = 'src/borg/item.pyx'
crc32_source = 'src/borg/algorithms/crc32.pyx'
platform_posix_source = 'src/borg/platform/posix.pyx'
platform_linux_source = 'src/borg/platform/linux.pyx'
platform_darwin_source = 'src/borg/platform/darwin.pyx'
platform_freebsd_source = 'src/borg/platform/freebsd.pyx'
cython_sources = [
compress_source,
crypto_source,
chunker_source,
hashindex_source,
item_source,
crc32_source,
platform_posix_source,
platform_linux_source,
platform_freebsd_source,
platform_darwin_source,
]
try:
from Cython.Distutils import build_ext
import Cython.Compiler.Main as cython_compiler
class Sdist(sdist):
def __init__(self, *args, **kwargs):
for src in cython_sources:
cython_compiler.compile(src, cython_compiler.default_options)
super().__init__(*args, **kwargs)
def make_distribution(self):
self.filelist.extend([
'src/borg/compress.c',
'src/borg/crypto.c',
'src/borg/algorithms/chunker.c', 'src/borg/algorithms/buzhash.c',
'src/borg/hashindex.c', 'src/borg/_hashindex.c',
'src/borg/item.c',
'src/borg/algorithms/crc32.c',
'src/borg/algorithms/crc32_dispatch.c', 'src/borg/algorithms/crc32_clmul.c', 'src/borg/algorithms/crc32_slice_by_8.c',
'src/borg/platform/posix.c',
'src/borg/platform/linux.c',
'src/borg/platform/freebsd.c',
'src/borg/platform/darwin.c',
])
super().make_distribution()
except ImportError:
class Sdist(sdist):
def __init__(self, *args, **kwargs):
raise Exception('Cython is required to run sdist')
compress_source = compress_source.replace('.pyx', '.c')
crypto_source = crypto_source.replace('.pyx', '.c')
chunker_source = chunker_source.replace('.pyx', '.c')
hashindex_source = hashindex_source.replace('.pyx', '.c')
item_source = item_source.replace('.pyx', '.c')
crc32_source = crc32_source.replace('.pyx', '.c')
platform_posix_source = platform_posix_source.replace('.pyx', '.c')
platform_linux_source = platform_linux_source.replace('.pyx', '.c')
platform_freebsd_source = platform_freebsd_source.replace('.pyx', '.c')
platform_darwin_source = platform_darwin_source.replace('.pyx', '.c')
from distutils.command.build_ext import build_ext
if not on_rtd and not all(os.path.exists(path) for path in [
compress_source, crypto_source, chunker_source, hashindex_source, item_source, crc32_source,
platform_posix_source, platform_linux_source, platform_freebsd_source, platform_darwin_source]):
raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.')
def detect_openssl(prefixes):
for prefix in prefixes:
filename = os.path.join(prefix, 'include', 'openssl', 'evp.h')
if os.path.exists(filename):
with open(filename, 'r') as fd:
if 'PKCS5_PBKDF2_HMAC(' in fd.read():
return prefix
def detect_lz4(prefixes):
for prefix in prefixes:
filename = os.path.join(prefix, 'include', 'lz4.h')
if os.path.exists(filename):
with open(filename, 'r') as fd:
if 'LZ4_decompress_safe' in fd.read():
return prefix
def detect_libb2(prefixes):
for prefix in prefixes:
filename = os.path.join(prefix, 'include', 'blake2.h')
if os.path.exists(filename):
with open(filename, 'r') as fd:
if 'blake2b_init' in fd.read():
return prefix
include_dirs = []
library_dirs = []
define_macros = []
crypto_libraries = ['crypto']
possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl',
'/usr/local/borg', '/opt/local', '/opt/pkg', ]
if os.environ.get('BORG_OPENSSL_PREFIX'):
possible_openssl_prefixes.insert(0, os.environ.get('BORG_OPENSSL_PREFIX'))
ssl_prefix = detect_openssl(possible_openssl_prefixes)
if not ssl_prefix:
raise Exception('Unable to find OpenSSL >= 1.0 headers. (Looked here: {})'.format(', '.join(possible_openssl_prefixes)))
include_dirs.append(os.path.join(ssl_prefix, 'include'))
library_dirs.append(os.path.join(ssl_prefix, 'lib'))
possible_lz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local/lz4',
'/usr/local/borg', '/opt/local', '/opt/pkg', ]
if os.environ.get('BORG_LZ4_PREFIX'):
possible_lz4_prefixes.insert(0, os.environ.get('BORG_LZ4_PREFIX'))
lz4_prefix = detect_lz4(possible_lz4_prefixes)
if lz4_prefix:
include_dirs.append(os.path.join(lz4_prefix, 'include'))
library_dirs.append(os.path.join(lz4_prefix, 'lib'))
elif not on_rtd:
raise Exception('Unable to find LZ4 headers. (Looked here: {})'.format(', '.join(possible_lz4_prefixes)))
possible_libb2_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libb2', '/usr/local/libb2',
'/usr/local/borg', '/opt/local', '/opt/pkg', ]
if os.environ.get('BORG_LIBB2_PREFIX'):
possible_libb2_prefixes.insert(0, os.environ.get('BORG_LIBB2_PREFIX'))
libb2_prefix = detect_libb2(possible_libb2_prefixes)
if libb2_prefix:
print('Detected and preferring libb2 over bundled BLAKE2')
include_dirs.append(os.path.join(libb2_prefix, 'include'))
library_dirs.append(os.path.join(libb2_prefix, 'lib'))
crypto_libraries.append('b2')
define_macros.append(('BORG_USE_LIBB2', 'YES'))
with open('README.rst', 'r') as fd:
long_description = fd.read()
# remove badges
long_description = re.compile(r'^\.\. start-badges.*^\.\. end-badges', re.M | re.S).sub('', long_description)
# remove |substitutions|
long_description = re.compile(r'\|screencast\|').sub('', long_description)
# remove unknown directives
long_description = re.compile(r'^\.\. highlight:: \w+$', re.M).sub('', long_description)
class build_usage(Command):
description = "generate usage for each command"
user_options = [
('output=', 'O', 'output directory'),
]
def initialize_options(self):
pass
def finalize_options(self):
pass
def run(self):
print('generating usage docs')
import borg
borg.doc_mode = 'build_man'
if not os.path.exists('docs/usage'):
os.mkdir('docs/usage')
# allows us to build docs without the C modules fully loaded during help generation
from borg.archiver import Archiver
parser = Archiver(prog='borg').build_parser()
self.generate_level("", parser, Archiver)
def generate_level(self, prefix, parser, Archiver):
is_subcommand = False
choices = {}
for action in parser._actions:
if action.choices is not None and 'SubParsersAction' in str(action.__class__):
is_subcommand = True
for cmd, parser in action.choices.items():
choices[prefix + cmd] = parser
if prefix and not choices:
return
print('found commands: %s' % list(choices.keys()))
for command, parser in sorted(choices.items()):
if command.startswith('debug'):
print('skipping', command)
continue
print('generating help for %s' % command)
if self.generate_level(command + " ", parser, Archiver):
continue
with open('docs/usage/%s.rst.inc' % command.replace(" ", "_"), 'w') as doc:
doc.write(".. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!\n\n")
if command == 'help':
for topic in Archiver.helptext:
params = {"topic": topic,
"underline": '~' * len('borg help ' + topic)}
doc.write(".. _borg_{topic}:\n\n".format(**params))
doc.write("borg help {topic}\n{underline}\n\n".format(**params))
doc.write(Archiver.helptext[topic])
else:
params = {"command": command,
"command_": command.replace(' ', '_'),
"underline": '-' * len('borg ' + command)}
doc.write(".. _borg_{command_}:\n\n".format(**params))
doc.write("borg {command}\n{underline}\n::\n\n borg {command}".format(**params))
self.write_usage(parser, doc)
epilog = parser.epilog
parser.epilog = None
self.write_options(parser, doc)
doc.write("\n\nDescription\n~~~~~~~~~~~\n")
doc.write(epilog)
if 'create' in choices:
common_options = [group for group in choices['create']._action_groups if group.title == 'Common options'][0]
with open('docs/usage/common-options.rst.inc', 'w') as doc:
self.write_options_group(common_options, doc, False)
return is_subcommand
def write_usage(self, parser, fp):
if any(len(o.option_strings) for o in parser._actions):
fp.write(' <options>')
for option in parser._actions:
if option.option_strings:
continue
fp.write(' ' + option.metavar)
def write_options(self, parser, fp):
for group in parser._action_groups:
if group.title == 'Common options':
fp.write('\n\n`Common options`_\n')
fp.write(' |')
else:
self.write_options_group(group, fp)
def write_options_group(self, group, fp, with_title=True):
def is_positional_group(group):
return any(not o.option_strings for o in group._group_actions)
def get_help(option):
text = textwrap.dedent((option.help or '') % option.__dict__)
return '\n'.join('| ' + line for line in text.splitlines())
def shipout(text):
fp.write(textwrap.indent('\n'.join(text), ' ' * 4))
if not group._group_actions:
return
if with_title:
fp.write('\n\n')
fp.write(group.title + '\n')
text = []
if is_positional_group(group):
for option in group._group_actions:
text.append(option.metavar)
text.append(textwrap.indent(option.help or '', ' ' * 4))
shipout(text)
return
options = []
for option in group._group_actions:
if option.metavar:
option_fmt = '``%%s %s``' % option.metavar
else:
option_fmt = '``%s``'
option_str = ', '.join(option_fmt % s for s in option.option_strings)
options.append((option_str, option))
for option_str, option in options:
help = textwrap.indent(get_help(option), ' ' * 4)
text.append(option_str)
text.append(help)
shipout(text)
class build_man(Command):
description = 'build man pages'
user_options = []
see_also = {
'create': ('delete', 'prune', 'check', 'patterns', 'placeholders', 'compression'),
'recreate': ('patterns', 'placeholders', 'compression'),
'list': ('info', 'diff', 'prune', 'patterns'),
'info': ('list', 'diff'),
'init': ('create', 'delete', 'check', 'list', 'key-import', 'key-export', 'key-change-passphrase'),
'key-import': ('key-export', ),
'key-export': ('key-import', ),
'mount': ('umount', 'extract'), # Would be cooler if these two were on the same page
'umount': ('mount', ),
'extract': ('mount', ),
}
rst_prelude = textwrap.dedent("""
.. role:: ref(title)
.. |project_name| replace:: Borg
""")
def initialize_options(self):
pass
def finalize_options(self):
pass
def run(self):
print('building man pages (in docs/man)', file=sys.stderr)
import borg
borg.doc_mode = 'build_man'
os.makedirs('docs/man', exist_ok=True)
# allows us to build docs without the C modules fully loaded during help generation
from borg.archiver import Archiver
parser = Archiver(prog='borg').build_parser()
self.generate_level('', parser, Archiver)
self.build_topic_pages(Archiver)
self.build_intro_page()
def generate_level(self, prefix, parser, Archiver):
is_subcommand = False
choices = {}
for action in parser._actions:
if action.choices is not None and 'SubParsersAction' in str(action.__class__):
is_subcommand = True
for cmd, parser in action.choices.items():
choices[prefix + cmd] = parser
if prefix and not choices:
return
for command, parser in sorted(choices.items()):
if command.startswith('debug') or command == 'help':
continue
man_title = 'borg-' + command.replace(' ', '-')
print('building man page', man_title + '(1)', file=sys.stderr)
is_intermediary = self.generate_level(command + ' ', parser, Archiver)
doc, write = self.new_doc()
self.write_man_header(write, man_title, parser.description)
self.write_heading(write, 'SYNOPSIS')
if is_intermediary:
subparsers = [action for action in parser._actions if 'SubParsersAction' in str(action.__class__)][0]
for subcommand in subparsers.choices:
write('| borg', command, subcommand, '...')
self.see_also.setdefault(command, []).append('%s-%s' % (command, subcommand))
else:
write('borg', command, end='')
self.write_usage(write, parser)
write('\n')
description, _, notes = parser.epilog.partition('\n.. man NOTES')
if description:
self.write_heading(write, 'DESCRIPTION')
write(description)
if not is_intermediary:
self.write_heading(write, 'OPTIONS')
write('See `borg-common(1)` for common options of Borg commands.')
write()
self.write_options(write, parser)
self.write_examples(write, command)
if notes:
self.write_heading(write, 'NOTES')
write(notes)
self.write_see_also(write, man_title)
self.gen_man_page(man_title, doc.getvalue())
# Generate the borg-common(1) man page with the common options.
if 'create' in choices:
doc, write = self.new_doc()
man_title = 'borg-common'
self.write_man_header(write, man_title, 'Common options of Borg commands')
common_options = [group for group in choices['create']._action_groups if group.title == 'Common options'][0]
self.write_heading(write, 'SYNOPSIS')
self.write_options_group(write, common_options)
self.write_see_also(write, man_title)
self.gen_man_page(man_title, doc.getvalue())
return is_subcommand
def build_topic_pages(self, Archiver):
for topic, text in Archiver.helptext.items():
doc, write = self.new_doc()
man_title = 'borg-' + topic
print('building man page', man_title + '(1)', file=sys.stderr)
self.write_man_header(write, man_title, 'Details regarding ' + topic)
self.write_heading(write, 'DESCRIPTION')
write(text)
self.gen_man_page(man_title, doc.getvalue())
def build_intro_page(self):
print('building man page borg(1)', file=sys.stderr)
with open('docs/man_intro.rst') as fd:
man_intro = fd.read()
self.gen_man_page('borg', self.rst_prelude + man_intro)
def new_doc(self):
doc = io.StringIO(self.rst_prelude)
doc.read()
write = self.printer(doc)
return doc, write
def printer(self, fd):
def write(*args, **kwargs):
print(*args, file=fd, **kwargs)
return write
def write_heading(self, write, header, char='-', double_sided=False):
write()
if double_sided:
write(char * len(header))
write(header)
write(char * len(header))
write()
def write_man_header(self, write, title, description):
self.write_heading(write, title, '=', double_sided=True)
self.write_heading(write, description, double_sided=True)
# man page metadata
write(':Author: The Borg Collective')
write(':Date:', datetime.utcnow().date().isoformat())
write(':Manual section: 1')
write(':Manual group: borg backup tool')
write()
def write_examples(self, write, command):
with open('docs/usage.rst') as fd:
usage = fd.read()
usage_include = '.. include:: usage/%s.rst.inc' % command
begin = usage.find(usage_include)
end = usage.find('.. include', begin + 1)
examples = usage[begin:end]
examples = examples.replace(usage_include, '')
examples = examples.replace('Examples\n~~~~~~~~', '')
examples = examples.replace('Miscellaneous Help\n------------------', '')
examples = re.sub('^(~+)$', lambda matches: '+' * len(matches.group(0)), examples, flags=re.MULTILINE)
examples = examples.strip()
if examples:
self.write_heading(write, 'EXAMPLES', '-')
write(examples)
def write_see_also(self, write, man_title):
see_also = self.see_also.get(man_title.replace('borg-', ''), ())
see_also = ['`borg-%s(1)`' % s for s in see_also]
see_also.insert(0, '`borg-common(1)`')
self.write_heading(write, 'SEE ALSO')
write(', '.join(see_also))
def gen_man_page(self, name, rst):
from docutils.writers import manpage
from docutils.core import publish_string
# We give the source_path so that docutils can find relative includes
# as-if the document where located in the docs/ directory.
man_page = publish_string(source=rst, source_path='docs/virtmanpage.rst', writer=manpage.Writer())
with open('docs/man/%s.1' % name, 'wb') as fd:
fd.write(man_page)
def write_usage(self, write, parser):
if any(len(o.option_strings) for o in parser._actions):
write(' <options> ', end='')
for option in parser._actions:
if option.option_strings:
continue
write(option.metavar, end=' ')
def write_options(self, write, parser):
for group in parser._action_groups:
if group.title == 'Common options' or not group._group_actions:
continue
title = 'arguments' if group.title == 'positional arguments' else group.title
self.write_heading(write, title, '+')
self.write_options_group(write, group)
def write_options_group(self, write, group):
def is_positional_group(group):
return any(not o.option_strings for o in group._group_actions)
if is_positional_group(group):
for option in group._group_actions:
write(option.metavar)
write(textwrap.indent(option.help or '', ' ' * 4))
return
opts = OrderedDict()
for option in group._group_actions:
if option.metavar:
option_fmt = '%s ' + option.metavar
else:
option_fmt = '%s'
option_str = ', '.join(option_fmt % s for s in option.option_strings)
option_desc = textwrap.dedent((option.help or '') % option.__dict__)
opts[option_str] = textwrap.indent(option_desc, ' ' * 4)
padding = len(max(opts)) + 1
for option, desc in opts.items():
write(option.ljust(padding), desc)
cmdclass = {
'build_ext': build_ext,
'build_usage': build_usage,
'build_man': build_man,
'sdist': Sdist
}
ext_modules = []
if not on_rtd:
ext_modules += [
Extension('borg.compress', [compress_source], libraries=['lz4'], include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
Extension('borg.crypto', [crypto_source], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
Extension('borg.hashindex', [hashindex_source]),
Extension('borg.item', [item_source]),
Extension('borg.algorithms.chunker', [chunker_source]),
Extension('borg.algorithms.crc32', [crc32_source]),
]
if not sys.platform.startswith(('win32', )):
ext_modules.append(Extension('borg.platform.posix', [platform_posix_source]))
if sys.platform == 'linux':
ext_modules.append(Extension('borg.platform.linux', [platform_linux_source], libraries=['acl']))
elif sys.platform.startswith('freebsd'):
ext_modules.append(Extension('borg.platform.freebsd', [platform_freebsd_source]))
elif sys.platform == 'darwin':
ext_modules.append(Extension('borg.platform.darwin', [platform_darwin_source]))
setup(
name='borgbackup',
use_scm_version={
'write_to': 'src/borg/_version.py',
},
author='The Borg Collective (see AUTHORS file)',
author_email='borgbackup@python.org',
url='https://borgbackup.readthedocs.io/',
description='Deduplicated, encrypted, authenticated and compressed backups',
long_description=long_description,
license='BSD',
platforms=['Linux', 'MacOS X', 'FreeBSD', 'OpenBSD', 'NetBSD', ],
classifiers=[
'Development Status :: 4 - Beta',
'Environment :: Console',
'Intended Audience :: System Administrators',
'License :: OSI Approved :: BSD License',
'Operating System :: POSIX :: BSD :: FreeBSD',
'Operating System :: POSIX :: BSD :: OpenBSD',
'Operating System :: POSIX :: BSD :: NetBSD',
'Operating System :: MacOS :: MacOS X',
'Operating System :: POSIX :: Linux',
'Programming Language :: Python',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Topic :: Security :: Cryptography',
'Topic :: System :: Archiving :: Backup',
],
packages=find_packages('src'),
package_dir={'': 'src'},
include_package_data=True,
zip_safe=False,
entry_points={
'console_scripts': [
'borg = borg.archiver:main',
'borgfs = borg.archiver:main',
]
},
package_data={
'borg': ['paperkey.html']
},
cmdclass=cmdclass,
ext_modules=ext_modules,
setup_requires=['setuptools_scm>=1.7'],
install_requires=install_requires,
extras_require=extras_require,
)