# -*- encoding: utf-8 *-* import os import io import re import sys from collections import OrderedDict from datetime import datetime from glob import glob from distutils.command.build import build from distutils.core import Command import textwrap min_python = (3, 4) my_python = sys.version_info if my_python < min_python: print("Borg requires Python %d.%d or later" % min_python) sys.exit(1) # Are we building on ReadTheDocs? on_rtd = os.environ.get('READTHEDOCS') install_requires = [ # msgpack pure python data corruption was fixed in 0.4.6. # Also, we might use some rather recent API features. 'msgpack-python>=0.4.6', 'pyzmq', ] # note for package maintainers: if you package borgbackup for distribution, # please add llfuse as a *requirement* on all platforms that have a working # llfuse package. "borg mount" needs llfuse to work. # if you do not have llfuse, do not require it, most of borgbackup will work. extras_require = { # llfuse 0.40 (tested, proven, ok), needs FUSE version >= 2.8.0 # llfuse 0.41 (tested shortly, looks ok), needs FUSE version >= 2.8.0 # llfuse 0.41.1 (tested shortly, looks ok), needs FUSE version >= 2.8.0 # llfuse 0.42 (tested shortly, looks ok), needs FUSE version >= 2.8.0 # llfuse 1.0 (tested shortly, looks ok), needs FUSE version >= 2.8.0 # llfuse 1.1.1 (tested shortly, looks ok), needs FUSE version >= 2.8.0 # llfuse 2.0 will break API 'fuse': ['llfuse<2.0', ], } if sys.platform.startswith('freebsd'): # llfuse was frequently broken / did not build on freebsd # llfuse 0.41.1, 1.1 are ok extras_require['fuse'] = ['llfuse <2.0, !=0.42.*, !=0.43, !=1.0', ] from setuptools import setup, find_packages, Extension from setuptools.command.sdist import sdist from distutils.command.clean import clean compress_source = 'src/borg/compress.pyx' crypto_ll_source = 'src/borg/crypto/low_level.pyx' crypto_helpers = 'src/borg/crypto/_crypto_helpers.c' chunker_source = 'src/borg/chunker.pyx' hashindex_source = 'src/borg/hashindex.pyx' item_source = 'src/borg/item.pyx' checksums_source = 'src/borg/algorithms/checksums.pyx' platform_posix_source = 'src/borg/platform/posix.pyx' platform_linux_source = 'src/borg/platform/linux.pyx' platform_darwin_source = 'src/borg/platform/darwin.pyx' platform_freebsd_source = 'src/borg/platform/freebsd.pyx' cython_sources = [ compress_source, crypto_ll_source, chunker_source, hashindex_source, item_source, checksums_source, platform_posix_source, platform_linux_source, platform_freebsd_source, platform_darwin_source, ] try: from Cython.Distutils import build_ext import Cython.Compiler.Main as cython_compiler class Sdist(sdist): def __init__(self, *args, **kwargs): for src in cython_sources: cython_compiler.compile(src, cython_compiler.default_options) super().__init__(*args, **kwargs) def make_distribution(self): self.filelist.extend([ 'src/borg/compress.c', 'src/borg/crypto/low_level.c', 'src/borg/chunker.c', 'src/borg/_chunker.c', 'src/borg/hashindex.c', 'src/borg/_hashindex.c', 'src/borg/cache_sync/cache_sync.c', 'src/borg/cache_sync/sysdep.h', 'src/borg/cache_sync/unpack.h', 'src/borg/cache_sync/unpack_define.h', 'src/borg/cache_sync/unpack_template.h', 'src/borg/item.c', 'src/borg/algorithms/checksums.c', 'src/borg/algorithms/crc32_dispatch.c', 'src/borg/algorithms/crc32_clmul.c', 'src/borg/algorithms/crc32_slice_by_8.c', 'src/borg/algorithms/xxh64/xxhash.h', 'src/borg/algorithms/xxh64/xxhash.c', 'src/borg/platform/posix.c', 'src/borg/platform/linux.c', 'src/borg/platform/freebsd.c', 'src/borg/platform/darwin.c', ]) super().make_distribution() except ImportError: class Sdist(sdist): def __init__(self, *args, **kwargs): raise Exception('Cython is required to run sdist') compress_source = compress_source.replace('.pyx', '.c') crypto_ll_source = crypto_ll_source.replace('.pyx', '.c') chunker_source = chunker_source.replace('.pyx', '.c') hashindex_source = hashindex_source.replace('.pyx', '.c') item_source = item_source.replace('.pyx', '.c') checksums_source = checksums_source.replace('.pyx', '.c') platform_posix_source = platform_posix_source.replace('.pyx', '.c') platform_linux_source = platform_linux_source.replace('.pyx', '.c') platform_freebsd_source = platform_freebsd_source.replace('.pyx', '.c') platform_darwin_source = platform_darwin_source.replace('.pyx', '.c') from distutils.command.build_ext import build_ext if not on_rtd and not all(os.path.exists(path) for path in [ compress_source, crypto_ll_source, chunker_source, hashindex_source, item_source, checksums_source, platform_posix_source, platform_linux_source, platform_freebsd_source, platform_darwin_source]): raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.') def detect_openssl(prefixes): for prefix in prefixes: filename = os.path.join(prefix, 'include', 'openssl', 'evp.h') if os.path.exists(filename): with open(filename, 'r') as fd: if 'PKCS5_PBKDF2_HMAC(' in fd.read(): return prefix def detect_lz4(prefixes): for prefix in prefixes: filename = os.path.join(prefix, 'include', 'lz4.h') if os.path.exists(filename): with open(filename, 'r') as fd: if 'LZ4_decompress_safe' in fd.read(): return prefix def detect_libb2(prefixes): for prefix in prefixes: filename = os.path.join(prefix, 'include', 'blake2.h') if os.path.exists(filename): with open(filename, 'r') as fd: if 'blake2b_init' in fd.read(): return prefix include_dirs = [] library_dirs = [] define_macros = [] crypto_libraries = ['crypto'] possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl', '/usr/local/borg', '/opt/local', '/opt/pkg', ] if os.environ.get('BORG_OPENSSL_PREFIX'): possible_openssl_prefixes.insert(0, os.environ.get('BORG_OPENSSL_PREFIX')) ssl_prefix = detect_openssl(possible_openssl_prefixes) if not ssl_prefix: raise Exception('Unable to find OpenSSL >= 1.0 headers. (Looked here: {})'.format(', '.join(possible_openssl_prefixes))) include_dirs.append(os.path.join(ssl_prefix, 'include')) library_dirs.append(os.path.join(ssl_prefix, 'lib')) possible_lz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local/lz4', '/usr/local/borg', '/opt/local', '/opt/pkg', ] if os.environ.get('BORG_LZ4_PREFIX'): possible_lz4_prefixes.insert(0, os.environ.get('BORG_LZ4_PREFIX')) lz4_prefix = detect_lz4(possible_lz4_prefixes) if lz4_prefix: include_dirs.append(os.path.join(lz4_prefix, 'include')) library_dirs.append(os.path.join(lz4_prefix, 'lib')) elif not on_rtd: raise Exception('Unable to find LZ4 headers. (Looked here: {})'.format(', '.join(possible_lz4_prefixes))) possible_libb2_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libb2', '/usr/local/libb2', '/usr/local/borg', '/opt/local', '/opt/pkg', ] if os.environ.get('BORG_LIBB2_PREFIX'): possible_libb2_prefixes.insert(0, os.environ.get('BORG_LIBB2_PREFIX')) libb2_prefix = detect_libb2(possible_libb2_prefixes) if libb2_prefix: print('Detected and preferring libb2 over bundled BLAKE2') include_dirs.append(os.path.join(libb2_prefix, 'include')) library_dirs.append(os.path.join(libb2_prefix, 'lib')) crypto_libraries.append('b2') define_macros.append(('BORG_USE_LIBB2', 'YES')) with open('README.rst', 'r') as fd: long_description = fd.read() # remove badges long_description = re.compile(r'^\.\. start-badges.*^\.\. end-badges', re.M | re.S).sub('', long_description) # remove |substitutions| long_description = re.compile(r'\|screencast\|').sub('', long_description) # remove unknown directives long_description = re.compile(r'^\.\. highlight:: \w+$', re.M).sub('', long_description) def format_metavar(option): if option.nargs in ('*', '...'): return '[%s...]' % option.metavar elif option.nargs == '?': return '[%s]' % option.metavar elif option.nargs is None: return option.metavar else: raise ValueError('Can\'t format metavar %s, unknown nargs %s!' % (option.metavar, option.nargs)) class build_usage(Command): description = "generate usage for each command" user_options = [ ('output=', 'O', 'output directory'), ] def initialize_options(self): pass def finalize_options(self): pass def run(self): print('generating usage docs') import borg borg.doc_mode = 'build_man' if not os.path.exists('docs/usage'): os.mkdir('docs/usage') # allows us to build docs without the C modules fully loaded during help generation from borg.archiver import Archiver parser = Archiver(prog='borg').build_parser() self.generate_level("", parser, Archiver) def generate_level(self, prefix, parser, Archiver): is_subcommand = False choices = {} for action in parser._actions: if action.choices is not None and 'SubParsersAction' in str(action.__class__): is_subcommand = True for cmd, parser in action.choices.items(): choices[prefix + cmd] = parser if prefix and not choices: return print('found commands: %s' % list(choices.keys())) for command, parser in sorted(choices.items()): if command.startswith('debug'): print('skipping', command) continue print('generating help for %s' % command) if self.generate_level(command + " ", parser, Archiver): continue with open('docs/usage/%s.rst.inc' % command.replace(" ", "_"), 'w') as doc: doc.write(".. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!\n\n") if command == 'help': for topic in Archiver.helptext: params = {"topic": topic, "underline": '~' * len('borg help ' + topic)} doc.write(".. _borg_{topic}:\n\n".format(**params)) doc.write("borg help {topic}\n{underline}\n\n".format(**params)) doc.write(Archiver.helptext[topic]) else: params = {"command": command, "command_": command.replace(' ', '_'), "underline": '-' * len('borg ' + command)} doc.write(".. _borg_{command_}:\n\n".format(**params)) doc.write("borg {command}\n{underline}\n.. code-block:: none\n\n borg [common options] {command}".format(**params)) self.write_usage(parser, doc) epilog = parser.epilog parser.epilog = None self.write_options(parser, doc) doc.write("\n\nDescription\n~~~~~~~~~~~\n") doc.write(epilog) if 'create' in choices: common_options = [group for group in choices['create']._action_groups if group.title == 'Common options'][0] with open('docs/usage/common-options.rst.inc', 'w') as doc: self.write_options_group(common_options, doc, False, base_indent=0) return is_subcommand def write_usage(self, parser, fp): if any(len(o.option_strings) for o in parser._actions): fp.write(' [options]') for option in parser._actions: if option.option_strings: continue fp.write(' ' + format_metavar(option)) fp.write('\n\n') def write_options(self, parser, fp): def is_positional_group(group): return any(not o.option_strings for o in group._group_actions) # HTML output: # A table using some column-spans def html_write(s): for line in s.splitlines(): fp.write(' ' + line + '\n') rows = [] for group in parser._action_groups: if group.title == 'Common options': # (no of columns used, columns, ...) rows.append((1, '.. class:: borg-common-opt-ref\n\n:ref:`common_options`')) else: if not group._group_actions: continue group_header = '**%s**' % group.title if group.description: group_header += ' — ' + group.description rows.append((1, group_header)) if is_positional_group(group): for option in group._group_actions: rows.append((3, '', '``%s``' % option.metavar, option.help or '')) else: for option in group._group_actions: if option.metavar: option_fmt = '``%s ' + option.metavar + '``' else: option_fmt = '``%s``' option_str = ', '.join(option_fmt % s for s in option.option_strings) option_desc = textwrap.dedent((option.help or '') % option.__dict__) rows.append((3, '', option_str, option_desc)) fp.write('.. only:: html\n\n') table = io.StringIO() table.write('.. class:: borg-options-table\n\n') self.rows_to_table(rows, table.write) fp.write(textwrap.indent(table.getvalue(), ' ' * 4)) # LaTeX output: # Regular rST option lists (irregular column widths) latex_options = io.StringIO() for group in parser._action_groups: if group.title == 'Common options': latex_options.write('\n\n:ref:`common_options`\n') latex_options.write(' |') else: self.write_options_group(group, latex_options) fp.write('\n.. only:: latex\n\n') fp.write(textwrap.indent(latex_options.getvalue(), ' ' * 4)) def rows_to_table(self, rows, write): def write_row_separator(): write('+') for column_width in column_widths: write('-' * (column_width + 1)) write('+') write('\n') # Find column count and width column_count = max(columns for columns, *_ in rows) column_widths = [0] * column_count for columns, *cells in rows: for i in range(columns): # "+ 1" because we want a space between the cell contents and the delimiting "|" in the output column_widths[i] = max(column_widths[i], len(cells[i]) + 1) for columns, *original_cells in rows: write_row_separator() # If a cell contains newlines, then the row must be split up in individual rows # where each cell contains no newline. rowspanning_cells = [] original_cells = list(original_cells) while any('\n' in cell for cell in original_cells): cell_bloc = [] for i, cell in enumerate(original_cells): pre, _, original_cells[i] = cell.partition('\n') cell_bloc.append(pre) rowspanning_cells.append(cell_bloc) rowspanning_cells.append(original_cells) for cells in rowspanning_cells: for i, column_width in enumerate(column_widths): if i < columns: write('| ') write(cells[i].ljust(column_width)) else: write(' ') write(''.ljust(column_width)) write('|\n') write_row_separator() # This bit of JavaScript kills the