# -*- encoding: utf-8 *-* # Support code for building docs (build_usage, build_man) import os import io import re import sys import textwrap from collections import OrderedDict from datetime import datetime from setuptools import Command def format_metavar(option): if option.nargs in ('*', '...'): return '[%s...]' % option.metavar elif option.nargs == '?': return '[%s]' % option.metavar elif option.nargs is None: return option.metavar else: raise ValueError('Can\'t format metavar %s, unknown nargs %s!' % (option.metavar, option.nargs)) class build_usage(Command): description = "generate usage for each command" user_options = [ ('output=', 'O', 'output directory'), ] def initialize_options(self): pass def finalize_options(self): pass def run(self): print('generating usage docs') import borg borg.doc_mode = 'build_man' if not os.path.exists('docs/usage'): os.mkdir('docs/usage') # allows us to build docs without the C modules fully loaded during help generation from borg.archiver import Archiver parser = Archiver(prog='borg').build_parser() # borgfs has a separate man page to satisfy debian's "every program from a package # must have a man page" requirement, but it doesn't need a separate HTML docs page #borgfs_parser = Archiver(prog='borgfs').build_parser() self.generate_level("", parser, Archiver) def generate_level(self, prefix, parser, Archiver, extra_choices=None): is_subcommand = False choices = {} for action in parser._actions: if action.choices is not None and 'SubParsersAction' in str(action.__class__): is_subcommand = True for cmd, parser in action.choices.items(): choices[prefix + cmd] = parser if extra_choices is not None: choices.update(extra_choices) if prefix and not choices: return print('found commands: %s' % list(choices.keys())) for command, parser in sorted(choices.items()): if command.startswith('debug'): print('skipping', command) continue print('generating help for %s' % command) if self.generate_level(command + " ", parser, Archiver): continue with open('docs/usage/%s.rst.inc' % command.replace(" ", "_"), 'w') as doc: doc.write(".. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!\n\n") if command == 'help': for topic in Archiver.helptext: params = {"topic": topic, "underline": '~' * len('borg help ' + topic)} doc.write(".. _borg_{topic}:\n\n".format(**params)) doc.write("borg help {topic}\n{underline}\n\n".format(**params)) doc.write(Archiver.helptext[topic]) else: params = {"command": command, "command_": command.replace(' ', '_'), "underline": '-' * len('borg ' + command)} doc.write(".. _borg_{command_}:\n\n".format(**params)) doc.write("borg {command}\n{underline}\n.. code-block:: none\n\n borg [common options] {command}".format(**params)) self.write_usage(parser, doc) epilog = parser.epilog parser.epilog = None self.write_options(parser, doc) doc.write("\n\nDescription\n~~~~~~~~~~~\n") doc.write(epilog) if 'create' in choices: common_options = [group for group in choices['create']._action_groups if group.title == 'Common options'][0] with open('docs/usage/common-options.rst.inc', 'w') as doc: self.write_options_group(common_options, doc, False, base_indent=0) return is_subcommand def write_usage(self, parser, fp): if any(len(o.option_strings) for o in parser._actions): fp.write(' [options]') for option in parser._actions: if option.option_strings: continue fp.write(' ' + format_metavar(option)) fp.write('\n\n') def write_options(self, parser, fp): def is_positional_group(group): return any(not o.option_strings for o in group._group_actions) # HTML output: # A table using some column-spans def html_write(s): for line in s.splitlines(): fp.write(' ' + line + '\n') rows = [] for group in parser._action_groups: if group.title == 'Common options': # (no of columns used, columns, ...) rows.append((1, '.. class:: borg-common-opt-ref\n\n:ref:`common_options`')) else: if not group._group_actions: continue group_header = '**%s**' % group.title if group.description: group_header += ' — ' + group.description rows.append((1, group_header)) if is_positional_group(group): for option in group._group_actions: rows.append((3, '', '``%s``' % option.metavar, option.help or '')) else: for option in group._group_actions: if option.metavar: option_fmt = '``%s ' + option.metavar + '``' else: option_fmt = '``%s``' option_str = ', '.join(option_fmt % s for s in option.option_strings) option_desc = textwrap.dedent((option.help or '') % option.__dict__) rows.append((3, '', option_str, option_desc)) fp.write('.. only:: html\n\n') table = io.StringIO() table.write('.. class:: borg-options-table\n\n') self.rows_to_table(rows, table.write) fp.write(textwrap.indent(table.getvalue(), ' ' * 4)) # LaTeX output: # Regular rST option lists (irregular column widths) latex_options = io.StringIO() for group in parser._action_groups: if group.title == 'Common options': latex_options.write('\n\n:ref:`common_options`\n') latex_options.write(' |') else: self.write_options_group(group, latex_options) fp.write('\n.. only:: latex\n\n') fp.write(textwrap.indent(latex_options.getvalue(), ' ' * 4)) def rows_to_table(self, rows, write): def write_row_separator(): write('+') for column_width in column_widths: write('-' * (column_width + 1)) write('+') write('\n') # Find column count and width column_count = max(columns for columns, *_ in rows) column_widths = [0] * column_count for columns, *cells in rows: for i in range(columns): # "+ 1" because we want a space between the cell contents and the delimiting "|" in the output column_widths[i] = max(column_widths[i], len(cells[i]) + 1) for columns, *original_cells in rows: write_row_separator() # If a cell contains newlines, then the row must be split up in individual rows # where each cell contains no newline. rowspanning_cells = [] original_cells = list(original_cells) while any('\n' in cell for cell in original_cells): cell_bloc = [] for i, cell in enumerate(original_cells): pre, _, original_cells[i] = cell.partition('\n') cell_bloc.append(pre) rowspanning_cells.append(cell_bloc) rowspanning_cells.append(original_cells) for cells in rowspanning_cells: for i, column_width in enumerate(column_widths): if i < columns: write('| ') write(cells[i].ljust(column_width)) else: write(' ') write(''.ljust(column_width)) write('|\n') write_row_separator() # This bit of JavaScript kills the that is invariably inserted by docutils, # but does absolutely no good here. It sets bogus column widths which cannot be overridden # with CSS alone. # Since this is HTML-only output, it would be possible to just generate a directly, # but then we'd lose rST formatting. write(textwrap.dedent(""" .. raw:: html """)) def write_options_group(self, group, fp, with_title=True, base_indent=4): def is_positional_group(group): return any(not o.option_strings for o in group._group_actions) indent = ' ' * base_indent if is_positional_group(group): for option in group._group_actions: fp.write(option.metavar + '\n') fp.write(textwrap.indent(option.help or '', ' ' * base_indent) + '\n') return if not group._group_actions: return if with_title: fp.write('\n\n') fp.write(group.title + '\n') opts = OrderedDict() for option in group._group_actions: if option.metavar: option_fmt = '%s ' + option.metavar else: option_fmt = '%s' option_str = ', '.join(option_fmt % s for s in option.option_strings) option_desc = textwrap.dedent((option.help or '') % option.__dict__) opts[option_str] = textwrap.indent(option_desc, ' ' * 4) padding = len(max(opts)) + 1 for option, desc in opts.items(): fp.write(indent + option.ljust(padding) + desc + '\n') class build_man(Command): description = 'build man pages' user_options = [] see_also = { 'create': ('delete', 'prune', 'check', 'patterns', 'placeholders', 'compression'), 'recreate': ('patterns', 'placeholders', 'compression'), 'list': ('info', 'diff', 'prune', 'patterns'), 'info': ('list', 'diff'), 'init': ('create', 'delete', 'check', 'list', 'key-import', 'key-export', 'key-change-passphrase'), 'key-import': ('key-export', ), 'key-export': ('key-import', ), 'mount': ('umount', 'extract'), # Would be cooler if these two were on the same page 'umount': ('mount', ), 'extract': ('mount', ), } rst_prelude = textwrap.dedent(""" .. role:: ref(title) .. |project_name| replace:: Borg """) usage_group = { 'break-lock': 'lock', 'with-lock': 'lock', 'change-passphrase': 'key', 'key_change-passphrase': 'key', 'key_export': 'key', 'key_import': 'key', 'key_migrate-to-repokey': 'key', 'export-tar': 'tar', 'benchmark_crud': 'benchmark', 'umount': 'mount', } def initialize_options(self): pass def finalize_options(self): pass def run(self): print('building man pages (in docs/man)', file=sys.stderr) import borg borg.doc_mode = 'build_man' os.makedirs('docs/man', exist_ok=True) # allows us to build docs without the C modules fully loaded during help generation from borg.archiver import Archiver parser = Archiver(prog='borg').build_parser() borgfs_parser = Archiver(prog='borgfs').build_parser() self.generate_level('', parser, Archiver, {'borgfs': borgfs_parser}) self.build_topic_pages(Archiver) self.build_intro_page() def generate_level(self, prefix, parser, Archiver, extra_choices=None): is_subcommand = False choices = {} for action in parser._actions: if action.choices is not None and 'SubParsersAction' in str(action.__class__): is_subcommand = True for cmd, parser in action.choices.items(): choices[prefix + cmd] = parser if extra_choices is not None: choices.update(extra_choices) if prefix and not choices: return for command, parser in sorted(choices.items()): if command.startswith('debug') or command == 'help': continue if command == "borgfs": man_title = command else: man_title = 'borg-' + command.replace(' ', '-') print('building man page', man_title + '(1)', file=sys.stderr) is_intermediary = self.generate_level(command + ' ', parser, Archiver) doc, write = self.new_doc() self.write_man_header(write, man_title, parser.description) self.write_heading(write, 'SYNOPSIS') if is_intermediary: subparsers = [action for action in parser._actions if 'SubParsersAction' in str(action.__class__)][0] for subcommand in subparsers.choices: write('| borg', '[common options]', command, subcommand, '...') self.see_also.setdefault(command, []).append('%s-%s' % (command, subcommand)) else: if command == "borgfs": write(command, end='') else: write('borg', '[common options]', command, end='') self.write_usage(write, parser) write('\n') description, _, notes = parser.epilog.partition('\n.. man NOTES') if description: self.write_heading(write, 'DESCRIPTION') write(description) if not is_intermediary: self.write_heading(write, 'OPTIONS') write('See `borg-common(1)` for common options of Borg commands.') write() self.write_options(write, parser) self.write_examples(write, command) if notes: self.write_heading(write, 'NOTES') write(notes) self.write_see_also(write, man_title) self.gen_man_page(man_title, doc.getvalue()) # Generate the borg-common(1) man page with the common options. if 'create' in choices: doc, write = self.new_doc() man_title = 'borg-common' self.write_man_header(write, man_title, 'Common options of Borg commands') common_options = [group for group in choices['create']._action_groups if group.title == 'Common options'][0] self.write_heading(write, 'SYNOPSIS') self.write_options_group(write, common_options) self.write_see_also(write, man_title) self.gen_man_page(man_title, doc.getvalue()) return is_subcommand def build_topic_pages(self, Archiver): for topic, text in Archiver.helptext.items(): doc, write = self.new_doc() man_title = 'borg-' + topic print('building man page', man_title + '(1)', file=sys.stderr) self.write_man_header(write, man_title, 'Details regarding ' + topic) self.write_heading(write, 'DESCRIPTION') write(text) self.gen_man_page(man_title, doc.getvalue()) def build_intro_page(self): print('building man page borg(1)', file=sys.stderr) with open('docs/man_intro.rst') as fd: man_intro = fd.read() self.gen_man_page('borg', self.rst_prelude + man_intro) def new_doc(self): doc = io.StringIO(self.rst_prelude) doc.read() write = self.printer(doc) return doc, write def printer(self, fd): def write(*args, **kwargs): print(*args, file=fd, **kwargs) return write def write_heading(self, write, header, char='-', double_sided=False): write() if double_sided: write(char * len(header)) write(header) write(char * len(header)) write() def write_man_header(self, write, title, description): self.write_heading(write, title, '=', double_sided=True) self.write_heading(write, description, double_sided=True) # man page metadata write(':Author: The Borg Collective') write(':Date:', datetime.utcnow().date().isoformat()) write(':Manual section: 1') write(':Manual group: borg backup tool') write() def write_examples(self, write, command): command = command.replace(' ', '_') with open('docs/usage/%s.rst' % self.usage_group.get(command, command)) as fd: usage = fd.read() usage_include = '.. include:: %s.rst.inc' % command begin = usage.find(usage_include) end = usage.find('.. include', begin + 1) # If a command has a dedicated anchor, it will occur before the command's include. if 0 < usage.find('.. _', begin + 1) < end: end = usage.find('.. _', begin + 1) examples = usage[begin:end] examples = examples.replace(usage_include, '') examples = examples.replace('Examples\n~~~~~~~~', '') examples = examples.replace('Miscellaneous Help\n------------------', '') examples = examples.replace('``docs/misc/prune-example.txt``:', '``docs/misc/prune-example.txt``.') examples = examples.replace('.. highlight:: none\n', '') # we don't support highlight examples = re.sub('^(~+)$', lambda matches: '+' * len(matches.group(0)), examples, flags=re.MULTILINE) examples = examples.strip() if examples: self.write_heading(write, 'EXAMPLES', '-') write(examples) def write_see_also(self, write, man_title): see_also = self.see_also.get(man_title.replace('borg-', ''), ()) see_also = ['`borg-%s(1)`' % s for s in see_also] see_also.insert(0, '`borg-common(1)`') self.write_heading(write, 'SEE ALSO') write(', '.join(see_also)) def gen_man_page(self, name, rst): from docutils.writers import manpage from docutils.core import publish_string from docutils.nodes import inline from docutils.parsers.rst import roles def issue(name, rawtext, text, lineno, inliner, options={}, content=[]): return [inline(rawtext, '#' + text)], [] roles.register_local_role('issue', issue) # We give the source_path so that docutils can find relative includes # as-if the document where located in the docs/ directory. man_page = publish_string(source=rst, source_path='docs/%s.rst' % name, writer=manpage.Writer()) with open('docs/man/%s.1' % name, 'wb') as fd: fd.write(man_page) def write_usage(self, write, parser): if any(len(o.option_strings) for o in parser._actions): write(' [options] ', end='') for option in parser._actions: if option.option_strings: continue write(format_metavar(option), end=' ') def write_options(self, write, parser): for group in parser._action_groups: if group.title == 'Common options' or not group._group_actions: continue title = 'arguments' if group.title == 'positional arguments' else group.title self.write_heading(write, title, '+') self.write_options_group(write, group) def write_options_group(self, write, group): def is_positional_group(group): return any(not o.option_strings for o in group._group_actions) if is_positional_group(group): for option in group._group_actions: write(option.metavar) write(textwrap.indent(option.help or '', ' ' * 4)) return opts = OrderedDict() for option in group._group_actions: if option.metavar: option_fmt = '%s ' + option.metavar else: option_fmt = '%s' option_str = ', '.join(option_fmt % s for s in option.option_strings) option_desc = textwrap.dedent((option.help or '') % option.__dict__) opts[option_str] = textwrap.indent(option_desc, ' ' * 4) padding = len(max(opts)) + 1 for option, desc in opts.items(): write(option.ljust(padding), desc)