borg/scripts/make.py

# Support code for building docs (build_usage, build_man)

import glob
import os
import io
import re
import sys
import textwrap
from collections import OrderedDict
from datetime import datetime, timezone
import time


def format_metavar(option):
    if option.nargs in ("*", "..."):
        return "[%s...]" % option.metavar
    elif option.nargs == "?":
        return "[%s]" % option.metavar
    elif option.nargs is None:
        return option.metavar
    else:
        raise ValueError(f"Can't format metavar {option.metavar}, unknown nargs {option.nargs}!")


class BuildUsage:
    """generate usage docs for each command"""

    def run(self):
        print("generating usage docs")
        import borg

        borg.doc_mode = "build_man"
        os.makedirs("docs/usage", exist_ok=True)
        # allows us to build docs without the C modules fully loaded during help generation
        from borg.archiver import Archiver

        parser = Archiver(prog="borg").build_parser()
        # borgfs has a separate man page to satisfy debian's "every program from a package
        # must have a man page" requirement, but it doesn't need a separate HTML docs page
        # borgfs_parser = Archiver(prog='borgfs').build_parser()

        self.generate_level("", parser, Archiver)
        return 0

    def generate_level(self, prefix, parser, Archiver, extra_choices=None):
        is_subcommand = False
        choices = {}
        for action in parser._actions:
            if action.choices is not None and "SubParsersAction" in str(action.__class__):
                is_subcommand = True
                for cmd, parser in action.choices.items():
                    choices[prefix + cmd] = parser
        if extra_choices is not None:
            choices.update(extra_choices)
        if prefix and not choices:
            return
        print("found commands: %s" % list(choices.keys()))

        for command, parser in sorted(choices.items()):
            if command.startswith("debug"):
                print("skipping", command)
                continue
            print("generating help for %s" % command)

            if self.generate_level(command + " ", parser, Archiver):
                continue

            with open("docs/usage/%s.rst.inc" % command.replace(" ", "_"), "w") as doc:
                doc.write(".. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!\n\n")
                if command == "help":
                    for topic in Archiver.helptext:
                        params = {"topic": topic, "underline": "~" * len("borg help " + topic)}
                        doc.write(".. _borg_{topic}:\n\n".format(**params))
                        doc.write("borg help {topic}\n{underline}\n\n".format(**params))
                        doc.write(Archiver.helptext[topic])
                else:
                    params = {
                        "command": command,
                        "command_": command.replace(" ", "_"),
                        "underline": "-" * len("borg " + command),
                    }
                    doc.write(".. _borg_{command_}:\n\n".format(**params))
                    doc.write(
                        "borg {command}\n{underline}\n.. code-block:: none\n\n    borg [common options] {command}".format(
                            **params
                        )
                    )
                    self.write_usage(parser, doc)
                    epilog = parser.epilog
                    parser.epilog = None
                    self.write_options(parser, doc)
                    doc.write("\n\nDescription\n~~~~~~~~~~~\n")
                    doc.write(epilog)

        if "create" in choices:
            common_options = [group for group in choices["create"]._action_groups if group.title == "Common options"][0]
            with open("docs/usage/common-options.rst.inc", "w") as doc:
                self.write_options_group(common_options, doc, False, base_indent=0)

        return is_subcommand

    def write_usage(self, parser, fp):
        if any(len(o.option_strings) for o in parser._actions):
            fp.write(" [options]")
        for option in parser._actions:
            if option.option_strings:
                continue
            fp.write(" " + format_metavar(option))
        fp.write("\n\n")

    def write_options(self, parser, fp):
        def is_positional_group(group):
            return any(not o.option_strings for o in group._group_actions)

        # HTML output:
        # A table using some column-spans

        rows = []
        for group in parser._action_groups:
            if group.title == "Common options":
                # (no of columns used, columns, ...)
                rows.append((1, ".. class:: borg-common-opt-ref\n\n:ref:`common_options`"))
            else:
                if not group._group_actions:
                    continue
                group_header = "**%s**" % group.title
                if group.description:
                    group_header += " — " + group.description
                rows.append((1, group_header))
                if is_positional_group(group):
                    for option in group._group_actions:
                        rows.append((3, "", "``%s``" % option.metavar, option.help or ""))
                else:
                    for option in group._group_actions:
                        if option.metavar:
                            option_fmt = "``%s " + option.metavar + "``"
                        else:
                            option_fmt = "``%s``"
                        option_str = ", ".join(option_fmt % s for s in option.option_strings)
                        option_desc = textwrap.dedent((option.help or "") % option.__dict__)
                        rows.append((3, "", option_str, option_desc))

        fp.write(".. only:: html\n\n")
        table = io.StringIO()
        table.write(".. class:: borg-options-table\n\n")
        self.rows_to_table(rows, table.write)
        fp.write(textwrap.indent(table.getvalue(), " " * 4))

        # LaTeX output:
        # Regular rST option lists (irregular column widths)
        latex_options = io.StringIO()
        for group in parser._action_groups:
            if group.title == "Common options":
                latex_options.write("\n\n:ref:`common_options`\n")
                latex_options.write("    |")
            else:
                self.write_options_group(group, latex_options)
        fp.write("\n.. only:: latex\n\n")
        fp.write(textwrap.indent(latex_options.getvalue(), " " * 4))

    def rows_to_table(self, rows, write):
        def write_row_separator():
            write("+")
            for column_width in column_widths:
                write("-" * (column_width + 1))
                write("+")
            write("\n")

        # Find column count and width
        column_count = max(columns for columns, *_ in rows)
        column_widths = [0] * column_count
        for columns, *cells in rows:
            for i in range(columns):
                # "+ 1" because we want a space between the cell contents and the delimiting "|" in the output
                column_widths[i] = max(column_widths[i], len(cells[i]) + 1)

        for columns, *original_cells in rows:
            write_row_separator()
            # If a cell contains newlines, then the row must be split up in individual rows
            # where each cell contains no newline.
            rowspanning_cells = []
            original_cells = list(original_cells)
            while any("\n" in cell for cell in original_cells):
                cell_bloc = []
                for i, cell in enumerate(original_cells):
                    pre, _, original_cells[i] = cell.partition("\n")
                    cell_bloc.append(pre)
                rowspanning_cells.append(cell_bloc)
            rowspanning_cells.append(original_cells)
            for cells in rowspanning_cells:
                for i, column_width in enumerate(column_widths):
                    if i < columns:
                        write("| ")
                        write(cells[i].ljust(column_width))
                    else:
                        write("  ")
                        write("".ljust(column_width))
                write("|\n")

        write_row_separator()
        # This bit of JavaScript kills the <colgroup> that is invariably inserted by docutils,
        # but does absolutely no good here. It sets bogus column widths which cannot be overridden
        # with CSS alone.
        # Since this is HTML-only output, it would be possible to just generate a <table> directly,
        # but then we'd lose rST formatting.
        write(
            textwrap.dedent(
                """
        .. raw:: html

            <script type='text/javascript'>
            $(document).ready(function () {
                $('.borg-options-table colgroup').remove();
            })
            </script>
        """
            )
        )

    def write_options_group(self, group, fp, with_title=True, base_indent=4):
        def is_positional_group(group):
            return any(not o.option_strings for o in group._group_actions)

        indent = " " * base_indent

        if is_positional_group(group):
            for option in group._group_actions:
                fp.write(option.metavar + "\n")
                fp.write(textwrap.indent(option.help or "", " " * base_indent) + "\n")
            return

        if not group._group_actions:
            return

        if with_title:
            fp.write("\n\n")
            fp.write(group.title + "\n")

        opts = OrderedDict()

        for option in group._group_actions:
            if option.metavar:
                option_fmt = "%s " + option.metavar
            else:
                option_fmt = "%s"
            option_str = ", ".join(option_fmt % s for s in option.option_strings)
            option_desc = textwrap.dedent((option.help or "") % option.__dict__)
            opts[option_str] = textwrap.indent(option_desc, " " * 4)

        padding = len(max(opts)) + 1

        for option, desc in opts.items():
            fp.write(indent + option.ljust(padding) + desc + "\n")


class BuildMan:
    """build man pages"""

    see_also = {
        "create": ("delete", "prune", "check", "patterns", "placeholders", "compression", "rcreate"),
        "recreate": ("patterns", "placeholders", "compression"),
        "list": ("info", "diff", "prune", "patterns", "rlist"),
        "info": ("list", "diff", "rinfo"),
        "rcreate": ("rdelete", "rlist", "check", "benchmark-cpu", "key-import", "key-export", "key-change-passphrase"),
        "key-import": ("key-export",),
        "key-export": ("key-import",),
        "mount": ("umount", "extract"),  # Would be cooler if these two were on the same page
        "umount": ("mount",),
        "extract": ("mount",),
        "delete": ("compact", "rdelete"),
        "prune": ("compact",),
    }

    rst_prelude = textwrap.dedent(
        """
    .. role:: ref(title)

    .. |project_name| replace:: Borg

    """
    )

    usage_group = {
        "break-lock": "lock",
        "with-lock": "lock",
        "key_change-passphrase": "key",
        "key_change-location": "key",
        "key_export": "key",
        "key_import": "key",
        "export-tar": "tar",
        "import-tar": "tar",
        "benchmark_crud": "benchmark",
        "benchmark_cpu": "benchmark",
        "umount": "mount",
    }

    def run(self):
        print("building man pages (in docs/man)", file=sys.stderr)
        import borg

        borg.doc_mode = "build_man"
        os.makedirs("docs/man", exist_ok=True)
        # allows us to build docs without the C modules fully loaded during help generation
        from borg.archiver import Archiver

        parser = Archiver(prog="borg").build_parser()
        borgfs_parser = Archiver(prog="borgfs").build_parser()

        self.generate_level("", parser, Archiver, {"borgfs": borgfs_parser})
        self.build_topic_pages(Archiver)
        self.build_intro_page()
        return 0

    def generate_level(self, prefix, parser, Archiver, extra_choices=None):
        is_subcommand = False
        choices = {}
        for action in parser._actions:
            if action.choices is not None and "SubParsersAction" in str(action.__class__):
                is_subcommand = True
                for cmd, parser in action.choices.items():
                    choices[prefix + cmd] = parser
        if extra_choices is not None:
            choices.update(extra_choices)
        if prefix and not choices:
            return

        for command, parser in sorted(choices.items()):
            if command.startswith("debug") or command == "help":
                continue

            if command == "borgfs":
                man_title = command
            else:
                man_title = "borg-" + command.replace(" ", "-")
            print("building man page", man_title + "(1)", file=sys.stderr)

            is_intermediary = self.generate_level(command + " ", parser, Archiver)

            doc, write = self.new_doc()
            self.write_man_header(write, man_title, parser.description)

            self.write_heading(write, "SYNOPSIS")
            if is_intermediary:
                subparsers = [action for action in parser._actions if "SubParsersAction" in str(action.__class__)][0]
                for subcommand in subparsers.choices:
                    write("| borg", "[common options]", command, subcommand, "...")
                    self.see_also.setdefault(command, []).append(f"{command}-{subcommand}")
            else:
                if command == "borgfs":
                    write(command, end="")
                else:
                    write("borg", "[common options]", command, end="")
                self.write_usage(write, parser)
            write("\n")

            description, _, notes = parser.epilog.partition("\n.. man NOTES")

            if description:
                self.write_heading(write, "DESCRIPTION")
                write(description)

            if not is_intermediary:
                self.write_heading(write, "OPTIONS")
                write("See `borg-common(1)` for common options of Borg commands.")
                write()
                self.write_options(write, parser)

                self.write_examples(write, command)

            if notes:
                self.write_heading(write, "NOTES")
                write(notes)

            self.write_see_also(write, man_title)

            self.gen_man_page(man_title, doc.getvalue())

        # Generate the borg-common(1) man page with the common options.
        if "create" in choices:
            doc, write = self.new_doc()
            man_title = "borg-common"
            self.write_man_header(write, man_title, "Common options of Borg commands")

            common_options = [group for group in choices["create"]._action_groups if group.title == "Common options"][0]

            self.write_heading(write, "SYNOPSIS")
            self.write_options_group(write, common_options)
            self.write_see_also(write, man_title)
            self.gen_man_page(man_title, doc.getvalue())

        return is_subcommand

    def build_topic_pages(self, Archiver):
        for topic, text in Archiver.helptext.items():
            doc, write = self.new_doc()
            man_title = "borg-" + topic
            print("building man page", man_title + "(1)", file=sys.stderr)

            self.write_man_header(write, man_title, "Details regarding " + topic)
            self.write_heading(write, "DESCRIPTION")
            write(text)
            self.gen_man_page(man_title, doc.getvalue())

    def build_intro_page(self):
        doc, write = self.new_doc()
        man_title = "borg"
        print("building man page borg(1)", file=sys.stderr)

        with open("docs/man_intro.rst") as fd:
            man_intro = fd.read()

        self.write_man_header(write, man_title, "deduplicating and encrypting backup tool")
        self.gen_man_page(man_title, doc.getvalue() + man_intro)

    def new_doc(self):
        doc = io.StringIO(self.rst_prelude)
        doc.read()
        write = self.printer(doc)
        return doc, write

    def printer(self, fd):
        def write(*args, **kwargs):
            print(*args, file=fd, **kwargs)

        return write

    def write_heading(self, write, header, char="-", double_sided=False):
        write()
        if double_sided:
            write(char * len(header))
        write(header)
        write(char * len(header))
        write()

    def write_man_header(self, write, title, description):
        self.write_heading(write, title, "=", double_sided=True)
        self.write_heading(write, description, double_sided=True)
        # man page metadata
        write(":Author: The Borg Collective")
        source_date_epoch = int(os.environ.get("SOURCE_DATE_EPOCH", time.time()))
        write(":Date:", datetime.fromtimestamp(source_date_epoch, timezone.utc).date().isoformat())
        write(":Manual section: 1")
        write(":Manual group: borg backup tool")
        write()

    def write_examples(self, write, command):
        command = command.replace(" ", "_")
        with open("docs/usage/%s.rst" % self.usage_group.get(command, command)) as fd:
            usage = fd.read()
            usage_include = ".. include:: %s.rst.inc" % command
            begin = usage.find(usage_include)
            end = usage.find(".. include", begin + 1)
            # If a command has a dedicated anchor, it will occur before the command's include.
            if 0 < usage.find(".. _", begin + 1) < end:
                end = usage.find(".. _", begin + 1)
            examples = usage[begin:end]
            examples = examples.replace(usage_include, "")
            examples = examples.replace("Examples\n~~~~~~~~", "")
            examples = examples.replace("Miscellaneous Help\n------------------", "")
            examples = examples.replace("``docs/misc/prune-example.txt``:", "``docs/misc/prune-example.txt``.")
            examples = examples.replace(".. highlight:: none\n", "")  # we don't support highlight
            examples = re.sub("^(~+)$", lambda matches: "+" * len(matches.group(0)), examples, flags=re.MULTILINE)
            examples = examples.strip()
        if examples:
            self.write_heading(write, "EXAMPLES", "-")
            write(examples)

    def write_see_also(self, write, man_title):
        see_also = self.see_also.get(man_title.replace("borg-", ""), ())
        see_also = ["`borg-%s(1)`" % s for s in see_also]
        see_also.insert(0, "`borg-common(1)`")
        self.write_heading(write, "SEE ALSO")
        write(", ".join(see_also))

    def gen_man_page(self, name, rst):
        from docutils.writers import manpage
        from docutils.core import publish_string
        from docutils.nodes import inline
        from docutils.parsers.rst import roles

        def issue(name, rawtext, text, lineno, inliner, options={}, content=[]):
            return [inline(rawtext, "#" + text)], []

        roles.register_local_role("issue", issue)
        # We give the source_path so that docutils can find relative includes
        # as-if the document where located in the docs/ directory.
        man_page = publish_string(source=rst, source_path="docs/%s.rst" % name, writer=manpage.Writer())
        with open("docs/man/%s.1" % name, "wb") as fd:
            fd.write(man_page)

    def write_usage(self, write, parser):
        if any(len(o.option_strings) for o in parser._actions):
            write(" [options] ", end="")
        for option in parser._actions:
            if option.option_strings:
                continue
            write(format_metavar(option), end=" ")

    def write_options(self, write, parser):
        for group in parser._action_groups:
            if group.title == "Common options" or not group._group_actions:
                continue
            title = "arguments" if group.title == "positional arguments" else group.title
            self.write_heading(write, title, "+")
            self.write_options_group(write, group)

    def write_options_group(self, write, group):
        def is_positional_group(group):
            return any(not o.option_strings for o in group._group_actions)

        if is_positional_group(group):
            for option in group._group_actions:
                write(option.metavar)
                write(textwrap.indent(option.help or "", " " * 4))
            return

        opts = OrderedDict()

        for option in group._group_actions:
            if option.metavar:
                option_fmt = "%s " + option.metavar
            else:
                option_fmt = "%s"
            option_str = ", ".join(option_fmt % s for s in option.option_strings)
            option_desc = textwrap.dedent((option.help or "") % option.__dict__)
            opts[option_str] = textwrap.indent(option_desc, " " * 4)

        padding = len(max(opts)) + 1

        for option, desc in opts.items():
            write(option.ljust(padding), desc)


cython_sources = """
src/borg/compress.pyx
src/borg/crypto/low_level.pyx
src/borg/chunker.pyx
src/borg/hashindex.pyx
src/borg/item.pyx
src/borg/checksums.pyx
src/borg/platform/posix.pyx
src/borg/platform/linux.pyx
src/borg/platform/syncfilerange.pyx
src/borg/platform/darwin.pyx
src/borg/platform/freebsd.pyx
src/borg/platform/windows.pyx
""".strip().splitlines()


def rm(file):
    try:
        os.unlink(file)
    except FileNotFoundError:
        return False
    else:
        return True


class Clean:
    def run(self):
        for source in cython_sources:
            genc = source.replace(".pyx", ".c")
            rm(genc)
            compiled_glob = source.replace(".pyx", ".cpython*")
            for compiled in sorted(glob.glob(compiled_glob)):
                rm(compiled)
        return 0


def usage():
    print(
        textwrap.dedent(
            """
        Usage:
            python scripts/make.py clean        # clean workdir (remove generated files)
            python scripts/make.py build_usage  # build usage documentation
            python scripts/make.py build_man    # build man pages
    """
        )
    )


def main(argv):
    if len(argv) < 2 or len(argv) == 2 and argv[1] in ("-h", "--help"):
        usage()
        return 0
    command = argv[1]
    if command == "clean":
        return Clean().run()
    if command == "build_usage":
        return BuildUsage().run()
    if command == "build_man":
        return BuildMan().run()
    usage()
    return 1


if __name__ == "__main__":
    rc = main(sys.argv)
    sys.exit(rc)