nixos-render-docs: add options commonmark converter

the old method of pasting parts of options.json into a markdown document
and hoping for the best no longer works now that options.json contains
more than just docbook. given the infrastructure we have now we can
actually render options.md properly, so we may as well do that.
This commit is contained in:
pennae 2023-02-04 23:16:30 +01:00 committed by pennae
parent 6c182075bb
commit 4d3aef762f
5 changed files with 408 additions and 35 deletions

View file

@ -98,11 +98,14 @@ in rec {
> $out
'';
optionsCommonMark = pkgs.runCommand "options.md" {} ''
${pkgs.python3Minimal}/bin/python ${./generateDoc.py} \
--format commonmark \
optionsCommonMark = pkgs.runCommand "options.md" {
nativeBuildInputs = [ pkgs.nixos-render-docs ];
} ''
nixos-render-docs -j $NIX_BUILD_CORES options commonmark \
--manpage-urls ${pkgs.path + "/doc/manpage-urls.json"} \
--revision ${lib.escapeShellArg revision} \
${optionsJSON}/share/doc/nixos/options.json \
> $out
$out
'';
optionsJSON = pkgs.runCommand "options.json"

View file

@ -2,7 +2,7 @@ import argparse
import json
import sys
formats = ['commonmark', 'asciidoc']
formats = ['asciidoc']
parser = argparse.ArgumentParser(
description = 'Generate documentation for a set of JSON-formatted NixOS options'
@ -38,33 +38,6 @@ class OptionsEncoder(json.JSONEncoder):
return super().encode(obj)
def generate_commonmark(options):
for (name, value) in options.items():
print('##', name.replace('<', '&lt;').replace('>', '&gt;'))
print(value['description'])
print()
if 'type' in value:
print('*_Type_*')
print ('```')
print(value['type'])
print ('```')
print()
print()
if 'default' in value:
print('*_Default_*')
print('```')
print(json.dumps(value['default'], cls=OptionsEncoder, ensure_ascii=False, separators=(',', ':')))
print('```')
print()
print()
if 'example' in value:
print('*_Example_*')
print('```')
print(json.dumps(value['example'], cls=OptionsEncoder, ensure_ascii=False, separators=(',', ':')))
print('```')
print()
print()
# TODO: declarations: link to github
def generate_asciidoc(options):
for (name, value) in options.items():
@ -103,9 +76,7 @@ def generate_asciidoc(options):
with open(args.nix_options_path) as nix_options_json:
options = json.load(nix_options_json)
if args.format == 'commonmark':
generate_commonmark(options)
elif args.format == 'asciidoc':
if args.format == 'asciidoc':
generate_asciidoc(options)
else:
raise Exception(f'Unsupported documentation format `--format {args.format}`')

View file

@ -0,0 +1,231 @@
from collections.abc import Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from typing import Any, cast, Optional
from .md import md_escape, md_make_code, Renderer
import markdown_it
from markdown_it.token import Token
from markdown_it.utils import OptionsDict
@dataclass(kw_only=True)
class List:
next_idx: Optional[int] = None
compact: bool
first_item_seen: bool = False
@dataclass
class Par:
indent: str
continuing: bool = False
class CommonMarkRenderer(Renderer):
__output__ = "commonmark"
_parstack: list[Par]
_link_stack: list[str]
_list_stack: list[List]
def __init__(self, manpage_urls: Mapping[str, str], parser: Optional[markdown_it.MarkdownIt] = None):
super().__init__(manpage_urls, parser)
self._parstack = [ Par("") ]
self._link_stack = []
self._list_stack = []
def _enter_block(self, extra_indent: str) -> None:
self._parstack.append(Par(self._parstack[-1].indent + extra_indent))
def _leave_block(self) -> None:
self._parstack.pop()
self._parstack[-1].continuing = True
def _break(self) -> str:
self._parstack[-1].continuing = True
return f"\n{self._parstack[-1].indent}"
def _maybe_parbreak(self) -> str:
result = f"\n{self._parstack[-1].indent}" * 2 if self._parstack[-1].continuing else ""
self._parstack[-1].continuing = True
return result
def _admonition_open(self, kind: str) -> str:
pbreak = self._maybe_parbreak()
self._enter_block("")
return f"{pbreak}**{kind}:** "
def _admonition_close(self) -> str:
self._leave_block()
return ""
def _indent_raw(self, s: str) -> str:
if '\n' not in s:
return s
return f"\n{self._parstack[-1].indent}".join(s.splitlines())
def text(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._parstack[-1].continuing = True
return self._indent_raw(md_escape(token.content))
def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._maybe_parbreak()
def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return ""
def hardbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return f" {self._break()}"
def softbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._break()
def code_inline(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._parstack[-1].continuing = True
return md_make_code(token.content)
def code_block(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self.fence(token, tokens, i, options, env)
def link_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._parstack[-1].continuing = True
self._link_stack.append(cast(str, token.attrs['href']))
return "["
def link_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return f"]({md_escape(self._link_stack.pop())})"
def list_item_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
lst = self._list_stack[-1]
lbreak = "" if not lst.first_item_seen else self._break() * (1 if lst.compact else 2)
lst.first_item_seen = True
head = " -"
if lst.next_idx is not None:
head = f" {lst.next_idx}."
lst.next_idx += 1
self._enter_block(" " * (len(head) + 1))
return f'{lbreak}{head} '
def list_item_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._leave_block()
return ""
def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._list_stack.append(List(compact=bool(token.meta['compact'])))
return self._maybe_parbreak()
def bullet_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._list_stack.pop()
return ""
def em_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return "*"
def em_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return "*"
def strong_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return "**"
def strong_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return "**"
def fence(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
code = token.content
if code.endswith('\n'):
code = code[:-1]
pbreak = self._maybe_parbreak()
return pbreak + self._indent_raw(md_make_code(code, info=token.info, multiline=True))
def blockquote_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
pbreak = self._maybe_parbreak()
self._enter_block("> ")
return pbreak + "> "
def blockquote_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._leave_block()
return ""
def note_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._admonition_open("Note")
def note_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._admonition_close()
def caution_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._admonition_open("Caution")
def caution_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._admonition_close()
def important_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._admonition_open("Important")
def important_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._admonition_close()
def tip_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._admonition_open("Tip")
def tip_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._admonition_close()
def warning_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._admonition_open("Warning")
def warning_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return self._admonition_close()
def dl_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._list_stack.append(List(compact=False))
return ""
def dl_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._list_stack.pop()
return ""
def dt_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
pbreak = self._maybe_parbreak()
self._enter_block(" ")
# add an opening zero-width non-joiner to separate *our* emphasis from possible
# emphasis in the provided term
return f'{pbreak} - *{chr(0x200C)}'
def dt_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return f"{chr(0x200C)}*"
def dd_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._parstack[-1].continuing = True
return ""
def dd_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._leave_block()
return ""
def myst_role(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._parstack[-1].continuing = True
content = md_make_code(token.content)
if token.meta['name'] == 'manpage' and (url := self._manpage_urls.get(token.content)):
return f"[{content}]({url})"
return content # no roles in regular commonmark
def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
# there's no way we can emit attrspans correctly in all cases. we could use inline
# html for ids, but that would not round-trip. same holds for classes. since this
# renderer is only used for approximate options export and all of these things are
# not allowed in options we can ignore them for now.
return ""
def attr_span_end(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return ""
def heading_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return token.markup + " "
def heading_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
return "\n"
def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._list_stack.append(
List(next_idx = cast(int, token.attrs.get('start', 1)),
compact = bool(token.meta['compact'])))
return self._maybe_parbreak()
def ordered_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
self._list_stack.pop()
return ""

View file

@ -13,6 +13,7 @@ from xml.sax.saxutils import escape, quoteattr
import markdown_it
from . import parallel
from .commonmark import CommonMarkRenderer
from .docbook import DocBookRenderer, make_xml_id
from .manpage import ManpageRenderer, man_escape
from .md import Converter, md_escape, md_make_code
@ -422,6 +423,59 @@ class ManpageConverter(BaseConverter):
return "\n".join(result)
class OptionsCommonMarkRenderer(OptionDocsRestrictions, CommonMarkRenderer):
pass
class CommonMarkConverter(BaseConverter):
__renderer__ = OptionsCommonMarkRenderer
__option_block_separator__ = ""
def _parallel_render_prepare(self) -> Any:
return (self._manpage_urls, self._revision, self._markdown_by_default)
@classmethod
def _parallel_render_init_worker(cls, a: Any) -> CommonMarkConverter:
return cls(*a)
def _render_code(self, option: dict[str, Any], key: str) -> list[str]:
# NOTE this duplicates the old direct-paste behavior, even if it is somewhat
# incorrect, since users rely on it.
if lit := option_is(option, key, 'literalDocBook'):
return [ f"*{key.capitalize()}:* {lit['text']}" ]
else:
return super()._render_code(option, key)
def _render_description(self, desc: str | dict[str, Any]) -> list[str]:
# NOTE this duplicates the old direct-paste behavior, even if it is somewhat
# incorrect, since users rely on it.
if isinstance(desc, str) and not self._markdown_by_default:
return [ desc ]
else:
return super()._render_description(desc)
def _related_packages_header(self) -> list[str]:
return [ "*Related packages:*" ]
def _decl_def_header(self, header: str) -> list[str]:
return [ f"*{header}:*" ]
def _decl_def_entry(self, href: Optional[str], name: str) -> list[str]:
if href is not None:
return [ f" - [{md_escape(name)}]({href})" ]
return [ f" - {md_escape(name)}" ]
def _decl_def_footer(self) -> list[str]:
return []
def finalize(self) -> str:
result = []
for (name, opt) in self._sorted_options():
result.append(f"## {md_escape(name)}\n")
result += opt.lines
result.append("\n\n")
return "\n".join(result)
def _build_cli_db(p: argparse.ArgumentParser) -> None:
p.add_argument('--manpage-urls', required=True)
p.add_argument('--revision', required=True)
@ -437,6 +491,13 @@ def _build_cli_manpage(p: argparse.ArgumentParser) -> None:
p.add_argument("infile")
p.add_argument("outfile")
def _build_cli_commonmark(p: argparse.ArgumentParser) -> None:
p.add_argument('--manpage-urls', required=True)
p.add_argument('--revision', required=True)
p.add_argument('--markdown-by-default', default=False, action='store_true')
p.add_argument("infile")
p.add_argument("outfile")
def _run_cli_db(args: argparse.Namespace) -> None:
with open(args.manpage_urls, 'r') as manpage_urls:
md = DocBookConverter(
@ -464,15 +525,30 @@ def _run_cli_manpage(args: argparse.Namespace) -> None:
with open(args.outfile, 'w') as f:
f.write(md.finalize())
def _run_cli_commonmark(args: argparse.Namespace) -> None:
with open(args.manpage_urls, 'r') as manpage_urls:
md = CommonMarkConverter(
json.load(manpage_urls),
revision = args.revision,
markdown_by_default = args.markdown_by_default)
with open(args.infile, 'r') as f:
md.add_options(json.load(f))
with open(args.outfile, 'w') as f:
f.write(md.finalize())
def build_cli(p: argparse.ArgumentParser) -> None:
formats = p.add_subparsers(dest='format', required=True)
_build_cli_db(formats.add_parser('docbook'))
_build_cli_manpage(formats.add_parser('manpage'))
_build_cli_commonmark(formats.add_parser('commonmark'))
def run_cli(args: argparse.Namespace) -> None:
if args.format == 'docbook':
_run_cli_db(args)
elif args.format == 'manpage':
_run_cli_manpage(args)
elif args.format == 'commonmark':
_run_cli_commonmark(args)
else:
raise RuntimeError('format not hooked up', args)

View file

@ -0,0 +1,92 @@
import nixos_render_docs
from sample_md import sample1
from typing import Mapping, Optional
import markdown_it
class Converter(nixos_render_docs.md.Converter):
__renderer__ = nixos_render_docs.commonmark.CommonMarkRenderer
# NOTE: in these tests we represent trailing spaces by ` ` and replace them with real space later,
# since a number of editors will strip trailing whitespace on save and that would break the tests.
def test_indented_fence() -> None:
c = Converter({})
s = """\
> - ```foo
> thing
>      
> rest
> ```\
""".replace(' ', ' ')
assert c._render(s) == s
def test_full() -> None:
c = Converter({ 'man(1)': 'http://example.org' })
assert c._render(sample1) == f"""\
**Warning:** foo
**Note:** nested
[
multiline
](link)
[` man(1) `](http://example.org) reference
some nested anchors
*emph* **strong** *nesting emph **and strong** and ` code `*
- wide bullet
- list
1. wide ordered
2. list
- narrow bullet
- list
1. narrow ordered
2. list
> quotes
> 
> > with *nesting*
> > 
> > ```
> > nested code block
> > ```
> 
> - and lists
> - ```
> containing code
> ```
> 
> and more quote
100. list starting at 100
101. goes on
- *deflist*
   
> with a quote
> and stuff
   
```
code block
```
   
```
fenced block
```
   
text
- *more stuff in same deflist*
   
foo""".replace(' ', ' ')