From 1ee5fcaf0c8a298f24ada56eadec0a83c382410c Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Tue, 23 Aug 2022 15:17:26 +0200 Subject: [PATCH 1/6] python3Packages.sphinxHook: Add support for multiple builders Removes the up until now unused option to specify a `sphinxOutdir` in favor of allowing to specify multiple builders, which is for example useful for generating both documentation and manpages using the hook. Since the output path cannot be determined from within the package we automatically generate it and add a diversion for manpages, so they land in the correct output and path. --- .../interpreters/python/hooks/default.nix | 3 +- .../interpreters/python/hooks/sphinx-hook.sh | 43 ++++++++++++++++--- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/pkgs/development/interpreters/python/hooks/default.nix b/pkgs/development/interpreters/python/hooks/default.nix index 0f175c90920..5d605b240ad 100644 --- a/pkgs/development/interpreters/python/hooks/default.nix +++ b/pkgs/development/interpreters/python/hooks/default.nix @@ -6,6 +6,7 @@ , isPy3k , ensureNewerSourcesForZipFilesHook , findutils +, installShellFiles }: let @@ -190,6 +191,6 @@ in rec { sphinxHook = callPackage ({ sphinx }: makeSetupHook { name = "python${python.pythonVersion}-sphinx-hook"; - deps = [ sphinx ]; + deps = [ sphinx installShellFiles ]; } ./sphinx-hook.sh) {}; } diff --git a/pkgs/development/interpreters/python/hooks/sphinx-hook.sh b/pkgs/development/interpreters/python/hooks/sphinx-hook.sh index 92cc9e52ed2..5bbb17e2126 100644 --- a/pkgs/development/interpreters/python/hooks/sphinx-hook.sh +++ b/pkgs/development/interpreters/python/hooks/sphinx-hook.sh @@ -11,10 +11,17 @@ # Sphinx build system can depend on arbitrary amount of python modules, client # code is responsible for ensuring that all dependencies are present. -buildSphinxPhase() { - local __sphinxRoot="" o +# shellcheck shell=bash +echo "Sourcing sphinx-hook" +declare -a __sphinxBuilders + +buildSphinxPhase() { + echo "Executing buildSphinxPhase" + + local __sphinxRoot="" o runHook preBuildSphinx + if [[ -n "${sphinxRoot:-}" ]] ; then # explicit root if ! [[ -f "${sphinxRoot}/conf.py" ]] ; then echo 2>&1 "$sphinxRoot/conf.py: no such file" @@ -35,20 +42,42 @@ buildSphinxPhase() { echo 2>&1 "Sphinx documentation not found, use 'sphinxRoot' variable" exit 1 fi - sphinx-build -M html "${__sphinxRoot}" ".sphinx/html" -v + + if [ -n "${sphinxBuilders-}" ]; then + eval "__sphinxBuilders=($sphinxBuilders)" + else + __sphinxBuilders=(html) + fi + + for __builder in "${__sphinxBuilders[@]}"; do + echo "Executing sphinx-build with ${__builder} builder" + sphinx-build -M "${__builder}" "${__sphinxRoot}" ".sphinx/${__builder}" -v + done runHook postBuildSphinx } installSphinxPhase() { + echo "Executing installSphinxPhase" + local docdir="" runHook preInstallSphinx - docdir="${doc:-$out}/share/doc/${sphinxOutdir:-$name}" - mkdir -p "$docdir" + for __builder in "${__sphinxBuilders[@]}"; do + # divert output for man builder + if [ "$__builder" == "man" ]; then + installManPage .sphinx/man/man/* - cp -r .sphinx/html/html "$docdir/" - rm -fr "${docdir}/html/_sources" "${docdir}/html/.buildinfo" + else + # shellcheck disable=2154 + docdir="${doc:-$out}/share/doc/${pname}" + + mkdir -p "$docdir" + + cp -r ".sphinx/${__builder}/${__builder}" "$docdir/" + rm -fr "${docdir}/${__builder}/_sources" "${docdir}/${__builder}/.buildinfo" + fi + done runHook postInstallSphinx } From 8a26deba0649a05bd978804a6af3a710548fcc0b Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Tue, 23 Aug 2022 15:45:14 +0200 Subject: [PATCH 2/6] doc/python: integrate sphinxHook docs Improve the documentation by moving it from the hook script into the Python section in the manual and adding code examples. --- doc/languages-frameworks/python.section.md | 48 +++++++++++++++++++ .../interpreters/python/hooks/sphinx-hook.sh | 13 ----- 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/doc/languages-frameworks/python.section.md b/doc/languages-frameworks/python.section.md index 8de523e89df..70ebe95a080 100644 --- a/doc/languages-frameworks/python.section.md +++ b/doc/languages-frameworks/python.section.md @@ -744,6 +744,53 @@ with the exception of `other` (see `format` in unittestFlags = [ "-s" "tests" "-v" ]; ``` +##### Using sphinxHook {#using-sphinxhook} + +The `sphinxHook` is a helpful tool to build documentation and manpages +using the popular Sphinx documentation generator. +It is setup to automatically find common documentation source paths and +render them using the default `html` style. + +``` + outputs = [ + "out" + "doc" + ]; + + nativeBuildInputs = [ + sphinxHook + ]; +``` + +The hook will automatically build and install the artifact into the +`doc` output, if it exists. It also provides an automatic diversion +for the artifacts of the `man` builder into the `man` target. + +``` + outputs = [ + "out" + "doc" + "man" + ]; + + # Use multiple builders + sphinxBuilders = [ + "singlehtml" + "man" + ]; +``` + +Overwrite `sphinxRoot` when the hook is unable to find your +documentation source root. + +``` + # Configure sphinxRoot for uncommon paths + sphinxRoot = "weird/docs/path"; +``` + +The hook is also available to packages outside the python ecosystem by +referencing it using `python3.pkgs.sphinxHook`. + ### Develop local package {#develop-local-package} As a Python developer you're likely aware of [development mode](http://setuptools.readthedocs.io/en/latest/setuptools.html#development-mode) @@ -1273,6 +1320,7 @@ are used in `buildPythonPackage`. - `pythonRemoveBinBytecode` to remove bytecode from the `/bin` folder. - `setuptoolsBuildHook` to build a wheel using `setuptools`. - `setuptoolsCheckHook` to run tests with `python setup.py test`. +- `sphinxHook` to build documentation and manpages using Sphinx. - `venvShellHook` to source a Python 3 `venv` at the `venvDir` location. A `venv` is created if it does not yet exist. `postVenvCreation` can be used to to run commands only after venv is first created. diff --git a/pkgs/development/interpreters/python/hooks/sphinx-hook.sh b/pkgs/development/interpreters/python/hooks/sphinx-hook.sh index 5bbb17e2126..ab5e0f83a33 100644 --- a/pkgs/development/interpreters/python/hooks/sphinx-hook.sh +++ b/pkgs/development/interpreters/python/hooks/sphinx-hook.sh @@ -1,16 +1,3 @@ -# This hook automatically finds Sphinx documentation, builds it in html format -# and installs it. -# -# This hook knows about several popular locations in which subdirectory -# documentation may be, but in very unusual cases $sphinxRoot directory can be -# set explicitly. -# -# Name of the directory relative to ${doc:-$out}/share/doc is normally also -# deduced automatically, but can be overridden with $sphinxOutdir variable. -# -# Sphinx build system can depend on arbitrary amount of python modules, client -# code is responsible for ensuring that all dependencies are present. - # shellcheck shell=bash echo "Sourcing sphinx-hook" From 5a852f4085cbf4d6521a4cbb775dc9a15bd0298c Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Tue, 23 Aug 2022 15:46:52 +0200 Subject: [PATCH 3/6] borgbackup: migrate to sphinxHook --- pkgs/tools/backup/borgbackup/default.nix | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/pkgs/tools/backup/borgbackup/default.nix b/pkgs/tools/backup/borgbackup/default.nix index 7ffca4203bb..adfd0b42e39 100644 --- a/pkgs/tools/backup/borgbackup/default.nix +++ b/pkgs/tools/backup/borgbackup/default.nix @@ -33,13 +33,15 @@ python3.pkgs.buildPythonApplication rec { setuptools-scm # docs - sphinx + sphinxHook guzzle_sphinx_theme # shell completions installShellFiles ]; + sphinxBuilders = [ "singlehtml" "man" ]; + buildInputs = [ libb2 lz4 @@ -67,14 +69,6 @@ python3.pkgs.buildPythonApplication rec { ]; postInstall = '' - make -C docs singlehtml - mkdir -p $out/share/doc/borg - cp -R docs/_build/singlehtml $out/share/doc/borg/html - - make -C docs man - mkdir -p $out/share/man - cp -R docs/_build/man $out/share/man/man1 - installShellCompletion --cmd borg \ --bash scripts/shell_completions/bash/borg \ --fish scripts/shell_completions/fish/borg.fish \ From caf2d010edbe5cd2193b9849798fa44858e08c00 Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Wed, 24 Aug 2022 03:32:13 +0200 Subject: [PATCH 4/6] python3Packages.sphinxHook: Rename loop var in source root detection The loop variable to find the documentation source root was unnecessarily short, rename it to something meaningful. --- .../interpreters/python/hooks/sphinx-hook.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkgs/development/interpreters/python/hooks/sphinx-hook.sh b/pkgs/development/interpreters/python/hooks/sphinx-hook.sh index ab5e0f83a33..e38562d8554 100644 --- a/pkgs/development/interpreters/python/hooks/sphinx-hook.sh +++ b/pkgs/development/interpreters/python/hooks/sphinx-hook.sh @@ -6,7 +6,7 @@ declare -a __sphinxBuilders buildSphinxPhase() { echo "Executing buildSphinxPhase" - local __sphinxRoot="" o + local __sphinxRoot="" runHook preBuildSphinx if [[ -n "${sphinxRoot:-}" ]] ; then # explicit root @@ -16,10 +16,10 @@ buildSphinxPhase() { fi __sphinxRoot=$sphinxRoot else - for o in doc docs doc/source docs/source ; do - if [[ -f "$o/conf.py" ]] ; then - echo "Sphinx documentation found in $o" - __sphinxRoot=$o + for candidate in doc docs doc/source docs/source ; do + if [[ -f "$candidate/conf.py" ]] ; then + echo "Sphinx documentation found in $candidate" + __sphinxRoot=$candidate break fi done From 19f4e14dcff3dd23885f3707838c91c36154a01a Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Wed, 24 Aug 2022 04:07:52 +0200 Subject: [PATCH 5/6] python3Packages.sphinxHook: run install phase in predist Run the installSphinxPhase earlier, so install errors are caught early, before long and expensive test are started. --- pkgs/development/interpreters/python/hooks/sphinx-hook.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkgs/development/interpreters/python/hooks/sphinx-hook.sh b/pkgs/development/interpreters/python/hooks/sphinx-hook.sh index e38562d8554..ae386ca859f 100644 --- a/pkgs/development/interpreters/python/hooks/sphinx-hook.sh +++ b/pkgs/development/interpreters/python/hooks/sphinx-hook.sh @@ -69,5 +69,4 @@ installSphinxPhase() { runHook postInstallSphinx } -preDistPhases+=" buildSphinxPhase" -postPhases+=" installSphinxPhase" +preDistPhases+=" buildSphinxPhase installSphinxPhase" From 986c31401e06456175d09e6fd4accc84b58b8b71 Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Wed, 24 Aug 2022 23:03:31 +0200 Subject: [PATCH 6/6] doc/python: Properly sort pythonRelaxDepsHook in hook list --- doc/languages-frameworks/python.section.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/languages-frameworks/python.section.md b/doc/languages-frameworks/python.section.md index 70ebe95a080..bde4183a65b 100644 --- a/doc/languages-frameworks/python.section.md +++ b/doc/languages-frameworks/python.section.md @@ -1317,6 +1317,8 @@ are used in `buildPythonPackage`. - `pytestCheckHook` to run tests with `pytest`. See [example usage](#using-pytestcheckhook). - `pythonCatchConflictsHook` to check whether a Python package is not already existing. - `pythonImportsCheckHook` to check whether importing the listed modules works. +- `pythonRelaxDepsHook` will relax Python dependencies restrictions for the package. + See [example usage](#using-pythonrelaxdepshook). - `pythonRemoveBinBytecode` to remove bytecode from the `/bin` folder. - `setuptoolsBuildHook` to build a wheel using `setuptools`. - `setuptoolsCheckHook` to run tests with `python setup.py test`. @@ -1326,8 +1328,6 @@ are used in `buildPythonPackage`. to run commands only after venv is first created. - `wheelUnpackHook` to move a wheel to the correct folder so it can be installed with the `pipInstallHook`. -- `pythonRelaxDepsHook` will relax Python dependencies restrictions for the package. - See [example usage](#using-pythonrelaxdepshook). - `unittestCheckHook` will run tests with `python -m unittest discover`. See [example usage](#using-unittestcheckhook). ### Development mode {#development-mode}