On Mon, Jan 22, 2024 at 11:35 AM Juraj Linkeš
<juraj.lin...@pantheon.tech> wrote:
>
> The tool used to generate developer docs is Sphinx, which is already in
> use in DPDK. The same configuration is used to preserve style, but it's
> been augmented with doc-generating configuration. There's a change that
> modifies how the sidebar displays the content hierarchy that's been put
> into an if block to not interfere with regular docs.
>
> Sphinx generates the documentation from Python docstrings. The docstring
> format is the Google format [0] which requires the sphinx.ext.napoleon
> extension. The other extension, sphinx.ext.intersphinx, enables linking
> to object in external documentations, such as the Python documentation.
>
> There are two requirements for building DTS docs:
> * The same Python version as DTS or higher, because Sphinx imports the
> code.
> * Also the same Python packages as DTS, for the same reason.
>
> [0]
> https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings
>
> Signed-off-by: Juraj Linkeš <juraj.lin...@pantheon.tech>
> ---
> buildtools/call-sphinx-build.py | 33 +++++++++++++++++++---------
> doc/api/doxy-api-index.md | 3 +++
> doc/api/doxy-api.conf.in | 2 ++
> doc/api/meson.build | 11 +++++++---
> doc/guides/conf.py | 39 ++++++++++++++++++++++++++++-----
> doc/guides/meson.build | 1 +
> doc/guides/tools/dts.rst | 34 +++++++++++++++++++++++++++-
> dts/doc/meson.build | 27 +++++++++++++++++++++++
> dts/meson.build | 16 ++++++++++++++
> meson.build | 1 +
> 10 files changed, 148 insertions(+), 19 deletions(-)
> create mode 100644 dts/doc/meson.build
> create mode 100644 dts/meson.build
>
> diff --git a/buildtools/call-sphinx-build.py b/buildtools/call-sphinx-build.py
> index 39a60d09fa..aea771a64e 100755
> --- a/buildtools/call-sphinx-build.py
> +++ b/buildtools/call-sphinx-build.py
> @@ -3,37 +3,50 @@
> # Copyright(c) 2019 Intel Corporation
> #
>
> +import argparse
> import sys
> import os
> from os.path import join
> from subprocess import run, PIPE, STDOUT
> from packaging.version import Version
>
> -# assign parameters to variables
> -(sphinx, version, src, dst, *extra_args) = sys.argv[1:]
> +parser = argparse.ArgumentParser()
> +parser.add_argument('sphinx')
> +parser.add_argument('version')
> +parser.add_argument('src')
> +parser.add_argument('dst')
> +parser.add_argument('--dts-root', default=None)
> +args, extra_args = parser.parse_known_args()
>
> # set the version in environment for sphinx to pick up
> -os.environ['DPDK_VERSION'] = version
> +os.environ['DPDK_VERSION'] = args.version
> +if args.dts_root:
> + os.environ['DTS_ROOT'] = args.dts_root
>
> # for sphinx version >= 1.7 add parallelism using "-j auto"
> -ver = run([sphinx, '--version'], stdout=PIPE,
> +ver = run([args.sphinx, '--version'], stdout=PIPE,
> stderr=STDOUT).stdout.decode().split()[-1]
> -sphinx_cmd = [sphinx] + extra_args
> +sphinx_cmd = [args.sphinx] + extra_args
> if Version(ver) >= Version('1.7'):
> sphinx_cmd += ['-j', 'auto']
>
> # find all the files sphinx will process so we can write them as dependencies
> srcfiles = []
> -for root, dirs, files in os.walk(src):
> +for root, dirs, files in os.walk(args.src):
> srcfiles.extend([join(root, f) for f in files])
>
> +if not os.path.exists(args.dst):
> + os.makedirs(args.dst)
> +
> # run sphinx, putting the html output in a "html" directory
> -with open(join(dst, 'sphinx_html.out'), 'w') as out:
> - process = run(sphinx_cmd + ['-b', 'html', src, join(dst, 'html')],
> - stdout=out)
> +with open(join(args.dst, 'sphinx_html.out'), 'w') as out:
> + process = run(
> + sphinx_cmd + ['-b', 'html', args.src, join(args.dst, 'html')],
> + stdout=out
> + )
>
> # create a gcc format .d file giving all the dependencies of this doc build
> -with open(join(dst, '.html.d'), 'w') as d:
> +with open(join(args.dst, '.html.d'), 'w') as d:
> d.write('html: ' + ' '.join(srcfiles) + '\n')
>
> sys.exit(process.returncode)
> diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
> index a6a768bd7c..b49b24acce 100644
> --- a/doc/api/doxy-api-index.md
> +++ b/doc/api/doxy-api-index.md
> @@ -241,3 +241,6 @@ The public API headers are grouped by topics:
> [experimental APIs](@ref rte_compat.h),
> [ABI versioning](@ref rte_function_versioning.h),
> [version](@ref rte_version.h)
> +
> +- **tests**:
> + [**DTS**](@dts_api_main_page)
> diff --git a/doc/api/doxy-api.conf.in b/doc/api/doxy-api.conf.in
> index e94c9e4e46..d53edeba57 100644
> --- a/doc/api/doxy-api.conf.in
> +++ b/doc/api/doxy-api.conf.in
> @@ -121,6 +121,8 @@ SEARCHENGINE = YES
> SORT_MEMBER_DOCS = NO
> SOURCE_BROWSER = YES
>
> +ALIASES = "dts_api_main_page=@DTS_API_MAIN_PAGE@"
> +
> EXAMPLE_PATH = @TOPDIR@/examples
> EXAMPLE_PATTERNS = *.c
> EXAMPLE_RECURSIVE = YES
> diff --git a/doc/api/meson.build b/doc/api/meson.build
> index 5b50692df9..ffc75d7b5a 100644
> --- a/doc/api/meson.build
> +++ b/doc/api/meson.build
> @@ -1,6 +1,7 @@
> # SPDX-License-Identifier: BSD-3-Clause
> # Copyright(c) 2018 Luca Boccassi <bl...@debian.org>
>
> +doc_api_build_dir = meson.current_build_dir()
> doxygen = find_program('doxygen', required: get_option('enable_docs'))
>
> if not doxygen.found()
> @@ -32,14 +33,18 @@ example = custom_target('examples.dox',
> # set up common Doxygen configuration
> cdata = configuration_data()
> cdata.set('VERSION', meson.project_version())
> -cdata.set('API_EXAMPLES', join_paths(dpdk_build_root, 'doc', 'api',
> 'examples.dox'))
> -cdata.set('OUTPUT', join_paths(dpdk_build_root, 'doc', 'api'))
> +cdata.set('API_EXAMPLES', join_paths(doc_api_build_dir, 'examples.dox'))
> +cdata.set('OUTPUT', doc_api_build_dir)
> cdata.set('TOPDIR', dpdk_source_root)
> -cdata.set('STRIP_FROM_PATH', ' '.join([dpdk_source_root,
> join_paths(dpdk_build_root, 'doc', 'api')]))
> +cdata.set('STRIP_FROM_PATH', ' '.join([dpdk_source_root, doc_api_build_dir]))
> cdata.set('WARN_AS_ERROR', 'NO')
> if get_option('werror')
> cdata.set('WARN_AS_ERROR', 'YES')
> endif
> +# A local reference must be relative to the main index.html page
> +# The path below can't be taken from the DTS meson file as that would
> +# require recursive subdir traversal (doc, dts, then doc again)
> +cdata.set('DTS_API_MAIN_PAGE', join_paths('..', 'dts', 'html', 'index.html'))
>
> # configure HTML Doxygen run
> html_cdata = configuration_data()
> diff --git a/doc/guides/conf.py b/doc/guides/conf.py
> index 0f7ff5282d..b442a1f76c 100644
> --- a/doc/guides/conf.py
> +++ b/doc/guides/conf.py
> @@ -7,10 +7,9 @@
> from sphinx import __version__ as sphinx_version
> from os import listdir
> from os import environ
> -from os.path import basename
> -from os.path import dirname
> +from os.path import basename, dirname
> from os.path import join as path_join
> -from sys import argv, stderr
> +from sys import argv, stderr, path
>
> import configparser
>
> @@ -24,6 +23,37 @@
> file=stderr)
> pass
>
> +# Napoleon enables the Google format of Python doscstrings, used in DTS
> +# Intersphinx allows linking to external projects, such as Python docs, also
> used in DTS
> +extensions = ['sphinx.ext.napoleon', 'sphinx.ext.intersphinx']
> +
> +# DTS Python docstring options
> +autodoc_default_options = {
> + 'members': True,
> + 'member-order': 'bysource',
> + 'show-inheritance': True,
> +}
> +autodoc_class_signature = 'separated'
> +autodoc_typehints = 'both'
> +autodoc_typehints_format = 'short'
> +autodoc_typehints_description_target = 'documented'
> +napoleon_numpy_docstring = False
> +napoleon_attr_annotations = True
> +napoleon_preprocess_types = True
> +add_module_names = False
> +toc_object_entries = True
> +toc_object_entries_show_parents = 'hide'
> +intersphinx_mapping = {'python': ('https://docs.python.org/3', None)}
> +
> +dts_root = environ.get('DTS_ROOT')
> +if dts_root:
> + path.append(dts_root)
> + # DTS Sidebar config
> + html_theme_options = {
> + 'collapse_navigation': False,
> + 'navigation_depth': -1,
> + }
> +
> stop_on_error = ('-W' in argv)
>
> project = 'Data Plane Development Kit'
> @@ -35,8 +65,7 @@
> html_show_copyright = False
> highlight_language = 'none'
>
> -release = environ.setdefault('DPDK_VERSION', "None")
> -version = release
> +version = environ.setdefault('DPDK_VERSION', "None")
>
> master_doc = 'index'
>
> diff --git a/doc/guides/meson.build b/doc/guides/meson.build
> index 51f81da2e3..8933d75f6b 100644
> --- a/doc/guides/meson.build
> +++ b/doc/guides/meson.build
> @@ -1,6 +1,7 @@
> # SPDX-License-Identifier: BSD-3-Clause
> # Copyright(c) 2018 Intel Corporation
>
> +doc_guides_source_dir = meson.current_source_dir()
> sphinx = find_program('sphinx-build', required: get_option('enable_docs'))
>
> if not sphinx.found()
> diff --git a/doc/guides/tools/dts.rst b/doc/guides/tools/dts.rst
> index 846696e14e..21d3d89fc2 100644
> --- a/doc/guides/tools/dts.rst
> +++ b/doc/guides/tools/dts.rst
> @@ -278,7 +278,12 @@ and try not to divert much from it.
> The :ref:`DTS developer tools <dts_dev_tools>` will issue warnings
> when some of the basics are not met.
>
> -The code must be properly documented with docstrings.
> +The API documentation, which is a helpful reference when developing, may be
> accessed
> +in the code directly or generated with the :ref:`API docs build steps
> <building_api_docs>`.
> +When adding new files or modifying the directory structure, the
> corresponding changes must
> +be made to DTS api doc sources in ``dts/doc``.
> +
> +Speaking of which, the code must be properly documented with docstrings.
> The style must conform to the `Google style
>
> <https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings>`_.
> See an example of the style `here
> @@ -413,6 +418,33 @@ the DTS code check and format script.
> Refer to the script for usage: ``devtools/dts-check-format.sh -h``.
>
>
> +.. _building_api_docs:
> +
> +Building DTS API docs
> +---------------------
> +
> +To build DTS API docs, install the dependencies with Poetry, then enter its
> shell:
> +
> +.. code-block:: console
> +
> + poetry install --with docs
> + poetry shell
> +
The only thing to note here is with newer versions of poetry this will
start to throw warnings because of the way we use poetry and don't
have a root package. It is just a warning message so it shouldn't
cause any real problems, but I believe the way we should be handling
it is passing --no-root into poetry install so that it knows not to
use the root package.
>
> +The documentation is built using the standard DPDK build system. After
> executing the meson command
> +and entering Poetry's shell, build the documentation with:
> +
> +.. code-block:: console
> +
> + ninja -C build dts-doc
> +
> +The output is generated in ``build/doc/api/dts/html``.
> +
> +.. Note::
> +
> + Make sure to fix any Sphinx warnings when adding or updating docstrings.
> Also make sure to run
> + the ``devtools/dts-check-format.sh`` script and address any issues it
> finds.
> +
> +
> Configuration Schema
> --------------------
>
> diff --git a/dts/doc/meson.build b/dts/doc/meson.build
> new file mode 100644
> index 0000000000..01b7b51034
> --- /dev/null
> +++ b/dts/doc/meson.build
> @@ -0,0 +1,27 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright(c) 2023 PANTHEON.tech s.r.o.
> +
> +sphinx = find_program('sphinx-build', required: false)
> +sphinx_apidoc = find_program('sphinx-apidoc', required: false)
> +
> +if not sphinx.found() or not sphinx_apidoc.found()
> + subdir_done()
> +endif
> +
> +dts_doc_api_build_dir = join_paths(doc_api_build_dir, 'dts')
> +
> +extra_sphinx_args = ['-E', '-c', doc_guides_source_dir, '--dts-root',
> dts_dir]
> +if get_option('werror')
> + extra_sphinx_args += '-W'
> +endif
> +
> +htmldir = join_paths(get_option('datadir'), 'doc', 'dpdk', 'dts')
> +dts_api_html = custom_target('dts_api_html',
> + output: 'html',
> + command: [sphinx_wrapper, sphinx, meson.project_version(),
> + meson.current_source_dir(), dts_doc_api_build_dir,
> extra_sphinx_args],
> + build_by_default: false,
> + install: get_option('enable_docs'),
> + install_dir: htmldir)
> +doc_targets += dts_api_html
> +doc_target_names += 'DTS_API_HTML'
> diff --git a/dts/meson.build b/dts/meson.build
> new file mode 100644
> index 0000000000..e8ce0f06ac
> --- /dev/null
> +++ b/dts/meson.build
> @@ -0,0 +1,16 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright(c) 2023 PANTHEON.tech s.r.o.
> +
> +doc_targets = []
> +doc_target_names = []
> +dts_dir = meson.current_source_dir()
> +
> +subdir('doc')
> +
> +if doc_targets.length() == 0
> + message = 'No docs targets found'
> +else
> + message = 'Built docs:'
> +endif
> +run_target('dts-doc', command: [echo, message, doc_target_names],
> + depends: doc_targets)
> diff --git a/meson.build b/meson.build
> index 5e161f43e5..001fdcbbbf 100644
> --- a/meson.build
> +++ b/meson.build
> @@ -87,6 +87,7 @@ subdir('app')
>
> # build docs
> subdir('doc')
> +subdir('dts')
>
> # build any examples explicitly requested - useful for developers - and
> # install any example code into the appropriate install path
> --
> 2.34.1
>
Reviewed-by: Jeremy Spewock <jspew...@iol.unh.edu>
