As the name suggests, the qapi2texi script converts JSON QAPI
description into a texi file suitable for different target
formats (info/man/txt/pdf/html...).
It parses the following kind of blocks:
Free-form:
##
# = Section
# == Subsection
#
# Some text foo with *emphasis*
# 1. with a list
# 2. like that
#
# And some code:
# | $ echo foo
# | -> do this
# | <- get that
#
##
Symbol:
##
# @symbol:
#
# Symbol body ditto ergo sum. Foo bar
# baz ding.
#
# @arg: foo
# @arg: #optional foo
#
# Returns: returns bla bla
# Or bla blah
#
# Since: version
# Notes: notes, comments can have
# - itemized list
# - like this
#
# Example:
#
# -> { "execute": "quit" }
# <- { "return": {} }
#
##
That's roughly following the following BNF grammar:
api_comment = "##\n" comment "##\n"
comment = freeform_comment | symbol_comment
freeform_comment = { "#" text "\n" }
symbol_comment = "#" "@" name ":\n" { freeform | member | meta }
member = "#" '@' name ':' [ text ] freeform_comment
meta = "#" ( "Returns:", "Since:", "Note:", "Notes:", "Example:", "Examples:" )
[ text ] freeform_comment
text = free-text markdown-like, "#optional" for members
Thanks to the following json expressions, the documentation is enhanced
with extra information about the type of arguments and return value
expected.
Signed-off-by: Marc-André Lureau <marcandre.lur...@redhat.com>
---
scripts/qapi.py | 175 ++++++++++++++++++++++++++-
scripts/qapi2texi.py | 316 +++++++++++++++++++++++++++++++++++++++++++++++++
docs/qapi-code-gen.txt | 44 +++++--
3 files changed, 524 insertions(+), 11 deletions(-)
create mode 100755 scripts/qapi2texi.py
diff --git a/scripts/qapi.py b/scripts/qapi.py
index 21bc32f..ed52ee4 100644
--- a/scripts/qapi.py
+++ b/scripts/qapi.py
@@ -122,6 +122,103 @@ class QAPIExprError(Exception):
"%s:%d: %s" % (self.info['file'], self.info['line'], self.msg)
+class QAPIDoc(object):
+ def __init__(self, parser):
+ self.parser = parser
+ self.symbol = None
+ self.body = []
+ # args is {'arg': 'doc', ...}
+ self.args = OrderedDict()
+ # meta is [(Since/Notes/Examples/Returns:, 'doc'), ...]
+ self.meta = []
+ # the current section to populate, array of [dict, key, comment...]
+ self.section = None
+ self.expr_elem = None
+
+ def get_body(self):
+ return "\n".join(self.body)
+
+ def has_meta(self, name):
+ """Returns True if the doc has a meta section 'name'"""
+ return next((True for i in self.meta if i[0] == name), False)
+
+ def append(self, line):
+ """Adds a # comment line, to be parsed and added in a section"""
+ line = line[1:]
+ if len(line) == 0:
+ self._append_section(line)
+ return
+
+ if line[0] != ' ':
+ raise QAPISchemaError(self.parser, "missing space after #")
+
+ line = line[1:]
+ # take the first word out
+ name = line.split(' ', 1)[0]
+ if name.startswith("@") and name.endswith(":"):
+ line = line[len(name):]
+ name = name[1:-1]
+ if self.symbol is None:
+ # the first is the symbol this APIDoc object documents
+ if len(self.body):
+ raise QAPISchemaError(self.parser, "symbol must come
first")
+ self.symbol = name
+ else:
+ # else an arg
+ self._start_args_section(name)
+ elif self.symbol and name in (
+ "Returns:", "Since:",
+ # those are often singular or plural
+ "Note:", "Notes:",
+ "Example:", "Examples:"):
+ # new "meta" section
+ line = line[len(name):]
+ self._start_meta_section(name[:-1])
+
+ self._append_section(line)
+
+ def _start_args_section(self, name):
+ self.end_section()
+ if self.args.has_key(name):
+ raise QAPISchemaError(self.parser, "'%s' arg duplicated" % name)
+ self.section = [self.args, name]
+
+ def _start_meta_section(self, name):
+ self.end_section()
+ if name in ("Returns", "Since") and self.has_meta(name):
+ raise QAPISchemaError(self.parser, "'%s' section duplicated" %
name)
+ self.section = [self.meta, name]
+
+ def _append_section(self, line):
+ """Add a comment to the current section, or the comment body"""
+ if self.section:
+ name = self.section[1]
+ if not name.startswith("Example"):
+ # an empty line ends the section, except with Example
+ if len(self.section) > 2 and len(line) == 0:
+ self.end_section()
+ return
+ # Example is verbatim
+ line = line.strip()
+ if len(line) > 0:
+ self.section.append(line)
+ else:
+ self.body.append(line.strip())
+
+ def end_section(self):
+ if self.section is not None:
+ target = self.section[0]
+ name = self.section[1]
+ if len(self.section) < 3:
+ raise QAPISchemaError(self.parser, "Empty doc section")
+ doc = "\n".join(self.section[2:])
+ if isinstance(target, dict):
+ target[name] = doc
+ else:
+ target.append((name, doc))
+ self.section = None
+
+
class QAPISchemaParser(object):
def __init__(self, fp, previously_included=[], incl_info=None):
@@ -137,9 +234,15 @@ class QAPISchemaParser(object):
self.line = 1
self.line_pos = 0
self.exprs = []
+ self.docs = []
self.accept()
while self.tok is not None:
+ if self.tok == '#' and self.val.startswith('##'):
+ doc = self.get_doc()
+ self.docs.append(doc)
+ continue
+
expr_info = {'file': fname, 'line': self.line,
'parent': self.incl_info}
expr = self.get_expr(False)
@@ -160,6 +263,7 @@ class QAPISchemaParser(object):
raise QAPIExprError(expr_info, "Inclusion loop for %s"
% include)
inf = inf['parent']
+
# skip multiple include of the same file
if incl_abs_fname in previously_included:
continue
@@ -171,12 +275,40 @@ class QAPISchemaParser(object):
exprs_include = QAPISchemaParser(fobj, previously_included,
expr_info)
self.exprs.extend(exprs_include.exprs)
+ self.docs.extend(exprs_include.docs)
else:
expr_elem = {'expr': expr,
'info': expr_info}
+ if len(self.docs) > 0:
+ self.docs[-1].expr_elem = expr_elem
self.exprs.append(expr_elem)
- def accept(self):
+ def get_doc(self):
+ if self.val != '##':
+ raise QAPISchemaError(self, "Doc comment not starting with '##'")
+
+ doc = QAPIDoc(self)
+ self.accept(False)
+ while self.tok == '#':
+ if self.val.startswith('##'):
+ # ## ends doc
+ if self.val != '##':
+ raise QAPISchemaError(self, "non-empty '##' line %s"
+ % self.val)
+ self.accept()
+ doc.end_section()
+ return doc
+ else:
+ doc.append(self.val)
+ self.accept(False)
+
+ if self.val != '##':
+ raise QAPISchemaError(self, "Doc comment not finishing with '##'")
+
+ doc.end_section()
+ return doc
+
+ def accept(self, skip_comment=True):
while True:
self.tok = self.src[self.cursor]
self.pos = self.cursor
@@ -184,7 +316,13 @@ class QAPISchemaParser(object):
self.val = None
if self.tok == '#':
+ if self.src[self.cursor] == '#':
+ # ## starts a doc comment
+ skip_comment = False
self.cursor = self.src.find('\n', self.cursor)
+ self.val = self.src[self.pos:self.cursor]
+ if not skip_comment:
+ return
elif self.tok in "{}:,[]":
return
elif self.tok == "'":
@@ -779,6 +917,41 @@ def check_exprs(exprs):
return exprs
+def check_docs(docs):
+ for doc in docs:
+ expr_elem = doc.expr_elem
+ if not expr_elem:
+ continue
+
+ expr = expr_elem['expr']
+ for i in ('enum', 'union', 'alternate', 'struct', 'command', 'event'):
+ if i in expr:
+ meta = i
+ break
+
+ info = expr_elem['info']
+ name = expr[meta]
+ if doc.symbol != name:
+ raise QAPIExprError(info,
+ "Documentation symbol mismatch '%s' != '%s'"
+ % (doc.symbol, name))
+ if not 'command' in expr and doc.has_meta('Returns'):
+ raise QAPIExprError(info, "Invalid return documentation")
+
+ doc_args = set(doc.args.keys())
+ if meta == 'union':
+ data = expr.get('base', [])
+ else:
+ data = expr.get('data', [])
+ if isinstance(data, dict):
+ data = data.keys()
+ args = set([k.strip('*') for k in data])
+ if meta == 'alternate' or \
+ (meta == 'union' and not expr.get('discriminator')):
+ args.add('type')
+ if not doc_args.issubset(args):
+ raise QAPIExprError(info, "Members documentation is not a subset
of"
+ " API %r > %r" % (list(doc_args), list(args)))
#
# Schema compiler frontend
diff --git a/scripts/qapi2texi.py b/scripts/qapi2texi.py
new file mode 100755
index 0000000..7e2440c
--- /dev/null
+++ b/scripts/qapi2texi.py
@@ -0,0 +1,316 @@
+#!/usr/bin/env python
+# QAPI texi generator
+#
+# This work is licensed under the terms of the GNU LGPL, version 2+.
+# See the COPYING file in the top-level directory.
+"""This script produces the documentation of a qapi schema in texinfo format"""
+import re
+import sys
+
+from qapi import *
+
+COMMAND_FMT = """
+@deftypefn {type} {{{ret}}} {name} @
+{{{args}}}
+
+{body}
+
+@end deftypefn
+
+""".format
+
+ENUM_FMT = """
+@deftp Enum {name}
+
+{body}
+
+@end deftp
+
+""".format
+
+STRUCT_FMT = """
+@deftp {type} {name} @
+{{{attrs}}}
+
+{body}
+
+@end deftp
+
+""".format
+
+EXAMPLE_FMT = """@example
+{code}
+@end example
+""".format
+
+
+def subst_strong(doc):
+ """Replaces *foo* by @strong{foo}"""
+ return re.sub(r'\*([^_\n]+)\*', r'@emph{\1}', doc)
+
+
+def subst_emph(doc):
+ """Replaces _foo_ by @emph{foo}"""
+ return re.sub(r'\s_([^_\n]+)_\s', r' @emph{\1} ', doc)
+
+
+def subst_vars(doc):
+ """Replaces @var by @var{var}"""
+ return re.sub(r'@([\w-]+)', r'@var{\1}', doc)
+
+
+def subst_braces(doc):
+ """Replaces {} with @{ @}"""
+ return doc.replace("{", "@{").replace("}", "@}")
+
+
+def texi_example(doc):
+ """Format @example"""
+ doc = subst_braces(doc).strip('\n')
+ return EXAMPLE_FMT(code=doc)
+
+
+def texi_comment(doc):
+ """
+ Format a comment
+
+ Lines starting with:
+ - |: generates an @example
+ - =: generates @section
+ - ==: generates @subsection
+ - 1. or 1): generates an @enumerate @item
+ - o/*/-: generates an @itemize list
+ """
+ lines = []
+ doc = subst_braces(doc)
+ doc = subst_vars(doc)
+ doc = subst_emph(doc)
+ doc = subst_strong(doc)
+ inlist = ""
+ lastempty = False
+ for line in doc.split('\n'):
+ empty = line == ""
+
+ if line.startswith("| "):
+ line = EXAMPLE_FMT(code=line[1:])
+ elif line.startswith("= "):
+ line = "@section " + line[1:]
+ elif line.startswith("== "):
+ line = "@subsection " + line[2:]
+ elif re.match("^([0-9]*[.)]) ", line):
+ if not inlist:
+ lines.append("@enumerate")
+ inlist = "enumerate"
+ line = line[line.find(" ")+1:]
+ lines.append("@item")
+ elif re.match("^[o*-] ", line):
+ if not inlist:
+ lines.append("@itemize %s" % {'o': "@bullet",
+ '*': "@minus",
+ '-': ""}[line[0]])
+ inlist = "itemize"
+ lines.append("@item")
+ line = line[2:]
+ elif lastempty and inlist:
+ lines.append("@end %s\n" % inlist)
+ inlist = ""
+
+ lastempty = empty
+ lines.append(line)
+
+ if inlist:
+ lines.append("@end %s\n" % inlist)
+ return "\n".join(lines)
+
+
+def texi_args(expr):
+ """
+ Format the functions/structure/events.. arguments/members
+ """
+ data = expr["data"] if "data" in expr else {}
+ if isinstance(data, str):
+ args = data
+ else:
+ arg_list = []
+ for name, typ in data.iteritems():
+ # optional arg
+ if name.startswith("*"):
+ name = name[1:]
+ arg_list.append("['%s': @var{%s}]" % (name, typ))
+ # regular arg
+ else:
+ arg_list.append("'%s': @var{%s}" % (name, typ))
+ args = ", ".join(arg_list)
+ return args
+
+def section_order(section):
+ return {"Returns": 0,
+ "Note": 1,
+ "Notes": 1,
+ "Since": 2,
+ "Example": 3,
+ "Examples": 3}[section]
+
+def texi_body(doc, arg="@var"):
+ """
+ Format the body of a symbol documentation:
+ - a table of arguments
+ - followed by "Returns/Notes/Since/Example" sections
+ """
+ body = "@table %s\n" % arg
+ for arg, desc in doc.args.iteritems():
+ if desc.startswith("#optional"):
+ desc = desc[10:]
+ arg += "*"
+ elif desc.endswith("#optional"):
+ desc = desc[:-10]
+ arg += "*"
+ body += "@item %s\n%s\n" % (arg, texi_comment(desc))
+ body += "@end table\n"
+ body += texi_comment(doc.get_body())
+
+ meta = sorted(doc.meta, key=lambda i: section_order(i[0]))
+ for m in meta:
+ key, doc = m
+ func = texi_comment
+ if key.startswith("Example"):
+ func = texi_example
+
+ body += "\n@quotation %s\n%s\n@end quotation" % \
+ (key, func(doc))
+ return body
+
+
+def texi_alternate(expr, doc):
+ """
+ Format an alternate to texi
+ """
+ args = texi_args(expr)
+ body = texi_body(doc)
+ return STRUCT_FMT(type="Alternate",
+ name=doc.symbol,
+ attrs="[ " + args + " ]",
+ body=body)
+
+
+def texi_union(expr, doc):
+ """
+ Format an union to texi
+ """
+ args = texi_args(expr)
+ body = texi_body(doc)
+ return STRUCT_FMT(type="Union",
+ name=doc.symbol,
+ attrs="[ " + args + " ]",
+ body=body)
+
+
+def texi_enum(_, doc):
+ """
+ Format an enum to texi
+ """
+ body = texi_body(doc, "@samp")
+ return ENUM_FMT(name=doc.symbol,
+ body=body)
+
+
+def texi_struct(expr, doc):
+ """
+ Format a struct to texi
+ """
+ args = texi_args(expr)
+ body = texi_body(doc)
+ return STRUCT_FMT(type="Struct",
+ name=doc.symbol,
+ attrs="@{ " + args + " @}",
+ body=body)
+
+
+def texi_command(expr, doc):
+ """
+ Format a command to texi
+ """
+ args = texi_args(expr)
+ ret = expr["returns"] if "returns" in expr else ""
+ body = texi_body(doc)
+ return COMMAND_FMT(type="Command",
+ name=doc.symbol,
+ ret=ret,
+ args="(" + args + ")",
+ body=body)
+
+
+def texi_event(expr, doc):
+ """
+ Format an event to texi
+ """
+ args = texi_args(expr)
+ body = texi_body(doc)
+ return COMMAND_FMT(type="Event",
+ name=doc.symbol,
+ ret="",
+ args="(" + args + ")",
+ body=body)
+
+
+def texi(docs):
+ """
+ Convert QAPI schema expressions to texi documentation
+ """
+ res = []
+ for doc in docs:
+ try:
+ expr_elem = doc.expr_elem
+ if expr_elem is None:
+ res.append(texi_body(doc))
+ continue
+
+ expr = expr_elem['expr']
+ (kind, _) = expr.items()[0]
+
+ fmt = {"command": texi_command,
+ "struct": texi_struct,
+ "enum": texi_enum,
+ "union": texi_union,
+ "alternate": texi_alternate,
+ "event": texi_event}
+ try:
+ fmt = fmt[kind]
+ except KeyError:
+ raise ValueError("Unknown expression kind '%s'" % kind)
+ res.append(fmt(expr, doc))
+ except:
+ print >>sys.stderr, "error at @%s" % qapi
+ raise
+
+ return '\n'.join(res)
+
+
+def parse_schema(fname):
+ """
+ Parse the given schema file and return the exprs
+ """
+ try:
+ schema = QAPISchemaParser(open(fname, "r"))
+ check_exprs(schema.exprs)
+ check_docs(schema.docs)
+ return schema.docs
+ except (QAPISchemaError, QAPIExprError), err:
+ print >>sys.stderr, err
+ exit(1)
+
+
+def main(argv):
+ """
+ Takes schema argument, prints result to stdout
+ """
+ if len(argv) != 2:
+ print >>sys.stderr, "%s: need exactly 1 argument: SCHEMA" % argv[0]
+ sys.exit(1)
+
+ docs = parse_schema(argv[1])
+ print texi(docs)
+
+
+if __name__ == "__main__":
+ main(sys.argv)
diff --git a/docs/qapi-code-gen.txt b/docs/qapi-code-gen.txt
index 2841c51..d82e251 100644
--- a/docs/qapi-code-gen.txt
+++ b/docs/qapi-code-gen.txt
@@ -45,16 +45,13 @@ QAPI parser does not). At present, there is no place where
a QAPI
schema requires the use of JSON numbers or null.
Comments are allowed; anything between an unquoted # and the following
-newline is ignored. Although there is not yet a documentation
-generator, a form of stylized comments has developed for consistently
-documenting details about an expression and when it was added to the
-schema. The documentation is delimited between two lines of ##, then
-the first line names the expression, an optional overview is provided,
-then individual documentation about each member of 'data' is provided,
-and finally, a 'Since: x.y.z' tag lists the release that introduced
-the expression. Optional members are tagged with the phrase
-'#optional', often with their default value; and extensions added
-after the expression was first released are also given a '(since
+newline is ignored. The documentation is delimited between two lines
+of ##, then the first line names the expression, an optional overview
+is provided, then individual documentation about each member of 'data'
+is provided, and finally, a 'Since: x.y.z' tag lists the release that
+introduced the expression. Optional members are tagged with the
+phrase '#optional', often with their default value; and extensions
+added after the expression was first released are also given a '(since
x.y.z)' comment. For example:
##
@@ -73,12 +70,39 @@ x.y.z)' comment. For example:
# (Since 2.0)
#
# Since: 0.14.0
+ #
+ # Notes: You can also make a list:
+ # - with items
+ # - like this
+ #
+ # Example:
+ #
+ # -> { "execute": ... }
+ # <- { "return": ... }
+ #
##
{ 'struct': 'BlockStats',
'data': {'*device': 'str', 'stats': 'BlockDeviceStats',
'*parent': 'BlockStats',
'*backing': 'BlockStats'} }
+It's also possible to create documentation sections, such as:
+
+ ##
+ # = Section
+ # == Subsection
+ #
+ # Some text foo with *emphasis*
+ # 1. with a list
+ # 2. like that
+ #
+ # And some code:
+ # | $ echo foo
+ # | -> do this
+ # | <- get that
+ #
+ ##
+
The schema sets up a series of types, as well as commands and events
that will use those types. Forward references are allowed: the parser
scans in two passes, where the first pass learns all type names, and
--
2.10.0
