this patch adds a command to auto-generate documentation from kernel's source tree.:: scripts/kerneldoc-src2rst --help E.g. to autodoc the kernel's ./include folder use:: scripts/kerneldoc-src2rst ./include /tmp/test123 >From the resulting reST-doctree you can build HTML rendered output like the one in [1]. I use it to see, if patches on the kernel_doc.py parser cause a regression (compare reST before/after the patch). Autodoc the whole source tree takes a long time. To speed up, the src2rst module uses multiprocessing from [2]. This means, it consumes all your CPUs. If you don't want this, use option '--threads=n'. [1] https://h2626237.stratoserver.net/kernel/linux_src_doc/index.html [2] https://docs.python.org/3.6/library/multiprocessing.html Signed-off-by: Markus Heiser <markus.heiser@xxxxxxxxxxx> --- Documentation/sphinx/src2rst.py | 229 ++++++++++++++++++++++++++++++++++++++++ scripts/kerneldoc-src2rst | 11 ++ 2 files changed, 240 insertions(+) create mode 100755 Documentation/sphinx/src2rst.py create mode 100755 scripts/kerneldoc-src2rst diff --git a/Documentation/sphinx/src2rst.py b/Documentation/sphinx/src2rst.py new file mode 100755 index 0000000..d8d6e7b --- /dev/null +++ b/Documentation/sphinx/src2rst.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8; mode: python -*- +# pylint: disable=C0103 + +u""" + src2rst + ~~~~~~~ + + Implementation of the ``kerneldoc-src2rst`` command. + + :copyright: Copyright (C) 2016 Markus Heiser + :license: GPL Version 2, June 1991 see Linux/COPYING for details. + + The ``kerneldoc-src2rst`` command extracts documentation from Linux kernel's + source code comments, see ``--help``:: + + $ kerneldoc-src2rst --help + +""" + +# ------------------------------------------------------------------------------ +# imports +# ------------------------------------------------------------------------------ + +import sys +import argparse +import re +import multiprocessing + +import six + +from fspath import FSPath, OS_ENV +import kernel_doc as kerneldoc + +# ------------------------------------------------------------------------------ +# config +# ------------------------------------------------------------------------------ + +MARKUP = "kernel-doc" # "reST" +MSG = lambda msg: sys.__stderr__.write("INFO : %s\n" % msg) +ERR = lambda msg: sys.__stderr__.write("ERROR: %s\n" % msg) +FATAL = lambda msg: sys.__stderr__.write("FATAL: %s\n" % msg) +IGNORE = ['kernel-doc.rst'] + +TEMPLATE_INDEX=u"""\ +.. -*- coding: utf-8; mode: rst -*- + +================================================================================ +%(title)s +================================================================================ + +.. toctree:: + :maxdepth: 1 + +""" + +CMD = None # global used by multiprocessing +SRCTREE = FSPath(OS_ENV.get("srctree", "")) + +# ------------------------------------------------------------------------------ +def main(): +# ------------------------------------------------------------------------------ + + global CMD # pylint: disable=W0603, W0621 + + CLI = argparse.ArgumentParser( + description = ("Parse *kernel-doc* comments from source code") + , formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + CLI.add_argument( + "srctree" + , help = "Folder of source code." + , type = lambda x: FSPath(x).ABSPATH) + + CLI.add_argument( + "doctree" + , help = "Folder to place reST documentation." + , type = lambda x: FSPath(x).ABSPATH) + + CLI.add_argument( + "--sloppy" + , action = "store_true" + , help = "Sloppy comment check, reports only severe errors.") + + CLI.add_argument( + "--force" + , action = "store_true" + , help = "Don't stop if doctree exists.") + + CLI.add_argument( + "--threads" + , type = int + , default = multiprocessing.cpu_count() + , help = "Use up to n threads.") + + CLI.add_argument( + "--markup" + , choices = ["reST", "kernel-doc", "auto"] + , default = "auto" + , help = ( + "Markup of the comments. Change this option only if you know" + " what you do. New comments must be marked up with reST!")) + + CMD = CLI.parse_args() + + if not CMD.srctree.EXISTS: + ERR("%s does not exists." % CMD.srctree) + sys.exit(42) + + if not CMD.srctree.ISDIR: + ERR("%s is not a folder." % CMD.srctree) + sys.exit(42) + + if not CMD.force and CMD.doctree.EXISTS: + ERR("%s is in the way, remove it first" % CMD.doctree) + sys.exit(42) + + CMD.rst_files = set() + if CMD.markup == "auto": + CMD.rst_files = docgrep(SRCTREE/"Documentation") + + pool = multiprocessing.Pool(CMD.threads) + pool.map(autodoc_file, gather_filenames(CMD)) + pool.close() + pool.join() + + insert_index_files(CMD.doctree) + +# ------------------------------------------------------------------------------ +def gather_filenames(cmd): +# ------------------------------------------------------------------------------ + MSG("gather files ...") + + for fname in cmd.srctree.reMatchFind(r"^.*\.[ch]$"): + if fname.startswith(CMD.srctree/"Documentation"): + continue + yield fname + +# ------------------------------------------------------------------------------ +def autodoc_file(fname): +# ------------------------------------------------------------------------------ + + fname = fname.relpath(CMD.srctree) + markup = CMD.markup + + if CMD.markup == "kernel-doc" and fname in CMD.rst_files: + markup = "reST" + + opts = kerneldoc.ParseOptions( + rel_fname = fname + , src_tree = CMD.srctree + , verbose_warn = not (CMD.sloppy) + , markup = markup ) + + parser = kerneldoc.Parser(opts, kerneldoc.NullTranslator()) + try: + parser.parse() + except Exception: # pylint: disable=W0703 + FATAL("kernel-doc markup of %s seems buggy / can't parse" % opts.fname) + return + + if not parser.ctx.dump_storage: + # no kernel-doc comments found + MSG("parsed: NONE comments: %s" % opts.fname) + return + + MSG("parsed: %4d comments: %s" % (len(parser.ctx.dump_storage), opts.fname)) + + try: + rst = six.StringIO() + translator = kerneldoc.ReSTTranslator() + opts.out = rst + + # First try to output reST, this might fail, because the kernel-doc + # parser part is to tollerant ("bad lines", "function name and function + # declaration are different", etc ...). + parser.parse_dump_storage(translator=translator) + + outFile = CMD.doctree / fname.replace(".","_") + ".rst" + outFile.DIRNAME.makedirs() + with outFile.openTextFile(mode="w") as out: + out.write(rst.getvalue()) + + except Exception: # pylint: disable=W0703 + FATAL("kernel-doc markup of %s seems buggy / can't parse" % opts.fname) + return + +# ------------------------------------------------------------------------------ +def insert_index_files(folder): +# ------------------------------------------------------------------------------ + + for folder, dirnames, filenames in folder.walk(): + ctx = kerneldoc.Container( title = folder.FILENAME ) + dirnames.sort() + filenames.sort() + indexFile = folder / "index.rst" + MSG("create index: %s" % indexFile) + with indexFile.openTextFile(mode="w") as index: + index.write(TEMPLATE_INDEX % ctx) + for d in dirnames: + index.write(" %s/index\n" % d.FILENAME) + for f in filenames: + if f.FILENAME == "index": + continue + index.write(" %s\n" % f.FILENAME) + +# ------------------------------------------------------------------------------ +def docgrep(folder): +# ------------------------------------------------------------------------------ + + # Hackisch script to grep all '.. kernel-doc::' directives. The assumption + # is, that comments from the source files used in those directives are + # allready migratetd to the reST format. I guess that (ATM) 95-99% of the + # comments are not migrated, those will be parsed with the old kernel-doc + # comment style introduced by the old DocBook toolchain. + + pat = re.compile(r"^\s*\.\.\s+kernel-doc::\s*([^\s]+)\s*$") + out = set() + for rstFile in folder.reMatchFind(r".*\.rst"): + if rstFile.BASENAME in IGNORE: + continue + #print(rstFile) + with rstFile.openTextFile() as f: + for l in f: + match = pat.search(l) + if match: + #print(match.group(1)) + out.add(match.group(1)) + return sorted(out) diff --git a/scripts/kerneldoc-src2rst b/scripts/kerneldoc-src2rst new file mode 100755 index 0000000..6c5e8b1 --- /dev/null +++ b/scripts/kerneldoc-src2rst @@ -0,0 +1,11 @@ +#!/usr/bin/python + +import sys +from os import path + +linuxdoc = path.abspath(path.join(path.dirname(__file__), '..')) +linuxdoc = path.join(linuxdoc, 'Documentation', 'sphinx') +sys.path.insert(0, linuxdoc) + +import src2rst +src2rst.main() -- 2.7.4 -- To unsubscribe from this list: send the line "unsubscribe linux-doc" in the body of a message to majordomo@xxxxxxxxxxxxxxx More majordomo info at http://vger.kernel.org/majordomo-info.html
