Automatic generation of config man page skeletondocgen

6 files changed, 457 insertions, 3 deletions

diff --git a/doc/Makefile b/doc/Makefile
index b3de194..528b6d5 100644
--- a/doc/Makefile
+++ b/doc/Makefile
@@ -1,7 +1,12 @@
 MANDOCS = catlfish.1 genconfig.1
 RONN = ronn --warnings --organization="FIXME:\$$version"
 
-all: man html
+all: configman man html
+
+configman:
+	../tools/compileconfig.py --manpagedir=.
+	make catlfish-log.cfg.in.5
+	make catlfish-node.cfg.5
 
 man: $(MANDOCS)
 
@@ -10,6 +15,9 @@ html: $(addsuffix .html,$(MANDOCS))
 %: %.md
 	$(RONN) --roff $^
 
+%: %.adoc
+	a2x --doctype manpage --format manpage $^
+
 %.html: %.md
 	$(RONN) --html $^
 
diff --git a/doc/catlfish-log.cfg.in.5.adoc b/doc/catlfish-log.cfg.in.5.adoc
new file mode 100644
index 0000000..1cc912f
--- /dev/null
+++ b/doc/catlfish-log.cfg.in.5.adoc
@@ -0,0 +1,95 @@
+:man source:   Catlfish
+:man manual:   Catlfish Manual
+CATLFISH-LOG.CFG.IN(5)
+======================
+
+NAME
+----
+catlfish-log.cfg.in - catlfish log configuration
+
+OPTIONS
+-------
+   **apikeys**: (list of items)::
+//    write description here
+
+      **nodename**: __nodename__:::
+//       write description here
+
+      **publickey**: __key__:::
+          BASE64-encoded key
+
+   **backup-quorum-size**: __number-of-nodes__::
+       number of secondary merge nodes that need to have an entry before the entry is considered committed
+
+   **baseurl**: __url__::
+//    write description here
+
+   **cafingerprint**: __fingerprint__::
+//    write description here
+
+   **frontendnodes**: (list of items)::
+//    write description here
+
+      **address**: __ip-address__:::
+//       write description here
+
+      **name**: __nodename__:::
+//       write description here
+
+      **publicaddress**: __ip-address__:::
+//       write description here
+
+   **logpublickey**: __key__::
+//    write description here
+
+   **mergenodes**: (list of items)::
+//    write description here
+
+      **address**: __ip-address__:::
+//       write description here
+
+      **name**: __nodename__:::
+//       write description here
+
+   **mmd**: __seconds__::
+//    write description here
+
+   **primarymergenode**: __nodename__::
+//    write description here
+
+   **signingnodes**: (list of items)::
+//    write description here
+
+      **address**: __ip-address__:::
+//       write description here
+
+      **name**: __nodename__:::
+//       write description here
+
+   **statusservers**: (list of items)::
+//    write description here
+
+      **address**: __ip-address__:::
+//       write description here
+
+      **name**: __nodename__:::
+//       write description here
+
+      **publicaddress**: __ip-address__:::
+//       write description here
+
+   **storage-quorum-size**: __number-of-nodes__::
+//    write description here
+
+   **storagenodes**: (list of items)::
+//    write description here
+
+      **address**: __ip-address__:::
+//       write description here
+
+      **name**: __nodename__:::
+//       write description here
+
+   **version**: __version__::
+//    write description here
+
diff --git a/doc/catlfish-node.cfg.5.adoc b/doc/catlfish-node.cfg.5.adoc
new file mode 100644
index 0000000..fabef29
--- /dev/null
+++ b/doc/catlfish-node.cfg.5.adoc
@@ -0,0 +1,113 @@
+:man source:   Catlfish
+:man manual:   Catlfish Manual
+CATLFISH-NODE.CFG(5)
+====================
+
+NAME
+----
+catlfish-node.cfg - catlfish node configuration
+
+OPTIONS
+-------
+   **configurl**: __url__::
+//    write description here
+
+   **ctapiaddress**: __ip-address__::
+//    write description here
+
+   **dbbackend**: **permdb**|**fsdb**::
+//    write description here
+
+   **frontendaddress**: __ip-address__::
+//    write description here
+
+   **logadminkey**: __key__::
+//    write description here
+
+   **merge**: ::
+//    write description here
+
+      **backup-sendentries-chunksize**: __number-of-entries__:::
+//       write description here
+
+      **backup-sendlog-chunksize**: __number-of-entries__:::
+//       write description here
+
+      **backup-window-size**: __number-of-entries__:::
+//       write description here
+
+      **dist-sendentries-chunksize**: __number-of-entries__:::
+//       write description here
+
+      **dist-sendlog-chunksize**: __number-of-entries__:::
+//       write description here
+
+      **dist-window-size**: __number-of-entries__:::
+//       write description here
+
+      **min-delay**: __seconds__:::
+//       write description here
+
+   **mergeaddress**: __ip-address__::
+//    write description here
+
+   **nodename**: __nodename__::
+//    write description here
+
+   **paths**: ::
+//    write description here
+
+      **configdir**: __path__:::
+//       write description here
+
+      **db**: __path__:::
+//       write description here
+
+      **https_cacertfile**: __path__:::
+//       write description here
+
+      **https_certfile**: __path__:::
+//       write description here
+
+      **https_keyfile**: __path__:::
+//       write description here
+
+      **knownroots**: __path__:::
+//       write description here
+
+      **logprivatekey**: __path__:::
+//       write description here
+
+      **logpublickey**: __path__:::
+//       write description here
+
+      **mergedb**: __path__:::
+//       write description here
+
+      **privatekeys**: __path__:::
+//       write description here
+
+      **public_cacertfile**: __path__:::
+//       write description here
+
+      **publickeys**: __path__:::
+//       write description here
+
+      **verifycert_bin**: __path__:::
+//       write description here
+
+   **publichttpaddress**: __ip-address__::
+//    write description here
+
+   **ratelimits**: ::
+//    write description here
+
+      **add_chain**: __rate__:::
+//       write description here
+
+   **signingaddress**: __ip-address__::
+//    write description here
+
+   **storageaddress**: __ip-address__::
+//    write description here
+
diff --git a/tools/compileconfig.py b/tools/compileconfig.py
index 10fc3d6..d6d65d4 100755
--- a/tools/compileconfig.py
+++ b/tools/compileconfig.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-# Copyright (c) 2014-2016, NORDUnet A/S.
+# Copyright (c) 2014-2017, NORDUnet A/S.
 # See LICENSE for licensing information.
 
 import argparse
@@ -9,6 +9,7 @@ import readconfig
 import re
 import base64
 from datetime import datetime
+import manpage
 
 class Symbol(str):
     pass
@@ -550,6 +551,10 @@ def printnodenames(config):
 
     print " ".join(frontendnodenames|storagenodenames|signingnodenames|mergenodenames|statusservernodenames)
 
+def gen_manpage(manpagedir):
+    manpage.rewrite_manpage(manpagedir + "/catlfish-log.cfg.in.5.adoc", globalconfigschema, "Catlfish", "Catlfish Manual", "CATLFISH-LOG.CFG.IN(5)", "catlfish-log.cfg.in - catlfish log configuration")
+    manpage.rewrite_manpage(manpagedir + "/catlfish-node.cfg.5.adoc", localconfigschema, "Catlfish", "Catlfish Manual", "CATLFISH-NODE.CFG(5)", "catlfish-node.cfg - catlfish node configuration")
+
 localconfigschema = [
     ("nodename", "string", "nodename"),
     ("frontendaddress", "string", "ip address"),
@@ -612,14 +617,19 @@ globalconfigschema = [
 def main():
     parser = argparse.ArgumentParser(description="")
     parser.add_argument('--config', help="System configuration")
+    parser.add_argument("--manpagedir", metavar="file", help="Generate manpages to directory")
     parser.add_argument('--localconfig', help="Local configuration")
     parser.add_argument("--testmakefile", metavar="file", help="Generate makefile variables for test")
     parser.add_argument("--testshellvars", metavar="file", help="Generate shell variable file for test")
     parser.add_argument("--getnodenames", action='store_true', help="Get list of node names")
     args = parser.parse_args()
 
+    if args.manpagedir:
+        gen_manpage(args.manpagedir)
+        sys.exit(0)
+
     if not args.config:
-        print >>sys.stderr, "--config is required"
+        print >>sys.stderr, "either --config or --manpage is required"
         sys.exit(1)
 
     if args.testmakefile:
diff --git a/tools/manpage.py b/tools/manpage.py
new file mode 100644
index 0000000..1ea8753
--- /dev/null
+++ b/tools/manpage.py
@@ -0,0 +1,183 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2017, NORDUnet A/S.
+# See LICENSE for licensing information.
+
+import argparse
+import sys
+import readconfig
+import re
+import base64
+import shutil
+from datetime import datetime
+from orderedtree import TreeNode
+
+def level_is_list(schema):
+    if len(schema.keys()) != 1:
+        return False
+    return schema.keys()[0] == "[]"
+
+def traverse_schema_part(schema):
+    tree = TreeNode()
+    for k in sorted(schema.keys()):
+        schema_part = schema.get(k)
+        result = None
+        if isinstance(schema_part, tuple):
+            (lowleveldatatype, highleveldatatype) = schema_part
+            if isinstance(highleveldatatype, list):
+                formatted_datatype = "|".join(["**"+t+"**" for t in highleveldatatype])
+            else:
+                formatted_datatype = "__" + highleveldatatype.replace(" ", "-") + "__"
+            if k == "[]":
+                result = "list of " + formatted_datatype
+            else:
+                result = "**" + k + "**: " + formatted_datatype
+
+            tree.add(k, (result, []))
+
+        elif isinstance(schema_part, dict):
+            if k == "[]":
+                result = "list of items"
+            else:
+                if level_is_list(schema_part):
+                    formatted_datatype = "(list of items)"
+                    schema_part = schema_part["[]"]
+                else:
+                    formatted_datatype = ""
+                result = "**"+k+"**: " + formatted_datatype
+            subtree = traverse_schema_part(schema_part)
+            tree.add(k, (result, []), subtree=subtree)
+        else:
+            print >>sys.stderr, "unknown type", type(schema_part)
+            sys.exit(1)
+    return tree
+
+def traverse_schema(schema):
+    transformed_schema = readconfig.transform_schema(schema)
+    tree = traverse_schema_part(transformed_schema)
+    return tree
+
+def is_adoc_header(row):
+    return set(row.rstrip()) == set("=")
+
+def is_adoc_section(row):
+    return set(row.rstrip()) == set("-")
+
+def parse_manpage(filename):
+    f = open(filename)
+    header = []
+    for row in f:
+        if is_adoc_header(row):
+            break
+        header.append(row.rstrip("\n"))
+    section_name = None
+    section = []
+    sections = []
+    for row in f:
+        if is_adoc_section(row):
+            if section_name:
+                sections.append((section_name, section[:-1]))
+            section_name = section[-1]
+            section = []
+        else:
+            section.append(row.rstrip("\n"))
+    if section_name:
+        sections.append((section_name, section))
+    return (header, sections)
+
+def is_manpage_option(row):
+    return row.endswith("::")
+
+def extract_option_name(row):
+    if row == None:
+        return (None, None)
+    (name_part, _, _) = row.lstrip().partition(":")
+    depth = len(row) - len(row.rstrip(":")) - 2
+    return (name_part.replace("*", ""), depth)
+
+def parse_manpage_options(option_rows):
+    option_name_row = None
+    options = []
+    option = []
+    for row in option_rows:
+        if is_manpage_option(row):
+            options.append((option_name_row, option))
+            option_name_row = row
+            option = []
+        else:
+            option.append(row)
+    options.append((option_name_row, option))
+    return options
+
+def build_tree(l, key):
+    tree = TreeNode()
+    curpath = []
+    for e in l:
+        (k, depth) = key(e)
+        if depth > len(curpath):
+            print >>sys.stderr, "depth", depth, "from", e, "greater than length of curpath", curpath
+            sys.exit(1)
+        curpath = curpath[:depth]
+        tree.walk(curpath).add(k, e)
+        curpath.append(k)
+    return tree
+
+def transfer_tree(current, wanted):
+    for name in wanted.iterkeys():
+        if name in current and name != None:
+            transfer_tree(current[name], wanted[name])
+    if wanted.entry:
+        wanted.entry = (wanted.entry[0], current.entry[1])
+
+def print_tree(f, tree, depth=0):
+    if tree.entry:
+        (section, rows) = tree.entry
+        print >>f, "   " * depth + section + (depth+1) * ":"
+        has_content = False
+        for row in rows:
+            if row.strip():
+                has_content = True
+            print >>f, row
+        if not has_content:
+            print >>f, "// " + "   " * depth + "write description here"
+            print >>f, ""
+    for name in tree.iterkeys():
+        print_tree(f, tree[name], depth=depth+1)
+
+def rewrite_options(f, schema, options):
+    wanted = traverse_schema(schema)
+    current = build_tree(options, lambda e: extract_option_name(e[0]))
+    transfer_tree(current, wanted)
+    print_tree(f, wanted)
+
+def rewrite_manpage(filename, schema, man_source, man_manual, title, name):
+    try:
+        (header, sections) = parse_manpage(filename)
+    except IOError:
+        sections = []
+    sections_dict = dict(sections)
+    section_names = [e for e, _ in sections]
+    options = parse_manpage_options(sections_dict.get("OPTIONS", []))
+    sections_dict["NAME"] = [name]
+    if "NAME" not in section_names:
+        section_names.append("NAME")
+    if "OPTIONS" not in section_names:
+        section_names.append("OPTIONS")
+    f = open(filename + ".new", "w")
+    print >>f, ":man source:   " + man_source
+    print >>f, ":man manual:   " + man_manual
+    print >>f, title
+    print >>f, len(title) * "="
+    print >>f, ""
+    for section_name in section_names:
+        print >>f, section_name
+        print >>f, len(section_name) * "-"
+        if section_name == "OPTIONS":
+            rewrite_options(f, schema, options)
+            continue
+        for row in sections_dict[section_name]:
+            print >>f, row
+        print >>f, ""
+    f.close()
+    shutil.move(filename + ".new", filename)
+
diff --git a/tools/orderedtree.py b/tools/orderedtree.py
new file mode 100644
index 0000000..def8928
--- /dev/null
+++ b/tools/orderedtree.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2017, NORDUnet A/S.
+# See LICENSE for licensing information.
+
+class OrderDict(dict):
+    def __init__(self):
+        self._order = []
+        dict.__init__({})
+    def __setitem__(self, key, value):
+        if key not in self:
+            self._order.append(key)
+        super(OrderDict, self).__setitem__(key, value)
+    def iterkeys(self):
+        return iter(self._order)
+
+class TreeNode():
+    def __init__(self):
+        self.entry = None
+        self._children = OrderDict()
+    def add(self, k, e, subtree=None):
+        if subtree != None:
+            self._children[k] = subtree
+        else:
+            self._children[k] = TreeNode()
+        self._children[k].entry = e
+    def __getitem__(self, key):
+        return self._children[key]
+    def iterkeys(self):
+        return self._children.iterkeys()
+    def __contains__(self, key):
+        return key in self._children
+    def walk(self, keys):
+        node = self
+        for k in keys:
+            node = node[k]
+        return node
+    def __str__(self):
+        s = str(self.entry) + "\n"
+        for k in self.iterkeys():
+            s += str(k) + ":\n"
+            for row in str(self._children[k]).split("\n"):
+                if row:
+                    s += "    " + row + "\n"
+        return s