Generate config from master config. Verify responses in merge.py.

1 files changed, 283 insertions, 0 deletions

diff --git a/tools/compileconfig.py b/tools/compileconfig.py
new file mode 100755
index 0000000..30424c5
--- /dev/null
+++ b/tools/compileconfig.py
@@ -0,0 +1,283 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2014, NORDUnet A/S.
+# See LICENSE for licensing information.
+
+import argparse
+import sys
+import yaml
+import re
+
+class Symbol(str):
+    pass
+
+clean_string = re.compile(r'^[-.:_/A-Za-z0-9 ]*$')
+clean_symbol = re.compile(r'^[_A-Za-z0-9]*$')
+
+def quote_erlang_string(s):
+    if clean_string.match(s):
+        return '"' + s + '"'
+    else:
+        return "[" + ",".join([str(ord(c)) for c in s]) + "]"
+
+def quote_erlang_symbol(s):
+    if clean_symbol.match(s):
+        return s
+    elif clean_string.match(s):
+        return "'" + s + "'"
+    else:
+        print >>sys.stderr, "Cannot generate symbol", s
+        sys.exit(1)
+
+def gen_erlang(term, level=1):
+    indent = " " * level
+    separator = ",\n" + indent
+    if isinstance(term, Symbol):
+        return quote_erlang_symbol(term)
+    elif isinstance(term, basestring):
+        return quote_erlang_string(term)
+    elif isinstance(term, int):
+        return str(term)
+    elif isinstance(term, tuple):
+        tuplecontents = [gen_erlang(e, level=level+1) for e in term]
+        if "\n" not in "".join(tuplecontents):
+            separator = ", "
+        return "{" + separator.join(tuplecontents) + "}"
+    elif isinstance(term, list):
+        listcontents = [gen_erlang(e, level=level+1) for e in term]
+        return "[" + separator.join(listcontents) + "]"
+    else:
+        print "unknown type", type(term)
+        sys.exit(1)
+
+saslconfig = [(Symbol("sasl_error_logger"), Symbol("false")),
+              (Symbol("errlog_type"), Symbol("error")),
+              (Symbol("error_logger_mf_dir"), "log"),
+              (Symbol("error_logger_mf_maxbytes"), 10485760),
+              (Symbol("error_logger_mf_maxfiles"), 10),
+              ]
+
+def parse_address(address):
+    parsed_address = address.split(":")
+    if len(parsed_address) != 2:
+        print >>sys.stderr, "Invalid address format", address
+        sys.exit(1)
+    return (parsed_address[0], int(parsed_address[1]))
+
+def get_node_config(nodename, config):
+    nodetype = None
+    nodeconfig = None
+    for t in ["frontendnodes", "storagenodes", "signingnodes"]:
+        for node in config[t]:
+            if node["name"] == nodename:
+                nodetype = t
+                nodeconfig = node
+    if nodeconfig == None:
+        print >>sys.stderr, "Cannot find config for node", nodename
+        sys.exit(1)
+    return (nodetype, nodeconfig)
+
+def gen_https_servers(nodetype, nodeconfig):
+    if nodetype == "frontendnodes":
+        (publichost, publicport) = parse_address(nodeconfig["publicaddress"])
+        (host, port) = parse_address(nodeconfig["address"])
+        return [(Symbol("external_https_api"), publichost, publicport, Symbol("v1")),
+                (Symbol("frontend_https_api"), host, port, Symbol("frontend"))]
+    elif nodetype == "storagenodes":
+        (host, port) = parse_address(nodeconfig["address"])
+        return [(Symbol("storage_https_api"), host, port, Symbol("storage"))]
+    elif nodetype == "signingnodes":
+        (host, port) = parse_address(nodeconfig["address"])
+        return [(Symbol("signing_https_api"), host, port, Symbol("signing"))]
+
+def allowed_clients_frontend(mergenodenames):
+    return [
+        ("/ct/frontend/sendentry", mergenodenames),
+        ("/ct/frontend/sendlog", mergenodenames),
+        ("/ct/frontend/sendsth", mergenodenames),
+        ("/ct/frontend/currentposition", mergenodenames),
+        ("/ct/frontend/missingentries", mergenodenames),
+    ]
+
+def allowed_clients_public():
+    noauth = Symbol("noauth")
+    return [
+        ("/ct/v1/add-chain", noauth),
+        ("/ct/v1/add-pre-chain", noauth),
+        ("/ct/v1/get-sth", noauth),
+        ("/ct/v1/get-sth-consistency", noauth),
+        ("/ct/v1/get-proof-by-hash", noauth),
+        ("/ct/v1/get-entries", noauth),
+        ("/ct/v1/get-entry-and-proof", noauth),
+        ("/ct/v1/get-roots", noauth),
+    ]
+
+def allowed_clients_signing(frontendnodenames, mergenodenames):
+    return [
+        ("/ct/signing/sct", frontendnodenames),
+        ("/ct/signing/sth", mergenodenames),
+    ]
+
+def allowed_clients_storage(frontendnodenames, mergenodenames):
+    return [
+        ("/ct/storage/sendentry", frontendnodenames),
+        ("/ct/storage/entrycommitted", frontendnodenames),
+        ("/ct/storage/fetchnewentries", mergenodenames),
+        ("/ct/storage/getentry", mergenodenames),
+    ]
+
+def allowed_servers_frontend(signingnodenames, storagenodenames):
+    return [
+        ("/ct/storage/sendentry", storagenodenames),
+        ("/ct/storage/entrycommitted", storagenodenames),
+        ("/ct/signing/sct", signingnodenames),
+    ]
+
+def gen_config(nodename, config, localconfig):
+    print "generating config for", nodename
+    paths = localconfig["paths"]
+    options = localconfig.get("options", [])
+
+    configfile = open(paths["configdir"] + nodename + ".config", "w")
+    print >>configfile, "%% catlfish configuration file (-*- erlang -*-)"
+
+    (nodetype, nodeconfig) = get_node_config(nodename, config)
+    https_servers = gen_https_servers(nodetype, nodeconfig)
+
+    catlfishconfig = []
+    plopconfig = []
+
+    if nodetype == "frontendnodes":
+        catlfishconfig.append((Symbol("known_roots_path"), localconfig["paths"]["knownroots"]))
+        if "sctcaching" in options:
+            catlfishconfig.append((Symbol("sctcache_root_path"), paths["db"] + "sctcache/"))
+
+    catlfishconfig += [
+        (Symbol("https_servers"), https_servers),
+        (Symbol("https_certfile"), paths["https_certfile"]),
+        (Symbol("https_keyfile"), paths["https_keyfile"]),
+        (Symbol("https_cacertfile"), paths["https_cacertfile"]),
+    ]
+
+    lagerconfig = [
+        (Symbol("handlers"), [
+            (Symbol("lager_console_backend"), Symbol("info")),
+            (Symbol("lager_file_backend"), [(Symbol("file"), nodename + "-error.log"), (Symbol("level"), Symbol("error"))]),
+            (Symbol("lager_file_backend"), [(Symbol("file"), nodename + "-debug.log"), (Symbol("level"), Symbol("debug"))]),
+            (Symbol("lager_file_backend"), [(Symbol("file"), nodename + "-console.log"), (Symbol("level"), Symbol("info"))]),
+        ])
+    ]
+
+    if nodetype in ("frontendnodes", "storagenodes"):
+        plopconfig += [
+            (Symbol("entry_root_path"), paths["db"] + "certentries/"),
+        ]
+    if nodetype == "frontendnodes":
+        plopconfig += [
+            (Symbol("index_path"), paths["db"] + "index"),
+        ]
+    elif nodetype == "storagenodes":
+        plopconfig += [
+            (Symbol("newentries_path"), paths["db"] + "newentries"),
+        ]
+    if nodetype in ("frontendnodes", "storagenodes"):
+        plopconfig += [
+            (Symbol("entryhash_root_path"), paths["db"] + "entryhash/"),
+            (Symbol("indexforhash_root_path"), paths["db"] + "certindex/"),
+        ]
+    if nodetype == "frontendnodes":
+        plopconfig += [
+            (Symbol("sth_path"), paths["db"] + "sth"),
+        ]
+
+    signingnode = config["signingnodes"][0]
+    mergenodenames = [node["name"] for node in config["mergenodes"]]
+    storagenodeaddresses = ["https://%s/ct/storage/" % node["address"] for node in config["storagenodes"]]
+    frontendnodenames = [node["name"] for node in config["frontendnodes"]]
+
+    allowed_clients = []
+    allowed_servers = []
+
+    if nodetype == "frontendnodes":
+        storagenodenames = [node["name"] for node in config["storagenodes"]]
+        plopconfig.append((Symbol("storage_nodes"), storagenodeaddresses))
+        plopconfig.append((Symbol("storage_nodes_quorum"), config["storage-quorum-size"]))
+        services = [Symbol("ht")]
+        allowed_clients += allowed_clients_frontend(mergenodenames)
+        allowed_clients += allowed_clients_public()
+        allowed_servers += allowed_servers_frontend([signingnode["name"]], storagenodenames)
+    elif nodetype == "storagenodes":
+        allowed_clients += allowed_clients_storage(frontendnodenames, mergenodenames)
+        services = []
+    elif nodetype == "signingnodes":
+        allowed_clients += allowed_clients_signing(frontendnodenames, mergenodenames)
+        services = [Symbol("sign")]
+
+    plopconfig += [
+        (Symbol("publickey_path"), paths["publickeys"]),
+        (Symbol("services"), services),
+    ]
+    if nodetype == "signingnodes":
+        plopconfig.append((Symbol("log_private_key"), paths["logprivatekey"]))
+    plopconfig += [
+        (Symbol("log_public_key"), paths["logpublickey"]),
+        (Symbol("own_key"), (nodename, "%s/%s-private.pem" % (paths["privatekeys"], nodename))),
+    ]
+    if nodetype == "frontendnodes":
+        plopconfig.append((Symbol("signing_node"), "https://%s/ct/signing/" % signingnode["address"]))
+    plopconfig += [
+        (Symbol("allowed_clients"), allowed_clients),
+        (Symbol("allowed_servers"), allowed_servers),
+    ]
+
+    erlangconfig = [
+        (Symbol("sasl"), saslconfig),
+        (Symbol("catlfish"), catlfishconfig),
+        (Symbol("lager"), lagerconfig),
+        (Symbol("plop"), plopconfig),
+    ]
+    
+    print >>configfile, gen_erlang(erlangconfig) + ".\n"
+    
+    configfile.close()
+
+
+def gen_testmakefile(config, testmakefile, machines):
+    configfile = open(testmakefile, "w")
+    frontendnodenames = [node["name"] for node in config["frontendnodes"]]
+    storagenodenames = [node["name"] for node in config["storagenodes"]]
+    signingnodename = [node["name"] for node in config["signingnodes"]]
+
+    frontendnodeaddresses = [node["publicaddress"] for node in config["frontendnodes"]]
+    storagenodeaddresses = [node["address"] for node in config["storagenodes"]]
+    signingnodeaddresses = [node["address"] for node in config["signingnodes"]]
+
+    print >>configfile, "NODES=" + " ".join(frontendnodenames+storagenodenames+signingnodename)
+    print >>configfile, "MACHINES=" + " ".join([str(e) for e in range(1, machines+1)])
+    print >>configfile, "TESTURLS=" + " ".join(frontendnodeaddresses+storagenodeaddresses+signingnodeaddresses)
+    print >>configfile, "BASEURL=" + config["baseurl"]
+
+    configfile.close()
+
+
+def main():
+    parser = argparse.ArgumentParser(description="")
+    parser.add_argument('--config', help="System configuration", required=True)
+    parser.add_argument('--localconfig', help="Local configuration")
+    parser.add_argument("--testmakefile", metavar="file", help="Generate makefile variables for test")
+    parser.add_argument("--machines", type=int, metavar="n", help="Number of machines")
+    args = parser.parse_args()
+
+    config = yaml.load(open(args.config))
+    if args.testmakefile and args.machines:
+        gen_testmakefile(config, args.testmakefile, args.machines)
+    elif args.localconfig:
+        localconfig = yaml.load(open(args.localconfig))
+        localnodes = localconfig["localnodes"]
+        for localnode in localnodes:
+            gen_config(localnode, config, localconfig)
+    else:
+        print >>sys.stderr, "Nothing to do"
+        sys.exit(1)
+
+main()