#!/usr/bin/env python # Copyright (c) 2014-2017, NORDUnet A/S. # See LICENSE for licensing information. import argparse import sys import readconfig import re import base64 from datetime import datetime import manpage class Symbol(str): pass class Binary(str): pass clean_string = re.compile(r'^[-.:_/A-Za-z0-9 ]*$') clean_symbol = re.compile(r'^[_A-Za-z0-9]*$') def quote_erlang_string(s): if clean_string.match(s): return '"' + s + '"' else: return "[" + ",".join([str(ord(c)) for c in s]) + "]" def quote_erlang_symbol(s): if clean_symbol.match(s): return s elif clean_string.match(s): return "'" + s + "'" else: print >>sys.stderr, "Cannot generate symbol", s sys.exit(1) def gen_erlang(term, level=1): indent = " " * level separator = ",\n" + indent if isinstance(term, Symbol): return quote_erlang_symbol(term) elif isinstance(term, Binary): return "<<" + ",".join([str(ord(c)) for c in term]) + ">>" elif isinstance(term, basestring): return quote_erlang_string(term) elif isinstance(term, int): return str(term) elif isinstance(term, tuple): tuplecontents = [gen_erlang(e, level=level+1) for e in term] if "\n" not in "".join(tuplecontents): separator = ", " return "{" + separator.join(tuplecontents) + "}" elif isinstance(term, list): listcontents = [gen_erlang(e, level=level+1) for e in term] return "[" + separator.join(listcontents) + "]" else: print "unknown type", type(term) sys.exit(1) saslconfig = [(Symbol("sasl_error_logger"), Symbol("false")), (Symbol("errlog_type"), Symbol("error")), (Symbol("error_logger_mf_dir"), "sasl_log"), (Symbol("error_logger_mf_maxbytes"), 10485760), (Symbol("error_logger_mf_maxfiles"), 10), ] def parse_address(address): parsed_address = address.split(":") if len(parsed_address) != 2: print >>sys.stderr, "Invalid address format", address sys.exit(1) return (parsed_address[0], int(parsed_address[1])) def get_node_config(nodename, config): nodetype = [] nodeconfig = {} for t in ["frontendnodes", "storagenodes", "signingnodes", "mergenodes", "statusservers"]: for node in config[t]: if node["name"] == nodename: nodetype.append(t) nodeconfig[t] = node if len(nodetype) == 0: print >>sys.stderr, "Cannot find config for node", nodename sys.exit(1) if len(nodetype) >= 2 and set(nodetype) != set(["frontendnodes", "storagenodes"]): print >>sys.stderr, "Node type unsupported:", nodetype sys.exit(1) return (set(nodetype), nodeconfig) def get_address(bind_address, nodeconfig): if bind_address: (host, port) = parse_address(bind_address) else: (_, port) = parse_address(nodeconfig["address"]) host = "0.0.0.0" return (host, port) def gen_http_servers(nodetype, nodeconfig, bind_addresses, bind_publicaddress, bind_publichttpaddress): http_servers = [] https_servers = [] if "frontendnodes" in nodetype and "mergenodes" in nodetype: print >>sys.stderr, "cannot have both frontend node and merge node at the same time", nodetype sys.exit(1) if "frontendnodes" in nodetype: (host, port) = get_address(bind_addresses["frontend"], nodeconfig["frontendnodes"]) if bind_publicaddress: (publichost, publicport) = parse_address(bind_publicaddress) else: (_, publicport) = parse_address(nodeconfig["frontendnodes"]["publicaddress"]) publichost = "0.0.0.0" if bind_publichttpaddress: (publichttphost, publichttpport) = parse_address(bind_publichttpaddress) http_servers.append((Symbol("external_http_api"), publichttphost, publichttpport, Symbol("v1"))) https_servers.append((Symbol("external_https_api"), publichost, publicport, Symbol("v1"))) https_servers.append((Symbol("frontend_https_api"), host, port, Symbol("frontend"))) if "storagenodes" in nodetype: (host, port) = get_address(bind_addresses["storage"], nodeconfig["storagenodes"]) https_servers.append((Symbol("storage_https_api"), host, port, Symbol("storage"))) if "signingnodes" in nodetype: (host, port) = get_address(bind_addresses["signing"], nodeconfig["signingnodes"]) https_servers.append((Symbol("signing_https_api"), host, port, Symbol("signing"))) if "mergenodes" in nodetype: (host, port) = get_address(bind_addresses["merge"], nodeconfig["mergenodes"]) https_servers.append((Symbol("frontend_https_api"), host, port, Symbol("frontend"))) if "statusservers" in nodetype: (host, port) = get_address(None, nodeconfig["statusservers"]) https_servers.append((Symbol("statusserver_https_api"), host, port, Symbol("statusserver"))) if bind_publicaddress: (publichost, publicport) = parse_address(bind_publicaddress) else: (_, publicport) = parse_address(nodeconfig["statusservers"]["publicaddress"]) publichost = "0.0.0.0" if bind_publichttpaddress: (publichttphost, publichttpport) = parse_address(bind_publichttpaddress) http_servers.append((Symbol("external_http_api"), publichttphost, publichttpport, Symbol("statusserver"))) https_servers.append((Symbol("external_https_api"), publichost, publicport, Symbol("statusserver"))) if nodetype - set(["frontendnodes", "storagenodes", "signingnodes", "mergenodes", "statusservers"]): print >>sys.stderr, "unknown nodetype", nodetype sys.exit(1) return (http_servers, https_servers) def allowed_clients_frontend(mergenodenames, primarymergenodename): return [ ("/plop/v1/frontend/sendentry", mergenodenames), ("/plop/v1/frontend/sendlog", mergenodenames), ("/plop/v1/frontend/publish-sth", [primarymergenodename]), ("/plop/v1/frontend/verify-entries", [primarymergenodename]), ("/plop/v1/frontend/currentposition", mergenodenames), ("/plop/v1/frontend/missingentries", mergenodenames), ] def allowed_clients_mergesecondary(primarymergenodename): return [ ("/plop/v1/merge/sendentry", [primarymergenodename]), ("/plop/v1/merge/sendlog", [primarymergenodename]), ("/plop/v1/merge/verifyroot", [primarymergenodename]), ("/plop/v1/merge/verifiedsize", [primarymergenodename]), ("/plop/v1/merge/setverifiedsize", [primarymergenodename]), ("/plop/v1/merge/missingentries", [primarymergenodename]), ] def allowed_clients_public(): noauth = Symbol("noauth") return [ ("/ct/v1/add-chain", noauth), ("/ct/v1/add-pre-chain", noauth), ("/ct/v1/get-sth", noauth), ("/ct/v1/get-sth-consistency", noauth), ("/ct/v1/get-proof-by-hash", noauth), ("/ct/v1/get-entries", noauth), ("/ct/v1/get-entry-and-proof", noauth), ("/ct/v1/get-roots", noauth), ] def allowed_clients_signing(frontendnodenames, primarymergenodename): return [ ("/plop/v1/signing/sct", frontendnodenames), ("/plop/v1/signing/sth", [primarymergenodename]), ] def allowed_clients_storage(frontendnodenames, mergenodenames): return [ ("/plop/v1/storage/sendentry", frontendnodenames), ("/plop/v1/storage/entrycommitted", frontendnodenames), ("/plop/v1/storage/fetchnewentries", mergenodenames), ("/plop/v1/storage/getentry", mergenodenames), ] def allowed_servers_frontend(signingnodenames, storagenodenames): return [ ("/plop/v1/storage/sendentry", storagenodenames), ("/plop/v1/storage/entrycommitted", storagenodenames), ("/plop/v1/signing/sct", signingnodenames), ] def allowed_servers_primarymerge(frontendnodenames, secondarymergenames, signingnodenames): return [ ("/plop/v1/frontend/verify-entries", frontendnodenames), ("/plop/v1/frontend/sendlog", frontendnodenames), ("/plop/v1/frontend/missingentries", frontendnodenames), ("/plop/v1/frontend/sendentry", frontendnodenames), ("/plop/v1/frontend/publish-sth", frontendnodenames), ("/plop/v1/merge/verifiedsize", secondarymergenames), ("/plop/v1/merge/verifyroot", secondarymergenames), ("/plop/v1/merge/setverifiedsize", secondarymergenames), ("/plop/v1/merge/sendlog", secondarymergenames), ("/plop/v1/merge/missingentries", secondarymergenames), ("/plop/v1/merge/sendentry", secondarymergenames), ("/plop/v1/signing/sth", signingnodenames), ] def parse_ratelimit_expression(expression): if expression == "none": return Symbol("none") parts = expression.split(" ") if not (len(parts) == 3 and parts[1] == 'per' and parts[2] in ["second", "minute", "hour"]): print >>sys.stderr, "Ratelimit expressions must have the format \" per second|minute|hour\" or \"none\"" sys.exit(1) return (int(parts[0]), Symbol(parts[2])) def parse_ratelimit((type, description)): descriptions = [parse_ratelimit_expression(s.strip()) for s in description.split(",")] if len(descriptions) != 1: print >>sys.stderr, "%s: Only one ratelimit expression supported right now" % (type,) return (Symbol(type), descriptions) def api_keys(config): return [(node["nodename"], Binary(base64.b64decode(node["publickey"]))) for node in config["apikeys"]] def gen_config(nodename, config, localconfig): print "generating config for", nodename paths = localconfig["paths"] apikeys = api_keys(config) bind_addresses = { "frontend": localconfig.get("frontendaddress"), "storage": localconfig.get("storageaddress"), "signing": localconfig.get("signingaddress"), "merge": localconfig.get("mergeaddress"), } bind_publicaddress = localconfig.get("ctapiaddress") bind_publichttpaddress = localconfig.get("publichttpaddress") options = localconfig.get("options", []) configfile = open(paths["configdir"] + "/" + nodename + ".config", "w") print >>configfile, "%% catlfish configuration file (-*- erlang -*-)" plopcontrolfilename = nodename + ".plopcontrol" plopconfigfilename = paths["configdir"] + "/" + nodename + ".plopconfig" plopconfigfile = open(plopconfigfilename, "w") print >>plopconfigfile, "%% plop configuration file (-*- erlang -*-)" (nodetype, nodeconfig) = get_node_config(nodename, config) if nodename == config["primarymergenode"]: (http_servers, https_servers) = [], [] else: (http_servers, https_servers) = gen_http_servers(nodetype, nodeconfig, bind_addresses, bind_publicaddress, bind_publichttpaddress=bind_publichttpaddress) catlfishconfig = [] plopconfig = [] reloadableplopconfig = [] if nodetype & set(["frontendnodes", "mergenodes"]): catlfishconfig.append((Symbol("known_roots_path"), paths["knownroots"])) if "frontendnodes" in nodetype: plopconfig.append((Symbol("sptcache_root_path"), paths["db"] + "sctcache")) if "ratelimits" in localconfig: ratelimits = map(parse_ratelimit, localconfig["ratelimits"].items()) catlfishconfig.append((Symbol("ratelimits"), ratelimits)) catlfishconfig += [ (Symbol("https_servers"), https_servers), (Symbol("http_servers"), http_servers), (Symbol("https_certfile"), paths["https_certfile"]), (Symbol("https_keyfile"), paths["https_keyfile"]), ] if "statusservers" in nodetype: plopconfig += [ (Symbol("https_servers"), https_servers), (Symbol("http_servers"), http_servers), (Symbol("https_certfile"), paths["https_certfile"]), (Symbol("https_keyfile"), paths["https_keyfile"]), ] catlfishconfig.append((Symbol("mmd"), config["mmd"])) lagerconfig = [ (Symbol("handlers"), [ (Symbol("lager_console_backend"), Symbol("info")), (Symbol("lager_file_backend"), [(Symbol("file"), nodename + "-error.log"), (Symbol("level"), Symbol("error"))]), (Symbol("lager_file_backend"), [(Symbol("file"), nodename + "-debug.log"), (Symbol("level"), Symbol("debug"))]), (Symbol("lager_file_backend"), [(Symbol("file"), nodename + "-console.log"), (Symbol("level"), Symbol("info"))]), ]) ] plopconfig += [ (Symbol("https_cacertfile"), paths["https_cacertfile"]), (Symbol("https_cacert_fingerprint"), Binary(base64.b16decode(config["cafingerprint"]))), ] if "dbbackend" in localconfig: dbbackend = localconfig["dbbackend"] if dbbackend not in ("fsdb", "permdb"): print >>sys.stderr, "DB backend not recognized:", dbbackend sys.exit(1) plopconfig += [ (Symbol("db_backend"), Symbol(dbbackend)), ] if dbbackend != "fsdb": plopconfig += [ (Symbol("fsync_parallel_tasks"), 4), ] #print "nodetype", ", ".join(nodetype) if nodetype & set(["frontendnodes", "storagenodes"]): plopconfig += [ (Symbol("entry_root_path"), paths["db"] + "certentries"), (Symbol("entryhash_root_path"), paths["db"] + "entryhash"), (Symbol("indexforhash_root_path"), paths["db"] + "certindex"), ] if "frontendnodes" in nodetype: plopconfig += [ (Symbol("index_path"), paths["db"] + "index"), (Symbol("sth_path"), paths["db"] + "sth"), (Symbol("sendsth_verified_path"), paths["db"] + "sendsth-verified"), (Symbol("entryhash_from_entry"), (Symbol("catlfish"), Symbol("entryhash_from_entry"))), (Symbol("spt_data"), (Symbol("catlfish"), Symbol("spt_data"))), ] if "storagenodes" in nodetype: plopconfig += [ (Symbol("newentries_path"), paths["db"] + "newentries"), (Symbol("lastverifiednewentry_path"), paths["db"] + "lastverifiednewentry"), (Symbol("spt_data"), (Symbol("catlfish"), Symbol("spt_data"))), ] if nodetype & set(["frontendnodes", "mergenodes"]): plopconfig += [ (Symbol("verify_entry"), (Symbol("catlfish"), Symbol("verify_entry"))), ] if "mergenodes" in nodetype: plopconfig += [ (Symbol("verifiedsize_path"), paths["mergedb"] + "/verifiedsize"), (Symbol("index_path"), paths["mergedb"] + "/logorder"), (Symbol("entry_root_path"), paths["mergedb"] + "/chains"), ] signingnodenames = [node["name"] for node in config["signingnodes"]] signingnodeaddresses = ["https://%s/plop/v1/signing/" % node["address"] for node in config["signingnodes"]] mergenodenames = [node["name"] for node in config["mergenodes"]] primarymergenodename = config["primarymergenode"] storagenodeaddresses = ["https://%s/plop/v1/storage/" % node["address"] for node in config["storagenodes"]] frontendnodenames = [node["name"] for node in config["frontendnodes"]] frontendnodeaddresses = ["https://%s/plop/v1/frontend/" % node["address"] for node in config["frontendnodes"]] statusservernames = [node["name"] for node in config["statusservers"]] statusserveraddresses = ["https://%s/plop/v1/status/" % node["address"] for node in config["statusservers"]] benchserveraddresses = ["https://%s/plop/v1/bench/" % node["address"] for node in config["statusservers"]] allowed_clients = [] allowed_servers = [] storagenodenames = [node["name"] for node in config["storagenodes"]] services = set() storage_sign_quorum = config.get("storage-sign-quorum-size", 0) allnodenames = set(signingnodenames + mergenodenames + frontendnodenames + statusservernames + storagenodenames) if "frontendnodes" in nodetype: reloadableplopconfig.append((Symbol("storage_nodes"), storagenodeaddresses)) reloadableplopconfig.append((Symbol("storage_nodes_quorum"), config["storage-quorum-size"])) reloadableplopconfig.append((Symbol("storage_sign_quorum"), storage_sign_quorum)) services.add(Symbol("ht")) allowed_clients += allowed_clients_frontend(mergenodenames, primarymergenodename) allowed_clients += allowed_clients_public() allowed_servers += allowed_servers_frontend(signingnodenames, storagenodenames) if "storagenodes" in nodetype: allowed_clients += allowed_clients_storage(frontendnodenames, mergenodenames) if "signingnodes" in nodetype: reloadableplopconfig.append((Symbol("storage_sign_quorum"), storage_sign_quorum)) allowed_clients += allowed_clients_signing(frontendnodenames, primarymergenodename) services = [Symbol("sign")] if "mergenodes" in nodetype: reloadableplopconfig.append((Symbol("storage_nodes"), storagenodeaddresses)) reloadableplopconfig.append((Symbol("storage_nodes_quorum"), config["storage-quorum-size"])) services.add(Symbol("ht")) if nodename == primarymergenodename: mergesecondarynames = [node["name"] for node in config["mergenodes"] if node["name"] != primarymergenodename] mergesecondaryaddresses = ["https://%s/plop/v1/merge/" % node["address"] for node in config["mergenodes"] if node["name"] != primarymergenodename] merge = localconfig["merge"] plopconfig.append((Symbol("db_backend_opt"), [(Symbol("write_flag"), Symbol("read"))])) plopconfig.append((Symbol("merge_delay"), merge["min-delay"])) plopconfig.append((Symbol("merge_dist_winsize"), min(50000, merge["dist-window-size"]))) plopconfig.append((Symbol("merge_dist_sendlog_chunksize"), merge["dist-sendlog-chunksize"])) plopconfig.append((Symbol("merge_dist_sendentries_chunksize"), merge["dist-sendentries-chunksize"])) reloadableplopconfig.append((Symbol("merge_backup_winsize"), min(50000, merge["backup-window-size"]))) reloadableplopconfig.append((Symbol("merge_backup_sendlog_chunksize"), merge["backup-sendlog-chunksize"])) reloadableplopconfig.append((Symbol("merge_backup_sendentries_chunksize"), merge["backup-sendentries-chunksize"])) reloadableplopconfig.append((Symbol("frontend_nodes"), zip(frontendnodenames, frontendnodeaddresses))) reloadableplopconfig.append((Symbol("merge_secondaries"), zip(mergesecondarynames, mergesecondaryaddresses))) plopconfig.append((Symbol("sth_path"), paths["mergedb"] + "/sth")) plopconfig.append((Symbol("fetched_path"), paths["mergedb"] + "/fetched")) plopconfig.append((Symbol("verified_path"), paths["mergedb"] + "/verified")) plopconfig.append((Symbol("minsize_path"), paths["mergedb"] + "/minsize")) reloadableplopconfig.append((Symbol("signing_nodes"), signingnodeaddresses)) reloadableplopconfig.append((Symbol("backup_quorum"), config["backup-quorum-size"])) allowed_servers += allowed_servers_primarymerge(frontendnodenames, mergesecondarynames, signingnodenames) else: allowed_clients += allowed_clients_mergesecondary(primarymergenodename) plopconfig += [ (Symbol("services"), list(services)), ] if "signingnodes" in nodetype: hsm = localconfig.get("hsm") if "logprivatekey" in paths: plopconfig.append((Symbol("log_private_key"), paths["logprivatekey"])) if hsm: plopconfig.append((Symbol("hsm"), [hsm.get("library"), str(hsm.get("slot")), "ecdsa", hsm.get("label"), hsm.get("pin")])) if not ("logprivatekey" in paths or hsm): print >>sys.stderr, "Neither logprivatekey nor hsm configured for signing node", nodename sys.exit(1) plopconfig += [ (Symbol("log_public_key"), Binary(base64.b64decode(config["logpublickey"]))), (Symbol("own_key"), (nodename, "%s/%s-private.pem" % (paths["privatekeys"], nodename))), ] if "frontendnodes" in nodetype: reloadableplopconfig.append((Symbol("signing_nodes"), signingnodeaddresses)) plopconfig += [ (Symbol("plopconfig"), plopconfigfilename), (Symbol("plopcontrol"), plopcontrolfilename), ] reloadableplopconfig.append((Symbol("statusservers"), statusserveraddresses)) reloadableplopconfig.append((Symbol("benchservers"), benchserveraddresses)) allowed_servers += [ ("/plop/v1/status/merge_dist", statusservernames), ("/plop/v1/status/merge_backup", statusservernames), ("/plop/v1/status/merge_sth", statusservernames), ("/plop/v1/status/merge_fetch", statusservernames), ("/plop/v1/status/storage", statusservernames), ("/plop/v1/status/merge_errors", statusservernames), ("/plop/v1/status/heartbeat", statusservernames), ("/plop/v1/bench/merge_dist", statusservernames), ("/plop/v1/bench/merge_backup", statusservernames), ("/plop/v1/bench/merge_sth", statusservernames), ("/plop/v1/bench/merge_fetch", statusservernames), ] if "statusservers" in nodetype: allowed_clients += [ ("/plop/v1/status/merge_dist", mergenodenames), ("/plop/v1/status/merge_backup", mergenodenames), ("/plop/v1/status/merge_sth", mergenodenames), ("/plop/v1/status/merge_fetch", mergenodenames), ("/plop/v1/status/merge_errors", mergenodenames), ("/plop/v1/status/storage", storagenodenames), ("/plop/v1/status/heartbeat", list(allnodenames)), ("/plop/v1/bench/merge_dist", mergenodenames), ("/plop/v1/bench/merge_backup", mergenodenames), ("/plop/v1/bench/merge_sth", mergenodenames), ("/plop/v1/bench/merge_fetch", mergenodenames), ("/status", Symbol("noauth")), ("/bench", Symbol("noauth")), ] reloadableplopconfig += [ (Symbol("allowed_clients"), list(allowed_clients)), (Symbol("allowed_servers"), list(allowed_servers)), (Symbol("storage_node_names"), list(storagenodenames)), (Symbol("apikeys"), apikeys), (Symbol("version"), config["version"]), ] erlangconfig = [ (Symbol("sasl"), saslconfig), (Symbol("catlfish"), catlfishconfig), (Symbol("lager"), lagerconfig), (Symbol("plop"), plopconfig), ] print >>configfile, gen_erlang(erlangconfig) + ".\n" print >>plopconfigfile, gen_erlang(reloadableplopconfig) + ".\n" configfile.close() plopconfigfile.close() def print_nodevar(configfile, delimiter, nodetype, bootname, names, testaddresses, gennodes=True, gentesturl=True): if gennodes: print >>configfile, nodetype + "NODES=" + delimiter + " ".join(['%s:%s' % (name, bootname) for name in names]) + delimiter for name in names: print >>configfile, name.upper().replace("-", "_") + "ONLY" + "NODES=" + delimiter + '%s:%s' % (name, bootname) + delimiter if gentesturl: print >>configfile, name.upper().replace("-", "_") + "ONLY" + "TESTURLS=" + delimiter + " ".join(testaddresses[name]) + delimiter else: print >>configfile, nodetype + "NODES=" + delimiter + delimiter if gentesturl: print >>configfile, nodetype + "TESTURLS=" + delimiter + " ".join([address for name in names for address in testaddresses[name]]) + delimiter else: print >>configfile, nodetype + "TESTURLS=" + delimiter + delimiter def multivaldict(dictlist): d = {} for (key, value) in dictlist: d.setdefault(key, []).append(value) return d def gen_testmakefile(config, testmakefile, shellvars=False): configfile = open(testmakefile, "w") print >>configfile, "#", testmakefile, "generated by", sys.argv[0], datetime.now() frontendnodenames = set([node["name"] for node in config["frontendnodes"]]) storagenodenames = set([node["name"] for node in config["storagenodes"]]) signingnodenames = set([node["name"] for node in config["signingnodes"]]) mergenodenames = set([node["name"] for node in config["mergenodes"]]) mergesecondarynodenames = set([node["name"] for node in config["mergenodes"] if node["name"] != config["primarymergenode"]]) statusservernodenames = set([node["name"] for node in config.get("statusservers")]) frontendnodenames_except_last = sorted(frontendnodenames)[:-1] frontendnodenames_except_first = sorted(frontendnodenames)[1:] allnodes = config["frontendnodes"] + config["storagenodes"] + config["signingnodes"] + config["mergenodes"] + config["statusservers"] testaddresses = multivaldict([(node["name"], node["address"]) for node in allnodes]) delimiter = '"' if shellvars else '' print_nodevar(configfile, delimiter, "FRONTEND", "catlfish", frontendnodenames, testaddresses) print_nodevar(configfile, delimiter, "FRONTENDEXCEPTLAST", "catlfish", frontendnodenames_except_last, testaddresses) print_nodevar(configfile, delimiter, "FRONTENDEXCEPTFIRST", "catlfish", frontendnodenames_except_first, testaddresses) print_nodevar(configfile, delimiter, "STORAGE", "catlfish", storagenodenames, testaddresses, gennodes=False) print_nodevar(configfile, delimiter, "SIGNING", "catlfish", signingnodenames, testaddresses) print_nodevar(configfile, delimiter, "MERGESECONDARY", "catlfish", mergesecondarynodenames, testaddresses) print_nodevar(configfile, delimiter, "MERGEPRIMARY", "merge", [config["primarymergenode"]], testaddresses, gentesturl=False) print_nodevar(configfile, delimiter, "STATUSSERVER", "statusserver", statusservernodenames, testaddresses) print >>configfile, "NODES=" + delimiter + " ".join(set([node["name"] for node in allnodes])) + delimiter print >>configfile, "MACHINES=" + delimiter + " ".join([str(e) for e in frontendnodenames]) + delimiter print >>configfile, "BASEURL=" + delimiter + config["baseurl"] + delimiter configfile.close() def printnodenames(config): frontendnodenames = set([node["name"] for node in config["frontendnodes"]]) storagenodenames = set([node["name"] for node in config["storagenodes"]]) signingnodenames = set([node["name"] for node in config["signingnodes"]]) mergenodenames = set([node["name"] for node in config["mergenodes"]]) statusservernodenames = set([node["name"] for node in config.get("statusservers")]) print " ".join(frontendnodenames|storagenodenames|signingnodenames|mergenodenames|statusservernodenames) def gen_manpage(manpagedir): manpage.rewrite_manpage(manpagedir + "/catlfish-log.cfg.in.5.adoc", globalconfigschema, "Catlfish", "Catlfish Manual", "CATLFISH-LOG.CFG.IN(5)", "catlfish-log.cfg.in - catlfish log configuration") manpage.rewrite_manpage(manpagedir + "/catlfish-node.cfg.5.adoc", localconfigschema, "Catlfish", "Catlfish Manual", "CATLFISH-NODE.CFG(5)", "catlfish-node.cfg - catlfish node configuration") localconfigschema = [ ("nodename", "string", "nodename"), ("frontendaddress", "string", "ip address"), ("ctapiaddress", "string", "ip address"), ("storageaddress", "string", "ip address"), ("signingaddress", "string", "ip address"), ("mergeaddress", "string", "ip address"), ("publichttpaddress", "string", "ip address"), ("configurl", "string", "url"), ("logadminkey", "string", "key"), ("dbbackend", "string", ["permdb", "fsdb"]), ("paths/configdir", "string", "path"), ("paths/db", "string", "path"), ("paths/https_cacertfile", "string", "path"), ("paths/https_certfile", "string", "path"), ("paths/https_keyfile", "string", "path"), ("paths/knownroots", "string", "path"), ("paths/logpublickey", "string", "path"), ("paths/privatekeys", "string", "path"), ("paths/public_cacertfile", "string", "path"), ("paths/publickeys", "string", "path"), ("paths/verifycert_bin", "string", "path"), ("paths/mergedb", "string", "path"), ("paths/logprivatekey", "string", "path"), ("ratelimits/add_chain", "string", "rate"), ("merge/min-delay", "integer", "seconds"), ("merge/backup-window-size", "integer", "number of entries"), ("merge/backup-sendlog-chunksize", "integer", "number of entries"), ("merge/backup-sendentries-chunksize", "integer", "number of entries"), ("merge/dist-window-size", "integer", "number of entries"), ("merge/dist-sendlog-chunksize", "integer", "number of entries"), ("merge/dist-sendentries-chunksize", "integer", "number of entries"), ] globalconfigschema = [ ("frontendnodes/[]/name", "string", "nodename"), ("frontendnodes/[]/address", "string", "ip address"), ("frontendnodes/[]/publicaddress", "string", "ip address"), ("mergenodes/[]/name", "string", "nodename"), ("mergenodes/[]/address", "string", "ip address"), ("signingnodes/[]/name", "string", "nodename"), ("signingnodes/[]/address", "string", "ip address"), ("storagenodes/[]/name", "string", "nodename"), ("storagenodes/[]/address", "string", "ip address"), ("statusservers/[]/name", "string", "nodename"), ("statusservers/[]/address", "string", "ip address"), ("statusservers/[]/publicaddress", "string", "ip address"), ("apikeys/[]/nodename", "string", "nodename"), ("apikeys/[]/publickey", "string", "key"), ("baseurl", "string", "url"), ("primarymergenode", "string", "nodename"), ("storage-quorum-size", "integer", "number of nodes"), ("backup-quorum-size", "integer", "number of nodes"), ("logpublickey", "string", "key"), ("cafingerprint", "string", "fingerprint"), ("version", "integer", "version"), ("mmd", "integer", "seconds"), ] def main(): parser = argparse.ArgumentParser(description="") parser.add_argument('--config', help="System configuration") parser.add_argument("--manpagedir", metavar="file", help="Generate manpages to directory") parser.add_argument('--localconfig', help="Local configuration") parser.add_argument("--testmakefile", metavar="file", help="Generate makefile variables for test") parser.add_argument("--testshellvars", metavar="file", help="Generate shell variable file for test") parser.add_argument("--getnodenames", action='store_true', help="Get list of node names") args = parser.parse_args() if args.manpagedir: gen_manpage(args.manpagedir) sys.exit(0) if not args.config: print >>sys.stderr, "either --config or --manpage is required" sys.exit(1) if args.testmakefile: config = readconfig.read_config(args.config, schema=globalconfigschema) gen_testmakefile(config, args.testmakefile) elif args.testshellvars: config = readconfig.read_config(args.config, schema=globalconfigschema) gen_testmakefile(config, args.testshellvars, shellvars=True) elif args.getnodenames: config = readconfig.read_config(args.config, schema=globalconfigschema) printnodenames(config) elif args.localconfig: localconfig = readconfig.read_config(args.localconfig, schema=localconfigschema) config = readconfig.verify_and_read_config(args.config, localconfig["logadminkey"], schema=globalconfigschema) gen_config(localconfig["nodename"], config, localconfig) else: print >>sys.stderr, "Nothing to do" sys.exit(1) main()