Parallelised merge, sth phase.

9 files changed, 86 insertions, 26 deletions

diff --git a/test/scripts/light-system-test-prepare-merge-takeover.sh b/test/scripts/light-system-test-prepare-merge-takeover.sh
index 67e5ac7..4b50ad7 100755
--- a/test/scripts/light-system-test-prepare-merge-takeover.sh
+++ b/test/scripts/light-system-test-prepare-merge-takeover.sh
@@ -9,8 +9,8 @@ top_srcdir=$(cd $(dirname $0)/../..; pwd)
 
 mv mergedb mergedb-down
 mv mergedb-secondary mergedb
-init_log
-cp mergedb-down/sth mergedb/    # After init_log.
+mv mergedb/verifiedsize mergedb/verifiedsize.OFF
+cp mergedb-down/logorder mergedb/
 mkdir mergedb-secondary
 touch mergedb-secondary/logorder
 printf 0 > mergedb-secondary/verifiedsize
diff --git a/test/scripts/light-system-test-start.sh b/test/scripts/light-system-test-start.sh
index 4520945..985a23c 100755
--- a/test/scripts/light-system-test-start.sh
+++ b/test/scripts/light-system-test-start.sh
@@ -14,8 +14,11 @@ start_node() {
                    "exec ../bin/erl -boot ${node_app} -config ${node}"
 }
 
-for node in ${ERLANGNODES}; do
-    start_node $(echo $node | tr ':' ' ')
+for nodegroup in $SIGNINGNODES $MERGESECONDARYNODES $STORAGENODES $FRONTENDNODES; do
+    for node in $nodegroup; do
+        echo "starting $node"
+        start_node $(echo $node | tr ':' ' ')
+    done
 done
 for i in 1 2 3 4 5 6 7 8 9 10; do
     echo "waiting for system to start"
@@ -37,3 +40,7 @@ for i in 1 2 3 4 5 6 7 8 9 10; do
         echo Not started: ${notstarted}
     fi
 done
+
+[ $# -gt 0 ] && [ "$1" = "initlog" ] && $top_srcdir/tools/initlog.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg
+
+start_node $(echo ${MERGEPRIMARYNODE} | tr ':' ' ')
diff --git a/test/scripts/light-system-test-stop.sh b/test/scripts/light-system-test-stop.sh
index 0bf4374..2265a34 100755
--- a/test/scripts/light-system-test-stop.sh
+++ b/test/scripts/light-system-test-stop.sh
@@ -12,6 +12,9 @@ stop_node() {
     ${top_srcdir}/tools/to_catlfish.py to_erl nodes/${node}/ "init:stop()"
 }
 
-for node in ${ERLANGNODES}; do
-    stop_node $(echo $node | tr ':' ' ')
+for nodegroup in $FRONTENDNODES $MERGEPRIMARYNODE $STORAGENODES $MERGESECONDARYNODES $SIGNINGNODES; do
+    for node in $nodegroup; do
+        echo "starting $node"
+        stop_node $(echo $node | tr ':' ' ')
+    done
 done
diff --git a/test/scripts/light-system-test.sh b/test/scripts/light-system-test.sh
index 1f761e8..d2d242c 100755
--- a/test/scripts/light-system-test.sh
+++ b/test/scripts/light-system-test.sh
@@ -10,7 +10,7 @@ top_srcdir=$(cd $(dirname $0)/../..; pwd)
 SCRIPTS=${top_srcdir}/test/scripts
 
 tests_start() {
-    ${SCRIPTS}/light-system-test-start.sh
+    ${SCRIPTS}/light-system-test-start.sh $@
 }
 
 tests_stop() {
@@ -18,8 +18,7 @@ tests_stop() {
 }
 
 ${SCRIPTS}/light-system-test-prepare.sh
-init_log
-tests_start
+tests_start initlog
 ${SCRIPTS}/light-system-test-run-1.sh || (echo "Tests failed"; sleep 5; tests_stop; false)
 sleep 5
 tests_stop
@@ -50,7 +49,7 @@ echo
 echo
 echo
 ${SCRIPTS}/light-system-test-prepare-merge-takeover.sh
-tests_start
+tests_start initlog
 ${SCRIPTS}/light-system-test-run-3.sh || (echo "Tests failed"; sleep 5; tests_stop; false)
 sleep 5
 tests_stop
diff --git a/test/scripts/testutils.sh b/test/scripts/testutils.sh
index 657607b..88e9a54 100644
--- a/test/scripts/testutils.sh
+++ b/test/scripts/testutils.sh
@@ -47,7 +47,3 @@ do_merge() {
         sleep 1
     done
 }
-
-init_log() {
-    ${top_srcdir}/tools/initlog.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || fail "Log init failed"
-}
diff --git a/tools/certtools.py b/tools/certtools.py
index 9febc69..82339aa 100644
--- a/tools/certtools.py
+++ b/tools/certtools.py
@@ -761,3 +761,21 @@ def write_file(fn, jsondata):
     tempname = fn + ".new"
     open(tempname, 'w').write(json.dumps(jsondata))
     mv_file(tempname, fn)
+
+def generate_tree_head_signature(signingnodes, own_key,
+                                 tree_size, timestamp, root_hash):
+    tree_head_signature = None
+    for signingnode in signingnodes:
+        try:
+            tree_head_signature = \
+              create_sth_signature(tree_size, timestamp,
+                                   root_hash,
+                                   "https://%s/" % signingnode["address"],
+                                   key=own_key)
+            break
+        except requests.exceptions.HTTPError, e:
+            logging.warning("create_sth_signature error: %s", e.response)
+    if tree_head_signature == None:
+        logging.error("Could not contact any signing nodes")
+
+    return tree_head_signature
diff --git a/tools/compileconfig.py b/tools/compileconfig.py
index abe59a3..a5f013a 100755
--- a/tools/compileconfig.py
+++ b/tools/compileconfig.py
@@ -186,7 +186,7 @@ def allowed_servers_frontend(signingnodenames, storagenodenames):
         ("/plop/v1/signing/sct", signingnodenames),
     ]
 
-def allowed_servers_primarymerge(frontendnodenames, secondarymergenames):
+def allowed_servers_primarymerge(frontendnodenames, secondarymergenames, signingnodenames):
     return [
         ("/plop/v1/frontend/verify-entries", frontendnodenames),
         ("/plop/v1/frontend/sendlog", frontendnodenames),
@@ -197,6 +197,7 @@ def allowed_servers_primarymerge(frontendnodenames, secondarymergenames):
         ("/plop/v1/merge/setverifiedsize", secondarymergenames),
         ("/plop/v1/merge/sendlog", secondarymergenames),
         ("/plop/v1/merge/sendentry", secondarymergenames),
+        ("/plop/v1/signing/sth", signingnodenames),
     ]
 
 def parse_ratelimit_expression(expression):
@@ -327,7 +328,7 @@ def gen_config(nodename, config, localconfig):
             (Symbol("entry_root_path"), paths["mergedb"] + "/chains"),
             ]
 
-    signingnodes = config["signingnodes"]
+    signingnodenames = [node["name"] for node in config["signingnodes"]]
     signingnodeaddresses = ["https://%s/plop/v1/signing/" % node["address"] for node in config["signingnodes"]]
     mergenodenames = [node["name"] for node in config["mergenodes"]]
     primarymergenodename = config["primarymergenode"]
@@ -346,7 +347,7 @@ def gen_config(nodename, config, localconfig):
         services.add(Symbol("ht"))
         allowed_clients += allowed_clients_frontend(mergenodenames, primarymergenodename)
         allowed_clients += allowed_clients_public()
-        allowed_servers += allowed_servers_frontend([node["name"] for node in signingnodes], storagenodenames)
+        allowed_servers += allowed_servers_frontend(signingnodenames, storagenodenames)
     if "storagenodes" in nodetype:
         allowed_clients += allowed_clients_storage(frontendnodenames, mergenodenames)
     if "signingnodes" in nodetype:
@@ -373,7 +374,9 @@ def gen_config(nodename, config, localconfig):
             plopconfig.append((Symbol("sth_path"), paths["mergedb"] + "/sth"))
             plopconfig.append((Symbol("fetched_path"), paths["mergedb"] + "/fetched"))
             plopconfig.append((Symbol("verified_path"), paths["mergedb"] + "/verified"))
-            allowed_servers += allowed_servers_primarymerge(frontendnodenames, mergesecondarynames)
+            reloadableplopconfig.append((Symbol("signing_nodes"), signingnodeaddresses))
+            reloadableplopconfig.append((Symbol("backup_quorum"), config["backup-quorum-size"]))
+            allowed_servers += allowed_servers_primarymerge(frontendnodenames, mergesecondarynames, signingnodenames)
         else:
             allowed_clients += allowed_clients_mergesecondary(primarymergenodename)
 
@@ -429,19 +432,25 @@ def gen_testmakefile(config, testmakefile, machines, shellvars=False):
     storagenodenames = set([node["name"] for node in config["storagenodes"]])
     signingnodenames = set([node["name"] for node in config["signingnodes"]])
     mergenodenames = set([node["name"] for node in config["mergenodes"]])
-    erlangnodenames_and_apps = ['%s:%s' % (nn, 'catlfish' if nn != config["primarymergenode"] else "merge") for nn in frontendnodenames | storagenodenames | signingnodenames | mergenodenames]
+    mergesecondarynodenames = set([node["name"] for node in config["mergenodes"] if node["name"] != config["primarymergenode"]])
 
     frontendnodeaddresses = [node["publicaddress"] for node in config["frontendnodes"]]
     storagenodeaddresses = [node["address"] for node in config["storagenodes"]]
     signingnodeaddresses = [node["address"] for node in config["signingnodes"]]
-    mergenodeaddresses = [node["address"] for node in config["mergenodes"] if node["name"] != config["primarymergenode"]]
+    mergesecondarynodeaddresses = [node["address"] for node in config["mergenodes"] if node["name"] != config["primarymergenode"]]
 
     delimiter = '"' if shellvars else ''
     
     print >>configfile, "NODES=" + delimiter + " ".join(frontendnodenames|storagenodenames|signingnodenames|mergenodenames) + delimiter
-    print >>configfile, "ERLANGNODES=" + delimiter + " ".join(erlangnodenames_and_apps) + delimiter
+
+    print >>configfile, "FRONTENDNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in frontendnodenames]) + delimiter
+    print >>configfile, "STORAGENODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in storagenodenames if name not in frontendnodenames]) + delimiter
+    print >>configfile, "SIGNINGNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in signingnodenames]) + delimiter
+    print >>configfile, "MERGESECONDARYNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in mergesecondarynodenames]) + delimiter
+    print >>configfile, "MERGEPRIMARYNODE=" '%s:%s' % (config["primarymergenode"], 'merge')
+
     print >>configfile, "MACHINES=" + delimiter + " ".join([str(e) for e in range(1, machines+1)]) + delimiter
-    print >>configfile, "TESTURLS=" + delimiter + " ".join(frontendnodeaddresses+storagenodeaddresses+signingnodeaddresses+mergenodeaddresses) + delimiter
+    print >>configfile, "TESTURLS=" + delimiter + " ".join(frontendnodeaddresses+storagenodeaddresses+signingnodeaddresses+mergesecondarynodeaddresses) + delimiter
     print >>configfile, "BASEURL=" + delimiter + config["baseurl"] + delimiter
 
     configfile.close()
diff --git a/tools/initlog.py b/tools/initlog.py
index 11ebc2e..3ef5720 100755
--- a/tools/initlog.py
+++ b/tools/initlog.py
@@ -14,7 +14,7 @@ import yaml
 import errno
 from time import time
 from base64 import b64encode
-from certtools import build_merkle_tree, write_file
+from certtools import build_merkle_tree, write_file, generate_tree_head_signature
 from mergetools import get_sth, perm, get_logorder
 
 def parse_args():
@@ -33,6 +33,9 @@ def parse_args():
 def main():
     """
     Initialise a log by creating
+    - sth file
+      - must not exist before
+      - consulting 'logorder' if it exists
     - perm database if it doesn't exist
     """
     args, config, localconfig = parse_args()
@@ -65,6 +68,31 @@ def main():
     # Create a chains database.
     chainsdb = perm(localconfig.get("dbbackend", "filedb"), mergedb + "/chains")
 
+    # Create sth file.
+    tree_size = 0
+    root_hash = build_merkle_tree('')[-1][0]
+    try:
+        logorder = get_logorder(logorderfile)
+        tree_size = len(logorder)
+        root_hash = build_merkle_tree(logorder[:tree_size])[-1][0]
+    except IOError, e:
+        if e.errno == errno.ENOENT:
+            pass
+    timestamp = int(time() * 1000)
+    tree_head_signature = \
+      generate_tree_head_signature(signingnodes, own_key,
+                                   tree_size, timestamp, root_hash)
+    if tree_head_signature == None:
+        return 1
+
+    sth = {"tree_size": tree_size,
+           "timestamp": timestamp,
+           "sha256_root_hash": b64encode(root_hash),
+           "tree_head_signature": b64encode(tree_head_signature)}
+    print "Creating sth file with tree size", tree_size, \
+      "and timestamp", timestamp
+    write_file(sthfile, sth)
+
     return 0
 
 if __name__ == '__main__':
diff --git a/tools/merge b/tools/merge
index d0198ec..19e22f8 100755
--- a/tools/merge
+++ b/tools/merge
@@ -5,6 +5,6 @@ set -o errexit
 BINDIR=$(dirname $0)
 
 $BINDIR/merge_fetch.py "$@"
-#$BINDIR/merge_backup.py "$@"
-$BINDIR/merge_sth.py "$@"
-#$BINDIR/merge_dist.py "$@"
+# $BINDIR/merge_backup.py "$@"
+# $BINDIR/merge_sth.py "$@"
+# $BINDIR/merge_dist.py "$@"