Improve start and stop of Erlang nodes

6 files changed, 97 insertions, 47 deletions

diff --git a/test/scripts/light-system-test-run-1.sh b/test/scripts/light-system-test-run-1.sh
index 585a829..7d52ed7 100755
--- a/test/scripts/light-system-test-run-1.sh
+++ b/test/scripts/light-system-test-run-1.sh
@@ -32,7 +32,8 @@ python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert5.tx
 python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/pre1.txt:${top_srcdir}/tools/testcerts/pre2.txt fetchcertstore/00000005:fetchcertstore/00000006 || fail"Verification failed"
 python ${top_srcdir}/tools/storagegc.py --config machine/machine-1/catlfish-test.cfg --localconfig machine/machine-1/catlfish-test-local-1.cfg || fail "GC failed"
 
-${top_srcdir}/tools/to_catlfish.py to_erl nodes/merge-2/ "init:stop()"
+${top_srcdir}/test/scripts/light-system-test-stop.sh mergesecondary
+
 python ${top_srcdir}/tools/submitcert.py --parallel=1 --store ${top_srcdir}/tools/testcerts/cert6.txt --check-sct --sct-file=submittedcerts ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Submission failed"
 echo NOTE: merge backup should fail with 111 Connection refused
 
@@ -42,12 +43,10 @@ sleep 3
 assert_equal "Tree size" "$(get_treesize)" 7
 
 check_sth
-../bin/run_erl -daemon nodes/merge-2/ nodes/merge-2/log/ "exec ../bin/erl -config merge-2"
-for i in 1 2 3 4 5 6 7 8 9 10; do
-    echo "waiting for system to start" ; \
-    sleep 0.5 ; \
-    if curl -s --cacert httpsca/demoCA/cacert.pem -4 https://localhost:8181 > /dev/null ; then break; fi
-done
+
+
+${top_srcdir}/test/scripts/light-system-test-start.sh mergesecondary
+
 do_merge 8
 
 assert_equal "Tree size" "$(get_treesize)" 8
diff --git a/test/scripts/light-system-test-start.sh b/test/scripts/light-system-test-start.sh
index cdfda8c..51d38ac 100755
--- a/test/scripts/light-system-test-start.sh
+++ b/test/scripts/light-system-test-start.sh
@@ -7,6 +7,8 @@ top_srcdir=$(cd $(dirname $0)/../..; pwd)
 
 . ${top_srcdir}/test/scripts/testutils.sh
 
+nodetypes="$@"
+
 start_node() {
     node=$1
     node_app=$2
@@ -14,18 +16,24 @@ start_node() {
                    "exec ../bin/erl -boot ${node_app} -config ${node}"
 }
 
-for nodegroup in $SIGNINGNODES $MERGESECONDARYNODES $STORAGENODES $FRONTENDNODES; do
-    for node in $nodegroup; do
-        echo "starting $node"
-        start_node $(echo $node | tr ':' ' ')
-    done
+nodes=
+testurls=
+for nodetype in $nodetypes; do
+    nodetype_uc=$(echo $nodetype | tr a-z A-Z)
+    nodes="$nodes $(eval echo \$${nodetype_uc}NODES)"
+    testurls="$testurls $(eval echo \$${nodetype_uc}TESTURLS)"
+done
+
+for node in $nodes; do
+    echo "starting $node"
+    start_node $(echo $node | tr ':' ' ')
 done
 for i in 1 2 3 4 5 6 7 8 9 10; do
     echo "waiting for system to start"
     sleep 0.5
     allstarted=1
     notstarted=
-    for testurl in ${TESTURLS}; do
+    for testurl in ${testurls}; do
 	if curl -s --cacert httpsca/demoCA/cacert.pem -4 https://${testurl} > /dev/null ; then
             :
         else
@@ -40,10 +48,3 @@ for i in 1 2 3 4 5 6 7 8 9 10; do
         echo Not started: ${notstarted}
     fi
 done
-
-if [ $# -gt 0 ] && [ "$1" = "initlog" ]; then
-    shift 1
-    $top_srcdir/tools/initlog.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg $@
-fi
-
-start_node $(echo ${MERGEPRIMARYNODE} | tr ':' ' ')
diff --git a/test/scripts/light-system-test-stop.sh b/test/scripts/light-system-test-stop.sh
index aef9c76..a946079 100755
--- a/test/scripts/light-system-test-stop.sh
+++ b/test/scripts/light-system-test-stop.sh
@@ -7,14 +7,20 @@ top_srcdir=$(cd $(dirname $0)/../..; pwd)
 
 . ${top_srcdir}/test/scripts/testutils.sh
 
+nodetypes="$@"
+
 stop_node() {
     node=$1
     ${top_srcdir}/tools/to_catlfish.py to_erl nodes/${node}/ "init:stop()"
 }
 
-for nodegroup in $FRONTENDNODES $MERGEPRIMARYNODE $STORAGENODES $MERGESECONDARYNODES $SIGNINGNODES; do
-    for node in $nodegroup; do
-        echo "stopping $node"
-        stop_node $(echo $node | tr ':' ' ')
-    done
+nodes=
+for nodetype in $nodetypes; do
+    nodetype_uc=$(echo $nodetype | tr a-z A-Z)
+    nodes="$nodes $(eval echo \$${nodetype_uc}NODES)"
+done
+
+for node in $nodes; do
+    echo "stopping $node"
+    stop_node $(echo $node | tr ':' ' ')
 done
diff --git a/test/scripts/light-system-test.sh b/test/scripts/light-system-test.sh
index ad3daca..217fb2c 100755
--- a/test/scripts/light-system-test.sh
+++ b/test/scripts/light-system-test.sh
@@ -10,18 +10,24 @@ top_srcdir=$(cd $(dirname $0)/../..; pwd)
 SCRIPTS=${top_srcdir}/test/scripts
 
 tests_start() {
-    ${SCRIPTS}/light-system-test-start.sh $@
+    ${SCRIPTS}/light-system-test-start.sh "$@"
 }
 
 tests_stop() {
-    ${SCRIPTS}/light-system-test-stop.sh
+    ${SCRIPTS}/light-system-test-stop.sh "$@"
+}
+
+tests_stop_all() {
+    ${SCRIPTS}/light-system-test-stop.sh mergeprimary mergesecondary frontend storage signing
 }
 
 ${SCRIPTS}/light-system-test-prepare.sh
-tests_start initlog
-${SCRIPTS}/light-system-test-run-1.sh || (echo "Tests failed"; sleep 5; tests_stop; false)
+tests_start signing mergesecondary storage frontend
+${top_srcdir}/tools/initlog.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg
+tests_start mergeprimary
+${SCRIPTS}/light-system-test-run-1.sh || (echo "Tests failed"; sleep 5; tests_stop_all; false)
 sleep 5
-tests_stop
+tests_stop_all
 sleep 5
 echo
 echo
@@ -33,10 +39,10 @@ echo
 echo
 echo
 echo
-tests_start
-${SCRIPTS}/light-system-test-run-2.sh || (echo "Tests failed"; sleep 5; tests_stop; false)
+tests_start signing mergesecondary storage frontend mergeprimary
+${SCRIPTS}/light-system-test-run-2.sh || (echo "Tests failed"; sleep 5; tests_stop_all; false)
 sleep 5
-tests_stop
+tests_stop_all
 sleep 5
 echo
 echo
@@ -49,10 +55,12 @@ echo
 echo
 echo
 ${SCRIPTS}/light-system-test-prepare-merge-takeover.sh
-tests_start initlog --promote-secondary
-${SCRIPTS}/light-system-test-run-3.sh || (echo "Tests failed"; sleep 5; tests_stop; false)
+tests_start signing mergesecondary storage frontend
+${top_srcdir}/tools/initlog.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg --promote-secondary
+tests_start mergeprimary
+${SCRIPTS}/light-system-test-run-3.sh || (echo "Tests failed"; sleep 5; tests_stop_all; false)
 sleep 5
-tests_stop
+tests_stop_all
 sleep 5
 echo
 echo
@@ -65,8 +73,8 @@ echo
 echo
 echo
 ${SCRIPTS}/light-system-test-prepare-redistribute-frontend.sh
-tests_start
-${SCRIPTS}/light-system-test-run-4.sh || (echo "Tests failed"; sleep 5; tests_stop; false)
+tests_start signing mergesecondary storage frontend mergeprimary
+${SCRIPTS}/light-system-test-run-4.sh || (echo "Tests failed"; sleep 5; tests_stop_all; false)
 
 echo
 echo
@@ -80,6 +88,6 @@ echo
 echo
 
 
-${SCRIPTS}/light-system-test-run-5.sh || (echo "Tests failed"; sleep 5; tests_stop; false)
+${SCRIPTS}/light-system-test-run-5.sh || (echo "Tests failed"; sleep 5; tests_stop_all; false)
 sleep 5
-tests_stop
+tests_stop_all
diff --git a/tools/compileconfig.py b/tools/compileconfig.py
index 84b2a4a..9169639 100755
--- a/tools/compileconfig.py
+++ b/tools/compileconfig.py
@@ -448,10 +448,14 @@ def gen_testmakefile(config, testmakefile, machines, shellvars=False):
     print >>configfile, "STORAGENODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in storagenodenames if name not in frontendnodenames]) + delimiter
     print >>configfile, "SIGNINGNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in signingnodenames]) + delimiter
     print >>configfile, "MERGESECONDARYNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in mergesecondarynodenames]) + delimiter
-    print >>configfile, "MERGEPRIMARYNODE=" '%s:%s' % (config["primarymergenode"], 'merge')
+    print >>configfile, "MERGEPRIMARYNODES=" + delimiter + '%s:%s' % (config["primarymergenode"], 'merge') + delimiter
 
     print >>configfile, "MACHINES=" + delimiter + " ".join([str(e) for e in range(1, machines+1)]) + delimiter
-    print >>configfile, "TESTURLS=" + delimiter + " ".join(frontendnodeaddresses+storagenodeaddresses+signingnodeaddresses+mergesecondarynodeaddresses) + delimiter
+    print >>configfile, "FRONTENDTESTURLS=" + delimiter + " ".join(frontendnodeaddresses) + delimiter
+    print >>configfile, "STORAGETESTURLS=" + delimiter + " ".join(storagenodeaddresses) + delimiter
+    print >>configfile, "SIGNINGTESTURLS=" + delimiter + " ".join(signingnodeaddresses) + delimiter
+    print >>configfile, "MERGESECONDARYTESTURLS=" + delimiter + " ".join(mergesecondarynodeaddresses) + delimiter
+    print >>configfile, "MERGEPRIMARYTESTURLS=" + delimiter + " ".join([]) + delimiter
     print >>configfile, "BASEURL=" + delimiter + config["baseurl"] + delimiter
 
     configfile.close()
diff --git a/tools/testcase1.py b/tools/testcase1.py
index 6a12301..63be2e1 100755
--- a/tools/testcase1.py
+++ b/tools/testcase1.py
@@ -165,18 +165,50 @@ def correct_tree_size(expected):
             return False
     return True
 
+import readconfig
+
+def mergestatus(configfile, localconfigfile):
+    localconfig = readconfig.read_config(localconfigfile)
+    config = readconfig.verify_and_read_config(configfile, localconfig["logadminkey"])
+
+    paths = localconfig["paths"]
+    mergenodes = config.get("mergenodes", [])
+    mergedb = paths["mergedb"]
+    sthfile = mergedb + "/sth"
+    currentsizefile = mergedb + "/fetched"
+
+    sth = json.loads(open(sthfile, "r").read())
+    currentsize = json.loads(open(currentsizefile, "r").read())
+    
+    print >>sys.stderr, currentsize["index"]+1,
+    
+    for mergenode in mergenodes:
+        if mergenode["name"] == config["primarymergenode"]:
+            continue
+        verifiedfile = mergedb + "/verified." + mergenode["name"]
+        try:
+            tree = json.loads(open(verifiedfile, "r").read())
+            print >>sys.stderr, tree["tree_size"],
+        except (IOError, ValueError):
+            pass
+    print >>sys.stderr, sth["tree_size"],
+    print
+
+
 def merge(expected=None, wait=0):
+    rv = subprocess.call([toolsdir + "/merge", "--config", testdir + "/catlfish-test.cfg",
+                          "--localconfig", testdir + "/catlfish-test-local-merge.cfg"])
+    if rv:
+        return rv
     for i in range(10):
-        rv = subprocess.call([toolsdir + "/merge", "--config", testdir + "/catlfish-test.cfg",
-                                "--localconfig", testdir + "/catlfish-test-local-merge.cfg"])
-        if rv:
-            return rv
+        mergestatus(testdir + "/catlfish-test.cfg", testdir + "/catlfish-test-local-merge.cfg")
         if i < wait:
             sleep(1)
             continue
         if correct_tree_size(expected):
             return 0
-    return 0
+        sleep(1)
+    return 1
 
 mergeresult = merge(expected=0, wait=3)
 assert_equal(mergeresult, 0, "merge", quiet=True, fatal=True)