summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--test/Makefile2
-rw-r--r--test/catlfish-test-local-frontend-1.cfg (renamed from test/catlfish-test-local-1.cfg)4
-rw-r--r--test/catlfish-test-local-frontend-2.cfg23
-rw-r--r--test/catlfish-test-local-frontend-3.cfg23
-rw-r--r--test/catlfish-test-local-merge-1.cfg (renamed from test/catlfish-test-local-merge.cfg)2
-rw-r--r--test/catlfish-test-local-merge-2.cfg14
-rw-r--r--test/catlfish-test-local-merge-3.cfg24
-rw-r--r--test/catlfish-test-local-signing-1.cfg (renamed from test/catlfish-test-local-signing.cfg)0
-rw-r--r--test/catlfish-test-mergefailover.cfg.in40
-rw-r--r--test/catlfish-test-redistribute-frontend.cfg.in45
-rw-r--r--test/catlfish-test.cfg.in15
-rwxr-xr-xtest/scripts/light-system-test-prepare-merge-takeover.sh11
-rwxr-xr-xtest/scripts/light-system-test-prepare-redistribute-frontend.sh16
-rwxr-xr-xtest/scripts/light-system-test-prepare.sh40
-rwxr-xr-xtest/scripts/light-system-test-run-1.sh12
-rwxr-xr-xtest/scripts/light-system-test-run-2.sh2
-rwxr-xr-xtest/scripts/light-system-test-run-5.sh7
-rwxr-xr-xtest/scripts/light-system-test-stop.sh1
-rwxr-xr-xtest/scripts/light-system-test.sh28
-rwxr-xr-xtest/scripts/perf-test.sh53
-rwxr-xr-xtest/scripts/reload-config.sh14
-rw-r--r--test/scripts/testutils.sh8
-rwxr-xr-xtools/compileconfig.py66
-rwxr-xr-xtools/merge_fetch.py23
-rwxr-xr-xtools/testcase1.py8
25 files changed, 340 insertions, 141 deletions
diff --git a/test/Makefile b/test/Makefile
index a033de6..1074348 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -11,7 +11,7 @@ tests-wait:
sleep 5
tests-makemk:
- $(PREFIX)/tools/compileconfig.py --config=$(PREFIX)/test/catlfish-test.cfg.in --testshellvars=$(PREFIX)/test/test.shvars --machines 1
+ $(PREFIX)/tools/compileconfig.py --config=$(PREFIX)/test/catlfish-test.cfg.in --testshellvars=$(PREFIX)/test/test.shvars
tests:
@make tests-makemk
diff --git a/test/catlfish-test-local-1.cfg b/test/catlfish-test-local-frontend-1.cfg
index 21f8c17..4867672 100644
--- a/test/catlfish-test-local-1.cfg
+++ b/test/catlfish-test-local-frontend-1.cfg
@@ -21,13 +21,13 @@ paths:
https_keyfile: httpscert/httpskey-1.pem
https_cacertfile: httpsca/demoCA/cacert.pem
public_cacertfile: httpsca/demoCA/cacert.pem
- db: machine/machine-1/db/
+ db: machine/frontend-1/db/
publickeys: publickeys
logpublickey: keys/logkey.pem
privatekeys: privatekeys
ratelimits:
- add_chain: 10 per second
+ add_chain: 1000 per second
logadminkey: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQ
Ah9sZ2CD+JeLbprS6AFcZbo0TGCH0rtEnr2Q3JW0ylhfA+
diff --git a/test/catlfish-test-local-frontend-2.cfg b/test/catlfish-test-local-frontend-2.cfg
new file mode 100644
index 0000000..2a6462a
--- /dev/null
+++ b/test/catlfish-test-local-frontend-2.cfg
@@ -0,0 +1,23 @@
+localnodes:
+ - frontend-2
+
+paths:
+ configdir: .
+ knownroots: known_roots
+ https_certfile: httpscert/httpscert-1.pem
+ https_keyfile: httpscert/httpskey-1.pem
+ https_cacertfile: httpsca/demoCA/cacert.pem
+ public_cacertfile: httpsca/demoCA/cacert.pem
+ db: machine/frontend-2/db/
+ publickeys: publickeys
+ logpublickey: keys/logkey.pem
+ privatekeys: privatekeys
+
+ratelimits:
+ add_chain: 1000 per second
+
+logadminkey: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQ
+ Ah9sZ2CD+JeLbprS6AFcZbo0TGCH0rtEnr2Q3JW0ylhfA+
+ 0/WLu755b3soVX/wI23vqCVGC7N9fOB2WUltveQ==
+
+configurl: file:///CURRENTWORKINGDIRECTORY/catlfish-test-generated.cfg
diff --git a/test/catlfish-test-local-frontend-3.cfg b/test/catlfish-test-local-frontend-3.cfg
new file mode 100644
index 0000000..430cef6
--- /dev/null
+++ b/test/catlfish-test-local-frontend-3.cfg
@@ -0,0 +1,23 @@
+localnodes:
+ - frontend-3
+
+paths:
+ configdir: .
+ knownroots: known_roots
+ https_certfile: httpscert/httpscert-1.pem
+ https_keyfile: httpscert/httpskey-1.pem
+ https_cacertfile: httpsca/demoCA/cacert.pem
+ public_cacertfile: httpsca/demoCA/cacert.pem
+ db: machine/frontend-3/db/
+ publickeys: publickeys
+ logpublickey: keys/logkey.pem
+ privatekeys: privatekeys
+
+ratelimits:
+ add_chain: 1000 per second
+
+logadminkey: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQ
+ Ah9sZ2CD+JeLbprS6AFcZbo0TGCH0rtEnr2Q3JW0ylhfA+
+ 0/WLu755b3soVX/wI23vqCVGC7N9fOB2WUltveQ==
+
+configurl: file:///CURRENTWORKINGDIRECTORY/catlfish-test-generated.cfg
diff --git a/test/catlfish-test-local-merge.cfg b/test/catlfish-test-local-merge-1.cfg
index 9283136..389a440 100644
--- a/test/catlfish-test-local-merge.cfg
+++ b/test/catlfish-test-local-merge-1.cfg
@@ -5,7 +5,7 @@ localnodes:
paths:
configdir: .
- mergedb: mergedb
+ mergedb: machine/merge-1/db
# TODO: https_certfile and https_keyfile not strictly necessary for primary merge
https_certfile: httpscert/httpscert-1.pem
https_keyfile: httpscert/httpskey-1.pem
diff --git a/test/catlfish-test-local-merge-2.cfg b/test/catlfish-test-local-merge-2.cfg
index e2baa01..d42cbf3 100644
--- a/test/catlfish-test-local-merge-2.cfg
+++ b/test/catlfish-test-local-merge-2.cfg
@@ -2,7 +2,7 @@ localnodes:
- merge-2
addresses:
- merge-2: 127.0.0.1:8181
+ merge-2: 127.0.0.1:8182
nodename: merge-2
@@ -11,16 +11,26 @@ nodename: merge-2
paths:
configdir: .
knownroots: known_roots
- mergedb: mergedb-secondary
+ mergedb: machine/merge-2/db
https_certfile: httpscert/httpscert-1.pem
https_keyfile: httpscert/httpskey-1.pem
https_cacertfile: httpsca/demoCA/cacert.pem
publickeys: publickeys
logpublickey: keys/logkey.pem
privatekeys: privatekeys
+ verifycert_bin: ../bin/verifycert.erl.escript
logadminkey: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQ
Ah9sZ2CD+JeLbprS6AFcZbo0TGCH0rtEnr2Q3JW0ylhfA+
0/WLu755b3soVX/wI23vqCVGC7N9fOB2WUltveQ==
+merge:
+ min-delay: 1
+ backup-window-size: 2
+ backup-sendlog-chunksize: 1
+ backup-sendentries-chunksize: 1
+ dist-window-size: 2
+ dist-sendlog-chunksize: 1
+ dist-sendentries-chunksize: 1
+
configurl: file:///CURRENTWORKINGDIRECTORY/catlfish-test-generated.cfg
diff --git a/test/catlfish-test-local-merge-3.cfg b/test/catlfish-test-local-merge-3.cfg
new file mode 100644
index 0000000..f6da89e
--- /dev/null
+++ b/test/catlfish-test-local-merge-3.cfg
@@ -0,0 +1,24 @@
+localnodes:
+ - merge-3
+
+addresses:
+ merge-3: 127.0.0.1:8183
+
+nodename: merge-3
+
+paths:
+ configdir: .
+ knownroots: known_roots
+ mergedb: machine/merge-3/db
+ https_certfile: httpscert/httpscert-1.pem
+ https_keyfile: httpscert/httpskey-1.pem
+ https_cacertfile: httpsca/demoCA/cacert.pem
+ publickeys: publickeys
+ logpublickey: keys/logkey.pem
+ privatekeys: privatekeys
+
+logadminkey: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQ
+ Ah9sZ2CD+JeLbprS6AFcZbo0TGCH0rtEnr2Q3JW0ylhfA+
+ 0/WLu755b3soVX/wI23vqCVGC7N9fOB2WUltveQ==
+
+configurl: file:///CURRENTWORKINGDIRECTORY/catlfish-test-generated.cfg
diff --git a/test/catlfish-test-local-signing.cfg b/test/catlfish-test-local-signing-1.cfg
index 0eccc94..0eccc94 100644
--- a/test/catlfish-test-local-signing.cfg
+++ b/test/catlfish-test-local-signing-1.cfg
diff --git a/test/catlfish-test-mergefailover.cfg.in b/test/catlfish-test-mergefailover.cfg.in
new file mode 100644
index 0000000..2e2cd1a
--- /dev/null
+++ b/test/catlfish-test-mergefailover.cfg.in
@@ -0,0 +1,40 @@
+baseurl: https://localhost:8080/
+
+frontendnodes:
+ - name: frontend-1
+ publicaddress: localhost:8080
+ address: localhost:8082
+ - name: frontend-2
+ publicaddress: localhost:8090
+ address: localhost:8092
+ - name: frontend-3
+ publicaddress: localhost:8000
+ address: localhost:8002
+
+storagenodes:
+ - name: frontend-1
+ address: localhost:8081
+ - name: frontend-2
+ address: localhost:8091
+ - name: frontend-3
+ address: localhost:8001
+
+signingnodes:
+ - name: signing-1
+ address: localhost:8088
+
+mergenodes:
+ - name: merge-1
+ address: localhost:8181
+ - name: merge-2
+ address: localhost:8182
+ - name: merge-3
+ address: localhost:8183
+
+primarymergenode: merge-2
+
+backup-quorum-size: 1
+
+storage-quorum-size: 1
+
+mmd: 86400
diff --git a/test/catlfish-test-redistribute-frontend.cfg.in b/test/catlfish-test-redistribute-frontend.cfg.in
new file mode 100644
index 0000000..725c7c5
--- /dev/null
+++ b/test/catlfish-test-redistribute-frontend.cfg.in
@@ -0,0 +1,45 @@
+baseurl: https://localhost:8000/
+
+frontendnodes:
+ - name: frontend-1
+ publicaddress: localhost:8080
+ address: localhost:8082
+ - name: frontend-2
+ publicaddress: localhost:8090
+ address: localhost:8092
+ - name: frontend-3
+ publicaddress: localhost:8000
+ address: localhost:8002
+
+storagenodes:
+ - name: frontend-1
+ address: localhost:8081
+ - name: frontend-2
+ address: localhost:8091
+ - name: frontend-3
+ address: localhost:8001
+
+signingnodes:
+ - name: signing-1
+ address: localhost:8088
+
+mergenodes:
+ - name: merge-1
+ address: localhost:8181
+ - name: merge-2
+ address: localhost:8182
+ - name: merge-3
+ address: localhost:8183
+
+primarymergenode: merge-2
+
+backup-quorum-size: 1
+
+storage-quorum-size: 1
+
+mmd: 86400
+
+statsservers:
+ - name: statsserver
+ address: localhost:9081
+ publicaddress: localhost:9082
diff --git a/test/catlfish-test.cfg.in b/test/catlfish-test.cfg.in
index 68c753f..c3b59b0 100644
--- a/test/catlfish-test.cfg.in
+++ b/test/catlfish-test.cfg.in
@@ -4,10 +4,20 @@ frontendnodes:
- name: frontend-1
publicaddress: localhost:8080
address: localhost:8082
+ - name: frontend-2
+ publicaddress: localhost:8090
+ address: localhost:8092
+ - name: frontend-3
+ publicaddress: localhost:8000
+ address: localhost:8002
storagenodes:
- name: frontend-1
address: localhost:8081
+ - name: frontend-2
+ address: localhost:8091
+ - name: frontend-3
+ address: localhost:8001
signingnodes:
- name: signing-1
@@ -15,8 +25,11 @@ signingnodes:
mergenodes:
- name: merge-1
- - name: merge-2
address: localhost:8181
+ - name: merge-2
+ address: localhost:8182
+ - name: merge-3
+ address: localhost:8183
primarymergenode: merge-1
diff --git a/test/scripts/light-system-test-prepare-merge-takeover.sh b/test/scripts/light-system-test-prepare-merge-takeover.sh
index 057af3d..f27b7f9 100755
--- a/test/scripts/light-system-test-prepare-merge-takeover.sh
+++ b/test/scripts/light-system-test-prepare-merge-takeover.sh
@@ -7,9 +7,8 @@ top_srcdir=$(cd $(dirname $0)/../..; pwd)
. ${top_srcdir}/test/scripts/testutils.sh
-mv mergedb mergedb-down
-mv mergedb-secondary mergedb
-mv mergedb/verifiedsize mergedb/verifiedsize.OFF
-mkdir mergedb-secondary
-touch mergedb-secondary/logorder
-printf 0 > mergedb-secondary/verifiedsize
+mv machine/merge-1/db/sth machine/merge-1/db/sth.OFF
+
+${top_srcdir}/tools/initlog.py --config machine/merge-2/catlfish-test.cfg --localconfig machine/merge-2/catlfish-test-local-merge-2.cfg --promote-secondary
+
+cp machine/merge-2/catlfish-test-local-merge-2.cfg current-merge-localconfig.cfg
diff --git a/test/scripts/light-system-test-prepare-redistribute-frontend.sh b/test/scripts/light-system-test-prepare-redistribute-frontend.sh
deleted file mode 100755
index c7b6f82..0000000
--- a/test/scripts/light-system-test-prepare-redistribute-frontend.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/sh
-
-set -o nounset
-set -o errexit
-
-top_srcdir=$(cd $(dirname $0)/../..; pwd)
-
-. ${top_srcdir}/test/scripts/testutils.sh
-
-mv machine/machine-1 machine/machine-1-down
-mkdir -p machine/machine-1/db
-touch machine/machine-1/db/index
-touch machine/machine-1/db/newentries
-cp machine/machine-1-down/catlfish-test-local-1.cfg machine/machine-1/
-cp machine/machine-1-down/catlfish-test.cfg machine/machine-1/
-cp machine/machine-1-down/catlfish-test.cfg.sig machine/machine-1/
diff --git a/test/scripts/light-system-test-prepare.sh b/test/scripts/light-system-test-prepare.sh
index 1f68ed1..9b500c9 100755
--- a/test/scripts/light-system-test-prepare.sh
+++ b/test/scripts/light-system-test-prepare.sh
@@ -41,39 +41,29 @@ createcert
mkdir keys
(cd keys ; ${top_srcdir}/tools/create-key.sh logkey)
openssl pkcs8 -topk8 -nocrypt -in keys/logkey-private.pem -out keys/logkey-private.pkcs8
-mkdir mergedb
-mkdir mergedb-secondary
-touch mergedb-secondary/logorder
-printf 0 > mergedb-secondary/verifiedsize
+
mkdir known_roots
cp ${top_srcdir}/tools/testcerts/roots/* known_roots
mkdir privatekeys
-for node in ${NODES}; do \
+for node in ${NODES}; do
(cd privatekeys ; ${top_srcdir}/tools/create-key.sh ${node})
mkdir -p nodes/${node}/log
+ mkdir -p machine/${node}/db
+ cp ${top_srcdir}/test/catlfish-test-local-${node}.cfg machine/${node}/
done
-${top_srcdir}/tools/genconfig.sh ${top_srcdir}/test/catlfish-test.cfg.in privatekeys keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test/logadminkey-private.pem catlfish-test-generated.cfg
-
-for machine in ${MACHINES}; do \
- mkdir -p machine/machine-${machine}/db
- cp ${top_srcdir}/test/catlfish-test-local-${machine}.cfg machine/machine-${machine}/
- ${top_srcdir}/tools/getconfig.py --localconfig machine/machine-${machine}/catlfish-test-local-${machine}.cfg --dest machine/machine-${machine}/catlfish-test.cfg
- ${top_srcdir}/tools/compileconfig.py --config machine/machine-${machine}/catlfish-test.cfg --localconfig machine/machine-${machine}/catlfish-test-local-${machine}.cfg
- touch machine/machine-${machine}/db/index && touch machine/machine-${machine}/db/newentries
+for node in ${MERGESECONDARYNODES}; do
+ nodename=$(echo ${node} | cut -f 1 -d ':')
+ printf 0 > machine/${nodename}/db/verifiedsize
done
-mkdir -p machine/merge
-cp ${top_srcdir}/test/catlfish-test-local-merge.cfg machine/merge/
-${top_srcdir}/tools/getconfig.py --localconfig machine/merge/catlfish-test-local-merge.cfg --dest machine/merge/catlfish-test.cfg
-${top_srcdir}/tools/compileconfig.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg
-mkdir -p machine/merge-2
-cp ${top_srcdir}/test/catlfish-test-local-merge-2.cfg machine/merge-2/
-${top_srcdir}/tools/getconfig.py --localconfig machine/merge-2/catlfish-test-local-merge-2.cfg --dest machine/merge-2/catlfish-test.cfg
-${top_srcdir}/tools/compileconfig.py --config machine/merge-2/catlfish-test.cfg --localconfig machine/merge-2/catlfish-test-local-merge-2.cfg
-mkdir -p machine/signing
-cp ${top_srcdir}/test/catlfish-test-local-signing.cfg machine/signing/
-${top_srcdir}/tools/getconfig.py --localconfig machine/signing/catlfish-test-local-signing.cfg --dest machine/signing/catlfish-test.cfg
-${top_srcdir}/tools/compileconfig.py --config machine/signing/catlfish-test.cfg --localconfig machine/signing/catlfish-test-local-signing.cfg
+
+
test -x ${SOFTHSM} && ${SOFTHSM} --init-token --slot=0 --label=mylabel --so-pin=ffff --pin=ffff || true
test -x ${SOFTHSM} && ${SOFTHSM} --import keys/logkey-private.pkcs8 --slot 0 --label mylabel --pin ffff --id 00 || true
+
+${top_srcdir}/tools/genconfig.sh ${top_srcdir}/test/catlfish-test.cfg.in privatekeys keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test/logadminkey-private.pem catlfish-test-generated.cfg
+
+${top_srcdir}/test/scripts/reload-config.sh
+
+cp machine/merge-1/catlfish-test-local-merge-1.cfg current-merge-localconfig.cfg
diff --git a/test/scripts/light-system-test-run-1.sh b/test/scripts/light-system-test-run-1.sh
index 7d52ed7..0695497 100755
--- a/test/scripts/light-system-test-run-1.sh
+++ b/test/scripts/light-system-test-run-1.sh
@@ -7,14 +7,14 @@ top_srcdir=$(cd $(dirname $0)/../..; pwd)
. ${top_srcdir}/test/scripts/testutils.sh
-python ${top_srcdir}/tools/testcase1.py https://localhost:8080/ keys/logkey.pem httpsca/demoCA/cacert.pem machine/merge || fail "Tests failed"
+python ${top_srcdir}/tools/testcase1.py https://localhost:8080/ keys/logkey.pem httpsca/demoCA/cacert.pem machine/merge-1 || fail "Tests failed"
check_sth
python ${top_srcdir}/tools/fetchallcerts.py ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Verification failed"
-python ${top_srcdir}/tools/storagegc.py --config machine/machine-1/catlfish-test.cfg --localconfig machine/machine-1/catlfish-test-local-1.cfg || fail "GC failed"
+python ${top_srcdir}/tools/storagegc.py --config machine/frontend-1/catlfish-test.cfg --localconfig machine/frontend-1/catlfish-test-local-frontend-1.cfg || fail "GC failed"
for certfile in ${top_srcdir}/tools/testcerts/cert[1-5].txt ${top_srcdir}/tools/testcerts/pre[12].txt; do
python ${top_srcdir}/tools/submitcert.py --parallel=1 --store $certfile --check-sct --sct-file=submittedcerts ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Submission failed"
done
-python ${top_srcdir}/tools/storagegc.py --config machine/machine-1/catlfish-test.cfg --localconfig machine/machine-1/catlfish-test-local-1.cfg || fail "GC failed"
+python ${top_srcdir}/tools/storagegc.py --config machine/frontend-1/catlfish-test.cfg --localconfig machine/frontend-1/catlfish-test-local-frontend-1.cfg || fail "GC failed"
do_merge 7
check_sth
@@ -30,9 +30,9 @@ python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert3.tx
python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert4.txt fetchcertstore/00000003 || fail "Verification failed"
python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/cert5.txt fetchcertstore/00000004 || fail "Verification failed"
python ${top_srcdir}/tools/comparecert.py ${top_srcdir}/tools/testcerts/pre1.txt:${top_srcdir}/tools/testcerts/pre2.txt fetchcertstore/00000005:fetchcertstore/00000006 || fail"Verification failed"
-python ${top_srcdir}/tools/storagegc.py --config machine/machine-1/catlfish-test.cfg --localconfig machine/machine-1/catlfish-test-local-1.cfg || fail "GC failed"
+python ${top_srcdir}/tools/storagegc.py --config machine/frontend-1/catlfish-test.cfg --localconfig machine/frontend-1/catlfish-test-local-frontend-1.cfg || fail "GC failed"
-${top_srcdir}/test/scripts/light-system-test-stop.sh mergesecondary
+${top_srcdir}/test/scripts/light-system-test-stop.sh merge_2only
python ${top_srcdir}/tools/submitcert.py --parallel=1 --store ${top_srcdir}/tools/testcerts/cert6.txt --check-sct --sct-file=submittedcerts ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Submission failed"
echo NOTE: merge backup should fail with 111 Connection refused
@@ -45,7 +45,7 @@ assert_equal "Tree size" "$(get_treesize)" 7
check_sth
-${top_srcdir}/test/scripts/light-system-test-start.sh mergesecondary
+${top_srcdir}/test/scripts/light-system-test-start.sh merge_2only
do_merge 8
diff --git a/test/scripts/light-system-test-run-2.sh b/test/scripts/light-system-test-run-2.sh
index bf80c97..88df7c1 100755
--- a/test/scripts/light-system-test-run-2.sh
+++ b/test/scripts/light-system-test-run-2.sh
@@ -11,4 +11,4 @@ python ${top_srcdir}/tools/verifysct.py --sct-file=submittedcerts --parallel 1 $
check_sth
-python ${top_srcdir}/tools/storagegc.py --config machine/machine-1/catlfish-test.cfg --localconfig machine/machine-1/catlfish-test-local-1.cfg || fail "GC failed"
+python ${top_srcdir}/tools/storagegc.py --config machine/frontend-1/catlfish-test.cfg --localconfig machine/frontend-1/catlfish-test-local-frontend-1.cfg || fail "GC failed"
diff --git a/test/scripts/light-system-test-run-5.sh b/test/scripts/light-system-test-run-5.sh
index cf0bfaf..73e8f5f 100755
--- a/test/scripts/light-system-test-run-5.sh
+++ b/test/scripts/light-system-test-run-5.sh
@@ -7,12 +7,7 @@ top_srcdir=$(cd $(dirname $0)/../..; pwd)
. ${top_srcdir}/test/scripts/testutils.sh
-${top_srcdir}/tools/genconfig.sh ${top_srcdir}/test/catlfish-test.cfg.in privatekeys keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test/logadminkey-private.pem catlfish-test-generated.cfg
-
-for machine in ${MACHINES}; do \
- ${top_srcdir}/tools/getconfig.sh machine/machine-${machine}/catlfish-test-local-${machine}.cfg machine/machine-${machine}/catlfish-test.cfg frontend-${machine}.plopcontrol
-done
-
+${top_srcdir}/test/scripts/reload-config.sh
do_merge 9
sleep 3
diff --git a/test/scripts/light-system-test-stop.sh b/test/scripts/light-system-test-stop.sh
index a946079..aa379cd 100755
--- a/test/scripts/light-system-test-stop.sh
+++ b/test/scripts/light-system-test-stop.sh
@@ -12,6 +12,7 @@ nodetypes="$@"
stop_node() {
node=$1
${top_srcdir}/tools/to_catlfish.py to_erl nodes/${node}/ "init:stop()"
+ rm ${node}.plopcontrol || true
}
nodes=
diff --git a/test/scripts/light-system-test.sh b/test/scripts/light-system-test.sh
index 217fb2c..69eb5c7 100755
--- a/test/scripts/light-system-test.sh
+++ b/test/scripts/light-system-test.sh
@@ -18,16 +18,18 @@ tests_stop() {
}
tests_stop_all() {
- ${SCRIPTS}/light-system-test-stop.sh mergeprimary mergesecondary frontend storage signing
+ ${SCRIPTS}/light-system-test-stop.sh mergeprimary mergesecondary frontend signing
}
${SCRIPTS}/light-system-test-prepare.sh
-tests_start signing mergesecondary storage frontend
-${top_srcdir}/tools/initlog.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg
+tests_start signing
+tests_start merge_2only frontendexceptlast
+${top_srcdir}/tools/initlog.py --config machine/merge-1/catlfish-test.cfg --localconfig machine/merge-1/catlfish-test-local-merge-1.cfg
tests_start mergeprimary
${SCRIPTS}/light-system-test-run-1.sh || (echo "Tests failed"; sleep 5; tests_stop_all; false)
+
sleep 5
-tests_stop_all
+tests_stop mergeprimary merge_2only frontendexceptlast
sleep 5
echo
echo
@@ -39,10 +41,10 @@ echo
echo
echo
echo
-tests_start signing mergesecondary storage frontend mergeprimary
+tests_start merge_2only frontendexceptlast mergeprimary
${SCRIPTS}/light-system-test-run-2.sh || (echo "Tests failed"; sleep 5; tests_stop_all; false)
sleep 5
-tests_stop_all
+tests_stop mergeprimary merge_2only
sleep 5
echo
echo
@@ -54,13 +56,13 @@ echo
echo
echo
echo
+${top_srcdir}/tools/genconfig.sh ${top_srcdir}/test/catlfish-test-mergefailover.cfg.in privatekeys keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test/logadminkey-private.pem catlfish-test-generated.cfg
+${top_srcdir}/test/scripts/reload-config.sh
${SCRIPTS}/light-system-test-prepare-merge-takeover.sh
-tests_start signing mergesecondary storage frontend
-${top_srcdir}/tools/initlog.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg --promote-secondary
-tests_start mergeprimary
+tests_start merge_3only mergeprimary
${SCRIPTS}/light-system-test-run-3.sh || (echo "Tests failed"; sleep 5; tests_stop_all; false)
sleep 5
-tests_stop_all
+tests_stop mergeprimary merge_3only frontendexceptlast
sleep 5
echo
echo
@@ -72,8 +74,9 @@ echo
echo
echo
echo
-${SCRIPTS}/light-system-test-prepare-redistribute-frontend.sh
-tests_start signing mergesecondary storage frontend mergeprimary
+${top_srcdir}/tools/genconfig.sh ${top_srcdir}/test/catlfish-test-redistribute-frontend.cfg.in privatekeys keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test/logadminkey-private.pem catlfish-test-generated.cfg
+${top_srcdir}/test/scripts/reload-config.sh
+tests_start merge_3only frontendexceptfirst mergeprimary
${SCRIPTS}/light-system-test-run-4.sh || (echo "Tests failed"; sleep 5; tests_stop_all; false)
echo
@@ -87,6 +90,7 @@ echo
echo
echo
+${top_srcdir}/tools/genconfig.sh ${top_srcdir}/test/catlfish-test-redistribute-frontend.cfg.in privatekeys keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test/logadminkey-private.pem catlfish-test-generated.cfg
${SCRIPTS}/light-system-test-run-5.sh || (echo "Tests failed"; sleep 5; tests_stop_all; false)
sleep 5
diff --git a/test/scripts/perf-test.sh b/test/scripts/perf-test.sh
index 3766cf1..085de14 100755
--- a/test/scripts/perf-test.sh
+++ b/test/scripts/perf-test.sh
@@ -18,66 +18,73 @@ tests_stop() {
}
tests_stop_all() {
- ${SCRIPTS}/light-system-test-stop.sh mergeprimary mergesecondary frontend storage signing
+ ${SCRIPTS}/light-system-test-stop.sh mergeprimary mergesecondary frontend signing
}
${SCRIPTS}/light-system-test-prepare.sh
-${top_srcdir}/tools/initlog.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg
+${top_srcdir}/tools/initlog.py --config machine/merge-1/catlfish-test.cfg --localconfig machine/merge-1/catlfish-test-local-merge-1.cfg
cp ${top_srcdir}/test/known_roots/* known_roots
tests_start signing
-tests_start mergesecondary storage frontend mergeprimary
+tests_start merge_2only frontendexceptlast mergeprimary
do_merge 0
check_sth
assert_equal "Tree size" "$(get_treesize)" 0
-python ${top_srcdir}/tools/submitcert.py --parallel=30 --store ${top_srcdir}/test/bulktestcerts/0000.zip --sct-file=submittedcerts ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || (tests_stop ; fail "Submission failed")
+python ${top_srcdir}/tools/submitcert.py --parallel=30 --store ${top_srcdir}/test/bulktestcerts/0000.zip --sct-file=submittedcerts ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || (tests_stop ; fail "Submission failed") &
do_merge 10000 || (tests_stop ; fail "Merge failed")
check_sth || (tests_stop ; fail "Check STH failed")
-wait_for_equal "Tree size" "${top_srcdir}/tools/mergestatus.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg" "10000 10000 10000" 400
+python ${top_srcdir}/tools/storagegc.py --config machine/machine-frontend-1/catlfish-test.cfg --localconfig machine/machine-frontend-1/catlfish-test-local-frontend-1.cfg || fail "GC failed" &
+python ${top_srcdir}/tools/storagegc.py --config machine/machine-frontend-1/catlfish-test.cfg --localconfig machine/machine-frontend-2/catlfish-test-local-frontend-2.cfg || fail "GC failed" &
+
+wait_for_equal "Tree size" "${top_srcdir}/tools/mergestatus.py --config machine/merge-1/catlfish-test.cfg --localconfig current-merge-localconfig.cfg" "10000 10000 10000" 400
+
sleep 5
-tests_stop mergesecondary storage frontend mergeprimary
+tests_stop merge_2only mergeprimary
sleep 5
-mv mergedb mergedb-down
-mv mergedb-secondary mergedb
-mkdir mergedb-secondary
-touch mergedb-secondary/logorder
-printf 0 > mergedb-secondary/verifiedsize
+${top_srcdir}/tools/genconfig.sh ${top_srcdir}/test/catlfish-test-mergefailover.cfg.in privatekeys keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test/logadminkey-private.pem catlfish-test-generated.cfg
+${top_srcdir}/test/scripts/reload-config.sh
+${SCRIPTS}/light-system-test-prepare-merge-takeover.sh
-${top_srcdir}/tools/initlog.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg --promote-secondary
+tests_start mergeprimary merge_3only
-tests_start mergesecondary storage frontend mergeprimary
+python ${top_srcdir}/tools/submitcert.py --parallel=1 --store ${top_srcdir}/tools/testcerts/cert3.txt --check-sct --sct-file=submittedcerts ${BASEURL} --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Submission failed"
-do_merge 10000 || (tests_stop ; fail "Merge failed")
+do_merge 10001 || (tests_stop ; fail "Merge failed")
check_sth || (tests_stop ; fail "Check STH failed")
-wait_for_equal "Tree size" "${top_srcdir}/tools/mergestatus.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg" "10000 10000 10000" 400
+
+wait_for_equal "Tree size" "${top_srcdir}/tools/mergestatus.py --config machine/merge-1/catlfish-test.cfg --localconfig current-merge-localconfig.cfg" "10001 10001 10001" 400
+
sleep 5
-tests_stop mergesecondary storage frontend mergeprimary
+tests_stop mergesecondary frontendexceptlast mergeprimary
sleep 5
+${top_srcdir}/tools/genconfig.sh ${top_srcdir}/test/catlfish-test-redistribute-frontend.cfg.in privatekeys keys/logkey.pem httpsca/demoCA/cacert.pem ${top_srcdir}/test/logadminkey-private.pem catlfish-test-generated.cfg
+${top_srcdir}/test/scripts/reload-config.sh
-mv machine/machine-1 machine/machine-1-down
-mkdir -p machine/machine-1/db
-touch machine/machine-1/db/index
-touch machine/machine-1/db/newentries
+. ${top_srcdir}/test/test.shvars
-tests_start mergesecondary storage frontend mergeprimary
+tests_start mergesecondary frontendexceptfirst mergeprimary
-do_merge 10000 || (tests_stop ; fail "Merge failed")
-wait_for_equal "Tree size" "${top_srcdir}/tools/mergestatus.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg" "10000 10000 10000" 400
+python ${top_srcdir}/tools/submitcert.py --parallel=1 --store ${top_srcdir}/tools/testcerts/cert2.txt --check-sct --sct-file=submittedcerts https://localhost:8090/ --publickey=keys/logkey.pem --cafile httpsca/demoCA/cacert.pem || fail "Submission failed"
+
+do_merge 10002 || (tests_stop ; fail "Merge failed")
+wait_for_equal "Tree size" "${top_srcdir}/tools/mergestatus.py --config machine/merge-1/catlfish-test.cfg --localconfig current-merge-localconfig.cfg" "10002 10002 10002" 800
check_sth || (tests_stop ; fail "Check STH failed")
sleep 5
tests_stop_all
sleep 5
+echo Success
+
#grep timing: bench-[123] > bench.txt
#${top_srcdir}/tools/parsebench.py bench.txt > bench.html
diff --git a/test/scripts/reload-config.sh b/test/scripts/reload-config.sh
new file mode 100755
index 0000000..7dbeebe
--- /dev/null
+++ b/test/scripts/reload-config.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+set -o nounset
+set -o errexit
+
+top_srcdir=$(cd $(dirname $0)/../..; pwd)
+
+. ${top_srcdir}/test/scripts/testutils.sh
+
+${top_srcdir}/tools/compileconfig.py --config=catlfish-test-generated.cfg --testshellvars=${top_srcdir}/test/test.shvars
+
+for node in ${NODES}; do
+ ${top_srcdir}/tools/getconfig.sh machine/${node}/catlfish-test-local-${node}.cfg machine/${node}/catlfish-test.cfg ${node}.plopcontrol
+done
diff --git a/test/scripts/testutils.sh b/test/scripts/testutils.sh
index 88e9a54..4d49be8 100644
--- a/test/scripts/testutils.sh
+++ b/test/scripts/testutils.sh
@@ -25,7 +25,7 @@ get_treesize() {
get_treesize_host ${BASEURL}
}
get_treesize_host() {
- ${top_srcdir}/tools/loginfo.py --localconfig machine/merge/catlfish-test-local-merge.cfg --treesize "$1"
+ ${top_srcdir}/tools/loginfo.py --localconfig current-merge-localconfig.cfg --treesize "$1"
}
check_sth() {
@@ -36,11 +36,11 @@ check_sth_host() {
}
do_merge() {
- n=15
+ n=3000
expected_treesize=$1
while [ $n -gt 0 ]; do
- ${top_srcdir}/tools/merge --config machine/merge/catlfish-test.cfg --timing --localconfig machine/merge/catlfish-test-local-merge.cfg || fail "Merge failed"
- ${top_srcdir}/tools/mergestatus.py --config machine/merge/catlfish-test.cfg --localconfig machine/merge/catlfish-test-local-merge.cfg
+ ${top_srcdir}/tools/merge --config machine/merge-1/catlfish-test.cfg --timing --localconfig current-merge-localconfig.cfg || fail "Merge failed"
+ ${top_srcdir}/tools/mergestatus.py --config machine/merge-1/catlfish-test.cfg --localconfig current-merge-localconfig.cfg
n=$((n-1))
treesize=$(get_treesize)
[ "${treesize}" = "${expected_treesize}" ] && return
diff --git a/tools/compileconfig.py b/tools/compileconfig.py
index 6ecfcad..d5a22df 100755
--- a/tools/compileconfig.py
+++ b/tools/compileconfig.py
@@ -434,7 +434,27 @@ def gen_config(nodename, config, localconfig):
plopconfigfile.close()
-def gen_testmakefile(config, testmakefile, machines, shellvars=False):
+def print_nodevar(configfile, delimiter, nodetype, bootname, names, testaddresses, gennodes=True, gentesturl=True):
+ if gennodes:
+ print >>configfile, nodetype + "NODES=" + delimiter + " ".join(['%s:%s' % (name, bootname) for name in names]) + delimiter
+ for name in names:
+ print >>configfile, name.upper().replace("-", "_") + "ONLY" + "NODES=" + delimiter + '%s:%s' % (name, bootname) + delimiter
+ if gentesturl:
+ print >>configfile, name.upper().replace("-", "_") + "ONLY" + "TESTURLS=" + delimiter + " ".join(testaddresses[name]) + delimiter
+ else:
+ print >>configfile, nodetype + "NODES=" + delimiter + delimiter
+ if gentesturl:
+ print >>configfile, nodetype + "TESTURLS=" + delimiter + " ".join([address for name in names for address in testaddresses[name]]) + delimiter
+ else:
+ print >>configfile, nodetype + "TESTURLS=" + delimiter + delimiter
+
+def multivaldict(dictlist):
+ d = {}
+ for (key, value) in dictlist:
+ d.setdefault(key, []).append(value)
+ return d
+
+def gen_testmakefile(config, testmakefile, shellvars=False):
configfile = open(testmakefile, "w")
print >>configfile, "#", testmakefile, "generated by", sys.argv[0], datetime.now()
@@ -444,27 +464,26 @@ def gen_testmakefile(config, testmakefile, machines, shellvars=False):
mergenodenames = set([node["name"] for node in config["mergenodes"]])
mergesecondarynodenames = set([node["name"] for node in config["mergenodes"] if node["name"] != config["primarymergenode"]])
- frontendnodeaddresses = [node["publicaddress"] for node in config["frontendnodes"]]
- storagenodeaddresses = [node["address"] for node in config["storagenodes"]]
- signingnodeaddresses = [node["address"] for node in config["signingnodes"]]
- mergesecondarynodeaddresses = [node["address"] for node in config["mergenodes"] if node["name"] != config["primarymergenode"]]
+ frontendnodenames_except_last = sorted(frontendnodenames)[:-1]
+ frontendnodenames_except_first = sorted(frontendnodenames)[1:]
+
+ allnodes = config["frontendnodes"] + config["storagenodes"] + config["signingnodes"] + config["mergenodes"]
+ testaddresses = multivaldict([(node["name"], node["address"]) for node in allnodes])
+
delimiter = '"' if shellvars else ''
+
+ print_nodevar(configfile, delimiter, "FRONTEND", "catlfish", frontendnodenames, testaddresses)
+ print_nodevar(configfile, delimiter, "FRONTENDEXCEPTLAST", "catlfish", frontendnodenames_except_last, testaddresses)
+ print_nodevar(configfile, delimiter, "FRONTENDEXCEPTFIRST", "catlfish", frontendnodenames_except_first, testaddresses)
+ print_nodevar(configfile, delimiter, "STORAGE", "catlfish", storagenodenames, testaddresses, gennodes=False)
+ print_nodevar(configfile, delimiter, "SIGNING", "catlfish", signingnodenames, testaddresses)
+ print_nodevar(configfile, delimiter, "MERGESECONDARY", "catlfish", mergesecondarynodenames, testaddresses)
+ print_nodevar(configfile, delimiter, "MERGEPRIMARY", "merge", [config["primarymergenode"]], testaddresses, gentesturl=False)
- print >>configfile, "NODES=" + delimiter + " ".join(frontendnodenames|storagenodenames|signingnodenames|mergenodenames) + delimiter
-
- print >>configfile, "FRONTENDNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in frontendnodenames]) + delimiter
- print >>configfile, "STORAGENODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in storagenodenames if name not in frontendnodenames]) + delimiter
- print >>configfile, "SIGNINGNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in signingnodenames]) + delimiter
- print >>configfile, "MERGESECONDARYNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in mergesecondarynodenames]) + delimiter
- print >>configfile, "MERGEPRIMARYNODES=" + delimiter + '%s:%s' % (config["primarymergenode"], 'merge') + delimiter
-
- print >>configfile, "MACHINES=" + delimiter + " ".join([str(e) for e in range(1, machines+1)]) + delimiter
- print >>configfile, "FRONTENDTESTURLS=" + delimiter + " ".join(frontendnodeaddresses) + delimiter
- print >>configfile, "STORAGETESTURLS=" + delimiter + " ".join(storagenodeaddresses) + delimiter
- print >>configfile, "SIGNINGTESTURLS=" + delimiter + " ".join(signingnodeaddresses) + delimiter
- print >>configfile, "MERGESECONDARYTESTURLS=" + delimiter + " ".join(mergesecondarynodeaddresses) + delimiter
- print >>configfile, "MERGEPRIMARYTESTURLS=" + delimiter + " ".join([]) + delimiter
+ print >>configfile, "NODES=" + delimiter + " ".join(set([node["name"] for node in allnodes])) + delimiter
+
+ print >>configfile, "MACHINES=" + delimiter + " ".join([str(e) for e in frontendnodenames]) + delimiter
print >>configfile, "BASEURL=" + delimiter + config["baseurl"] + delimiter
configfile.close()
@@ -484,15 +503,14 @@ def main():
parser.add_argument("--testmakefile", metavar="file", help="Generate makefile variables for test")
parser.add_argument("--testshellvars", metavar="file", help="Generate shell variable file for test")
parser.add_argument("--getnodenames", action='store_true', help="Get list of node names")
- parser.add_argument("--machines", type=int, metavar="n", help="Number of machines")
args = parser.parse_args()
- if args.testmakefile and args.machines:
+ if args.testmakefile:
config = readconfig.read_config(args.config)
- gen_testmakefile(config, args.testmakefile, args.machines)
- elif args.testshellvars and args.machines:
+ gen_testmakefile(config, args.testmakefile)
+ elif args.testshellvars:
config = readconfig.read_config(args.config)
- gen_testmakefile(config, args.testshellvars, args.machines, shellvars=True)
+ gen_testmakefile(config, args.testshellvars, shellvars=True)
elif args.getnodenames:
config = readconfig.read_config(args.config)
printnodenames(config)
diff --git a/tools/merge_fetch.py b/tools/merge_fetch.py
index 42a3089..e71d3f1 100755
--- a/tools/merge_fetch.py
+++ b/tools/merge_fetch.py
@@ -38,14 +38,17 @@ def merge_fetch(args, config, localconfig):
entries_to_fetch = {}
for storagenode in storagenodes:
- print >>sys.stderr, "getting new entries from", storagenode["name"]
- sys.stderr.flush()
- new_entries_per_node[storagenode["name"]] = \
- set(get_new_entries(storagenode["name"],
- "https://%s/" % storagenode["address"],
- own_key, paths))
- new_entries.update(new_entries_per_node[storagenode["name"]])
- entries_to_fetch[storagenode["name"]] = []
+ try:
+ print >>sys.stderr, "getting new entries from", storagenode["name"]
+ sys.stderr.flush()
+ new_entries_per_node[storagenode["name"]] = \
+ set(get_new_entries(storagenode["name"],
+ "https://%s/" % storagenode["address"],
+ own_key, paths))
+ new_entries.update(new_entries_per_node[storagenode["name"]])
+ entries_to_fetch[storagenode["name"]] = []
+ except requests.exceptions.ConnectionError:
+ pass
timing_point(timing, "get new entries")
new_entries -= certsinlog
@@ -54,6 +57,8 @@ def merge_fetch(args, config, localconfig):
for ehash in new_entries:
for storagenode in storagenodes:
+ if storagenode["name"] not in new_entries_per_node:
+ continue
if ehash in new_entries_per_node[storagenode["name"]]:
entries_to_fetch[storagenode["name"]].append(ehash)
break
@@ -64,6 +69,8 @@ def merge_fetch(args, config, localconfig):
added_entries = 0
for storagenode in storagenodes:
+ if storagenode["name"] not in entries_to_fetch:
+ continue
print >>sys.stderr, "getting %d entries from %s:" % \
(len(entries_to_fetch[storagenode["name"]]), storagenode["name"]),
sys.stderr.flush()
diff --git a/tools/testcase1.py b/tools/testcase1.py
index fb081e5..a616841 100755
--- a/tools/testcase1.py
+++ b/tools/testcase1.py
@@ -192,11 +192,11 @@ def mergestatus(configfile, localconfigfile):
def merge(expected=None, wait=0):
rv = subprocess.call([toolsdir + "/merge", "--config", testdir + "/catlfish-test.cfg",
- "--localconfig", testdir + "/catlfish-test-local-merge.cfg"])
+ "--localconfig", testdir + "/catlfish-test-local-merge-1.cfg"])
if rv:
return rv
for i in range(10):
- mergestatus(testdir + "/catlfish-test.cfg", testdir + "/catlfish-test-local-merge.cfg")
+ mergestatus(testdir + "/catlfish-test.cfg", testdir + "/catlfish-test-local-merge-1.cfg")
if i < wait:
sleep(1)
continue
@@ -214,6 +214,7 @@ for baseurl in baseurls:
testgroup("cert1")
result1 = do_add_chain(cc1, baseurls[0])
+result2 = do_add_chain(cc1, baseurls[0])
mergeresult = merge(expected=1)
assert_equal(mergeresult, 0, "merge", quiet=True, fatal=True)
@@ -224,9 +225,10 @@ for baseurl in baseurls:
print_and_check_tree_size(1, baseurl)
size_sth[1] = base64.b64decode(get_sth(baseurls[0])["sha256_root_hash"])
-result2 = do_add_chain(cc1, baseurls[0])
+result3 = do_add_chain(cc1, baseurls[0])
assert_equal(result2["timestamp"], result1["timestamp"], "timestamp")
+assert_equal(result3["timestamp"], result1["timestamp"], "timestamp")
mergeresult = merge(expected=1, wait=3)
assert_equal(mergeresult, 0, "merge", quiet=True, fatal=True)