From b7b8903b6a7c3342933b9984afa72fb6527b5f72 Mon Sep 17 00:00:00 2001 From: Linus Nordberg Date: Tue, 24 Jan 2017 09:16:28 +0100 Subject: Parallelised merge, distribution phase. --- reltool.config | 6 +- src/catlfish_sup.erl | 2 +- test/catlfish-test-local-merge.cfg | 15 ++++- test/catlfish-test.cfg.in | 1 - .../light-system-test-prepare-merge-takeover.sh | 1 + test/scripts/light-system-test-prepare.sh | 1 + test/scripts/light-system-test-start.sh | 9 ++- test/scripts/light-system-test-stop.sh | 7 +- test/scripts/light-system-test.sh | 3 +- test/scripts/testutils.sh | 27 +++++++- tools/compileconfig.py | 75 +++++++++++++++------- tools/initlog.py | 71 ++++++++++++++++++++ tools/merge | 2 +- tools/merge_fetch.py | 2 +- tools/testcase1.py | 5 +- 15 files changed, 190 insertions(+), 37 deletions(-) create mode 100755 tools/initlog.py diff --git a/reltool.config b/reltool.config index b00f48b..5f837d1 100644 --- a/reltool.config +++ b/reltool.config @@ -3,13 +3,15 @@ {erts, [{mod_cond, derived}, {app_file, strip}]}, {app_file, strip}, {rel, "catlfish", "0.10.0-dev", [kernel, stdlib, sasl, catlfish]}, + {rel, "merge", "0.10.0-dev", [sasl, merge]}, {boot_rel, "catlfish"}, {profile, standalone}, {incl_sys_filters, ["^bin/", "^erts-.*/", "^lib"]}, - {incl_app_filters, ["^ebin/", "^priv/", "^src/"]}, - {app, catlfish, [{app_file, all}, {lib_dir, "."}]}, + {incl_app_filters, ["ebin/", "priv/", "src/"]}, {escript, "./verifycert.erl", [{incl_cond, include}]}, + {app, catlfish, [{app_file, all}, {lib_dir, "."}]}, {app, plop, [{app_file, all}, {lib_dir, "../plop"}]}, + {app, merge, [{app_file, all}, {lib_dir, "../plop/merge"}]}, {app, mochiweb, [{app_file, all}, {lib_dir, "../mochiweb"}]}, {app, idna, [{app_file, all}, {lib_dir, "../hackney/deps/idna"}]}, {app, hackney, [{app_file, all}, {lib_dir, "../hackney"}]}, diff --git a/src/catlfish_sup.erl b/src/catlfish_sup.erl index 8b43faf..f0fcee2 100644 --- a/src/catlfish_sup.erl +++ b/src/catlfish_sup.erl @@ -37,7 +37,7 @@ init([]) -> SSLOptions = [{certfile, application:get_env(catlfish, https_certfile, none)}, {keyfile, application:get_env(catlfish, https_keyfile, none)}, - {cacertfile, application:get_env(catlfish, https_cacertfile, none)}], + {cacertfile, application:get_env(plop, https_cacertfile, none)}], Servers = lists:map(fun (Config) -> gen_http_config(Config, SSLOptions, true) diff --git a/test/catlfish-test-local-merge.cfg b/test/catlfish-test-local-merge.cfg index 3b4d45f..bf1d3c1 100644 --- a/test/catlfish-test-local-merge.cfg +++ b/test/catlfish-test-local-merge.cfg @@ -1,14 +1,27 @@ nodename: merge-1 +localnodes: + - merge-1 + paths: + configdir: . mergedb: mergedb + # TODO: https_certfile and https_keyfile not strictly necessary for primary merge + https_certfile: httpscert/httpscert-1.pem + https_keyfile: httpscert/httpskey-1.pem https_cacertfile: httpsca/demoCA/cacert.pem publickeys: publickeys logpublickey: keys/logkey.pem privatekeys: privatekeys verifycert_bin: ../bin/verifycert.erl.escript - known_roots: known_roots/ + knownroots: known_roots/ logadminkey: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQ Ah9sZ2CD+JeLbprS6AFcZbo0TGCH0rtEnr2Q3JW0ylhfA+ 0/WLu755b3soVX/wI23vqCVGC7N9fOB2WUltveQ== + +merge: + min-delay: 1 + dist-window-size: 2 + dist-sendlog-chunksize: 1 + dist-sendentries-chunksize: 1 diff --git a/test/catlfish-test.cfg.in b/test/catlfish-test.cfg.in index 39288c7..ed3494b 100644 --- a/test/catlfish-test.cfg.in +++ b/test/catlfish-test.cfg.in @@ -15,7 +15,6 @@ signingnodes: mergenodes: - name: merge-1 - address: localhost:8180 - name: merge-2 address: localhost:8181 diff --git a/test/scripts/light-system-test-prepare-merge-takeover.sh b/test/scripts/light-system-test-prepare-merge-takeover.sh index c84a17e..085a6a2 100755 --- a/test/scripts/light-system-test-prepare-merge-takeover.sh +++ b/test/scripts/light-system-test-prepare-merge-takeover.sh @@ -9,6 +9,7 @@ top_srcdir=$(cd $(dirname $0)/../..; pwd) mv mergedb mergedb-down mv mergedb-secondary mergedb +init_log mkdir mergedb-secondary touch mergedb-secondary/logorder printf 0 > mergedb-secondary/verifiedsize diff --git a/test/scripts/light-system-test-prepare.sh b/test/scripts/light-system-test-prepare.sh index 84b06cd..536c0db 100755 --- a/test/scripts/light-system-test-prepare.sh +++ b/test/scripts/light-system-test-prepare.sh @@ -71,6 +71,7 @@ for machine in ${MACHINES}; do \ mkdir -p machine/machine-${machine}/db touch machine/machine-${machine}/db/index && touch machine/machine-${machine}/db/newentries done +${top_srcdir}/tools/compileconfig.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg ${top_srcdir}/tools/compileconfig.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge-2.cfg ${top_srcdir}/tools/compileconfig.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-signing.cfg test -x ${SOFTHSM} && ${SOFTHSM} --init-token --slot=0 --label=mylabel --so-pin=ffff --pin=ffff || true diff --git a/test/scripts/light-system-test-start.sh b/test/scripts/light-system-test-start.sh index 1623d7e..4520945 100755 --- a/test/scripts/light-system-test-start.sh +++ b/test/scripts/light-system-test-start.sh @@ -7,8 +7,15 @@ top_srcdir=$(cd $(dirname $0)/../..; pwd) . ${top_srcdir}/test/scripts/testutils.sh +start_node() { + node=$1 + node_app=$2 + ../bin/run_erl -daemon nodes/${node}/ nodes/${node}/log/ \ + "exec ../bin/erl -boot ${node_app} -config ${node}" +} + for node in ${ERLANGNODES}; do - (../bin/run_erl -daemon nodes/${node}/ nodes/${node}/log/ "exec ../bin/erl -config ${node}") + start_node $(echo $node | tr ':' ' ') done for i in 1 2 3 4 5 6 7 8 9 10; do echo "waiting for system to start" diff --git a/test/scripts/light-system-test-stop.sh b/test/scripts/light-system-test-stop.sh index 575ef38..0bf4374 100755 --- a/test/scripts/light-system-test-stop.sh +++ b/test/scripts/light-system-test-stop.sh @@ -7,6 +7,11 @@ top_srcdir=$(cd $(dirname $0)/../..; pwd) . ${top_srcdir}/test/scripts/testutils.sh -for node in ${ERLANGNODES}; do +stop_node() { + node=$1 ${top_srcdir}/tools/to_catlfish.py to_erl nodes/${node}/ "init:stop()" +} + +for node in ${ERLANGNODES}; do + stop_node $(echo $node | tr ':' ' ') done diff --git a/test/scripts/light-system-test.sh b/test/scripts/light-system-test.sh index eee1078..816bd67 100755 --- a/test/scripts/light-system-test.sh +++ b/test/scripts/light-system-test.sh @@ -5,7 +5,7 @@ set -o errexit top_srcdir=$(cd $(dirname $0)/../..; pwd) -. ${top_srcdir}/test/test.shvars +. ${top_srcdir}/test/scripts/testutils.sh SCRIPTS=${top_srcdir}/test/scripts @@ -18,6 +18,7 @@ tests_stop() { } ${SCRIPTS}/light-system-test-prepare.sh +init_log tests_start ${SCRIPTS}/light-system-test-run-1.sh || (echo "Tests failed"; sleep 5; tests_stop; false) sleep 5 diff --git a/test/scripts/testutils.sh b/test/scripts/testutils.sh index e779e07..ba4221c 100644 --- a/test/scripts/testutils.sh +++ b/test/scripts/testutils.sh @@ -9,14 +9,37 @@ assert_equal() { test "$2" = "$3" || fail "$1 $2 != expected $3" } +wait_for_equal() { + n=$4 + while [ $n -gt 0 ]; do + currentvalue="$(eval "$2")" + [ "${currentvalue}" = "$3" ] && return + echo "still ${currentvalue}, not $3" + n=$((n-1)) + sleep 1 + done + assert_equal "$1" "$(eval "$2")" "$3" +} + get_treesize() { - ${top_srcdir}/tools/loginfo.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg --treesize https://localhost:8080/ + get_treesize_host ${BASEURL} +} +get_treesize_host() { + ${top_srcdir}/tools/loginfo.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg --treesize "$1" } check_sth() { - ${top_srcdir}/tools/check-sth.py ${BASEURL} --publickey=keys/logkey.pem --cur-sth=stored-sth --cafile httpsca/demoCA/cacert.pem || fail "Check STH failed" + check_sth_host ${BASEURL} +} +check_sth_host() { + ${top_srcdir}/tools/check-sth.py "$1" --publickey=keys/logkey.pem --cur-sth=stored-sth --cafile httpsca/demoCA/cacert.pem || fail "Check STH failed" } do_merge() { ${top_srcdir}/tools/merge --config ${top_srcdir}/test/catlfish-test.cfg --timing --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || fail "Merge failed" + sleep 5 # FIXME: Just wait for dist instead. +} + +init_log() { + ${top_srcdir}/tools/initlog.py --config ${top_srcdir}/test/catlfish-test.cfg --localconfig ${top_srcdir}/test/catlfish-test-local-merge.cfg || fail "Log init failed" } diff --git a/tools/compileconfig.py b/tools/compileconfig.py index b5e5053..aea7ace 100755 --- a/tools/compileconfig.py +++ b/tools/compileconfig.py @@ -8,6 +8,7 @@ import sys import readconfig import re import base64 +from datetime import datetime class Symbol(str): pass @@ -131,24 +132,24 @@ def gen_http_servers(nodetype, nodeconfig, bind_addresses, bind_publicaddress, b return (http_servers, https_servers) -def allowed_clients_frontend(mergenodenames, primarymergenode): +def allowed_clients_frontend(mergenodenames, primarymergenodename): return [ ("/plop/v1/frontend/sendentry", mergenodenames), ("/plop/v1/frontend/sendlog", mergenodenames), - ("/plop/v1/frontend/publish-sth", [primarymergenode]), - ("/plop/v1/frontend/verify-entries", [primarymergenode]), + ("/plop/v1/frontend/publish-sth", [primarymergenodename]), + ("/plop/v1/frontend/verify-entries", [primarymergenodename]), ("/plop/v1/frontend/currentposition", mergenodenames), ("/plop/v1/frontend/missingentries", mergenodenames), ] -def allowed_clients_mergesecondary(primarymergenode): +def allowed_clients_mergesecondary(primarymergenodename): return [ - ("/plop/v1/merge/sendentry", [primarymergenode]), - ("/plop/v1/merge/sendlog", [primarymergenode]), - ("/plop/v1/merge/verifyroot", [primarymergenode]), - ("/plop/v1/merge/verifiedsize", [primarymergenode]), - ("/plop/v1/merge/setverifiedsize", [primarymergenode]), - ("/plop/v1/merge/missingentries", [primarymergenode]), + ("/plop/v1/merge/sendentry", [primarymergenodename]), + ("/plop/v1/merge/sendlog", [primarymergenodename]), + ("/plop/v1/merge/verifyroot", [primarymergenodename]), + ("/plop/v1/merge/verifiedsize", [primarymergenodename]), + ("/plop/v1/merge/setverifiedsize", [primarymergenodename]), + ("/plop/v1/merge/missingentries", [primarymergenodename]), ] def allowed_clients_public(): @@ -164,10 +165,10 @@ def allowed_clients_public(): ("/ct/v1/get-roots", noauth), ] -def allowed_clients_signing(frontendnodenames, primarymergenode): +def allowed_clients_signing(frontendnodenames, primarymergenodename): return [ ("/plop/v1/signing/sct", frontendnodenames), - ("/plop/v1/signing/sth", [primarymergenode]), + ("/plop/v1/signing/sth", [primarymergenodename]), ] def allowed_clients_storage(frontendnodenames, mergenodenames): @@ -185,6 +186,14 @@ def allowed_servers_frontend(signingnodenames, storagenodenames): ("/plop/v1/signing/sct", signingnodenames), ] +def allowed_servers_primarymerge(frontendnodenames): + return [ + ("/plop/v1/frontend/verify-entries", frontendnodenames), + ("/plop/v1/frontend/sendlog", frontendnodenames), + ("/plop/v1/frontend/sendentry", frontendnodenames), + ("/plop/v1/frontend/publish-sth", frontendnodenames), + ] + def parse_ratelimit_expression(expression): if expression == "none": return Symbol("none") @@ -228,7 +237,10 @@ def gen_config(nodename, config, localconfig): print >>plopconfigfile, "%% plop configuration file (-*- erlang -*-)" (nodetype, nodeconfig) = get_node_config(nodename, config) - (http_servers, https_servers) = gen_http_servers(nodetype, nodeconfig, bind_addresses, bind_publicaddress, bind_publichttpaddress=bind_publichttpaddress) + if nodename == config["primarymergenode"]: + (http_servers, https_servers) = [], [] + else: + (http_servers, https_servers) = gen_http_servers(nodetype, nodeconfig, bind_addresses, bind_publicaddress, bind_publichttpaddress=bind_publichttpaddress) catlfishconfig = [] plopconfig = [] @@ -248,8 +260,6 @@ def gen_config(nodename, config, localconfig): (Symbol("http_servers"), http_servers), (Symbol("https_certfile"), paths["https_certfile"]), (Symbol("https_keyfile"), paths["https_keyfile"]), - (Symbol("https_cacertfile"), paths["https_cacertfile"]), - (Symbol("https_cacert_fingerprint"), Binary(base64.b16decode(config["cafingerprint"]))), ] catlfishconfig.append((Symbol("mmd"), config["mmd"])) @@ -263,6 +273,11 @@ def gen_config(nodename, config, localconfig): ]) ] + plopconfig += [ + (Symbol("https_cacertfile"), paths["https_cacertfile"]), + (Symbol("https_cacert_fingerprint"), Binary(base64.b16decode(config["cafingerprint"]))), + ] + if "dbbackend" in localconfig: dbbackend = localconfig["dbbackend"] if dbbackend not in ("fsdb", "permdb"): @@ -275,7 +290,7 @@ def gen_config(nodename, config, localconfig): print >>sys.stderr, "When using permdb, all services have to be in the same node" sys.exit(1) - print "nodetype", ", ".join(nodetype) + #print "nodetype", ", ".join(nodetype) if nodetype & set(["frontendnodes", "storagenodes"]): plopconfig += [ (Symbol("entry_root_path"), paths["db"] + "certentries"), @@ -310,9 +325,10 @@ def gen_config(nodename, config, localconfig): signingnodes = config["signingnodes"] signingnodeaddresses = ["https://%s/plop/v1/signing/" % node["address"] for node in config["signingnodes"]] mergenodenames = [node["name"] for node in config["mergenodes"]] - primarymergenode = config["primarymergenode"] + primarymergenodename = config["primarymergenode"] storagenodeaddresses = ["https://%s/plop/v1/storage/" % node["address"] for node in config["storagenodes"]] frontendnodenames = [node["name"] for node in config["frontendnodes"]] + frontendnodeaddresses = ["https://%s/plop/v1/frontend/" % node["address"] for node in config["frontendnodes"]] allowed_clients = [] allowed_servers = [] @@ -323,20 +339,30 @@ def gen_config(nodename, config, localconfig): reloadableplopconfig.append((Symbol("storage_nodes"), storagenodeaddresses)) reloadableplopconfig.append((Symbol("storage_nodes_quorum"), config["storage-quorum-size"])) services.add(Symbol("ht")) - allowed_clients += allowed_clients_frontend(mergenodenames, primarymergenode) + allowed_clients += allowed_clients_frontend(mergenodenames, primarymergenodename) allowed_clients += allowed_clients_public() allowed_servers += allowed_servers_frontend([node["name"] for node in signingnodes], storagenodenames) if "storagenodes" in nodetype: allowed_clients += allowed_clients_storage(frontendnodenames, mergenodenames) if "signingnodes" in nodetype: - allowed_clients += allowed_clients_signing(frontendnodenames, primarymergenode) + allowed_clients += allowed_clients_signing(frontendnodenames, primarymergenodename) services = [Symbol("sign")] if "mergenodes" in nodetype: - storagenodenames = [node["name"] for node in config["storagenodes"]] reloadableplopconfig.append((Symbol("storage_nodes"), storagenodeaddresses)) reloadableplopconfig.append((Symbol("storage_nodes_quorum"), config["storage-quorum-size"])) services.add(Symbol("ht")) - allowed_clients += allowed_clients_mergesecondary(primarymergenode) + if nodename == primarymergenodename: + merge = localconfig["merge"] + plopconfig.append((Symbol("db_backend_opt"), [(Symbol("write_flag"), Symbol("read"))])) + plopconfig.append((Symbol("merge_delay"), merge["min-delay"])) + plopconfig.append((Symbol("merge_dist_winsize"), merge["dist-window-size"])) + plopconfig.append((Symbol("merge_dist_sendlog_chunksize"), merge["dist-sendlog-chunksize"])) + plopconfig.append((Symbol("merge_dist_sendentries_chunksize"), merge["dist-sendentries-chunksize"])) + plopconfig.append((Symbol("frontend_nodes"), frontendnodeaddresses)) + plopconfig.append((Symbol("sth_path"), paths["mergedb"] + "/sth")) + allowed_servers += allowed_servers_primarymerge(frontendnodenames) + else: + allowed_clients += allowed_clients_mergesecondary(primarymergenodename) plopconfig += [ (Symbol("services"), list(services)), @@ -383,12 +409,13 @@ def gen_config(nodename, config, localconfig): def gen_testmakefile(config, testmakefile, machines, shellvars=False): configfile = open(testmakefile, "w") + print >>configfile, "#", testmakefile, "generated by", sys.argv[0], datetime.now() + frontendnodenames = set([node["name"] for node in config["frontendnodes"]]) storagenodenames = set([node["name"] for node in config["storagenodes"]]) signingnodenames = set([node["name"] for node in config["signingnodes"]]) mergenodenames = set([node["name"] for node in config["mergenodes"]]) - erlangnodenames = frontendnodenames | storagenodenames | signingnodenames | \ - set(filter(lambda name: name != config["primarymergenode"], mergenodenames)) + erlangnodenames_and_apps = ['%s:%s' % (nn, 'catlfish' if nn != config["primarymergenode"] else "merge") for nn in frontendnodenames | storagenodenames | signingnodenames | mergenodenames] frontendnodeaddresses = [node["publicaddress"] for node in config["frontendnodes"]] storagenodeaddresses = [node["address"] for node in config["storagenodes"]] @@ -398,7 +425,7 @@ def gen_testmakefile(config, testmakefile, machines, shellvars=False): delimiter = '"' if shellvars else '' print >>configfile, "NODES=" + delimiter + " ".join(frontendnodenames|storagenodenames|signingnodenames|mergenodenames) + delimiter - print >>configfile, "ERLANGNODES=" + delimiter + " ".join(erlangnodenames) + delimiter + print >>configfile, "ERLANGNODES=" + delimiter + " ".join(erlangnodenames_and_apps) + delimiter print >>configfile, "MACHINES=" + delimiter + " ".join([str(e) for e in range(1, machines+1)]) + delimiter print >>configfile, "TESTURLS=" + delimiter + " ".join(frontendnodeaddresses+storagenodeaddresses+signingnodeaddresses+mergenodeaddresses) + delimiter print >>configfile, "BASEURL=" + delimiter + config["baseurl"] + delimiter diff --git a/tools/initlog.py b/tools/initlog.py new file mode 100755 index 0000000..11ebc2e --- /dev/null +++ b/tools/initlog.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2017, NORDUnet A/S. +# See LICENSE for licensing information. +# +# Initialise a new CT log. +# + +import sys +import os +import argparse +import yaml +import errno +from time import time +from base64 import b64encode +from certtools import build_merkle_tree, write_file +from mergetools import get_sth, perm, get_logorder + +def parse_args(): + parser = argparse.ArgumentParser(description="") + parser.add_argument('--config', help="System configuration", + required=True) + parser.add_argument('--localconfig', help="Local configuration", + required=True) + + args = parser.parse_args() + config = yaml.load(open(args.config)) + localconfig = yaml.load(open(args.localconfig)) + + return (args, config, localconfig) + +def main(): + """ + Initialise a log by creating + - perm database if it doesn't exist + """ + args, config, localconfig = parse_args() + signingnodes = config["signingnodes"] + paths = localconfig["paths"] + own_key = (localconfig["nodename"], + "%s/%s-private.pem" % (paths["privatekeys"], + localconfig["nodename"])) + mergedb = paths["mergedb"] + logorderfile = mergedb + "/logorder" + sthfile = mergedb + "/sth" + + # Don't do anything if there's already an sth file. + sth = get_sth(sthfile) + if sth['tree_size'] >= 0: + print >>sys.stderr, \ + "This log has an STH file with tree size %s." % sth['tree_size'] + print >>sys.stderr, "I refuse to destroy this log." + return 1 + + # Ensure that we can find our keyfile. + try: + os.stat(own_key[1]) + except OSError, e: + if e.errno == errno.ENOENT: + print >>sys.stderr, "Unable to open keyfile: %s" % own_key[1] + return 1 + raise + + # Create a chains database. + chainsdb = perm(localconfig.get("dbbackend", "filedb"), mergedb + "/chains") + + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/merge b/tools/merge index b5a50d5..0d3f36c 100755 --- a/tools/merge +++ b/tools/merge @@ -7,4 +7,4 @@ BINDIR=$(dirname $0) $BINDIR/merge_fetch.py "$@" $BINDIR/merge_backup.py "$@" $BINDIR/merge_sth.py "$@" -$BINDIR/merge_dist.py "$@" +#$BINDIR/merge_dist.py "$@" diff --git a/tools/merge_fetch.py b/tools/merge_fetch.py index 8f94aed..42a3089 100755 --- a/tools/merge_fetch.py +++ b/tools/merge_fetch.py @@ -59,7 +59,7 @@ def merge_fetch(args, config, localconfig): break verifycert = subprocess.Popen( - [paths["verifycert_bin"], paths["known_roots"]], + [paths["verifycert_bin"], paths["knownroots"]], stdin=subprocess.PIPE, stdout=subprocess.PIPE) added_entries = 0 diff --git a/tools/testcase1.py b/tools/testcase1.py index dbafe7a..5192074 100755 --- a/tools/testcase1.py +++ b/tools/testcase1.py @@ -13,6 +13,7 @@ import struct import hashlib import itertools import os.path +from time import sleep from certtools import * baseurls = [sys.argv[1]] @@ -148,8 +149,10 @@ def get_and_check_entry(timestamp, chain, leaf_index, baseurl): len(submittedcertchain)) def merge(): - return subprocess.call([toolsdir + "/merge", "--config", testdir + "/catlfish-test.cfg", + rv = subprocess.call([toolsdir + "/merge", "--config", testdir + "/catlfish-test.cfg", "--localconfig", testdir + "/catlfish-test-local-merge.cfg"]) + sleep(5) # FIXME: Just wait for dist instead. + return rv mergeresult = merge() assert_equal(mergeresult, 0, "merge", quiet=True, fatal=True) -- cgit v1.1