summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLinus Nordberg <linus@nordu.net>2017-02-02 02:47:49 +0100
committerLinus Nordberg <linus@nordu.net>2017-02-02 02:47:49 +0100
commit6610afbf6a5e802fc15733d6a499281cd981af13 (patch)
treedfdfc005008c84abfd7623d9de15970f3ed687e0
parentb7b8903b6a7c3342933b9984afa72fb6527b5f72 (diff)
Parallelised merge, backup phase.
-rw-r--r--test/catlfish-test-local-merge.cfg3
-rwxr-xr-xtools/compileconfig.py17
-rwxr-xr-xtools/merge2
3 files changed, 19 insertions, 3 deletions
diff --git a/test/catlfish-test-local-merge.cfg b/test/catlfish-test-local-merge.cfg
index bf1d3c1..e0841e9 100644
--- a/test/catlfish-test-local-merge.cfg
+++ b/test/catlfish-test-local-merge.cfg
@@ -22,6 +22,9 @@ logadminkey: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQ
merge:
min-delay: 1
+ backup-window-size: 2
+ backup-sendlog-chunksize: 1
+ backup-sendentries-chunksize: 1
dist-window-size: 2
dist-sendlog-chunksize: 1
dist-sendentries-chunksize: 1
diff --git a/tools/compileconfig.py b/tools/compileconfig.py
index aea7ace..47fd19a 100755
--- a/tools/compileconfig.py
+++ b/tools/compileconfig.py
@@ -186,12 +186,17 @@ def allowed_servers_frontend(signingnodenames, storagenodenames):
("/plop/v1/signing/sct", signingnodenames),
]
-def allowed_servers_primarymerge(frontendnodenames):
+def allowed_servers_primarymerge(frontendnodenames, secondarymergenames):
return [
("/plop/v1/frontend/verify-entries", frontendnodenames),
("/plop/v1/frontend/sendlog", frontendnodenames),
("/plop/v1/frontend/sendentry", frontendnodenames),
("/plop/v1/frontend/publish-sth", frontendnodenames),
+ ("/plop/v1/merge/verifiedsize", secondarymergenames),
+ ("/plop/v1/merge/verifyroot", secondarymergenames),
+ ("/plop/v1/merge/setverifiedsize", secondarymergenames),
+ ("/plop/v1/merge/sendlog", secondarymergenames),
+ ("/plop/v1/merge/sendentry", secondarymergenames),
]
def parse_ratelimit_expression(expression):
@@ -352,15 +357,23 @@ def gen_config(nodename, config, localconfig):
reloadableplopconfig.append((Symbol("storage_nodes_quorum"), config["storage-quorum-size"]))
services.add(Symbol("ht"))
if nodename == primarymergenodename:
+ mergesecondarynames = [node["name"] for node in config["mergenodes"] if node["name"] != primarymergenodename]
+ mergesecondaryaddresses = ["https://%s/plop/v1/merge/" % node["address"] for node in config["mergenodes"] if node["name"] != primarymergenodename]
merge = localconfig["merge"]
plopconfig.append((Symbol("db_backend_opt"), [(Symbol("write_flag"), Symbol("read"))]))
plopconfig.append((Symbol("merge_delay"), merge["min-delay"]))
plopconfig.append((Symbol("merge_dist_winsize"), merge["dist-window-size"]))
plopconfig.append((Symbol("merge_dist_sendlog_chunksize"), merge["dist-sendlog-chunksize"]))
plopconfig.append((Symbol("merge_dist_sendentries_chunksize"), merge["dist-sendentries-chunksize"]))
+ reloadableplopconfig.append((Symbol("merge_backup_winsize"), merge["backup-window-size"]))
+ reloadableplopconfig.append((Symbol("merge_backup_sendlog_chunksize"), merge["backup-sendlog-chunksize"]))
+ reloadableplopconfig.append((Symbol("merge_backup_sendentries_chunksize"), merge["backup-sendentries-chunksize"]))
plopconfig.append((Symbol("frontend_nodes"), frontendnodeaddresses))
+ reloadableplopconfig.append((Symbol("merge_secondaries"), zip(mergesecondarynames, mergesecondaryaddresses)))
plopconfig.append((Symbol("sth_path"), paths["mergedb"] + "/sth"))
- allowed_servers += allowed_servers_primarymerge(frontendnodenames)
+ plopconfig.append((Symbol("fetched_path"), paths["mergedb"] + "/fetched"))
+ plopconfig.append((Symbol("verified_path"), paths["mergedb"] + "/verified"))
+ allowed_servers += allowed_servers_primarymerge(frontendnodenames, mergesecondarynames)
else:
allowed_clients += allowed_clients_mergesecondary(primarymergenodename)
diff --git a/tools/merge b/tools/merge
index 0d3f36c..d0198ec 100755
--- a/tools/merge
+++ b/tools/merge
@@ -5,6 +5,6 @@ set -o errexit
BINDIR=$(dirname $0)
$BINDIR/merge_fetch.py "$@"
-$BINDIR/merge_backup.py "$@"
+#$BINDIR/merge_backup.py "$@"
$BINDIR/merge_sth.py "$@"
#$BINDIR/merge_dist.py "$@"