summaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
authorMagnus Ahltorp <map@kth.se>2016-07-08 16:51:55 +0200
committerMagnus Ahltorp <map@kth.se>2016-07-08 16:51:55 +0200
commit9039104a2203e3467a2420348e5cee43eb5bd5af (patch)
tree6f6e565cf504aaab6b43fb60734b89142dc25477 /tools
parented94e0682912a8ee16cb5a1427166c3e77b51b50 (diff)
Added permdb support to merge
Diffstat (limited to 'tools')
-rwxr-xr-xtools/merge_backup.py8
-rwxr-xr-xtools/merge_dist.py6
-rwxr-xr-xtools/merge_fetch.py9
-rw-r--r--tools/mergetools.py31
4 files changed, 43 insertions, 11 deletions
diff --git a/tools/merge_backup.py b/tools/merge_backup.py
index 48ac133..ab2625d 100755
--- a/tools/merge_backup.py
+++ b/tools/merge_backup.py
@@ -12,9 +12,9 @@ from time import sleep
from certtools import timing_point, build_merkle_tree, write_file, \
create_ssl_context
from mergetools import chunks, backup_sendlog, get_logorder, \
- get_verifiedsize, get_missingentriesforbackup, read_chain, \
+ get_verifiedsize, get_missingentriesforbackup, \
hexencode, setverifiedsize, sendentries_merge, verifyroot, \
- get_nfetched, parse_args
+ get_nfetched, parse_args, perm
def backup_loop(nodename, nodeaddress, own_key, paths, verifiedsize, chunk):
for trynumber in range(5, 0, -1):
@@ -37,7 +37,7 @@ def merge_backup(args, config, localconfig, secondaries):
"%s/%s-private.pem" % (paths["privatekeys"],
localconfig["nodename"]))
mergedb = paths["mergedb"]
- chainsdir = mergedb + "/chains"
+ chainsdb = perm(localconfig.get("dbbackend", "filedb"), mergedb + "/chains")
logorderfile = mergedb + "/logorder"
currentsizefile = mergedb + "/fetched"
timing = timing_point()
@@ -119,7 +119,7 @@ def merge_backup(args, config, localconfig, secondaries):
with requests.sessions.Session() as session:
for missingentry_chunk in chunks(missingentries, 100):
missingentry_hashes = [base64.b64decode(missingentry) for missingentry in missingentry_chunk]
- hashes_and_entries = [(hash, read_chain(chainsdir, hash)) for hash in missingentry_hashes]
+ hashes_and_entries = [(hash, chainsdb.get(hash)) for hash in missingentry_hashes]
sendentryresult = sendentries_merge(nodename, nodeaddress,
own_key, paths,
hashes_and_entries, session)
diff --git a/tools/merge_dist.py b/tools/merge_dist.py
index 9831f97..2af1d6c 100755
--- a/tools/merge_dist.py
+++ b/tools/merge_dist.py
@@ -11,7 +11,7 @@ from base64 import b64encode, b64decode
from certtools import timing_point, \
create_ssl_context
from mergetools import get_curpos, get_logorder, chunks, get_missingentries, \
- sendsth, sendlog, sendentry, read_chain, parse_args
+ sendsth, sendlog, sendentry, parse_args, perm
def merge_dist(args, localconfig, frontendnodes, timestamp):
paths = localconfig["paths"]
@@ -19,7 +19,7 @@ def merge_dist(args, localconfig, frontendnodes, timestamp):
"%s/%s-private.pem" % (paths["privatekeys"],
localconfig["nodename"]))
mergedb = paths["mergedb"]
- chainsdir = mergedb + "/chains"
+ chainsdb = perm(localconfig.get("dbbackend", "filedb"), mergedb + "/chains")
logorderfile = mergedb + "/logorder"
sthfile = mergedb + "/sth"
create_ssl_context(cafile=paths["https_cacertfile"])
@@ -92,7 +92,7 @@ def merge_dist(args, localconfig, frontendnodes, timestamp):
for missingentry in missingentries:
ehash = b64decode(missingentry)
sendentryresult = sendentry(nodename, nodeaddress, own_key, paths,
- read_chain(chainsdir, ehash), ehash)
+ chainsdb.get(ehash), ehash)
if sendentryresult["result"] != "ok":
print >>sys.stderr, "sendentry:", sendentryresult
sys.exit(1)
diff --git a/tools/merge_fetch.py b/tools/merge_fetch.py
index bb1e17e..b77dea6 100755
--- a/tools/merge_fetch.py
+++ b/tools/merge_fetch.py
@@ -9,8 +9,8 @@ import struct
import subprocess
from time import sleep
from mergetools import get_logorder, verify_entry, get_new_entries, \
- chunks, fsync_logorder, get_entries, write_chain, add_to_logorder, \
- hexencode, parse_args
+ chunks, fsync_logorder, get_entries, add_to_logorder, \
+ hexencode, parse_args, perm
from certtools import timing_point, write_file, create_ssl_context
def merge_fetch(args, config, localconfig):
@@ -18,7 +18,7 @@ def merge_fetch(args, config, localconfig):
storagenodes = config["storagenodes"]
mergedb = paths["mergedb"]
logorderfile = mergedb + "/logorder"
- chainsdir = mergedb + "/chains"
+ chainsdb = perm(localconfig.get("dbbackend", "filedb"), mergedb + "/chains")
own_key = (localconfig["nodename"],
"%s/%s-private.pem" % (paths["privatekeys"],
localconfig["nodename"]))
@@ -70,7 +70,7 @@ def merge_fetch(args, config, localconfig):
for ehash in chunk:
entry = entries[ehash]
verify_entry(verifycert, entry, ehash)
- write_chain(ehash, entry, chainsdir)
+ chainsdb.add(ehash, entry)
add_to_logorder(logorderfile, ehash)
logorder.append(ehash)
certsinlog.add(ehash)
@@ -79,6 +79,7 @@ def merge_fetch(args, config, localconfig):
sys.stderr.flush()
print >>sys.stderr
sys.stderr.flush()
+ chainsdb.commit()
fsync_logorder(logorderfile)
timing_point(timing, "add entries")
print >>sys.stderr, "added", added_entries, "entries"
diff --git a/tools/mergetools.py b/tools/mergetools.py
index ec4fd2a..80fbf0b 100644
--- a/tools/mergetools.py
+++ b/tools/mergetools.py
@@ -10,6 +10,10 @@ import json
import yaml
import argparse
import requests
+try:
+ import permdb
+except ImportError:
+ pass
from certtools import get_leaf_hash, http_request, get_leaf_hash
def parselogrow(row):
@@ -409,3 +413,30 @@ def parse_args():
localconfig = yaml.load(open(args.localconfig))
return (args, config, localconfig)
+
+def perm(dbtype, path):
+ if dbtype == "filedb":
+ return FileDB(path)
+ elif dbtype == "permdb":
+ return PermDB(path)
+ assert False
+
+class FileDB:
+ def __init__(self, path):
+ self.path = path
+ def get(self, key):
+ return read_chain(self.path, key)
+ def add(self, key, value):
+ return write_chain(key, value, self.path)
+ def commit(self):
+ pass
+
+class PermDB:
+ def __init__(self, path):
+ self.permdbobj = permdb.alloc(path)
+ def get(self, key):
+ return permdb.getvalue(self.permdbobj, key)
+ def add(self, key, value):
+ return permdb.addvalue(self.permdbobj, key, value)
+ def commit(self):
+ permdb.committree(self.permdbobj)