From 4e1bcab3f91f975a19710a4350bbee0e9af5168e Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Tue, 3 Mar 2015 14:03:05 +0100 Subject: Move http_request to certtools --- tools/merge.py | 35 ++++++++++------------------------- 1 file changed, 10 insertions(+), 25 deletions(-) (limited to 'tools/merge.py') diff --git a/tools/merge.py b/tools/merge.py index 6becf7e..c9f99af 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -14,7 +14,7 @@ import time import ecdsa import hashlib import urlparse -from certtools import build_merkle_tree, create_sth_signature, check_sth_signature, get_eckey_from_file, timing_point +from certtools import build_merkle_tree, create_sth_signature, check_sth_signature, get_eckey_from_file, timing_point, http_request parser = argparse.ArgumentParser(description="") parser.add_argument("--baseurl", metavar="url", help="Base URL for CT server", required=True) @@ -34,6 +34,8 @@ storagenodes = args.storage chainsdir = args.mergedb + "/chains" logorderfile = args.mergedb + "/logorder" +own_key = (args.own_keyname, args.own_keyfile) + def parselogrow(row): return base64.b16decode(row) @@ -57,26 +59,9 @@ def add_to_logorder(key): f.write(base64.b16encode(key) + "\n") f.close() -def http_request(url, data=None): - req = urllib2.Request(url, data) - keyname = args.own_keyname - privatekey = get_eckey_from_file(args.own_keyfile) - sk = ecdsa.SigningKey.from_der(privatekey) - parsed_url = urlparse.urlparse(url) - if data == None: - data = parsed_url.query - method = "GET" - else: - method = "POST" - signature = sk.sign("%s\0%s\0%s" % (method, parsed_url.path, data), hashfunc=hashlib.sha256, - sigencode=ecdsa.util.sigencode_der) - req.add_header('X-Catlfish-Auth', base64.b64encode(signature) + ";key=" + keyname) - result = urllib2.urlopen(req).read() - return result - def get_new_entries(baseurl): try: - result = http_request(baseurl + "ct/storage/fetchnewentries") + result = http_request(baseurl + "ct/storage/fetchnewentries", key=own_key) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": return [base64.b64decode(entry) for entry in parsed_result[u"entries"]] @@ -89,7 +74,7 @@ def get_new_entries(baseurl): def get_entries(baseurl, hashes): try: params = urllib.urlencode({"hash":[base64.b64encode(hash) for hash in hashes]}, doseq=True) - result = http_request(baseurl + "ct/storage/getentry?" + params) + result = http_request(baseurl + "ct/storage/getentry?" + params, key=own_key) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": entries = dict([(base64.b64decode(entry["hash"]), base64.b64decode(entry["entry"])) for entry in parsed_result[u"entries"]]) @@ -104,7 +89,7 @@ def get_entries(baseurl, hashes): def get_curpos(baseurl): try: - result = http_request(baseurl + "ct/frontend/currentposition") + result = http_request(baseurl + "ct/frontend/currentposition", key=own_key) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": return parsed_result[u"position"] @@ -117,7 +102,7 @@ def get_curpos(baseurl): def sendlog(baseurl, submission): try: result = http_request(baseurl + "ct/frontend/sendlog", - json.dumps(submission)) + json.dumps(submission), key=own_key) return json.loads(result) except urllib2.HTTPError, e: print "ERROR: sendlog", e.read() @@ -133,7 +118,7 @@ def sendlog(baseurl, submission): def sendentry(baseurl, entry, hash): try: result = http_request(baseurl + "ct/frontend/sendentry", - json.dumps({"entry":base64.b64encode(entry), "treeleafhash":base64.b64encode(hash)})) + json.dumps({"entry":base64.b64encode(entry), "treeleafhash":base64.b64encode(hash)}), key=own_key) return json.loads(result) except urllib2.HTTPError, e: print "ERROR: sendentry", e.read() @@ -149,7 +134,7 @@ def sendentry(baseurl, entry, hash): def sendsth(baseurl, submission): try: result = http_request(baseurl + "ct/frontend/sendsth", - json.dumps(submission)) + json.dumps(submission), key=own_key) return json.loads(result) except urllib2.HTTPError, e: print "ERROR: sendsth", e.read() @@ -164,7 +149,7 @@ def sendsth(baseurl, submission): def get_missingentries(baseurl): try: - result = http_request(baseurl + "ct/frontend/missingentries") + result = http_request(baseurl + "ct/frontend/missingentries", key=own_key) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": return parsed_result[u"entries"] -- cgit v1.1 From ff18e0fdd57a6b485f427173fe7febee03345037 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Tue, 3 Mar 2015 15:33:39 +0100 Subject: merge.py: use external signing --- tools/merge.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'tools/merge.py') diff --git a/tools/merge.py b/tools/merge.py index c9f99af..0996ec9 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -21,7 +21,7 @@ parser.add_argument("--baseurl", metavar="url", help="Base URL for CT server", r parser.add_argument("--frontend", action="append", metavar="url", help="Base URL for frontend server", required=True) parser.add_argument("--storage", action="append", metavar="url", help="Base URL for storage server", required=True) parser.add_argument("--mergedb", metavar="dir", help="Merge database directory", required=True) -parser.add_argument("--keyfile", metavar="keyfile", help="File containing log key", required=True) +parser.add_argument("--signing", metavar="url", help="Base URL for signing server", required=True) parser.add_argument("--own-keyname", metavar="keyname", help="The key name of the merge node", required=True) parser.add_argument("--own-keyfile", metavar="keyfile", help="The file containing the private key of the merge node", required=True) parser.add_argument("--nomerge", action='store_true', help="Don't actually do merge") @@ -215,10 +215,9 @@ tree = build_merkle_tree(logorder) tree_size = len(logorder) root_hash = tree[-1][0] timestamp = int(time.time() * 1000) -privatekey = get_eckey_from_file(args.keyfile) tree_head_signature = create_sth_signature(tree_size, timestamp, - root_hash, privatekey) + root_hash, args.signing, key=own_key) sth = {"tree_size": tree_size, "timestamp": timestamp, "sha256_root_hash": base64.b64encode(root_hash), -- cgit v1.1 From e0f11a58033d52c70bc76b4b5611cb88485d4653 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Wed, 4 Mar 2015 15:42:59 +0100 Subject: Save STH instead of calculating a new one each time. --- tools/merge.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'tools/merge.py') diff --git a/tools/merge.py b/tools/merge.py index 0996ec9..c137f4b 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -239,7 +239,10 @@ for frontendnode in frontendnodes: print "current position", curpos entries = [base64.b64encode(entry) for entry in logorder[curpos:]] for chunk in chunks(entries, 1000): - sendlog(frontendnode, {"start": curpos, "hashes": chunk}) + sendlogresult = sendlog(frontendnode, {"start": curpos, "hashes": chunk}) + if sendlogresult["result"] != "ok": + print "sendlog:", sendlogresult + sys.exit(1) curpos += len(chunk) print curpos, sys.stdout.flush() @@ -250,8 +253,14 @@ for frontendnode in frontendnodes: print "missing entries:", len(missingentries) for missingentry in missingentries: hash = base64.b64decode(missingentry) - sendentry(frontendnode, read_chain(hash), hash) + sendentryresult = sendentry(frontendnode, read_chain(hash), hash) + if sendentryresult["result"] != "ok": + print "send sth:", sendentryresult + sys.exit(1) timing_point(timing, "send missing") - sendsth(frontendnode, sth) + sendsthresult = sendsth(frontendnode, sth) + if sendsthresult["result"] != "ok": + print "send sth:", sendsthresult + sys.exit(1) timing_point(timing, "send sth") print timing["deltatimes"] -- cgit v1.1 From 8a7bcb91f440502b24d92842e0c89f2fcdf90c04 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sun, 8 Mar 2015 00:03:36 +0100 Subject: merge.py: Store certs in hashed directory structure --- tools/merge.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) (limited to 'tools/merge.py') diff --git a/tools/merge.py b/tools/merge.py index c137f4b..1b94581 100755 --- a/tools/merge.py +++ b/tools/merge.py @@ -14,6 +14,7 @@ import time import ecdsa import hashlib import urlparse +import os from certtools import build_merkle_tree, create_sth_signature, check_sth_signature, get_eckey_from_file, timing_point, http_request parser = argparse.ArgumentParser(description="") @@ -36,6 +37,8 @@ logorderfile = args.mergedb + "/logorder" own_key = (args.own_keyname, args.own_keyfile) +hashed_dir = True + def parselogrow(row): return base64.b16decode(row) @@ -44,12 +47,26 @@ def get_logorder(): return [parselogrow(row.rstrip()) for row in f] def write_chain(key, value): - f = open(chainsdir + "/" + base64.b16encode(key), "w") + filename = base64.b16encode(key) + if hashed_dir: + path = chainsdir + "/" + filename[0:2] + "/" + filename[2:4] + "/" + filename[4:6] + try: + os.makedirs(path) + except Exception, e: + print e + else: + path = chainsdir + f = open(path + "/" + filename, "w") f.write(value) f.close() def read_chain(key): - f = open(chainsdir + "/" + base64.b16encode(key), "r") + filename = base64.b16encode(key) + path = chainsdir + "/" + filename[0:2] + "/" + filename[2:4] + "/" + filename[4:6] + try: + f = open(path + "/" + filename, "r") + except IOError, e: + f = open(chainsdir + "/" + filename, "r") value = f.read() f.close() return value -- cgit v1.1