Verify new entries when merging.

Minor tweaks on map/mergeverify2.
Closes CATLFISH-36.

1 files changed, 45 insertions, 4 deletions

diff --git a/tools/merge.py b/tools/merge.py
index f9c93d9..ce3bf0b 100755
--- a/tools/merge.py
+++ b/tools/merge.py
@@ -16,9 +16,11 @@ import hashlib
 import urlparse
 import os
 import yaml
+import select
+import struct
 from certtools import build_merkle_tree, create_sth_signature, \
     check_sth_signature, get_eckey_from_file, timing_point, http_request, \
-    get_public_key_from_file
+    get_public_key_from_file, get_leaf_hash, decode_certificate_chain
 
 parser = argparse.ArgumentParser(description="")
 parser.add_argument('--config', help="System configuration", required=True)
@@ -61,7 +63,7 @@ def write_chain(key, value):
         try:
             os.makedirs(path)
         except Exception, e:
-            print e
+            pass
     else:
         path = chainsdir
     f = open(path + "/" + filename, "w")
@@ -131,7 +133,7 @@ def sendlog(node, baseurl, submission):
         return json.loads(result)
     except urllib2.HTTPError, e:
         print "ERROR: sendlog", e.read()
-        sys.exit(1)
+        return None
     except ValueError, e:
         print "==== FAILED REQUEST ===="
         print submission
@@ -206,6 +208,32 @@ for storagenode in storagenodes:
     new_entries.update(new_entries_per_node[storagenode["name"]])
     entries_to_fetch[storagenode["name"]] = []
 
+def unpack_entry(entry):
+    pieces = []
+    while len(entry):
+        (length,) = struct.unpack(">I", entry[0:4])
+        data = entry[4:4+length]
+        entry = entry[4+length:]
+        pieces.append(data)
+    return pieces
+
+import subprocess
+
+def verify_entry(verifycert, entry, hash):
+    unpacked = unpack_entry(entry)
+    mtl = unpacked[0]
+    assert hash == get_leaf_hash(mtl)
+    s = struct.pack(">I", len(entry)) + entry
+    verifycert.stdin.write(s)
+    result_length_packed = verifycert.stdout.read(4)
+    (result_length,) = struct.unpack(">I", result_length_packed)
+    result = verifycert.stdout.read(result_length)
+    assert len(result) == result_length
+    (error_code,) = struct.unpack("B", result[0:1])
+    if error_code != 0:
+        print >>sys.stderr, result[1:]
+        sys.exit(1)
+
 timing_point(timing, "get new entries")
 
 new_entries -= certsinlog
@@ -221,6 +249,8 @@ for hash in new_entries:
             entries_to_fetch[storagenode["name"]].append(hash)
             break
 
+verifycert = subprocess.Popen(["../verifycert.erl"],
+                               stdin=subprocess.PIPE, stdout=subprocess.PIPE)
 
 added_entries = 0
 for storagenode in storagenodes:
@@ -229,6 +259,7 @@ for storagenode in storagenodes:
         entries = get_entries(storagenode["name"], "https://%s/" % storagenode["address"], chunk)
         for hash in chunk:
             entry = entries[hash]
+            verify_entry(verifycert, entry, hash)
             write_chain(hash, entry)
             add_to_logorder(hash)
             logorder.append(hash)
@@ -237,6 +268,8 @@ for storagenode in storagenodes:
 timing_point(timing, "add entries")
 print "added", added_entries, "entries"
 
+verifycert.communicate(struct.pack("I", 0))
+
 tree = build_merkle_tree(logorder)
 tree_size = len(logorder)
 root_hash = tree[-1][0]
@@ -277,7 +310,15 @@ for frontendnode in frontendnodes:
     print "current position", curpos
     entries = [base64.b64encode(entry) for entry in logorder[curpos:]]
     for chunk in chunks(entries, 1000):
-        sendlogresult = sendlog(nodename, nodeaddress, {"start": curpos, "hashes": chunk})
+        for trynumber in range(5, 0, -1):
+            sendlogresult = sendlog(nodename, nodeaddress, {"start": curpos, "hashes": chunk})
+            if sendlogresult == None:
+                if trynumber == 1:
+                    sys.exit(1)
+                select.select([], [], [], 10.0)
+                print "tries left:", trynumber
+                continue
+            break
         if sendlogresult["result"] != "ok":
             print "sendlog:", sendlogresult
             sys.exit(1)