fetchallcerts.py: handle precerts

submitcert.py: handle .zip files
fetchallcerts.py: Always calculate full tree
fetchallcerts.py: Cache level 16 hashes
fetchallcerts.py: Save STH

1 files changed, 90 insertions, 3 deletions

diff --git a/tools/certtools.py b/tools/certtools.py
index e1ca57a..6a144c9 100644
--- a/tools/certtools.py
+++ b/tools/certtools.py
@@ -11,6 +11,8 @@ import sys
 import hashlib
 import ecdsa
 import datetime
+import cStringIO
+import zipfile
 
 publickeys = {
     "https://ct.googleapis.com/pilot/":
@@ -44,11 +46,14 @@ def get_cert_info(s):
 
 
 def get_pemlike(filename, marker):
+    return get_pemlike_from_file(open(filename), marker)
+
+def get_pemlike_from_file(f, marker):
     entries = []
     entry = ""
     inentry = False
 
-    for line in open(filename):
+    for line in f:
         line = line.strip()
         if line == "-----BEGIN " + marker + "-----":
             entry = ""
@@ -63,6 +68,10 @@ def get_pemlike(filename, marker):
 def get_certs_from_file(certfile):
     return get_pemlike(certfile, "CERTIFICATE")
 
+def get_certs_from_string(s):
+    f = cStringIO.StringIO(s)
+    return get_pemlike_from_file(f, "CERTIFICATE")
+
 def get_eckey_from_file(keyfile):
     keys = get_pemlike(keyfile, "EC PRIVATE KEY")
     assert len(keys) == 1
@@ -138,6 +147,10 @@ def get_entries(baseurl, start, end):
         print "ERROR:", e.read()
         sys.exit(1)
 
+def extract_precertificate(precert_chain_entry):
+    (precert, certchain) = unpack_tls_array(precert_chain_entry, 3)
+    return (precert, certchain)
+
 def decode_certificate_chain(packed_certchain):
     (unpacked_certchain, rest) = unpack_tls_array(packed_certchain, 3)
     assert len(rest) == 0
@@ -235,8 +248,13 @@ def unpack_mtl(merkle_tree_leaf):
     leaf_type = merkle_tree_leaf[1:2]
     timestamped_entry = merkle_tree_leaf[2:]
     (timestamp, entry_type) = struct.unpack(">QH", timestamped_entry[0:10])
-    (leafcert, rest_entry) = unpack_tls_array(timestamped_entry[10:], 3)
-    return (leafcert, timestamp)
+    if entry_type == 0:
+        issuer_key_hash = None
+        (leafcert, rest_entry) = unpack_tls_array(timestamped_entry[10:], 3)
+    elif entry_type == 1:
+        issuer_key_hash = timestamped_entry[10:42]
+        (leafcert, rest_entry) = unpack_tls_array(timestamped_entry[42:], 3)
+    return (leafcert, timestamp, issuer_key_hash)
 
 def get_leaf_hash(merkle_tree_leaf):
     leaf_hash = hashlib.sha256()
@@ -284,3 +302,72 @@ def build_merkle_tree(layer0):
         current_layer = next_merkle_layer(current_layer)
         layers.append(current_layer)
     return layers
+
+def print_inclusion_proof(proof):
+    audit_path = proof[u'audit_path']
+    n = proof[u'leaf_index']
+    level = 0
+    for s in audit_path:
+        entry = base64.b16encode(base64.b64decode(s))
+        n ^= 1
+        print level, n, entry
+        n >>= 1
+        level += 1
+
+def get_one_cert(store, i):
+    filename = i / 10000
+    zf = zipfile.ZipFile("%s/%04d.zip" % (store, i / 10000))
+    cert = zf.read("%08d" % i)
+    zf.close()
+    return cert
+
+def get_hash_from_certfile(cert):
+    for line in cert.split("\n"):
+        if line.startswith("-----"):
+            return None
+        if line.startswith("Leafhash: "):
+            return base64.b16decode(line[len("Leafhash: "):])
+    return None
+
+def get_proof(store, tree_size, n):
+    hash = get_hash_from_certfile(get_one_cert(store, n))
+    return get_proof_by_hash(args.baseurl, hash, tree_size)
+
+def get_certs_from_zipfiles(zipfiles, firstleaf, lastleaf):
+    for i in range(firstleaf, lastleaf + 1):
+        try:
+            yield zipfiles[i / 10000].read("%08d" % i)
+        except KeyError:
+            return
+
+def get_merkle_hash_64k(store, blocknumber, write_to_cache=False):
+    hashfilename = "%s/%04x.64khash" % (store, blocknumber)
+    try:
+        hash = base64.b16decode(open(hashfilename).read())
+        assert len(hash) == 32
+        return ("hash", hash)
+    except IOError:
+        pass
+    firstleaf = blocknumber * 65536
+    lastleaf = firstleaf + 65535
+    firstfile = firstleaf / 10000
+    lastfile = lastleaf / 10000
+    zipfiles = {}
+    for i in range(firstfile, lastfile + 1):
+        try:
+            zipfiles[i] = zipfile.ZipFile("%s/%04d.zip" % (store, i))
+        except IOError:
+            break
+    certs = get_certs_from_zipfiles(zipfiles, firstleaf, lastleaf)
+    layer0 = [get_hash_from_certfile(cert) for cert in certs]
+    tree = build_merkle_tree(layer0)
+    calculated_hash = tree[-1][0]
+    for zf in zipfiles.values():
+        zf.close()
+    if len(layer0) != 65536:
+        return ("incomplete", (len(layer0), calculated_hash))
+    if write_to_cache:
+        f = open(hashfilename, "w")
+        f.write(base64.b16encode(calculated_hash))
+        f.close()
+    return ("hash", calculated_hash)