fetchallcerts.py: handle precerts

submitcert.py: handle .zip files
fetchallcerts.py: Always calculate full tree
fetchallcerts.py: Cache level 16 hashes
fetchallcerts.py: Save STH

1 files changed, 39 insertions, 12 deletions

diff --git a/tools/submitcert.py b/tools/submitcert.py
index 1b87b53..04b6ebe 100755
--- a/tools/submitcert.py
+++ b/tools/submitcert.py
@@ -15,6 +15,7 @@ from certtools import *
 import os
 import signal
 import select
+import zipfile
 
 from multiprocessing import Pool
 
@@ -29,13 +30,13 @@ if certfilepath[-1] == "/":
 else:
     certfiles = [certfilepath]
 
-def submitcert(certfile):
+def submitcert((certfile, cert)):
     timing = timing_point()
-    certs = get_certs_from_file(certfile)
+    certchain = get_certs_from_string(cert)
     timing_point(timing, "readcerts")
 
     try:
-        result = add_chain(baseurl, {"chain":map(base64.b64encode, certs)})
+        result = add_chain(baseurl, {"chain":map(base64.b64encode, certchain)})
     except SystemExit:
         print "EXIT:", certfile
         select.select([], [], [], 1.0)
@@ -49,7 +50,7 @@ def submitcert(certfile):
 
     try:
         if check_sig:
-            check_sct_signature(baseurl, certs[0], result)
+            check_sct_signature(baseurl, certchain[0], result)
             timing_point(timing, "checksig")
     except AssertionError, e:
         print "ERROR:", certfile, e
@@ -63,7 +64,7 @@ def submitcert(certfile):
 
     if lookup_in_log:
 
-        merkle_tree_leaf = pack_mtl(result["timestamp"], certs[0])
+        merkle_tree_leaf = pack_mtl(result["timestamp"], certchain[0])
 
         leaf_hash = get_leaf_hash(merkle_tree_leaf)
 
@@ -84,14 +85,14 @@ def submitcert(certfile):
 
         certchain = decode_certificate_chain(base64.decodestring(extra_data))
 
-        submittedcertchain = certs[1:]
+        submittedcertchain = certchain[1:]
 
         for (submittedcert, fetchedcert, i) in zip(submittedcertchain,
                                                    certchain, itertools.count(1)):
             print "cert", i, "in chain is the same:", submittedcert == fetchedcert
 
         if len(certchain) == len(submittedcertchain) + 1:
-            last_issuer = get_cert_info(certs[-1])["issuer"]
+            last_issuer = get_cert_info(certchain[-1])["issuer"]
             root_subject = get_cert_info(certchain[-1])["subject"]
             print "issuer of last cert in submitted chain and " \
                 "subject of last cert in fetched chain is the same:", \
@@ -105,20 +106,46 @@ def submitcert(certfile):
     timing_point(timing, "lookup")
     return timing["deltatimes"]
 
+def get_ncerts(certfiles):
+    n = 0
+    for certfile in certfiles:
+        if certfile.endswith(".zip"):
+            zf = zipfile.ZipFile(certfile)
+            n += len(zf.namelist())
+            zf.close()
+        else:
+            n += 1
+    return n
+
+def get_all_certificates(certfiles):
+    for certfile in certfiles:
+        if certfile.endswith(".zip"):
+            zf = zipfile.ZipFile(certfile)
+            for name in zf.namelist():
+                yield (name, zf.read(name))
+            zf.close()
+        else:
+            yield (certfile, open(certfile).read())
+
 p = Pool(16, lambda: signal.signal(signal.SIGINT, signal.SIG_IGN))
 
 nsubmitted = 0
 lastprinted = 0
-starttime = datetime.datetime.now()
 
-print len(certfiles), "certs"
+ncerts = get_ncerts(certfiles)
 
-submitcert(certfiles[0])
+print ncerts, "certs"
+
+certs = get_all_certificates(certfiles)
+
+submitcert(certs.next())
 nsubmitted += 1
 select.select([], [], [], 3.0)
 
+starttime = datetime.datetime.now()
+
 try:
-    for timing in p.imap_unordered(submitcert, certfiles[1:]):
+    for timing in p.imap_unordered(submitcert, certs):
         if timing == None:
             print "error"
             print "submitted", nsubmitted
@@ -129,7 +156,7 @@ try:
         deltatime = datetime.datetime.now() - starttime
         deltatime_f = deltatime.seconds + deltatime.microseconds / 1000000.0
         rate = nsubmitted / deltatime_f
-        if nsubmitted > lastprinted + len(certfiles) / 10:
+        if nsubmitted > lastprinted + ncerts / 10:
             print nsubmitted, "rate %.1f" % rate
             lastprinted = nsubmitted
         #print timing, "rate %.1f" % rate