Merge branch 'master' of ssh://git.nordu.net/user/josef/catlfish

5 files changed, 50 insertions, 27 deletions

diff --git a/monitor/josef_experimental.py b/monitor/josef_experimental.py
index 579331b..d64012d 100755
--- a/monitor/josef_experimental.py
+++ b/monitor/josef_experimental.py
@@ -70,23 +70,32 @@ def update_roots(log):
 
 if __name__ == '__main__':
 
+    dbdir = "tmpdb/"
     # entry = get_entries(ctlogs[0]["url"], 1,1)["entries"]
     # print extract_original_entry(entry[0])
-    for url in [CTLOGS[0]["url"],CTLOGS[5]["url"],CTLOGS[6]["url"]]:
+    for url in [CTLOGS[6]["url"]]:
+    # for url in [CTLOGS[0]["url"],CTLOGS[5]["url"],CTLOGS[6]["url"]]:
     # for log in CTLOGS:
     #     url = log["url"]
     # url = CTLOGS[1]["url"]
-        # result = urlopen(url + "ct/v1/get-sth").info()
-        # print get_sth(url)
-        import urllib2, socket, urlparse
-        # set up your request as before, then:
-        data = urllib2.urlopen(url+ "ct/v1/get-sth")
-        addr = socket.gethostbyname(urlparse.urlparse(data.geturl()).hostname)
-        print addr, data.fp._sock.fp._sock.getpeername()[0]
-
-        # data =  urllib2.urlopen(request)
-
-        # print addr
+        entries = get_entries(url, 3638637,3638637)["entries"]
+        # entries = get_entries(url, first, last)["entries"]
+        tmp_cert_data = []
+        for item in entries:
+            tmp_data = check_domain(item, url)
+            entry_hash = get_leaf_hash(base64.b64decode(item["leaf_input"]))
+            if tmp_data:
+                tmp_data["leaf_hash"] = base64.b64encode(entry_hash)
+                tmp_cert_data.append(tmp_data)
+                print tmp_data
+            # new_leafs.append(entry_hash)
+        # if self.dbdir:/
+
+        # db_add_certs(dbdir, tmp_cert_data)
+        
+        # if CONFIG.DEFAULT_CERT_FILE:
+        #     append_file(CONFIG.DEFAULT_CERT_FILE, tmp_cert_data)
+        # subtree = reduce_tree(new_leafs, subtree)
 
 
 
diff --git a/monitor/josef_leveldb.py b/monitor/josef_leveldb.py
index 49c5594..ef6437e 100755
--- a/monitor/josef_leveldb.py
+++ b/monitor/josef_leveldb.py
@@ -61,12 +61,14 @@ def db_append(db, key, val):
         pass
     else:
         tmpl.append(val)
+        # print "Storing:",key,SEP.join(tmpl)
         db.Put(key,SEP.join(tmpl))
 
 def db_add_domain_2(db_dir, key_in, val_in, db_in=None):
     if db_in == None:
         val = "###" + ast.literal_eval(val_in)["leaf_hash"]
         cert_db = db_open(db_dir + "certificates")
+        # print "Storing: ",val,val_in
         cert_db.Put(val, val_in)
 
         key_list  = key_in.split('/')[0].split(',')[0].split('.')
@@ -88,9 +90,11 @@ def db_add_domain_2(db_dir, key_in, val_in, db_in=None):
             return
         else:
             tmpl.append(val)
+            # print "Storing: ",key,SEP.join(tmpl)
             db.Put(key,SEP.join(tmpl))
             
     except KeyError:
+        # print "Storing: ",key,SEP.join([val])
         db.Put(key,SEP.join([val]))
         new_key = key.split('.',1)[-1]
         if new_key != key:
diff --git a/monitor/josef_monitor.py b/monitor/josef_monitor.py
index 86b6852..3896493 100755
--- a/monitor/josef_monitor.py
+++ b/monitor/josef_monitor.py
@@ -84,7 +84,6 @@ class ctlog:
                     try:    
                         self.subtree, self.entries = self.fetch_and_increment_subtree(self.entries, self.sth["tree_size"] -1, self.url, self.subtree)
                     except Exception, e:
-                        # print ERROR_STR  + "Failed fetch and increment for " + self.name
                         self.log(ERROR_STR + "Failed fetch and increment tree. Current Size: " + str(self.entries) + " Sth: " + str(self.sth) + " Error: " + str(e))
                         self.rollback()
                         return
@@ -192,7 +191,6 @@ class ctlog:
             self.log(ERROR_STR + "Could not verify STH signature " + str(new_sth))
             self.rollback()
 
-        # sth_time = datetime.datetime.fromtimestamp(new_sth['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S")
         sth_time = time_str(new_sth["timestamp"])
         if new_sth["timestamp"] != self.sth["timestamp"]:
             self.log("STH updated. Size: " + str(new_sth["tree_size"]) + ", Time: " + sth_time)
@@ -260,22 +258,17 @@ class ctlog:
         try:
             if new["tree_size"] == old["tree_size"]:
                 if old["sha256_root_hash"] != new["sha256_root_hash"]:
-                    # print ERROR_STR + "Root hash is different for same tree size in " + self.name
                     self.log(ERROR_STR + "New root hash for same tree size! Old:" + str(old) + " New:" + str(new))
                     self.rollback()
             elif new["tree_size"] < old["tree_size"]:
-                # print ERROR_STR + "New tree smaller than previous tree (%d < %d) in %s" % \
-                #   (new["tree_size"], old["tree_size"], self.name)
                 self.log(ERROR_STR + "New tree is smaller than old tree! Old:" + str(old) + " New:" + str(new))
                 self.rollback()
 
             if new["timestamp"] < old["timestamp"]:
                 self.log(ERROR_STR + "Regression in timestamps! Old:" + str(old) + " New:" + str(new))
                 self.rollback()
-                # print ERROR_STR + " Regression in timestamps in " + self.name
             else:
                 age = time.time() - new["timestamp"]/1000
-                # sth_time = datetime.datetime.fromtimestamp(new['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S")
                 sth_time = time_str(new["timestamp"])
                 roothash = new['sha256_root_hash']
                 if age > 24 * 3600:
@@ -291,7 +284,6 @@ class ctlog:
         except Exception, e:
             self.log(ERROR_STR + "Failed to verify progress! Old:" + str(old) + " New:" + str(new) + " Exception: " + str(e))
             self.rollback()
-            # print "Failed to verify progress in " + self.name
 
     def verify_consistency(self, old):
         new = self.sth
@@ -307,17 +299,14 @@ class ctlog:
                     self.log(ERROR_STR + "Verification of consistency for old hash failed! Old:" \
                         + str(old) + " New:" + str(new) + " Proof:" + str(consistency_proof))
                     self.rollback()
-                    # print ERROR_STR + "Failed to verify consistency for " + self.name
                 elif new["sha256_root_hash"] != str(base64.b64encode(res[1])):
                     self.log(ERROR_STR + "Verification of consistency for new hash failed! Old:" \
                         + str(old) + " New:" + str(new) + " Proof:" + str(consistency_proof))
                     self.rollback()
-                    # print ERROR_STR + "Failed to verify consistency for " + self.name
 
         except Exception, e:
             self.log(ERROR_STR + "Could not verify consistency! " + " Old:" + str(old) + " New:" + str(new) + " Error:"  + str(e))
             self.rollback()
-            # print ERROR_STR + "Could not verify consistency for " + self.url
 
 
 # def verify_inclusion_all(old, new):
diff --git a/monitor/josef_reader.py b/monitor/josef_reader.py
index 9eb5094..bd069bb 100755
--- a/monitor/josef_reader.py
+++ b/monitor/josef_reader.py
@@ -16,7 +16,7 @@ except:
     print "No database support found."
 from datetime import datetime as dt
 import ast
-from monitor_conf import DB_PATH, MONITORED_DOMAINS, OUTPUT_DIR
+from monitor_conf import DB_PATH, MONITORED_DOMAINS, OUTPUT_DIR, ERROR_STR
 
 
 
@@ -165,12 +165,27 @@ def db_monitor_domain(domain, log=None, exclude_invalid=None, get_cert=None, iss
 
         # Print matching
         if success:
-            me = monitored_entry(entry["subject"].split("CN=")[1], \
-                entry["issuer"].split("CN=")[1], \
+            try:
+                tmp_issuer = entry["issuer"].split("CN=")[1]
+            except:
+                if "issuer" in entry:
+                    tmp_issuer = entry["issuer"]
+                else:
+                    tmp_issuer = "No Issuer Found"
+
+            try:
+                tmp_subject = entry["subject"].split("CN=")[1]
+            except:
+                if "subject" in entry:
+                    tmp_subject = entry["subject"]
+                else:
+                    tmp_subject = "No Subject Found"
+
+            me = monitored_entry(tmp_subject, \
+                tmp_issuer, \
                 entry["log"], \
                 valid, \
                 entry["leaf_hash"])
-            # print str(me)
 
             if get_cert:
                 print get_full_cert(entry)
diff --git a/monitor/monitor_conf.py b/monitor/monitor_conf.py
index 5a3737c..674c565 100644
--- a/monitor/monitor_conf.py
+++ b/monitor/monitor_conf.py
@@ -85,6 +85,12 @@ CTLOGS = [
     "id" : "rDua7X+pZ0dXFZ5tfVdWcvnZgQCUHpve/+yhMTt1eC0=",
     "build" : True},
 
+    {"name" : "wosign",
+    "url" : "https://ct.wosign.com/",
+    "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE1+wvK3VPN7yjQ7qLZWY8fWrlDCqmwuUm/gx9TnzwOrzi0yLcAdAfbkOcXG6DrZwV9sSNYLUdu6NiaX7rp6oBmw==",
+    "id" : "nk/3PcPOIgtpIXyJnkaAdqv414Y21cz8haMadWKLqIs=",
+    "build" : True},    
+
 ]