#!/usr/bin/python # -*- coding: utf-8 -*- # Selected dependencies # python-dev (apt) # python-crypto (apt) # leveldb (pip) import time import datetime import base64 import argparse import errno from copy import deepcopy from josef_lib import * from josef_reader import monitored_domain try: from josef_leveldb import db_add_certs, db_open except: print "No database support found" import os.path parser = argparse.ArgumentParser(description="") parser.add_argument('--config', default="monitor_conf.py") args = parser.parse_args() # Import from config file if os.path.isfile(args.config): modules = map(__import__, [args.config[:-2]]) CONFIG = modules[0] ERROR_STR = CONFIG.ERROR_STR else: print "Config file not found!" ERROR_STR = "(local)ERROR: " sys.exit() class ctlog: def __init__(self, name, url, key, log_id=None, build=True): self.name = name self.url = url self.key = key self.log_id = log_id self.logfile = CONFIG.OUTPUT_DIR + name + ".log" self.savefile = CONFIG.OUTPUT_DIR + name + "-state-info.json" self.subtree = [[]] self.fe_ips = {} self.sth = None self.entries = 0 self.root_hash = None self.build = build self.saved_sth = None self.saved_entries = None self.saved_subtree = None if CONFIG.DB_PATH: self.dbdir = CONFIG.DB_PATH # self.dbdir = CONFIG.DB_PATH + name + "/" if not os.path.exists(self.dbdir): os.makedirs(self.dbdir) else: self.dbdir = None self.log("Starting monitor") def incremental_build(self): # Keeps state current during build, partial builds are possible. try: self.sth = get_sth(self.url) except Exception, e: self.log("Failed to fetch STH. " + str(e)) return if self.build: start_size = self.entries try: while self.entries < self.sth["tree_size"]: tmp_size = self.entries try: self.subtree, self.entries = self.fetch_and_increment_subtree(self.entries, self.sth["tree_size"] -1, self.url, self.subtree) except Exception, e: # print ERROR_STR + "Failed fetch and increment for " + self.name self.log(ERROR_STR + "Failed fetch and increment tree. Current Size: " + str(self.entries) + " Sth: " + str(self.sth) + " Error: " + str(e)) self.rollback() return if tmp_size != self.entries: self.log("Got entries " + str(tmp_size) + " to " \ + str(self.entries -1 ) + " of " + str(self.sth["tree_size"]-1)) if self.entries != start_size: if verify_subtree(self.sth, self.subtree, self.url): pass else: self.log(ERROR_STR + "Failed to verify newly built subtree!") self.rollback() except Exception, e: # print "Failed incremental build for " + self.name self.log(ERROR_STR + "Failed incremental build. Error: " + str(e)) self.rollback() def save_state(self): self.saved_sth = self.sth self.saved_subtree = self.subtree self.saved_entries = self.entries def rollback(self): if self.saved_entries and self.saved_subtree and self.saved_sth: self.log("Rolling back to last saved state") self.sth = self.saved_sth self.subtree = self.saved_subtree self.entries = self.saved_entries else: self.log(ERROR_STR + "Could not roll back, no saved state found!") def fetch_and_increment_subtree(self, first, last, url, subtree =[[]]): new_leafs = [] if first <= last: entries = get_entries(url, first, last)["entries"] tmp_cert_data = [] for item in entries: tmp_data = check_domain(item, url) entry_hash = get_leaf_hash(base64.b64decode(item["leaf_input"])) if tmp_data: tmp_data["leaf_hash"] = base64.b64encode(entry_hash) tmp_cert_data.append(tmp_data) new_leafs.append(entry_hash) if self.dbdir: db_add_certs(self.dbdir, tmp_cert_data) if CONFIG.DEFAULT_CERT_FILE: append_file(CONFIG.DEFAULT_CERT_FILE, tmp_cert_data) subtree = reduce_tree(new_leafs, subtree) return subtree, len(new_leafs) + first def to_dict(self): d = {} d["entries"] = self.entries d["subtree"] = encode_tree(self.subtree) d["sth"] = self.sth d["fe_ips"] = self.fe_ips return d def save(self): self.log("Saving state to file") open(self.savefile, 'w').write(json.dumps(self.to_dict())) def load(self): self.log("Loading state from file") try: f = open(self.savefile) s = f.read() d = json.loads(s) self.subtree = decode_tree(d["subtree"]) self.sth = d["sth"] self.entries = d["entries"] if "fe_ips" in d: self.fe_ips = d["fe_ips"] else: self.fe_ips = {} except IOError, e: if e.errno == errno.ENOENT: return None raise e def log(self, string): s = time_str() + " " + string with open(self.logfile, 'a') as f: f.write(s + "\n") f.close() def update_sth(self): try: new_sth, ip = get_sth_and_ip(self.url) if not ip in self.fe_ips: self.log("New Front end IP: " + ip) self.fe_ips[ip] = time_str() except Exception, e: self.log(ERROR_STR + "Failed to fetch STH and IP. " +str(e)) return try: check_sth_signature(self.url, new_sth, base64.b64decode(self.key)) except: self.log(ERROR_STR + "Could not verify STH signature " + str(new_sth)) self.rollback() # sth_time = datetime.datetime.fromtimestamp(new_sth['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S") sth_time = time_str(new_sth["timestamp"]) if new_sth["timestamp"] != self.sth["timestamp"]: self.log("STH updated. Size: " + str(new_sth["tree_size"]) + ", Time: " + sth_time) self.sth = new_sth def get_all_roots(self): result = urlopen(self.url + "ct/v1/get-roots").read() certs = json.loads(result)["certificates"] return certs def update_roots(self): try: roots = self.get_all_roots() except Exception, e: self.log(ERROR_STR + "Failed to fetch roots. " + str(e)) return new_root_hash = str(hash(str(roots))) if new_root_hash != self.root_hash: self.root_hash = new_root_hash cert_dir = CONFIG.OUTPUT_DIR + self.name + "-roots" if not os.path.exists(cert_dir): os.makedirs(cert_dir) hash_list = [] for cert in roots: h = str(hash(str(cert))) hash_list.append(h) loaded_list = os.listdir(cert_dir) added, removed = compare_lists(hash_list, loaded_list) if len(added) != 0: for item in added: root_cert = base64.decodestring(roots[hash_list.index(item)]) subject = get_cert_info(root_cert)["subject"] issuer = get_cert_info(root_cert)["issuer"] if subject == issuer: self.log("Added Root: " + item + ", " + subject) else: self.log(ERROR_STR + "Non self-signed root cert added! Subject:" + subject + " Issuer:" + issuer) fn = cert_dir + "/" + item tempname = fn + ".new" data = roots[hash_list.index(item)] open(tempname, 'w').write(data) mv_file(tempname, fn) if len(removed) != 0: for item in removed: data = open(cert_dir + "/" + item).read() root_cert = base64.decodestring(data) subject = get_cert_info(root_cert)["subject"] issuer = get_cert_info(root_cert)["issuer"] if subject == issuer: self.log("Removed Root: " + item + ", " + subject) else: self.log(ERROR_STR + "Non self-signed root cert removed! Subject:" + subject + " Issuer:" + issuer) def verify_progress(self, old): new = self.sth try: if new["tree_size"] == old["tree_size"]: if old["sha256_root_hash"] != new["sha256_root_hash"]: # print ERROR_STR + "Root hash is different for same tree size in " + self.name self.log(ERROR_STR + "New root hash for same tree size! Old:" + str(old) + " New:" + str(new)) self.rollback() elif new["tree_size"] < old["tree_size"]: # print ERROR_STR + "New tree smaller than previous tree (%d < %d) in %s" % \ # (new["tree_size"], old["tree_size"], self.name) self.log(ERROR_STR + "New tree is smaller than old tree! Old:" + str(old) + " New:" + str(new)) self.rollback() if new["timestamp"] < old["timestamp"]: self.log(ERROR_STR + "Regression in timestamps! Old:" + str(old) + " New:" + str(new)) self.rollback() # print ERROR_STR + " Regression in timestamps in " + self.name else: age = time.time() - new["timestamp"]/1000 # sth_time = datetime.datetime.fromtimestamp(new['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S") sth_time = time_str(new["timestamp"]) roothash = new['sha256_root_hash'] if age > 24 * 3600: s = ERROR_STR + "STH is older than 24h: %s UTC" % (sth_time) self.log(s + str(new)) print s elif age > 12 * 3600: s = "WARNING: STH is older than 12h: %s UTC" % (sth_time) self.log(s) elif age > 6 * 3600: s = "WARNING: STH is older than 6h: %s UTC" % (sth_time) self.log(s) except Exception, e: self.log(ERROR_STR + "Failed to verify progress! Old:" + str(old) + " New:" + str(new) + " Exception: " + str(e)) self.rollback() # print "Failed to verify progress in " + self.name def verify_consistency(self, old): new = self.sth try: if old["tree_size"]!= new["tree_size"]: consistency_proof = get_consistency_proof(self.url, old["tree_size"], new["tree_size"]) decoded_consistency_proof = [] for item in consistency_proof: decoded_consistency_proof.append(base64.b64decode(item)) res = verify_consistency_proof(decoded_consistency_proof, old["tree_size"], new["tree_size"], old["sha256_root_hash"]) if old["sha256_root_hash"] != str(base64.b64encode(res[0])): self.log(ERROR_STR + "Verification of consistency for old hash failed! Old:" \ + str(old) + " New:" + str(new) + " Proof:" + str(consistency_proof)) self.rollback() # print ERROR_STR + "Failed to verify consistency for " + self.name elif new["sha256_root_hash"] != str(base64.b64encode(res[1])): self.log(ERROR_STR + "Verification of consistency for new hash failed! Old:" \ + str(old) + " New:" + str(new) + " Proof:" + str(consistency_proof)) self.rollback() # print ERROR_STR + "Failed to verify consistency for " + self.name except Exception, e: self.log(ERROR_STR + "Could not verify consistency! " + " Old:" + str(old) + " New:" + str(new) + " Error:" + str(e)) self.rollback() # print ERROR_STR + "Could not verify consistency for " + self.url # def verify_inclusion_all(old, new): # for url in old: # try: # if old[url] and new[url]: # if old[url]["tree_size"]!= new[url]["tree_size"]: # entries = [] # while len(entries) + old[url]["tree_size"]!= new[url]["tree_size"]: # entries += get_entries(url, str(int(old[url]["tree_size"]) + len(entries)), new[url]["tree_size"] -1)["entries"] # print "Got " + str(len(entries)) + " entries..." # success = True # for i in entries: # h = get_leaf_hash(base64.b64decode(i["leaf_input"])) # if not verify_inclusion_by_hash(url, h): # success = False # if success: # print time.strftime("%H:%M:%S") + " Verifying inclusion for " + str(len(entries)) + " new entries in " + url + " ...OK" # else: # print time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url # errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url) # except: # print time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url # errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url) # def verify_inclusion_by_hash(base_url, leaf_hash): # try: # tmp_sth = get_sth(base_url) # proof = get_proof_by_hash(base_url, leaf_hash, tmp_sth["tree_size"]) # decoded_inclusion_proof = [] # for item in proof["audit_path"]: # decoded_inclusion_proof.append(base64.b64decode(item)) # root = base64.b64encode(verify_inclusion_proof(decoded_inclusion_proof, proof["leaf_index"], tmp_sth["tree_size"], leaf_hash)) # if tmp_sth["sha256_root_hash"] == root: # return True # else: # print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(proof["leaf_index"]) + " in " + base_url # return False # except: # print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for hashed entry in " + base_url # return False # def verify_inclusion_by_index(base_url, index): # try: # tmp_sth = get_sth(base_url) # proof = get_proof_by_index(base_url, index, tmp_sth["tree_size"]) # decoded_inclusion_proof = [] # for item in proof["audit_path"]: # decoded_inclusion_proof.append(base64.b64decode(item)) # root = base64.b64encode(verify_inclusion_proof(decoded_inclusion_proof, index, tmp_sth["tree_size"], get_leaf_hash(base64.b64decode(proof["leaf_input"])))) # if tmp_sth["sha256_root_hash"] == root: # print time.strftime('%H:%M:%S') + " Verifying inclusion for entry " + str(index) + " in " + base_url + "...OK." # else: # print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url # errors.append(time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url) # except: # print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url # errors.append(time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url) # def get_proof_by_index(baseurl, index, tree_size): # try: # params = urllib.urlencode({"leaf_index":index, # "tree_size":tree_size}) # result = \ # urlopen(baseurl + "ct/v1/get-entry-and-proof?" + params).read() # return json.loads(result) # except urllib2.HTTPError, e: # print "ERROR:", e.read() # sys.exit(0) def time_str(ts = None): if ts is None: return datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') else: return datetime.datetime.fromtimestamp(ts / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S") def setup_domain_monitoring(): monitored_domains = [] try: with open(CONFIG.DOMAINS_FILE) as fp: for line in fp: tmp = json.loads(line) for domain in tmp: if domain["url"] in CONFIG.MONITORED_DOMAINS: md = monitored_domain(domain["url"]) md.load_entries(domain["entries"]) monitored_domains.append(md) except IOError: pass for md in CONFIG.MONITORED_DOMAINS: tmp = monitored_domain(md) if not tmp in monitored_domains: print "New domain (not in file) " + md tmp.set() monitored_domains.append(tmp) return monitored_domains def main(args): monitored_domains = setup_domain_monitoring() # Create logs logs = [] try: # Create log objects for item in CONFIG.CTLOGS: logs.append(ctlog(item["name"], item["url"], item["key"], item["id"], item["build"])) print time_str() + " Setting up monitor for " + str(len(logs)) + " logs..." # Set up state for log in logs: if os.path.isfile(log.savefile): log.load() # Build new entries for log in logs: log.incremental_build() # Main loop: Monitor print time_str() + " Running... (see logfiles for output)" while True: time.sleep(CONFIG.INTERVAL) for log in logs: log.update_roots() old_sth = log.sth log.save_state() # Create rollback point log.update_sth() if old_sth["timestamp"] != log.sth["timestamp"]: log.verify_progress(old_sth) # Does rollback on critical fail log.verify_consistency(old_sth) # Does rollback on critical fail log.incremental_build() # Does rollback on critical fail for md in monitored_domains: md.update() # Normal exit of the program except KeyboardInterrupt: print time_str() + ' Received interrupt from user. Saving and exiting....' for log in logs: log.save() # Save info about monitored domains domain_dict = [] for md in monitored_domains: domain_dict.append(md.to_dict()) open(CONFIG.DOMAINS_FILE, 'w').write(json.dumps(domain_dict)) # Something went horribly wrong! except Exception, err: print Exception, err for log in logs: log.save() # Save info about monitored domains domain_dict = [] if len(monitored_domains) > 0: for md in monitored_domains: domain_dict.append(md.to_dict()) open(CONFIG.DOMAINS_FILE, 'w').write(json.dumps(domain_dict)) if __name__ == '__main__': if CONFIG.OUTPUT_DIR and not os.path.exists(CONFIG.OUTPUT_DIR): os.makedirs(CONFIG.OUTPUT_DIR) if CONFIG.DB_PATH and not os.path.exists(CONFIG.DB_PATH): os.makedirs(CONFIG.DB_PATH) main(args)