diff options
author | josef <josef.gson@gmail.com> | 2015-12-08 10:05:29 +0100 |
---|---|---|
committer | josef <josef.gson@gmail.com> | 2015-12-08 10:05:29 +0100 |
commit | 1cada068124a50b319ed67177b6c5b151948c256 (patch) | |
tree | fcf4db84db6549ed8184066ed63ef9e352fe1758 | |
parent | 934cbe0afe1bf1838bb32250f22057572d71abfe (diff) |
adding gaol
-rw-r--r-- | monitor/gaol/gaol.ct.nordu.net.pem | 4 | ||||
-rwxr-xr-x | monitor/gaol/gaol_auditor.py | 220 | ||||
-rw-r--r-- | monitor/gaol/gaol_conf.py | 29 | ||||
-rwxr-xr-x | monitor/gaol/gaol_lib.py | 66 | ||||
-rw-r--r-- | monitor/gaol/gaol_sct.txt | 17 | ||||
-rwxr-xr-x | monitor/gaol/gaol_test.py | 33 | ||||
-rwxr-xr-x | monitor/josef_experimental.py | 6 |
7 files changed, 372 insertions, 3 deletions
diff --git a/monitor/gaol/gaol.ct.nordu.net.pem b/monitor/gaol/gaol.ct.nordu.net.pem new file mode 100644 index 0000000..c75c001 --- /dev/null +++ b/monitor/gaol/gaol.ct.nordu.net.pem @@ -0,0 +1,4 @@ +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEE0E7UyZA2XkTJC2Rlcx0DbXkI1Y0 ++2OgoJQ0O+pwtVRCiVGHdU08i4m8MKx2r8FWbpHVgt6V0rLS8rYBErfSVA== +-----END PUBLIC KEY----- diff --git a/monitor/gaol/gaol_auditor.py b/monitor/gaol/gaol_auditor.py new file mode 100755 index 0000000..d2da59b --- /dev/null +++ b/monitor/gaol/gaol_auditor.py @@ -0,0 +1,220 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + + +import time +import datetime +import base64 +import argparse +import errno +import subprocess + +import json + +from gaol_lib import * +from lib import * +import os.path + + +parser = argparse.ArgumentParser(description="") +parser.add_argument('--config', default="gaol_conf.py") +args = parser.parse_args() + +# Import from config file +if os.path.isfile(args.config): + modules = map(__import__, [args.config[:-2]]) + CONFIG = modules[0] + ERROR_STR = CONFIG.ERROR_STR +else: + print "Config file not found!" + ERROR_STR = "(local)ERROR: " + sys.exit() + +def email(s): + for addr in CONFIG.EMAIL_ADDR: + p = subprocess.Popen( + ["mail", "-s", CONFIG.EMAIL_SUBJECT, addr], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + p.communicate(s) + + +class ctlog: + def __init__(self, name, url, key, log_id=None, build=True): + self.name = name + self.url = url + self.key = key + self.log_id = log_id + self.logfile = CONFIG.OUTPUT_DIR + name + ".log" + self.savefile = CONFIG.OUTPUT_DIR + name + "-state-info.json" + self.subtree = [[]] + # self.fe_ips = {} + self.sth = None + self.entries = 0 + self.root_hash = None + self.build = build + + self.saved_sth = None + self.saved_entries = None + self.saved_subtree = None + + self.log("Starting monitor") + + + + def to_dict(self): + d = {} + d["sth"] = self.sth + return d + + def save(self): + self.log("Saving state to file") + open(self.savefile, 'w').write(json.dumps(self.to_dict())) + + def load(self): + self.log("Loading state from file") + try: + f = open(self.savefile) + s = f.read() + d = json.loads(s) + self.sth = d["sth"] + + except IOError, e: + if e.errno == errno.ENOENT: + return None + raise e + + + def log(self, string): + s = time_str() + " " + string + with open(self.logfile, 'a') as f: + f.write(s + "\n") + f.close() + email(s) + + def update_sth(self): + try: + new_sth = get_sth(self.url) + except Exception, e: + self.log(ERROR_STR + "Failed to fetch STH. " +str(e)) + return + + try: + check_sth_signature(self.url, new_sth, base64.b64decode(self.key)) + except: + self.log(ERROR_STR + "Could not verify STH signature " + str(new_sth)) + + if self.sth: + sth_time = time_str(new_sth["timestamp"]) + if new_sth["timestamp"] != self.sth["timestamp"]: + self.log("STH updated. Size: " + str(new_sth["tree_size"]) + ", Time: " + sth_time) + self.sth = new_sth + else: + self.log("Setting initial STH: " + str(new_sth)) + self.sth = new_sth + + + def verify_progress(self, old): + new = self.sth + try: + if new["tree_size"] == old["tree_size"]: + if old["sha256_root_hash"] != new["sha256_root_hash"]: + self.log(ERROR_STR + "New root hash for same tree size! Old:" + str(old) + " New:" + str(new)) + elif new["tree_size"] < old["tree_size"]: + self.log(ERROR_STR + "New tree is smaller than old tree! Old:" + str(old) + " New:" + str(new)) + + if new["timestamp"] < old["timestamp"]: + self.log(ERROR_STR + "Regression in timestamps! Old:" + str(old) + " New:" + str(new)) + else: + age = time.time() - new["timestamp"]/1000 + sth_time = time_str(new["timestamp"]) + roothash = new['sha256_root_hash'] + if age > 24 * 3600: + s = ERROR_STR + "STH is older than 24h: %s UTC" % (sth_time) + self.log(s + str(new)) + print s + elif age > 12 * 3600: + s = "WARNING: STH is older than 12h: %s UTC" % (sth_time) + self.log(s) + elif age > 6 * 3600: + s = "WARNING: STH is older than 6h: %s UTC" % (sth_time) + self.log(s) + except Exception, e: + self.log(ERROR_STR + "Failed to verify progress! Old:" + str(old) + " New:" + str(new) + " Exception: " + str(e)) + + def verify_consistency(self, old): + new = self.sth + try: + if old["tree_size"]!= new["tree_size"]: + consistency_proof = get_consistency_proof(self.url, old["tree_size"], new["tree_size"]) + decoded_consistency_proof = [] + for item in consistency_proof: + decoded_consistency_proof.append(base64.b64decode(item)) + res = verify_consistency_proof(decoded_consistency_proof, old["tree_size"], new["tree_size"], base64.b64decode(old["sha256_root_hash"])) + + if old["sha256_root_hash"] != str(base64.b64encode(res[0])): + self.log(ERROR_STR + "Verification of consistency for old hash failed! Old:" \ + + str(old) + " New:" + str(new) + " Calculated:" + str(base64.b64encode(res[0]))\ + + " Proof:" + str(consistency_proof)) + elif new["sha256_root_hash"] != str(base64.b64encode(res[1])): + self.log(ERROR_STR + "Verification of consistency for new hash failed! Old:" \ + + str(old) + " New:" + str(new) + " Proof:" + str(consistency_proof)) + + except Exception, e: + self.log(ERROR_STR + "Could not verify consistency! " + " Old:" + str(old) + " New:" + str(new) + " Error:" + str(e)) + + + +def main(args): + # Create logs + logs = [] + try: + # Create log objects + for item in CONFIG.CTLOGS: + logs.append(ctlog(item["name"], item["url"], item["key"], item["id"], item["build"])) + print time_str() + " Setting up monitor for " + str(len(logs)) + " logs..." + + # Set up state + for log in logs: + if os.path.isfile(log.savefile): + log.load() + + + # Main loop: Auditor + print time_str() + " Running... (see logfiles for output)" + while True: + for log in logs: + old_sth = log.sth + + log.update_sth() + if old_sth and old_sth["timestamp"] != log.sth["timestamp"]: + log.verify_progress(old_sth) + log.verify_consistency(old_sth) # Does rollback on critical fail + pass + time.sleep(CONFIG.INTERVAL) + + # Normal exit of the program + except KeyboardInterrupt: + print time_str() + ' Received interrupt from user. Saving and exiting....' + for log in logs: + log.save() + + # Something went horribly wrong! + except Exception, err: + print Exception, err + for log in logs: + log.save() + + + +if __name__ == '__main__': + if CONFIG.OUTPUT_DIR and not os.path.exists(CONFIG.OUTPUT_DIR): + os.makedirs(CONFIG.OUTPUT_DIR) + + main(args) + + + + + + diff --git a/monitor/gaol/gaol_conf.py b/monitor/gaol/gaol_conf.py new file mode 100644 index 0000000..7019989 --- /dev/null +++ b/monitor/gaol/gaol_conf.py @@ -0,0 +1,29 @@ + # All configuration for the CT monitor is done from this file! + +# interval (in seconds) between updates +INTERVAL = 60 + +# Directories for various output files +OUTPUT_DIR = "output/" + +# Some strings +ERROR_STR = "ERROR: " +EMAIL_SUBJECT = "GAOL" + +# Email addresses for error messages +EMAIL_ADDR = [ +"josef@nordu.net", +] + +# CT logs and associated keys +CTLOGS = [ + {"name" : "gaol", + "url" : "https://gaol.ct.nordu.net/open/", + "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEE0E7UyZA2XkTJC2Rlcx0DbXkI1Y0+2OgoJQ0O+pwtVRCiVGHdU08i4m8MKx2r8FWbpHVgt6V0rLS8rYBErfSVA==", + "id" : "5WUP4zAPa2LwNM1oepj+x7cM8LlqB4fstdOMzzj4MzM=", + "build" : False}, +] + + + + diff --git a/monitor/gaol/gaol_lib.py b/monitor/gaol/gaol_lib.py new file mode 100755 index 0000000..07e674b --- /dev/null +++ b/monitor/gaol/gaol_lib.py @@ -0,0 +1,66 @@ +import json +import urllib +import urllib2 +import ssl +import base64 + +from lib import * + +class sslparameters: + sslcontext = None + +def get_opener(): + try: + opener = urllib2.build_opener(urllib2.HTTPSHandler(context=sslparameters.sslcontext)) + except TypeError: + opener = urllib2.build_opener(urllib2.HTTPSHandler()) + return opener + +def urlopen(url, data=None): + return get_opener().open(url, data) + +def get_sth(baseurl): + result = urlopen(baseurl + "gaol/v1/get-sth").read() + return json.loads(result) + +def get_entries(baseurl, start, end): + params = urllib.urlencode({"start":start, "end":end}) + # try: + result = urlopen(baseurl + "gaol/v1/get-entries?" + params).read() + return json.loads(result) + +def get_consistency_proof(baseurl, tree_size1, tree_size2): + # try: + params = urllib.urlencode({"first":tree_size1, + "second":tree_size2}) + result = \ + urlopen(baseurl + "gaol/v1/get-sth-consistency?" + params).read() + return json.loads(result)["consistency"] + # except urllib2.HTTPError, e: + # print "ERROR:", e.read() + # sys.exit(1) + +def extract_original_entry(entry): + leaf_input = base64.decodestring(entry["leaf_input"]) + (data_blob, timestamp, issuer_key_hash) = unpack_mtl(leaf_input) + return (data_blob, timestamp) + +def make_blob(data): + return base64.b64encode(data) + +def add_blob(baseurl, blob): + try: + result = urlopen(baseurl + "gaol/v1/add-blob", json.dumps({"blob" : blob})).read() + return json.loads(result) + except urllib2.HTTPError, e: + return "ERROR " + str(e.code) + " : " + e.read() + # if e.code == 400: + return None + # sys.exit(1) + except ValueError, e: + print "==== FAILED REQUEST ====" + print submission + print "======= RESPONSE =======" + print result + print "========================" + raise e
\ No newline at end of file diff --git a/monitor/gaol/gaol_sct.txt b/monitor/gaol/gaol_sct.txt new file mode 100644 index 0000000..569a7b2 --- /dev/null +++ b/monitor/gaol/gaol_sct.txt @@ -0,0 +1,17 @@ +{u'timestamp': 1448874793847, u'signature': u'BAMASDBGAiEAhlobK0j+9Ya8ssxJ7Fbhn0MP9gXW1YihNKRyhkwGak0CIQCLpQ6bIRI+7l0E7umLCPIkW0e79S3g5jYo9iNNVBSrzA==', u'sct_version': 0, u'id': u'5WUP4zAPa2LwNM1oepj+x7cM8LlqB4fstdOMzzj4MzM=', u'extensions': u''} + +{u'timestamp': 1448876147795, u'signature': u'BAMARjBEAiB83TwGbzXwgoE0xR3/LYlADdvEexqciFne1Rh68B2wWQIgGoROCsKWHGrRjfeOVHzPutJlUQ3ahfKfMO2ffa+zLuc=', u'sct_version': 0, u'id': u'5WUP4zAPa2LwNM1oepj+x7cM8LlqB4fstdOMzzj4MzM=', u'extensions': u''} + +{"sth": {"timestamp": 1448639930117, "sha256_root_hash": "FliDqU/S5o+qMPS42nwYy9b1/kKEldeedad/5vDl+UA=", "tree_size": 2, "tree_head_signature": "BAMARzBFAiBEOSbEdnzBGeUV+AGzJUs/ekf5Sq/SRbrxxiDKN23yNAIhAOTrwhYFHg3a6iLrGoz0QoIjOW8PckRJ/1oayj3993Cl"}} + +{"sth": {"timestamp": 1448878465713, "sha256_root_hash": "uaFzynY1Y6csM1ZeJfL73C+pqTt3NOI2kMXwEZxe7B0=", "tree_size": 4, "tree_head_signature": "BAMARzBFAiEAiUfFg4YCjjAb5bkJz0FfqQUYT/noY+6Y1b+hjvbySGoCIH0ItuAWS5LPQjhAKTXAJtu+ODxXNQmgcUu/bNfhV01r"}} + +{"sth" : {"timestamp":1448893523957,"sha256_root_hash":"APf+TzOLNooNAF0Ka59C9n+dLRccMKZ/S5TNKULuolM=","tree_size":5,"tree_head_signature":"BAMARjBEAiAmmxUE3xC1HePGEZvUaubi0WlzwhW+SiTu5hvvxquLhQIgAoRms8ogotonyuoFHZaxWxJMkh9d9l+ZsJ/Jee2VTTo="}} + +{u'timestamp': 1448886901898, u'signature': u'BAMARTBDAh9nairFQko6KHZGpS+X4PmCwTUmUP1MpAOSphmA+Zf+AiBW5Ok+y1aDCNqKfmRyaY/sBSMWDI8NNdE56ouRvR07BQ==', u'sct_version': 0, u'id': u'5WUP4zAPa2LwNM1oepj+x7cM8LlqB4fstdOMzzj4MzM=', u'extensions': u''} + + +{"sth": {"timestamp": 1447820379698, "sha256_root_hash": "adJNwPSYP0XOadNQEVF1nZSVZ+ojN9ORPXEdjRMhC8U=", "tree_size": 9867417, "tree_head_signature": "BAMARzBFAiEAwa1SAOMb5k/QQTDQvTPqDkGi43esqg04Em1MHJ6tpckCIBedSUATxgSB3AfIxq/UlP5KtlfuaMGRZYcbUIF0UWyC"}} + +{u'timestamp': 1448895405459, u'signature': u'BAMARzBFAiEA5BtisYNDhFY3GmNCi+KPrXhi2yHc4uETnthF6QGJ44UCICDYBNrcwgSjKt2BSVFn7Jn8G9StLo3NrsXLUKYQZFoz', u'sct_version': 0, u'id': u'5WUP4zAPa2LwNM1oepj+x7cM8LlqB4fstdOMzzj4MzM=', u'extensions': u''} + diff --git a/monitor/gaol/gaol_test.py b/monitor/gaol/gaol_test.py new file mode 100755 index 0000000..dbcadc4 --- /dev/null +++ b/monitor/gaol/gaol_test.py @@ -0,0 +1,33 @@ +#!/usr/bin/python + +import base64 + +from gaol_conf import * +from gaol_lib import * + +def test_get_entry(url, idx): + print "Testing to fetch an entry..." + entry = get_entries(url,idx,idx)["entries"][0] + print "Received: " + extract_original_entry(entry)[0] + +def test_submission(url, data): + print "\nTesting to submitt a sample text..." + blob = make_blob(data) + res = add_blob(url,blob) + print res + +def test_consistency_proof(url, idx1, idx2): + print "\nTesing a consistency proof" + res = get_consistency_proof(url, idx1, idx2) + print res + +if __name__ == '__main__': + url = CTLOGS[0]["url"] + test_get_entry(url,1) + test_get_entry(url,2) + test_get_entry(url,3) + test_get_entry(url,4) + test_get_entry(url,5) + # test_get_entry(url,6) + # test_submission(url, "Progress (n.): The process through which the Internet has evolved from smart people in front of dumb terminals to dumb people in front of smart terminals.") + # test_consistency_proof(url, 2, 3)
\ No newline at end of file diff --git a/monitor/josef_experimental.py b/monitor/josef_experimental.py index 5939b82..b3acf1c 100755 --- a/monitor/josef_experimental.py +++ b/monitor/josef_experimental.py @@ -175,11 +175,11 @@ def email(s): # command = 'echo "' + s + '" | mail -s "' + EMAIL_SUBJECT + '" ' + addr # os.system("bash -c '" + command + "'") p = subprocess.Popen( - ["mail", "-s", '"' + EMAIL_SUBJECT + '"', addr], + ["mail", "-s", EMAIL_SUBJECT, addr], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - parsed = p.communicate(s) - print parsed + p.communicate(s) + # print parsed if __name__ == '__main__': |