From fae6b9c78099fa40bcbf9ed451979cdca715fef0 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Fri, 31 Jul 2015 23:31:24 +0200 Subject: Added storage gc script --- Makefile | 3 +++ tools/certtools.py | 9 +++++++ tools/compileconfig.py | 1 + tools/storagegc.py | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+) create mode 100755 tools/storagegc.py diff --git a/Makefile b/Makefile index 4762424..e658b78 100644 --- a/Makefile +++ b/Makefile @@ -80,6 +80,7 @@ tests-run: @(cd $(INSTDIR) && python ../tools/check-sth.py --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem https://localhost:8080/) || (echo "Check failed" ; false) @(cd $(INSTDIR) && python ../tools/fetchallcerts.py $(BASEURL) --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem) || (echo "Verification failed" ; false) @(cd $(INSTDIR) && rm -f submittedcerts) + @(cd $(INSTDIR) && python ../tools/storagegc.py --config ../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-1.cfg) || (echo "GC failed" ; false) @(cd $(INSTDIR) && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/cert1.txt --check-sct --sct-file=submittedcerts $(BASEURL) --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem) || (echo "Submission failed" ; false) @(cd $(INSTDIR) && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/cert2.txt --check-sct --sct-file=submittedcerts $(BASEURL) --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem) || (echo "Submission failed" ; false) @(cd $(INSTDIR) && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/cert3.txt --check-sct --sct-file=submittedcerts $(BASEURL) --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem) || (echo "Submission failed" ; false) @@ -87,9 +88,11 @@ tests-run: @(cd $(INSTDIR) && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/cert5.txt --check-sct --sct-file=submittedcerts $(BASEURL) --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem) || (echo "Submission failed" ; false) @(cd $(INSTDIR) && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/pre1.txt --check-sct --sct-file=submittedcerts $(BASEURL) --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem) || (echo "Submission failed" ; false) @(cd $(INSTDIR) && python ../tools/submitcert.py --parallel=1 --store ../tools/testcerts/pre2.txt --check-sct --sct-file=submittedcerts $(BASEURL) --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem) || (echo "Submission failed" ; false) + @(cd $(INSTDIR) && python ../tools/storagegc.py --config ../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-1.cfg) || (echo "GC failed" ; false) @(cd $(INSTDIR) && python ../tools/merge.py --config ../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-merge.cfg) || (echo "Merge failed" ; false) @diff -r -x nursery -x verifiedsize catlfish/tests/mergedb catlfish/tests/mergedb-secondary || (echo "Merge databases not matching" ; false) @(cd $(INSTDIR) && python ../tools/check-sth.py --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem https://localhost:8080/) || (echo "Check failed" ; false) + @(cd $(INSTDIR) && python ../tools/storagegc.py --config ../test/catlfish-test.cfg --localconfig ../test/catlfish-test-local-1.cfg) || (echo "GC failed" ; false) tests-run2: @(cd $(INSTDIR) ; python ../tools/verifysct.py --sct-file=submittedcerts --parallel 1 $(BASEURL) --publickey=tests/keys/logkey.pem --cafile tests/httpsca/demoCA/cacert.pem) || echo "Verification of SCT:s failed" diff --git a/tools/certtools.py b/tools/certtools.py index ed8ab30..dfd5b24 100644 --- a/tools/certtools.py +++ b/tools/certtools.py @@ -15,6 +15,7 @@ import ecdsa import datetime import cStringIO import zipfile +import shutil from certkeys import publickeys def get_cert_info(s): @@ -734,3 +735,11 @@ def extract_original_entry(entry): leaf_cert = precert certchain = decode_certificate_chain(extra_data) return ([leaf_cert] + certchain, timestamp, issuer_key_hash) + +def mv_file(fromfn, tofn): + shutil.move(fromfn, tofn) + +def write_file(fn, sth): + tempname = fn + ".new" + open(tempname, 'w').write(json.dumps(sth)) + mv_file(tempname, fn) diff --git a/tools/compileconfig.py b/tools/compileconfig.py index 1e1fa41..d493226 100755 --- a/tools/compileconfig.py +++ b/tools/compileconfig.py @@ -237,6 +237,7 @@ def gen_config(nodename, config, localconfig): elif nodetype == "storagenodes": plopconfig += [ (Symbol("newentries_path"), paths["db"] + "newentries"), + (Symbol("lastverifiednewentry_path"), paths["db"] + "lastverifiednewentry"), ] if nodetype in ("frontendnodes", "storagenodes"): plopconfig += [ diff --git a/tools/storagegc.py b/tools/storagegc.py new file mode 100755 index 0000000..c13dcb5 --- /dev/null +++ b/tools/storagegc.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +# Copyright (c) 2014, NORDUnet A/S. +# See LICENSE for licensing information. + +import argparse +import urllib2 +import urllib +import json +import base64 +import sys +import yaml +from certtools import * + +parser = argparse.ArgumentParser(description='') +parser.add_argument('--config', help="System configuration", required=True) +parser.add_argument('--localconfig', help="Local configuration", required=True) +args = parser.parse_args() + +config = yaml.load(open(args.config)) +localconfig = yaml.load(open(args.localconfig)) + +paths = localconfig["paths"] +db_path = paths["db"] +create_ssl_context(cafile=paths["https_cacertfile"]) + +baseurl = config["baseurl"] + +sth = get_sth(baseurl) + +def verifyleafhash(leaf_hash): + try: + proof = get_proof_by_hash(baseurl, leaf_hash, sth["tree_size"]) + except SystemExit: + return False + + leaf_index = proof["leaf_index"] + inclusion_proof = [base64.b64decode(e) for e in proof["audit_path"]] + + calc_root_hash = verify_inclusion_proof(inclusion_proof, leaf_index, sth["tree_size"], leaf_hash) + + root_hash = base64.b64decode(sth["sha256_root_hash"]) + if root_hash != calc_root_hash: + print "sth calculation incorrect:" + print base64.b16encode(root_hash) + print base64.b16encode(calc_root_hash) + sys.exit(1) + + return True + +starttime = datetime.datetime.now() + +lastverified = (-1, None) + +try: + for i, line in enumerate(open(db_path + "newentries")): + leaf_hash = base64.b16decode(line.strip(), casefold=True) + result = verifyleafhash(leaf_hash) + if not result: + break + lastverified = {"index": i, "hash": base64.b16encode(leaf_hash).lower()} + if lastverified["index"] >= 0: + write_file(db_path + "lastverifiednewentry", lastverified) + print "lastverified", lastverified +except KeyboardInterrupt: + pass -- cgit v1.1