Merge remote-tracking branch 'refs/remotes/map/storagegc'

Conflicts:
	Makefile

1 files changed, 66 insertions, 0 deletions

diff --git a/tools/storagegc.py b/tools/storagegc.py
new file mode 100755
index 0000000..c13dcb5
--- /dev/null
+++ b/tools/storagegc.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2014, NORDUnet A/S.
+# See LICENSE for licensing information.
+
+import argparse
+import urllib2
+import urllib
+import json
+import base64
+import sys
+import yaml
+from certtools import *
+
+parser = argparse.ArgumentParser(description='')
+parser.add_argument('--config', help="System configuration", required=True)
+parser.add_argument('--localconfig', help="Local configuration", required=True)
+args = parser.parse_args()
+
+config = yaml.load(open(args.config))
+localconfig = yaml.load(open(args.localconfig))
+
+paths = localconfig["paths"]
+db_path = paths["db"]
+create_ssl_context(cafile=paths["https_cacertfile"])
+
+baseurl = config["baseurl"]
+
+sth = get_sth(baseurl)
+
+def verifyleafhash(leaf_hash):
+    try:
+        proof = get_proof_by_hash(baseurl, leaf_hash, sth["tree_size"])
+    except SystemExit:
+        return False
+
+    leaf_index = proof["leaf_index"]
+    inclusion_proof = [base64.b64decode(e) for e in proof["audit_path"]]
+
+    calc_root_hash = verify_inclusion_proof(inclusion_proof, leaf_index, sth["tree_size"], leaf_hash)
+
+    root_hash = base64.b64decode(sth["sha256_root_hash"])
+    if root_hash != calc_root_hash:
+        print "sth calculation incorrect:"
+        print base64.b16encode(root_hash)
+        print base64.b16encode(calc_root_hash)
+        sys.exit(1)
+
+    return True
+
+starttime = datetime.datetime.now()
+
+lastverified = (-1, None)
+
+try:
+    for i, line in enumerate(open(db_path + "newentries")):
+        leaf_hash = base64.b16decode(line.strip(), casefold=True)
+        result = verifyleafhash(leaf_hash)
+        if not result:
+            break
+        lastverified = {"index": i, "hash": base64.b16encode(leaf_hash).lower()}
+    if lastverified["index"] >= 0:
+        write_file(db_path + "lastverifiednewentry", lastverified)
+    print "lastverified", lastverified
+except KeyboardInterrupt:
+    pass