bugfix: fetching more than chunksize entris when verifying inclusion

2 files changed, 13 insertions, 23 deletions

diff --git a/tools/josef_experimental_auditor.py b/tools/josef_experimental_auditor.py
index 135bb46..120f4e6 100755
--- a/tools/josef_experimental_auditor.py
+++ b/tools/josef_experimental_auditor.py
@@ -171,7 +171,7 @@ def verify_progress(old, new):
 
 def verify_consistency(old, new):
     for url in old:
-        # try:
+        try:
             if old[url] and new[url] and old[url]["tree_size"]!= new[url]["tree_size"]:
                 consistency_proof = get_consistency_proof(url, old[url]["tree_size"], new[url]["tree_size"])
                 decoded_consistency_proof = []
@@ -189,15 +189,20 @@ def verify_consistency(old, new):
                     print time.strftime("%H:%M:%S") + " New STH from " + url + ", timestamp: " + \
                         str(new[url]["timestamp"]) + ", size: " + str(new[url]["tree_size"]) + "...OK."
 
-        # except:
-        #   print "ERROR: Could not verify consistency for " + url
+        except:
+          print "ERROR: Could not verify consistency for " + url
 
 def verify_inclusion_all(old, new):
     for url in old:
         try:
             if old[url] and new[url]:
                 if old[url]["tree_size"]!= new[url]["tree_size"]:
-                    entries = get_entries(url, old[url]["tree_size"], new[url]["tree_size"] -1)["entries"]
+                    entries = []
+
+                    while len(entries) + old[url]["tree_size"]!= new[url]["tree_size"]:
+                        entries = get_entries(url, str(int(old[url]["tree_size"]) + len(entries)), new[url]["tree_size"] -1)["entries"]
+                        print "Got " + str(len(entries)) + " entries..."
+
                     success = True
                     for i in entries:
                         h = get_leaf_hash(base64.b64decode(i["leaf_input"]))
diff --git a/tools/josef_nagios_auditor.py b/tools/josef_nagios_auditor.py
index 74cc522..e950c8b 100755
--- a/tools/josef_nagios_auditor.py
+++ b/tools/josef_nagios_auditor.py
@@ -106,24 +106,6 @@ def fetch_all_sth():
             errors.append(error_str)
             continue
 
-        # Add timing info 
-        # try:
-        #     if base_url not in timings:
-        #         timings[base_url] = {"last":sths[base_url]["timestamp"], "longest":0}
-        #     else:
-        #         then = datetime.datetime.fromtimestamp(int(timings[base_url]["last"])/1000)
-        #         now = datetime.datetime.fromtimestamp(int(sths[base_url]["timestamp"])/1000)
-        #         tdelta = now - then
-
-        #         timings[base_url]["last"] = sths[base_url]["timestamp"]
-
-        #         if tdelta.total_seconds() > timings[base_url]["longest"]:
-        #             timings[base_url]["longest"] = tdelta.total_seconds()
-
-        # except Exception, err:
-        #     print Exception, err
-        #     print time.strftime('%H:%M:%S') + "ERROR: Failed to set TIME info for STH"
-
     return sths
 
 def verify_progress(url, old, new):
@@ -181,7 +163,10 @@ def verify_inclusion_all(url, old, new):
     if old and new:
         try:
             if old["tree_size"]!= new["tree_size"]:
-                entries = get_entries(url, old["tree_size"], new["tree_size"] -1)["entries"]
+                entries = []
+                while len(entries) + old[url]["tree_size"]!= new[url]["tree_size"]:
+                    entries = get_entries(url, str(int(old[url]["tree_size"]) + len(entries)), new[url]["tree_size"] -1)["entries"]
+
                 success = True
                 for i in entries:
                     h = get_leaf_hash(base64.b64decode(i["leaf_input"]))