adding diff counter for history in logreader

4 files changed, 34 insertions, 17 deletions

diff --git a/monitor/josef_experimental.py b/monitor/josef_experimental.py
index eab57a1..979bae8 100755
--- a/monitor/josef_experimental.py
+++ b/monitor/josef_experimental.py
@@ -134,7 +134,9 @@ if __name__ == '__main__':
     #         get_entries(url,8,8)
     #     except Exception, e:
     #         print "Failed to get entry from " + log["name"], e
-
-    check_inclusion_by_submission(1,1,[CTLOGS[3]],[CTLOGS[3]])  
+    log = CTLOGS[9]
+    entries = get_entries(log["url"],0,5000)["entries"]
+    print log["name"], len(entries)
+    # check_inclusion_by_submission(1,1,[CTLOGS[3]],[CTLOGS[3]])  
 
 
diff --git a/monitor/josef_logreader.py b/monitor/josef_logreader.py
index 8dadc7a..cac5812 100755
--- a/monitor/josef_logreader.py
+++ b/monitor/josef_logreader.py
@@ -165,13 +165,23 @@ def write_history_data(l):
     with open(logfile, 'a') as f:
         prev_val = 0
         for item in data_list:
+            s = str(item[0])
             if item[1] > prev_val:
                 val = item[1]
             else:
                 val = prev_val
+
+            if prev_val == 0:
+                diff = 0
+            else:
+                diff = val - prev_val
+
             prev_val = val
-            # print item[0], val
-            f.write(str(item[0]) + " " + str(val) + "\n")
+
+            s += " " + str(val)
+            s += " " + str(diff)
+            s += "\n"
+            f.write(s)
         f.close()
 
 
@@ -188,9 +198,10 @@ if __name__ == "__main__":
         if log == "monitor.log":
             pass
         elif log[- len(OVERLAP_STR):] == OVERLAP_STR:
-            # print_overlap(log)
+            print_overlap(log)
             pass
         else:
+            pass
             # print log
             # print_log_stats(log)
             # print_average_age(log)
diff --git a/monitor/josef_monitor.py b/monitor/josef_monitor.py
index 75553cd..a05a898 100755
--- a/monitor/josef_monitor.py
+++ b/monitor/josef_monitor.py
@@ -402,7 +402,7 @@ def main(args):
 
                 log.save_state()                         # Create rollback point 
                 log.update_sth()
-                if old_sth["timestamp"] != log.sth["timestamp"]:
+                if old_sth and old_sth["timestamp"] != log.sth["timestamp"]:
                     log.verify_progress(old_sth)         # Does rollback on critical fail
                     log.verify_consistency(old_sth)      # Does rollback on critical fail
                     log.incremental_build()              # Does rollback on critical fail
diff --git a/monitor/josef_mover.py b/monitor/josef_mover.py
index a985f65..2267d00 100755
--- a/monitor/josef_mover.py
+++ b/monitor/josef_mover.py
@@ -128,13 +128,14 @@ def move_entry(first, last, source, dest):
 
 
 def check_overlap(source, dests):
-    PROCESS_COUNT = 100
+    PROCESS_COUNT = 50
 
     for log in source:
         print "Checking overlap from " + log["name"]
         sth = get_sth(log["url"])
         first = 0
-        last = int(sth["tree_size"])
+        # last = int(sth["tree_size"])
+        last = 23000
 
         # split into tasks 
         processes = []
@@ -155,6 +156,7 @@ def check_overlap(source, dests):
                 args=(tmp_start, tmp_last, [log], dests, filename))
             p.start()
             processes.append(p)
+            time.sleep(1.3)
 
         for p in processes:
             p.join()
@@ -192,8 +194,8 @@ def check_submission_inner(d_log, item, inclusions):
         inclusions.append(d_log["name"])
 
 def check_submission_outer(first, last, s_log, dest, logfile=None):
-    MAX_CHUNK_SIZE = 50
-    MAX_RETRIES = 2
+    MAX_CHUNK_SIZE = 65
+    MAX_RETRIES = 3
     idx = 0
     while first + idx < last:
         if first + idx + MAX_CHUNK_SIZE < last:
@@ -204,15 +206,16 @@ def check_submission_outer(first, last, s_log, dest, logfile=None):
         retries = 0
         while retries <= MAX_RETRIES:
             try:
-                # print "Getting " + str(first + idx) + " to " + str(tmp_last)
+                print "Getting " + str(first + idx) + " to " + str(tmp_last)
                 entries = get_entries(s_log["url"], first + idx, tmp_last)["entries"]
                 break
             except Exception, e:
                 time.sleep(1)
-                # if retries == MAX_RETRIES:
-                #     print "FAILED!", Exception, e
-                # else:
-                #     print "Failed, retrying...(" + str(retries) + ")"
+                if retries == MAX_RETRIES:
+                    print "ERROR, could not download entries "  + str(first + idx) + " to " + str(tmp_last), Exception, e
+                    raise Exception("Failed to get entries")
+                else:
+                    print "Failed, retrying...(" + str(retries) + ") ", Exception, e
                 retries += 1   
 
 
@@ -242,7 +245,7 @@ def check_submission_outer(first, last, s_log, dest, logfile=None):
         idx += len(entries)
 
 def check_stress_submission(first, last, source, dest, filename=None):
-    MAX_RETRIES = 2
+    MAX_RETRIES = 4
 
     for s_log in source:
         retries = 0
@@ -258,6 +261,7 @@ def check_stress_submission(first, last, source, dest, filename=None):
                     print "FAILED!", Exception, e
                 else:
                     print "Failed, retrying...(" + str(retries) + ")"
+                    pass
                 retries += 1 
 
   
@@ -292,7 +296,7 @@ def log(fn, string):
 
 
 if __name__ == "__main__":
-    source = [CTLOGS[2]]
+    source = [CTLOGS[4]]
     dests = CTLOGS
 
     # stress_test([CTLOGS[0]])