Merge remote-tracking branch 'refs/remotes/map/robust-distribution'

1 files changed, 41 insertions, 14 deletions

diff --git a/tools/mergetools.py b/tools/mergetools.py
index 80fbf0b..94901a9 100644
--- a/tools/mergetools.py
+++ b/tools/mergetools.py
@@ -179,13 +179,13 @@ def get_new_entries(node, baseurl, own_key, paths):
         print >>sys.stderr, "ERROR: fetchnewentries", e.response
         sys.exit(1)
 
-def get_entries(node, baseurl, own_key, paths, hashes):
+def get_entries(node, baseurl, own_key, paths, hashes, session=None):
     try:
         params = {"hash":[base64.b64encode(ehash) for ehash in hashes]}
         result = http_request(baseurl + "plop/v1/storage/getentry",
                               params=params,
                               key=own_key, verifynode=node,
-                              publickeydir=paths["publickeys"])
+                              publickeydir=paths["publickeys"], session=session)
         parsed_result = json.loads(result)
         if parsed_result.get(u"result") == u"ok":
             entries = dict([(base64.b64decode(entry["hash"]),
@@ -214,6 +214,25 @@ def get_curpos(node, baseurl, own_key, paths):
         print >>sys.stderr, "ERROR: currentposition", e.response
         sys.exit(1)
 
+def get_frontend_verifiedsize(node, baseurl, own_key, paths):
+    return frontend_verify_entries(node, baseurl, own_key, paths, 0)
+
+def frontend_verify_entries(node, baseurl, own_key, paths, size):
+    try:
+        arguments = {"verify_to": size}
+        result = http_request(baseurl + "plop/v1/frontend/verify-entries",
+                              json.dumps(arguments),
+                              key=own_key, verifynode=node,
+                              publickeydir=paths["publickeys"])
+        parsed_result = json.loads(result)
+        if parsed_result.get(u"result") == u"ok":
+            return parsed_result[u"verified"]
+        print >>sys.stderr, "ERROR: verify-entries", parsed_result
+        sys.exit(1)
+    except requests.exceptions.HTTPError, e:
+        print >>sys.stderr, "ERROR: verify-entries", e.response
+        sys.exit(1)
+
 def get_verifiedsize(node, baseurl, own_key, paths):
     try:
         result = http_request(baseurl + "plop/v1/merge/verifiedsize",
@@ -258,6 +277,10 @@ def backup_sendlog(node, baseurl, own_key, paths, submission):
         print >>sys.stderr, "ERROR: backup_sendlog", e.response
         sys.stderr.flush()
         return None
+    except requests.packages.urllib3.exceptions.NewConnectionError, e:
+        print >>sys.stderr, "ERROR: backup_sendlog new connection error"
+        sys.stderr.flush()
+        return None
     except ValueError, e:
         print >>sys.stderr, "==== FAILED REQUEST ===="
         print >>sys.stderr, submission
@@ -267,16 +290,17 @@ def backup_sendlog(node, baseurl, own_key, paths, submission):
         sys.stderr.flush()
         raise e
 
-def sendentry(node, baseurl, own_key, paths, entry, ehash):
+def sendentries(node, baseurl, own_key, paths, entries, session=None):
     try:
+        json_entries = [{"entry":base64.b64encode(entry), "treeleafhash":base64.b64encode(hash)} for hash, entry in entries]
         result = http_request(
             baseurl + "plop/v1/frontend/sendentry",
-            json.dumps({"entry":base64.b64encode(entry),
-                        "treeleafhash":base64.b64encode(ehash)}),
-            key=own_key, verifynode=node, publickeydir=paths["publickeys"])
+            json.dumps(json_entries),
+            key=own_key, verifynode=node, publickeydir=paths["publickeys"],
+            session=session)
         return json.loads(result)
     except requests.exceptions.HTTPError, e:
-        print >>sys.stderr, "ERROR: sendentry", e.reponse
+        print >>sys.stderr, "ERROR: sendentries", e.response
         sys.exit(1)
     except ValueError, e:
         print >>sys.stderr, "==== FAILED REQUEST ===="
@@ -286,9 +310,9 @@ def sendentry(node, baseurl, own_key, paths, entry, ehash):
         print >>sys.stderr, "========================"
         sys.stderr.flush()
         raise e
-
-def sendentry_merge(node, baseurl, own_key, paths, entry, ehash):
-    return sendentries_merge(node, baseurl, own_key, paths, [(ehash, entry)])
+    except requests.exceptions.ConnectionError, e:
+        print >>sys.stderr, "ERROR: sendentries", baseurl, e.request, e.response
+        sys.exit(1)
 
 def sendentries_merge(node, baseurl, own_key, paths, entries, session=None):
     try:
@@ -300,7 +324,7 @@ def sendentries_merge(node, baseurl, own_key, paths, entries, session=None):
             session=session)
         return json.loads(result)
     except requests.exceptions.HTTPError, e:
-        print >>sys.stderr, "ERROR: sendentry_merge", e.response
+        print >>sys.stderr, "ERROR: sendentries_merge", e.response
         sys.exit(1)
     except ValueError, e:
         print >>sys.stderr, "==== FAILED REQUEST ===="
@@ -310,15 +334,18 @@ def sendentries_merge(node, baseurl, own_key, paths, entries, session=None):
         print >>sys.stderr, "========================"
         sys.stderr.flush()
         raise e
+    except requests.exceptions.ConnectionError, e:
+        print >>sys.stderr, "ERROR: sendentries_merge", baseurl, e.request, e.response
+        sys.exit(1)
 
-def sendsth(node, baseurl, own_key, paths, submission):
+def publish_sth(node, baseurl, own_key, paths, submission):
     try:
-        result = http_request(baseurl + "plop/v1/frontend/sendsth",
+        result = http_request(baseurl + "plop/v1/frontend/publish-sth",
                               json.dumps(submission), key=own_key,
                               verifynode=node, publickeydir=paths["publickeys"])
         return json.loads(result)
     except requests.exceptions.HTTPError, e:
-        print >>sys.stderr, "ERROR: sendsth", e.response
+        print >>sys.stderr, "ERROR: publish-sth", e.response
         sys.exit(1)
     except ValueError, e:
         print >>sys.stderr, "==== FAILED REQUEST ===="