From b4dc2c37be0ed51b629e91f4d3e1dfc44cb192c0 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Wed, 2 Nov 2016 03:34:35 +0100 Subject: Send list of entries to frontend Handle connection error --- tools/mergetools.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) (limited to 'tools/mergetools.py') diff --git a/tools/mergetools.py b/tools/mergetools.py index 80fbf0b..bea09e9 100644 --- a/tools/mergetools.py +++ b/tools/mergetools.py @@ -258,6 +258,10 @@ def backup_sendlog(node, baseurl, own_key, paths, submission): print >>sys.stderr, "ERROR: backup_sendlog", e.response sys.stderr.flush() return None + except requests.packages.urllib3.exceptions.NewConnectionError, e: + print >>sys.stderr, "ERROR: backup_sendlog new connection error" + sys.stderr.flush() + return None except ValueError, e: print >>sys.stderr, "==== FAILED REQUEST ====" print >>sys.stderr, submission @@ -267,16 +271,17 @@ def backup_sendlog(node, baseurl, own_key, paths, submission): sys.stderr.flush() raise e -def sendentry(node, baseurl, own_key, paths, entry, ehash): +def sendentries(node, baseurl, own_key, paths, entries, session=None): try: + json_entries = [{"entry":base64.b64encode(entry), "treeleafhash":base64.b64encode(hash)} for hash, entry in entries] result = http_request( baseurl + "plop/v1/frontend/sendentry", - json.dumps({"entry":base64.b64encode(entry), - "treeleafhash":base64.b64encode(ehash)}), - key=own_key, verifynode=node, publickeydir=paths["publickeys"]) + json.dumps(json_entries), + key=own_key, verifynode=node, publickeydir=paths["publickeys"], + session=session) return json.loads(result) except requests.exceptions.HTTPError, e: - print >>sys.stderr, "ERROR: sendentry", e.reponse + print >>sys.stderr, "ERROR: sendentries", e.response sys.exit(1) except ValueError, e: print >>sys.stderr, "==== FAILED REQUEST ====" @@ -286,9 +291,9 @@ def sendentry(node, baseurl, own_key, paths, entry, ehash): print >>sys.stderr, "========================" sys.stderr.flush() raise e - -def sendentry_merge(node, baseurl, own_key, paths, entry, ehash): - return sendentries_merge(node, baseurl, own_key, paths, [(ehash, entry)]) + except requests.exceptions.ConnectionError, e: + print >>sys.stderr, "ERROR: sendentries", baseurl, e.request, e.response + sys.exit(1) def sendentries_merge(node, baseurl, own_key, paths, entries, session=None): try: @@ -300,7 +305,7 @@ def sendentries_merge(node, baseurl, own_key, paths, entries, session=None): session=session) return json.loads(result) except requests.exceptions.HTTPError, e: - print >>sys.stderr, "ERROR: sendentry_merge", e.response + print >>sys.stderr, "ERROR: sendentries_merge", e.response sys.exit(1) except ValueError, e: print >>sys.stderr, "==== FAILED REQUEST ====" @@ -310,6 +315,9 @@ def sendentries_merge(node, baseurl, own_key, paths, entries, session=None): print >>sys.stderr, "========================" sys.stderr.flush() raise e + except requests.exceptions.ConnectionError, e: + print >>sys.stderr, "ERROR: sendentries_merge", baseurl, e.request, e.response + sys.exit(1) def sendsth(node, baseurl, own_key, paths, submission): try: -- cgit v1.1 From b48c1689e36bdcc65a34b4ab12763478b072a716 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Wed, 2 Nov 2016 13:10:37 +0100 Subject: Change algorithm for merge backup and merge dist --- tools/mergetools.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) (limited to 'tools/mergetools.py') diff --git a/tools/mergetools.py b/tools/mergetools.py index bea09e9..ff3d08c 100644 --- a/tools/mergetools.py +++ b/tools/mergetools.py @@ -214,6 +214,25 @@ def get_curpos(node, baseurl, own_key, paths): print >>sys.stderr, "ERROR: currentposition", e.response sys.exit(1) +def get_frontend_verifiedsize(node, baseurl, own_key, paths): + return frontend_verify_entries(node, baseurl, own_key, paths, 0) + +def frontend_verify_entries(node, baseurl, own_key, paths, size): + try: + arguments = {"verify_to": size} + result = http_request(baseurl + "plop/v1/frontend/verify-entries", + json.dumps(arguments), + key=own_key, verifynode=node, + publickeydir=paths["publickeys"]) + parsed_result = json.loads(result) + if parsed_result.get(u"result") == u"ok": + return parsed_result[u"verified"] + print >>sys.stderr, "ERROR: verify-entries", parsed_result + sys.exit(1) + except requests.exceptions.HTTPError, e: + print >>sys.stderr, "ERROR: verify-entries", e.response + sys.exit(1) + def get_verifiedsize(node, baseurl, own_key, paths): try: result = http_request(baseurl + "plop/v1/merge/verifiedsize", @@ -319,14 +338,14 @@ def sendentries_merge(node, baseurl, own_key, paths, entries, session=None): print >>sys.stderr, "ERROR: sendentries_merge", baseurl, e.request, e.response sys.exit(1) -def sendsth(node, baseurl, own_key, paths, submission): +def publish_sth(node, baseurl, own_key, paths, submission): try: - result = http_request(baseurl + "plop/v1/frontend/sendsth", + result = http_request(baseurl + "plop/v1/frontend/publish-sth", json.dumps(submission), key=own_key, verifynode=node, publickeydir=paths["publickeys"]) return json.loads(result) except requests.exceptions.HTTPError, e: - print >>sys.stderr, "ERROR: sendsth", e.response + print >>sys.stderr, "ERROR: publish-sth", e.response sys.exit(1) except ValueError, e: print >>sys.stderr, "==== FAILED REQUEST ====" -- cgit v1.1 From aa7be9990183d6e9a87037b88a1e33dca35304e3 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Fri, 18 Nov 2016 17:13:31 +0100 Subject: Use sessions in merge_fetch --- tools/mergetools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tools/mergetools.py') diff --git a/tools/mergetools.py b/tools/mergetools.py index ff3d08c..94901a9 100644 --- a/tools/mergetools.py +++ b/tools/mergetools.py @@ -179,13 +179,13 @@ def get_new_entries(node, baseurl, own_key, paths): print >>sys.stderr, "ERROR: fetchnewentries", e.response sys.exit(1) -def get_entries(node, baseurl, own_key, paths, hashes): +def get_entries(node, baseurl, own_key, paths, hashes, session=None): try: params = {"hash":[base64.b64encode(ehash) for ehash in hashes]} result = http_request(baseurl + "plop/v1/storage/getentry", params=params, key=own_key, verifynode=node, - publickeydir=paths["publickeys"]) + publickeydir=paths["publickeys"], session=session) parsed_result = json.loads(result) if parsed_result.get(u"result") == u"ok": entries = dict([(base64.b64decode(entry["hash"]), -- cgit v1.1