From b4dc2c37be0ed51b629e91f4d3e1dfc44cb192c0 Mon Sep 17 00:00:00 2001
From: Magnus Ahltorp <map@kth.se>
Date: Wed, 2 Nov 2016 03:34:35 +0100
Subject: Send list of entries to frontend Handle connection error

---
 tools/merge_dist.py | 52 ++++++++++++++++++++++++++++++++--------------------
 1 file changed, 32 insertions(+), 20 deletions(-)

(limited to 'tools/merge_dist.py')

diff --git a/tools/merge_dist.py b/tools/merge_dist.py
index 2af1d6c..9d66cfd 100755
--- a/tools/merge_dist.py
+++ b/tools/merge_dist.py
@@ -6,12 +6,14 @@
 
 import sys
 import json
+import base64
+import requests
 from time import sleep
 from base64 import b64encode, b64decode
 from certtools import timing_point, \
      create_ssl_context
 from mergetools import get_curpos, get_logorder, chunks, get_missingentries, \
-     sendsth, sendlog, sendentry, parse_args, perm
+     sendsth, sendlog, sendentries, parse_args, perm
 
 def merge_dist(args, localconfig, frontendnodes, timestamp):
     paths = localconfig["paths"]
@@ -84,25 +86,35 @@ def merge_dist(args, localconfig, frontendnodes, timestamp):
                                             paths)
         timing_point(timing, "get missing")
 
-        print >>sys.stderr, "missing entries:", len(missingentries)
-        sys.stderr.flush()
-        sent_entries = 0
-        print >>sys.stderr, "send missing entries",
-        sys.stderr.flush()
-        for missingentry in missingentries:
-            ehash = b64decode(missingentry)
-            sendentryresult = sendentry(nodename, nodeaddress, own_key, paths,
-                                        chainsdb.get(ehash), ehash)
-            if sendentryresult["result"] != "ok":
-                print >>sys.stderr, "sendentry:", sendentryresult
-                sys.exit(1)
-            sent_entries += 1
-            if sent_entries % 1000 == 0:
-                print >>sys.stderr, sent_entries,
-                sys.stderr.flush()
-        print >>sys.stderr
-        sys.stderr.flush()
-        timing_point(timing, "send missing")
+
+
+        while missingentries:
+            print >>sys.stderr, "missing entries:", len(missingentries)
+            sys.stderr.flush()
+
+            sent_entries = 0
+            print >>sys.stderr, "sending missing entries",
+            sys.stderr.flush()
+            with requests.sessions.Session() as session:
+                for missingentry_chunk in chunks(missingentries, 100):
+                    missingentry_hashes = [base64.b64decode(missingentry) for missingentry in missingentry_chunk]
+                    hashes_and_entries = [(hash, chainsdb.get(hash)) for hash in missingentry_hashes]
+                    sendentryresult = sendentries(nodename, nodeaddress,
+                                                 own_key, paths,
+                                                 hashes_and_entries, session)
+                    if sendentryresult["result"] != "ok":
+                        print >>sys.stderr, "sendentries:", sendentryresult
+                        sys.exit(1)
+                    sent_entries += len(missingentry_hashes)
+                    print >>sys.stderr, sent_entries,
+                    sys.stderr.flush()
+            print >>sys.stderr
+            sys.stderr.flush()
+            timing_point(timing, "send missing")
+
+            missingentries = get_missingentries(nodename, nodeaddress,
+                                                         own_key, paths)
+            timing_point(timing, "get missing")
 
         print >>sys.stderr, "sending sth to node", nodename
         sys.stderr.flush()
-- 
cgit v1.1