Add back chunking for get_entries in parallelised merge_fetch.

Even if it's not important for parallelism on the merge node, it's not
a bad idea to keep the requests from growing.

1 files changed, 9 insertions, 10 deletions

diff --git a/tools/merge_fetch.py b/tools/merge_fetch.py
index 10fd406..be9edce 100755
--- a/tools/merge_fetch.py
+++ b/tools/merge_fetch.py
@@ -127,17 +127,16 @@ def merge_fetch_worker(args, localconfig, storagenode, pipe):
         if to_fetch:
             logging.info("%s: fetching %d entries", storagenode["name"],
                          len(to_fetch))
-            fetchlist = list(to_fetch)
             with requests.sessions.Session() as session:
-                entries = get_entries(storagenode["name"],
-                                      "https://%s/" % storagenode["address"],
-                                      own_key, paths, fetchlist,
-                                      session=session)
-                for ehash in fetchlist:
-                    entry = entries[ehash]
-                    verify_entry(verifycert, entry, ehash)
-                    pipe.send(('FETCHED', ehash, entry))
-                    to_fetch.remove(ehash)
+                for chunk in chunks(list(to_fetch), 100):
+                    entries = get_entries(storagenode["name"],
+                                          "https://%s/" % storagenode["address"],
+                                          own_key, paths, chunk, session=session)
+                    for ehash in chunk:
+                        entry = entries[ehash]
+                        verify_entry(verifycert, entry, ehash)
+                        pipe.send(('FETCHED', ehash, entry))
+                        to_fetch.remove(ehash)
 
         new_entries = get_new_entries(storagenode["name"],
                                       "https://%s/" % storagenode["address"],