From c7094dd2108a9583f7704c156ae34264edf26090 Mon Sep 17 00:00:00 2001
From: Magnus Ahltorp <map@kth.se>
Date: Tue, 29 Sep 2015 00:54:10 +0200
Subject: Only return first 100000 missing entries and make sendlog faster.

---
 src/frontend.erl | 57 +++++++++++++++++++++++++++++++++-----------------------
 1 file changed, 34 insertions(+), 23 deletions(-)

diff --git a/src/frontend.erl b/src/frontend.erl
index bce26b7..3dd178f 100644
--- a/src/frontend.erl
+++ b/src/frontend.erl
@@ -26,13 +26,7 @@ request(post, "plop/v1/frontend/sendlog", Input) ->
         {struct, PropList} ->
             Start = proplists:get_value(<<"start">>, PropList),
             Hashes = lists:map(fun (S) -> base64:decode(S) end, proplists:get_value(<<"hashes">>, PropList)),
-
-            Indices = lists:seq(Start, Start + length(Hashes) - 1),
-            lists:foreach(fun ({Hash, Index}) ->
-                                  ok = db:add_index_nosync_noreverse(Hash, Index)
-                          end, lists:zip(Hashes, Indices)),
-            ok = db:index_sync(),
-            success({[{result, <<"ok">>}]})
+            write_or_verify_index(Start, Hashes)
     end;
 
 request(post, "plop/v1/frontend/sendsth", Input) ->
@@ -79,7 +73,7 @@ request(get, "plop/v1/frontend/currentposition", _Query) ->
 
 request(get, "plop/v1/frontend/missingentries", _Query) ->
     Size = db:size(),
-    Missing = fetchmissingentries(Size),
+    Missing = fetchmissingentries(Size, 100000),
     lager:debug("missingentries: ~p", [Missing]),
     success({[{result, <<"ok">>},
               {entries, lists:map(fun (Entry) -> base64:encode(Entry) end,
@@ -104,13 +98,7 @@ request(post, "plop/v1/merge/sendlog", Input) ->
         {struct, PropList} ->
             Start = proplists:get_value(<<"start">>, PropList),
             Hashes = lists:map(fun (S) -> base64:decode(S) end, proplists:get_value(<<"hashes">>, PropList)),
-
-            Indices = lists:seq(Start, Start + length(Hashes) - 1),
-            lists:foreach(fun ({Hash, Index}) ->
-                                  ok = db:add_index_nosync_noreverse(Hash, Index)
-                          end, lists:zip(Hashes, Indices)),
-            ok = db:index_sync(),
-            success({[{result, <<"ok">>}]})
+            write_or_verify_index(Start, Hashes)
     end;
 
 request(post, "plop/v1/merge/verifyroot", Input) ->
@@ -171,7 +159,7 @@ request(post, "plop/v1/merge/setverifiedsize", Input) ->
 
 request(get, "plop/v1/merge/missingentries", _Query) ->
     Size = db:verifiedsize(),
-    Missing = fetchmissingentries(Size),
+    Missing = fetchmissingentries(Size, 100000),
     lager:debug("missingentries: ~p", [Missing]),
     success({[{result, <<"ok">>},
               {entries, lists:map(fun (Entry) -> base64:encode(Entry) end,
@@ -306,13 +294,15 @@ check_entry_noreverse(LeafHash, Index) ->
             end
     end.
 
--spec fetchmissingentries(non_neg_integer()) -> [binary() | noentry].
-fetchmissingentries(Index) ->
-    lists:reverse(fetchmissingentries(Index, [])).
+-spec fetchmissingentries(non_neg_integer(), non_neg_integer()) -> [binary() | noentry].
+fetchmissingentries(Index, MaxEntries) ->
+    lists:reverse(fetchmissingentries(Index, [], MaxEntries)).
 
--spec fetchmissingentries(non_neg_integer(), [binary() | noentry]) ->
+-spec fetchmissingentries(non_neg_integer(), [binary() | noentry], non_neg_integer()) ->
                                  [binary() | noentry].
-fetchmissingentries(Index, Acc) ->
+fetchmissingentries(_Index, Acc, 0) ->
+    Acc;
+fetchmissingentries(Index, Acc, MaxEntries) ->
     lager:debug("index ~p", [Index]),
     case db:leafhash_for_index(Index) of
         noentry ->
@@ -321,12 +311,33 @@ fetchmissingentries(Index, Acc) ->
             case db:entry_for_leafhash(Hash) of
                 noentry ->
                     lager:debug("didn't find hash ~p", [Hash]),
-                    fetchmissingentries(Index + 1, [Hash | Acc]);
+                    fetchmissingentries(Index + 1, [Hash | Acc], MaxEntries - 1);
                 _ ->
-                    fetchmissingentries(Index + 1, Acc)
+                    fetchmissingentries(Index + 1, Acc, MaxEntries)
             end
     end.
 
+write_or_verify_index(Start, Hashes) ->
+    End = Start + length(Hashes) - 1,
+    case db:indexsize() of
+        IndexSize when IndexSize > End ->
+            FetchedHashes = db:leafhash_for_indices(Start, End),
+            case FetchedHashes of
+                Hashes ->
+                    success({[{result, <<"ok">>}]});
+                _ ->
+                    util:exit_with_error(invalid, index,
+                                         "Written content not the" ++
+                                             " same as old content")
+            end;
+        _ ->
+            Indices = lists:seq(Start, End),
+            lists:foreach(fun ({Hash, Index}) ->
+                                  ok = db:add_index_nosync_noreverse(Hash, Index)
+                          end, lists:zip(Hashes, Indices)),
+            ok = db:index_sync(),
+            success({[{result, <<"ok">>}]})
+    end.
 
 %% Private functions.
 html(Text, Input) ->
-- 
cgit v1.1