From 2483f0cf09ccc4cf73558c7a85bbb51a72d29c3a Mon Sep 17 00:00:00 2001
From: Magnus Ahltorp <map@kth.se>
Date: Sat, 25 Oct 2014 15:22:09 +0200
Subject: Optimize db:get_by_indices by not fetching entry and implementing
 index:getrange

---
 src/db.erl    | 12 +++++++-----
 src/index.erl | 41 ++++++++++++++++++++++++-----------------
 src/plop.erl  |  7 ++++++-
 3 files changed, 37 insertions(+), 23 deletions(-)

(limited to 'src')

diff --git a/src/db.erl b/src/db.erl
index fade7ce..f7c2057 100644
--- a/src/db.erl
+++ b/src/db.erl
@@ -129,6 +129,9 @@ index_for_leafhash(LeafHash) ->
 leafhash_for_index(Index) ->
     index:get(index_path(), Index).
 
+leafhash_for_indices(Start, End) ->
+    index:getrange(index_path(), Start, End).
+
 leafhash_for_entryhash(EntryHash) ->
     perm:readfile(entryhash_root_path(), EntryHash).
 
@@ -138,11 +141,10 @@ get_by_indices_helper(Start, End) ->
     EndBound = min(End, size() - 1),
     case Start =< EndBound of
         true ->
-            lists:map(fun (Index) ->
-                              LeafHash = leafhash_for_index(Index),
-                              Entry = entry_for_leafhash(LeafHash),
-                              {Index, LeafHash, Entry}
-                      end, lists:seq(Start, EndBound));
+            lists:map(fun ({LeafHash, Index}) ->
+                              {Index, LeafHash, notfetched}
+                      end, lists:zip(leafhash_for_indices(Start, EndBound),
+                                     lists:seq(Start, EndBound)));
         false ->
             []
     end.
diff --git a/src/index.erl b/src/index.erl
index 5169fbb..bbc9a10 100644
--- a/src/index.erl
+++ b/src/index.erl
@@ -12,7 +12,7 @@
 %% TODO: Checksums
 
 -module(index).
--export([get/2, add/3, addlast/2]).
+-export([get/2, getrange/3, add/3, addlast/2]).
 
 -define(ENTRYSIZE, 32).
 -define(ENTRYSIZEINFILE, (?ENTRYSIZE*2+1)).
@@ -66,31 +66,38 @@ add(Basepath, Index, Entry) when is_binary(Entry), size(Entry) == ?ENTRYSIZE ->
 addlast(Basepath, Entry) ->
     add(Basepath, last, Entry).
 
-%% From lib/stdlib/src/lists.erl. For supporting < R17.
--spec droplast(nonempty_list()) -> list().
-droplast([_T]) -> [];
-droplast([H|T]) -> [H|droplast(T)].
+decodedata(Binary) ->
+    lists:reverse(decodedata(Binary, [])).
 
-decodedata(EntryText) when length(EntryText) == ?ENTRYSIZEINFILE ->
-    case [lists:last(EntryText)] of
-        "\n" ->
-            hex:hexstr_to_bin(droplast(EntryText));
-        _ ->
-            util:exit_with_error(badformat, readindex,
-                                 "Index line not ending with linefeed")
-    end.
+decodedata(<<>>, Acc) ->
+    Acc;
+decodedata(<<Entry:?ENTRYSIZE/binary-unit:16, "\n", Rest/binary>>, Acc) ->
+    decodedata(Rest, [mochihex:to_bin(binary_to_list(Entry)) | Acc]);
+decodedata(<<_:?ENTRYSIZE/binary-unit:16, _>>, _Acc) ->
+    util:exit_with_error(badformat, readindex,
+                         "Index line not ending with linefeed").
 
 -spec get(string(), integer()) -> binary().
 get(Basepath, Index) ->
+    case getrange(Basepath, Index, Index) of
+        noentry ->
+            noentry;
+        [Entry] ->
+            Entry
+    end.
+
+-spec getrange(string(), integer(), integer()) -> [binary()].
+getrange(Basepath, Start, End) when Start =< End ->
     case file:open(Basepath, [read, binary]) of
         {ok, File} ->
             {ok, Filesize} = file:position(File, eof),
             if
-                Index * ?ENTRYSIZEINFILE + ?ENTRYSIZEINFILE =< Filesize ->
+                End * ?ENTRYSIZEINFILE + ?ENTRYSIZEINFILE =< Filesize ->
                     {ok, _Position} = file:position(File,
-                                                    Index * ?ENTRYSIZEINFILE),
-                    {ok, EntryText} = file:read(File, ?ENTRYSIZEINFILE),
-                    Entry = decodedata(binary_to_list(EntryText)),
+                                                    Start * ?ENTRYSIZEINFILE),
+                    {ok, EntryText} =
+                        file:read(File, ?ENTRYSIZEINFILE * (End - Start + 1)),
+                    Entry = decodedata(EntryText),
                     file:close(File),
                     Entry;
                 true ->
diff --git a/src/plop.erl b/src/plop.erl
index 0523613..0c85b21 100644
--- a/src/plop.erl
+++ b/src/plop.erl
@@ -248,12 +248,17 @@ store_at_all_nodes(Nodes, {LogEntry, TreeLeafHash, EntryHash}, From, State) ->
                     end, PlopWithOwn, RequestIds),
     PlopWithRequests.
 
+fill_in_entry({_Index, LeafHash, notfetched}) ->
+    db:get_by_leaf_hash(LeafHash).
+
 %%%%%%%%%%%%%%%%%%%%
 handle_call(stop, _From, Plop) ->
     {stop, normal, stopped, Plop};
 
 handle_call({get, {index, Start, End}}, _From, Plop) ->
-    {reply, db:get_by_indices(Start, End, {sorted, false}), Plop};
+    {reply, lists:map(fun (E) -> fill_in_entry(E) end,
+                      db:get_by_indices(Start, End, {sorted, false})),
+     Plop};
 
 handle_call({get, {hash, EntryHash}}, _From, Plop) ->
     {reply, db:get_by_entry_hash(EntryHash), Plop};
-- 
cgit v1.1