Preliminary merge secondary implementation.

2 files changed, 141 insertions, 20 deletions

diff --git a/src/db.erl b/src/db.erl
index f53cd19..feb864b 100644
--- a/src/db.erl
+++ b/src/db.erl
@@ -8,6 +8,8 @@
 -export([start_link/0, stop/0]).
 -export([create_size_table/0]).
 -export([add/2, add_entryhash/2, add_index_nosync/2, set_treesize/1, size/0]).
+-export([add_index_nosync_noreverse/2]).
+-export([verifiedsize/0, set_verifiedsize/1]).
 -export([get_by_index/1, get_by_indices/3, get_by_leaf_hash/1]).
 -export([get_by_entry_hash/1, entry_for_leafhash/1, leafhash_for_index/1]).
 -export([leafhash_for_indices/2, indexsize/0]).
@@ -26,6 +28,9 @@ size() ->
     [{_, Size}] = ets:lookup(?DB_SIZE_TABLE, db_size),
     Size.
 
+verifiedsize() ->
+    binary_to_integer(atomic:readfile(verifiedsize_path())).
+
 indexsize() ->
     index:indexsize(index_path()).
 
@@ -65,11 +70,19 @@ add_entryhash(LeafHash, EntryHash) ->
 add_index_nosync(LeafHash, Index) ->
     call(?MODULE, {add_index_nosync, {LeafHash, Index}}).
 
+-spec add_index_nosync_noreverse(binary(), non_neg_integer()) -> ok.
+add_index_nosync_noreverse(LeafHash, Index) ->
+    call(?MODULE, {add_index_nosync_noreverse, {LeafHash, Index}}).
+
 -spec set_treesize(non_neg_integer()) -> ok.
 set_treesize(Size) ->
     true = ets:insert(?DB_SIZE_TABLE, {db_size, Size}),
     ok.
 
+-spec set_verifiedsize(non_neg_integer()) -> ok.
+set_verifiedsize(Size) ->
+    ok = atomic:replacefile(verifiedsize_path(), integer_to_binary(Size)).
+
 -spec get_by_indices(integer(), integer(), {sorted, true|false}) ->
                             [{non_neg_integer(), binary(), notfetched}].
 get_by_indices(Start, End, {sorted, _Sorted}) ->
@@ -151,6 +164,11 @@ entryhash_root_path() ->
     {ok, Value} = application:get_env(plop, entryhash_root_path),
     Value.
 
+% File that stores the number of verified entries
+verifiedsize_path() ->
+    {ok, Value} = application:get_env(plop, verifiedsize_path),
+    Value.
+
 entry_for_leafhash(LeafHash) ->
     perm:readfile(entry_root_path(), LeafHash).
 
@@ -192,6 +210,10 @@ handle_call({add_index_nosync, {LeafHash, Index}}, _From, State) ->
     ok = perm:ensurefile_nosync(indexforhash_root_path(),
                                 LeafHash, integer_to_binary(Index)),
     ok = index:add_nosync(index_path(), Index, LeafHash),
+    {reply, ok, State};
+
+handle_call({add_index_nosync_noreverse, {LeafHash, Index}}, _From, State) ->
+    ok = index:add_nosync(index_path(), Index, LeafHash),
     {reply, ok, State}.
 
 indexforhash_sync(LeafHash, Index) ->
diff --git a/src/frontend.erl b/src/frontend.erl
index 3a45f7b..3924fb8 100644
--- a/src/frontend.erl
+++ b/src/frontend.erl
@@ -55,16 +55,7 @@ request(post, "ct/frontend/sendsth", Input) ->
                     html("Size is older than current size", OldSize);
                 Treesize == 0, OldSize == 0 ->
                     lager:debug("both old and new size is 0, saving sth"),
-                    OwnRootHash = ht:root(-1),
-                    case {plop:verify_sth(Treesize, Timestamp, RootHash, Signature), OwnRootHash} of
-                        {true, RootHash} ->
-                            ok = plop:save_sth({struct, PropList}),
-                            success({[{result, <<"ok">>}]});
-                        {false, RootHash} ->
-                            html("Verification failed", hex:bin_to_hexstr(RootHash));
-                        _ ->
-                            html("Root hash not the same", hex:bin_to_hexstr(OwnRootHash))
-                    end;
+                    verify_and_save_sth(Treesize, Timestamp, RootHash, Signature, PropList);
                 Treesize > Indexsize ->
                     html("Has too few entries", Indexsize);
                 true ->
@@ -77,22 +68,14 @@ request(post, "ct/frontend/sendsth", Input) ->
                     case Errors of
                         [] ->
                             ht:load_tree(Treesize - 1),
-                            OwnRootHash = ht:root(Treesize - 1),
-                            case {plop:verify_sth(Treesize, Timestamp, RootHash, Signature), OwnRootHash} of
-                                {true, RootHash} ->
-                                    ok = plop:save_sth({struct, PropList}),
-                                    success({[{result, <<"ok">>}]});
-                                {false, RootHash} ->
-                                    html("Verification failed", hex:bin_to_hexstr(RootHash));
-                                _ ->
-                                    html("Root hash not the same", hex:bin_to_hexstr(OwnRootHash))
-                            end;
+                            verify_and_save_sth(Treesize, Timestamp, RootHash, Signature, PropList);
                         _ ->
                             html("Database not complete", Errors)
                     end
             end
     end;
 
+
 request(get, "ct/frontend/currentposition", _Query) ->
     Size = db:size(),
     success({[{result, <<"ok">>},
@@ -104,8 +87,94 @@ request(get, "ct/frontend/missingentries", _Query) ->
     lager:debug("missingentries: ~p", [Missing]),
     success({[{result, <<"ok">>},
               {entries, lists:map(fun (Entry) -> base64:encode(Entry) end,
+                                  Missing)}]});
+
+request(post, "catlfish/merge/sendentry", Input) ->
+    case (catch mochijson2:decode(Input)) of
+        {error, E} ->
+            html("sendentry: bad input:", E);
+        {struct, PropList} ->
+            LogEntry = base64:decode(proplists:get_value(<<"entry">>, PropList)),
+            TreeLeafHash = base64:decode(proplists:get_value(<<"treeleafhash">>, PropList)),
+
+            ok = db:add(TreeLeafHash, LogEntry),
+            success({[{result, <<"ok">>}]})
+    end;
+
+request(post, "catlfish/merge/sendlog", Input) ->
+    case (catch mochijson2:decode(Input)) of
+        {error, E} ->
+            html("sendentry: bad input:", E);
+        {struct, PropList} ->
+            Start = proplists:get_value(<<"start">>, PropList),
+            Hashes = lists:map(fun (S) -> base64:decode(S) end, proplists:get_value(<<"hashes">>, PropList)),
+
+            Indices = lists:seq(Start, Start + length(Hashes) - 1),
+            lists:foreach(fun ({Hash, Index}) ->
+                                  ok = db:add_index_nosync_noreverse(Hash, Index)
+                          end, lists:zip(Hashes, Indices)),
+            ok = db:index_sync(),
+            success({[{result, <<"ok">>}]})
+    end;
+
+request(post, "catlfish/merge/verifyroot", Input) ->
+    case (catch mochijson2:decode(Input)) of
+        {error, E} ->
+            html("sendentry: bad input:", E);
+        {struct, PropList} ->
+            OldSize = db:verifiedsize(),
+            Treesize = proplists:get_value(<<"tree_size">>, PropList),
+            Indexsize = db:indexsize(),
+
+            if
+                Treesize > Indexsize ->
+                    html("Has too few entries", Indexsize);
+                true ->
+                    NewEntries = get_new_entries(OldSize, Treesize),
+                    lager:debug("old size: ~p new size: ~p entries: ~p",
+                                [OldSize, Treesize, NewEntries]),
+
+                    Errors = check_entries_noreverse(NewEntries, OldSize, Treesize - 1),
+
+                    case Errors of
+                        [] ->
+                            ht:load_tree(Treesize - 1),
+                            RootHash = ht:root(Treesize - 1),
+                            success({[{result, <<"ok">>}, {root_hash, base64:encode(RootHash)}]});
+                        _ ->
+                            html("Database not complete", Errors)
+                    end
+            end
+    end;
+
+%% implement setverifiedsize
+
+request(get, "catlfish/merge/verifiedsize", _Query) ->
+    Size = db:verifiedsize(),
+    success({[{result, <<"ok">>},
+              {size, Size}]});
+
+request(get, "catlfish/merge/missingentries", _Query) ->
+    Size = db:verifiedsize(),
+    Missing = fetchmissingentries(Size),
+    lager:debug("missingentries: ~p", [Missing]),
+    success({[{result, <<"ok">>},
+              {entries, lists:map(fun (Entry) -> base64:encode(Entry) end,
                                   Missing)}]}).
 
+verify_and_save_sth(Treesize, Timestamp, RootHash, Signature, PropList) ->
+    OwnRootHash = ht:root(Treesize - 1),
+    case {plop:verify_sth(Treesize, Timestamp, RootHash, Signature), OwnRootHash} of
+        {true, RootHash} ->
+            ok = plop:save_sth({struct, PropList}),
+            success({[{result, <<"ok">>}]});
+        {false, RootHash} ->
+            html("Verification failed", hex:bin_to_hexstr(RootHash));
+        _ ->
+            html("Root hash not the same", hex:bin_to_hexstr(OwnRootHash))
+    end.
+
+
 get_new_entries(OldSize, Treesize) when OldSize < Treesize ->
     db:leafhash_for_indices(OldSize, Treesize - 1);
 get_new_entries(OldSize, Treesize) when OldSize == Treesize ->
@@ -121,6 +190,16 @@ check_entries(Entries, Start, End) ->
                         end
                 end, [], lists:zip(Entries, lists:seq(Start, End))).
 
+check_entries_noreverse(Entries, Start, End) ->
+    lists:foldl(fun ({Hash, Index}, Acc) ->
+                        case check_entry_noreverse(Hash, Index) of
+                            ok ->
+                                Acc;
+                            Error ->
+                                [Error | Acc]
+                        end
+                end, [], lists:zip(Entries, lists:seq(Start, End))).
+
 entryhash_from_entry(Entry) ->
     {ok, {Module, Function}} = application:get_env(plop, entryhash_from_entry),
     Module:Function(Entry).
@@ -151,6 +230,26 @@ check_entry(LeafHash, Index) ->
             end
     end.
 
+check_entry_noreverse(LeafHash, Index) ->
+    case db:entry_for_leafhash(LeafHash) of
+        notfound ->
+            {notfound, Index};
+        Entry ->
+            case verify_entry(Entry) of
+                {ok, LeafHash} ->
+                    ok;
+                {ok, DifferentLeafHash} ->
+                    lager:error("leaf hash not correct: filename is ~p " ++
+                                    "and contents are ~p",
+                                [hex:bin_to_hexstr(LeafHash),
+                                 hex:bin_to_hexstr(DifferentLeafHash)]),
+                    {error, differentleafhash};
+                {error, Reason} ->
+                    lager:error("verification failed: ~p", [Reason]),
+                    {error, verificationfailed}
+            end
+    end.
+
 -spec fetchmissingentries(non_neg_integer()) -> [binary() | noentry].
 fetchmissingentries(Index) ->
     lists:reverse(fetchmissingentries(Index, [])).