diff options
| -rw-r--r-- | ebin/plop.app | 2 | ||||
| -rw-r--r-- | merge/ebin/merge.app | 2 | ||||
| -rw-r--r-- | merge/src/merge_dist.erl | 4 | ||||
| -rw-r--r-- | merge/src/merge_util.erl | 20 | ||||
| -rw-r--r-- | src/atomic.erl | 8 |
5 files changed, 21 insertions, 15 deletions
diff --git a/ebin/plop.app b/ebin/plop.app index c3308fa..7a463e4 100644 --- a/ebin/plop.app +++ b/ebin/plop.app @@ -5,7 +5,7 @@ {application, plop, [{description, "The plop store"}, - {vsn, "0.10.1"}, + {vsn, "1.0.1-alpha-dev"}, {modules, [plop_app, plop_sup, plop, db, ht, hex, tlv]}, {applications, [kernel, stdlib]}, % crypto, public_key {registered, [plop, ht, db]}, diff --git a/merge/ebin/merge.app b/merge/ebin/merge.app index 7d2a9f4..b34334a 100644 --- a/merge/ebin/merge.app +++ b/merge/ebin/merge.app @@ -5,7 +5,7 @@ {application, merge, [{description, "Plop merge"}, - {vsn, "0.10.1"}, + {vsn, "1.0.1-alpha-dev"}, {modules, [merge_app, merge_dist, merge_dist_sup, merge_sup]}, {applications, [kernel, stdlib, lager, plop]}, {registered, [merge_dist, merge_dist_sup, merge_sup]}, diff --git a/merge/src/merge_dist.erl b/merge/src/merge_dist.erl index 2e7013f..f8f0c7c 100644 --- a/merge/src/merge_dist.erl +++ b/merge/src/merge_dist.erl @@ -98,10 +98,12 @@ do_dist(NodeAddress, Start, NTotal) -> SendentriesChunksize = application:get_env(plop, merge_dist_sendentries_chunksize, 100), ok = merge_util:sendlog(NodeAddress, Start, Hashes, SendlogChunksize), {ok, HashesMissingEncoded} = merge_util:missingentries(NodeAddress), + lager:debug("number of missing entries: ~B", [length(HashesMissingEncoded)]), HashesMissing = lists:map(fun base64:decode/1, HashesMissingEncoded), ok = merge_util:sendentries(NodeAddress, HashesMissing, SendentriesChunksize), {ok, NewSize} = frontend_verify_entries(NodeAddress, Start + N), - lager:info("~p: Done distributing ~B entries.", [NodeAddress, NewSize-Start]), + lager:info("~p: Done distributing ~B out of ~B entries.", + [NodeAddress, NewSize-Start, NTotal]), true = NTotal >= NewSize - Start, do_dist(NodeAddress, NewSize, NTotal - (NewSize - Start)). diff --git a/merge/src/merge_util.erl b/merge/src/merge_util.erl index a6c435b..7598e40 100644 --- a/merge/src/merge_util.erl +++ b/merge/src/merge_util.erl @@ -25,11 +25,13 @@ request(DebugTag, URL, Headers, RequestBody) -> end. sendlog(NodeAddress, Start, Hashes, Chunksize) -> + lager:debug("sending log: start=~B, N=~B, chunksize=~B", [Start, length(Hashes), Chunksize]), sendlog_chunk(NodeAddress, Start, lists:split(min(Chunksize, length(Hashes)), Hashes), Chunksize). sendlog_chunk(_, _, {[], _}, _) -> ok; sendlog_chunk(NodeAddress, Start, {Chunk, Rest}, Chunksize) -> + lager:debug("sending log chunk: start=~B, N=~B", [Start, length(Chunk)]), ok = sendlog_request(NodeAddress, Start, Chunk), sendlog_chunk(NodeAddress, Start + length(Chunk), lists:split(min(Chunksize, length(Rest)), Rest), Chunksize). @@ -59,17 +61,25 @@ missingentries(NodeAddress) -> end. sendentries(NodeAddress, Hashes, Chunksize) -> + lager:debug("sending entries: N=~B, chunksize=~B", [length(Hashes), Chunksize]), {ChunkOfHashes, RestOfHashes} = lists:split(min(Chunksize, length(Hashes)), Hashes), sendentries_chunk(NodeAddress, {ChunkOfHashes, RestOfHashes}, Chunksize). sendentries_chunk(_, {[], _}, _) -> ok; sendentries_chunk(NodeAddress, {Chunk, Rest}, Chunksize) -> - HashesAndEntries = lists:zip(Chunk, [db:entry_for_leafhash(H) || H <- Chunk]), - ok = sendentries_request(NodeAddress, HashesAndEntries), - sendentries_chunk(NodeAddress, - lists:split(min(Chunksize, length(Rest)), Rest), - Chunksize). + lager:debug("sending entries chunk: N=~B", [length(Chunk)]), + HashesAndEntries = lists:zip(Chunk, lists:map(fun db:entry_for_leafhash/1, Chunk)), + case lists:keysearch(noentry, 2, HashesAndEntries) of + false -> + ok = sendentries_request(NodeAddress, HashesAndEntries), + sendentries_chunk(NodeAddress, + lists:split(min(Chunksize, length(Rest)), Rest), + Chunksize); + Missing -> + lager:error("Entries not in db: ~p", [Missing]), + {error, entrynotindb} + end. sendentries_request(NodeAddress, HashesAndEntries) -> DebugTag = io_lib:format("sendentry ~B", [length(HashesAndEntries)]), diff --git a/src/atomic.erl b/src/atomic.erl index 580fd17..36fba81 100644 --- a/src/atomic.erl +++ b/src/atomic.erl @@ -2,7 +2,7 @@ %%% See LICENSE for licensing information. -module(atomic). --export([replacefile/2, readfile/1, openfile/2, readline/1]). +-export([replacefile/2, readfile/1]). -spec replacefile(string(), binary()) -> ok. replacefile(Path, Content) -> @@ -20,9 +20,3 @@ readfile(Path) -> {error, Error} -> util:exit_with_error(readfile, Error, "Error reading file") end. - -openfile(Path, Modes) -> - file:open(Path, Modes). - -readline(IoDevice) -> - file:read_line(IoDevice). |