From dc6b383aa4a9cc1e2793c4d9f3be81bd676b7a20 Mon Sep 17 00:00:00 2001 From: Linus Nordberg Date: Sat, 27 Sep 2014 15:45:14 +0200 Subject: wip --- src/db.erl | 126 ++++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 75 insertions(+), 51 deletions(-) (limited to 'src/db.erl') diff --git a/src/db.erl b/src/db.erl index 3ab2d1b..ddebbeb 100644 --- a/src/db.erl +++ b/src/db.erl @@ -7,7 +7,8 @@ %% API. -export([start_link/0, stop/0]). -export([init_db/0, init_db/1, init_tables/0, init_tables/1]). --export([add/1, find/2, get_by_index/2, get_by_index_sorted/2, size/0]). +-export([add/4, size/0]). +-export([get_by_index/1, get_by_indices/3, get_by_leaf_hash/1, get_by_entry_hash/1]). %% API for testing. -export([dump/1, destroy_tables/0, info_tables/0, dump_to_file/1]). %% gen_server callbacks. @@ -72,24 +73,31 @@ stop() -> %%%%%%%%%%%%%%%%%%%% %% Public API. --spec add(plop()) -> {atomic, ok}. -add(Entry) -> - gen_server:call(?MODULE, {add, Entry}). +-spec add(binary(), binary(), binary(), non_neg_integer()) -> ok. +add(LeafHash, EntryHash, Data, Index) -> + gen_server:call(?MODULE, {add, {LeafHash, EntryHash, Data, Index}}). -%% @doc Find one entry. --spec find(entryhash | mtlhash | index, binary()) -> - [] | plop() | duplicate_hash_in_db. -find(Type, Hash) -> - gen_server:call(?MODULE, {find, Type, Hash}). +-spec get_by_indices(non_neg_integer(), + non_neg_integer(), + {sorted, true|false}) -> + [{non_neg_integer(), binary(), binary()}]. +get_by_indices(Start, End, {sorted, Sorted}) -> + gen_server:call(?MODULE, {get_by_indices, {Start, End, Sorted}}). --spec get_by_index(non_neg_integer(), non_neg_integer()) -> [{mtl(), binary()}]. -get_by_index(Start, End) -> - gen_server:call(?MODULE, {get_by_index, {Start, End}}). +-spec get_by_index(binary()) -> notfound | + {non_neg_integer(), binary(), binary()}. +get_by_index(Index) -> + gen_server:call(?MODULE, {get_by_index, Index}). --spec get_by_index_sorted(non_neg_integer(), non_neg_integer()) -> - [{mtl(), binary()}]. -get_by_index_sorted(Start, End) -> - gen_server:call(?MODULE, {get_by_index_sorted, {Start, End}}). +-spec get_by_leaf_hash(binary()) -> notfound | + {non_neg_integer(), binary(), binary()}. +get_by_leaf_hash(LeafHash) -> + gen_server:call(?MODULE, {get_by_leaf_hash, LeafHash}). + +-spec get_by_entry_hash(binary()) -> notfound | + {non_neg_integer(), binary(), binary()}. +get_by_entry_hash(EntryHash) -> + gen_server:call(?MODULE, {get_by_entry_hash, EntryHash}). %% Testing and debugging. dump(Table) -> @@ -117,50 +125,64 @@ terminate(_Reason, _State) -> handle_call(stop, _From, State) -> {stop, normal, stopped, State}; -handle_call({add, Entry}, _From, State) -> - F = fun() -> - mnesia:write(Entry) - end, - Res = mnesia:transaction(F), - {reply, Res, State}; - -handle_call({dump, Table}, _From, State) -> - F = fun() -> - Q = qlc:q([E || E <- mnesia:table(Table)]), - qlc:e(Q) +handle_call({add, {LeafHash, EntryHash, Data, Index}}, _From, State) -> + R = mnesia:transaction( + fun() -> + mnesia:write( + #plop{ + index = Index, + mtlhash = LeafHash, + entryhash = EntryHash, + logentry = Data}) + end), + {reply, R, State}; + +handle_call({get_by_indices, {Start, End, Sorted}}, _From, State) -> + R = case Sorted of + false -> + select_index(Start, End); + true -> + %% FIXME: RAM hog -- how bad is it? + lists:sort(select_index(Start, End)) end, - Res = mnesia:transaction(F), - {reply, Res, State}; + {reply, R, State}; -handle_call({find, entryhash, Hash}, _From, State) -> +handle_call({get_by_index, Index}, _From, State) -> {reply, - find_entry(fun() -> mnesia:index_read(plop, Hash, #plop.entryhash) end), - State}; -handle_call({find, mtlhash, Hash}, _From, State) -> - {reply, - find_entry(fun() -> mnesia:index_read(plop, Hash, #plop.mtlhash) end), + find_entry(fun() -> mnesia:read(plop, Index) end), State}; -handle_call({find, index, Index}, _From, State) -> + +handle_call({get_by_leaf_hash, LeafHash}, _From, State) -> {reply, - find_entry(fun() -> mnesia:read(plop, Index) end), + find_entry(fun() -> + mnesia:index_read(plop, LeafHash, #plop.mtlhash) + end), State}; -handle_call({get_by_index, {Start, End}}, _From, State) -> - Res = [{MTL, Extra} || [_Index, MTL, Extra] <- select_index(Start, End)], - {reply, Res, State}; +handle_call({get_by_entry_hash, EntryHash}, _From, State) -> + {reply, + find_entry(fun() -> + mnesia:index_read(plop, EntryHash, #plop.entryhash) + end), + State}; -handle_call({get_by_index_sorted, {Start, End}}, _From, State) -> - %% FIXME: RAM hog -- how bad is it? - Res = [{MTL, Extra} || [_Index, MTL, Extra] <- lists:sort(select_index(Start, End))], - {reply, Res, State}. +handle_call({dump, Table}, _From, State) -> + R = mnesia:transaction( + fun() -> + Q = qlc:q([E || E <- mnesia:table(Table)]), + qlc:e(Q) + end), + {reply, R, State}. %%%%%%%%%%%%%%%%%%%% %% Helper functions. +-spec select_index(non_neg_integer(), non_neg_integer()) -> + [{non_neg_integer(), binary(), binary()}]. select_index(Start, End) -> F = fun() -> - %% Get index, mtl and extra_data. - MatchHead = {plop, '$1', '_', '_', '$2', '$3', '_'}, + %% Get index, mtlhash and logentry. + MatchHead = {plop, '$1', '$2', '_', '$3'}, Guard = [{'>=', '$1', Start}, {'=<', '$1', End}], Result = ['$$'], mnesia:select(plop, [{MatchHead, Guard, Result}]) @@ -168,11 +190,13 @@ select_index(Start, End) -> {atomic, Res} = mnesia:transaction(F), Res. --spec find_entry(fun()) -> [] | plop() | duplicate_hash_in_db. +-spec find_entry(fun()) -> notfound | + {non_neg_integer(), binary(), binary()} | + duplicate. find_entry(Fun) -> {atomic, Result} = mnesia:transaction(Fun), - case length(Result) of - 0 -> []; - 1 -> hd(Result); - _ -> duplicate_hash_in_db % FIXME: log an error? + case Result of + [] -> notfound; + [#plop{index = I, mtlhash = H, logentry = E}] -> {I, H, E}; + _ -> duplicate end. -- cgit v1.1 From b3d01b2105b1e1df6804d6463ae465bc91ae4e8d Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sun, 28 Sep 2014 01:50:48 +0200 Subject: Use raw file storage --- src/db.erl | 185 ++++++++++++++++++++++++------------------------------------- 1 file changed, 71 insertions(+), 114 deletions(-) (limited to 'src/db.erl') diff --git a/src/db.erl b/src/db.erl index ddebbeb..45d0b57 100644 --- a/src/db.erl +++ b/src/db.erl @@ -6,11 +6,9 @@ %% API. -export([start_link/0, stop/0]). --export([init_db/0, init_db/1, init_tables/0, init_tables/1]). +-export([]). -export([add/4, size/0]). -export([get_by_index/1, get_by_indices/3, get_by_leaf_hash/1, get_by_entry_hash/1]). -%% API for testing. --export([dump/1, destroy_tables/0, info_tables/0, dump_to_file/1]). %% gen_server callbacks. -export([init/1, handle_call/3, terminate/2, handle_cast/2, handle_info/2, code_change/3]). @@ -19,50 +17,11 @@ -include("db.hrl"). -include("$CTROOT/plop/include/plop.hrl"). -%% @doc Set up a database schema on all nodes that are to be included -%% in the "database cluster". Has to be run _before_ mnesia has been -%% started. -init_db() -> - init_db([node()]). -init_db(Nodes) -> - ok = mnesia:create_schema(Nodes), - rpc:multicall(Nodes, application, start, [mnesia]), - init_tables(Nodes), - rpc:multicall(Nodes, application, stop, [mnesia]). - -%% @doc Run once, or rather every time you start on a new database. -%% If run more than once, we'll get {aborted, {already_exists, TABLE}}. -init_tables() -> - init_tables([node()]). -init_tables(Nodes) -> - %% We've once upon a time invoked mnesia:create_schema/1 with the - %% nodes that will be part of the database. - RamCopies = [], - DiscCopies = [], - DiscOnlyCopies = Nodes, - mnesia:start(), - {atomic, ok} = - mnesia:create_table(plop, - [{type, set}, - {ram_copies, RamCopies}, - {disc_copies, DiscCopies}, - {disc_only_copies, DiscOnlyCopies}, - {attributes, record_info(fields, plop)}, - {majority, true}]), - {atomic, ok} = mnesia:add_table_index(plop, entryhash), - {atomic, ok} = mnesia:add_table_index(plop, mtlhash). - -destroy_tables() -> - mnesia:delete_table(plop). -info_tables() -> - mnesia:table_info(plop, all). -dump_to_file(Filename) -> - mnesia:dump_to_textfile(Filename). size() -> - mnesia:table_info(plop, size). + binary_to_integer(atomic:readfile(treesize_path())). init(_Args) -> - {mnesia:wait_for_tables([plop], 5000), []}. + {ok, []}. start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). @@ -99,10 +58,6 @@ get_by_leaf_hash(LeafHash) -> get_by_entry_hash(EntryHash) -> gen_server:call(?MODULE, {get_by_entry_hash, EntryHash}). -%% Testing and debugging. -dump(Table) -> - gen_server:call(?MODULE, {dump, Table}). - %%%%%%%%%%%%%%%%%%%% %% gen_server callbacks. @@ -122,81 +77,83 @@ terminate(_Reason, _State) -> %%%%%%%%%%%%%%%%%%%% %% The meat. +% Table for Leaf hash -> Entry +entry_root_path() -> + {ok, Value} = application:get_env(plop, entry_root_path), + Value. + +% Table for Leaf hash -> Entry +indexforhash_root_path() -> + {ok, Value} = application:get_env(plop, indexforhash_root_path), + Value. + +% Table for Index -> Leaf hash +index_path() -> + {ok, Value} = application:get_env(plop, index_path), + Value. + +% Table for Entry hash -> Leaf hash +entryhash_root_path() -> + {ok, Value} = application:get_env(plop, entryhash_root_path), + Value. + +% File that stores tree size +treesize_path() -> + {ok, Value} = application:get_env(plop, treesize_path), + Value. + + +entry_for_leafhash(LeafHash) -> + perm:readfile(entry_root_path(), LeafHash). + +index_for_leafhash(LeafHash) -> + binary_to_integer(perm:readfile(indexforhash_root_path(), LeafHash)). + +leafhash_for_index(Index) -> + index:get(index_path(), Index). + +leafhash_for_entryhash(EntryHash) -> + perm:readfile(entryhash_root_path(), EntryHash). + handle_call(stop, _From, State) -> {stop, normal, stopped, State}; handle_call({add, {LeafHash, EntryHash, Data, Index}}, _From, State) -> - R = mnesia:transaction( - fun() -> - mnesia:write( - #plop{ - index = Index, - mtlhash = LeafHash, - entryhash = EntryHash, - logentry = Data}) - end), - {reply, R, State}; - -handle_call({get_by_indices, {Start, End, Sorted}}, _From, State) -> - R = case Sorted of - false -> - select_index(Start, End); - true -> - %% FIXME: RAM hog -- how bad is it? - lists:sort(select_index(Start, End)) - end, + ok = perm:ensurefile(entry_root_path(), LeafHash, Data), + ok = perm:ensurefile(entryhash_root_path(), EntryHash, LeafHash), + ok = perm:ensurefile(indexforhash_root_path(), + LeafHash, integer_to_binary(Index)), + ok = index:add(index_path(), Index, LeafHash), + ok = atomic:replacefile(treesize_path(), integer_to_list(Index+1)), + {reply, ok, State}; + +handle_call({get_by_indices, {Start, End, _Sorted}}, _From, State) -> + R = lists:map(fun (Index) -> + LeafHash = leafhash_for_index(Index), + Entry = entry_for_leafhash(LeafHash), + {Index, LeafHash, Entry} + end, lists:seq(Start, End)), {reply, R, State}; handle_call({get_by_index, Index}, _From, State) -> - {reply, - find_entry(fun() -> mnesia:read(plop, Index) end), - State}; + LeafHash = leafhash_for_index(Index), + Entry = entry_for_leafhash(LeafHash), + R = {Index, LeafHash, Entry}, + {reply, R, State}; handle_call({get_by_leaf_hash, LeafHash}, _From, State) -> - {reply, - find_entry(fun() -> - mnesia:index_read(plop, LeafHash, #plop.mtlhash) - end), - State}; + Entry = entry_for_leafhash(LeafHash), + Index = index_for_leafhash(LeafHash), + R = {Index, LeafHash, Entry}, + {reply, R, State}; handle_call({get_by_entry_hash, EntryHash}, _From, State) -> - {reply, - find_entry(fun() -> - mnesia:index_read(plop, EntryHash, #plop.entryhash) - end), - State}; - -handle_call({dump, Table}, _From, State) -> - R = mnesia:transaction( - fun() -> - Q = qlc:q([E || E <- mnesia:table(Table)]), - qlc:e(Q) - end), - {reply, R, State}. - -%%%%%%%%%%%%%%%%%%%% -%% Helper functions. - --spec select_index(non_neg_integer(), non_neg_integer()) -> - [{non_neg_integer(), binary(), binary()}]. -select_index(Start, End) -> - F = fun() -> - %% Get index, mtlhash and logentry. - MatchHead = {plop, '$1', '$2', '_', '$3'}, - Guard = [{'>=', '$1', Start}, {'=<', '$1', End}], - Result = ['$$'], - mnesia:select(plop, [{MatchHead, Guard, Result}]) + R = case leafhash_for_entryhash(EntryHash) of + noentry -> + notfound; + LeafHash -> + Entry = entry_for_leafhash(LeafHash), + Index = index_for_leafhash(LeafHash), + {Index, LeafHash, Entry} end, - {atomic, Res} = mnesia:transaction(F), - Res. - --spec find_entry(fun()) -> notfound | - {non_neg_integer(), binary(), binary()} | - duplicate. -find_entry(Fun) -> - {atomic, Result} = mnesia:transaction(Fun), - case Result of - [] -> notfound; - [#plop{index = I, mtlhash = H, logentry = E}] -> {I, H, E}; - _ -> duplicate - end. + {reply, R, State}. -- cgit v1.1