From 29ac49eabca61c4a9e0c3a0d8f9ba57ab516ebae Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Thu, 25 Sep 2014 01:35:33 +0200 Subject: Permanent storage implementation --- Makefile | 2 + c_src/erlport.c | 105 +++++++++++++++++++++++++++++++++++++++++++++++++ c_src/erlport.h | 15 +++++++ c_src/fsynchelper.c | 64 ++++++++++++++++++++++++++++++ c_src/net_read_write.c | 93 +++++++++++++++++++++++++++++++++++++++++++ c_src/net_read_write.h | 10 +++++ src/fsyncport.erl | 88 +++++++++++++++++++++++++++++++++++++++++ src/perm.erl | 95 ++++++++++++++++++++++++++++++++++++++++++++ src/plop_sup.erl | 5 +++ 9 files changed, 477 insertions(+) create mode 100644 c_src/erlport.c create mode 100644 c_src/erlport.h create mode 100644 c_src/fsynchelper.c create mode 100644 c_src/net_read_write.c create mode 100644 c_src/net_read_write.h create mode 100644 src/fsyncport.erl create mode 100644 src/perm.erl diff --git a/Makefile b/Makefile index 2efdd34..4e54096 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,6 @@ build all: + (cd c_src && make all) + cp c_src/fsynchelper priv/fsynchelper erl -make clean: -rm ebin/*.beam diff --git a/c_src/erlport.c b/c_src/erlport.c new file mode 100644 index 0000000..5e5c17c --- /dev/null +++ b/c_src/erlport.c @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2014 Kungliga Tekniska Högskolan + * (KTH Royal Institute of Technology, Stockholm, Sweden). + */ + +#include +#include +#include +#include +#include +#include + +#include "net_read_write.h" +#include "erlport.h" + +static ssize_t +read_length(size_t length_size) +{ + unsigned char buf[2]; + + if (length_size != 2) { + return -1; + } + + if (length_size > sizeof(buf)) { + return -1; + } + + ssize_t ret; + + ret = net_read(0, (char *)buf, length_size); + + if (ret != (ssize_t) length_size) { + return -1; + } + + return (ssize_t)(((unsigned long)buf[0] << 8) | (unsigned long)buf[1]); +} + +ssize_t +read_command(char *buf, size_t maxlen) +{ + ssize_t len; + + len = read_length(2); + + if (len < 0) { + return -1; + } + + if (len > (ssize_t) maxlen) { + return -1; + } + return net_read(0, buf, (size_t)len); +} + +static int +write_length(size_t len, size_t length_size) +{ + unsigned char buf[2]; + + if (length_size != 2) { + return -1; + } + + buf[0] = (len >> 8) & 0xff; + buf[1] = len & 0xff; + + ssize_t ret; + + ret = net_write(1, (char *)buf, length_size); + + if (ret < 0) { + return -1; + } + + if (ret != (ssize_t) length_size) { + return -1; + } + + return 0; +} + +static int +write_reply(char *msg, size_t len) +{ + ssize_t ret; + + ret = write_length(len, 2); + if (ret < 0) { + return -1; + } + ret = net_write(1, msg, len); + if (ret < 0) { + return -1; + } + + return 0; +} + +int +write_status(char *msg) +{ + return write_reply(msg, strlen(msg)); +} diff --git a/c_src/erlport.h b/c_src/erlport.h new file mode 100644 index 0000000..49e1b7c --- /dev/null +++ b/c_src/erlport.h @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2014 Kungliga Tekniska Högskolan + * (KTH Royal Institute of Technology, Stockholm, Sweden). + */ + +#ifndef ERLPORT_H +#define ERLPORT_H + +ssize_t +read_command(char *buf, size_t len); + +int +write_status(char *msg); + +#endif diff --git a/c_src/fsynchelper.c b/c_src/fsynchelper.c new file mode 100644 index 0000000..e6a04be --- /dev/null +++ b/c_src/fsynchelper.c @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2014 Kungliga Tekniska Högskolan + * (KTH Royal Institute of Technology, Stockholm, Sweden). + */ + +#include +#include +#include +#include + +#include +#include + +#include "erlport.h" + +static int +dosync(int fd) +{ +#ifdef F_FULLFSYNC + int ret = fcntl(fd, F_FULLFSYNC); +#else + int ret = fsync(fd); +#endif + return ret; +} + +int +main() +{ + char buf[100]; + ssize_t len; + + /* XXX: exits when command size is 0 */ + + while ((len = read_command(buf, sizeof(buf)-1)) > 0) { + buf[len] = '\0'; + while (1) { + int fd; + + fd = open(buf, O_RDONLY); + if (fd == -1) { + /* XXX: better errors */ + write_status("openerror"); + break; + } + + if (dosync(fd) == 0) { + write_status("ok"); + } else if (errno == EBADF) { + write_status("ebadf"); + } else if (errno == EINTR) { + close(fd); + continue; + } else { + write_status("fsyncerror"); + } + + close(fd); + break; + } + } + + return 0; +} diff --git a/c_src/net_read_write.c b/c_src/net_read_write.c new file mode 100644 index 0000000..f8f14f0 --- /dev/null +++ b/c_src/net_read_write.c @@ -0,0 +1,93 @@ +/* + * Copyright (c) 1995, 1996, 1997, 1998 Kungliga Tekniska Högskolan + * (Royal Institute of Technology, Stockholm, Sweden). + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the Institute nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include + +#include "net_read_write.h" + +/* + * Like read but never return partial data. + */ + +ssize_t +net_read (int fd, void *buf, size_t nbytes) +{ + char *cbuf = (char *)buf; + ssize_t count; + size_t rem = nbytes; + + while (rem > 0) { + count = read (fd, cbuf, rem); + if (count < 0) { + if (errno == EINTR) + continue; + else + return count; + } else if (count == 0) { + return count; + } + cbuf += (size_t) count; + rem -= (size_t) count; + } + return (ssize_t)nbytes; +} + +/* + * Like write but never return partial data. + */ + +ssize_t +net_write (int fd, const void *buf, size_t nbytes) +{ + const char *cbuf = (const char *)buf; + ssize_t count; + size_t rem = nbytes; + + while (rem > 0) { + count = write (fd, cbuf, rem); + if (count < 0) { + if (errno == EINTR) + continue; + else + return count; + } + cbuf += (size_t)count; + rem -= (size_t)count; + } + return (ssize_t)nbytes; +} diff --git a/c_src/net_read_write.h b/c_src/net_read_write.h new file mode 100644 index 0000000..80b92b3 --- /dev/null +++ b/c_src/net_read_write.h @@ -0,0 +1,10 @@ +#ifndef NET_READ_WRITE_H +#define NET_READ_WRITE_H + +ssize_t +net_read (int, void *, size_t); + +ssize_t +net_write (int, const void *, size_t); + +#endif diff --git a/src/fsyncport.erl b/src/fsyncport.erl new file mode 100644 index 0000000..8bc8c60 --- /dev/null +++ b/src/fsyncport.erl @@ -0,0 +1,88 @@ +%% +%% Copyright (c) 2014 Kungliga Tekniska Högskolan +%% (KTH Royal Institute of Technology, Stockholm, Sweden). +%% + +-module(fsyncport). +-export([start_link/0, stop/0, init/1]). +-export([fsync/1]). + +start_link() -> + Pid = spawn(?MODULE, init, [code:priv_dir(plop) ++ "/fsynchelper"]), + {ok, Pid}. +stop() -> + fsyncport ! stop. + +fsync(Path) -> + call_port({fsync, Path}). + +call_port(Msg) -> + fsyncport ! {call, self(), Msg}, + receive + {fsyncport, Result} -> + Result + end. + +init(ExtPrg) -> + register(fsyncport, self()), + process_flag(trap_exit, true), + Ports = lists:map(fun(_N) -> open_port({spawn_executable, ExtPrg}, + [{packet, 2}]) end, + lists:seq(1, 32)), + loop(Ports). + +loop(Ports) -> + loop(Ports, dict:new(), queue:new()). +loop(IdlePorts, BusyPorts, Waiting) -> + receive + {call, Caller, {fsync, Path}} -> + case IdlePorts of + [] -> + loop(IdlePorts, + BusyPorts, + queue:in({Caller, Path}, Waiting)); + [Port | Rest] -> + Port ! {self(), {command, Path}}, + loop(Rest, + dict:store(Port, {Caller, os:timestamp()}, BusyPorts), + Waiting) + end; + + {Port, {data, Data}} when is_port(Port) -> + {Caller, Starttime} = dict:fetch(Port, BusyPorts), + Stoptime = os:timestamp(), + statreport({fsync, Stoptime, Starttime}), + Caller ! {fsyncport, list_to_atom(Data)}, + case queue:out(Waiting) of + {empty, _} -> + loop([Port | IdlePorts], + dict:erase(Port, BusyPorts), + Waiting); + {{value, {NewCaller, NewPath}}, NewWaiting} -> + IdlePorts = [], + Port ! {self(), {command, NewPath}}, + loop(IdlePorts, + dict:store(Port, {NewCaller, os:timestamp()}, + BusyPorts), + NewWaiting) + end; + stop -> + lists:foreach(fun (Port) -> + Port ! {self(), close} + end, + IdlePorts), + lists:foreach(fun ({Port, {_Caller, _Starttime}}) -> + Port ! {self(), close} + end, + dict:to_list(BusyPorts)), + receive + {Port, closed} when is_port(Port) -> + exit(normal) %% XXX exits when first port is closed + end; + {'EXIT', Port, _Reason} when is_port(Port) -> + %% XXX supervisor doesn't restart fsyncport, why? + exit(port_terminated) + end. + +statreport(_Entry) -> + none. diff --git a/src/perm.erl b/src/perm.erl new file mode 100644 index 0000000..2ce5b46 --- /dev/null +++ b/src/perm.erl @@ -0,0 +1,95 @@ +%% +%% Copyright (c) 2014 Kungliga Tekniska Högskolan +%% (KTH Royal Institute of Technology, Stockholm, Sweden). +%% + +-module(perm). +-export([ensurefile/3]). + +fsync(Name) -> + fsyncport:fsync(Name). + +readfile_and_verify(Name, Content) -> + case file:read_file(Name) of + {ok, ContentsReadBinary} -> + ContentsRead = binary_to_list(ContentsReadBinary), + if Content == ContentsRead -> + ok; + true -> + {error, "File contents differ"} + end; + {error, Error} -> + {error, Error} + end. + +writefile(Name, NurseryName, Content) -> + case file:open(NurseryName, [write, exclusive]) of + {ok, File} -> + %io:format("Write file: ~p~n", [Name]), + ok = file:write(File, Content), + file:close(File), + Result = file:rename(NurseryName, Name), + Result; + {error, eexist} -> + %% Should not happen, file name should be unique + {error, eexist}; + {error, Error} -> + {error, Error} + end. + +make_dir(Name) -> + case file:make_dir(Name) of + ok -> + ok; + {error, eexist} -> + ok; + {error, Error} -> + {error, Error} + end. + +make_dirs([]) -> + ok; +make_dirs([Name | Rest]) -> + case make_dir(Name) of + ok -> + make_dirs(Rest); + {error, Error} -> + {error, Error} + end. + +path_for_key(Rootdir, Key) -> + Name = hex:bin_to_hexstr(Key), + [C1, C2, C3, C4, C5, C6 | _] = Name, + Firstlevel = Rootdir ++ [C1, C2], + Secondlevel = Firstlevel ++ "/" ++ [C3, C4], + Thirdlevel = Secondlevel ++ "/" ++ [C5, C6], + Fullpath = Thirdlevel ++ "/" ++ Name, + {[Firstlevel, Secondlevel, Thirdlevel], Fullpath}. + +tempfilename(Base) -> + {MegaSecs, Secs, MicroSecs} = now(), + Filename = io_lib:format("~s-~s-~p.~p", [Base, os:getpid(), + MegaSecs * 1000000 + Secs, MicroSecs]), + Filename. + +ensurefile(Rootdir, Key, Content) -> + {Dirs, Path} = path_for_key(Rootdir, Key), + case readfile_and_verify(Path, Content) of + ok -> + lists:foreach(fun (Dir) -> fsync(Dir) end, [Path, Rootdir | Dirs]); + {error, enoent} -> + case make_dirs([Rootdir, Rootdir ++ "nursery/"] ++ Dirs) of + ok -> + NurseryName = Rootdir ++ "nursery/" ++ + tempfilename(hex:bin_to_hexstr(Key)), + _Result = writefile(Path, NurseryName, Content), + lists:foreach(fun (Dir) -> + fsync(Dir) + end, + [Path, Rootdir | Dirs]); %% XXX check results + {error, Error} -> + io:format("Error creating directory: ~w~n", [Error]) + end; + {error, Error} -> + exit({perm, fileerror, "Error reading file", Error}) + end. diff --git a/src/plop_sup.erl b/src/plop_sup.erl index a5ce905..bcb9756 100644 --- a/src/plop_sup.erl +++ b/src/plop_sup.erl @@ -23,6 +23,11 @@ init(Args) -> permanent, 10000, worker, [db]}, + {fsync, + {fsyncport, start_link, []}, + permanent, + 10000, + worker, [fsyncport]}, {the_ht, {ht, start_link, []}, permanent, -- cgit v1.1 From eb95f6951e7a4abd2b7685b2de07de90b90ee0d2 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Thu, 25 Sep 2014 08:35:07 +0200 Subject: perm: Don't crash if file content is different, tell caller instead. Better error handling. --- src/perm.erl | 54 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/src/perm.erl b/src/perm.erl index 2ce5b46..34f431c 100644 --- a/src/perm.erl +++ b/src/perm.erl @@ -6,17 +6,25 @@ -module(perm). -export([ensurefile/3]). -fsync(Name) -> - fsyncport:fsync(Name). +fsync([]) -> + ok; +fsync([Name | Rest]) -> + case fsyncport:fsync(Name) of + ok -> + fsync(Rest); + {error, Error} -> + {error, Error} + end. readfile_and_verify(Name, Content) -> case file:read_file(Name) of {ok, ContentsReadBinary} -> ContentsRead = binary_to_list(ContentsReadBinary), - if Content == ContentsRead -> + if + Content == ContentsRead -> ok; - true -> - {error, "File contents differ"} + true -> + differ end; {error, Error} -> {error, Error} @@ -72,24 +80,32 @@ tempfilename(Base) -> MegaSecs * 1000000 + Secs, MicroSecs]), Filename. +exit_with_error(Error, Message) -> + io:format("~s: ~w~n", [Message, Error]), + exit({perm, fileerror, Message, Error}). + +check_error(ReturnValue, ErrorMessage) -> + case ReturnValue of + ok -> + ok; + {error, Error} -> + exit_with_error(Error, ErrorMessage) + end. + ensurefile(Rootdir, Key, Content) -> {Dirs, Path} = path_for_key(Rootdir, Key), case readfile_and_verify(Path, Content) of ok -> - lists:foreach(fun (Dir) -> fsync(Dir) end, [Path, Rootdir | Dirs]); + check_error(fsync([Path, Rootdir | Dirs]), "Error in fsync"); + differ -> + differ; {error, enoent} -> - case make_dirs([Rootdir, Rootdir ++ "nursery/"] ++ Dirs) of - ok -> - NurseryName = Rootdir ++ "nursery/" ++ - tempfilename(hex:bin_to_hexstr(Key)), - _Result = writefile(Path, NurseryName, Content), - lists:foreach(fun (Dir) -> - fsync(Dir) - end, - [Path, Rootdir | Dirs]); %% XXX check results - {error, Error} -> - io:format("Error creating directory: ~w~n", [Error]) - end; + check_error(make_dirs([Rootdir, Rootdir ++ "nursery/"] ++ Dirs), + "Error creating directory"), + NurseryName = Rootdir ++ "nursery/" ++ + tempfilename(hex:bin_to_hexstr(Key)), + _Result = writefile(Path, NurseryName, Content), + check_error(fsync([Path, Rootdir | Dirs]), "Error in fsync"); {error, Error} -> - exit({perm, fileerror, "Error reading file", Error}) + exit_with_error(Error, "Error reading file") end. -- cgit v1.1 From 11f8efc7fb27935761c38cf32f41836193ae97f4 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Thu, 25 Sep 2014 09:04:48 +0200 Subject: perm: Added readfile function. --- src/perm.erl | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/perm.erl b/src/perm.erl index 34f431c..5cd2889 100644 --- a/src/perm.erl +++ b/src/perm.erl @@ -4,7 +4,7 @@ %% -module(perm). --export([ensurefile/3]). +-export([ensurefile/3, readfile/2]). fsync([]) -> ok; @@ -18,14 +18,10 @@ fsync([Name | Rest]) -> readfile_and_verify(Name, Content) -> case file:read_file(Name) of - {ok, ContentsReadBinary} -> - ContentsRead = binary_to_list(ContentsReadBinary), - if - Content == ContentsRead -> - ok; - true -> - differ - end; + {ok, ContentsRead} when Content == ContentsRead -> + ok; + {ok, _ContentsRead} -> + differ; {error, Error} -> {error, Error} end. @@ -109,3 +105,14 @@ ensurefile(Rootdir, Key, Content) -> {error, Error} -> exit_with_error(Error, "Error reading file") end. + +readfile(Rootdir, Key) -> + {_Dirs, Path} = path_for_key(Rootdir, Key), + case file:read_file(Path) of + {ok, Contents} -> + Contents; + {error, enoent} -> + noentry; + {error, Error} -> + exit_with_error(Error, "Error reading file") + end. -- cgit v1.1 From 62e2a6e4849d342f90a3860554bf44df4e563d3b Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Thu, 25 Sep 2014 15:18:39 +0200 Subject: Added atomic module --- src/atomic.erl | 24 +++++++++++++++++++ src/hex.erl | 1 + src/perm.erl | 73 ++++++++++++---------------------------------------------- src/util.erl | 57 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 58 deletions(-) create mode 100644 src/atomic.erl create mode 100644 src/util.erl diff --git a/src/atomic.erl b/src/atomic.erl new file mode 100644 index 0000000..5bf5670 --- /dev/null +++ b/src/atomic.erl @@ -0,0 +1,24 @@ +%% +%% Copyright (c) 2014 Kungliga Tekniska Högskolan +%% (KTH Royal Institute of Technology, Stockholm, Sweden). +%% + +-module(atomic). +-export([replacefile/2, readfile/1]). + +-spec replacefile(string(), binary()) -> ok. +replacefile(Path, Content) -> + TempName = util:tempfilename(Path), + util:write_tempfile_and_rename(Path, TempName, Content), + util:fsync([Path, filename:dirname(Path)]). + +-spec readfile(string()) -> binary(). +readfile(Path) -> + case file:read_file(Path) of + {ok, Contents} -> + Contents; + {error, enoent} -> + noentry; + {error, Error} -> + util:exit_with_error(readfile, Error, "Error reading file") + end. diff --git a/src/hex.erl b/src/hex.erl index e3c8441..1eb1e6a 100644 --- a/src/hex.erl +++ b/src/hex.erl @@ -4,6 +4,7 @@ -module(hex). -export([bin_to_hexstr/1,hexstr_to_bin/1]). +-spec bin_to_hexstr(binary()) -> string(). bin_to_hexstr(Bin) -> lists:flatten([io_lib:format("~2.16.0B", [X]) || X <- binary_to_list(Bin)]). diff --git a/src/perm.erl b/src/perm.erl index 5cd2889..ccb23bc 100644 --- a/src/perm.erl +++ b/src/perm.erl @@ -6,16 +6,7 @@ -module(perm). -export([ensurefile/3, readfile/2]). -fsync([]) -> - ok; -fsync([Name | Rest]) -> - case fsyncport:fsync(Name) of - ok -> - fsync(Rest); - {error, Error} -> - {error, Error} - end. - +-spec readfile_and_verify(string(), binary()) -> ok | differ | {error, atom()}. readfile_and_verify(Name, Content) -> case file:read_file(Name) of {ok, ContentsRead} when Content == ContentsRead -> @@ -26,21 +17,7 @@ readfile_and_verify(Name, Content) -> {error, Error} end. -writefile(Name, NurseryName, Content) -> - case file:open(NurseryName, [write, exclusive]) of - {ok, File} -> - %io:format("Write file: ~p~n", [Name]), - ok = file:write(File, Content), - file:close(File), - Result = file:rename(NurseryName, Name), - Result; - {error, eexist} -> - %% Should not happen, file name should be unique - {error, eexist}; - {error, Error} -> - {error, Error} - end. - +-spec make_dir(string()) -> ok | {error, atom()}. make_dir(Name) -> case file:make_dir(Name) of ok -> @@ -51,6 +28,7 @@ make_dir(Name) -> {error, Error} end. +-spec make_dirs([string()]) -> ok | {error, atom()}. make_dirs([]) -> ok; make_dirs([Name | Rest]) -> @@ -61,6 +39,7 @@ make_dirs([Name | Rest]) -> {error, Error} end. +-spec path_for_key(string(), binary()) -> {[string()], string()}. path_for_key(Rootdir, Key) -> Name = hex:bin_to_hexstr(Key), [C1, C2, C3, C4, C5, C6 | _] = Name, @@ -70,49 +49,27 @@ path_for_key(Rootdir, Key) -> Fullpath = Thirdlevel ++ "/" ++ Name, {[Firstlevel, Secondlevel, Thirdlevel], Fullpath}. -tempfilename(Base) -> - {MegaSecs, Secs, MicroSecs} = now(), - Filename = io_lib:format("~s-~s-~p.~p", [Base, os:getpid(), - MegaSecs * 1000000 + Secs, MicroSecs]), - Filename. - -exit_with_error(Error, Message) -> - io:format("~s: ~w~n", [Message, Error]), - exit({perm, fileerror, Message, Error}). - -check_error(ReturnValue, ErrorMessage) -> - case ReturnValue of - ok -> - ok; - {error, Error} -> - exit_with_error(Error, ErrorMessage) - end. - +-spec ensurefile(string(), binary(), binary()) -> ok | differ. ensurefile(Rootdir, Key, Content) -> {Dirs, Path} = path_for_key(Rootdir, Key), case readfile_and_verify(Path, Content) of ok -> - check_error(fsync([Path, Rootdir | Dirs]), "Error in fsync"); + util:fsync([Path, Rootdir | Dirs]); differ -> differ; {error, enoent} -> - check_error(make_dirs([Rootdir, Rootdir ++ "nursery/"] ++ Dirs), - "Error creating directory"), + util:check_error(make_dirs([Rootdir, Rootdir ++ "nursery/"] + ++ Dirs), + makedir, "Error creating directory"), NurseryName = Rootdir ++ "nursery/" ++ - tempfilename(hex:bin_to_hexstr(Key)), - _Result = writefile(Path, NurseryName, Content), - check_error(fsync([Path, Rootdir | Dirs]), "Error in fsync"); + util:tempfilename(hex:bin_to_hexstr(Key)), + util:write_tempfile_and_rename(Path, NurseryName, Content), + util:fsync([Path, Rootdir | Dirs]); {error, Error} -> - exit_with_error(Error, "Error reading file") + util:exit_with_error(Error, readfile, "Error reading file") end. +-spec readfile(string(), binary()) -> binary(). readfile(Rootdir, Key) -> {_Dirs, Path} = path_for_key(Rootdir, Key), - case file:read_file(Path) of - {ok, Contents} -> - Contents; - {error, enoent} -> - noentry; - {error, Error} -> - exit_with_error(Error, "Error reading file") - end. + atomic:readfile(Path). diff --git a/src/util.erl b/src/util.erl new file mode 100644 index 0000000..48ebbb0 --- /dev/null +++ b/src/util.erl @@ -0,0 +1,57 @@ +%% +%% Copyright (c) 2014 Kungliga Tekniska Högskolan +%% (KTH Royal Institute of Technology, Stockholm, Sweden). +%% + +-module(util). +-export([tempfilename/1, fsync/1, exit_with_error/3, + check_error/3, write_tempfile_and_rename/3]). + +-spec tempfilename(string()) -> string(). +tempfilename(Base) -> + {MegaSecs, Secs, MicroSecs} = now(), + Filename = io_lib:format("~s-~s-~p.~p", [Base, os:getpid(), + MegaSecs * 1000000 + Secs, MicroSecs]), + Filename. + +-spec fsync([string()]) -> ok. +fsync([]) -> + ok; +fsync([Name | Rest]) -> + case fsyncport:fsync(Name) of + ok -> + fsync(Rest); + {error, Error} -> + exit_with_error(fsync, Error, "Error in fsync") + end. + +-spec exit_with_error(atom(), atom(), string()) -> no_return(). +exit_with_error(Operation, Error, ErrorMessage) -> + io:format("~s(~w): ~w~n", [ErrorMessage, Operation, Error]), + exit({fileerror, Operation, Error, ErrorMessage}). + +-spec check_error(any(), atom(), string()) -> ok. +check_error(ReturnValue, Operation, ErrorMessage) -> + case ReturnValue of + ok -> + ok; + {error, Error} -> + exit_with_error(Operation, Error, ErrorMessage) + end. + +-spec write_tempfile_and_rename(string(), string(), binary()) -> ok. +write_tempfile_and_rename(Name, NurseryName, Content) -> + case file:open(NurseryName, [write, exclusive]) of + {ok, File} -> + ok = file:write(File, Content), + file:close(File), + check_error(file:rename(NurseryName, Name), rename, + "Error when renaming tempfile to final file"); + {error, eexist} -> + %% Should not happen, file name should be unique + exit_with_error(writefile, eexist, + "File existed when creating tempfile"); + {error, Error} -> + exit_with_error(writefile, Error, + "Error when creating tempfile") + end. -- cgit v1.1 From f5fd5ac81904160c6bd4ccda04e333ca44461b22 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Thu, 25 Sep 2014 17:57:31 +0200 Subject: Add Makefile to c_src --- Makefile | 5 ++++- c_src/Makefile | 13 +++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 c_src/Makefile diff --git a/Makefile b/Makefile index 4e54096..b4bb715 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ build all: (cd c_src && make all) - cp c_src/fsynchelper priv/fsynchelper + mkdir -p priv + cp c_src/fsynchelper priv/ erl -make clean: + (cd c_src && make clean) + -rm priv/fsynchelper -rm ebin/*.beam diff --git a/c_src/Makefile b/c_src/Makefile new file mode 100644 index 0000000..338dc6d --- /dev/null +++ b/c_src/Makefile @@ -0,0 +1,13 @@ +CC = gcc +CFLAGS = -Wall +LDFLAGS = + +PORTS = fsynchelper + +all: $(PORTS) + +clean: + rm -f *.o $(PORTS) + +fsynchelper: net_read_write.o erlport.o fsynchelper.o + $(CC) $(LDFLAGS) -o fsynchelper net_read_write.o erlport.o fsynchelper.o -- cgit v1.1 From 7fe4225f5969d0c5135363dcfa99d7511100a0b8 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Fri, 26 Sep 2014 03:27:43 +0200 Subject: Added implementation of index file --- src/index.erl | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 src/index.erl diff --git a/src/index.erl b/src/index.erl new file mode 100644 index 0000000..5382485 --- /dev/null +++ b/src/index.erl @@ -0,0 +1,81 @@ +%% +%% Copyright (c) 2014 Kungliga Tekniska Högskolan +%% (KTH Royal Institute of Technology, Stockholm, Sweden). +%% + +%% Implements an interface to a file pair (basename and basename.chksum) +%% that stores an ordered list of fixed-size entries. Entries can be +%% added at the end and are retrieved by index. The list can also be +%% truncated. + +-module(index). +-export([get/2, add/3, addlast/2, truncate/2]). + +-define(ENTRYSIZE, 32). +-define(ENTRYSIZEINFILE, (?ENTRYSIZE*2+1)). + +-spec add(string(), integer() | last, binary()) -> ok. +add(Basepath, Index, Entry) when is_binary(Entry), size(Entry) == ?ENTRYSIZE -> + case file:open(Basepath, [read, write, binary]) of + {ok, File} -> + {ok, Position} = file:position(File, eof), + case Index of + last when Position rem ?ENTRYSIZEINFILE == 0 -> + ok; + Index when is_integer(Index), + Index * ?ENTRYSIZEINFILE == Position -> + ok + end, + EntryText = hex:bin_to_hexstr(Entry) ++ "\n", + ok = file:write(File, EntryText), + file:close(File); + {error, Error} -> + util:exit_with_error(Error, writefile, + "Error opening file for writing") + end. + +truncate(Basepath, Index) -> + case file:open(Basepath, [read, write, binary]) of + {ok, File} -> + {ok, _Position} = file:position(File, Index * ?ENTRYSIZEINFILE), + ok = file:truncate(File), + file:close(File); + {error, Error} -> + util:exit_with_error(Error, writefile, + "Error opening file for writing") + end. + + +-spec addlast(string(), integer()) -> ok. +addlast(Basepath, Entry) -> + add(Basepath, last, Entry). + +decodedata(EntryText) when length(EntryText) == ?ENTRYSIZEINFILE -> + case [lists:last(EntryText)] of + "\n" -> + hex:hexstr_to_bin(lists:droplast(EntryText)); + _ -> + util:exit_with_error(badformat, readindex, + "Index line not ending with linefeed") + end. + +-spec get(string(), integer()) -> binary(). +get(Basepath, Index) -> + case file:open(Basepath, [read, binary]) of + {ok, File} -> + {ok, Filesize} = file:position(File, eof), + if + Index * ?ENTRYSIZEINFILE + ?ENTRYSIZEINFILE =< Filesize -> + {ok, _Position} = file:position(File, + Index * ?ENTRYSIZEINFILE), + {ok, EntryText} = file:read(File, ?ENTRYSIZEINFILE), + Entry = decodedata(binary_to_list(EntryText)), + file:close(File), + Entry; + true -> + noentry + end; + {error, Error} -> + util:exit_with_error(Error, readfile, + "Error opening file for reading") + end. -- cgit v1.1 From 9c0d01c3059a7f82ac9acd5574755feed2f351a8 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Fri, 26 Sep 2014 10:28:47 +0200 Subject: index: Added fsync --- src/index.erl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/index.erl b/src/index.erl index 5382485..5fd468b 100644 --- a/src/index.erl +++ b/src/index.erl @@ -7,6 +7,10 @@ %% that stores an ordered list of fixed-size entries. Entries can be %% added at the end and are retrieved by index. The list can also be %% truncated. +%% +%% Writes(add, truncate, addlast) need to be serialized. + +%% TODO: Checksums -module(index). -export([get/2, add/3, addlast/2, truncate/2]). @@ -28,7 +32,8 @@ add(Basepath, Index, Entry) when is_binary(Entry), size(Entry) == ?ENTRYSIZE -> end, EntryText = hex:bin_to_hexstr(Entry) ++ "\n", ok = file:write(File, EntryText), - file:close(File); + ok = file:close(File), + util:fsync([Basepath, filename:dirname(Basepath)]); {error, Error} -> util:exit_with_error(Error, writefile, "Error opening file for writing") @@ -39,7 +44,8 @@ truncate(Basepath, Index) -> {ok, File} -> {ok, _Position} = file:position(File, Index * ?ENTRYSIZEINFILE), ok = file:truncate(File), - file:close(File); + ok = file:close(File), + util:fsync([Basepath, filename:dirname(Basepath)]); {error, Error} -> util:exit_with_error(Error, writefile, "Error opening file for writing") -- cgit v1.1 From dc6b383aa4a9cc1e2793c4d9f3be81bd676b7a20 Mon Sep 17 00:00:00 2001 From: Linus Nordberg Date: Sat, 27 Sep 2014 15:45:14 +0200 Subject: wip --- README | 2 +- include/plop.hrl | 36 ------- src/db.erl | 126 ++++++++++++++---------- src/db.hrl | 19 ++-- src/ht.erl | 26 ++--- src/plop.erl | 283 ++++++++++++++--------------------------------------- test/plop_test.erl | 94 ------------------ 7 files changed, 167 insertions(+), 419 deletions(-) delete mode 100644 test/plop_test.erl diff --git a/README b/README index b5910a5..5d26ec6 100644 --- a/README +++ b/README @@ -13,7 +13,7 @@ Compile the application Start the application locally Very first time, before there is a database: - $ erl -boot start_sasl -pa ebin -eval "plop_app:install([node()])." + $ erl -boot start_sasl -pa ebin -eval "plop_app:install([node()])."xo There should now exist a directory Mnesia.nonode@nohost/ with four files in it. diff --git a/include/plop.hrl b/include/plop.hrl index b62a04a..b04af0d 100644 --- a/include/plop.hrl +++ b/include/plop.hrl @@ -5,46 +5,12 @@ %%% database storage, some for interfacing with consumers and some are %%% for serialisation. --define(PLOPVERSION, 0). - --type signature_type() :: certificate_timestamp | tree_hash | test. % uint8 --type entry_type() :: x509 | precert | test. % uint16 --type leaf_type() :: timestamped_entry | test. % uint8 - %% @doc Merkle Tree Leaf -- what's sent as 'leaf_input' in response to %% get-entries requests and also the input to the hash function for %% leaf hashes in the tree. RFC 6962 sect 3.4. --record(mtl, { - version = ?PLOPVERSION :: non_neg_integer(), - leaf_type = timestamped_entry :: leaf_type(), - entry :: timestamped_entry() - }). --type mtl() :: #mtl{}. - --record(spt, { - version :: non_neg_integer(), % uint8 - logid :: binary(), % SHA-256 over DER encoded public log key - timestamp :: integer(), % uint64 - signature :: signature() - }). --type spt() :: #spt{}. - -%% A plop entry with timestamp. Part of the Merkle Tree Leaf -%% structure. --record(timestamped_entry, { - timestamp = now :: now | integer(), - entry :: plop_entry() - }). --type timestamped_entry() :: #timestamped_entry{}. %% An entry, without the timestamp. This is what we hash over and %% store in the the database for finding duplicated submissions. --record(plop_entry, { - type :: entry_type(), - data :: binary() - }). --type plop_entry() :: #plop_entry{}. - -record(sth, { treesize :: integer(), timestamp :: integer(), @@ -69,5 +35,3 @@ signature :: binary() }). -type signature() :: #signature{}. - --export_type([timestamped_entry/0, mtl/0, entry_type/0]). diff --git a/src/db.erl b/src/db.erl index 3ab2d1b..ddebbeb 100644 --- a/src/db.erl +++ b/src/db.erl @@ -7,7 +7,8 @@ %% API. -export([start_link/0, stop/0]). -export([init_db/0, init_db/1, init_tables/0, init_tables/1]). --export([add/1, find/2, get_by_index/2, get_by_index_sorted/2, size/0]). +-export([add/4, size/0]). +-export([get_by_index/1, get_by_indices/3, get_by_leaf_hash/1, get_by_entry_hash/1]). %% API for testing. -export([dump/1, destroy_tables/0, info_tables/0, dump_to_file/1]). %% gen_server callbacks. @@ -72,24 +73,31 @@ stop() -> %%%%%%%%%%%%%%%%%%%% %% Public API. --spec add(plop()) -> {atomic, ok}. -add(Entry) -> - gen_server:call(?MODULE, {add, Entry}). +-spec add(binary(), binary(), binary(), non_neg_integer()) -> ok. +add(LeafHash, EntryHash, Data, Index) -> + gen_server:call(?MODULE, {add, {LeafHash, EntryHash, Data, Index}}). -%% @doc Find one entry. --spec find(entryhash | mtlhash | index, binary()) -> - [] | plop() | duplicate_hash_in_db. -find(Type, Hash) -> - gen_server:call(?MODULE, {find, Type, Hash}). +-spec get_by_indices(non_neg_integer(), + non_neg_integer(), + {sorted, true|false}) -> + [{non_neg_integer(), binary(), binary()}]. +get_by_indices(Start, End, {sorted, Sorted}) -> + gen_server:call(?MODULE, {get_by_indices, {Start, End, Sorted}}). --spec get_by_index(non_neg_integer(), non_neg_integer()) -> [{mtl(), binary()}]. -get_by_index(Start, End) -> - gen_server:call(?MODULE, {get_by_index, {Start, End}}). +-spec get_by_index(binary()) -> notfound | + {non_neg_integer(), binary(), binary()}. +get_by_index(Index) -> + gen_server:call(?MODULE, {get_by_index, Index}). --spec get_by_index_sorted(non_neg_integer(), non_neg_integer()) -> - [{mtl(), binary()}]. -get_by_index_sorted(Start, End) -> - gen_server:call(?MODULE, {get_by_index_sorted, {Start, End}}). +-spec get_by_leaf_hash(binary()) -> notfound | + {non_neg_integer(), binary(), binary()}. +get_by_leaf_hash(LeafHash) -> + gen_server:call(?MODULE, {get_by_leaf_hash, LeafHash}). + +-spec get_by_entry_hash(binary()) -> notfound | + {non_neg_integer(), binary(), binary()}. +get_by_entry_hash(EntryHash) -> + gen_server:call(?MODULE, {get_by_entry_hash, EntryHash}). %% Testing and debugging. dump(Table) -> @@ -117,50 +125,64 @@ terminate(_Reason, _State) -> handle_call(stop, _From, State) -> {stop, normal, stopped, State}; -handle_call({add, Entry}, _From, State) -> - F = fun() -> - mnesia:write(Entry) - end, - Res = mnesia:transaction(F), - {reply, Res, State}; - -handle_call({dump, Table}, _From, State) -> - F = fun() -> - Q = qlc:q([E || E <- mnesia:table(Table)]), - qlc:e(Q) +handle_call({add, {LeafHash, EntryHash, Data, Index}}, _From, State) -> + R = mnesia:transaction( + fun() -> + mnesia:write( + #plop{ + index = Index, + mtlhash = LeafHash, + entryhash = EntryHash, + logentry = Data}) + end), + {reply, R, State}; + +handle_call({get_by_indices, {Start, End, Sorted}}, _From, State) -> + R = case Sorted of + false -> + select_index(Start, End); + true -> + %% FIXME: RAM hog -- how bad is it? + lists:sort(select_index(Start, End)) end, - Res = mnesia:transaction(F), - {reply, Res, State}; + {reply, R, State}; -handle_call({find, entryhash, Hash}, _From, State) -> +handle_call({get_by_index, Index}, _From, State) -> {reply, - find_entry(fun() -> mnesia:index_read(plop, Hash, #plop.entryhash) end), - State}; -handle_call({find, mtlhash, Hash}, _From, State) -> - {reply, - find_entry(fun() -> mnesia:index_read(plop, Hash, #plop.mtlhash) end), + find_entry(fun() -> mnesia:read(plop, Index) end), State}; -handle_call({find, index, Index}, _From, State) -> + +handle_call({get_by_leaf_hash, LeafHash}, _From, State) -> {reply, - find_entry(fun() -> mnesia:read(plop, Index) end), + find_entry(fun() -> + mnesia:index_read(plop, LeafHash, #plop.mtlhash) + end), State}; -handle_call({get_by_index, {Start, End}}, _From, State) -> - Res = [{MTL, Extra} || [_Index, MTL, Extra] <- select_index(Start, End)], - {reply, Res, State}; +handle_call({get_by_entry_hash, EntryHash}, _From, State) -> + {reply, + find_entry(fun() -> + mnesia:index_read(plop, EntryHash, #plop.entryhash) + end), + State}; -handle_call({get_by_index_sorted, {Start, End}}, _From, State) -> - %% FIXME: RAM hog -- how bad is it? - Res = [{MTL, Extra} || [_Index, MTL, Extra] <- lists:sort(select_index(Start, End))], - {reply, Res, State}. +handle_call({dump, Table}, _From, State) -> + R = mnesia:transaction( + fun() -> + Q = qlc:q([E || E <- mnesia:table(Table)]), + qlc:e(Q) + end), + {reply, R, State}. %%%%%%%%%%%%%%%%%%%% %% Helper functions. +-spec select_index(non_neg_integer(), non_neg_integer()) -> + [{non_neg_integer(), binary(), binary()}]. select_index(Start, End) -> F = fun() -> - %% Get index, mtl and extra_data. - MatchHead = {plop, '$1', '_', '_', '$2', '$3', '_'}, + %% Get index, mtlhash and logentry. + MatchHead = {plop, '$1', '$2', '_', '$3'}, Guard = [{'>=', '$1', Start}, {'=<', '$1', End}], Result = ['$$'], mnesia:select(plop, [{MatchHead, Guard, Result}]) @@ -168,11 +190,13 @@ select_index(Start, End) -> {atomic, Res} = mnesia:transaction(F), Res. --spec find_entry(fun()) -> [] | plop() | duplicate_hash_in_db. +-spec find_entry(fun()) -> notfound | + {non_neg_integer(), binary(), binary()} | + duplicate. find_entry(Fun) -> {atomic, Result} = mnesia:transaction(Fun), - case length(Result) of - 0 -> []; - 1 -> hd(Result); - _ -> duplicate_hash_in_db % FIXME: log an error? + case Result of + [] -> notfound; + [#plop{index = I, mtlhash = H, logentry = E}] -> {I, H, E}; + _ -> duplicate end. diff --git a/src/db.hrl b/src/db.hrl index bea9131..9e09b66 100644 --- a/src/db.hrl +++ b/src/db.hrl @@ -1,17 +1,10 @@ %%% Copyright (c) 2014, NORDUnet A/S. %%% See LICENSE for licensing information. -%% @doc What's stored in the database. -%% 'index' is the primary key, 'entryhash' and 'mtlhash' are also -%% indexed, see init_tables/1. -%% NOTE: Don't change anything here without also fixing -%% select_index/2, which depends on the order of fields. +%% @doc What's stored in the database. +%% 'mtlhash' and 'entryhash' are also indexed, see init_tables/1. -record(plop, { - index :: non_neg_integer(), % Primary key. - entryhash :: binary(), % Hash over #plop_entry{} in mtl. - mtlhash :: binary(), % Merkle Tree Leaf hash. - mtl :: mtl(), % Merkle Tree Leaf. - extra_data :: binary(), % Data not part of mtl. - spt :: spt() % Signed Plop Timestamp. - }). --type plop() :: #plop{}. + index :: non_neg_integer(), % Primary key. + mtlhash :: binary(), % Merkle Tree Leaf hash. + entryhash :: binary(), % Hash for duplicate detection. + logentry :: binary()}). % Data. diff --git a/src/ht.erl b/src/ht.erl index 74f8ab4..cd4e57c 100644 --- a/src/ht.erl +++ b/src/ht.erl @@ -51,8 +51,8 @@ stop() -> gen_server:call(?MODULE, stop). size() -> gen_server:call(?MODULE, size). -add(Entry) -> - gen_server:call(?MODULE, {add, Entry}). +add(Hash) -> + gen_server:call(?MODULE, {add, Hash}). root() -> gen_server:call(?MODULE, root). root(Version) -> @@ -91,8 +91,8 @@ handle_call(stop, _From, State) -> {stop, normal, stopped, State}; handle_call(size, _From, State) -> {reply, State#tree.version + 1, State}; -handle_call({add, Entry}, _From, State) -> - {reply, ok, add(State, Entry)}; +handle_call({add, Hash}, _From, State) -> + {reply, ok, add(State, Hash)}; handle_call(root, _From, State) -> {NewState, Hash} = head(State, State#tree.version), {reply, Hash, NewState}; @@ -245,11 +245,10 @@ first_left_node(Layer, Index, BAL) -> false -> {Layer, Index} end. -%% @doc Add an entry but don't update the tree. +%% @doc Add a hash but don't update the tree. -spec add(tree(), binary()) -> tree(). -add(Tree = #tree{version = V, store = Store}, Entry) -> - Tree#tree{version = V + 1, - store = ts:add(Store, 0, mkleafhash(Entry))}. +add(Tree = #tree{version = V, store = Store}, Hash) -> + Tree#tree{version = V + 1, store = ts:add(Store, 0, Hash)}. %% @doc Return a new tree. -spec new(list()) -> tree(). @@ -261,13 +260,14 @@ new([-1]) -> new([]); %% Initialise tree from db. new([Version]) when is_integer(Version) -> - foldl(fun(MTL, Tree) -> + foldl(fun(Hash, Tree) -> %% Return value becomes Tree in next invocation. - add(Tree, plop:serialise(MTL)) - end, new([]), [X || {X, _} <- db:get_by_index_sorted(0, Version)]); -%% Initialise tree from List. + add(Tree, Hash) + end, new([]), [H || {_I, H, _E} <- + db:get_by_indices(0, Version, {sorted, true})]); +%% Initialise tree from List with hashes. new([List]) when is_list(List) -> - foldl(fun(SerialisedMTL, Tree) -> add(Tree, SerialisedMTL) end, + foldl(fun(Hash, Tree) -> add(Tree, Hash) end, new([]), List). update(Tree) -> diff --git a/src/plop.erl b/src/plop.erl index 5443183..b59e19b 100644 --- a/src/plop.erl +++ b/src/plop.erl @@ -26,7 +26,8 @@ %% API. -export([start_link/2, stop/0]). -export([get_logid/0, serialise/1]). --export([add/2, sth/0, get/2, consistency/2, inclusion/2, inclusion_and_entry/2]). +-export([add/4, sth/0, get/1, get/2, spt/2, consistency/2, inclusion/2, inclusion_and_entry/2]). +-export([generate_timestamp/0]). %% API for tests. -export([read_keyfile_rsa/2, read_keyfiles_ec/2]). -export([testing_get_pubkey/0]). @@ -35,7 +36,7 @@ handle_cast/2, handle_info/2, code_change/3]). -include("$CTROOT/plop/include/plop.hrl"). --include("db.hrl"). +%%-include("db.hrl"). -include_lib("public_key/include/public_key.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -46,6 +47,12 @@ privkey :: public_key:rsa_private_key(), logid :: binary()}). +%%%%% moved from plop.hrl, maybe remove +-define(PLOPVERSION, 0). +-type signature_type() :: certificate_timestamp | tree_hash | test. % uint8 +%%%%% + + %% @doc The parts of an STH which is to be signed. Used as the %% interface to plop:sth/1, for testing. -record(sth_signed, { @@ -57,17 +64,6 @@ }). -type sth_signed() :: #sth_signed{}. -%% @doc What's signed in an SPT. Used for serialisation before hasning -%% and signing. FIXME: Overlapping #spt{} -- merge somehow? --record(spt_signed, { - version = ?PLOPVERSION :: non_neg_integer(), - signature_type :: signature_type(), - timestamp :: integer(), - entry_type :: entry_type(), - signed_entry :: binary() - }). --type spt_signed() :: #spt_signed{}. - start_link(Keyfile, Passphrase) -> gen_server:start_link({local, ?MODULE}, ?MODULE, [Keyfile, Passphrase], []). @@ -101,22 +97,31 @@ terminate(_Reason, _State) -> ok. %%%%%%%%%%%%%%%%%%%% --spec add(timestamped_entry(), binary()) -> spt(). -add(Entry, ExtraData) -> - gen_server:call(?MODULE, {add, {Entry, ExtraData}}). +-spec add(binary(), binary(), binary(), binary()) -> ok. +add(LogEntry, TreeLeafHash, EntryHash, DataToSign) -> + gen_server:call(?MODULE, + {add, {LogEntry, TreeLeafHash, EntryHash, DataToSign}}). + sth() -> gen_server:call(?MODULE, {sth, []}). --spec get(non_neg_integer(), non_neg_integer()) -> [{mtl(), binary()}]. + +-spec get(non_neg_integer(), non_neg_integer()) -> + [{non_neg_integer(), binary(), binary()}]. get(Start, End) -> - gen_server:call(?MODULE, {get, {Start, End}}). + gen_server:call(?MODULE, {get, {index, Start, End}}). + +get(Hash) -> + gen_server:call(?MODULE, {get, {hash, Hash}}). + consistency(TreeSizeFirst, TreeSizeSecond) -> gen_server:call(?MODULE, {consistency, {TreeSizeFirst, TreeSizeSecond}}). -spec inclusion(binary(), non_neg_integer()) -> - {ok, mtl()} | {notfound, string()}. + {ok, {binary(), binary()}} | {notfound, string()}. inclusion(Hash, TreeSize) -> gen_server:call(?MODULE, {inclusion, {Hash, TreeSize}}). -spec inclusion_and_entry(non_neg_integer(), non_neg_integer()) -> - {ok, {mtl(), binary()}} | {notfound, string()}. + {ok, {binary(), binary()}} | + {notfound, string()}. inclusion_and_entry(Index, TreeSize) -> gen_server:call(?MODULE, {inclusion_and_entry, {Index, TreeSize}}). get_logid() -> @@ -124,118 +129,61 @@ get_logid() -> testing_get_pubkey() -> gen_server:call(?MODULE, {test, pubkey}). %%%%%%%%%%%%%%%%%%%% -handle_call(stop, _From, State) -> - {stop, normal, stopped, State}; - -%% FIXME: What's the right interface for add()? Need to be able to set -%% version and signature type, at least. That's missing from -%% #timestamped_entry, so add it somehow. -handle_call({add, - {#timestamped_entry{timestamp = Timestamp_in, entry = Entry}, - ExtraData}}, - _From, - State = #state{privkey = Privkey, logid = LogID}) -> - TimestampedEntry = #timestamped_entry{ - timestamp = timestamp(Timestamp_in), - entry = Entry}, - {ok, SPT} = do_add(TimestampedEntry, ExtraData, Privkey, LogID), - {reply, SPT, State}; +handle_call(stop, _From, Plop) -> + {stop, normal, stopped, Plop}; -handle_call({sth, Data}, _From, - Plop = #state{privkey = PrivKey}) -> - {reply, sth(PrivKey, Data), Plop}; +handle_call({get, {index, Start, End}}, _From, Plop) -> + {reply, db:get_by_indices(Start, End, {sorted, false}), Plop}; -handle_call({get, {Start, End}}, _From, Plop) -> - {reply, db:get_by_index(Start, End), Plop}; +handle_call({get, {hash, EntryHash}}, _From, Plop) -> + {reply, db:get_by_entry_hash(EntryHash), Plop}; handle_call({get, logid}, _From, Plop = #state{logid = LogID}) -> {reply, LogID, Plop}; +handle_call({add, {LogEntry, TreeLeafHash, EntryHash}}, _From, Plop) -> + ok = ht:add(TreeLeafHash), + ok = db:add(TreeLeafHash, EntryHash, LogEntry, ht:size()), + {reply, ok, Plop}; + +handle_call({sth, Data}, _From, + Plop = #state{privkey = PrivKey}) -> + {reply, sth(PrivKey, Data), Plop}; + handle_call({consistency, {First, Second}}, _From, Plop) -> {reply, ht:consistency(First - 1, Second - 1), Plop}; handle_call({inclusion, {Hash, TreeSize}}, _From, Plop) -> - R = case db:find(mtlhash, Hash) of - [] -> + R = case db:get_by_entry_hash(Hash) of + notfound -> {notfound, "Unknown hash"}; % FIXME: include Hash - {plop, I, _EntryHash, _MTLHash, _MTL, _ExtraData, _SPT} -> + {I, _MTLHash, _Entry} -> {ok, I, ht:path(I, TreeSize - 1)} end, {reply, R, Plop}; handle_call({inclusion_and_entry, {Index, TreeSize}}, _From, Plop) -> - R = case db:find(index, Index) of - [] -> + R = case db:get_by_index(Index) of + notfound -> {notfound, "Unknown index"}; % FIXME: include Index - {plop, I, _EntryHash, _MTLHash, MTL, ExtraData, _SPT} -> - {ok, MTL, ExtraData, ht:path(I, TreeSize - 1)} - end, + {I, _MTLHash, Entry} -> + {ok, Entry, ht:path(I, TreeSize - 1)} + end, {reply, R, Plop}; handle_call({test, pubkey}, _From, Plop = #state{pubkey = PK}) -> {reply, PK, Plop}. -%%%%%%%%%%%%%%%%%%%% -%% db_get_single_entry(N) -> -%% [#mtl{entry = #timestamped_entry{entry = #plop_entry{data = Data}}}] = -%% db:get_by_index(N, N), -%% Data. - --spec do_add(timestamped_entry(), - binary(), - public_key:rsa_private_key(), - binary()) -> {ok, spt()} | {error, any()}. -do_add(TimestampedEntry, ExtraData, Privkey, LogID) -> - DB_hash = crypto:hash(sha256, - serialise(TimestampedEntry#timestamped_entry.entry)), - Record = db:find(entryhash, DB_hash), - case Record of - #plop{index = _I, mtl = MTL, spt = SPT} -> - %% Costly database consistency checking. FIXME: Remove. - Record = Record#plop{ - entryhash = DB_hash, - mtlhash = ht:leaf_hash(serialise(MTL)), - mtl = MTL}, - {ok, SPT}; - [] -> - NewSPT = spt(LogID, Privkey, TimestampedEntry), - MTL = #mtl{entry = TimestampedEntry}, - MTLtext = serialise(MTL), - DB_data = #plop{index = ht:size(), - entryhash = DB_hash, - mtlhash = ht:leaf_hash(MTLtext), - mtl = MTL, - extra_data = ExtraData, - spt = NewSPT}, - {atomic, ok} = db:add(DB_data), - {ht:add(MTLtext), NewSPT}; - Err -> {error, Err} - end. - %% @doc Signed Plop Timestamp, conformant to an SCT in RFC6962 3.2 and %% RFC5246 4.7. --spec spt(binary(), public_key:rsa_private_key(), timestamped_entry()) -> spt(). -spt(LogID, PrivKey, #timestamped_entry{ - timestamp = Timestamp, - entry = #plop_entry{type = EntryType, data = EntryData} - }) -> - BinToSign = serialise(#spt_signed{ - signature_type = certificate_timestamp, - timestamp = Timestamp, - entry_type = EntryType, - signed_entry = EntryData}), - Signature = #signature{ - algorithm = #sig_and_hash_alg{ - hash_alg = sha256, - signature_alg = ecdsa}, - signature = signhash(BinToSign, PrivKey)}, - #spt{ - version = ?PLOPVERSION, - logid = LogID, - timestamp = Timestamp, - signature = Signature}. +-spec spt(public_key:ec_private_key(), binary()) -> signature(). +spt(PrivKey, SerialisedData) -> + #signature{algorithm = #sig_and_hash_alg{ + hash_alg = sha256, + signature_alg = ecdsa}, + signature = signhash(SerialisedData, PrivKey)}. %% @doc Signed Tree Head as specified in RFC6962 section 3.2. -spec sth(#'ECPrivateKey'{}, sth_signed() | list()) -> sth(). @@ -336,15 +284,6 @@ signature_type(certificate_timestamp) -> 0; signature_type(tree_hash) -> 1; signature_type(test) -> 2. --spec entry_type(entry_type()) -> integer(). -entry_type(x509) -> 0; -entry_type(precert) -> 1; -entry_type(test) -> 2. - --spec leaf_type(leaf_type()) -> integer(). -leaf_type(timestamped_entry) -> 0; -leaf_type(test) -> 1. - -spec hash_alg_type(hash_alg_type()) -> integer(). hash_alg_type(none) -> 0; hash_alg_type(md5) -> 1; @@ -360,6 +299,7 @@ signature_alg_type(rsa) -> 1; signature_alg_type(dsa) -> 2; signature_alg_type(ecdsa) -> 3. +%% TODO: Remove. -spec timestamp(now | integer()) -> integer(). timestamp(Timestamp) -> case Timestamp of @@ -371,58 +311,15 @@ timestamp(Timestamp) -> _ -> Timestamp end. -serialise_tls_vector(Binary, LengthLen) -> - Length = byte_size(Binary), - <>. +-spec generate_timestamp() -> integer(). +generate_timestamp() -> + {NowMegaSec, NowSec, NowMicroSec} = now(), + trunc(NowMegaSec * 1.0e9 + + NowSec * 1.0e3 + + NowMicroSec / 1.0e3). --spec serialise(plop_entry() | timestamped_entry() | mtl() | - spt() | spt_signed() | sth() | sth_signed() | - sig_and_hash_alg() | signature()) -> binary(). -serialise(#plop_entry{ - type = TypeAtom, - data = Data - }) -> - EntryType = entry_type(TypeAtom), - DataVector = serialise_tls_vector(Data, 3), - <>; -serialise(#timestamped_entry{ - timestamp = Timestamp, - entry = PlopEntry - }) -> - Extensions = <<>>, - list_to_binary([<>, serialise(PlopEntry), serialise_tls_vector(Extensions, 2)]); -serialise(#spt{ - version = Version, - logid = LogID, - timestamp = Timestamp, - signature = Signature - }) -> - list_to_binary([<>, - serialise(Signature)]); -serialise(#spt_signed{ - version = Version, - signature_type = SigtypeAtom, - timestamp = Timestamp, - entry_type = EntrytypeAtom, - signed_entry = Entry - }) -> - Sigtype = signature_type(SigtypeAtom), - Entrytype = entry_type(EntrytypeAtom), - Extensions = <<>>, - list_to_binary( - [<>, - serialise_tls_vector(Entry, 3), - serialise_tls_vector(Extensions, 2)]); -serialise(#mtl{ % Merkle Tree Leaf. - version = Version, - leaf_type = TypeAtom, - entry = TimestampedEntry - }) -> - LeafType = leaf_type(TypeAtom), - list_to_binary([<>, serialise(TimestampedEntry)]); +-spec serialise(sth() | sth_signed() | sig_and_hash_alg() | signature()) -> + binary(). serialise(#sth_signed{ % Signed Tree Head. version = Version, signature_type = SigtypeAtom, @@ -456,48 +353,12 @@ serialise(#signature{ %%%%%%%%%%%%%%%%%%%% %% Internal tests. For more tests see ../test/. -serialise_test_() -> - [?_assertEqual( - <<0:8, 0:8, 0:64, 0:16, "foo">>, - serialise(#spt_signed{ - version = 0, - signature_type = certificate_timestamp, - timestamp = 0, - entry_type = x509, - signed_entry = <<"foo">>}))]. -add_test() -> - {ok, S} = init([?TESTPRIVKEYFILE, ?TESTPUBKEYFILE]), - - Data1 = <<"some data">>, - ExtraData1 = <<"some extra data">>, - {_Tree, SPT} = - do_add(#timestamped_entry{ - timestamp = 4711, - entry = #plop_entry{type = test, data = Data1}}, - ExtraData1, - S#state.privkey, - S#state.logid), - {_Tree1, SPT1} = - do_add(#timestamped_entry{ - timestamp = 4712, - entry = #plop_entry{type = test, data = Data1}}, - ExtraData1, - S#state.privkey, - S#state.logid), - ?assertEqual(SPT, SPT1), - - TE = #timestamped_entry{ - timestamp = 0, - entry = #plop_entry{type = test, data = <<"some data">>}}, - SPTeq1 = spt(S#state.logid, S#state.privkey, TE), - SPTeq2 = spt(S#state.logid, S#state.privkey, TE), - ?assertNotEqual(SPTeq1, SPTeq2), % DSA signatures differ! - ok. - -%% add_random_data(N, Size) -> -%% lists:foreach( -%% fun(_) -> plop:add(#timestamped_entry -%% {entry = #plop_entry -%% {type = test, -%% data = crypto:rand_bytes(Size)}}) end, -%% lists:seq(1, N)). +%% serialise_test_() -> +%% [?_assertEqual( +%% <<0:8, 0:8, 0:64, 0:16, "foo">>, +%% serialise(#spt_signed{ +%% version = 0, +%% signature_type = certificate_timestamp, +%% timestamp = 0, +%% entry_type = x509, +%% signed_entry = <<"foo">>}))]. diff --git a/test/plop_test.erl b/test/plop_test.erl deleted file mode 100644 index ce2a052..0000000 --- a/test/plop_test.erl +++ /dev/null @@ -1,94 +0,0 @@ --module(plop_test). --include("plop.hrl"). --include_lib("eunit/include/eunit.hrl"). - -%% start_stop_test_() -> -%% {"The server can be started, stopped and is regsitered", -%% {setup, fun start/0, fun stop/1, fun is_registered/1}}. - -%% "Entries can be added and the STH changes." -%% FIXME: This way, if a test fails, we don't stop plop. The tests -%% must run and be validated in strict order though. -adding_verifying_test() -> - %%Pid = start(), - Pubkey = plop:testing_get_pubkey(), - add_sth_add_sth(Pubkey). - %%stop(Pid). - -%% "Entries can be retrieved." -get_entries_test_() -> - Entries = plop:get(1, 2), - [?_assertEqual(2, length(Entries)), - ?_assertMatch(#mtl{}, hd(Entries))]. - -%%% Setup. -%% start() -> -%% {ok, Pid} = plop:start_link("../test/rsakey.pem", "sikrit"), -%% Pid. - -%% stop(_) -> -%% plop:stop(). - -%%% Tests. -%% is_registered(Pid) -> -%% [?_assert(erlang:is_process_alive(Pid)), -%% ?_assertEqual(Pid, whereis(plop))]. - -%%% Helpers. - -add_sth_add_sth(Pubkey) -> - add(0, Pubkey), - STH0 = sth(Pubkey), - add(1, Pubkey), - STH1 = sth(Pubkey), - ?assertNotEqual(STH0, STH1). - -add(0, Pubkey) -> - Msg = crypto:rand_bytes(32), - Entry = #timestamped_entry{ - timestamp = 4711, - entry = #plop_entry{ - type = test, - data = Msg}}, - DataSigned = <<0:8, 0:8, 4711:64, 2:16, Msg/binary>>, - #spt{ - version = Version, - timestamp = Timestamp, - signature = #signature{signature = Signature} - } = plop:add(Entry), - ?assertEqual(0, Version), - ?assertEqual(4711, Timestamp), - ?assert(public_key:verify(DataSigned, sha256, Signature, Pubkey)); -add(1, Pubkey) -> - Msg = crypto:rand_bytes(32), - Entry = #timestamped_entry{ - timestamp = 4712, - entry = #plop_entry{ - type = test, - data = Msg}}, - DataSigned = <<0:8, 0:8, 4712:64, 2:16, Msg/binary>>, - #spt{ - version = Version, - timestamp = Timestamp, - signature = #signature{signature = Signature} - } = plop:add(Entry), - ?assertEqual(0, Version), - ?assertEqual(4712, Timestamp), - ?assert(public_key:verify(DataSigned, sha256, Signature, Pubkey)). - -%% TODO -%% add(2) -> -%% TestVector = <<>>, -%% %% Same data as in 0, should not result in new database entry. - -sth(Pubkey) -> - #sth{ - treesize = Treesize, - timestamp = Timestamp, - roothash = Roothash, - signature = #signature{signature = Signature} - } = STH = plop:sth(), - Data = list_to_binary([<<0:8, 1:8, Timestamp:64, Treesize:64>>, Roothash]), - ?assert(public_key:verify(Data, sha256, Signature, Pubkey)), - STH. - -- cgit v1.1 From f5951fdc069dc4c15b2c42d139d074ed55386e05 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sat, 27 Sep 2014 18:36:39 +0200 Subject: Fix api problems --- src/plop.erl | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/plop.erl b/src/plop.erl index b59e19b..9ea6aec 100644 --- a/src/plop.erl +++ b/src/plop.erl @@ -26,7 +26,7 @@ %% API. -export([start_link/2, stop/0]). -export([get_logid/0, serialise/1]). --export([add/4, sth/0, get/1, get/2, spt/2, consistency/2, inclusion/2, inclusion_and_entry/2]). +-export([add/3, sth/0, get/1, get/2, spt/1, consistency/2, inclusion/2, inclusion_and_entry/2]). -export([generate_timestamp/0]). %% API for tests. -export([read_keyfile_rsa/2, read_keyfiles_ec/2]). @@ -97,10 +97,10 @@ terminate(_Reason, _State) -> ok. %%%%%%%%%%%%%%%%%%%% --spec add(binary(), binary(), binary(), binary()) -> ok. -add(LogEntry, TreeLeafHash, EntryHash, DataToSign) -> +-spec add(binary(), binary(), binary()) -> ok. +add(LogEntry, TreeLeafHash, EntryHash) -> gen_server:call(?MODULE, - {add, {LogEntry, TreeLeafHash, EntryHash, DataToSign}}). + {add, {LogEntry, TreeLeafHash, EntryHash}}). sth() -> gen_server:call(?MODULE, {sth, []}). @@ -113,6 +113,9 @@ get(Start, End) -> get(Hash) -> gen_server:call(?MODULE, {get, {hash, Hash}}). +spt(Data) -> + gen_server:call(?MODULE, {spt, Data}). + consistency(TreeSizeFirst, TreeSizeSecond) -> gen_server:call(?MODULE, {consistency, {TreeSizeFirst, TreeSizeSecond}}). -spec inclusion(binary(), non_neg_integer()) -> @@ -144,13 +147,17 @@ handle_call({get, logid}, _From, handle_call({add, {LogEntry, TreeLeafHash, EntryHash}}, _From, Plop) -> ok = ht:add(TreeLeafHash), - ok = db:add(TreeLeafHash, EntryHash, LogEntry, ht:size()), + {atomic, ok} = db:add(TreeLeafHash, EntryHash, LogEntry, ht:size()), {reply, ok, Plop}; handle_call({sth, Data}, _From, Plop = #state{privkey = PrivKey}) -> {reply, sth(PrivKey, Data), Plop}; +handle_call({spt, Data}, _From, + Plop = #state{privkey = PrivKey}) -> + {reply, spt(PrivKey, Data), Plop}; + handle_call({consistency, {First, Second}}, _From, Plop) -> {reply, ht:consistency(First - 1, Second - 1), Plop}; -- cgit v1.1 From c3111912df87159d1c3bccac351f614a25a3c553 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sat, 27 Sep 2014 20:27:51 +0200 Subject: Fix api problems --- src/plop.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plop.erl b/src/plop.erl index 9ea6aec..c93b26d 100644 --- a/src/plop.erl +++ b/src/plop.erl @@ -146,8 +146,8 @@ handle_call({get, logid}, _From, {reply, LogID, Plop}; handle_call({add, {LogEntry, TreeLeafHash, EntryHash}}, _From, Plop) -> - ok = ht:add(TreeLeafHash), {atomic, ok} = db:add(TreeLeafHash, EntryHash, LogEntry, ht:size()), + ok = ht:add(TreeLeafHash), {reply, ok, Plop}; handle_call({sth, Data}, _From, @@ -162,7 +162,7 @@ handle_call({consistency, {First, Second}}, _From, Plop) -> {reply, ht:consistency(First - 1, Second - 1), Plop}; handle_call({inclusion, {Hash, TreeSize}}, _From, Plop) -> - R = case db:get_by_entry_hash(Hash) of + R = case db:get_by_leaf_hash(Hash) of notfound -> {notfound, "Unknown hash"}; % FIXME: include Hash {I, _MTLHash, _Entry} -> -- cgit v1.1 From b3d01b2105b1e1df6804d6463ae465bc91ae4e8d Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sun, 28 Sep 2014 01:50:48 +0200 Subject: Use raw file storage --- src/db.erl | 185 +++++++++++++++++++++++------------------------------------ src/plop.erl | 2 +- 2 files changed, 72 insertions(+), 115 deletions(-) diff --git a/src/db.erl b/src/db.erl index ddebbeb..45d0b57 100644 --- a/src/db.erl +++ b/src/db.erl @@ -6,11 +6,9 @@ %% API. -export([start_link/0, stop/0]). --export([init_db/0, init_db/1, init_tables/0, init_tables/1]). +-export([]). -export([add/4, size/0]). -export([get_by_index/1, get_by_indices/3, get_by_leaf_hash/1, get_by_entry_hash/1]). -%% API for testing. --export([dump/1, destroy_tables/0, info_tables/0, dump_to_file/1]). %% gen_server callbacks. -export([init/1, handle_call/3, terminate/2, handle_cast/2, handle_info/2, code_change/3]). @@ -19,50 +17,11 @@ -include("db.hrl"). -include("$CTROOT/plop/include/plop.hrl"). -%% @doc Set up a database schema on all nodes that are to be included -%% in the "database cluster". Has to be run _before_ mnesia has been -%% started. -init_db() -> - init_db([node()]). -init_db(Nodes) -> - ok = mnesia:create_schema(Nodes), - rpc:multicall(Nodes, application, start, [mnesia]), - init_tables(Nodes), - rpc:multicall(Nodes, application, stop, [mnesia]). - -%% @doc Run once, or rather every time you start on a new database. -%% If run more than once, we'll get {aborted, {already_exists, TABLE}}. -init_tables() -> - init_tables([node()]). -init_tables(Nodes) -> - %% We've once upon a time invoked mnesia:create_schema/1 with the - %% nodes that will be part of the database. - RamCopies = [], - DiscCopies = [], - DiscOnlyCopies = Nodes, - mnesia:start(), - {atomic, ok} = - mnesia:create_table(plop, - [{type, set}, - {ram_copies, RamCopies}, - {disc_copies, DiscCopies}, - {disc_only_copies, DiscOnlyCopies}, - {attributes, record_info(fields, plop)}, - {majority, true}]), - {atomic, ok} = mnesia:add_table_index(plop, entryhash), - {atomic, ok} = mnesia:add_table_index(plop, mtlhash). - -destroy_tables() -> - mnesia:delete_table(plop). -info_tables() -> - mnesia:table_info(plop, all). -dump_to_file(Filename) -> - mnesia:dump_to_textfile(Filename). size() -> - mnesia:table_info(plop, size). + binary_to_integer(atomic:readfile(treesize_path())). init(_Args) -> - {mnesia:wait_for_tables([plop], 5000), []}. + {ok, []}. start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). @@ -99,10 +58,6 @@ get_by_leaf_hash(LeafHash) -> get_by_entry_hash(EntryHash) -> gen_server:call(?MODULE, {get_by_entry_hash, EntryHash}). -%% Testing and debugging. -dump(Table) -> - gen_server:call(?MODULE, {dump, Table}). - %%%%%%%%%%%%%%%%%%%% %% gen_server callbacks. @@ -122,81 +77,83 @@ terminate(_Reason, _State) -> %%%%%%%%%%%%%%%%%%%% %% The meat. +% Table for Leaf hash -> Entry +entry_root_path() -> + {ok, Value} = application:get_env(plop, entry_root_path), + Value. + +% Table for Leaf hash -> Entry +indexforhash_root_path() -> + {ok, Value} = application:get_env(plop, indexforhash_root_path), + Value. + +% Table for Index -> Leaf hash +index_path() -> + {ok, Value} = application:get_env(plop, index_path), + Value. + +% Table for Entry hash -> Leaf hash +entryhash_root_path() -> + {ok, Value} = application:get_env(plop, entryhash_root_path), + Value. + +% File that stores tree size +treesize_path() -> + {ok, Value} = application:get_env(plop, treesize_path), + Value. + + +entry_for_leafhash(LeafHash) -> + perm:readfile(entry_root_path(), LeafHash). + +index_for_leafhash(LeafHash) -> + binary_to_integer(perm:readfile(indexforhash_root_path(), LeafHash)). + +leafhash_for_index(Index) -> + index:get(index_path(), Index). + +leafhash_for_entryhash(EntryHash) -> + perm:readfile(entryhash_root_path(), EntryHash). + handle_call(stop, _From, State) -> {stop, normal, stopped, State}; handle_call({add, {LeafHash, EntryHash, Data, Index}}, _From, State) -> - R = mnesia:transaction( - fun() -> - mnesia:write( - #plop{ - index = Index, - mtlhash = LeafHash, - entryhash = EntryHash, - logentry = Data}) - end), - {reply, R, State}; - -handle_call({get_by_indices, {Start, End, Sorted}}, _From, State) -> - R = case Sorted of - false -> - select_index(Start, End); - true -> - %% FIXME: RAM hog -- how bad is it? - lists:sort(select_index(Start, End)) - end, + ok = perm:ensurefile(entry_root_path(), LeafHash, Data), + ok = perm:ensurefile(entryhash_root_path(), EntryHash, LeafHash), + ok = perm:ensurefile(indexforhash_root_path(), + LeafHash, integer_to_binary(Index)), + ok = index:add(index_path(), Index, LeafHash), + ok = atomic:replacefile(treesize_path(), integer_to_list(Index+1)), + {reply, ok, State}; + +handle_call({get_by_indices, {Start, End, _Sorted}}, _From, State) -> + R = lists:map(fun (Index) -> + LeafHash = leafhash_for_index(Index), + Entry = entry_for_leafhash(LeafHash), + {Index, LeafHash, Entry} + end, lists:seq(Start, End)), {reply, R, State}; handle_call({get_by_index, Index}, _From, State) -> - {reply, - find_entry(fun() -> mnesia:read(plop, Index) end), - State}; + LeafHash = leafhash_for_index(Index), + Entry = entry_for_leafhash(LeafHash), + R = {Index, LeafHash, Entry}, + {reply, R, State}; handle_call({get_by_leaf_hash, LeafHash}, _From, State) -> - {reply, - find_entry(fun() -> - mnesia:index_read(plop, LeafHash, #plop.mtlhash) - end), - State}; + Entry = entry_for_leafhash(LeafHash), + Index = index_for_leafhash(LeafHash), + R = {Index, LeafHash, Entry}, + {reply, R, State}; handle_call({get_by_entry_hash, EntryHash}, _From, State) -> - {reply, - find_entry(fun() -> - mnesia:index_read(plop, EntryHash, #plop.entryhash) - end), - State}; - -handle_call({dump, Table}, _From, State) -> - R = mnesia:transaction( - fun() -> - Q = qlc:q([E || E <- mnesia:table(Table)]), - qlc:e(Q) - end), - {reply, R, State}. - -%%%%%%%%%%%%%%%%%%%% -%% Helper functions. - --spec select_index(non_neg_integer(), non_neg_integer()) -> - [{non_neg_integer(), binary(), binary()}]. -select_index(Start, End) -> - F = fun() -> - %% Get index, mtlhash and logentry. - MatchHead = {plop, '$1', '$2', '_', '$3'}, - Guard = [{'>=', '$1', Start}, {'=<', '$1', End}], - Result = ['$$'], - mnesia:select(plop, [{MatchHead, Guard, Result}]) + R = case leafhash_for_entryhash(EntryHash) of + noentry -> + notfound; + LeafHash -> + Entry = entry_for_leafhash(LeafHash), + Index = index_for_leafhash(LeafHash), + {Index, LeafHash, Entry} end, - {atomic, Res} = mnesia:transaction(F), - Res. - --spec find_entry(fun()) -> notfound | - {non_neg_integer(), binary(), binary()} | - duplicate. -find_entry(Fun) -> - {atomic, Result} = mnesia:transaction(Fun), - case Result of - [] -> notfound; - [#plop{index = I, mtlhash = H, logentry = E}] -> {I, H, E}; - _ -> duplicate - end. + {reply, R, State}. diff --git a/src/plop.erl b/src/plop.erl index c93b26d..30d05ca 100644 --- a/src/plop.erl +++ b/src/plop.erl @@ -146,7 +146,7 @@ handle_call({get, logid}, _From, {reply, LogID, Plop}; handle_call({add, {LogEntry, TreeLeafHash, EntryHash}}, _From, Plop) -> - {atomic, ok} = db:add(TreeLeafHash, EntryHash, LogEntry, ht:size()), + ok = db:add(TreeLeafHash, EntryHash, LogEntry, ht:size()), ok = ht:add(TreeLeafHash), {reply, ok, Plop}; -- cgit v1.1 From 409ea0e5857acffe36ebc977bdce843f994a00aa Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Sun, 28 Sep 2014 02:11:12 +0200 Subject: Remove reference to creating database. Remove mnesia. --- README | 25 ++----------------------- ebin/plop.app | 2 +- 2 files changed, 3 insertions(+), 24 deletions(-) diff --git a/README b/README index 5d26ec6..d315080 100644 --- a/README +++ b/README @@ -8,31 +8,10 @@ Requires Erlang/OTP 17 [erts-6.0] or later. Compile the application - $ erl -make - -Start the application locally - - Very first time, before there is a database: - $ erl -boot start_sasl -pa ebin -eval "plop_app:install([node()])."xo - There should now exist a directory Mnesia.nonode@nohost/ with four - files in it. - - Start the application: - $ erl -boot start_sasl -pa ebin \ - -eval "application:start(mnesia), application:start(plop)." - - FIXME: mnesia isn't starting automagically, why? - TODO: -plop Keyfile "test/rsakey.pem" -plop Passphrase "sikrit" + $ make Test the application [FIXME] -Moving the database files - - Add `-mnesia dir "/some/path"' to the list of arguments to erl. - -Debugging - - Dump the database to a file: - 1> db:dump_to_file("dump.txt"). +TODO: -plop Keyfile "test/rsakey.pem" -plop Passphrase "sikrit" diff --git a/ebin/plop.app b/ebin/plop.app index d017331..a2a0dd4 100644 --- a/ebin/plop.app +++ b/ebin/plop.app @@ -6,7 +6,7 @@ [{description, "The plop store"}, {vsn, "0.2.0-dev"}, {modules, [plop_app, plop_sup, plop, db, ht, hex]}, - {applications, [kernel, stdlib, mnesia]}, % crypto, public_key + {applications, [kernel, stdlib]}, % crypto, public_key {registered, [plop, ht, db]}, {mod, {plop_app, ["test/eckey.pem", "test/eckey-public.pem"]}} ]}. -- cgit v1.1