From 29ac49eabca61c4a9e0c3a0d8f9ba57ab516ebae Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Thu, 25 Sep 2014 01:35:33 +0200 Subject: Permanent storage implementation --- src/perm.erl | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 src/perm.erl (limited to 'src/perm.erl') diff --git a/src/perm.erl b/src/perm.erl new file mode 100644 index 0000000..2ce5b46 --- /dev/null +++ b/src/perm.erl @@ -0,0 +1,95 @@ +%% +%% Copyright (c) 2014 Kungliga Tekniska Högskolan +%% (KTH Royal Institute of Technology, Stockholm, Sweden). +%% + +-module(perm). +-export([ensurefile/3]). + +fsync(Name) -> + fsyncport:fsync(Name). + +readfile_and_verify(Name, Content) -> + case file:read_file(Name) of + {ok, ContentsReadBinary} -> + ContentsRead = binary_to_list(ContentsReadBinary), + if Content == ContentsRead -> + ok; + true -> + {error, "File contents differ"} + end; + {error, Error} -> + {error, Error} + end. + +writefile(Name, NurseryName, Content) -> + case file:open(NurseryName, [write, exclusive]) of + {ok, File} -> + %io:format("Write file: ~p~n", [Name]), + ok = file:write(File, Content), + file:close(File), + Result = file:rename(NurseryName, Name), + Result; + {error, eexist} -> + %% Should not happen, file name should be unique + {error, eexist}; + {error, Error} -> + {error, Error} + end. + +make_dir(Name) -> + case file:make_dir(Name) of + ok -> + ok; + {error, eexist} -> + ok; + {error, Error} -> + {error, Error} + end. + +make_dirs([]) -> + ok; +make_dirs([Name | Rest]) -> + case make_dir(Name) of + ok -> + make_dirs(Rest); + {error, Error} -> + {error, Error} + end. + +path_for_key(Rootdir, Key) -> + Name = hex:bin_to_hexstr(Key), + [C1, C2, C3, C4, C5, C6 | _] = Name, + Firstlevel = Rootdir ++ [C1, C2], + Secondlevel = Firstlevel ++ "/" ++ [C3, C4], + Thirdlevel = Secondlevel ++ "/" ++ [C5, C6], + Fullpath = Thirdlevel ++ "/" ++ Name, + {[Firstlevel, Secondlevel, Thirdlevel], Fullpath}. + +tempfilename(Base) -> + {MegaSecs, Secs, MicroSecs} = now(), + Filename = io_lib:format("~s-~s-~p.~p", [Base, os:getpid(), + MegaSecs * 1000000 + Secs, MicroSecs]), + Filename. + +ensurefile(Rootdir, Key, Content) -> + {Dirs, Path} = path_for_key(Rootdir, Key), + case readfile_and_verify(Path, Content) of + ok -> + lists:foreach(fun (Dir) -> fsync(Dir) end, [Path, Rootdir | Dirs]); + {error, enoent} -> + case make_dirs([Rootdir, Rootdir ++ "nursery/"] ++ Dirs) of + ok -> + NurseryName = Rootdir ++ "nursery/" ++ + tempfilename(hex:bin_to_hexstr(Key)), + _Result = writefile(Path, NurseryName, Content), + lists:foreach(fun (Dir) -> + fsync(Dir) + end, + [Path, Rootdir | Dirs]); %% XXX check results + {error, Error} -> + io:format("Error creating directory: ~w~n", [Error]) + end; + {error, Error} -> + exit({perm, fileerror, "Error reading file", Error}) + end. -- cgit v1.1 From eb95f6951e7a4abd2b7685b2de07de90b90ee0d2 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Thu, 25 Sep 2014 08:35:07 +0200 Subject: perm: Don't crash if file content is different, tell caller instead. Better error handling. --- src/perm.erl | 54 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 19 deletions(-) (limited to 'src/perm.erl') diff --git a/src/perm.erl b/src/perm.erl index 2ce5b46..34f431c 100644 --- a/src/perm.erl +++ b/src/perm.erl @@ -6,17 +6,25 @@ -module(perm). -export([ensurefile/3]). -fsync(Name) -> - fsyncport:fsync(Name). +fsync([]) -> + ok; +fsync([Name | Rest]) -> + case fsyncport:fsync(Name) of + ok -> + fsync(Rest); + {error, Error} -> + {error, Error} + end. readfile_and_verify(Name, Content) -> case file:read_file(Name) of {ok, ContentsReadBinary} -> ContentsRead = binary_to_list(ContentsReadBinary), - if Content == ContentsRead -> + if + Content == ContentsRead -> ok; - true -> - {error, "File contents differ"} + true -> + differ end; {error, Error} -> {error, Error} @@ -72,24 +80,32 @@ tempfilename(Base) -> MegaSecs * 1000000 + Secs, MicroSecs]), Filename. +exit_with_error(Error, Message) -> + io:format("~s: ~w~n", [Message, Error]), + exit({perm, fileerror, Message, Error}). + +check_error(ReturnValue, ErrorMessage) -> + case ReturnValue of + ok -> + ok; + {error, Error} -> + exit_with_error(Error, ErrorMessage) + end. + ensurefile(Rootdir, Key, Content) -> {Dirs, Path} = path_for_key(Rootdir, Key), case readfile_and_verify(Path, Content) of ok -> - lists:foreach(fun (Dir) -> fsync(Dir) end, [Path, Rootdir | Dirs]); + check_error(fsync([Path, Rootdir | Dirs]), "Error in fsync"); + differ -> + differ; {error, enoent} -> - case make_dirs([Rootdir, Rootdir ++ "nursery/"] ++ Dirs) of - ok -> - NurseryName = Rootdir ++ "nursery/" ++ - tempfilename(hex:bin_to_hexstr(Key)), - _Result = writefile(Path, NurseryName, Content), - lists:foreach(fun (Dir) -> - fsync(Dir) - end, - [Path, Rootdir | Dirs]); %% XXX check results - {error, Error} -> - io:format("Error creating directory: ~w~n", [Error]) - end; + check_error(make_dirs([Rootdir, Rootdir ++ "nursery/"] ++ Dirs), + "Error creating directory"), + NurseryName = Rootdir ++ "nursery/" ++ + tempfilename(hex:bin_to_hexstr(Key)), + _Result = writefile(Path, NurseryName, Content), + check_error(fsync([Path, Rootdir | Dirs]), "Error in fsync"); {error, Error} -> - exit({perm, fileerror, "Error reading file", Error}) + exit_with_error(Error, "Error reading file") end. -- cgit v1.1 From 11f8efc7fb27935761c38cf32f41836193ae97f4 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Thu, 25 Sep 2014 09:04:48 +0200 Subject: perm: Added readfile function. --- src/perm.erl | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) (limited to 'src/perm.erl') diff --git a/src/perm.erl b/src/perm.erl index 34f431c..5cd2889 100644 --- a/src/perm.erl +++ b/src/perm.erl @@ -4,7 +4,7 @@ %% -module(perm). --export([ensurefile/3]). +-export([ensurefile/3, readfile/2]). fsync([]) -> ok; @@ -18,14 +18,10 @@ fsync([Name | Rest]) -> readfile_and_verify(Name, Content) -> case file:read_file(Name) of - {ok, ContentsReadBinary} -> - ContentsRead = binary_to_list(ContentsReadBinary), - if - Content == ContentsRead -> - ok; - true -> - differ - end; + {ok, ContentsRead} when Content == ContentsRead -> + ok; + {ok, _ContentsRead} -> + differ; {error, Error} -> {error, Error} end. @@ -109,3 +105,14 @@ ensurefile(Rootdir, Key, Content) -> {error, Error} -> exit_with_error(Error, "Error reading file") end. + +readfile(Rootdir, Key) -> + {_Dirs, Path} = path_for_key(Rootdir, Key), + case file:read_file(Path) of + {ok, Contents} -> + Contents; + {error, enoent} -> + noentry; + {error, Error} -> + exit_with_error(Error, "Error reading file") + end. -- cgit v1.1 From 62e2a6e4849d342f90a3860554bf44df4e563d3b Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Thu, 25 Sep 2014 15:18:39 +0200 Subject: Added atomic module --- src/perm.erl | 73 +++++++++++++----------------------------------------------- 1 file changed, 15 insertions(+), 58 deletions(-) (limited to 'src/perm.erl') diff --git a/src/perm.erl b/src/perm.erl index 5cd2889..ccb23bc 100644 --- a/src/perm.erl +++ b/src/perm.erl @@ -6,16 +6,7 @@ -module(perm). -export([ensurefile/3, readfile/2]). -fsync([]) -> - ok; -fsync([Name | Rest]) -> - case fsyncport:fsync(Name) of - ok -> - fsync(Rest); - {error, Error} -> - {error, Error} - end. - +-spec readfile_and_verify(string(), binary()) -> ok | differ | {error, atom()}. readfile_and_verify(Name, Content) -> case file:read_file(Name) of {ok, ContentsRead} when Content == ContentsRead -> @@ -26,21 +17,7 @@ readfile_and_verify(Name, Content) -> {error, Error} end. -writefile(Name, NurseryName, Content) -> - case file:open(NurseryName, [write, exclusive]) of - {ok, File} -> - %io:format("Write file: ~p~n", [Name]), - ok = file:write(File, Content), - file:close(File), - Result = file:rename(NurseryName, Name), - Result; - {error, eexist} -> - %% Should not happen, file name should be unique - {error, eexist}; - {error, Error} -> - {error, Error} - end. - +-spec make_dir(string()) -> ok | {error, atom()}. make_dir(Name) -> case file:make_dir(Name) of ok -> @@ -51,6 +28,7 @@ make_dir(Name) -> {error, Error} end. +-spec make_dirs([string()]) -> ok | {error, atom()}. make_dirs([]) -> ok; make_dirs([Name | Rest]) -> @@ -61,6 +39,7 @@ make_dirs([Name | Rest]) -> {error, Error} end. +-spec path_for_key(string(), binary()) -> {[string()], string()}. path_for_key(Rootdir, Key) -> Name = hex:bin_to_hexstr(Key), [C1, C2, C3, C4, C5, C6 | _] = Name, @@ -70,49 +49,27 @@ path_for_key(Rootdir, Key) -> Fullpath = Thirdlevel ++ "/" ++ Name, {[Firstlevel, Secondlevel, Thirdlevel], Fullpath}. -tempfilename(Base) -> - {MegaSecs, Secs, MicroSecs} = now(), - Filename = io_lib:format("~s-~s-~p.~p", [Base, os:getpid(), - MegaSecs * 1000000 + Secs, MicroSecs]), - Filename. - -exit_with_error(Error, Message) -> - io:format("~s: ~w~n", [Message, Error]), - exit({perm, fileerror, Message, Error}). - -check_error(ReturnValue, ErrorMessage) -> - case ReturnValue of - ok -> - ok; - {error, Error} -> - exit_with_error(Error, ErrorMessage) - end. - +-spec ensurefile(string(), binary(), binary()) -> ok | differ. ensurefile(Rootdir, Key, Content) -> {Dirs, Path} = path_for_key(Rootdir, Key), case readfile_and_verify(Path, Content) of ok -> - check_error(fsync([Path, Rootdir | Dirs]), "Error in fsync"); + util:fsync([Path, Rootdir | Dirs]); differ -> differ; {error, enoent} -> - check_error(make_dirs([Rootdir, Rootdir ++ "nursery/"] ++ Dirs), - "Error creating directory"), + util:check_error(make_dirs([Rootdir, Rootdir ++ "nursery/"] + ++ Dirs), + makedir, "Error creating directory"), NurseryName = Rootdir ++ "nursery/" ++ - tempfilename(hex:bin_to_hexstr(Key)), - _Result = writefile(Path, NurseryName, Content), - check_error(fsync([Path, Rootdir | Dirs]), "Error in fsync"); + util:tempfilename(hex:bin_to_hexstr(Key)), + util:write_tempfile_and_rename(Path, NurseryName, Content), + util:fsync([Path, Rootdir | Dirs]); {error, Error} -> - exit_with_error(Error, "Error reading file") + util:exit_with_error(Error, readfile, "Error reading file") end. +-spec readfile(string(), binary()) -> binary(). readfile(Rootdir, Key) -> {_Dirs, Path} = path_for_key(Rootdir, Key), - case file:read_file(Path) of - {ok, Contents} -> - Contents; - {error, enoent} -> - noentry; - {error, Error} -> - exit_with_error(Error, "Error reading file") - end. + atomic:readfile(Path). -- cgit v1.1