Merge remote-tracking branch 'refs/remotes/map/external-merge3' into merging-external-merge

Conflicts:
	src/v1.erl
	tools/merge.py
	tools/testcase1.py

22 files changed, 688 insertions, 201 deletions

diff --git a/Makefile b/Makefile
index aa96290..0239a6c 100644
--- a/Makefile
+++ b/Makefile
@@ -20,3 +20,36 @@ release:
 	printf "0" > rel/db/treesize
 	cp -r webroot rel/catlfish
 	test -d rel/catlfish/webroot/log || mkdir rel/catlfish/webroot/log
+
+tests-prepare:
+	-rm -r rel/known_roots
+	mkdir rel/known_roots
+	cp tools/testcerts/roots/* rel/known_roots
+
+	mkdir -p test/nodes/frontend-1/log
+	mkdir -p test/nodes/storage-1/log
+	mkdir -p test/nodes/storage-2/log
+	cp test/config/frontend-1.config rel
+	cp test/config/storage-1.config rel
+	-rm -r rel/tests
+	mkdir -p rel/tests/machine/machine-1/db
+	printf "0" > rel/tests/machine/machine-1/db/treesize
+	mkdir -p rel/tests/machine/machine-2/db
+	printf "0" > rel/tests/machine/machine-2/db/treesize
+	touch rel/tests/machine/machine-1/db/index
+	touch rel/tests/machine/machine-1/db/newentries
+
+tests-start:
+	(cd rel ; bin/run_erl -daemon ../test/nodes/frontend-1/ ../test/nodes/frontend-1/log/ "exec bin/erl -config frontend-1 -name frontend-1")
+	(cd rel ; bin/run_erl -daemon ../test/nodes/storage-1/ ../test/nodes/storage-1/log/ "exec bin/erl -config storage-1 -name storage-1")
+	sleep 1
+
+tests-run:
+	(cd tools ; python testcase1.py ) || echo "Tests failed"
+
+tests-stop:
+	sleep 5
+	echo "halt()." | ./rel/bin/to_erl test/nodes/frontend-1/
+	echo "halt()." | ./rel/bin/to_erl test/nodes/storage-1/
+
+tests: tests-prepare tests-start tests-run tests-stop
diff --git a/catlfish.config b/catlfish.config
index 672f997..91868e5 100644
--- a/catlfish.config
+++ b/catlfish.config
@@ -10,7 +10,7 @@
  {catlfish,
   [{known_roots_path, "known_roots"},
    {https_servers,
-    [{"127.0.0.1", 8080, v1}
+    [{external_https_api, "127.0.0.1", 8080, v1}
     ]},
    {https_certfile, "catlfish/webroot/certs/webcert.pem"},
    {https_keyfile, "catlfish/webroot/keys/webkey.pem"},
diff --git a/ebin/catlfish.app b/ebin/catlfish.app
index beea7d6..75bc876 100644
--- a/ebin/catlfish.app
+++ b/ebin/catlfish.app
@@ -8,5 +8,5 @@
  [{description, "catlfish -- Certificate Transparency Log Server"},
   {vsn, "0.2.0-dev"},
   {modules, [v1, catlfish_app]},
-  {applications, [kernel, stdlib, plop, inets, jiffy, lager, mochiweb]},
+  {applications, [kernel, stdlib, plop, inets, lager, mochiweb]},
   {mod, {catlfish_app, []}}]}.
diff --git a/src/catlfish.erl b/src/catlfish.erl
index 73066bb..5d96278 100644
--- a/src/catlfish.erl
+++ b/src/catlfish.erl
@@ -5,6 +5,7 @@
 -export([add_chain/2, entries/2, entry_and_proof/2]).
 -export([known_roots/0, update_known_roots/0]).
 -include_lib("eunit/include/eunit.hrl").
+-include("catlfish.hrl").
 
 -define(PROTOCOL_VERSION, 0).
 
@@ -98,13 +99,11 @@ add_chain(LeafCert, CertChain) ->
         plop:spt(list_to_binary([<<?PROTOCOL_VERSION:8>>,
                                  serialise_signature_type(certificate_timestamp),
                                  serialise(TimestampedEntry)])),
-    binary_to_list(
-      jiffy:encode(
-        {[{sct_version, ?PROTOCOL_VERSION},
-          {id, base64:encode(plop:get_logid())},
-          {timestamp, TimestampedEntry#timestamped_entry.timestamp},
-          {extensions, base64:encode(<<>>)},
-          {signature, base64:encode(plop:serialise(SCT_sig))}]})).
+    {[{sct_version, ?PROTOCOL_VERSION},
+      {id, base64:encode(plop:get_logid())},
+      {timestamp, TimestampedEntry#timestamped_entry.timestamp},
+      {extensions, base64:encode(<<>>)},
+      {signature, base64:encode(plop:serialise(SCT_sig))}]}.
 
 -spec serialise_logentry(integer(), binary(), [binary()]) -> binary().
 serialise_logentry(Timestamp, LeafCert, CertChain) ->
@@ -118,24 +117,21 @@ serialise_logentry(Timestamp, LeafCert, CertChain) ->
 
 -spec entries(non_neg_integer(), non_neg_integer()) -> list().
 entries(Start, End) ->
-    binary_to_list(
-      jiffy:encode({[{entries, x_entries(plop:get(Start, End))}]})).
+    {[{entries, x_entries(plop:get(Start, End))}]}.
 
 -spec entry_and_proof(non_neg_integer(), non_neg_integer()) -> list().
 entry_and_proof(Index, TreeSize) ->
-    binary_to_list(
-      jiffy:encode(
-        case plop:inclusion_and_entry(Index, TreeSize) of
-            {ok, Entry, Path} ->
-                {Timestamp, LeafCertVector, CertChainVector} = unpack_entry(Entry),
-                MTL = build_mtl(Timestamp, LeafCertVector),
-                {[{leaf_input, base64:encode(MTL)},
-                  {extra_data, base64:encode(CertChainVector)},
-                  {audit_path, [base64:encode(X) || X <- Path]}]};
-            {notfound, Msg} ->
-                {[{success, false},
-                  {error_message, list_to_binary(Msg)}]}
-        end)).
+    case plop:inclusion_and_entry(Index, TreeSize) of
+        {ok, Entry, Path} ->
+            {Timestamp, LeafCertVector, CertChainVector} = unpack_entry(Entry),
+            MTL = build_mtl(Timestamp, LeafCertVector),
+            {[{leaf_input, base64:encode(MTL)},
+              {extra_data, base64:encode(CertChainVector)},
+              {audit_path, [base64:encode(X) || X <- Path]}]};
+        {notfound, Msg} ->
+            {[{success, false},
+              {error_message, list_to_binary(Msg)}]}
+    end.
 
 %% Private functions.
 unpack_entry(Entry) ->
@@ -164,7 +160,7 @@ decode_tls_vector(Binary, LengthLen) ->
     <<ExtractedBinary:Length/binary-unit:8, Rest2/binary>> = Rest,
     {ExtractedBinary, Rest2}.
 
--define(ROOTS_TABLE, catlfish_roots).
+-define(ROOTS_CACHE_KEY, roots).
 
 update_known_roots() ->
     case application:get_env(catlfish, known_roots_path) of
@@ -183,22 +179,20 @@ known_roots() ->
 
 -spec known_roots(file:filename(), use_cache|update_tab) -> list().
 known_roots(Directory, CacheUsage) ->
-    case ets:info(?ROOTS_TABLE) of
-        undefined ->
-            read_pemfiles_from_dir(
-              ets:new(?ROOTS_TABLE, [set, protected, named_table]),
-              Directory);
-        _ ->
-            case CacheUsage of
-                use_cache ->
-                    ets:lookup_element(?ROOTS_TABLE, list, 2);
-                update_tab ->
-                    read_pemfiles_from_dir(?ROOTS_TABLE, Directory)
-            end
+    case CacheUsage of
+        use_cache ->
+            case ets:lookup(?CACHE_TABLE, ?ROOTS_CACHE_KEY) of
+                [] ->
+                    read_pemfiles_from_dir(Directory);
+                [{roots, DerList}] ->
+                    DerList
+            end;
+        update_tab ->
+            read_pemfiles_from_dir(Directory)
     end.
 
--spec read_pemfiles_from_dir(ets:tab(), file:filename()) -> list().
-read_pemfiles_from_dir(Tab, Dir) ->
+-spec read_pemfiles_from_dir(file:filename()) -> list().
+read_pemfiles_from_dir(Dir) ->
     DerList =
         case file:list_dir(Dir) of
             {error, enoent} ->
@@ -213,7 +207,7 @@ read_pemfiles_from_dir(Tab, Dir) ->
                           Filenames),
                 ders_from_pemfiles(Dir, Files)
         end,
-    true = ets:insert(Tab, {list, DerList}),
+    true = ets:insert(?CACHE_TABLE, {?ROOTS_CACHE_KEY, DerList}),
     DerList.
 
 ders_from_pemfiles(Dir, Filenames) ->
@@ -256,7 +250,7 @@ read_pemfiles_test_() ->
      fun() -> {known_roots(?PEMFILES_DIR_OK, use_cache),
                known_roots(?PEMFILES_DIR_OK, use_cache)}
      end,
-     fun(_) -> ets:delete(?ROOTS_TABLE) end,
+     fun(_) -> ets:delete(?CACHE_TABLE, ?ROOTS_CACHE_KEY) end,
      fun({L, LCached}) ->
              [?_assertMatch(7, length(L)),
               ?_assertEqual(L, LCached)]
@@ -265,5 +259,5 @@ read_pemfiles_test_() ->
 read_pemfiles_fail_test_() ->
     {setup,
      fun() -> known_roots(?PEMFILES_DIR_NONEXISTENT, use_cache) end,
-     fun(_) -> ets:delete(?ROOTS_TABLE) end,
+     fun(_) -> ets:delete(?CACHE_TABLE, ?ROOTS_CACHE_KEY) end,
      fun(Empty) -> [?_assertMatch([], Empty)] end}.
diff --git a/src/catlfish.hrl b/src/catlfish.hrl
new file mode 100644
index 0000000..46e882b
--- /dev/null
+++ b/src/catlfish.hrl
@@ -0,0 +1,4 @@
+%%% Copyright (c) 2014, NORDUnet A/S.
+%%% See LICENSE for licensing information.
+
+-define(CACHE_TABLE, catlfish_cache).
diff --git a/src/catlfish_app.erl b/src/catlfish_app.erl
index cfb55cd..e24a1bb 100644
--- a/src/catlfish_app.erl
+++ b/src/catlfish_app.erl
@@ -8,11 +8,20 @@
 %% Application callbacks
 -export([start/2, stop/1]).
 
+-include("catlfish.hrl").
+
 %% ===================================================================
 %% Application callbacks
 %% ===================================================================
 
 start(normal, Args) ->
+    case ets:info(?CACHE_TABLE) of
+	undefined ->
+	    ok;
+	_ ->
+	    ets:delete(?CACHE_TABLE)
+    end,
+    ets:new(?CACHE_TABLE, [set, public, named_table]),
     catlfish_sup:start_link(Args).
 
 stop(_State) ->
diff --git a/src/catlfish_sup.erl b/src/catlfish_sup.erl
index abdac44..0b6c306 100644
--- a/src/catlfish_sup.erl
+++ b/src/catlfish_sup.erl
@@ -16,7 +16,7 @@ init([]) ->
          {cacertfile, application:get_env(catlfish, https_cacertfile, none)}],
     Servers =
         lists:map(fun (Config) ->
-                          {IpAddress, Port, Module} = Config,
+                          {ChildName, IpAddress, Port, Module} = Config,
                           {ok, IPv4Address} =
                               inet:parse_ipv4strict_address(IpAddress),
                           WebConfig = [{ip, IPv4Address},
@@ -24,11 +24,12 @@ init([]) ->
                                        {ssl, true},
                                        {ssl_opts, SSLOptions}
                                       ],
-                          {catlfish_web,
+                          {ChildName,
                            {catlfish_web, start, [WebConfig, Module]},
                            permanent, 5000,
                            worker, dynamic}
                   end, application:get_env(catlfish, https_servers, [])),
+    lager:debug("Starting servers ~p", [Servers]),
     {ok,
      {{one_for_one, 3, 10},
       Servers}}.
diff --git a/src/catlfish_web.erl b/src/catlfish_web.erl
index cdc1a39..f3231e4 100644
--- a/src/catlfish_web.erl
+++ b/src/catlfish_web.erl
@@ -2,16 +2,14 @@
 %%% See LICENSE for licensing information.
 
 -module(catlfish_web).
--export([start/2, stop/0, loop/2]).
+-export([start/2, loop/2]).
 
 start(Options, Module) ->
+    lager:debug("Starting catlfish web server: ~p", [Module]),
     Loop = fun (Req) ->
                    ?MODULE:loop(Req, Module)
            end,
-    mochiweb_http:start([{name, ?MODULE}, {loop, Loop} | Options]).
-
-stop() ->
-    mochiweb_http:stop(?MODULE).
+    mochiweb_http:start([{name, Module}, {loop, Loop} | Options]).
 
 loop(Req, Module) ->
     "/" ++ Path = Req:get(path),
@@ -42,7 +40,8 @@ loop(Req, Module) ->
         end
     catch
         Type:What ->
-            lager:error("Crash in ~p for path ~p: ~p ~n~p~n~p~n", [Module, Path, Type, What, erlang:get_stacktrace()]),
+            [CrashFunction | Stack] = erlang:get_stacktrace(),
+            lager:error("Crash in ~p for path ~p: ~p ~p~n~p~n~p~n", [Module, Path, Type, What, CrashFunction, Stack]),
             Req:respond({500, [{"Content-Type", "text/plain"}],
                          "Internal Server Error\n"})
     end.
diff --git a/src/v1.erl b/src/v1.erl
index 707f8ea..d9796fa 100644
--- a/src/v1.erl
+++ b/src/v1.erl
@@ -9,32 +9,31 @@
 
 %% Public functions, i.e. part of URL.
 request(post, "ct/v1/add-chain", Input) ->
-    R = case (catch jiffy:decode(Input)) of
-            {error, E} ->
-                html("add-chain: bad input:", E);
-            {[{<<"chain">>, ChainBase64}]} ->
-                case (catch [base64:decode(X) || X <- ChainBase64]) of
-                    {'EXIT', _} ->
-                        html("add-chain: invalid base64-encoded chain: ",
-                             [ChainBase64]);
-                    [LeafCert | CertChain] ->
-                        Roots = catlfish:known_roots(),
-                        case x509:normalise_chain(Roots, [LeafCert|CertChain]) of
-                            {ok, [Leaf | Chain]} ->
-                                io:format("[info] adding ~p~n",
-                                          [x509:cert_string(LeafCert)]),
-                                success(catlfish:add_chain(Leaf, Chain));
-                            {error, Reason} ->
-                                io:format("[info] rejecting ~p: ~p~n",
-                                          [x509:cert_string(LeafCert), Reason]),
-                                html("add-chain: invalid chain", Reason)
-                        end;
-                    Invalid ->
-                        html("add-chain: chain is not a list: ", [Invalid])
-                end;
-            _ -> html("add-chain: missing input: chain", Input)
-        end,
-    R;
+    case (catch mochijson2:decode(Input)) of
+        {error, E} ->
+            html("add-chain: bad input:", E);
+        {struct, [{<<"chain">>, ChainBase64}]} ->
+            case (catch [base64:decode(X) || X <- ChainBase64]) of
+                {'EXIT', _} ->
+                    html("add-chain: invalid base64-encoded chain: ",
+                         [ChainBase64]);
+                [LeafCert | CertChain] ->
+                    Roots = catlfish:known_roots(),
+                    case x509:normalise_chain(Roots, [LeafCert|CertChain]) of
+                        {ok, [Leaf | Chain]} ->
+                            lager:info("adding ~p",
+                                           [x509:cert_string(LeafCert)]),
+                            success(catlfish:add_chain(Leaf, Chain));
+                        {error, Reason} ->
+                            lager:info("rejecting ~p: ~p",
+                                       [x509:cert_string(LeafCert), Reason]),
+                            html("add-chain: invalid chain", Reason)
+                    end;
+                Invalid ->
+                    html("add-chain: chain is not a list: ", [Invalid])
+            end;
+        _ -> html("add-chain: missing input: chain", Input)
+    end;
 
 request(post, "ct/v1/add-pre-chain", _Input) ->
     niy();
@@ -49,91 +48,83 @@ request(get, "ct/v1/get-sth", _Query) ->
          {sha256_root_hash, base64:encode(Roothash)},
          {tree_head_signature, base64:encode(
                                  plop:serialise(Signature))}],
-    success(jiffy:encode({R}));
+    success({R});
 
 request(get, "ct/v1/get-sth-consistency", Query) ->
-    R = case lists:sort(Query) of
-            [{"first", FirstInput}, {"second", SecondInput}] ->
-                {First, _} = string:to_integer(FirstInput),
-                {Second, _} = string:to_integer(SecondInput),
-                case lists:member(error, [First, Second]) of
-                    true ->
-                        html("get-sth-consistency: bad input:",
-                             [FirstInput, SecondInput]);
-                    false ->
-                        success(
-                          jiffy:encode(
-                            {[{consistency,
-                               [base64:encode(X) ||
-                                   X <- plop:consistency(First, Second)]}]}))
-                end;
-            _ -> html("get-sth-consistency: bad input:", Query)
-        end,
-    R;
+    case lists:sort(Query) of
+        [{"first", FirstInput}, {"second", SecondInput}] ->
+            {First, _} = string:to_integer(FirstInput),
+            {Second, _} = string:to_integer(SecondInput),
+            case lists:member(error, [First, Second]) of
+                true ->
+                    html("get-sth-consistency: bad input:",
+                         [FirstInput, SecondInput]);
+                false ->
+                    success(
+                      {[{consistency,
+                         [base64:encode(X) ||
+                             X <- plop:consistency(First, Second)]}]})
+            end;
+        _ -> html("get-sth-consistency: bad input:", Query)
+    end;
 
 request(get, "ct/v1/get-proof-by-hash", Query) ->
-    R = case lists:sort(Query) of
-            [{"hash", HashInput}, {"tree_size", TreeSizeInput}] ->
-                Hash = case (catch base64:decode(HashInput)) of
-                           {'EXIT', _} -> error;
-                           H -> H
-                       end,
-                {TreeSize, _} = string:to_integer(TreeSizeInput),
-                case lists:member(error, [Hash, TreeSize]) of
-                    true ->
-                        html("get-proof-by-hash: bad input:",
-                             [HashInput, TreeSizeInput]);
-                    false ->
-                        success(
-                          jiffy:encode(
-                            case plop:inclusion(Hash, TreeSize) of
-                                {ok, Index, Path} ->
-                                    {[{leaf_index, Index},
-                                      {audit_path,
-                                       [base64:encode(X) || X <- Path]}]};
-                                {notfound, Msg} ->
-                                    %% FIXME: http status 400
-                                    {[{success, false},
-                                      {error_message, list_to_binary(Msg)}]}
-                            end))
-                end;
-            _ -> html("get-proof-by-hash: bad input:", Query)
-        end,
-    R;
+    case lists:sort(Query) of
+        [{"hash", HashInput}, {"tree_size", TreeSizeInput}] ->
+            Hash = case (catch base64:decode(HashInput)) of
+                       {'EXIT', _} -> error;
+                       H -> H
+                   end,
+            {TreeSize, _} = string:to_integer(TreeSizeInput),
+            case lists:member(error, [Hash, TreeSize]) of
+                true ->
+                    html("get-proof-by-hash: bad input:",
+                         [HashInput, TreeSizeInput]);
+                false ->
+                      case plop:inclusion(Hash, TreeSize) of
+                          {ok, Index, Path} ->
+                              success({[{leaf_index, Index},
+                                        {audit_path,
+                                         [base64:encode(X) || X <- Path]}]});
+                          {notfound, Msg} ->
+                              html("get-proof-by-hash: hash not found", Msg)
+                      end
+            end;
+        _ -> html("get-proof-by-hash: bad input:", Query)
+    end;
 
 request(get, "ct/v1/get-entries", Query) ->
-    R = case lists:sort(Query) of
-            [{"end", EndInput}, {"start", StartInput}] ->
-                {Start, _} = string:to_integer(StartInput),
-                {End, _} = string:to_integer(EndInput),
-                case lists:member(error, [Start, End]) of
-                    true -> html("get-entries: bad input:", [Start, End]);
-                    false -> success(catlfish:entries(Start, min(End, Start + 999)))
-                end;
-            _ -> html("get-entries: bad input:", Query)
-        end,
-    R;
+    case lists:sort(Query) of
+        [{"end", EndInput}, {"start", StartInput}] ->
+            {Start, _} = string:to_integer(StartInput),
+            {End, _} = string:to_integer(EndInput),
+            case lists:member(error, [Start, End]) of
+                true -> html("get-entries: bad input:", [Start, End]);
+                false -> success(
+                           catlfish:entries(Start, min(End, Start + 999)))
+            end;
+        _ -> html("get-entries: bad input:", Query)
+    end;
 
 request(get, "ct/v1/get-entry-and-proof", Query) ->
-    R = case lists:sort(Query) of
-            [{"leaf_index", IndexInput}, {"tree_size", TreeSizeInput}] ->
-                {Index, _} = string:to_integer(IndexInput),
-                {TreeSize, _} = string:to_integer(TreeSizeInput),
-                case lists:member(error, [Index, TreeSize]) of
-                    true ->
-                        html("get-entry-and-proof: not integers: ",
-                             [IndexInput, TreeSizeInput]);
-                    false -> success(catlfish:entry_and_proof(Index, TreeSize))
-                end;
-            _ -> html("get-entry-and-proof: bad input:", Query)
-        end,
-    R;
+    case lists:sort(Query) of
+        [{"leaf_index", IndexInput}, {"tree_size", TreeSizeInput}] ->
+            {Index, _} = string:to_integer(IndexInput),
+            {TreeSize, _} = string:to_integer(TreeSizeInput),
+            case lists:member(error, [Index, TreeSize]) of
+                true ->
+                    html("get-entry-and-proof: not integers: ",
+                         [IndexInput, TreeSizeInput]);
+                false -> success(catlfish:entry_and_proof(Index, TreeSize))
+            end;
+        _ -> html("get-entry-and-proof: bad input:", Query)
+    end;
 
 request(get, "ct/v1/get-roots", _Query) ->
     R = [{certificates,
           [base64:encode(Der) ||
-              Der <- catlfish:known_roots()]}],
-    success(jiffy:encode({R}));
+              Der <- catlfish:update_known_roots()]}],
+    success({R});
 
 request(_Method, _Path, _) ->
     none.
@@ -151,4 +142,4 @@ niy() ->
     html("NIY - Not Implemented Yet|", []).
 
 success(Data) ->
-    {200, [{"Content-Type", "text/json"}], Data}.
+    {200, [{"Content-Type", "text/json"}], mochijson2:encode(Data)}.
diff --git a/test/config/frontend-1.config b/test/config/frontend-1.config
new file mode 100644
index 0000000..79d887d
--- /dev/null
+++ b/test/config/frontend-1.config
@@ -0,0 +1,35 @@
+%% catlfish configuration file (-*- erlang -*-)
+
+[{sasl,
+  [{sasl_error_logger, false},
+   {errlog_type, error},
+   {error_logger_mf_dir, "log"},
+   {error_logger_mf_maxbytes, 10485760},	% 10 MB
+   {error_logger_mf_maxfiles, 10}]},
+ {catlfish,
+  [{known_roots_path, "known_roots"},
+   {https_servers,
+    [{external_https_api, "127.0.0.1", 8080, v1},
+     {frontend_https_api, "127.0.0.1", 8082, frontend}
+    ]},
+   {https_certfile, "catlfish/webroot/certs/webcert.pem"},
+   {https_keyfile, "catlfish/webroot/keys/webkey.pem"},
+   {https_cacertfile, "catlfish/webroot/certs/webcert.pem"}
+  ]},
+ {lager,
+  [{handlers,
+    [{lager_console_backend, info},
+     {lager_file_backend, [{file, "frontend-1-error.log"}, {level, error}]},
+     {lager_file_backend, [{file, "frontend-1-debug.log"}, {level, debug}]},
+     {lager_file_backend, [{file, "frontend-1-console.log"}, {level, info}]}
+    ]}
+  ]},
+ {plop,
+  [{entry_root_path, "tests/machine/machine-1/db/certentries/"},
+   {index_path, "tests/machine/machine-1/db/index"},
+   {entryhash_root_path, "tests/machine/machine-1/db/entryhash/"},
+   {treesize_path, "tests/machine/machine-1/db/treesize"},
+   {indexforhash_root_path, "tests/machine/machine-1/db/certindex/"},
+   {storage_nodes, ["https://127.0.0.1:8081/ct/storage/"]},
+   {storage_nodes_quorum, 1}
+  ]}].
diff --git a/test/config/storage-1.config b/test/config/storage-1.config
new file mode 100644
index 0000000..b176e1f
--- /dev/null
+++ b/test/config/storage-1.config
@@ -0,0 +1,31 @@
+%% catlfish configuration file (-*- erlang -*-)
+
+[{sasl,
+  [{sasl_error_logger, false},
+   {errlog_type, error},
+   {error_logger_mf_dir, "log"},
+   {error_logger_mf_maxbytes, 10485760},	% 10 MB
+   {error_logger_mf_maxfiles, 10}]},
+ {catlfish,
+  [{https_servers,
+    [{storage_https_api, "127.0.0.1", 8081, storage}
+    ]},
+   {https_certfile, "catlfish/webroot/certs/webcert.pem"},
+   {https_keyfile, "catlfish/webroot/keys/webkey.pem"},
+   {https_cacertfile, "catlfish/webroot/certs/webcert.pem"}
+  ]},
+ {lager,
+  [{handlers,
+    [{lager_console_backend, info},
+     {lager_file_backend, [{file, "storage-1-error.log"}, {level, error}]},
+     {lager_file_backend, [{file, "storage-1-debug.log"}, {level, debug}]},
+     {lager_file_backend, [{file, "storage-1-console.log"}, {level, info}]}
+    ]}
+  ]},
+ {plop,
+  [{entry_root_path, "tests/machine/machine-1/db/certentries/"},
+   {index_path, "tests/machine/machine-1/db/index"},
+   {newentries_path, "tests/machine/machine-1/db/newentries"},
+   {entryhash_root_path, "tests/machine/machine-1/db/entryhash/"},
+   {treesize_path, "tests/machine/machine-1/db/treesize"},
+   {indexforhash_root_path, "tests/machine/machine-1/db/certindex/"}]}].
diff --git a/tools/certtools.py b/tools/certtools.py
index 8d64ee4..cbb4ff7 100644
--- a/tools/certtools.py
+++ b/tools/certtools.py
@@ -10,6 +10,7 @@ import struct
 import sys
 import hashlib
 import ecdsa
+import datetime
 
 publickeys = {
     "https://ct.googleapis.com/pilot/":
@@ -142,6 +143,11 @@ def decode_signature(signature):
     assert rest == ""
     return (hash_alg, signature_alg, unpacked_signature)
 
+def encode_signature(hash_alg, signature_alg, unpacked_signature):
+    signature = struct.pack(">bb", hash_alg, signature_alg)
+    signature += tls_array(unpacked_signature, 2)
+    return signature
+
 def check_signature(baseurl, signature, data):
     publickey = base64.decodestring(publickeys[baseurl])
     (hash_alg, signature_alg, unpacked_signature) = decode_signature(signature)
@@ -154,6 +160,12 @@ def check_signature(baseurl, signature, data):
     vk.verify(unpacked_signature, data, hashfunc=hashlib.sha256,
               sigdecode=ecdsa.util.sigdecode_der)
 
+def create_signature(privatekey, data):
+    sk = ecdsa.SigningKey.from_der(privatekey)
+    unpacked_signature = sk.sign(data, hashfunc=hashlib.sha256,
+                                 sigencode=ecdsa.util.sigencode_der)
+    return encode_signature(4, 3, unpacked_signature)
+
 def check_sth_signature(baseurl, sth):
     signature = base64.decodestring(sth["tree_head_signature"])
 
@@ -166,6 +178,15 @@ def check_sth_signature(baseurl, sth):
 
     check_signature(baseurl, signature, tree_head)
 
+def create_sth_signature(tree_size, timestamp, root_hash, privatekey):
+    version = struct.pack(">b", 0)
+    signature_type = struct.pack(">b", 1)
+    timestamp_packed = struct.pack(">Q", timestamp)
+    tree_size_packed = struct.pack(">Q", tree_size)
+    tree_head = version + signature_type + timestamp_packed + tree_size_packed + root_hash
+
+    return create_signature(privatekey, tree_head)
+
 def check_sct_signature(baseurl, leafcert, sct):
     publickey = base64.decodestring(publickeys[baseurl])
     calculated_logid = hashlib.sha256(publickey).digest()
@@ -198,9 +219,57 @@ def pack_mtl(timestamp, leafcert):
     merkle_tree_leaf = version + leaf_type + timestamped_entry
     return merkle_tree_leaf
 
+def unpack_mtl(merkle_tree_leaf):
+    version = merkle_tree_leaf[0:1]
+    leaf_type = merkle_tree_leaf[1:2]
+    timestamped_entry = merkle_tree_leaf[2:]
+    (timestamp, entry_type) = struct.unpack(">QH", timestamped_entry[0:10])
+    (leafcert, rest_entry) = unpack_tls_array(timestamped_entry[10:], 3)
+    return (leafcert, timestamp)
+
 def get_leaf_hash(merkle_tree_leaf):
     leaf_hash = hashlib.sha256()
     leaf_hash.update(struct.pack(">b", 0))
     leaf_hash.update(merkle_tree_leaf)
 
     return leaf_hash.digest()
+
+def timing_point(timer_dict=None, name=None):
+    t = datetime.datetime.now()
+    if timer_dict:
+        starttime = timer_dict["lasttime"]
+        stoptime = t
+        deltatime = stoptime - starttime
+        timer_dict["deltatimes"].append((name, deltatime.seconds * 1000000 + deltatime.microseconds))
+        timer_dict["lasttime"] = t
+        return None
+    else:
+        timer_dict = {"deltatimes":[], "lasttime":t}
+        return timer_dict
+
+def internal_hash(pair):
+    if len(pair) == 1:
+        return pair[0]
+    else:
+        hash = hashlib.sha256()
+        hash.update(struct.pack(">b", 1))
+        hash.update(pair[0])
+        hash.update(pair[1])
+        return hash.digest()
+
+def chunks(l, n):
+    return [l[i:i+n] for i in range(0, len(l), n)]
+
+def next_merkle_layer(layer):
+    return [internal_hash(pair) for pair in chunks(layer, 2)]
+
+def build_merkle_tree(layer0):
+    if len(layer0) == 0:
+        return [[hashlib.sha256().digest()]]
+    layers = []
+    current_layer = layer0
+    layers.append(current_layer)
+    while len(current_layer) > 1:
+        current_layer = next_merkle_layer(current_layer)
+        layers.append(current_layer)
+    return layers
diff --git a/tools/fetchallcerts.py b/tools/fetchallcerts.py
new file mode 100644
index 0000000..dad5241
--- /dev/null
+++ b/tools/fetchallcerts.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2014, NORDUnet A/S.
+# See LICENSE for licensing information.
+
+import argparse
+import urllib2
+import urllib
+import json
+import base64
+import sys
+import struct
+import hashlib
+import itertools
+from certtools import *
+
+parser = argparse.ArgumentParser(description='')
+parser.add_argument('baseurl', help="Base URL for CT server")
+parser.add_argument('--store', default=None, metavar="dir", help='Store certificates in directory dir')
+args = parser.parse_args()
+
+def extract_original_entry(entry):
+    leaf_input =  base64.decodestring(entry["leaf_input"])
+    (leaf_cert, timestamp) = unpack_mtl(leaf_input)
+    extra_data = base64.decodestring(entry["extra_data"])
+    certchain = decode_certificate_chain(extra_data)
+    return [leaf_cert] + certchain
+
+def get_entries_wrapper(baseurl, start, end):
+    fetched_entries = []
+    while start + len(fetched_entries) < (end + 1):
+        print "fetching from", start + len(fetched_entries)
+        entries = get_entries(baseurl, start + len(fetched_entries), end)["entries"]
+        if len(entries) == 0:
+            break
+        fetched_entries.extend(entries)
+    return fetched_entries
+
+def print_layer(layer):
+    for entry in layer:
+        print base64.b16encode(entry)
+
+sth = get_sth(args.baseurl)
+tree_size = sth["tree_size"]
+root_hash = base64.decodestring(sth["sha256_root_hash"])
+
+print "tree size", tree_size
+print "root hash", base64.b16encode(root_hash)
+
+entries = get_entries_wrapper(args.baseurl, 0, tree_size - 1)
+
+print "fetched", len(entries), "entries"
+
+layer0 = [get_leaf_hash(base64.decodestring(entry["leaf_input"])) for entry in entries]
+
+tree = build_merkle_tree(layer0)
+
+calculated_root_hash = tree[-1][0]
+
+print "calculated root hash", base64.b16encode(calculated_root_hash)
+
+if calculated_root_hash != root_hash:
+    print "fetched root hash and calculated root hash different, aborting"
+    sys.exit(1)
+
+if args.store:
+    for entry, i in zip(entries, range(0, len(entries))):
+        chain = extract_original_entry(entry)
+        f = open(args.store + "/" + ("%06d" % i), "w")
+        for cert in chain:
+            print >> f, "-----BEGIN CERTIFICATE-----"
+            print >> f, base64.encodestring(cert).rstrip()
+            print >> f, "-----END CERTIFICATE-----"
+            print >> f, ""
diff --git a/tools/merge.py b/tools/merge.py
index 41144ea..e007d7c 100755
--- a/tools/merge.py
+++ b/tools/merge.py
@@ -9,8 +9,11 @@ import base64
 import urllib
 import urllib2
 import sys
+import time
+from certtools import build_merkle_tree, create_sth_signature, check_sth_signature
 
-frontendnodes = ["https://127.0.0.1:8080/"]
+ctbaseurl = "https://127.0.0.1:8080/"
+frontendnodes = ["https://127.0.0.1:8082/"]
 storagenodes = ["https://127.0.0.1:8081/"]
 
 chainsdir = "../rel/mergedb/chains"
@@ -79,6 +82,22 @@ def sendlog(baseurl, submission):
         print "========================"
         raise e
 
+def sendentry(baseurl, entry, hash):
+    try:
+        result = urllib2.urlopen(baseurl + "ct/frontend/sendentry",
+            json.dumps({"entry":base64.b64encode(entry), "treeleafhash":base64.b64encode(hash)})).read()
+        return json.loads(result)
+    except urllib2.HTTPError, e:
+        print "ERROR: sendentry", e.read()
+        sys.exit(1)
+    except ValueError, e:
+        print "==== FAILED REQUEST ===="
+        print hash
+        print "======= RESPONSE ======="
+        print result
+        print "========================"
+        raise e
+
 def sendsth(baseurl, submission):
     try:
         result = urllib2.urlopen(baseurl + "ct/frontend/sendsth",
@@ -113,6 +132,8 @@ certsinlog = set(logorder)
 
 new_entries = [entry for storagenode in storagenodes for entry in get_new_entries(storagenode)]
 
+print "adding entries"
+added_entries = 0
 for new_entry in new_entries:
     hash = base64.b64decode(new_entry["hash"])
     entry = base64.b64decode(new_entry["entry"])
@@ -121,13 +142,41 @@ for new_entry in new_entries:
         add_to_logorder(hash)
         logorder.append(hash)
         certsinlog.add(hash)
-        print "added", base64.b16encode(hash)
+        added_entries += 1
+print "added", added_entries, "entries"
+
+tree = build_merkle_tree(logorder)
+tree_size = len(logorder)
+root_hash = tree[-1][0]
+timestamp = int(time.time() * 1000)
+privatekey = base64.decodestring(
+    "MHcCAQEEIMM/FjZ4FSzfENTTwGpTve6CP+IVr"
+    "Y7p8OKV634uJI/foAoGCCqGSM49AwEHoUQDQg"
+    "AE4qWq6afhBUi0OdcWUYhyJLNXTkGqQ9PMS5l"
+    "qoCgkV2h1ZvpNjBH2u8UbgcOQwqDo66z6BWQJ"
+    "GolozZYmNHE2kQ==")
+
+tree_head_signature = create_sth_signature(tree_size, timestamp,
+                                           root_hash, privatekey)
+
+sth = {"tree_size": tree_size, "timestamp": timestamp,
+       "sha256_root_hash": base64.b64encode(root_hash),
+       "tree_head_signature": base64.b64encode(tree_head_signature)}
+
+check_sth_signature(ctbaseurl, sth)
+
+print "root hash", base64.b16encode(root_hash)
 
 for frontendnode in frontendnodes:
+    print "distributing for node", frontendnode
     curpos = get_curpos(frontendnode)
+    print "current position", curpos
     entries = [base64.b64encode(entry) for entry in logorder[curpos:]]
     sendlog(frontendnode, {"start": curpos, "hashes": entries})
+    print "log sent"
     missingentries = get_missingentries(frontendnode)
     print "missing entries:", missingentries
-    # XXX: no test case for missing entries yet, waiting to implement
-    sendsth(frontendnode, {"tree_size": len(logorder)})
+    for missingentry in missingentries:
+        hash = base64.b64decode(missingentry)
+        sendentry(frontendnode, read_chain(hash), hash)
+    sendsth(frontendnode, sth)
diff --git a/tools/submitcert.py b/tools/submitcert.py
index 4f1609c..80a3e37 100755
--- a/tools/submitcert.py
+++ b/tools/submitcert.py
@@ -12,63 +12,87 @@ import struct
 import hashlib
 import itertools
 from certtools import *
+import os
+
+from multiprocessing import Pool
 
 baseurl = sys.argv[1]
-certfile = sys.argv[2]
+certfilepath = sys.argv[2]
+
+lookup_in_log = False
+check_sig = False
+
+if certfilepath[-1] == "/":
+    certfiles = [certfilepath + filename for filename in sorted(os.listdir(certfilepath))]
+else:
+    certfiles = [certfilepath]
+
+def submitcert(certfile):
+    timing = timing_point()
+    certs = get_certs_from_file(certfile)
+    timing_point(timing, "readcerts")
+
+    result = add_chain(baseurl, {"chain":map(base64.b64encode, certs)})
+
+    timing_point(timing, "addchain")
+
+    try:
+        if check_sig:
+            check_sct_signature(baseurl, certs[0], result)
+            timing_point(timing, "checksig")
+    except AssertionError, e:
+        print "ERROR:", e
+        sys.exit(1)
+    except ecdsa.keys.BadSignatureError, e:
+        print "ERROR: bad signature"
+        sys.exit(1)
 
-lookup_in_log = True
+    if lookup_in_log:
 
-certs = get_certs_from_file(certfile)
+        merkle_tree_leaf = pack_mtl(result["timestamp"], certs[0])
 
-result = add_chain(baseurl, {"chain":map(base64.b64encode, certs)})
+        leaf_hash = get_leaf_hash(merkle_tree_leaf)
 
-try:
-    check_sct_signature(baseurl, certs[0], result)
-except AssertionError, e:
-    print "ERROR:", e
-    sys.exit(1)
-except ecdsa.keys.BadSignatureError, e:
-    print "ERROR: bad signature"
-    sys.exit(1)
-print "signature check succeeded"
+        sth = get_sth(baseurl)
 
-if lookup_in_log:
+        proof = get_proof_by_hash(baseurl, leaf_hash, sth["tree_size"])
 
-    merkle_tree_leaf = pack_mtl(result["timestamp"], certs[0])
+        leaf_index = proof["leaf_index"]
 
-    leaf_hash = get_leaf_hash(merkle_tree_leaf)
+        entries = get_entries(baseurl, leaf_index, leaf_index)
 
-    sth = get_sth(baseurl)
+        fetched_entry = entries["entries"][0]
 
-    proof = get_proof_by_hash(baseurl, leaf_hash, sth["tree_size"])
+        print "does the leaf_input of the fetched entry match what we calculated:", \
+          base64.decodestring(fetched_entry["leaf_input"]) == merkle_tree_leaf
 
-    leaf_index = proof["leaf_index"]
+        extra_data = fetched_entry["extra_data"]
 
-    entries = get_entries(baseurl, leaf_index, leaf_index)
+        certchain = decode_certificate_chain(base64.decodestring(extra_data))
 
-    fetched_entry = entries["entries"][0]
+        submittedcertchain = certs[1:]
 
-    print "does the leaf_input of the fetched entry match what we calculated:", \
-      base64.decodestring(fetched_entry["leaf_input"]) == merkle_tree_leaf
+        for (submittedcert, fetchedcert, i) in zip(submittedcertchain,
+                                                   certchain, itertools.count(1)):
+            print "cert", i, "in chain is the same:", submittedcert == fetchedcert
 
-    extra_data = fetched_entry["extra_data"]
+        if len(certchain) == len(submittedcertchain) + 1:
+            last_issuer = get_cert_info(certs[-1])["issuer"]
+            root_subject = get_cert_info(certchain[-1])["subject"]
+            print "issuer of last cert in submitted chain and " \
+                "subject of last cert in fetched chain is the same:", \
+                last_issuer == root_subject
+        elif len(certchain) == len(submittedcertchain):
+            print "cert chains are the same length"
+        else:
+            print "ERROR: fetched cert chain has length", len(certchain),
+            print "and submitted chain has length", len(submittedcertchain)
 
-    certchain = decode_certificate_chain(base64.decodestring(extra_data))
+    timing_point(timing, "lookup")
+    return timing["deltatimes"]
 
-    submittedcertchain = certs[1:]
+p = Pool(1)
 
-    for (submittedcert, fetchedcert, i) in zip(submittedcertchain,
-                                               certchain, itertools.count(1)):
-        print "cert", i, "in chain is the same:", submittedcert == fetchedcert
+for timing in p.imap_unordered(submitcert, certfiles):
+    print timing
 
-    if len(certchain) == len(submittedcertchain) + 1:
-        last_issuer = get_cert_info(certs[-1])["issuer"]
-        root_subject = get_cert_info(certchain[-1])["subject"]
-        print "issuer of last cert in submitted chain and " \
-            "subject of last cert in fetched chain is the same:", \
-            last_issuer == root_subject
-    elif len(certchain) == len(submittedcertchain):
-        print "cert chains are the same length"
-    else:
-        print "ERROR: fetched cert chain has length", len(certchain),
-        print "and submitted chain has length", len(submittedcertchain)
diff --git a/tools/testcase1.py b/tools/testcase1.py
index 2d5e0e8..639cd69 100755
--- a/tools/testcase1.py
+++ b/tools/testcase1.py
@@ -105,7 +105,7 @@ def get_and_check_entry(timestamp, chain, leaf_index):
         assert_equal(fetchedcert, submittedcert, "cert %d in chain" % (i,))
 
     if len(certchain) == len(submittedcertchain) + 1:
-        last_issuer = get_cert_info(certs[-1])["issuer"]
+        last_issuer = get_cert_info(submittedcertchain[-1])["issuer"]
         root_subject = get_cert_info(certchain[-1])["subject"]
         if last_issuer == root_subject:
             print_success("fetched chain has an appended root cert")
@@ -122,11 +122,15 @@ def get_and_check_entry(timestamp, chain, leaf_index):
 
 print_and_check_tree_size(0)
 
+mergeresult = subprocess.call(["./merge.py"])
+assert_equal(mergeresult, 0, "merge", quiet=True)
+
 testgroup("cert1")
 
 result1 = do_add_chain(cc1)
 
-subprocess.call(["./merge.py"])
+mergeresult = subprocess.call(["./merge.py"])
+assert_equal(mergeresult, 0, "merge", quiet=True)
 
 print_and_check_tree_size(1)
 
@@ -134,7 +138,8 @@ result2 = do_add_chain(cc1)
 
 assert_equal(result2["timestamp"], result1["timestamp"], "timestamp")
 
-subprocess.call(["./merge.py"])
+mergeresult = subprocess.call(["./merge.py"])
+assert_equal(mergeresult, 0, "merge", quiet=True)
 
 print_and_check_tree_size(1)
 
@@ -147,7 +152,8 @@ testgroup("cert2")
 
 result3 = do_add_chain(cc2)
 
-subprocess.call(["./merge.py"])
+mergeresult = subprocess.call(["./merge.py"])
+assert_equal(mergeresult, 0, "merge", quiet=True)
 
 print_and_check_tree_size(2)
 
@@ -158,7 +164,8 @@ testgroup("cert3")
 
 result4 = do_add_chain(cc3)
 
-subprocess.call(["./merge.py"])
+mergeresult = subprocess.call(["./merge.py"])
+assert_equal(mergeresult, 0, "merge", quiet=True)
 
 print_and_check_tree_size(3)
 
@@ -170,7 +177,8 @@ testgroup("cert4")
 
 result5 = do_add_chain(cc4)
 
-subprocess.call(["./merge.py"])
+mergeresult = subprocess.call(["./merge.py"])
+assert_equal(mergeresult, 0, "merge", quiet=True)
 
 print_and_check_tree_size(4)
 
@@ -183,7 +191,8 @@ testgroup("cert5")
 
 result6 = do_add_chain(cc5)
 
-subprocess.call(["./merge.py"])
+mergeresult = subprocess.call(["./merge.py"])
+assert_equal(mergeresult, 0, "merge", quiet=True)
 
 print_and_check_tree_size(5)
 
diff --git a/tools/testcerts/cert3.txt b/tools/testcerts/cert3.txt
index d12e485..a776b58 100644
--- a/tools/testcerts/cert3.txt
+++ b/tools/testcerts/cert3.txt
@@ -38,6 +38,36 @@ SbDmRK4Rxa5UmgfZnezD0snHVUCrzKzP
 subject=/OU=Domain Control Validated/CN=*.nordu.net
 issuer=/C=NL/O=TERENA/CN=TERENA SSL CA
 ---
+
+Manually added intermediate certificate:
+-----BEGIN CERTIFICATE-----
+MIIEmDCCA4CgAwIBAgIQS8gUAy8H+mqk8Nop32F5ujANBgkqhkiG9w0BAQUFADCB
+lzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug
+Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho
+dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3Qt
+SGFyZHdhcmUwHhcNMDkwNTE4MDAwMDAwWhcNMjAwNTMwMTA0ODM4WjA2MQswCQYD
+VQQGEwJOTDEPMA0GA1UEChMGVEVSRU5BMRYwFAYDVQQDEw1URVJFTkEgU1NMIENB
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAw+NIxC9cwcupmf0booNd
+ij2tOtDipEMfTQ7+NSUwpWkbxOjlwY9UfuFqoppcXN49/ALOlrhfj4NbzGBAkPjk
+tjolnF8UUeyx56+eUKExVccCvaxSin81joL6hK0V/qJ/gxA6VVOULAEWdJRUYyij
+8lspPZSIgCDiFFkhGbSkmOFg5vLrooCDQ+CtaPN5GYtoQ1E/iptBhQw1jF218bbl
+p8ODtWsjb9Sl61DllPFKX+4nSxQSFSRMDc9ijbcAIa06Mg9YC18em9HfnY6pGTVQ
+L0GprTvG4EWyUzl/Ib8iGodcNK5Sbwd9ogtOnyt5pn0T3fV/g3wvWl13eHiRoBS/
+fQIDAQABo4IBPjCCATowHwYDVR0jBBgwFoAUoXJfJhsomEOVXQc31YWWnUvSw0Uw
+HQYDVR0OBBYEFAy9k2gM896ro0lrKzdXR+qQ47ntMA4GA1UdDwEB/wQEAwIBBjAS
+BgNVHRMBAf8ECDAGAQH/AgEAMBgGA1UdIAQRMA8wDQYLKwYBBAGyMQECAh0wRAYD
+VR0fBD0wOzA5oDegNYYzaHR0cDovL2NybC51c2VydHJ1c3QuY29tL1VUTi1VU0VS
+Rmlyc3QtSGFyZHdhcmUuY3JsMHQGCCsGAQUFBwEBBGgwZjA9BggrBgEFBQcwAoYx
+aHR0cDovL2NydC51c2VydHJ1c3QuY29tL1VUTkFkZFRydXN0U2VydmVyX0NBLmNy
+dDAlBggrBgEFBQcwAYYZaHR0cDovL29jc3AudXNlcnRydXN0LmNvbTANBgkqhkiG
+9w0BAQUFAAOCAQEATiPuSJz2hYtxxApuc5NywDqOgIrZs8qy1AGcKM/yXA4hRJML
+thoh45gBlA5nSYEevj0NTmDa76AxTpXv8916WoIgQ7ahY0OzUGlDYktWYrA0irkT
+Q1mT7BR5iPNIk+idyfqHcgxrVqDDFY1opYcfcS3mWm08aXFABFXcoEOUIEU4eNe9
+itg5xt8Jt1qaqQO4KBB4zb8BG1oRPjj02Bs0ec8z0gH9rJjNbUcRkEy7uVvYcOfV
+r7bMxIbmdcCeKbYrDyqlaQIN4+mitF3A884saoU4dmHGSYKrUbOCprlBmCiY+2v+
+ihb/MX5UR6g83EMmqZsFt57ANEORMNQywxFa4Q==
+-----END CERTIFICATE-----
+
 No client certificate CA names sent
 ---
 SSL handshake has read 4093 bytes and written 434 bytes
diff --git a/tools/testcerts/cert4.txt b/tools/testcerts/cert4.txt
index 1762e35..57559e9 100644
--- a/tools/testcerts/cert4.txt
+++ b/tools/testcerts/cert4.txt
@@ -49,6 +49,37 @@ FceHmpqlkA2AvjdvSvwnODux3QPbMucIaJXrUUwf
 subject=/businessCategory=Private Organization/1.3.6.1.4.1.311.60.2.1.3=US/1.3.6.1.4.1.311.60.2.1.2=Delaware/serialNumber=3359300/street=16 Allen Rd/postalCode=03894-4801/C=US/ST=NH/L=Wolfeboro,/O=Python Software Foundation/CN=www.python.org
 issuer=/C=US/O=DigiCert Inc/OU=www.digicert.com/CN=DigiCert SHA2 Extended Validation Server CA
 ---
+
+Manually added intermediate certificate:
+-----BEGIN CERTIFICATE-----
+MIIEtjCCA56gAwIBAgIQDHmpRLCMEZUgkmFf4msdgzANBgkqhkiG9w0BAQsFADBs
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSswKQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5j
+ZSBFViBSb290IENBMB4XDTEzMTAyMjEyMDAwMFoXDTI4MTAyMjEyMDAwMFowdTEL
+MAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3
+LmRpZ2ljZXJ0LmNvbTE0MDIGA1UEAxMrRGlnaUNlcnQgU0hBMiBFeHRlbmRlZCBW
+YWxpZGF0aW9uIFNlcnZlciBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
+ggEBANdTpARR+JmmFkhLZyeqk0nQOe0MsLAAh/FnKIaFjI5j2ryxQDji0/XspQUY
+uD0+xZkXMuwYjPrxDKZkIYXLBxA0sFKIKx9om9KxjxKws9LniB8f7zh3VFNfgHk/
+LhqqqB5LKw2rt2O5Nbd9FLxZS99RStKh4gzikIKHaq7q12TWmFXo/a8aUGxUvBHy
+/Urynbt/DvTVvo4WiRJV2MBxNO723C3sxIclho3YIeSwTQyJ3DkmF93215SF2AQh
+cJ1vb/9cuhnhRctWVyh+HA1BV6q3uCe7seT6Ku8hI3UarS2bhjWMnHe1c63YlC3k
+8wyd7sFOYn4XwHGeLN7x+RAoGTMCAwEAAaOCAUkwggFFMBIGA1UdEwEB/wQIMAYB
+Af8CAQAwDgYDVR0PAQH/BAQDAgGGMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEF
+BQcDAjA0BggrBgEFBQcBAQQoMCYwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRp
+Z2ljZXJ0LmNvbTBLBgNVHR8ERDBCMECgPqA8hjpodHRwOi8vY3JsNC5kaWdpY2Vy
+dC5jb20vRGlnaUNlcnRIaWdoQXNzdXJhbmNlRVZSb290Q0EuY3JsMD0GA1UdIAQ2
+MDQwMgYEVR0gADAqMCgGCCsGAQUFBwIBFhxodHRwczovL3d3dy5kaWdpY2VydC5j
+b20vQ1BTMB0GA1UdDgQWBBQ901Cl1qCt7vNKYApl0yHU+PjWDzAfBgNVHSMEGDAW
+gBSxPsNpA/i/RwHUmCYaCALvY2QrwzANBgkqhkiG9w0BAQsFAAOCAQEAnbbQkIbh
+hgLtxaDwNBx0wY12zIYKqPBKikLWP8ipTa18CK3mtlC4ohpNiAexKSHc59rGPCHg
+4xFJcKx6HQGkyhE6V6t9VypAdP3THYUYUN9XR3WhfVUgLkc3UHKMf4Ib0mKPLQNa
+2sPIoc4sUqIAY+tzunHISScjl2SFnjgOrWNoPLpSgVh5oywM395t6zHyuqB8bPEs
+1OG9d4Q3A84ytciagRpKkk47RpqF/oOi+Z6Mo8wNXrM9zwR4jxQUezKcxwCmXMS1
+oVWNWlZopCJwqjyBcdmdqEU79OX2olHdx3ti6G8MdOu42vi/hw15UJGQmxg7kVkn
+8TUoE6smftX3eg==
+-----END CERTIFICATE-----
+
 No client certificate CA names sent
 ---
 SSL handshake has read 3662 bytes and written 434 bytes
diff --git a/tools/testcerts/cert5.txt b/tools/testcerts/cert5.txt
index 0f3f8f1..14af5fd 100644
--- a/tools/testcerts/cert5.txt
+++ b/tools/testcerts/cert5.txt
@@ -67,6 +67,46 @@ uMko54p5i2QMvXtvIr/a3Nzlx6CiavI=
 subject=/C=US/ST=California/L=San Francisco/O=Wikimedia Foundation, Inc./CN=*.wikipedia.org
 issuer=/C=US/O=DigiCert Inc/OU=www.digicert.com/CN=DigiCert High Assurance CA-3
 ---
+
+Manually added intermediate certificate:
+-----BEGIN CERTIFICATE-----
+MIIGWDCCBUCgAwIBAgIQCl8RTQNbF5EX0u/UA4w/OzANBgkqhkiG9w0BAQUFADBs
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSswKQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5j
+ZSBFViBSb290IENBMB4XDTA4MDQwMjEyMDAwMFoXDTIyMDQwMzAwMDAwMFowZjEL
+MAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3
+LmRpZ2ljZXJ0LmNvbTElMCMGA1UEAxMcRGlnaUNlcnQgSGlnaCBBc3N1cmFuY2Ug
+Q0EtMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL9hCikQH17+NDdR
+CPge+yLtYb4LDXBMUGMmdRW5QYiXtvCgFbsIYOBC6AUpEIc2iihlqO8xB3RtNpcv
+KEZmBMcqeSZ6mdWOw21PoF6tvD2Rwll7XjZswFPPAAgyPhBkWBATaccM7pxCUQD5
+BUTuJM56H+2MEb0SqPMV9Bx6MWkBG6fmXcCabH4JnudSREoQOiPkm7YDr6ictFuf
+1EutkozOtREqqjcYjbTCuNhcBoz4/yO9NV7UfD5+gw6RlgWYw7If48hl66l7XaAs
+zPw82W3tzPpLQ4zJ1LilYRyyQLYoEt+5+F/+07LJ7z20Hkt8HEyZNp496+ynaF4d
+32duXvsCAwEAAaOCAvowggL2MA4GA1UdDwEB/wQEAwIBhjCCAcYGA1UdIASCAb0w
+ggG5MIIBtQYLYIZIAYb9bAEDAAIwggGkMDoGCCsGAQUFBwIBFi5odHRwOi8vd3d3
+LmRpZ2ljZXJ0LmNvbS9zc2wtY3BzLXJlcG9zaXRvcnkuaHRtMIIBZAYIKwYBBQUH
+AgIwggFWHoIBUgBBAG4AeQAgAHUAcwBlACAAbwBmACAAdABoAGkAcwAgAEMAZQBy
+AHQAaQBmAGkAYwBhAHQAZQAgAGMAbwBuAHMAdABpAHQAdQB0AGUAcwAgAGEAYwBj
+AGUAcAB0AGEAbgBjAGUAIABvAGYAIAB0AGgAZQAgAEQAaQBnAGkAQwBlAHIAdAAg
+AEMAUAAvAEMAUABTACAAYQBuAGQAIAB0AGgAZQAgAFIAZQBsAHkAaQBuAGcAIABQ
+AGEAcgB0AHkAIABBAGcAcgBlAGUAbQBlAG4AdAAgAHcAaABpAGMAaAAgAGwAaQBt
+AGkAdAAgAGwAaQBhAGIAaQBsAGkAdAB5ACAAYQBuAGQAIABhAHIAZQAgAGkAbgBj
+AG8AcgBwAG8AcgBhAHQAZQBkACAAaABlAHIAZQBpAG4AIABiAHkAIAByAGUAZgBl
+AHIAZQBuAGMAZQAuMBIGA1UdEwEB/wQIMAYBAf8CAQAwNAYIKwYBBQUHAQEEKDAm
+MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5jb20wgY8GA1UdHwSB
+hzCBhDBAoD6gPIY6aHR0cDovL2NybDMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0SGln
+aEFzc3VyYW5jZUVWUm9vdENBLmNybDBAoD6gPIY6aHR0cDovL2NybDQuZGlnaWNl
+cnQuY29tL0RpZ2lDZXJ0SGlnaEFzc3VyYW5jZUVWUm9vdENBLmNybDAfBgNVHSME
+GDAWgBSxPsNpA/i/RwHUmCYaCALvY2QrwzAdBgNVHQ4EFgQUUOpzidsp+xCPnuUB
+INTeeZlIg/cwDQYJKoZIhvcNAQEFBQADggEBAB7ipUiebNtTOA/vphoqrOIDQ+2a
+vD6OdRvw/S4iWawTwGHi5/rpmc2HCXVUKL9GYNy+USyS8xuRfDEIcOI3ucFbqL2j
+CwD7GhX9A61YasXHJJlIR0YxHpLvtF9ONMeQvzHB+LGEhtCcAarfilYGzjrpDq6X
+dF3XcZpCdF/ejUN83ulV7WkAywXgemFhM9EZTfkI7qA5xSU1tyvED7Ld8aW3DiTE
+JiiNeXf1L/BXunwH1OH8zVowV36GEEfdMR/X/KLCvzB8XSSq6PmuX2p0ws5rs0bY
+Ib4p1I5eFdZCSucyb6Sxa1GDWL4/bcf72gMhy2oWGU4K8K2Eyl2Us1p292E=
+-----END CERTIFICATE-----
+
+
 No client certificate CA names sent
 ---
 SSL handshake has read 4905 bytes and written 434 bytes
diff --git a/tools/testcerts/roots/root1.pem b/tools/testcerts/roots/root1.pem
new file mode 100644
index 0000000..e077900
--- /dev/null
+++ b/tools/testcerts/roots/root1.pem
@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIIEdDCCA1ygAwIBAgIQRL4Mi1AAJLQR0zYq/mUK/TANBgkqhkiG9w0BAQUFADCBlzELMAkGA1UE
+BhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2UgQ2l0eTEeMBwGA1UEChMVVGhl
+IFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExhodHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xHzAd
+BgNVBAMTFlVUTi1VU0VSRmlyc3QtSGFyZHdhcmUwHhcNOTkwNzA5MTgxMDQyWhcNMTkwNzA5MTgx
+OTIyWjCBlzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2UgQ2l0
+eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExhodHRwOi8vd3d3LnVz
+ZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3QtSGFyZHdhcmUwggEiMA0GCSqGSIb3
+DQEBAQUAA4IBDwAwggEKAoIBAQCx98M4P7Sof885glFn0G2f0v9Y8+efK+wNiVSZuTiZFvfgIXlI
+wrthdBKWHTxqctU8EGc6Oe0rE81m65UJM6Rsl7HoxuzBdXmcRl6Nq9Bq/bkqVRcQVLMZ8Jr28bFd
+tqdt++BxF2uiiPsA3/4aMXcMmgF6sTLjKwEHOG7DpV4jvEWbe1DByTCP2+UretNb+zNAHqDVmBe8
+i4fDidNdoI6yqqr2jmmIBsX6iSHzCJ1pLgkzmykNRg+MzEk0sGlRvfkGzWitZky8PqxhvQqIDsjf
+Pe58BEydCl5rkdbux+0ojatNh4lz0G6k0B4WixThdkQDf2Os5M1JnMWS9KsyoUhbAgMBAAGjgbkw
+gbYwCwYDVR0PBAQDAgHGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFKFyXyYbKJhDlV0HN9WF
+lp1L0sNFMEQGA1UdHwQ9MDswOaA3oDWGM2h0dHA6Ly9jcmwudXNlcnRydXN0LmNvbS9VVE4tVVNF
+UkZpcnN0LUhhcmR3YXJlLmNybDAxBgNVHSUEKjAoBggrBgEFBQcDAQYIKwYBBQUHAwUGCCsGAQUF
+BwMGBggrBgEFBQcDBzANBgkqhkiG9w0BAQUFAAOCAQEARxkP3nTGmZev/K0oXnWO6y1n7k57K9cM
+//bey1WiCuFMVGWTYGufEpytXoMs61quwOQt9ABjHbjAbPLPSbtNk28GpgoiskliCE7/yMgUsogW
+XecB5BKV5UU0s4tpvc+0hY91UZ59Ojg6FEgSxvunOxqNDYJAB+gECJChicsZUN/KHAG8HQQZexB2
+lzvukJDKxA4fFm517zP4029bHpbj4HR3dHuKom4t3XbWOTCC8KucUvIqx69JXn7HaOWCgchqJ/kn
+iCrVWFCVH/A7HFe7fRQ5YiuayZSSKqMiDP+JJn1fIytH1xUdqWqeUQ0qUZ6B+dQ7XnASfxAynB67
+nfhmqA==
+-----END CERTIFICATE-----
diff --git a/tools/testcerts/roots/root2.pem b/tools/testcerts/roots/root2.pem
new file mode 100644
index 0000000..bdb6474
--- /dev/null
+++ b/tools/testcerts/roots/root2.pem
@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIENjCCAx6gAwIBAgIBATANBgkqhkiG9w0BAQUFADBvMQswCQYDVQQGEwJTRTEUMBIGA1UEChML
+QWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFkZFRydXN0IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYD
+VQQDExlBZGRUcnVzdCBFeHRlcm5hbCBDQSBSb290MB4XDTAwMDUzMDEwNDgzOFoXDTIwMDUzMDEw
+NDgzOFowbzELMAkGA1UEBhMCU0UxFDASBgNVBAoTC0FkZFRydXN0IEFCMSYwJAYDVQQLEx1BZGRU
+cnVzdCBFeHRlcm5hbCBUVFAgTmV0d29yazEiMCAGA1UEAxMZQWRkVHJ1c3QgRXh0ZXJuYWwgQ0Eg
+Um9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALf3GjPm8gAELTngTlvtH7xsD821
++iO2zt6bETOXpClMfZOfvUq8k+0DGuOPz+VtUFrWlymUWoCwSXrbLpX9uMq/NzgtHj6RQa1wVsfw
+Tz/oMp50ysiQVOnGXw94nZpAPA6sYapeFI+eh6FqUNzXmk6vBbOmcZSccbNQYArHE504B4YCqOmo
+aSYYkKtMsE8jqzpPhNjfzp/haW+710LXa0Tkx63ubUFfclpxCDezeWWkWaCUN/cALw3CknLa0Dhy
+2xSoRcRdKn23tNbE7qzNE0S3ySvdQwAl+mG5aWpYIxG3pzOPVnVZ9c0p10a3CitlttNCbxWyuHv7
+7+ldU9U0WicCAwEAAaOB3DCB2TAdBgNVHQ4EFgQUrb2YejS0Jvf6xCZU7wO94CTLVBowCwYDVR0P
+BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wgZkGA1UdIwSBkTCBjoAUrb2YejS0Jvf6xCZU7wO94CTL
+VBqhc6RxMG8xCzAJBgNVBAYTAlNFMRQwEgYDVQQKEwtBZGRUcnVzdCBBQjEmMCQGA1UECxMdQWRk
+VHJ1c3QgRXh0ZXJuYWwgVFRQIE5ldHdvcmsxIjAgBgNVBAMTGUFkZFRydXN0IEV4dGVybmFsIENB
+IFJvb3SCAQEwDQYJKoZIhvcNAQEFBQADggEBALCb4IUlwtYj4g+WBpKdQZic2YR5gdkeWxQHIzZl
+j7DYd7usQWxHYINRsPkyPef89iYTx4AWpb9a/IfPeHmJIZriTAcKhjW88t5RxNKWt9x+Tu5w/Rw5
+6wwCURQtjr0W4MHfRnXnJK3s9EK0hZNwEGe6nQY1ShjTK3rMUUKhemPR5ruhxSvCNr4TDea9Y355
+e6cJDUCrat2PisP29owaQgVR1EX1n6diIWgVIEM8med8vSTYqZEXc4g/VhsxOBi0cQ+azcgOno4u
+G+GMmIPLHzHxREzGBHNJdmAPx/i9F4BrLunMTA5amnkPIAou1Z5jJh5VkpTYghdae9C8x49OhgQ=
+-----END CERTIFICATE-----
diff --git a/tools/testcerts/roots/root3.pem b/tools/testcerts/roots/root3.pem
new file mode 100644
index 0000000..81c8a7d
--- /dev/null
+++ b/tools/testcerts/roots/root3.pem
@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDxTCCAq2gAwIBAgIQAqxcJmoLQJuPC3nyrkYldzANBgkqhkiG9w0BAQUFADBsMQswCQYDVQQG
+EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQuY29tMSsw
+KQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5jZSBFViBSb290IENBMB4XDTA2MTExMDAwMDAw
+MFoXDTMxMTExMDAwMDAwMFowbDELMAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZ
+MBcGA1UECxMQd3d3LmRpZ2ljZXJ0LmNvbTErMCkGA1UEAxMiRGlnaUNlcnQgSGlnaCBBc3N1cmFu
+Y2UgRVYgUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMbM5XPm+9S75S0t
+Mqbf5YE/yc0lSbZxKsPVlDRnogocsF9ppkCxxLeyj9CYpKlBWTrT3JTWPNt0OKRKzE0lgvdKpVMS
+OO7zSW1xkX5jtqumX8OkhPhPYlG++MXs2ziS4wblCJEMxChBVfvLWokVfnHoNb9Ncgk9vjo4UFt3
+MRuNs8ckRZqnrG0AFFoEt7oT61EKmEFBIk5lYYeBQVCmeVyJ3hlKV9Uu5l0cUyx+mM0aBhakaHPQ
+NAQTXKFx01p8VdteZOE3hzBWBOURtCmAEvF5OYiiAhF8J2a3iLd48soKqDirCmTCv2ZdlYTBoSUe
+h10aUAsgEsxBu24LUTi4S8sCAwEAAaNjMGEwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQFMAMB
+Af8wHQYDVR0OBBYEFLE+w2kD+L9HAdSYJhoIAu9jZCvDMB8GA1UdIwQYMBaAFLE+w2kD+L9HAdSY
+JhoIAu9jZCvDMA0GCSqGSIb3DQEBBQUAA4IBAQAcGgaX3NecnzyIZgYIVyHbIUf4KmeqvxgydkAQ
+V8GK83rZEWWONfqe/EW1ntlMMUu4kehDLI6zeM7b41N5cdblIZQB2lWHmiRk9opmzN6cN82oNLFp
+myPInngiK3BD41VHMWEZ71jFhS9OMPagMRYjyOfiZRYzy78aG6A9+MpeizGLYAiJLQwGXFK3xPkK
+mNEVX58Svnw2Yzi9RKR/5CYrCsSXaQ3pjOLAEFe4yHYSkVXySGnYvCoCWw9E1CAx2/S6cCZdkGCe
+vEsXCS+0yx5DaMkHJ8HSXPfqIbloEpw8nL+e/IBcm2PN7EeqJSdnoDfzAIJ9VNep+OkuE6N36B9K
+-----END CERTIFICATE-----