From cf6e64bf94677092ef94a6f4ed7c1391dcd43f23 Mon Sep 17 00:00:00 2001
From: Linus Nordberg <linus@nordberg.se>
Date: Mon, 13 Apr 2015 14:19:38 +0200
Subject: Run all types of nodes from the same docker image.

---
 packaging/docker/base-debian:jessie/Dockerfile   |  2 --
 packaging/docker/catlfish-dev/Dockerfile         | 27 ++++++++++------
 packaging/docker/catlfish-dev/merge.sh           | 29 +++++++++++++++++
 packaging/docker/catlfish-dev/start.sh           | 22 +++++++++++++
 packaging/docker/catlfish-dev/supervisord.conf   |  2 +-
 packaging/docker/catlfish-merge/Dockerfile       | 41 ------------------------
 packaging/docker/catlfish-merge/merge.sh         | 27 ----------------
 packaging/docker/catlfish-merge/supervisord.conf |  6 ----
 reltool.config                                   |  2 +-
 test/catlfish-test-local-merge.cfg               |  1 +
 tools/merge.py                                   | 15 +++++++--
 verifycert.erl                                   |  6 ++--
 12 files changed, 87 insertions(+), 93 deletions(-)
 create mode 100755 packaging/docker/catlfish-dev/merge.sh
 create mode 100755 packaging/docker/catlfish-dev/start.sh
 delete mode 100644 packaging/docker/catlfish-merge/Dockerfile
 delete mode 100755 packaging/docker/catlfish-merge/merge.sh
 delete mode 100644 packaging/docker/catlfish-merge/supervisord.conf

diff --git a/packaging/docker/base-debian:jessie/Dockerfile b/packaging/docker/base-debian:jessie/Dockerfile
index dbd5d37..864c239 100644
--- a/packaging/docker/base-debian:jessie/Dockerfile
+++ b/packaging/docker/base-debian:jessie/Dockerfile
@@ -2,5 +2,3 @@ FROM debian:jessie
 RUN apt-get update
 RUN echo 'debconf debconf/frontend select noninteractive' | debconf-set-selections
 RUN apt-get -y -q upgrade
-RUN apt-get -y -q install supervisor
-RUN mkdir -p /var/log/supervisor
diff --git a/packaging/docker/catlfish-dev/Dockerfile b/packaging/docker/catlfish-dev/Dockerfile
index 11ebed5..241c58d 100644
--- a/packaging/docker/catlfish-dev/Dockerfile
+++ b/packaging/docker/catlfish-dev/Dockerfile
@@ -2,18 +2,29 @@
 # /usr/local/etc/catlfish/catlfish.config so mounting
 # /usr/local/etc/catlfish is recommended. This can be done using the
 # `-v' flag to `docker run'.
-
+#
 # NOTE: The directory on the host system that's mounted at
 # /var/local/db/catlfish in the container has to be writable by a host
 # user with uid 147.
-
+#
+# Example, running a frontend node:
+# $ docker run -v /etc/catlfish:/usr/local/etc/catlfish:ro catlfish
+#       frontend /usr/local/catlfish
+#
+# Example, running a merge node:
 # $ docker run -v /etc/catlfish:/usr/local/etc/catlfish:ro catlfish
+#       merge /usr/local/catlfish /var/local/db/catlfish-merge
 
 FROM erlang
 RUN apt-get update
 RUN echo 'debconf debconf/frontend select noninteractive' | debconf-set-selections
+
+# For building.
 RUN apt-get -y -q install gcc git make curl
 
+# For merge.
+RUN apt-get -y -q install git python-ecdsa python-yaml
+
 # Build dependencies in /usr/local/src.
 WORKDIR /usr/local/src
 
@@ -41,10 +52,8 @@ RUN make -C plop
 RUN git clone https://git.nordu.net/catlfish.git
 RUN make -C catlfish PREFIX=/usr/local all release
 
-# Config dir and database dir are mounted from host using `-v' to
-# 'docker run'.
+# Config dir is mounted from host using `-v' to 'docker run'.
 VOLUME /usr/local/etc/catlfish
-VOLUME /var/local/db/catlfish
 
 # Create a catlfish user.
 RUN groupadd --gid 147 catlfish
@@ -54,12 +63,12 @@ RUN useradd --uid 147 --gid 147 catlfish
 # /var/run/catlfish and not in /usr/local/etc/catlfish, so symlink.
 RUN mkdir /var/run/catlfish
 WORKDIR /var/run/catlfish
-RUN mkdir erlang_log sasl_log supervisord_log
+RUN mkdir erlang_log sasl_log merge_log
 RUN chown -R catlfish:catlfish /var/run/catlfish
 RUN ln -s /usr/local/etc/catlfish/catlfish.config /var/run/catlfish/
 
-# Run supervisord.
-ADD supervisord.conf /etc/supervisor/
+ADD merge.sh /usr/local/catlfish/
+ADD start.sh /var/run/catlfish/
 WORKDIR /var/run/catlfish
 USER catlfish
-CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]
+ENTRYPOINT ["/var/run/catlfish/start.sh"]
diff --git a/packaging/docker/catlfish-dev/merge.sh b/packaging/docker/catlfish-dev/merge.sh
new file mode 100755
index 0000000..304d8a4
--- /dev/null
+++ b/packaging/docker/catlfish-dev/merge.sh
@@ -0,0 +1,29 @@
+#! /bin/sh
+
+# Default intervals
+# - 5m before first merge
+# - 20m between subsequent merges
+S1=300; [ -n "$1" ] && S1=$1
+S2=1200; [ -n "$2" ] && S2=$2
+
+DBDIR="$3"
+
+[ -d $DBDIR ] || mkdir $DBDIR
+[ -d $DBDIR/chains ] || mkdir $DBDIR/chains
+[ -e $DBDIR/logorder ] || touch $DBDIR/logorder
+
+echo "merge: ERL_LIBS: $ERL_LIBS"
+
+date
+echo "merge: waiting $(expr $S1 / 60)m$(expr $S1 % 60)s before merging for the first time"
+sleep $S1
+
+while true; do
+    echo "$0: merging"
+    date
+    python /usr/local/src/catlfish/tools/merge.py \
+           --config /usr/local/etc/catlfish/system.cfg \
+           --localconfig /usr/local/etc/catlfish/merge.cfg
+    echo "merge: waiting $(expr $S2 / 60)m$(expr $S2 % 60)s before merging again"
+    sleep $S2
+done
diff --git a/packaging/docker/catlfish-dev/start.sh b/packaging/docker/catlfish-dev/start.sh
new file mode 100755
index 0000000..8b3697d
--- /dev/null
+++ b/packaging/docker/catlfish-dev/start.sh
@@ -0,0 +1,22 @@
+#! /bin/sh
+
+role=$1; [ -n "$1" ] && shift
+erlbase=$1; [ -n "$1" ] && shift
+database=$1; [ -n "$1" ] && shift
+
+case $role in
+    frontend|storage|signing)
+        $erlbase/bin/run_erl \
+            /var/run/catlfish/ \
+            /var/run/catlfish/erlang_log/ \
+            "exec $erlbase/bin/erl -config catlfish"
+        ;;
+    merge)
+        ERL_LIBS=$erlbase/lib/catlfish-0.6.0-dev.ez/catlfish-0.6.0-dev
+        ERL_LIBS=$ERL_LIBS:$erlbase/lib/lager-2.1.1.ez/lager-2.1.1
+        export ERL_LIBS
+        $erlbase/merge.sh 60 3600 $database > merge_log/stdout 2> merge_log/stderr
+        ;;
+    *)
+        echo "catlfish: unknown role: $role"
+esac
diff --git a/packaging/docker/catlfish-dev/supervisord.conf b/packaging/docker/catlfish-dev/supervisord.conf
index f3493e5..8eda9b9 100644
--- a/packaging/docker/catlfish-dev/supervisord.conf
+++ b/packaging/docker/catlfish-dev/supervisord.conf
@@ -3,4 +3,4 @@ nodaemon=true
 
 [program:catlfish]
 childlogdir=/var/run/catlfish/supervisord_log
-command=/usr/local/catlfish/bin/run_erl /var/run/catlfish/ /var/run/catlfish/erlang_log/ "exec /usr/local/catlfish/bin/erl -config catlfish"
+command=/usr/local/catlfish/start.sh
diff --git a/packaging/docker/catlfish-merge/Dockerfile b/packaging/docker/catlfish-merge/Dockerfile
deleted file mode 100644
index 9a82d5f..0000000
--- a/packaging/docker/catlfish-merge/Dockerfile
+++ /dev/null
@@ -1,41 +0,0 @@
-# Docker file for catlfish merge.
-#
-# NOTE: The directory on the host system that's mounted at
-# /var/local/db/catlfish-merge in the container has to be writable by
-# a host user with uid 147.
-# 
-# $ docker run \
-#	-v /etc/catlfish:/usr/local/etc/catlfish:ro \
-#	-v /var/local/db/catlfish-merge:/var/local/db/catlfish-merge \
-#	catlfish-merge
-
-FROM erlang
-RUN apt-get update
-RUN echo 'debconf debconf/frontend select noninteractive' | debconf-set-selections
-RUN apt-get -y -q install git python-ecdsa python-yaml
-
-WORKDIR /usr/local/src
-RUN git clone https://git.nordu.net/catlfish.git
-
-# Config dir and database dir are mounted from host using `-v' to
-# 'docker run'.
-VOLUME /usr/local/etc/catlfish
-VOLUME /var/local/db/catlfish-merge
-
-# Copy merge.sh.
-RUN mkdir -p /usr/local/catlfish/bin
-ADD merge.sh /usr/local/catlfish/bin/
-
-# Create a catlfish user.
-RUN groupadd --gid 147 catlfish
-RUN useradd --uid 147 --gid 147 catlfish
-
-# We run from /var/run/catlfish.
-RUN mkdir -p /var/run/catlfish/supervisord_log
-RUN chown -R catlfish:catlfish /var/run/catlfish
-WORKDIR /var/run/catlfish
-
-# Run supervisord.
-ADD supervisord.conf /etc/supervisor/
-USER catlfish
-CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]
diff --git a/packaging/docker/catlfish-merge/merge.sh b/packaging/docker/catlfish-merge/merge.sh
deleted file mode 100755
index b3658cd..0000000
--- a/packaging/docker/catlfish-merge/merge.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#! /bin/sh
-
-# Default intervals
-# - 5m before first merge
-# - 20m between subsequent merges
-S1=300; [ -n "$1" ] && S1=$1
-S2=1200; [ -n "$2" ] && S2=$2
-
-DBDIR="$3"
-
-[ -d $DBDIR ] || mkdir $DBDIR
-[ -d $DBDIR/chains ] || mkdir $DBDIR/chains
-[ -e $DBDIR/logorder ] || touch $DBDIR/logorder
-
-date
-echo "merge: waiting $(expr $S1 / 60)m$(expr $S1 % 60)s before merging for the first time"
-sleep $S1
-
-while true; do
-    echo "$0: merging"
-    date
-    python /usr/local/src/catlfish/tools/merge.py \
-           --config /usr/local/etc/catlfish/system.cfg \
-           --localconfig /usr/local/etc/catlfish/merge.cfg
-    echo "merge: waiting $(expr $S2 / 60)m$(expr $S2 % 60)s before merging again"
-    sleep $S2
-done
diff --git a/packaging/docker/catlfish-merge/supervisord.conf b/packaging/docker/catlfish-merge/supervisord.conf
deleted file mode 100644
index deaf308..0000000
--- a/packaging/docker/catlfish-merge/supervisord.conf
+++ /dev/null
@@ -1,6 +0,0 @@
-[supervisord]
-nodaemon=true
-
-[program:catlfish-merge]
-childlogdir=/var/run/catlfish
-command=/usr/local/catlfish/bin/merge.sh 300 1200 /var/local/db/catlfish-merge
diff --git a/reltool.config b/reltool.config
index 31fd1b0..c40be85 100644
--- a/reltool.config
+++ b/reltool.config
@@ -2,7 +2,7 @@
 {sys, [
        {erts, [{mod_cond, derived}, {app_file, strip}]},
        {app_file, strip},
-       {rel, "catlfish", "0.2.0-dev",
+       {rel, "catlfish", "0.6.0-dev",
         [
          kernel,
          stdlib,
diff --git a/test/catlfish-test-local-merge.cfg b/test/catlfish-test-local-merge.cfg
index c4dffbb..766c872 100644
--- a/test/catlfish-test-local-merge.cfg
+++ b/test/catlfish-test-local-merge.cfg
@@ -7,3 +7,4 @@ paths:
         logpublickey: tests/keys/logkey.pem
         privatekeys: tests/privatekeys
         verifycert_bin: ../verifycert.erl
+        known_roots: tests/known_roots/
diff --git a/tools/merge.py b/tools/merge.py
index 0435b77..76ffede 100755
--- a/tools/merge.py
+++ b/tools/merge.py
@@ -226,7 +226,16 @@ def verify_entry(verifycert, entry, hash):
     mtl = unpacked[0]
     assert hash == get_leaf_hash(mtl)
     s = struct.pack(">I", len(entry)) + entry
-    verifycert.stdin.write(s)
+    try:
+        verifycert.stdin.write(s)
+    except IOError, e:
+        sys.stderr.write("merge: unable to write to verifycert process: ")
+        while 1:
+            line = verifycert.stdout.readline()
+            if line:
+                sys.stderr.write(line)
+            else:
+                sys.exit(1)
     result_length_packed = verifycert.stdout.read(4)
     (result_length,) = struct.unpack(">I", result_length_packed)
     result = verifycert.stdout.read(result_length)
@@ -251,8 +260,8 @@ for hash in new_entries:
             entries_to_fetch[storagenode["name"]].append(hash)
             break
 
-verifycert = subprocess.Popen(paths["verifycert_bin"],
-                               stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+verifycert = subprocess.Popen([paths["verifycert_bin"], paths["known_roots"]],
+                              stdin=subprocess.PIPE, stdout=subprocess.PIPE)
 
 added_entries = 0
 for storagenode in storagenodes:
diff --git a/verifycert.erl b/verifycert.erl
index b9a3753..d364adf 100755
--- a/verifycert.erl
+++ b/verifycert.erl
@@ -1,6 +1,6 @@
 #!/usr/bin/env escript
 %% -*- erlang -*-
-%%! -pa ebin -pa lib/catlfish-0.6.0-dev.ez/catlfish-0.6.0-dev/ebin -pa lib/lager-2.1.1.ez/lager-2.1.1/ebin
+%%! -pa lib/catlfish-0.6.0-dev.ez/catlfish-0.6.0-dev/ebin -pa lib/lager-2.1.1.ez/lager-2.1.1/ebin
 
 write_reply(Bin) ->
     Length = size(Bin),
@@ -36,6 +36,6 @@ loop(RootCerts) ->
             loop(RootCerts)
     end.
 
-main(_) ->
-    Certs = x509:read_pemfiles_from_dir("tests/known_roots/"),
+main([KnownRoots]) ->
+    Certs = x509:read_pemfiles_from_dir(KnownRoots),
     loop(Certs).
-- 
cgit v1.1