Merge branch 'multiverse'

Conflicts:
	global/overlay/etc/puppet/cosmos-modules.conf

9 files changed, 125 insertions, 35 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..0d20b64
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+*.pyc
diff --git a/edit-secrets b/edit-secrets
index 08ec257..742321f 100755
--- a/edit-secrets
+++ b/edit-secrets
@@ -32,19 +32,13 @@ if [ "x$1" != "x-l" ]; then
 
     trap "rm -f $TMPFILE $TMPFILE2" EXIT
 
-    ssh -t $host /var/cache/cosmos/repo/edit-secrets -l
-    scp -q $host:$LAST_OUTPUT_FILENAME $TMPFILE
+    ssh -t root@$host /var/cache/cosmos/repo/edit-secrets -l
+    scp -q root@$host:$LAST_OUTPUT_FILENAME $TMPFILE
 
     if grep ^"STATUS=UPDATED" $TMPFILE > /dev/null; then
 	# extract the path of the file that should be updated in the Cosmos repo
-        save_to=$(grep ^"SAVE_TO=" $TMPFILE | cut -d = -f 2-)
-	if [ ! -f $save_to ]; then
-	    echo "$0: Output file $save_to doesn't exist"
-	    echo "    (leaving output in $TMPFILE)"
-	    rm $TMPFILE2
-	    trap EXIT  # clear trap command to preserve $TMPFILE
-	    exit 1
-	fi
+        save_to="${host}/overlay/etc/hiera/data/secrets.yaml.asc"
+        mkdir -p "`dirname $save_to`"
 	# extract the GPG output
 	perl -e '$a = 0; while (<>) { $a = 1 if ($_ =~ /-+BEGIN PGP MESSAGE-+/);
                  print $_ if $a; $a = 0 if ($_ =~ /-+END PGP MESSAGE-+/); }' < $TMPFILE > $TMPFILE2
@@ -132,17 +126,10 @@ else
     # figure out this hosts gpg key id
     recipient=$($GPG --list-secret-key | grep ^sec | head -1 | awk '{print $2}' | cut -d / -f 2)
 
-    save_to="`hostname --fqdn`/overlay${SECRETFILE}"
     echo ""
     (
 	echo "STATUS=UPDATED"
-	echo "SAVE_TO=$save_to"
 	echo ""
     ) > $LAST_OUTPUT_FILENAME
     $GPG --output - --armor --recipient $recipient --sign --encrypt $TMPFILE >> $LAST_OUTPUT_FILENAME
-    echo ""
-    echo "GPG output saved in $LAST_OUTPUT_FILENAME - save it in Cosmos as"
-    echo ""
-    echo "  $save_to"
-    echo ""
 fi
diff --git a/fabfile/__init__.py b/fabfile/__init__.py
index d87fbdd..8db5748 100644
--- a/fabfile/__init__.py
+++ b/fabfile/__init__.py
@@ -17,7 +17,7 @@ def all():
     env.hosts = cosmos_db()['members']['all']
 
 def cosmos():
-    run("cosmos update ; cosmos -v apply");
+    run("/usr/local/bin/run-cosmos");
 
 def upgrade():
     run("apt-get -qq update && apt-get -y -q dist-upgrade");
diff --git a/global/overlay/etc/cron.d/cosmos b/global/overlay/etc/cron.d/cosmos
index 70af3a4..4eab8de 100644
--- a/global/overlay/etc/cron.d/cosmos
+++ b/global/overlay/etc/cron.d/cosmos
@@ -1,4 +1,4 @@
 SHELL=/bin/sh
 PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
 
-*/15 * * * *   root   test -f /etc/no-automatic-cosmos || (cosmos update ; cosmos apply)
+*/15 * * * *   root   test -f /etc/no-automatic-cosmos || /usr/local/bin/run-cosmos
diff --git a/global/overlay/etc/logrotate.d/docker-containers b/global/overlay/etc/logrotate.d/docker-containers
new file mode 100644
index 0000000..e9c90b8
--- /dev/null
+++ b/global/overlay/etc/logrotate.d/docker-containers
@@ -0,0 +1,7 @@
+/var/lib/docker/containers/*/*.log {                                                                                                                                     
+  rotate 7                                                                                                                                                               
+  daily
+  compress
+  delaycompress
+  copytruncate
+}
diff --git a/global/overlay/etc/puppet/cosmos-modules.conf b/global/overlay/etc/puppet/cosmos-modules.conf
index 6a89c4f..20c6106 100644
--- a/global/overlay/etc/puppet/cosmos-modules.conf
+++ b/global/overlay/etc/puppet/cosmos-modules.conf
@@ -1,12 +1,36 @@
+# name source (puppetlabs fq name or git url) upgrade (yes/no) tag-pattern
 #
-# name source (puppetlabs fq name or git url) upgrade (yes/no)
+# NOTE that Git packages MUST be tagged with signatures by someone
+# in the Cosmos trust list. That is why all the URLs point to forked
+# versions in the SUNET github organization.
 #
-concat puppetlabs/concat no
-stdlib puppetlabs/stdlib no
-cosmos git://github.com/SUNET/puppet-cosmos.git yes ct-ops-*
-ufw attachmentgenie/ufw no
-apt puppetlabs/apt no
-vcsrepo puppetlabs/vcsrepo no
-hiera-gpg git://github.com/SUNET/hiera-gpg.git no ct-ops-*
-docker git://github.com/SUNET/garethr-docker.git yes ct-ops-*
-augeas git://github.com/SUNET/puppet-augeas.git yes ct-ops-*
+concat          git://github.com/SUNET/puppetlabs-concat.git yes sunet-*
+stdlib          git://github.com/SUNET/puppetlabs-stdlib.git yes sunet-*
+cosmos          git://github.com/SUNET/puppet-cosmos.git yes sunet-*
+ufw             git://github.com/SUNET/puppet-module-ufw.git yes sunet_dev-*
+apt             git://github.com/SUNET/puppetlabs-apt.git yes sunet_dev-*
+vcsrepo         git://github.com/SUNET/puppetlabs-vcsrepo.git yes sunet-*
+xinetd          git://github.com/SUNET/puppetlabs-xinetd.git yes sunet-*
+hiera-gpg       git://github.com/SUNET/hiera-gpg.git yes sunet-*
+augeas 		git://github.com/SUNET/puppet-augeas.git yes sunet-*
+docker          git://github.com/SUNET/garethr-docker.git yes sunet_dev-*
+#
+# Alternate sources you might or might not want to use:
+#concat          puppetlabs/concat no
+#stdlib          puppetlabs/stdlib no
+#ufw             attachmentgenie/ufw no
+#apt             puppetlabs/apt no
+#vcsrepo         puppetlabs/vcsrepo no
+#xinetd          puppetlabs/xinetd no
+#cosmos          git://github.com/leifj/puppet-cosmos.git yes
+#python          git://github.com/SUNET/puppet-python.git yes sunet-*
+#erlang          git://github.com/SUNET/garethr-erlang.git yes sunet-*
+#rabbitmq        git://github.com/SUNET/puppetlabs-rabbitmq.git yes sunet_dev-*
+#pound           git://github.com/SUNET/puppet-pound.git yes sunet_dev-*
+#augeas          git://github.com/SUNET/puppet-augeas.git yes sunet-*
+#bastion         git://github.com/SUNET/puppet-bastion.git yes sunet-*
+#postgresql      git://github.com/SUNET/puppetlabs-postgresql.git yes sunet_dev-*
+#munin           git://github.com/SUNET/ssm-munin.git yes sunet-*
+#nagios          git://github.com/SUNET/puppet-nagios.git yes sunet-*
+#staging         git://github.com/SUNET/puppet-staging.git yes sunet-*
+#apparmor        git://github.com/SUNET/puppet-apparmor.git yes sunet-*
diff --git a/global/overlay/etc/puppet/hiera.yaml b/global/overlay/etc/puppet/hiera.yaml
index cd619bb..3663305 100644
--- a/global/overlay/etc/puppet/hiera.yaml
+++ b/global/overlay/etc/puppet/hiera.yaml
@@ -1,13 +1,16 @@
 ---
-:backends: - yaml
-           - gpg
+:backends:
+  - yaml
+  - gpg
 
 :logger: console
 
-:hierarchy: - %{env}/%{location}/%{calling_module}
-            - %{env}/%{calling_module}
-            - secrets.yaml
-            - common
+:hierarchy:
+  - "%{env}/%{location}/%{calling_module}"
+  - "%{env}/%{calling_module}"
+  - local
+  - secrets.yaml
+  - common
 
 
 :yaml:
diff --git a/global/overlay/usr/local/bin/docker-cleanup b/global/overlay/usr/local/bin/docker-cleanup
new file mode 100755
index 0000000..f46942b
--- /dev/null
+++ b/global/overlay/usr/local/bin/docker-cleanup
@@ -0,0 +1,46 @@
+#!/bin/sh
+# Cleanup docker files: untagged containers and images.
+#
+# Use `docker-cleanup -n` for a dry run to see what would be deleted.
+
+untagged_containers() {
+	# Print containers using untagged images: $1 is used with awk's print: 0=line, 1=column 1.
+	# NOTE: "[0-9a-f]{12}" does not work with GNU Awk 3.1.7 (RHEL6).
+	#       Ref: https://github.com/blueyed/dotfiles/commit/a14f0b4b#commitcomment-6736470
+	docker ps -a | tail -n +2 | awk '$2 ~ "^[0-9a-f]+$" {print $'$1'}'
+}
+
+untagged_images() {
+	# Print untagged images: $1 is used with awk's print: 0=line, 3=column 3.
+	# NOTE: intermediate images (via -a) seem to only cause
+	# "Error: Conflict, foobarid wasn't deleted" messages.
+	# Might be useful sometimes when Docker messed things up?!
+	# docker images -a | awk '$1 == "<none>" {print $'$1'}'
+	docker images | tail -n +2 | awk '$1 == "<none>" {print $'$1'}'
+}
+
+# Dry-run.
+if [ "$1" = "-n" ]; then
+	echo "=== Containers with uncommitted images: ==="
+	untagged_containers 0
+	echo
+
+	echo "=== Uncommitted images: ==="
+	untagged_images 0
+
+	exit
+fi
+if [ -n "$1" ]; then
+	echo "Cleanup docker files: remove untagged containers and images."
+	echo "Usage: ${0##*/} [-n]"
+	echo "   -n: dry run: display what would get removed."
+	exit 1
+fi
+
+# Remove containers with untagged images.
+echo "Removing containers:" >&2
+untagged_containers 1 | xargs --no-run-if-empty docker rm --volumes=true
+
+# Remove untagged images
+echo "Removing images:" >&2
+untagged_images 3 | xargs --no-run-if-empty docker rmi
diff --git a/global/overlay/usr/local/bin/run-cosmos b/global/overlay/usr/local/bin/run-cosmos
new file mode 100755
index 0000000..a37d49f
--- /dev/null
+++ b/global/overlay/usr/local/bin/run-cosmos
@@ -0,0 +1,22 @@
+#!/bin/sh
+#
+# Simplify running cosmos, with serialization if flock is available.
+#
+
+set -e
+
+FLOCK=`which flock`
+
+if [ -x "$FLOCK" ]; then
+    ($FLOCK --exclusive --wait 60 9 || exit 1
+	cosmos $* update
+	cosmos $* apply
+    )9>/var/lock/run-cosmos
+else
+    cosmos $* update
+    cosmos $* apply
+fi
+
+touch /var/run/last-cosmos-ok.stamp
+
+find /var/lib/puppet/reports/ -type f -mtime +10 | xargs rm -f