summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--global/overlay/etc/cron.d/cosmos2
-rwxr-xr-xglobal/overlay/etc/cron.daily/scriptherder_cleanup10
-rw-r--r--global/overlay/etc/puppet/facter/cosmos.rb22
-rw-r--r--global/overlay/etc/puppet/puppet.conf4
-rw-r--r--global/overlay/etc/scriptherder/check/cosmos.ini3
-rwxr-xr-xglobal/overlay/usr/local/bin/scriptherder302
-rwxr-xr-xglobal/post-tasks.d/015cosmos-trust7
7 files changed, 276 insertions, 74 deletions
diff --git a/global/overlay/etc/cron.d/cosmos b/global/overlay/etc/cron.d/cosmos
index 70af3a4..58b45af 100644
--- a/global/overlay/etc/cron.d/cosmos
+++ b/global/overlay/etc/cron.d/cosmos
@@ -1,4 +1,4 @@
SHELL=/bin/sh
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
-*/15 * * * * root test -f /etc/no-automatic-cosmos || (cosmos update ; cosmos apply)
+*/15 * * * * root test -f /etc/no-automatic-cosmos || scriptherder --mode wrap --syslog --name cosmos -- /usr/local/bin/run-cosmos -v
diff --git a/global/overlay/etc/cron.daily/scriptherder_cleanup b/global/overlay/etc/cron.daily/scriptherder_cleanup
new file mode 100755
index 0000000..08ec7f5
--- /dev/null
+++ b/global/overlay/etc/cron.daily/scriptherder_cleanup
@@ -0,0 +1,10 @@
+#!/bin/sh
+#
+# Remove scriptherder data older than 7 days.
+#
+
+DIR="/var/cache/scriptherder/"
+
+test -d ${DIR} || exit 0
+
+find ${DIR} -type f -mtime +7 -print0 | xargs -0 rm -f
diff --git a/global/overlay/etc/puppet/facter/cosmos.rb b/global/overlay/etc/puppet/facter/cosmos.rb
new file mode 100644
index 0000000..d810082
--- /dev/null
+++ b/global/overlay/etc/puppet/facter/cosmos.rb
@@ -0,0 +1,22 @@
+#
+# Extract local Cosmos configuration
+#
+require 'facter'
+Facter.add(:cosmos_repo) do
+ setcode do
+ Facter::Util::Resolution.exec("sh -c '. /etc/cosmos/cosmos.conf && echo $COSMOS_REPO'")
+ end
+end
+
+Facter.add(:cosmos_tag_pattern) do
+ setcode do
+ Facter::Util::Resolution.exec("sh -c '. /etc/cosmos/cosmos.conf && echo $COSMOS_UPDATE_VERIFY_GIT_TAG_PATTERN'")
+ end
+end
+
+Facter.add(:cosmos_repo_origin_url) do
+ setcode do
+ Facter::Util::Resolution.exec("sh -c '. /etc/cosmos/cosmos.conf && cd $COSMOS_REPO && git remote show -n origin | grep \"Fetch URL\" | awk \"{print \\$NF }\"'")
+ end
+end
+
diff --git a/global/overlay/etc/puppet/puppet.conf b/global/overlay/etc/puppet/puppet.conf
index a269892..96f7b44 100644
--- a/global/overlay/etc/puppet/puppet.conf
+++ b/global/overlay/etc/puppet/puppet.conf
@@ -3,7 +3,9 @@ logdir=/var/log/puppet
vardir=/var/lib/puppet
ssldir=/var/lib/puppet/ssl
rundir=/var/run/puppet
-factpath=$vardir/lib/facter
+# factpath is supposed to be colon-delimeted, but that does not appear to work
+# (tested with 'strace -f facter --puppet something' - does not split on colon in Puppet 3.4.2).
+factpath=/etc/puppet/facter
templatedir=$confdir/templates
node_terminus = exec
external_nodes = /etc/puppet/cosmos_enc.py
diff --git a/global/overlay/etc/scriptherder/check/cosmos.ini b/global/overlay/etc/scriptherder/check/cosmos.ini
new file mode 100644
index 0000000..b44a537
--- /dev/null
+++ b/global/overlay/etc/scriptherder/check/cosmos.ini
@@ -0,0 +1,3 @@
+[check]
+ok = exit_status=0, max_age=35m
+warning = exit_status=0, max_age=1h
diff --git a/global/overlay/usr/local/bin/scriptherder b/global/overlay/usr/local/bin/scriptherder
index c11383a..1e00ec0 100755
--- a/global/overlay/usr/local/bin/scriptherder
+++ b/global/overlay/usr/local/bin/scriptherder
@@ -68,6 +68,28 @@ exit_status = {'OK': 0,
}
+class ScriptHerderError(Exception):
+ """
+ Base exception class for scriptherder.
+ """
+
+ def __init__(self, reason, filename):
+ self.reason = reason
+ self.filename = filename
+
+
+class JobLoadError(ScriptHerderError):
+ """
+ Raised when loading a job file fails.
+ """
+
+
+class CheckLoadError(ScriptHerderError):
+ """
+ Raised when loading a check file fails.
+ """
+
+
class Job(object):
"""
Representation of an execution of a job.
@@ -109,6 +131,21 @@ class Job(object):
exit = self.exit_status,
)
+ def status_summary(self):
+ """
+ Return short string with status of job.
+
+ E.g. 'name[exit=0,age=19h]'
+ """
+ if self._end_time is None or self._start_time is None:
+ return '{name}[not_running]'.format(name = self.name)
+ age = _time_to_str(time.time() - self._start_time)
+ return '{name}[exit={exit_status},age={age}]'.format(
+ name = self.name,
+ exit_status = self._exit_status,
+ age = age,
+ )
+
@property
def name(self):
"""
@@ -167,11 +204,10 @@ class Job(object):
@rtype: string
"""
+ if self._end_time is None or self._start_time is None:
+ return 'NaN'
duration = self._end_time - self._start_time
- if duration < 1:
- # milliseconds
- return '{:0.3f}ms'.format(duration * 1000)
- return '{:0.3f}s'.format(duration)
+ return _time_to_str(duration)
@property
def exit_status(self):
@@ -326,7 +362,7 @@ class Job(object):
#self._output_size = data.get('output_size') # currently not used in scriptherder
self._filename = filename
else:
- raise AssertionError('Unknown version in file {!r}: {!r}'.format(filename, data.get('version')))
+ raise JobLoadError('Unknown version: {!r}'.format(data.get('version')), filename=filename)
return self
@@ -355,7 +391,7 @@ class Check(object):
self.logger = logger
self.config = ConfigParser.ConfigParser(_check_defaults)
if not self.config.read([filename]):
- raise ValueError("Failed loading config file {!r}".format(filename))
+ raise ScriptHerderError('Failed loading config file', filename)
_section = 'check'
self._ok_criteria = [x.strip() for x in self.config.get(_section, 'ok').split(',')]
self._warning_criteria = [x.strip() for x in self.config.get(_section, 'warning').split(',')]
@@ -422,6 +458,130 @@ class Check(object):
return False
+class CheckStatus(object):
+ """
+ Aggregated status of job invocations for --mode check.
+
+ Attributes:
+
+ checks_ok: List of checks in OK state ([Job()]).
+ checks_warning: List of checks in WARNING state ([Job()]).
+ checks_critical: List of checks in CRITICAL state ([Job()]).
+ """
+
+ def __init__(self, args, logger):
+ """
+ @param args: Parsed command line arguments
+ @param logger: logging logger
+ """
+
+ self.checks_ok = []
+ self.checks_warning = []
+ self.checks_critical = []
+
+ self._jobs = _get_job_results(args, logger)
+ # group the jobs by their name
+ _by_name = {}
+ for this in self._jobs:
+ if this.name not in _by_name:
+ _by_name[this.name] = []
+ _by_name[this.name].append(this)
+ self._jobs_by_name = _by_name
+
+ self._job_count = len(_by_name)
+
+ self._check_running_jobs(args, logger)
+ if not args.cmd:
+ self._check_not_running(args, logger)
+
+ def _check_running_jobs(self, args, logger):
+ """
+ Look for job execution entrys (parsed into Job() instances), group them
+ per check name and determine the status. For each group, append status
+ to one of the three aggregate status lists of this object (checks_ok,
+ checks_warning or checks_critical).
+
+ @param args: Parsed command line arguments
+ @param logger: logging logger
+ """
+ # determine total check status based on all logged invocations of this job
+ for (name, jobs) in self._jobs_by_name.items():
+ # Load the evaluation criterias for this job
+ check_filename = os.path.join(args.checkdir, name + '.ini')
+ logger.debug("Loading check definition from {!r}".format(check_filename))
+ try:
+ check = Check(check_filename, logger)
+ except ScriptHerderError as exc:
+ logger.warning("Failed loading check: {!r}".format(exc), exc_info=True)
+ raise CheckLoadError('Failed loading check', filename = check_filename)
+
+ # Sort jobs, oldest first
+ jobs = sorted(jobs, key=lambda x: x.start_time)
+ logger.debug("Checking {!r}: {!r}".format(name, jobs))
+
+ jobs_ok = []
+ jobs_warning = []
+ jobs_critical = []
+ for job in jobs:
+ if check.job_is_ok(job):
+ jobs_ok.append(job)
+ elif check.job_is_warning(job):
+ jobs_warning.append(job)
+ else:
+ jobs_critical.append(job)
+
+ logger.debug("Raw status OK : {!r}".format(jobs_ok))
+ logger.debug("Raw status WARN : {!r}".format(jobs_warning))
+ logger.debug("Raw status CRITICAL: {!r}".format(jobs_critical))
+
+ # add most recent job status to the totals
+ if jobs_ok:
+ self.checks_ok.append(jobs_ok[-1])
+ elif jobs_warning:
+ self.checks_warning.append(jobs_warning[-1])
+ else:
+ self.checks_critical.append(jobs_critical[-1])
+
+ def _check_not_running(self, args, logger):
+ """
+ Look for job execution entrys (parsed into Job() instances), group them
+ per check name and determine the status. For each group, append status
+ to one of the three aggregate status lists of this object (checks_ok,
+ checks_warning or checks_critical).
+
+ @param args: Parsed command line arguments
+ @param logger: logging logger
+ """
+ files = [f for f in os.listdir(args.checkdir) if os.path.isfile(os.path.join(args.checkdir, f))]
+ for this in files:
+ if not this.endswith('.ini'):
+ continue
+ filename = os.path.join(args.checkdir, this)
+ logger.debug("Loading check definition from {!r}".format(filename))
+ try:
+ # validate check loads
+ Check(filename, logger)
+ except ValueError as exc:
+ logger.warning("Failed loading check: {!r}".format(exc), exc_info=True)
+ raise CheckLoadError(filename = filename)
+ name = this[:-4] # remove the '.ini' suffix
+ if name not in self._jobs_by_name:
+ logger.debug('Check {!r} (filename {!r}) not found in jobs'.format(name, filename))
+ job = Job(name=name)
+ self.checks_critical.append(job)
+ self._job_count += 1
+ else:
+ logger.debug('Check {!r} has {!r} logged results'.format(name, len(self._jobs_by_name[name])))
+
+ def num_jobs(self):
+ """
+ Return number of jobs processed. This is number of different jobs running + not running.
+
+ @rtype: int
+ """
+ return self._job_count
+
+
def job_from_file(filename):
"""
Recreate Job() instance from saved file.
@@ -488,6 +648,7 @@ def parse_args(defaults):
)
args = parser.parse_args()
+
return args
@@ -537,88 +698,61 @@ def mode_check(args, logger):
@param args: Parsed command line arguments
@param logger: logging logger
"""
- jobs = _get_job_results(args, logger)
- # group the jobs by their name
- by_name = {}
- for this in jobs:
- if this.name not in by_name:
- by_name[this.name] = []
- by_name[this.name].append(this)
-
- total_ok = []
- total_warning = []
- total_critical = []
-
- # determine total check status based on all logged invocations of this job
- for (name, jobs) in by_name.items():
- # Sort jobs, oldest first
- jobs = sorted(jobs, key=lambda x: x.start_time)
- # Load the evaluation criterias for this job
- check_filename = os.path.join(args.checkdir, name + '.ini')
- logger.debug("Loading check definition from {!r}".format(check_filename))
- check = Check(check_filename, logger)
- logger.debug("Checking {!r}: {!r}".format(name, jobs))
-
- jobs_ok = []
- jobs_warning = []
- jobs_critical = []
- for job in jobs:
- if check.job_is_ok(job):
- jobs_ok.append(job)
- elif check.job_is_warning(job):
- jobs_warning.append(job)
- else:
- jobs_critical.append(job)
- logger.debug("Raw status OK : {!r}".format(jobs_ok))
- logger.debug("Raw status WARN : {!r}".format(jobs_warning))
- logger.debug("Raw status CRITICAL: {!r}".format(jobs_critical))
- if jobs_ok:
- total_ok.append(jobs_ok[-1])
- elif jobs_warning:
- total_warning.append(jobs_warning[-1])
- else:
- total_critical.append(jobs_critical[-1])
+ try:
+ status = CheckStatus(args, logger)
+ except CheckLoadError as exc:
+ print("UNKNOWN: Failed loading check from file '{!s}' ({!s})".format(exc.filename, exc.reason))
+ return exit_status['UNKNOWN']
if args.cmd:
# Single job check requested, output detailed information
- if total_ok:
- print('OK: {!s}'.format(total_ok[-1]))
+ if status.checks_ok:
+ print('OK: {!s}'.format(status.checks_ok[-1]))
return exit_status['OK']
- if total_warning:
- print('WARNING: {!s}'.format(total_warning[-1]))
+ if status.checks_warning:
+ print('WARNING: {!s}'.format(status.checks_warning[-1]))
return exit_status['WARNING']
- if total_critical:
- print('CRITICAL: {!s}'.format(total_critical[-1]))
+ if status.checks_critical:
+ print('CRITICAL: {!s}'.format(status.checks_critical[-1]))
return exit_status['CRITICAL']
print "UNKNOWN - no jobs found for {!r}?".format(args.cmd)
return exit_status['UNKNOWN']
- # When not looking at multiple jobs at once, logic gets a bit reversed - if ANY
+ # When looking at multiple jobs at once, logic gets a bit reversed - if ANY
# job invocation is CRITICAL/WARNING, the aggregate message given to
# Nagios will have to be a failure.
- if total_critical:
- print("CRITICAL: {num} job(s) in this state: {names}".format(
- num = len(total_critical),
- names = ', '.join([str(x.name) for x in total_critical]),
- ))
+ if status.checks_critical:
+ print('CRITICAL: {!s}'.format(
+ _status_summary(status.num_jobs(), status.checks_critical)))
return exit_status['CRITICAL']
- if total_warning:
- print("WARNING: {num} job(s) in this state: {names}".format(
- num = len(total_warning),
- names = ', '.join([str(x.name) for x in total_warning]),
- ))
+ if status.checks_warning:
+ print('WARNING: {!s}'.format(
+ _status_summary(status.num_jobs(), status.checks_warning)))
return exit_status['WARNING']
- if total_ok:
- print("OK: {num} job(s) in this state: {names}".format(
- num = len(total_ok),
- names = ', '.join([x.name for x in total_ok]),
- ))
+ if status.checks_ok:
+ print('OK: {!s}'.format(
+ _status_summary(status.num_jobs(), status.checks_ok)))
return exit_status['OK']
print "UNKNOWN - no jobs found?"
return exit_status['UNKNOWN']
+def _status_summary(num_jobs, failed):
+ """
+ String format routine used in output of checks status.
+ """
+ fmt = '1 job in this state: {summary}'
+ if len(failed) == 1:
+ fmt = '{jobs}/{num_jobs} job in this state: {summary}'
+
+ summary = ', '.join(sorted([str(x.status_summary()) for x in failed]))
+ return fmt.format(jobs = len(failed),
+ num_jobs = num_jobs,
+ summary = summary,
+ )
+
+
def _get_job_results(args, logger):
"""
Load all jobs matching any specified name on the command line.
@@ -634,7 +768,10 @@ def _get_job_results(args, logger):
if not this.endswith('.json'):
continue
filename = os.path.join(args.datadir, this)
- job = job_from_file(filename)
+ try:
+ job = job_from_file(filename)
+ except JobLoadError as exc:
+ logger.warning("Failed loading job file '{!s}' ({!s})".format(exc.filename, exc.reason))
if args.cmd:
if args.cmd[0] != job.name:
logger.debug("Skipping '{!s}' not matching '{!s}' (file {!s})".format(job.name, args.cmd[0], filename))
@@ -666,6 +803,27 @@ def _parse_time_value(value):
return num
+def _time_to_str(value):
+ """
+ Format number of seconds to short readable string.
+
+ @type value: float or int
+
+ @rtype: string
+ """
+ if value < 1:
+ # milliseconds
+ return '{:0.3f}ms'.format(value * 1000)
+ if value < 60:
+ return '{!s}s'.format(int(value))
+ if value < 3600:
+ return '{!s}m'.format(int(value))
+ if value < 86400:
+ return '{!s}h'.format(int(value / 3600))
+ days = int(value / 86400)
+ return '{!s}d{!s}h'.format(days, int((value % 86400) / 3600))
+
+
def main(myname = 'scriptherder', args = None, logger = None, defaults=_defaults):
"""
Main entry point for either wrapping a script, or checking the status of it.
@@ -699,6 +857,10 @@ def main(myname = 'scriptherder', args = None, logger = None, defaults=_defaults
syslog_h.setFormatter(formatter)
logger.addHandler(syslog_h)
+ if args.name and args.mode != 'wrap':
+ logger.error('Argument --name only applicable for --mode wrap')
+ return False
+
if args.mode == 'wrap':
return mode_wrap(args, logger)
elif args.mode == 'ls':
diff --git a/global/post-tasks.d/015cosmos-trust b/global/post-tasks.d/015cosmos-trust
index 447d875..5c3359b 100755
--- a/global/post-tasks.d/015cosmos-trust
+++ b/global/post-tasks.d/015cosmos-trust
@@ -4,12 +4,15 @@ if [ -z "$COSMOS_KEYS" ]; then
COSMOS_KEYS=/etc/cosmos/keys
fi
+# Install new keys discovered in the $COSMOS_KEYS directory
for k in $COSMOS_KEYS/*.pub; do
fp=`cosmos gpg --with-colons --with-fingerprint < $k| awk -F: '$1 == "pub" {print $5}'`
- cosmos gpg --with-colons --fingerprint | grep -q ":$fp:" || cosmos gpg --import < $k
+ # The removal of any ^pub:e: entrys means to ignore expired keys - thereby importing them again.
+ cosmos gpg --with-colons --fingerprint | grep -v "^pub:e:" | grep -q ":$fp:" || cosmos gpg --import < $k
done
-for fp in `cosmos gpg --with-colons --fingerprint | awk -F: '$1 == "pub" {print $5}'`; do
+# Delete keys no longer present in $COSMOS_KEYS directory
+for fp in `cosmos gpg --with-colons --fingerprint | awk -F: '$1 == "pub" {print $5 }'`; do
seen="no"
for k in $COSMOS_KEYS/*.pub; do
cosmos gpg --with-colons --with-fingerprint < $k | grep -q ":$fp:" && seen="yes"