inn2 2.5.2+20110413-1build1 / usr / lib / news / bin / innwatch

#! /bin/bash
. /usr/lib/news/innshellvars

##  $Id: innwatch.in 7823 2008-05-06 13:04:27Z iulius $
##  Watch the state of the system relative to the news subsystem.
##  As controlled by the control file, when space or inodes are almost
##  exhausted, innd is throttled or paused, similarly if the load is
##  too high; when conditions revert to normal, innd is restarted.
##  No logging is done here, watch for syslog reports from innd.
##  Written by Mike Cooper <mcooper@usc.edu>.
##  Extensively modified by <kre@munnari.oz.au>.
##  Watch a log file and send mail when it gets new output by
##	Steve Groom <stevo@elroy.Jpl.Nasa.Gov>.
##  Steve's extensions merged in innwatch by
##	<Christophe.Wolfhugel@grasp.insa-lyon.fr>.

PROGNAME=innwatch
LOCK=${LOCKS}/LOCK.${PROGNAME}
DAILY=${LOCKS}/LOCK.news.daily
##  Where to put the timestamp file (directory and filename).
TIMESTAMP=${LOCKS}/${PROGNAME}.time

##  Logfile to watch.  Comment out if no logwatch.
LOGFILE=${MOST_LOGS}/news.crit

##  Default value in case there is no definition in inn.conf.
: ${INNWATCHPAUSELOAD:=1500}
: ${INNWATCHHILOAD:=2000}
: ${INNWATCHLOLOAD:=1000}
: ${INNWATCHSPOOLSPACE:=25000}
: ${INNWATCHBATCHSPACE:=4000}
: ${INNWATCHLIBSPACE:=25000}
: ${INNWATCHSPOOLNODES:=200}
: ${INNWATCHSLEEPTIME:=600}

##  Parse JCL.
while [ $# -gt 0 ] ; do
    case X"$1" in
    X-f)
	FILE=$2
	shift
	;;
    X-f*)
	FILE=`expr "$1" : '-s\(.*\)'`
	;;
    X-l)
	LOGFILE=$2
	shift
	;;
    X-l*)
	LOGFILE=`expr "$1" : '-s\(.*\)'`
	;;
    X-t)
	INNWATCHSLEEPTIME=$2
	shift
	;;
    X-t*)
	INNWATCHSLEEPTIME=`expr "$1" : '-t\(.*\)'`
	;;
    X--)
	shift
	break
	;;
    X-*)
	echo "${PROGNAME}:  Unknown flag $1" 1>&2
	exit 1
	;;
    *)
	break
	;;
    esac
    shift
done

##  Process arguments.
if [ $# -ne 0 ] ; then
    echo "Usage:  ${PROGNAME} [flags]" 1>&2
    exit 1
fi

trap '' 2

##  Anyone else there?
shlock -p $$ -f ${LOCK} || {
    echo "${PROGNAME}: [$$] locked by [`cat ${LOCK}`]"
    exit 0
}

trap 'rm -f ${LOCK} ${WATCHPID} ; exit 1' 1 3 15
echo "$$" > ${WATCHPID}

##  The reason why we turned innd off, and its, and our current state.
REASON=''
INND=''
STATE=''

trap '(
	echo "${PROGNAME} waiting for INND to start (pid: $$)"
	date
    ) >${INNWSTATUS}' 2

##  We need to remember the process ID of innd, in case one exits.
##  But we need to wait for innd to start before we can do that.
while PID=`cat ${SERVERPID} 2>/dev/null`; test -z "${PID}"; do
    sleep ${INNWATCHSLEEPTIME}
done

trap '(
	if [ -z "${STATE}" ]; then
	    echo "${PROGNAME} state RUN interval ${INNWATCHSLEEPTIME} pid $$"
	else
	    echo "${PROGNAME} state ${STATE} interval ${INNWATCHSLEEPTIME} pid $$"
	fi
	if [ -z "${INND}" ]; then
	    X=GO
	else
	    X="${INND}"
	fi
	test -n "${REASON}" && X="${X}: ${REASON}"
	echo "INND state ${X}"
	date
    ) >${INNWSTATUS}' 2

cd ${SPOOL}

NEXTSLEEP=1
HASEXITED=false

while { sleep ${NEXTSLEEP} & wait; } ; : ; do
    NEXTSLEEP=${INNWATCHSLEEPTIME}

    ##  If news.daily is running, idle:  we don't want to change the
    ##  status of anything while news.daily may be depending on what we
    ##  have done.
    test -f "${DAILY}" && continue

    ##  Check to see if innd is running.
    ##  Notify the newsmaster if it has stopped or just restarted.
    if ctlinnd -s -t 120 mode 2>/dev/null ; then
	${HASEXITED} && {
	    HASEXITED=false
	    ${MAILCMD} -s "INND is now running" ${NEWSMASTER} </dev/null
	}
    else
	${HASEXITED} || {
	    HASEXITED=true
	    ${MAILCMD} -s "INND is NOT running" ${NEWSMASTER} </dev/null
	}
	continue
    fi

    ##  If innd has exited & restarted, put the new one into the
    ##  same state the old one was in.
    nPID=`cat ${SERVERPID} 2>/dev/null`
    test -n "${nPID}" -a "${PID}" -ne "${nPID}" && {
	test -n "${INND}" -a "${INND}" != go && ctlinnd -s "${INND}" "${REASON}"
	PID="${nPID}"
    }

    VALUE=0
    PREVEXP=''

    exec 3<&0
    exec 0<${CTLWATCH}

    LINE=0
    while read line ; do
	LINE=`expr ${LINE} + 1`
	test -z "$line" && continue

	##  The first character on the line is the field delimiter,
	##  except '#' which marks the line as a comment.
	delim=`expr "${line}" : '\(.\).*'`
	test "X${delim}" = 'X#' && continue

	##  Parse the line into seven fields, and assign them to local vars.
	##  You're welcome to work out what's going on with quoting in
	##  the next few lines if you feel inclined.
	eval `trap '' 2; echo "${line}" \
		| ${SED} -e "s/'/'\"'\"'/g" \
		    -e "s/[ 	]*\\\\${delim}[ 	]*/\\\\${delim}/g" \
		| ${AWK} -F"${delim}" '{ print	"LAB='"'"'" $2 "'"'"'", \
						"CND='"'"'" $3 "'"'"'", \
						"EXP='"'"'" $4 "'"'"'", \
						"TST='"'"'" $5 "'"'"'", \
						"LIM=" $6, \
						"CMD='"'"'" $7 "'"'"'", \
						"CMT='"'"'" $8 "'"'"'" }'`

	##  If there's no label, the label is the line number.
	test -z "${LAB}" && LAB=${LINE}

	##  Should we act on this line?  We will if one (or more) of the
	##  specified conditions is satisfied.
	for X in a b; do	# Meaningless trash because we have no goto.
	    if [ -z "${CND}" ]; then
		X=-
	    else
		X="${CND}"
	    fi
	    set -$- X ${X}; shift
	    for cnd
	    do
		case "${cnd}" in
		-)
		    test -n "${STATE}" -a "X${STATE}" != "X${LAB}" && continue
		    ;;
		+)
		    test -n "${STATE}" && continue
		    ;;
		'*')
		    ;;
		-*)
		    test "X-${STATE}" = "X${cnd}" && continue
		    ;;
		*)
		    test "X${STATE}" != "X${cnd}" && continue;
		    ;;
		esac
		break 2;	# OK, continue with this line.
	    done
	    continue 2;		# No, skip it.
	done

	##  Evaluate the expression, if there is one, and if that works.
	if [ -z "${EXP}" -o "${EXP}" = "${PREVEXP}" ] \
	 || { PREVEXP="${EXP}"; VALUE=`trap '' 2;eval "${EXP}"`; }; then
	    ##  If innd is running, and test "succeeds", stop it.
	    case "${CMD}" in
	    throttle|pause)
		OK=n
		;;
	    *)
		OK=y
		;;
	    esac

	    if [ \( -z "${STATE}" -o "${STATE}" != "${LAB}" -o "${OK}" = y \) \
		    -a "${VALUE}" "-${TST}" "${LIM}" ] ; then
		R="${CMT} [${PROGNAME}:${LAB}] ${VALUE} ${TST} ${LIM}"
		O=
		case "${CMD}" in
		throttle)
		    case "${STATE}" in
		    ''|go)
			REASON="${R}"
			;;
		    *)
			;;
		    esac
		    O="${LAB}"
		    ARG="${REASON}"
		    ;;
		pause)
		    O="${LAB}"
		    REASON="${R}"
		    ARG="${REASON}"
		    ;;
		shutdown)
		    ARG="${R}"
		    ;;
		flush)
		    ARG=''
		    O="${STATE}"
		    ARG="${REASON}"
		    ;;
		go)
		    ARG="${REASON}"
		    NEXTSLEEP=1
		    REASON=''
		    ;;
		exit)
		    exit 0
		    ;;
		*)
		    break
		    ;;
		esac

		ctlinnd -s "${CMD}" "${ARG}" && STATE="${O}" && INND="${CMD}"
		break

	    ##  Otherwise, if innd is not running, and reverse test succeeds,
	    ##  restart it.
	    elif [ "${STATE}" = "${LAB}" -a \
		    \( "${CMD}" = "throttle" -o "${CMD}" = pause \) -a \
		    ! "${VALUE}" "-${TST}" "${LIM}" ] ; then
		ctlinnd -s go "${REASON}"
		STATE=''
		REASON=''
		INND=''

		##  If we have started innd, run all tests again quickly in
		##  case there is some other condition that should stop it.
		NEXTSLEEP=1
		break
	    fi
	fi

    done

    exec 0<&3
    exec 3<&-

    if [ -n "${LOGFILE}" -a -f "${LOGFILE}" ]; then
	if [ ! -f ${TIMESTAMP} ]; then
	    DOIT=${LOGFILE}
	else
	    # Use ls to print most recently modified file first.
	    # If that's ${LOGFILE}, it has changed since the last pass.
	    DOIT="`ls -t ${TIMESTAMP} ${LOGFILE} | ${SED} -e 1q | grep ${LOGFILE}`"
	fi

	# If the file has been modified more recently than the timestamp,
	# and the file has length greater than 0, send the warning.
	if [ -n "${DOIT}" -a -s ${LOGFILE} ]; then
	    date >${TIMESTAMP}
	    (
		ls -l ${LOGFILE}
		echo "-----"
		ctlinnd -t120 mode
		echo "-----"
		cat ${LOGFILE}
	    ) 2>&1 \
            | sed -e 's/^~/~~/' \
	    | ${MAILCMD} -s "${PROGNAME} warning: messages in ${LOGFILE}" \
		${NEWSMASTER}
	fi
    fi

done

rm -f ${LOCK}