check-mk-server 1.2.2p3-1 / usr / share / check_mk / checks / logwatch

#!/usr/bin/python
# -*- encoding: utf-8; py-indent-offset: 4 -*-
# +------------------------------------------------------------------+
# |             ____ _               _        __  __ _  __           |
# |            / ___| |__   ___  ___| | __   |  \/  | |/ /           |
# |           | |   | '_ \ / _ \/ __| |/ /   | |\/| | ' /            |
# |           | |___| | | |  __/ (__|   <    | |  | | . \            |
# |            \____|_| |_|\___|\___|_|\_\___|_|  |_|_|\_\           |
# |                                                                  |
# | Copyright Mathias Kettner 2013             mk@mathias-kettner.de |
# +------------------------------------------------------------------+
#
# This file is part of Check_MK.
# The official homepage is at http://mathias-kettner.de/check_mk.
#
# check_mk is free software;  you can redistribute it and/or modify it
# under the  terms of the  GNU General Public License  as published by
# the Free Software Foundation in version 2.  check_mk is  distributed
# in the hope that it will be useful, but WITHOUT ANY WARRANTY;  with-
# out even the implied warranty of  MERCHANTABILITY  or  FITNESS FOR A
# PARTICULAR PURPOSE. See the  GNU General Public License for more de-
# ails.  You should have  received  a copy of the  GNU  General Public
# License along with GNU Make; see the file  COPYING.  If  not,  write
# to the Free Software Foundation, Inc., 51 Franklin St,  Fifth Floor,
# Boston, MA 02110-1301 USA.

# Configuration variables in main.mk needed during the actual check
logwatch_dir = var_dir + '/logwatch'
logwatch_patterns = { }
logwatch_rules = []
logwatch_max_filesize = 500000 # do not save more than 500k of message (configurable)
logwatch_service_output = "default"
logwatch_forward_to_ec = False

# Variables embedded in precompiled checks
check_config_variables += [ "logwatch_dir", "logwatch_max_filesize", "logwatch_service_output" ]

def inventory_logwatch(info):
    if logwatch_forward_to_ec:
        # this is the regular logwatch inventory. Terminate when in logwatch forward mode
        return []

    inventory = []
    for l in info:
        line = " ".join(l)
        if len(line) > 6 and line[0:3] == "[[[" and line[-3:] == "]]]" \
           and ':missing' not in line and ':cannotopen' not in line:
            inventory.append((line[3:-3], "", '""'))
    return inventory

#logwatch_patterns = {
#    'System': [
#    ( 'W', 'sshd' ),
#    ( ['host1', 'host2'],        'C', 'ssh' ), # only applies to certain hosts
#    ( ['lnx', 'dmz'], ALL_HOSTS, 'C', 'ssh' ), # only applies to host having certain tags
#    ( ALL_HOSTS, (10, 20), 'x' ), # at 10 messages per interval warn, at 20 crit
#    ( 'I', '0' )
#    ],
#    'Application': [
#    ( 'W', 'crash.exe' ),
#    ( 'E', 'ssh' )
#    ]
#    }

# New rule-stule logwatch_rules in WATO friendly consistent rule notation:
#
# logwatch_rules = [
#   ( [ PATTERNS ], ALL_HOSTS, [ "Application", "System" ] ),
# ]
# All [ PATTERNS ] of matching rules will be concatenated in order of
# appearance.
#
# PATTERN is a list like:
# [ ( 'O',      ".*ssh.*" ),          # Make informational (OK) messages from these
#   ( (10, 20), "login"   ),          # Warning at 10 messages, Critical at 20
#   ( 'C',      "bad"     ),          # Always critical
#   ( 'W',      "not entirely bad" ), # Always warning
# ]
#


def logwatch_state(state):
    if state == 1:
        return "WARN"
    elif state != 0:
        return "CRIT"
    else:
        return "OK"

def logwatch_level_name(level):
    if   level == 'O': return 'OK'
    elif level == 'W': return 'WARN'
    elif level == 'u': return 'WARN' # undefined states are treated as warning
    elif level == 'C': return 'CRIT'
    else: return 'IGN'

def logwatch_level_worst(worst, level):
    if   level == 'O': return max(worst, 0)
    elif level == 'W': return max(worst, 1)
    elif level == 'u': return max(worst, 1)
    elif level == 'C': return max(worst, 2)
    else: return worst

# Extracts patterns that are relevant for the current host and item.
# Constructs simple list of pairs: [ ('W', 'crash.exe'), ('C', 'sshd.*test') ]
def logwatch_precompile(hostname, item, _unused):
    # Initialize the patterns list with the logwatch_rules
    params = []
    description = check_info['logwatch']['service_description'] % item

    # This is the new (-> WATO controlled) variable
    rules = service_extra_conf(hostname, item, logwatch_rules)
    for rule in rules:
        for pattern in rule:
            params.append((pattern[0], pattern[1]))

    # Now load the old logwatch_patterns var
    patterns = logwatch_patterns.get(item)
    if patterns:
        for entry in patterns:
            hostlist = None
            tags = []

            pattern = entry[-1]
            level = entry[-2]

            if len(entry) >= 3:    # found optional host list
                hostlist = entry[-3]
            if len(entry) >= 4:    # found optional host tags
                tags = entry[-4]

            if hostlist and not \
                   (hosttags_match_taglist(tags_of_host(hostname), tags) and \
                    in_extraconf_hostlist(hostlist, hostname)):
                continue

            params.append((level, pattern))

    return params


def logwatch_reclassify(counts, patterns, text):
    for level, pattern in patterns:
        reg = compiled_regexes.get(pattern)
        if not reg:
            reg = re.compile(pattern)
            compiled_regexes[pattern] = reg
        if reg.search(text):
            # If the level is not fixed like 'C' or 'W' but a pair like (10, 20),
            # then we count how many times this pattern has already matched and
            # assign the levels according to the number of matches of this pattern.
            if type(level) == tuple:
                warn, crit = level
                newcount = counts.setdefault(id(pattern), 0) + 1
                counts[id(pattern)] = newcount
                if newcount >= crit:
                    return 'C'
                elif newcount >= warn:
                    return 'W'
                else:
                    return 'I'
            else:
                return level
    return None

def logwatch_parse_line(line):
    parts = line.split(None, 1)
    level = parts[0]
    if len(parts) > 1:
        text = parts[1]
    else:
        text = ""
    return level, text

# In case of a precompiled check, params contains the precompiled
# logwatch_patterns for the logfile we deal with. If using check_mk
# without precompiled checks, the params must be None an will be
# ignored.
def check_logwatch(item, params, info):
    found = False
    loglines = []
    for l in info:
        line = " ".join(l)
        if line == "[[[%s]]]" % item:
            found = True
        elif len(line) > 6 and line[0:3] == "[[[" and line[-3:] == "]]]":
            if found:
                break
            found = False
        elif found:
            loglines.append(line)

    # Create directories, if neccessary
    try:
        logdir = logwatch_dir + "/" + g_hostname
        if not os.path.exists(logwatch_dir):
            os.mkdir(logwatch_dir)
        if not os.path.exists(logdir):
            os.mkdir(logdir)
            if www_group != None:
                try:
                    if i_am_root():
                        import pwd
                        to_user = pwd.getpwnam(nagios_user)[2]
                    else:
                        to_user = -1 # keep user unchanged
                    os.chown(logdir, to_user, www_group)
                    os.chmod(logdir, 0775)
                except Exception, e:
                    os.rmdir(logdir)
                    raise MKGeneralException(("User %s cannot chown directory to group id %d: %s. Please make sure "+
                                             "that %s is a member of that group.") %
                                             (username(), www_group, e, username()))

    except MKGeneralException:
        raise
    except Exception, e:
        raise MKGeneralException("User %s cannot create logwatch directory: %s" % \
                                 (username(), e) )

    logfile = logdir + "/" + item.replace("/", "\\")

    # Logfile (=item) section not found and no local file found. This usually
    # means, that the corresponding logfile also vanished on the target host.
    if found == False and not os.path.exists(logfile):
        return (3, "UNKNOWN - log not present anymore")

    # if logfile has reached maximum size, abort with critical state
    if os.path.exists(logfile) and os.path.getsize(logfile) > logwatch_max_filesize:
        return (2, "CRIT - unacknowledged messages have exceeded max size (%d Bytes)" % logwatch_max_filesize)

    #
    # Write out new log lines (no reclassify here. It is done later in general for all logs)
    #
    if len(loglines) > 0:
        worst = -1
        for line in loglines:
            worst = logwatch_level_worst(worst, logwatch_parse_line(line)[0])

        # Ignore log lines which result in an "I" -> "Ignore" state
        if worst > -1:
            try:
                logarch = file(logfile, "a+")
                logarch.write(time.strftime("<<<%Y-%m-%d %H:%M:%S " + logwatch_state(worst) + ">>>\n"))
                logarch.write("\n".join(loglines) + "\n")
                logarch.close()
            except Exception, e:
                raise MKGeneralException("User %s cannot create logfile: %s" % \
                                         (username(), e))

    # Get the patterns (either compile or reuse the precompiled ones)
    # Check_MK creates an empty string if the precompile function has
    # not been exectued yet. The precomile function creates an empty
    # list when no rules/patterns are defined.
    if params != '':
        patterns = params # patterns already precompiled
    else:
        patterns = logwatch_precompile(g_hostname, item, None)

    def reclassify_line(counts, patterns, text, level):
        if patterns:
            newlevel = logwatch_reclassify(counts, patterns, text)
            if newlevel != None:
                level = newlevel
        return level

    #
    # Read current log messages, reclassify all messages and write out the
    # whole file again if at least one line has been reclassified
    #
    worst = 0
    last_worst_line = ''
    reclassified_lines = []
    reclassified = False
    counts = {} # for counting number of matches of a certain pattern
    state_counts = {} # for counting number of blocks with a certain state
    block_header = None
    block_worst = -1
    block_lines = []
    block_last_worst_line = ''

    def finish_block(header, block_worst, block_lines):
        start = ' '.join(header.split(' ')[0:2])
        return [ start + ' %s>>>' % logwatch_state(block_worst) ] + block_lines

    if os.path.exists(logfile):
        for line in file(logfile):
            line = line.rstrip('\n')
            # Skip empty lines
            if not line:
                continue
            if line.startswith('<<<') and line.endswith('>>>'):
                if block_header:
                    # The section is finished here. Add it to the list of reclassified lines if the
                    # state of the block is not "I" -> "ignore"
                    if block_worst > -1:
                        reclassified_lines += finish_block(block_header, block_worst, block_lines)

                        if block_worst >= worst:
                            worst = block_worst
                            last_worst_line = block_last_worst_line

                    block_lines = []
                    block_worst = -1
                block_header = line
                counts = {}
            else:
                old_level, text = logwatch_parse_line(line)
                level = reclassify_line(counts, patterns, text, old_level)

                if old_level != level:
                    reclassified = True

                old_block_worst = block_worst
                block_worst = logwatch_level_worst(block_worst, level)

                # Save the last worst line of this block
                if old_block_worst != block_worst or level == block_worst:
                    block_last_worst_line = text

                # Count the number of lines by state
                if level != '.':
                    try:
                        state_counts[level] += 1
                    except KeyError:
                        state_counts[level] = 1

                block_lines.append(level + ' ' + text)

        # The last section is finished here. Add it to the list of reclassified lines if the
        # state of the block is not "I" -> "ignore"
        if block_header and block_worst > -1:
            reclassified_lines += finish_block(block_header, block_worst, block_lines)

            if block_worst >= worst:
                worst = block_worst
                last_worst_line = block_last_worst_line


        # Only rewrite the lines if at least one line has been reclassified
        if reclassified:
            if reclassified_lines:
                file(logfile, 'w').write('\n'.join(reclassified_lines) + '\n')
            else:
                os.unlink(logfile)

    #
    # Render output
    #

    state = logwatch_state(worst)

    if len(reclassified_lines) == 0:
        return (0, "OK - no error messages")
    else:
        count_txt = []
        for level, num in state_counts.iteritems():
            count_txt.append('%d %s' % (num, logwatch_level_name(level)))
        if logwatch_service_output == 'default':
            return (worst, state + " - %s messages (Last worst: \"%s\")" %
                                           (', '.join(count_txt), last_worst_line))
        else:
            return (worst, state + " - %s messages" % ', '.join(count_txt))


check_info['logwatch'] = {
    'check_function':      check_logwatch,
    'service_description': "LOG %s",
    'has_perfdata':        0,
    'inventory_function':  inventory_logwatch,
    'group':               'logwatch',
}

precompile_params['logwatch'] = logwatch_precompile

#   .----------------------------------------------------------------------.
#   |      _____ ____   _____ ___  ______        ___    ____  ____         |
#   |     | ____/ ___| |  ___/ _ \|  _ \ \      / / \  |  _ \|  _ \        |
#   |     |  _|| |     | |_ | | | | |_) \ \ /\ / / _ \ | |_) | | | |       |
#   |     | |__| |___  |  _|| |_| |  _ < \ V  V / ___ \|  _ <| |_| |       |
#   |     |_____\____| |_|   \___/|_| \_\ \_/\_/_/   \_\_| \_\____/        |
#   |                                                                      |
#   +----------------------------------------------------------------------+
#   | Forwarding logwatch messages to event console                        |
#   '----------------------------------------------------------------------'

import socket, time

def inventory_logwatch_ec(info):
    if logwatch_forward_to_ec and info:
        # this is the logwatch event console forward inventory. Add one service per host.
        return [(None, "{}")]

    # this is the logwatch event console forward inventory. Terminate when not in logwatch forward mode
    return []

# OK   -> priority 5
# WARN -> priority 4
# CRIT -> priority 2
# u = Uknown
def logwatch_to_prio(level):
    if level == 'W':
        return 4
    elif level == 'C':
        return 2
    elif level == 'O':
        return 5
    elif level == 'u':
        return 4

def syslog_time():
    localtime = time.localtime()
    day = int(time.strftime("%d", localtime)) # strip leading 0
    value = time.strftime("%b %%d %H:%M:%S", localtime)
    return value % day

def check_logwatch_ec(_unused, params, info):
    # 1. Parse lines in info and separate by logfile
    logs = []
    logfile = None
    for l in info:
        line = " ".join(l)
        if len(line) > 6 and line[0:3] == "[[[" and line[-3:] == "]]]":
            # new logfile, extract name
            logfile = line[3:-3]

        elif logfile and line:
            # new regular line, skip context lines
            if line[0] != '.':
                logs.append((logfile, line))

    # 2. create syslog message of each line
    # <128> Oct 24 10:44:27 Klappspaten /var/log/syslog: Oct 24 10:44:27 Klappspaten logger: asdasdad as
    # <facility+priority> timestamp hostname logfile: message
    facility = params.get('facility', 17) << 3 # default to "local1"
    messages = []
    cur_time = syslog_time()
    for logfile, line in logs:
        msg = '<%d>' % (facility + logwatch_to_prio(line[0]),)
        msg += '%s %s %s: %s' % (cur_time, g_hostname, logfile, line[2:])
        messages.append(msg)

    # 3. send lines to event console
    # a) local in same omd site
    # b) local pipe
    # c) remote via udp
    # d) remote via tcp
    method = params.get('method')
    if method == None:
        method = os.getenv('OMD_ROOT') + "/tmp/run/mkeventd/eventsocket"

    try:
        if isinstance(method, tuple):
            # connect either via tcp or udp
            if method[0] == 'udp':
                sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
            else:
                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            sock.connect((method[1], method[2]))
            for message in messages:
                sock.send(message)
            sock.close()

        else:
            # write into local event pipe
            # Important: When the event daemon is stopped, then the pipe
            # is *not* existing! This prevents us from hanging in such
            # situations. So we must make sure that we do not create a file
            # instead of the pipe!
            sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
            sock.connect(method)
            sock.send('\n'.join(messages) + '\n')
            sock.close()

        num_messages = len(messages)
        return (0, 'OK - Forwarded %d messages to event console' % num_messages, [('messages', num_messages)])
    except Exception, e:
        return (2, 'CRITICAL - Unable to forward messages to event console (%s)' % e)

check_info['logwatch.ec'] = {
    'check_function':      check_logwatch_ec,
    'service_description': "Log Forwarding",
    'has_perfdata':        0,
    'inventory_function':  inventory_logwatch_ec,
    'group':               'logwatch_ec',
}