xen-utils-4.9 4.9.2-0ubuntu1 / usr / lib / xen-4.9 / bin / xen-ringwatch

#! /usr/bin/python2
#
# Copyright (C) 2011 Citrix Systems, Inc.
#
# This library is free software; you can redistribute it and/or modify
# it under the terms of version 2.1 of the GNU Lesser General Public
# License as published by the Free Software Foundation.
#
# This library is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; If not, see <http://www.gnu.org/licenses/>.
#

"""Overview:

        - Gather Xen I/O ring states
          (from %s/*/ring)

        - Update ring states every -T seconds.

        - Determine if rings are idle or make progress.

        - Determine if idle rings dropped notifications (%s).

        - Instruct stuck backends to reissue notifications.
"""

import os
import glob

class Pattern(object):
    """A regex pattern. Compiled on demand, then persisted."""

    def __init__(self, regex):
        self.regex     = regex
        self.__pattern = None

    def get(self):
        import re

        if not self.__pattern:
            self.__pattern = re.compile(self.regex)

        return self.__pattern

    def search(self, s):
        return self.get().search(s)

class XenBackend(object):
    """A Xen I/O backend."""

    SYSFS_BASEDIR = "/sys/devices/xen-backend"

    def __init__(self, rd, devid):
        self.rd    = int(rd)
        self.devid = int(devid)

    def __repr__(self):
        return "%s(%d, %d)" % (type(self).__name__,
                               self.rd, self.devid)

    def name(self):
        raise NotImplementedError

    def path(self):
        return "%s/%s" % (self.SYSFS_BASEDIR, self.name())

    _name_pattern = None

    @classmethod
    def from_name(cls, name):
        match = cls._name_pattern.search(name)
        if not match:
            raise Exception, "Malformed %s name: %s" % \
                (type(self).__name__, name)

        rd    = match.group(1)
        devid = match.group(2)

        return cls(rd, devid)

    _name_glob = None

    @classmethod
    def find(cls):
        paths = glob.glob("%s/%s" % (cls.SYSFS_BASEDIR,
                                     cls._name_glob))
        for path in paths:
            name = os.path.basename(path)
            yield cls.from_name(name)

    def find_rings(self):
        for ring in self.Ring.find(self):
            yield ring

    class Ring(object):

        def __init__(self, backend, name):
            self.backend = backend
            self.name    = name

        __size = None

        def key(self):
            return "%s/%s" % (self.backend.name(),
                              self.name)

        def __str__(self):
            return "%s(%s)" % (type(self).__name__, self.key())

        @classmethod
        def from_name(cls, backend, name):
            return cls(backend, name)

        _name_glob = None

        @classmethod
        def find(cls, backend):
            paths = glob.glob("%s/%s" % (backend.path(),
                                         cls._name_glob))
            for path in paths:
                name = os.path.basename(path)
                yield cls.from_name(backend, name)

        def path(self):
            return "%s/%s" % (self.backend.path(),
                              self.name)

        def read(self):
            state = RingState.from_sysfs(self.path())
            return state

        def write(self, cmd):
            f = file(self.path(), 'w')
            try:
                f.write(cmd.rstrip())
            finally:
                f.close()

        def kick(self):
            self.write("kick")

        def poll(self):
            self.write("poll")

    __ring = None

    TYPES = {}
    XEN_BACKEND_NAME = None

    @classmethod
    def register(cls):
        XenBackend.TYPES[cls.XEN_BACKEND_NAME] = cls

class VBD(XenBackend):
    """Xen blkif backends."""

    XEN_BACKEND_NAME = 'vbd'

    _name_pattern = Pattern("vbd-(\d+)-(\d+)")
    _name_glob    = "vbd-*-*"

    def name(self):
        return "vbd-%d-%d" % (self.rd, self.devid)

    class Ring(XenBackend.Ring):
        _name_glob = "io_ring"

VBD.register()

class VIF(XenBackend):
    """Xen netif backends."""

    XEN_BACKEND_NAME = 'vif'

    _name_pattern = Pattern("vif-(\d+)-(\d+)")
    _name_glob    = "vif-*-*"

    def name(self):
        return "vif-%d-%d" % (self.rd, self.devid)

    class Ring(XenBackend.Ring):
        _name_glob = "{rx,tx}_ring"

#VIF.register()

class RingState(object):
    """Overall backend ring state. Comprising req and rsp queue
    indexes, and analysis."""

    def __init__(self, size, req, rsp):
        self.size = int(size)
        self.req  = req
        self.rsp  = rsp

    _size_pattern = Pattern("nr_ents (\d+)")

    @classmethod
    def from_sysfs(cls, path):

        f = file(path, "r")
        try:
            s = f.read()
        finally:
            f.close()

        try:
            (_nr_ents, _req, _rsp, _) = s.split("\n")

            match   = cls._size_pattern.search(_nr_ents)
            nr_ents = int(match.group(1))

        except Exception, e:
            raise Exception, "Malformed %s input: %s (%s)" % \
                (cls.__name__, repr(s), str(e))

        req = cls.Req.from_sysfs(_req, size=nr_ents)
        rsp = cls.Rsp.from_sysfs(_rsp, size=nr_ents)

        return cls(nr_ents, req, rsp)

    class Queue(dict):

        def __init__(self, size):
            self.size = int(size)

        prod = None

        @classmethod
        def from_sysfs(cls, line, **d):

            match = cls._pattern.search(line)
            if not match:
                raise Exception, "Malformed %s input: %s" % \
                    (cls.__name__, repr(s))

            i = iter(match.groups())
            for k in i:
                d[k] = i.next()

            return cls(**d)

        def is_consumed(self):
            return self.prod == self._cons()

    class Req(Queue):

        _pattern = Pattern("req (prod) (\d+) (cons) (\d+) (event) (\d+)")

        def __init__(self, prod, cons, event, **d):
            RingState.Queue.__init__(self, **d)
            self.prod  = int(prod)
            self.cons  = int(cons)
            self.event = int(event)

        def __repr__(self):
            return "%s(prod=%d, cons=%d, event=%d)" % \
                (type(self).__name__, self.prod, self.cons, self.event)

        def _cons(self):
            return self.cons

        def __eq__(self, other):
            return \
                self.prod  == other.prod and \
                self.cons  == other.cons and \
                self.event == other.event

    class Rsp(Queue):

        _pattern = Pattern("rsp (prod) (\d+) (pvt) (\d+) (event) (\d+)")

        def __init__(self, prod, pvt, event, **d):
            RingState.Queue.__init__(self, **d)
            self.prod  = int(prod)
            self.pvt   = int(pvt)
            self.event = int(event)

        def __repr__(self):
            return "%s(prod=%d, pvt=%d, event=%d)" % \
                (type(self).__name__, self.prod, self.pvt, self.event)

        def _cons(self):
            return self.event - 1

        def __eq__(self, other):
            return \
                self.prod  == other.prod and \
                self.pvt   == other.pvt  and \
                self.event == other.event

    def is_consumed(self):
        return \
            self.rsp.is_consumed() and \
            self.req.is_consumed()

    def is_pending(self):
        return self.rsp.prod != self.req.prod

    def kick(self, ring):
        action = False

        if not self.req.is_consumed():
            action |= True
            ring.poll()

        if not self.rsp.is_consumed():
            action |= True
            ring.kick()

        return action

    def __eq__(self, other):
        return \
            self.size == other.size and \
            self.req == other.req and \
            self.rsp == other.rsp

    def __repr__(self):
        return "%s(size=%d, %s, %s)" % \
            (type(self).__name__, self.size, self.req, self.rsp)

    def display(self):
        complete = { True: "complete", False: "pending" }

        io  = complete[not self.is_pending()]
        req = complete[self.req.is_consumed()]
        rsp = complete[self.rsp.is_consumed()]

        return "%s: io: %s, req: %s, rsp: %s" % (self, io, req, rsp)

class RingWatch(object):
    """State machine watching I/O individual ring state"""

    _NEW  = "_NEW"
    BUSY  = "BUSY"
    IDLE  = "IDLE"
    STCK  = "STCK"

    COMMENTS = { BUSY: "Message traffic observed (OK)",
                 IDLE: "No messages observed (Ring OK, I/O depends)",
                 STCK: "No pending req/rsp consumer progress observed (BUG)" }

    def __init__(self, ring, state):
        self.ring   = ring
        self.state  = state
        self.status = RingWatch._NEW

    @classmethod
    def new(cls, ring):
        state = ring.read()
        return cls(ring, state)

    def __str__(self):
        return "%s(%s)[%s]" % \
            (type(self).__name__, self.ring.key(), self.status)

    def is_stuck(self):
        return self.status == self.STCK

    def is_idle(self):
        return self.status == self.IDLE

    def kick(self):
        if self.is_stuck():
            return self.state.kick(self.ring)

    def update(self):

        prev = self.state
        curr = self.ring.read()

        if curr == prev:
            if not curr.is_consumed():
                self.status = self.STCK
            else:
                self.status = self.IDLE
        else:
            self.status = self.BUSY

        self.state = curr

    def display(self):
        return "%s: %s" % (self,
                           self.state.display())

class WatchList(object):
    """Managed collection of I/O rings under surveillance."""

    def __init__(self, gen):
        self.gen  = gen
        self.list = {}

    def update(self):

        # NB. clear the watch list, then rebuild it. new entries get
        # added, existing ones updates, those gone discarded.
        prev      = self.list
        self.list = {}

        for ring in self.gen():

            key   = ring.key()
            entry = prev.get(key)

            try:
                if not entry:
                    entry = RingWatch.new(ring)
                else:
                    entry.update()

            except IOError, e:
                pass
                # NB. racing unplug, any ring.read() may raise.
                # nothing left to memorize then.
            else:
                self.list[key] = entry

    def __iter__(self):
        return self.list.itervalues()

    def pending(self):
        for entry in self:
            if entry.is_idle() and entry.state.is_pending():
                yield entry

    def stuck(self):
        for entry in self:
            if entry.is_stuck():
                yield entry

    def kick(self):
        for entry in self.stuck():
            try:
                entry.kick()
            except IOError:
                # NB. racing unplug, any ring.write() may raise.
                pass

if __name__ == '__main__':
    from sys import argv, stdout, stderr, exit
    from getopt import gnu_getopt, GetoptError
    from pprint import pprint

    DEFAULT_PERIOD = 1 # secs

    verbose  = 0
    period   = DEFAULT_PERIOD
    backends = XenBackend.TYPES.values()
    kick     = False
    iowatch  = False

    OPTIONS = ((('h', 'help'),
                "Print this help screen."),

               (('v', 'verbose'),
                "Increase output verbosity level (use n-times)."),

               (('I', 'io'),
                "Watch out for stuck I/O (not messaging), too. (%s)" % \
                    (iowatch)),

               (('t', 'types'),
                "Comma separated list of backend types to watch. (%s)" % \
                    ",".join(map(lambda t: t.XEN_BACKEND_NAME, backends))),

               (('T', 'period'),
                "Watch update period. (%d) [secs]" % \
                    (period)),

               (('k', 'kick'),
                "Kick broken guests out of cardiac arrest. (%s)" % \
                    (kick))
               )

    COMMANDS = {"check":
                    "Single iteration quick test (takes -T seconds)."}

    def usage(stream):
        prog = os.path.basename(argv[0])

        print >>stream

        print >>stream, "Usage:"
        print >>stream, "\t%s [options] {%s}" % (prog, "|".join(COMMANDS))

        print >>stream

        print >>stream, "Commands:"
        for (name, desc) in COMMANDS.iteritems():
            print >>stream, "\t%s: \t%s" % (name, desc)

        print >>stream

        print >>stream, "Options:"
        for ((short, _long), desc) in OPTIONS:
            print >>stream, "\t-%s, --%s: \t%s" % (short, _long, desc)

        print >>stream

    def fail(msg = None):
        if msg: print >>stderr, "Error: %s" % msg
        usage(stderr)
        exit(1)

    def help():

        usage(stdout)

        print __doc__ % (XenBackend.SYSFS_BASEDIR, RingWatch.STCK)

        print "Backend Types:"
        for k, v in XenBackend.TYPES.iteritems():
            print "\t%s: \t%s (%s)" % (k, v.__doc__, v._name_glob)

        print
        print "Ring States:"
        for k, v in RingWatch.COMMENTS.iteritems():
            print "\t%s: \t%s" % (k, v)

        print

    try:
        opts, args = gnu_getopt(argv[1:],
                                "hIkt:vT:",
                                ["help",
                                 "io",
                                 "kick",
                                 "type=",
                                 "verbose",
                                 "period="])
    except GetoptError, e:
        fail(str(e))

    for (o, arg) in opts:
        try:
            if o in ('-h', '--help'):
                help()
                exit(0)

            elif o in ['-v', '--verbose']:
                verbose += 1

            elif o in ['-I', '--io']:
                iowatch = True

            elif o in ('-T', '--period'):
                period = int(arg)

            elif o in ('-t', '--type'):
                backends = ",".split(arg)
                backends = map(lambda t: XenBackend.TYPES[t], backends)

            elif o in ('-k', '--kick'):
                kick = True

            else:
                raise "BUG: option %s unhandled." % o

        except ValueError:
            fail("%s: invalid argument '%s'." % (o, arg))

    try:
        cmd = args[0]
    except IndexError:
        fail("Missing command.")

    def ring_select():
        for _type in backends:
            for backend in _type.find():
                for ring in backend.find_rings():
                    yield ring

    def show(entries):
        for watch in entries:
            print watch.display()

    def pause():
        import time
        time.sleep(period)

    watches = WatchList(ring_select)

    if cmd == "check":

        # init
        watches.update()

        if verbose >= 2:
            show(watches)

        # watch for one round
        pause()
        watches.update()

        # show result
        crit  = list(watches.stuck())
        stuck = bool(crit)

        if (iowatch):
            crit.extend(watches.pending())

        if verbose >= 1:
            show(watches)
        elif crit:
            show(crit)

        if stuck and kick:
            # deal with it
            watches.kick()

    else:
        fail("Invalid command.")