trac-bitten 0.6+final-3 / usr / share / pyshared / bitten / master.py

# -*- coding: utf-8 -*-
#
# Copyright (C) 2007-2010 Edgewall Software
# Copyright (C) 2005-2007 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://bitten.edgewall.org/wiki/License.

"""Build master implementation."""

import calendar
import re
import time
from StringIO import StringIO

from trac.attachment import Attachment
from trac.config import BoolOption, IntOption, Option
from trac.core import *
from trac.resource import ResourceNotFound
from trac.web import IRequestHandler, RequestDone

from bitten import PROTOCOL_VERSION
from bitten.model import BuildConfig, Build, BuildStep, BuildLog, Report, \
                     TargetPlatform

from bitten.main import BuildSystem
from bitten.queue import BuildQueue
from bitten.recipe import Recipe
from bitten.util import xmlio

__all__ = ['BuildMaster']
__docformat__ = 'restructuredtext en'


HTTP_BAD_REQUEST = 400
HTTP_FORBIDDEN = 403
HTTP_NOT_FOUND = 404
HTTP_METHOD_NOT_ALLOWED = 405
HTTP_CONFLICT = 409


class BuildMaster(Component):
    """Trac request handler implementation for the build master."""

    implements(IRequestHandler)

    # Configuration options

    adjust_timestamps = BoolOption('bitten', 'adjust_timestamps', False, doc=
        """Whether the timestamps of builds should be adjusted to be close
        to the timestamps of the corresponding changesets.""")

    build_all = BoolOption('bitten', 'build_all', False, doc=
        """Whether to request builds of older revisions even if a younger
        revision has already been built.""")

    stabilize_wait = IntOption('bitten', 'stabilize_wait', 0, doc=
        """The time in seconds to wait for the repository to stabilize before
        queuing up a new build.  This allows time for developers to check in
        a group of related changes back to back without spawning multiple
        builds.""")

    slave_timeout = IntOption('bitten', 'slave_timeout', 3600, doc=
        """The time in seconds after which a build is cancelled if the slave
        does not report progress.""")

    logs_dir = Option('bitten', 'logs_dir', "log/bitten", doc=
         """The directory on the server in which client log files will be stored.""")

    quick_status = BoolOption('bitten', 'quick_status', False, doc=
         """Whether to show the current build status within the Trac main
            navigation bar. '''Note:''' The feature requires expensive database and
            repository checks for every page request, and should not be enabled
            if the project has a large repository or uses a non-Subversion
            repository such as Mercurial or Git.""")

    def __init__(self):
        self.env.systeminfo.append(('Bitten',
                __import__('bitten', ['__version__']).__version__))

    # IRequestHandler methods

    def match_request(self, req):
        match = re.match(r'/builds(?:/(\d+)(?:/(\w+)/([^/]+)?)?)?$',
                         req.path_info)
        if match:
            if match.group(1):
                req.args['id'] = match.group(1)
                req.args['collection'] = match.group(2)
                req.args['member'] = match.group(3)
            return True

    def process_request(self, req):
        req.perm.assert_permission('BUILD_EXEC')

        if 'trac_auth' in req.incookie:
            slave_token = req.incookie['trac_auth'].value
        else:
            slave_token = req.session.sid

        if 'id' not in req.args:
            if req.method != 'POST':
                self._send_response(req,
                                body='Only POST allowed for build creation.')
            return self._process_build_creation(req, slave_token)

        build = Build.fetch(self.env, req.args['id'])
        if not build:
            self._send_error(req, HTTP_NOT_FOUND,
                                  'No such build (%s)' % req.args['id'])

        build_token = build.slave_info.get('token', '')
        if build_token != slave_token:
            self._send_error(req, HTTP_CONFLICT,
                          'Token mismatch (wrong slave): slave=%s, build=%s' \
                               % (slave_token, build_token))

        config = BuildConfig.fetch(self.env, build.config)

        if not req.args['collection']:
            if req.method == 'DELETE':
                return self._process_build_cancellation(req, config, build)
            else:
                return self._process_build_initiation(req, config, build)

        if req.method != 'POST':
            self._send_error(req, HTTP_METHOD_NOT_ALLOWED,
                                  'Method %s not allowed' % req.method)

        if req.args['collection'] == 'steps':
            return self._process_build_step(req, config, build)
        elif req.args['collection'] == 'attach':
            return self._process_attachment(req, config, build)
        elif req.args['collection'] == 'keepalive':
            return self._process_keepalive(req, config, build)
        else:
            self._send_error(req, HTTP_NOT_FOUND,
                    "No such collection '%s'" % req.args['collection'])

    # Internal methods

    def _send_response(self, req, code=200, body='', headers=None):
        """ Formats and sends the response, raising ``RequestDone``. """
        if isinstance(body, unicode):
            body = body.encode('utf-8')
        req.send_response(code)
        headers = headers or {}
        headers.setdefault('Content-Length', len(body))
        for header in headers:
            req.send_header(header, headers[header])
        req.write(body)
        raise RequestDone

    def _send_error(self, req, code=500, message=''):
        """ Formats and sends the error, raising ``RequestDone``. """
        headers = {'Content-Type': 'text/plain',
                   'Content-Length': str(len(message))}
        self._send_response(req, code, body=message, headers=headers)

    def _process_build_creation(self, req, slave_token):
        queue = BuildQueue(self.env, build_all=self.build_all,
                           stabilize_wait=self.stabilize_wait,
                           timeout=self.slave_timeout)
        try:
            queue.populate()
        except AssertionError, e:
            self.log.error(e.message, exc_info=True)
            self._send_error(req, HTTP_BAD_REQUEST, e.message)

        try:
            elem = xmlio.parse(req.read())
        except xmlio.ParseError, e:
            self.log.error('Error parsing build initialization request: %s', e,
                           exc_info=True)
            self._send_error(req, HTTP_BAD_REQUEST, 'XML parser error')

        slave_version = int(elem.attr.get('version', 1))

        # FIXME: Remove version compatibility code.
        # The initial difference between protocol version 3 and 4 is that
        # the master allows keepalive requests-- the master must be
        # at least 4 before slaves supporting version 4 are allowed. When
        # the first force master/slave upgrade requirement comes in
        # (or we bump the) version number again, remove this code.
        if slave_version == 3 and PROTOCOL_VERSION == 4:
            self.log.info('Allowing slave version %d to process build for '
                          'compatibility. Upgrade slave to support build '
                          'keepalives.', slave_version)
        elif slave_version != PROTOCOL_VERSION:
            self._send_error(req, HTTP_BAD_REQUEST,
                    "Master-Slave version mismatch: master=%d, slave=%d" % \
                                (PROTOCOL_VERSION, slave_version))

        slavename = elem.attr['name']
        properties = {'name': slavename, Build.IP_ADDRESS: req.remote_addr,
                    Build.TOKEN: slave_token}
        self.log.info('Build slave %r connected from %s with token %s',
                    slavename, req.remote_addr, slave_token)

        for child in elem.children():
            if child.name == 'platform':
                properties[Build.MACHINE] = child.gettext()
                properties[Build.PROCESSOR] = child.attr.get('processor')
            elif child.name == 'os':
                properties[Build.OS_NAME] = child.gettext()
                properties[Build.OS_FAMILY] = child.attr.get('family')
                properties[Build.OS_VERSION] = child.attr.get('version')
            elif child.name == 'package':
                for name, value in child.attr.items():
                    if name == 'name':
                        continue
                    properties[child.attr['name'] + '.' + name] = value

        self.log.debug('Build slave configuration: %r', properties)

        build = queue.get_build_for_slave(slavename, properties)
        if not build:
            self._send_response(req, 204, '', {})

        self._send_response(req, 201, 'Build pending', headers={
                            'Content-Type': 'text/plain',
                            'Location': req.abs_href.builds(build.id)})

    def _process_build_cancellation(self, req, config, build):
        self.log.info('Build slave %r cancelled build %d', build.slave,
                      build.id)
        build.status = Build.PENDING
        build.slave = None
        build.slave_info = {}
        build.started = 0
        db = self.env.get_db_cnx()
        for step in list(BuildStep.select(self.env, build=build.id, db=db)):
            step.delete(db=db)
        build.update(db=db)

        Attachment.delete_all(self.env, 'build', build.resource.id, db)

        db.commit()

        for listener in BuildSystem(self.env).listeners:
            listener.build_aborted(build)

        self._send_response(req, 204, '', {})

    def _process_build_initiation(self, req, config, build):
        self.log.info('Build slave %r initiated build %d', build.slave,
                      build.id)
        build.started = int(time.time())
        build.last_activity = build.started
        build.update()

        for listener in BuildSystem(self.env).listeners:
            listener.build_started(build)

        xml = xmlio.parse(config.recipe)
        xml.attr['path'] = config.path
        xml.attr['revision'] = build.rev
        xml.attr['config'] = config.name
        xml.attr['build'] = str(build.id)
        target_platform = TargetPlatform.fetch(self.env, build.platform)
        xml.attr['platform'] = target_platform.name
        xml.attr['name'] = build.slave
        xml.attr['form_token'] = req.form_token # For posting attachments
        body = str(xml)

        self.log.info('Build slave %r initiated build %d', build.slave,
                      build.id)

        # create the first step, mark it as in-progress.

        recipe = Recipe(xmlio.parse(config.recipe))
        stepname = recipe.__iter__().next().id

        step = self._start_new_step(build, stepname)
        step.insert()

        self._send_response(req, 200, body, headers={
                    'Content-Type': 'application/x-bitten+xml',
                    'Content-Length': str(len(body)),
                    'Content-Disposition':
                        'attachment; filename=recipe_%s_r%s.xml' %
                        (config.name, build.rev)})

    def _process_build_step(self, req, config, build):
        try:
            elem = xmlio.parse(req.read())
        except xmlio.ParseError, e:
            self.log.error('Error parsing build step result: %s', e,
                           exc_info=True)
            self._send_error(req, HTTP_BAD_REQUEST, 'XML parser error')
        stepname = elem.attr['step']

        # we should have created this step previously; if it hasn't,
        # the master and slave are processing steps out of order.
        step = BuildStep.fetch(self.env, build=build.id, name=stepname)
        if not step:
            self._send_error(req, HTTP_CONFLICT, 'Build step has not been created.')

        recipe = Recipe(xmlio.parse(config.recipe))
        index = None
        current_step = None
        for num, recipe_step in enumerate(recipe):
            if recipe_step.id == stepname:
                index = num
                current_step = recipe_step
        if index is None:
            self._send_error(req, HTTP_FORBIDDEN,
                                'No such build step' % stepname)
        last_step = index == num

        self.log.debug('Slave %s (build %d) completed step %d (%s) with '
                       'status %s', build.slave, build.id, index, stepname,
                       elem.attr['status'])

        db = self.env.get_db_cnx()

        step.stopped = int(time.time())

        if elem.attr['status'] == 'failure':
            self.log.warning('Build %s step %s failed', build.id, stepname)
            step.status = BuildStep.FAILURE
            if current_step.onerror == 'fail':
                last_step = True
        else:
            step.status = BuildStep.SUCCESS
        step.errors += [error.gettext() for error in elem.children('error')]

        # TODO: step.update(db=db)
        step.delete(db=db)
        step.insert(db=db)

        # Collect log messages from the request body
        for idx, log_elem in enumerate(elem.children('log')):
            build_log = BuildLog(self.env, build=build.id, step=stepname,
                                 generator=log_elem.attr.get('generator'),
                                 orderno=idx)
            for message_elem in log_elem.children('message'):
                build_log.messages.append((message_elem.attr['level'],
                                           message_elem.gettext()))
            build_log.insert(db=db)

        # Collect report data from the request body
        for report_elem in elem.children('report'):
            report = Report(self.env, build=build.id, step=stepname,
                            category=report_elem.attr.get('category'),
                            generator=report_elem.attr.get('generator'))
            for item_elem in report_elem.children():
                item = {'type': item_elem.name}
                item.update(item_elem.attr)
                for child_elem in item_elem.children():
                    item[child_elem.name] = child_elem.gettext()
                report.items.append(item)
            report.insert(db=db)

        # If this was the last step in the recipe we mark the build as
        # completed otherwise just update last_activity
        if last_step:
            self.log.info('Slave %s completed build %d ("%s" as of [%s])',
                          build.slave, build.id, build.config, build.rev)
            build.stopped = step.stopped
            build.last_activity = build.stopped

            # Determine overall outcome of the build by checking the outcome
            # of the individual steps against the "onerror" specification of
            # each step in the recipe
            for num, recipe_step in enumerate(recipe):
                step = BuildStep.fetch(self.env, build.id, recipe_step.id)
                if step.status == BuildStep.FAILURE:
                    if recipe_step.onerror == 'fail' or \
                            recipe_step.onerror == 'continue':
                        build.status = Build.FAILURE
                        break
            else:
                build.status = Build.SUCCESS

            build.update(db=db)
        else:
            build.last_activity = step.stopped
            build.update(db=db)

            # start the next step.
            for num, recipe_step in enumerate(recipe):
                if num == index + 1:
                    next_step = recipe_step
            if next_step is None:
                self._send_error(req, HTTP_FORBIDDEN,
                                 'Unable to find step after ' % stepname)

            step = self._start_new_step(build, next_step.id)
            step.insert(db=db)

        db.commit()

        if last_step:
            for listener in BuildSystem(self.env).listeners:
                listener.build_completed(build)

        body = 'Build step processed'
        self._send_response(req, 201, body, {
                            'Content-Type': 'text/plain',
                            'Content-Length': str(len(body)),
                            'Location': req.abs_href.builds(
                                    build.id, 'steps', stepname)})

    def _process_attachment(self, req, config, build):
        resource_id = req.args['member'] == 'config' \
                    and build.config or build.resource.id
        upload = req.args['file']
        if not upload.file:
            send_error(req, message="Attachment not received.")
        self.log.debug('Received attachment %s for attaching to build:%s',
                      upload.filename, resource_id)

        # Determine size of file
        upload.file.seek(0, 2) # to the end
        size = upload.file.tell()
        upload.file.seek(0)    # beginning again

        # Delete attachment if it already exists
        try:
            old_attach = Attachment(self.env, 'build',
                            parent_id=resource_id, filename=upload.filename)
            old_attach.delete()
        except ResourceNotFound:
            pass

        # Save new attachment
        attachment = Attachment(self.env, 'build', parent_id=resource_id)
        attachment.description = req.args.get('description', '')
        attachment.author = req.authname
        attachment.insert(upload.filename, upload.file, size)

        self._send_response(req, 201, 'Attachment created', headers={
                            'Content-Type': 'text/plain',
                            'Content-Length': str(len('Attachment created'))})

    def _process_keepalive(self, req, config, build):
        build.last_activity = int(time.time())
        build.update()

        self.log.info('Slave %s build %d keepalive ("%s" as of [%s])',
                      build.slave, build.id, build.config, build.rev)

        body = 'Keepalive processed'
        self._send_response(req, 200, body, {
                            'Content-Type': 'text/plain',
                            'Content-Length': str(len(body))})

    def _start_new_step(self, build, stepname):
        """Creates the in-memory representation for a newly started
        step, ready to be persisted to the database.
        """
        step = BuildStep(self.env, build=build.id, name=stepname)
        step.status = BuildStep.IN_PROGRESS
        step.started = int(time.time())
        step.stopped = 0

        return step