autopkgtest 2.14.1 / usr / bin / adt-run

#!/usr/bin/python
#
# adt-run is part of autopkgtest
# autopkgtest is a tool for testing Debian binary packages
#
# autopkgtest is Copyright (C) 2006-2007, 2013 Canonical Ltd.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
# See the file CREDITS for a full list of credits information (often
# installed as /usr/share/doc/autopkgtest/CREDITS).

import signal
import optparse
import tempfile
import sys
import subprocess
import traceback
import urllib
import string
import re
import os
import errno
import fnmatch
import shutil
import copy
import time
import atexit
import pipes

try:
    from debian import deb822, debian_support
    deb822, debian_support  # pyflakes
except ImportError:
    from debian_bundle import deb822, debian_support

from optparse import OptionParser

signal.signal(signal.SIGINT, signal.SIG_DFL)  # undo stupid Python SIGINT thing

try:
    our_base = os.environ['AUTOPKGTEST_BASE'] + '/lib'
except KeyError:
    our_base = '/usr/share/autopkgtest/python'
sys.path.insert(1, our_base)

#---------- global variables

tmp = None		# pathstring on host
testbed = None		# Testbed
shared_downtmp = None   # testbed's downtmp on the host, if supported
opts = None             # optparse options
errorcode = 0		# exit status that we are going to use
timeouts = {'short': 100, 'copy': 300, 'install': 3000, 'test': 10000, 'build': 100000}
binaries = None		# Binaries (.debs we have registered)
build_essential = ['build-essential']
dpkg_buildpackage = 'dpkg-buildpackage -us -uc -b'

os.putenv('APT_LISTBUGS_FRONTEND', 'none')
          # do not consider using apt-listbugs
os.putenv('APT_LISTCHANGES_FRONTEND', 'none')
          # do not consider using apt-listchanges


#
# logging/error functions
#

summary_stream = None


def error(message):
    '''Write an error message to stderr'''

    print >> sys.stderr, message


def debug(message, minlevel=0, timestamp=False):
    '''Write a debug message to stderr according to --quiet'''

    if opts.debuglevel < minlevel:
        return

    p = 'adt-run'
    if minlevel:
        p += str(minlevel)
    if timestamp:
        p += ' [%s]' % time.strftime('%Y-%m-%d %H:%M:%S')
    p += ': '
    log_msg(message, p)


def log_msg(message, prefix=''):
    '''Write a string to stderr according to --quiet'''

    if opts.quiet:
        return
    for line in message.rstrip('\n').splitlines():
        s = prefix + line
        if not opts.quiet:
            print >> sys.stderr, s


def log_file(path, minlevel=0):
    '''Write a file to stderr according to --quiet'''

    if opts.debuglevel < minlevel or opts.quiet:
        return
    rc = subprocess.call(['cat', path], stdout=sys.stderr)
    if rc:
        bomb('cat failed for %s, exit code %d' % (path, rc))


def debug_subprocess(what, argv, script=None):
    '''Log a subprocess call for debugging'''

    o = '$ ' + what + ':'
    if argv is not None:
        ol = []
        for x in argv:
            if x is script:
                x = '<SCRIPT>'
            ol.append(x.replace('\\', '\\\\').replace(' ', '\\ '))
        o += ' ' + ' '.join(ol)
    debug(o)
    if script is not None and opts.debuglevel >= 1:
        o = ''
        for l in script.rstrip('\n').split('\n'):
            o += '$     ' + l + '\n'
        debug(o, 1)


def script_out(argv, what=None, script=None, **kwargs):
    '''Call a script and get its return code, and optionally stdout.

    If what/script are given, log these for debugging.

    Return (exitcode, stdout). stdout will be a string if kwargs contains
    stdout=subprocess.PIPE, otherwise None.
    '''
    if what:
        debug_subprocess(what, argv, script=script)

    process = subprocess.Popen(argv, **kwargs)
    output = process.communicate()[0]
    return (process.returncode, output)


def psummary(m):
    if summary_stream is not None:
        summary_stream.write(m.encode('UTF-8'))
        summary_stream.write('\n')


def preport(m):
    sys.stdout.write(m.encode('UTF-8'))
    sys.stdout.write('\n')
    sys.stdout.flush()
    psummary(m)


def report(tname, result):
    preport('%-20s %s' % (tname, result.decode('UTF-8', 'replace')))

#---------- errors we define


class Quit:

    def __init__(self, ec, m):
        self.ec = ec
        self.m = m


def bomb(m):
    raise Quit(20, 'unexpected error: %s' % m)


def badpkg(m):
    preport('blame: ' + ' '.join(testbed.blamed))
    preport('badpkg: ' + m)
    raise Quit(12, 'erroneous package: %s' % m)


class Unsupported(Exception):

    def __init__(self, lno, m):
        if lno >= 0:
            self.m = '%s (control line %d)' % (m, lno)
        else:
            self.m = m

    def report(self, tname):
        global errorcode
        errorcode |= 2
        report(tname, 'SKIP %s' % self.m)

#---------- convenience function


def mkdir_okexist(pathname, mode=02755):
    try:
        os.mkdir(pathname, mode)
    except (IOError, OSError), oe:
        if oe.errno != errno.EEXIST:
            raise


def rmtree(what, pathname):
    debug('/ %s rmtree %s' % (what, pathname), 2)
    try:
        shutil.rmtree(pathname)
    except (IOError, OSError), oe:
        if oe.errno not in (errno.EEXIST, errno.ENOENT):
            raise


def flatten(l):
    return reduce((lambda a, b: a + b), l, [])

#---------- parsing and representation of the arguments


class Action:

    def __init__(self, kind, arg, arghandling, what):
        # extra attributes get added during processing
        self.kind = kind
        self.arg = arg
        self.ah = arghandling
        self.what = what
        self.missing_tests_control = False

    def __repr__(self):
        return '<Action %s %s %s>' % (self.kind, self.what, self.arg)


def parse_args():
    global opts, timeouts
    global n_non_actions  # argh, stupid python scoping rules
    usage = '%prog <options> --- <virt-server>...'
    parser = OptionParser(usage=usage)

    arghandling = {
        'dsc_tests': True,
        'dsc_filter': '*',
        'deb_forbuilds': 'auto',
        'deb_fortests': 'auto',
        'set_lang': 'C'
    }
    initial_arghandling = arghandling.copy()
    n_non_actions = 0

    #----------
    # actions (ie, test sets to run, sources to build, binaries to use):

    def cb_action(op, optstr, value, parser, kindpath, is_act):
        global n_non_actions
        parser.largs.append((value, kindpath))
        n_non_actions += not(is_act)

    def pa_action(longname, metavar, kindpath, help, is_act=True):
        parser.add_option('--' + longname, action='callback', callback=cb_action,
                          nargs=1, type='string', callback_args=(kindpath, is_act),
                          help=help)

    pa_action('built-tree', 'TREE', '@/', help='run tests from build tree TREE')
    pa_action('unbuilt-tree', 'TREE', '@//', help='run tests from build tree TREE')

    pa_action('source', 'DSC', '@.dsc',
              help='build DSC and use its tests and/or generated binary packages')

    pa_action('binary', 'DEB', '@.deb',
              help='use binary package DEB according to most recent --binaries-* settings')

    pa_action('apt-source', 'SRCPKG', '@.apt',
              help='download with apt-get source in testbed and use its tests')

    def cb_actnoarg(op, optstr, value, parser, largsentry):
        parser.largs.append(largsentry)

    parser.add_option('--instantiate', action='callback', callback=cb_actnoarg,
                      callback_args=((None, ('instantiate',)),),
                      help='instantiate testbed now (during testing phase)'
                      ' and install packages'
                      ' selected for automatic installation, even'
                      ' if this might apparently not be required otherwise')

    #----------
    # argument handling settings (what ways to use action
    #  arguments, and pathname processing):

    def cb_setah(option, opt_str, value, parser, toset, setval):
        if type(setval) == list:
            if not value in setval:
                parser.error('value for %s option (%s) is not '
                             'one of the permitted values (%s)' %
                             (value, opt_str, setval.join(' ')))
        elif setval is not None:
            value = setval
        for v in toset:
            arghandling[v] = value
        parser.largs.append(arghandling.copy())

    def pa_setah(short, long, affected, effect, metavar=None, **kwargs):
        type = metavar
        if type is not None:
            type = 'string'
        parser.add_option(long, short, action='callback', callback=cb_setah,
                          callback_args=(affected, effect), **kwargs)

    #---- source processing settings:

    pa_setah(None, '--sources-tests', ['dsc_tests'], True,
             help='run tests from builds of subsequent sources')
    pa_setah(None, '--sources-no-tests', ['dsc_tests'], False,
             help='do not run tests from builds of subsequent sources')

    pa_setah(None, '--built-binaries-filter', ['dsc_filter'], None,
             type='string', metavar='PATTERN-LIST',
             help='from subsequent sources, use binaries matching'
             ' PATTERN-LIST (comma-separated glob patterns)'
             ' according to most recent --binaries-* settings')
    pa_setah('-B', '--no-built-binaries', ['dsc_filter'], '_',
             help='from subsequent sources, do not use any binaries')
    #---- binary package processing settings:

    def pa_setahbins(long, toset, how):
        pa_setah(None, long, toset, ['ignore', 'auto', 'install'],
                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
                 help=how + ' ignore binaries, install them as needed'
                 ' for dependencies, or unconditionally install'
                 ' them, respectively')
    pa_setahbins('--binaries', ['deb_forbuilds', 'deb_fortests'], '')
    pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
    pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')

    #----------
    # general options:

    def cb_vserv(op, optstr, value, parser):
        parser.values.vserver = list(parser.rargs)
        del parser.rargs[:]

    parser.add_option('--leave-lang', dest='set_lang', action='store_false',
                      help="leave LANG on testbed set to testbed's default")
    parser.add_option('--set-lang', metavar='LANGVAL',
                      help='set LANG on testbed to LANGVAL', default='C')

    parser.add_option('-o', '--output-dir',
                      help='write test artifacts (stdout/err, log, debs, etc.) '
                      'to OUTPUT-DIR, emptying it beforehand')
    # backwards compatible alias
    parser.add_option('--tmp-dir', dest='output_dir',
                      help='alias for --output-dir for backwards compatibility')

    parser.add_option('-l', '--log-file', dest='logfile',
                      help='write the log LOGFILE, emptying it beforehand,'
                      ' instead of using OUTPUT-DIR/log')
    parser.add_option('--summary-file', dest='summary',
                      help='write a summary report to SUMMARY, emptying it beforehand')

    for k in timeouts.keys():
        parser.add_option('--timeout-' + k, type='int', dest='timeout_' + k,
                          metavar='T', help='set %s timeout to T' % k)
    parser.add_option('--timeout-factor', type='float', metavar='FACTOR',
                      default=1.0,
                      help='multiply all default timeouts by FACTOR')

    parser.add_option('-u', '--user', help='run tests as USER (needs root on testbed)')
    parser.add_option('--gain-root', type='string', dest='gainroot',
                      help='command to gain root during package build, passed to dpkg-buildpackage -r')
    parser.add_option('-q', '--quiet', action='store_true', dest='quiet', default=False)
    parser.add_option('-d', '--debug', action='count', dest='debuglevel', default=0)
    parser.add_option('-s', '--shell-fail', action='store_true',
                      help='run a shell in the testbed after every failed test')
    parser.add_option('--shell', action='store_true',
                      help='run a shell in the testbed after every test')
    parser.add_option('--gnupg-home', type='string', dest='gnupghome',
                      default='~/.autopkgtest/gpg',
                      help='use GNUPGHOME rather than ~/.autopkgtest (for'
                      ' signing private apt archive);'
                      ' "fresh" means generate new key each time.')
    parser.add_option('--setup-commands', metavar='COMMANDS_OR_PATH',
                      action='append', default=[],
                      help='Run these commands after opening the testbed '
                      '(e. g. "apt-get update" or adding apt sources); can be '
                      'a string with the commands, or a file containing the '
                      'commands')
    parser.add_option('-U', '--apt-upgrade', dest='setup_commands',
                      action='append_const',
                      const='apt-get update && apt-get dist-upgrade -y -o Dpkg::Options::="--force-confnew"',
                      help='Run apt update/dist-upgrade before the tests')
    parser.add_option('--apt-pocket', metavar='POCKETNAME', action='append',
                      default=[],
                      help='Enable additional apt source for POCKETNAME')

    #----------
    # actual meat:

    class SpecialOption(optparse.Option):
        pass
    vs_op = SpecialOption('', '--VSERVER-DUMMY')
    vs_op.action = 'callback'
    vs_op.type = None
    vs_op.default = None
    vs_op.nargs = 0
    vs_op.callback = cb_vserv
    vs_op.callback_args = ()
    vs_op.callback_kwargs = {}
    vs_op.help = 'introduces virtualisation server and args'
    vs_op._short_opts = []
    vs_op._long_opts = ['---']

    parser.add_option(vs_op)

    (opts, args) = parser.parse_args()
    if not hasattr(opts, 'vserver'):
        parser.error('you must specifiy --- <virt-server>...')
    if not opts.vserver[0].startswith('adt-virt-'):
        opts.vserver[0] = 'adt-virt-' + opts.vserver[0]

    if n_non_actions >= len(parser.largs):
        parser.error('nothing to do specified')

    for k in timeouts.keys():
        t = getattr(opts, 'timeout_' + k)
        if t is None:
            t = timeouts[k] * opts.timeout_factor
        timeouts[k] = int(t)

    # this timeout is for adt-virt-*, so pass it down via environment
    os.environ['ADT_VIRT_COPY_TIMEOUT'] = str(timeouts['copy'])

    arghandling = initial_arghandling
    opts.actions = []
    ix = 0
    for act in args:
        if type(act) == dict:
            arghandling = act
            continue
        elif type(act) == tuple:
            pass
        elif type(act) == str:
            act = (act, act)
        else:
            raise Exception("unknown action in list `%s' having"
                            " type `%s'" % (act, type(act)))
        (pathstr, kindpath) = act

        if type(kindpath) is tuple:
            kind = kindpath[0]
        elif kindpath.endswith('.deb'):
            kind = 'deb'
        elif kindpath.endswith('.dsc'):
            kind = 'dsc'
        elif kindpath.endswith('.apt'):
            kind = 'apt'
        elif kindpath.endswith('//'):
            kind = 'ubtree'
        elif kindpath.endswith('/'):
            kind = 'tree'
        elif re.match('[0-9a-z][0-9a-z.+-]+', kindpath):
            kind = 'apt'
        else:
            parser.error("do not know how to handle filename `%s';"
                         " specify --source, --binary, --built-tree, --unbuilt-tree, or --apt-source" %
                         kindpath)

        what = '%s%s' % (kind, ix)
        ix += 1

        # "no built binaries" for apt sources
        if kind == 'apt':
            arghandling['dsc_filter'] = '_'

        opts.actions.append(Action(kind, pathstr, arghandling, what))

        # if we have --setup-commands and it points to a file, read its contents
        for i, c in enumerate(opts.setup_commands):
            if os.path.exists(c):
                with open(c) as f:
                    opts.setup_commands[i] = f.read()


def setup_trace():
    global tmp, summary_stream

    if opts.output_dir is not None:
        rmtree('tmp(specified)', opts.output_dir)
        mkdir_okexist(opts.output_dir, 0755)
        tmp = opts.output_dir
    else:
        assert(tmp is None)
        tmp = tempfile.mkdtemp()
        os.chmod(tmp, 0755)

    if opts.logfile is None and opts.output_dir is not None:
        opts.logfile = opts.output_dir + '/log'

    if opts.logfile is not None:
        # tee stdout/err into log file
        fifo_log = os.path.join(tmp, 'fifo_log')
        os.mkfifo(fifo_log)
        atexit.register(os.unlink, fifo_log)
        out_tee = subprocess.Popen(['tee', fifo_log],
                                   stdin=subprocess.PIPE)
        err_tee = subprocess.Popen(['tee', fifo_log, '-a', '/dev/stderr'],
                                   stdin=subprocess.PIPE,
                                   stdout=open('/dev/null', 'wb'))
        subprocess.Popen(['cat', fifo_log], stdout=open(opts.logfile, 'wb'))
        os.dup2(out_tee.stdin.fileno(), sys.stdout.fileno())
        os.dup2(err_tee.stdin.fileno(), sys.stderr.fileno())
        atexit.register(out_tee.wait)
        atexit.register(out_tee.terminate)
        atexit.register(err_tee.wait)
        atexit.register(err_tee.terminate)

    if opts.summary is not None:
        summary_stream = open(opts.summary, 'w', 0)

    debug('options: ' + str(opts) + '; timeouts: ' + str(timeouts), 1)


def finalise_options():
    global opts, testbed, build_essential, dpkg_buildpackage

    if opts.user is None and 'root-on-testbed' in testbed.caps:
        su = 'suggested-normal-user='
        ul = [
            e[len(su):]
            for e in testbed.caps
            if e.startswith(su)
        ]
        if ul:
            opts.user = ul[0]
        else:
            opts.user = ''

    if opts.user:
        if 'root-on-testbed' not in testbed.caps:
            error('warning: virtualisation system does not offer root on '
                  'testbed but --user option specified: failure likely')
        opts.user_wrap = lambda x: "su -s /bin/sh %s -c '%s'" % (opts.user, x)
    else:
        opts.user_wrap = lambda x: x

    if opts.gainroot is None:
        if (opts.user or
                # FIXME: work around chown root not working in qemu's 9p mount
                'qemu' in opts.vserver[0] or
                'root-on-testbed' not in testbed.caps):
            opts.gainroot = 'fakeroot'
            build_essential += ['fakeroot']
    if opts.gainroot:
        dpkg_buildpackage += ' -r' + opts.gainroot

    if opts.gnupghome.startswith('~/'):
        opts.gnupghome = os.path.expanduser(opts.gnupghome)
    elif opts.gnupghome == 'fresh':
        opts.gnupghome = None

#---------- testbed management - the Testbed class


class Testbed:

    def __init__(self):
        self.sp = None
        self.lastsend = None
        self.scratch = None
        self.modified = False
        self.blamed = []
        self._debug('init', 1)
        self._need_reset_apt = False
        self.stop_sent = False
        self.ec_auxverbscript = None

    def _debug(self, m, minlevel=0):
        debug('** ' + m, minlevel)

    def start(self):
        debug('version 2.14.1', timestamp=True)
        debug_subprocess('vserver', opts.vserver)
        self.sp = subprocess.Popen(opts.vserver,
                                   stdin=subprocess.PIPE,
                                   stdout=subprocess.PIPE)
        self.expect('ok')

    def stop(self):
        self._debug('stop', 1)
        if self.stop_sent:
            # avoid endless loop
            return
        self.stop_sent = True

        self.close()
        if self.sp is None:
            return
        ec = self.sp.returncode
        if ec is None:
            self.sp.stdout.close()
            self.send('quit')
            self.sp.stdin.close()
            ec = self.sp.wait()
        if ec:
            self.bomb('testbed gave exit status %d after quit' % ec)
        self.sp = None

    def open(self):
        self._debug('open, scratch=%s' % self.scratch, 1)
        if self.scratch is not None:
            return
        pl = self.commandr('open')
        self._opened(pl)

    def _opened(self, pl):
        global shared_downtmp

        self.scratch = pl[0]
        self.deps_processed = []
        self.caps = self.commandr('capabilities')
        debug('testbed capabilities: %s' % self.caps, 1)
        for c in self.caps:
            if c.startswith('downtmp-host='):
                shared_downtmp = c.split('=', 1)[1]
        self._auxverbscript_make()
        self.run_setup_commands()

    def _auxverbscript_make(self):
        pec = self.commandr('print-auxverb-command')
        if len(pec) < 1:
            self.bomb('too few results from print-execute-command')
        cmdl = map(urllib.unquote, pec[0].split(','))

        self._debug('cmdl = %s' % str(cmdl), 1)

        # turn cmdl into a shell-qouted string
        cmdl_str = ' '.join(map(pipes.quote, cmdl))

        self.ec_auxverbscript = os.path.join(tmp, 'satdep-auxverb')
        with open(self.ec_auxverbscript, 'w') as f:
            f.write('''#!/bin/sh
set -e
if [ $# = 2 ] && [ "x$1" = xdpkg-architecture ] && [ "x$2" = x-qDEB_HOST_ARCH ]; then
    # This is a pretty nasty hack.  Hopefully it can go away
    #  eventually.  See #635763.
    set -- dpkg --print-architecture
fi
export DEBIAN_FRONTEND=noninteractive
exec ''' + cmdl_str + ' "$@"\n')
        os.chmod(self.ec_auxverbscript, 0755)

    def mungeing_apt(self):
        if not 'revert' in self.caps:
            self._need_reset_apt = True

    def reset_apt(self):
        if not self._need_reset_apt:
            return
        self._need_reset_apt = False
        what = 'aptget-update-reset'
        cmdl = ['sh', '-c', 'apt-get -qy update 2>&1']
        rc = self.execute(what, cmdl, kind='install')
        if rc:
            error('\nwarning: failed to restore testbed apt cache, '
                  'exit code %d' % rc)
        what = 'aptconf-reset'
        cmdl = ['rm', '-f', '/etc/apt/apt.conf.d/90autopkgtest',
                '/etc/apt/sources.list.d/autopkgtest.list',
                '/etc/apt/preferences.d/90autopkgtest']
        rc = self.execute(what, cmdl, kind='install')
        if rc:
            error('\nwarning: failed to reset changes '
                  'made to testbed apt configuration, exit code %d' % rc)

    def close(self):
        global shared_downtmp

        self._debug('close, scratch=%s' % self.scratch, 1)
        if self.ec_auxverbscript:
            os.unlink(self.ec_auxverbscript)
            self.ec_auxverbscript = None
        if self.scratch is None:
            return
        self.scratch = None
        if self.sp is None:
            return
        self.command('close')
        shared_downtmp = None

    def run_setup_commands(self):
        if not opts.setup_commands and not opts.apt_pocket:
            return

        debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ test bed setup')
        for p in opts.apt_pocket:
            script = '''awk '/^deb / { if ($3 !~ /-/) { ''' \
                '''$3 = $3"-%s"; print; $1="deb-src"; print; exit}}' ''' \
                '''/etc/apt/sources.list > /etc/apt/sources.list.d/%s.list''' % (p, p)

            rc = testbed.execute('apt-pocket-' + p, ['sh', '-ec', script],
                                 script=script)

        for c in opts.setup_commands:
            rc = testbed.execute('setup-commands',
                                 ['sh', '-ec', c],
                                 script=c,
                                 so='/dev/stdout',
                                 kind='install')
            if rc:
                bomb('testbed setup commands failed with status %i' % rc)

    def prepare1(self, deps_new):
        self._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
                    (self.modified, self.deps_processed, deps_new), 1)
        if 'revert' in self.caps and (
                self.modified or [d for d in self.deps_processed if d not in deps_new]):
            self._debug('reset **', 1)
            pl = self.commandr('revert')
            self._opened(pl)
        self.modified = False

    def prepare2(self, deps_new):
        self._debug('prepare2, deps_new=%s' % deps_new, 1)
        binaries.publish()
        self._install_deps(deps_new)

    def prepare(self, deps_new):
        self.prepare1(deps_new)
        self.prepare2(deps_new)

    def _install_deps(self, deps_new):
        self._debug(' installing dependencies ' + str(deps_new), 1)
        self.deps_processed = deps_new
        if not deps_new:
            return
        self.satisfy_dependencies_string(', '.join(deps_new), 'install-deps')

    def needs_reset(self):
        # show what caused a reset
        (fname, lineno, function, code) = traceback.extract_stack(limit=2)[0]
        self._debug('needs_reset, previously=%s, requested by %s() line %i' %
                    (self.modified, function, lineno), 1)
        self.modified = True

    def blame(self, m):
        self._debug('blame += %s' % m, 1)
        self.blamed.append(m)

    def bomb(self, m):
        self._debug('bomb %s' % m, 1)
        self.reset_apt()
        self.stop()
        raise Quit(16, 'testbed failed: %s' % m)

    def send(self, string):
        self.sp.stdin
        try:
            debug('>> ' + string, 2)
            print >>self.sp.stdin, string
            self.sp.stdin.flush()
            self.lastsend = string
        except:
            (type, value, dummy) = sys.exc_info()
            self.bomb('cannot send to testbed: %s' % traceback.
                      format_exception_only(type, value))

    def expect(self, keyword, nresults=None):
        l = self.sp.stdout.readline()
        if not l:
            self.bomb('unexpected eof from the testbed')
        if not l.endswith('\n'):
            self.bomb('unterminated line from the testbed')
        l = l.rstrip('\n')
        debug('<< ' + l, 2)
        ll = l.split()
        if not ll:
            self.bomb('unexpected whitespace-only line from the testbed')
        if ll[0] != keyword:
            if self.lastsend is None:
                self.bomb("got banner `%s', expected `%s...'" %
                          (l, keyword))
            else:
                self.bomb("sent `%s', got `%s', expected `%s...'" %
                          (self.lastsend, l, keyword))
        ll = ll[1:]
        if nresults is not None and len(ll) != nresults:
            self.bomb("sent `%s', got `%s' (%d result parameters),"
                      " expected %d result parameters" %
                      (self.lastsend, l, len(ll), nresults))
        return ll

    def commandr(self, cmd, args=(), nresults=None, unquote=True):
        # pass args=[None,...] or =(None,...) to avoid more url quoting
        if type(cmd) is str:
            cmd = [cmd]
        if len(args) and args[0] is None:
            args = args[1:]
        else:
            args = map(urllib.quote, args)
        al = cmd + args
        self.send(string.join(al))
        ll = self.expect('ok', nresults)
        if unquote:
            ll = map(urllib.unquote, ll)
        return ll

    def command(self, cmd, args=()):
        self.commandr(cmd, args, 0)

    def commandr1(self, cmd, args=()):
        rl = self.commandr(cmd, args, 1)
        return rl[0]

    def execute(self, what, cmdl,
                si='/dev/null', so='/dev/null', se=None, cwd=None,
                script=False, xenv=[], kind='short'):
        # Options for script:
        #   False - do not call debug_subprocess, no synch. reporting required
        #   None or string - call debug_subprocess with that value,
        #			plumb stderr through synchronously if possible
        # Options for se:
        #   None - usual logging (output is of kind 2c)
        #   string - path on testbed (output is of kind 2d)

        timeout = timeouts[kind]

        if script is not False:
            debug_subprocess(what, cmdl, script=script)
        if cwd is None:
            cwd = self.scratch

        xdump = None
        if se is None:
            se_catch = TempTestbedPath(testbed, what + '-xerr')
            se_use = se_catch.tb
            if not opts.quiet:
                xdump = 'debug=2-2'
        else:
            se_catch = None
            se_use = se

        cmdl = [None,
                ','.join(map(urllib.quote, cmdl)),
                si, so, se_use, cwd]

        if timeout is not None and timeout > 0:
            cmdl.append('timeout=%d' % timeout)

        if xdump is not None and 'execute-debug' in self.caps:
            cmdl += [xdump]
        for e in xenv:
            cmdl.append('env=%s' % e)
        if kind == 'install':
            cmdl.append('env=DEBIAN_FRONTEND=noninteractive')
        if opts.set_lang is not False:
            cmdl.append('env=LANG=%s' % opts.set_lang)

        rc = self.commandr1('execute', cmdl)
        try:
            rc = int(rc)
        except ValueError:
            bomb("execute for %s gave invalid response `%s'"
                 % (what, rc))

        if se_catch is not None:
            se_catch.copyup()
            log_file(se_catch.host)

        return rc

    def satisfy_dependencies_string(self, deps, what):
        # Must have called Binaries.configure_apt
        debug('dependencies: %s: satisfying %s' % (what, deps), 1)
        dsc = os.path.join(tmp, 'deps.dsc')
        with open(dsc, 'w') as f:
            f.write('Build-Depends: %s\n\n' % deps)
        # pbuilder-satisfydepends has a bug where it ignores the
        # Build-Depends if it's the last line in the dsc (#635696)
        self.satisfy_dependencies_dsc(dsc, what)
        os.unlink(dsc)

    def satisfy_dependencies_dsc(self, dsc, what):
        if 'root-on-testbed' not in self.caps:
            error('WARNING: virtualisation system does not offer root on '
                  'testbed, cannot install any packages; failure likely')
            return

        # Must have called Binaries.configure_apt
        cmdl = ['/usr/lib/pbuilder/pbuilder-satisfydepends-classic',
                '--binary-all',  # --check-key
                '--internal-chrootexec', self.ec_auxverbscript,
                '-c', dsc
                ]
        # The --internal-chrootexec option is really handy but
        # perhaps we are not supposed to use it ?  See also #635697.
        debug('dependencies: %s: running %s' % (what, str(cmdl)))
        rc = subprocess.call(cmdl, stdout=None, stderr=None)
        if rc:
            badpkg('dependency install failed, exit code %d' % rc)


class TestbedPath:
    '''Represent a file/dir with a host and a testbed path'''

    def __init__(self, testbed, host, tb, is_dir=None):
        '''Create a TestbedPath object.

        The object itself is just a pair of file names, nothing more. They do
        not need to exist until you call copyup() or copydown() on them.

        testbed: the Testbed object which this refers to
        host: path of the file on the host
        tb: path of the file in testbed
        is_dir: whether path is a directory; None for "unspecified" if you only
                need copydown()
        '''
        self.testbed = testbed
        self.host = host
        self.tb = tb
        self.is_dir = is_dir

    def copydown(self):
        '''Copy file from the host to the testbed'''

        # create directory on testbed
        rc = testbed.execute('mkdir',
                             ['sh', '-ec', 'test -d "$1" || mkdir -p "$1"',
                              'x', os.path.dirname(self.tb)])
        if rc:
            bomb('failed to create directory %s' % os.path.dirname(self.tb))

        if os.path.isdir(self.host):
            # directories need explicit '/' appended for VirtSubproc
            testbed.command('copydown', (self.host + '/', self.tb + '/'))
        else:
            testbed.command('copydown', (self.host, self.tb))

        # we usually want our files be readable for the non-root user
        if opts.user:
            rc = testbed.execute('copydown-chown-' + os.path.basename(self.host),
                                 ['chown', '-R', opts.user, '--', self.tb])
            if rc:
                bomb('failed to chown %s' % self.tb)

    def copyup(self):
        '''Copy file from the testbed to the host'''

        mkdir_okexist(os.path.dirname(self.host))
        assert self.is_dir is not None
        if self.is_dir:
            testbed.command('copyup', (self.tb + '/', self.host + '/'))
        else:
            testbed.command('copyup', (self.tb, self.host))


class TempTestbedPath(TestbedPath):
    '''Represent a file in the hosts'/testbed's temporary directories

    These are only guaranteed to exit within one testbed run.
    '''
    def __init__(self, testbed, name, is_dir=False, autoclean=True):
        '''Create a temporary TestbedPath object.

        The object itself is just a pair of file names, nothing more. They do
        not need to exist until you call copyup() or copydown() on them.

        testbed: the Testbed object which this refers to
        name: name of the temporary file (without path); host and tb
              will then be derived from that
        is_dir: whether path is a directory; None for "unspecified" if you only
                need copydown()
        autoclean: If True (default), remove file when adt-run finishes. Should
                be set to False for files which you want to keep in the
                --output-dir which are useful for reporting results, like test
                stdout/err, log files, and binaries.
        '''
        # if the testbed supports a shared downtmp, use that to avoid
        # unnecessary copying, unless we want to permanently keep the file
        if shared_downtmp and (not opts.output_dir or autoclean):
            host = shared_downtmp
        else:
            host = tmp
        TestbedPath.__init__(self, testbed, os.path.join(host, name),
                             os.path.join(testbed.scratch, name),
                             is_dir)
        self.autoclean = autoclean

    def __del__(self):
        if self.autoclean:
            if os.path.exists(self.host):
                os.unlink(self.host)

#---------- representation of test control files: Field*, Test, etc.


class FieldBase:

    def __init__(self, fname, stz, base, tnames, vl):
        assert(vl)
        self.stz = stz
        self.fname = fname
        self.base = base
        self.tnames = tnames
        self.vl = vl

    def words(self):
        def distribute(vle):
            (lno, v) = vle
            r = v.split()
            r = map((lambda w: (lno, w)), r)
            return r
        return flatten(map(distribute, self.vl))

    def atmostone(self):
        if len(self.vl) == 1:
            (self.lno, self.v) = self.vl[0]
        else:
            raise Unsupported(self.vl[1][0],
                              'only one %s field allowed' % self.fname)
        return self.v


class FieldIgnore(FieldBase):

    def parse(self):
        pass


class Restriction:

    def __init__(self, rname, base):
        pass


class Restriction_rw_build_tree(Restriction):
    pass


class Restriction_build_needed(Restriction):
    pass


class Restriction_allow_stderr(Restriction):
    pass


class Restriction_isolation_container(Restriction):

    def __init__(self, rname, base):
        if 'isolation-container' not in testbed.caps and 'isolation-machine' not in testbed.caps:
            raise Unsupported(-1, 'Test requires container-level isolation but testbed does not provide that')


class Restriction_isolation_machine(Restriction):

    def __init__(self, rname, base):
        if 'isolation-machine' not in testbed.caps:
            raise Unsupported(-1, 'Test requires machine-level isolation but testbed does not provide that')


class Restriction_breaks_testbed(Restriction):

    def __init__(self, rname, base):
        if 'revert-full-system' not in testbed.caps:
            raise Unsupported(-1, 'Test breaks testbed but testbed does not '
                              'advertise revert-full-system')


class Restriction_needs_root(Restriction):

    def __init__(self, rname, base):
        if 'root-on-testbed' not in testbed.caps:
            raise Unsupported(-1,
                              'Test needs root on testbed which is not available')


class Field_Restrictions(FieldBase):

    def parse(self):
        for wle in self.words():
            (lno, rname) = wle
            nrname = rname.replace('-', '_')
            try:
                rclass = globals()['Restriction_' + nrname]
            except KeyError:
                raise Unsupported(lno,
                                  'unknown restriction %s' % rname)
            r = rclass(nrname, self.base)
            self.base['restriction_names'].append(rname)
            self.base['restrictions'].append(r)


class Field_Features(FieldIgnore):

    def parse(self):
        for wle in self.words():
            (lno, fname) = wle
            self.base['feature_names'].append(fname)
            nfname = fname.replace('-', '_')
            try:
                fclass = globals()['Feature_' + nfname]
            except KeyError:
                continue
            ft = fclass(nfname, self.base)
            self.base['features'].append(ft)


class Field_Tests(FieldIgnore):
    pass


class Field_Depends(FieldBase):

    def parse(self):
        debug('Field_Depends: %s %s %s %s' %
              (self.stz, self.base, self.tnames, self.vl), 2)
        dl = map(lambda x: x.strip(),
                 flatten(map(lambda (lno, v): v.split(','), self.vl)))
        # Remove empty dependencies
        dl = filter(None, dl)

        dep_re = re.compile(
            r'(?P<package>[a-z0-9+-.]+)\s*(\((?P<relation><<|<=|>=|=|>>)\s*(?P<version>[^\)]*)\))?(\s*\[[[a-z0-9+-. ]+\])?$')
        for di in dl:
            for d in di.split('|'):
                d = d.strip()
                if d == '@':
                    continue  # Expanded to binary packages
                if d == '@builddeps@':
                    continue  # Expanded to build dependencies
                m = dep_re.match(d)
                if not m:
                    badpkg("Test Depends field contains an invalid "
                           "dependency `%s'" % d)
                if m.group("version"):
                    try:
                        debian_support.NativeVersion(m.group('version'))
                    except ValueError:
                        badpkg("Test Depends field contains dependency"
                               " `%s' with an invalid version" % d)
                    except AttributeError:
                        # too old python-debian, skip the check
                        pass
        self.base['depends'] = dl


class Field_Tests_directory(FieldBase):

    def parse(self):
        td = self.atmostone()
        if td.startswith('/'):
            raise Unsupported(self.lno,
                              'Tests-Directory may not be absolute')
        self.base['testsdir'] = td


def run_tests(stanzas, tree):
    global errorcode, testbed

    # record tested package version
    with open(os.path.join(tree.host, 'debian', 'changelog')) as f:
        m = re.match('^([^ ]+) \(([^ ]+)\) ', f.readline())
        if m:
            debug('testing package %s version %s' % (m.group(1), m.group(2)))
            if opts.output_dir:
                with open(os.path.join(tmp, 'testpkg-version'), 'w') as f:
                    f.write(m.group(1))
                    f.write(' ')
                    f.write(m.group(2))
                    f.write('\n')
        else:
            error('debian/changelog is invalid, cannot parse version')

    # record package versions of pristine testbed
    if opts.output_dir:
        pkglist = TempTestbedPath(testbed, 'testbed-packages', autoclean=False)
        rc = testbed.execute(
            'record-pkgversions-testbed',
            ['sh', '-ec', "dpkg-query --show -f '${Package}\\t${Version}\\n' > %s" % pkglist.tb])
        if rc:
            bomb('failed to get package list')
        pkglist.copyup()

    if stanzas == ():
        report('*', 'SKIP no tests in this package')
        errorcode |= 8
    for stanza in stanzas:
        tests = stanza[' tests']
        if not tests:
            # if we have skipped tests, don't claim that we don't have any
            if not errorcode & 2:
                report('*', 'SKIP package has metadata but no tests')
                errorcode |= 8
        for t in tests:
            t.prepare(tree)
            t.run(tree)
            if 'breaks-testbed' in t.restriction_names:
                testbed.needs_reset()
        testbed.needs_reset()


class Test:

    def __init__(self, tname, base, act):
        if '/' in tname:
            raise Unsupported(base[' lno'],
                              'test name may not contain / character')
        for k in base:
            setattr(self, k, base[k])
        self.tname = tname
        self.act = act
        self.what = act.what + 't-' + tname
        if len(base['testsdir']):
            self.path = base['testsdir'] + '/' + tname
        else:
            self.path = tname
        debug('constructed; path=%s' % self.path, 1)
        debug(' .depends=%s' % self.depends, 1)

    def _debug(self, m, timestamp=False):
        debug('& %s: %s' % (self.what, m), timestamp=timestamp)

    def report(self, m):
        report(self.what, m)

    def reportfail(self, m):
        global errorcode
        errorcode |= 4
        report(self.what, 'FAIL ' + m)

    def prepare(self, tree):
        self._debug('preparing', True)
        dn = []
        for d in self.depends:
            self._debug(' processing dependency ' + d)
            if d == '@':
                # expand to all binaries from that source
                for pkg in packages_from_source(self.act, tree):
                    dp = d.replace('@', pkg)
                    self._debug('  synthesised dependency ' + dp)
                    dn.append(dp)
            elif d == '@builddeps@':
                for dp in build_deps_from_source(self.act, tree):
                    self._debug('  synthesised dependency ' + dp)
                    dn.append(dp)
            else:
                self._debug('  literal dependency ' + d)
                dn.append(d)
        testbed.prepare(dn)

    def run(self, tree):
        # record installed package versions
        if opts.output_dir:
            pkglist = TempTestbedPath(testbed, self.what + '-packages.all', autoclean=False)
            rc = testbed.execute(
                'record-pkgversions-' + self.what,
                ['sh', '-ec', "dpkg-query --show -f '${Package}\\t${Version}\\n' > %s" % pkglist.tb])
            if rc:
                bomb('failed to get package list')
            pkglist.copyup()

            # filter out packages from the base system
            with open(pkglist.host[:-4], 'w') as out:
                rc = script_out(['join', '-v2', '-t\t',
                                 os.path.join(opts.output_dir, 'testbed-packages'),
                                 pkglist.host], stdout=out, env={})[0]
            if rc:
                badpkg('failed to call join for test specific package list, code %d' % rc)
            os.unlink(pkglist.host)

        # ensure our tests are in the testbed
        tree.copydown()

        # stdout/err files in testbed
        so = TempTestbedPath(testbed, self.what + '-stdout', autoclean=False)
        se = TempTestbedPath(testbed, self.what + '-stderr', autoclean=False)

        tb_test_path = os.path.join(tree.tb, self.path)
        xenv = []

        rc = testbed.execute('testchmod-' + self.what, ['chmod', '+x', '--', tb_test_path])
        if rc:
            bomb('failed to chmod +x %s' % tb_test_path)

        testtmp = '%s/%s-testtmp' % (testbed.scratch, self.what)

        script = 'buildtree="$1"; shift\n'
        script += 'rm -rf -- "$@"; mkdir -m 755 -- "$@"\n'

        if 'needs-root' not in self.restriction_names and opts.user is not None:
            if 'root-on-testbed' not in testbed.caps:
                bomb('cannot change to user %s without root-on-testbed' % opts.user)
            test_argv = ['su', '-s', '/bin/sh', opts.user, '-c', tb_test_path]
            if opts.user:
                script += 'chown %s "$@"\n' % opts.user
                if 'rw-build-tree' in self.restriction_names:
                    script += ('chown -R %s "$buildtree"\n'
                               % opts.user)
        else:
            test_argv = [tb_test_path]

        test_adttmp = testtmp + '/adttmp'
        xenv.append('ADTTMP=%s' % test_adttmp)

        test_artifacts = testtmp + '/artifacts'
        script += 'mkdir -m 1777 "%s"\n' % test_artifacts
        xenv.append('ADT_ARTIFACTS=%s' % test_artifacts)

        rc = testbed.execute('mktmpdir-' + self.what,
                             ['sh', '-ec', script, 'x', tree.tb,
                              testtmp, test_adttmp,
                              ])
        if rc:
            bomb("could not create test tmp dirs in `%s', exit code %d"
                 % (testtmp, rc))

        # If we have a shared downtmp and can create a FIFO in it,
        # route stdout/stderr through a common FIFO to watch them in realtime.
        # Note that we can't use the coreutils tee with pipes, as it aborts on
        # the first read error.
        show_realtime = False
        if shared_downtmp and 'downtmp-shared-fifo' in testbed.caps:
            fifo_out = TempTestbedPath(testbed, 'test_stdout')
            fifo_err = TempTestbedPath(testbed, 'test_stderr')
            os.mkfifo(fifo_out.host)
            os.mkfifo(fifo_err.host)
            show_realtime = True
            debug('shared downtmp with FIFO supported, showing test output in realtime', 1)
        else:
            debug('shared_downtmp with FIFO not supported', 1)

        if show_realtime:
            self._debug('[-----------------------', True)
            debug('teeing to stdout: %s, stderr: %s' % (fifo_out.host, fifo_err.host), 2)
            tee_out = os.fork()
            if tee_out == 0:
                fd_in = os.open(fifo_out.host, os.O_RDONLY)
                fd_out = os.open(so.host, os.O_CREAT | os.O_WRONLY)

                while True:
                    block = os.read(fd_in, 1024)
                    if not block:
                        break
                    os.write(fd_out, block)
                    os.write(1, block)
                os._exit(0)

            tee_err = os.fork()
            if tee_err == 0:
                fd_in = os.open(fifo_err.host, os.O_RDONLY)
                fd_out = os.open(se.host, os.O_CREAT | os.O_WRONLY)

                while True:
                    block = os.read(fd_in, 1024)
                    if not block:
                        break
                    os.write(fd_out, block)
                    os.write(2, block)
                os._exit(0)

            try:
                rc = testbed.execute('test-' + self.what, test_argv,
                                     so=fifo_out.tb, se=fifo_err.tb,
                                     cwd=tree.tb,
                                     xenv=xenv, kind='test')
                debug('testbed executing test finished with exit status %i' % rc, 1)
            finally:
                # give tee childs another second to mop up last output
                time.sleep(1)
                os.kill(tee_out, signal.SIGTERM)
                os.kill(tee_err, signal.SIGTERM)
                os.waitpid(tee_out, 0)
                os.waitpid(tee_err, 0)
            self._debug('-----------------------]', True)

        # without a shared downtmp we cannot show realtime output and need
        # to show stdout afterwards
        else:
            self._debug('running...', True)
            rc = testbed.execute('test-' + self.what, test_argv,
                                 so=so.tb, se=se.tb, cwd=tree.tb,
                                 xenv=xenv, kind='test')
            self._debug('finished', True)
            debug('testbed executing test finished with exit status %i' % rc, 1)

            # copy stdout/err files to host
            so.copyup()
            se.copyup()

        se_size = os.path.getsize(se.host)

        # avoid mixing up stdout (from report) and stderr (from _debug) in output
        sys.stdout.flush()
        sys.stderr.flush()
        time.sleep(0.1)

        self._debug(' - - - - - - - - - - results - - - - - - - - - -')

        passed = False
        if rc != 0:
            self.reportfail('non-zero exit status %d' % rc)
        elif se_size != 0 and 'allow-stderr' not in self.restriction_names:
            with open(se.host) as f:
                stderr_top = f.readline().rstrip('\n \t\r')
            self.reportfail('status: %d, stderr: %s' % (rc, stderr_top))
        else:
            self.report('PASS')
            passed = True

        # avoid mixing up stdout (from report) and stderr (from _debug) in output
        sys.stdout.flush()
        sys.stderr.flush()
        time.sleep(0.1)

        if os.path.getsize(so.host) != 0:
            if not show_realtime:
                self._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
                log_file(so.host)
        else:
            # don't produce empty -stdout files in --output-dir
            os.unlink(so.host)

        if se_size != 0:
            if not show_realtime or 'allow-stderr' not in self.restriction_names:
                self._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
                log_file(se.host)
        else:
            # don't produce empty -stderr files in --output-dir
            os.unlink(se.host)

        # copy artifacts to host, if we have --output-dir
        if opts.output_dir:
            ap = TestbedPath(testbed,
                             os.path.join(opts.output_dir, 'artifacts'),
                             test_artifacts, is_dir=True)
            ap.copyup()
            # don't keep an empty artifacts dir around
            if not os.listdir(ap.host):
                os.rmdir(ap.host)

        if opts.shell or (opts.shell_fail and not passed):
            self._debug(' - - - - - - - - - - running shell - - - - - - - - - -')
            testbed.command('shell', (os.path.realpath('/dev/stdin'),
                                      os.path.realpath('/dev/stdout'),
                                      os.path.realpath('/dev/stderr'),
                                      tree.tb))


def read_stanzas(path):
    stanzas = []

    try:
        control = open(path, 'r')
    except (IOError, OSError), oe:
        if oe[0] != errno.ENOENT:
            raise
        return []

    # filter out comments, python-debian doesn't do that
    # (http://bugs.debian.org/743174)
    lines = []
    for l in control:
        # completely ignore ^# as that breaks continuation lines
        if l.startswith('#'):
            continue
        # filter out comments which don't start on first column
        l = l.split('#', 1)[0]
        lines.append(l)
    control.close()

    lno = 0
    stz = {}  # stz[field_name][index] = (lno, value)
                # special field names:
                # stz[' lno'] = number
                # stz[' tests'] = list of Test objects
                # empty dictionary means we're between stanzas
    for paragraph in deb822.Deb822.iter_paragraphs(lines):
        lno += 1
        stz = {' lno': lno, ' tests': []}
        for field, value in paragraph.iteritems():
            v = ''.join(value.split('\n')).replace('  ', ' ')
            field = string.capwords(field)
            stz[field] = [(lno, v)]
            lno += 1 + value.count('\n')  # Count multilines fields
        stanzas.append(stz.copy())

    return stanzas


def read_control(act, control_path):

    if act.missing_tests_control:
        return ()
    stanzas = read_stanzas(control_path)

    for stz in stanzas:
        try:
            try:
                tnames = stz['Tests']
            except KeyError:
                tnames = ['*']
                raise Unsupported(stz[' lno'],
                                  'no Tests field')
            tnames = map((lambda lt: lt[1]), tnames)
            tnames = string.join(tnames).split()
            base = {
                'restriction_names': [],
                'restrictions': [],
                'feature_names': [],
                'features': [],
                'testsdir': 'debian/tests',
                'depends': '@',
                ' lno': stz[' lno'],
            }
            for fname in stz.keys():
                if fname.startswith(' '):
                    continue
                vl = stz[fname]
                try:
                    fclass = globals()['Field_' +
                                       fname.replace('-', '_')]
                except KeyError:
                    raise Unsupported(vl[0][0],
                                      'unknown metadata field %s' % fname)
                f = fclass(stz, fname, base, tnames, vl)
                f.parse()
            for tname in tnames:
                try:
                    t = Test(tname, base, act)
                    stz[' tests'].append(t)
                except Unsupported, u:
                    u.report(tname)
                    continue
        except Unsupported, u:
            for tname in tnames:
                u.report(tname)
            continue

    return stanzas


def print_exception(ei, msgprefix=''):
    if msgprefix:
        error(msgprefix)
    (et, q, tb) = ei
    if et is Quit:
        error('adt-run: ' + q.m)
        psummary('quitting: ' + q.m)
        return q.ec
    else:
        error('adt-run: unexpected, exceptional, error:')
        psummary('quitting: unexpected error, consult transcript')
        traceback.print_exc(None, sys.stderr)
        return 20


def cleanup():
    try:
        if testbed is not None:
            testbed.reset_apt()
            testbed.stop()
        if opts.output_dir is None and tmp is not None:
            rmtree('tmp', tmp)
    except:
        print_exception(sys.exc_info(),
                        '\nadt-run: error cleaning up:\n')
        sys.exit(20)

#---------- registration, installation etc. of .deb's: Binaries


def determine_package(act):
    cmd = 'dpkg-deb --info --'.split(' ') + [act.arg, 'control']
    (rc, output) = script_out(cmd, stdout=subprocess.PIPE)
    if rc:
        badpkg('failed to parse binary package, code %d' % rc)
    pkg_re = re.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
    act.pkg = None
    for l in output.split('\n'):
        m = pkg_re.match(l)
        if not m:
            continue
        if act.pkg:
            badpkg('two Package: lines in control file')
        act.pkg = m.groups()[0]
    if not act.pkg:
        badpkg('no good Package: line in control file')


def packages_from_source(act, tree):
    (rc, output) = script_out(['dh_listpackages'], stdout=subprocess.PIPE, cwd=tree.host)
    if rc:
        badpkg('failed to parse packages built from source, code %d' % rc)

    # filter out empty lines
    packages = [p for p in output.split() if p]

    # filter out udebs
    for st in read_stanzas(os.path.join(tree.host, 'debian/control')):
        if 'Package' not in st:
                    # source stanza
            continue
        if 'Xc-package-type' in st:
            try:
                packages.remove(st['Package'][0][1])
            except ValueError:
                pass

    return packages


def build_deps_from_source(act, tree):
    deps = []
    for st in read_stanzas(os.path.join(tree.host, 'debian/control')):
        if 'Build-depends' in st:
            for d in st['Build-depends'][-1][1].split(','):
                dp = d.strip()
                if dp:
                    deps.append(dp)
        if 'Build-depends-indep' in st:
            for d in st['Build-depends-indep'][-1][1].split(','):
                dp = d.strip()
                if dp:
                    deps.append(dp)
    # we also almost always need make installed to run "make installcheck" or
    # similar, so provide it
    deps.append('make')
    return deps


class Binaries:

    def __init__(self, tb):
        # the binary dir must exist across tb reopenings, so don't use a
        # TempTestbedPath
        self.dir = TestbedPath(tb,
                               os.path.join(tmp, 'binaries'),
                               os.path.join(tb.scratch, 'binaries'),
                               is_dir=True)
        os.mkdir(self.dir.host)
        # clean up an empty binaries output dir
        atexit.register(lambda: os.path.exists(self.dir.host) and (
            os.listdir(self.dir.host) or os.rmdir(self.dir.host)))
        ok = False

        if opts.gnupghome is None:
            opts.gnupghome = tmp + '/gnupg'

        self._debug('initialising')
        try:
            for x in ['pubring', 'secring']:
                os.stat(opts.gnupghome + '/' + x + '.gpg')
            ok = True
        except (IOError, OSError), oe:
            if oe.errno != errno.ENOENT:
                raise

        if ok:
            self._debug('no key generation needed')
        else:
            self.genkey()

        self.apt_get_cmd = 'apt-get -q -o Debug::pkgProblemResolver=true ' \
            '-o APT::Get::force-yes=true -o APT::Get::Assume-Yes=true'

    def _debug(self, s):
        debug('* ' + s, 1)

    def genkey(self):
        self._debug('preparing for key generation')

        mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
        mkdir_okexist(opts.gnupghome, 0700)

        script = '''
  exec >&2
  cd "$1"
  cat <<"END" >key-gen-params
Key-Type: DSA
Key-Length: 1024
Key-Usage: sign
Name-Real: autopkgtest per-run key
Name-Comment: do not trust this key
Name-Email: autopkgtest@example.com
END
  set -x
  gpg --homedir="$1" --batch --gen-key key-gen-params
'''
        cmdl = ['sh', '-ec', script, 'x', opts.gnupghome]
        rc = script_out(cmdl, what='genkey', script=script)[0]
        if rc:
            bomb('key generation failed, code %d' % rc)

    def _configure_apt(self, tb):
        prefs = TestbedPath(testbed, os.path.join(tmp, 'apt-prefs'),
                            '/etc/apt/preferences.d/90autopkgtest')
        with open(prefs.host, 'w') as f:
            f.write('''Package: *
Pin: origin ""
Pin-Priority: 1002
''')
        prefs.copydown()
        os.unlink(prefs.host)

    def reset(self):
        self._debug('reset')
        rmtree('binaries', self.dir.host)
        os.mkdir(self.dir.host)
        self.install = []
        self.blamed = []
        self.registered = set()

    def register(self, act, pkg, path, forwhat, blamed):
        self._debug('register what=%s deb_%s=%s pkg=%s path=%s' %
                    (act.what, forwhat, act.ah['deb_' + forwhat], pkg, path))

        if act.ah['deb_' + forwhat] == 'ignore':
            return

        self.blamed += testbed.blamed

        dest = os.path.join(self.dir.host, pkg + '.deb')

        try:
            os.remove(dest)
        except (IOError, OSError), oe:
            if oe.errno != errno.ENOENT:
                raise oe

        try:
            os.link(path, dest)
        except (IOError, OSError), oe:
            if oe.errno != errno.EXDEV:
                raise oe
            shutil.copy(path, dest)
        # clean up locally built debs (what=ubtreeN) to keep a clean
        # --output-dir, but don't clean up --binary arguments
        if opts.output_dir and path.startswith(opts.output_dir):
            atexit.register(lambda f: os.path.exists(f) and os.unlink(f), path)

        if act.ah['deb_' + forwhat] == 'install':
            self.install.append(pkg)

        self.registered.add(pkg)

    def publish(self):
        self._debug('publish')
        if not self.registered:
            self._debug('no registered binaries, not publishing anything')
            return

        self._configure_apt(testbed)

        script = '''
  exec >&2
  cd "$1"
  apt-ftparchive packages . >Packages
  gzip <Packages >Packages.gz
  apt-ftparchive release . >Release
  rm -f Release.gpg
  gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
  gpg --homedir="$2" --batch --export >archive-key.pgp
'''
        cmdl = ['sh', '-ec', script, 'x', self.dir.host, opts.gnupghome]
        rc = script_out(cmdl, what='ftparchive', script=script)[0]
        if rc:
            bomb('apt-ftparchive or signature failed, code %d' % rc)

        # copy binaries directory to testbed; self.dir.tb might have changed
        # since last time due to a reset, so update it
        self.dir.tb = os.path.join(testbed.scratch, 'binaries')
        testbed.execute('clean-binaries', ['rm', '-rf', self.dir.tb])
        self.dir.copydown()

        aptkey_out = TempTestbedPath(testbed, 'apt-key.out')
        script = '''
  exec 3>&1 >&2
  apt-key add archive-key.pgp
  echo "deb file://''' + self.dir.tb + ''' /" >/etc/apt/sources.list.d/autopkgtest.list
  if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
    echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
  fi
''' + self.apt_get_cmd + ''' update >&2
  cat /var/lib/dpkg/status >&3
'''
        testbed.mungeing_apt()
        rc = testbed.execute('apt-key', ['sh', '-ec', script],
                             so=aptkey_out.tb, cwd=self.dir.tb,
                             script=script, kind='install')
        if rc:
            bomb('apt setup failed with exit code %d' % rc)

        testbed.blamed += self.blamed

        aptkey_out.copyup()

        self._debug('publish reinstall checking...')
        pkgs_reinstall = set()
        pkg = None
        for l in open(aptkey_out.host):
            if l.startswith('Package: '):
                pkg = l[9:].rstrip()
            elif l.startswith('Status: install '):
                if pkg in self.registered:
                    pkgs_reinstall.add(pkg)
                    self._debug(' publish reinstall needs ' + pkg)

        if pkgs_reinstall:
            for pkg in pkgs_reinstall:
                testbed.blame(pkg)
            what = 'apt-get-reinstall'
            cmdl = (self.apt_get_cmd + ' --reinstall install ' +
                    ' '.join([pkg for pkg in pkgs_reinstall]) + ' >&2')
            cmdl = ['sh', '-c', cmdl]
            rc = testbed.execute(what, cmdl, script=None, kind='install')
            if rc:
                badpkg('installation of basic binaries failed,'
                       ' exit code %d' % rc)

        self._debug('publish install...')
        for pkg in self.install:
            what = 'apt-get-install-%s' % pkg
            testbed.blame(pkg)
            cmdl = self.apt_get_cmd + ' install ' + pkg + ' >&2'
            cmdl = ['sh', '-c', cmdl]
            rc = testbed.execute(what, cmdl, script=None, kind='install')
            if rc:
                badpkg('installation of %s failed, exit code %d'
                       % (pkg, rc))

        self._debug('publish done')

#---------- processing of sources (building)


def source_rules_command(script, what, which, cwd,
                         results_lines=0, xargs=[]):
    if opts.debuglevel >= 1:
        script = ['exec 3>&1 >&2', 'set -x'] + script
    else:
        script = ['exec 3>&1 >&2'] + script
    script = '\n'.join(script)
    so = TempTestbedPath(testbed, '%s-%s-results' % (what, which))
    rc = testbed.execute('%s-%s' % (what, which),
                         ['sh', '-ec', script] + xargs, script=script,
                         so=so.tb, cwd=cwd, kind='build')
    so.copyup()
    with open(so.host) as f:
        results = f.read().rstrip('\n')
    if len(results):
        results = results.split('\n')
    else:
        results = []
    if rc:
        badpkg('rules %s failed with exit code %d' % (which, rc))
    if results_lines is not None and len(results) != results_lines:
        badpkg('got %d lines of results from %s where %d expected'
               % (len(results), which, results_lines))
    if results_lines == 1:
        return results[0]
    return results


def build_source(act):
    act.blame = 'arg:' + act.arg
    testbed.blame(act.blame)
    testbed.prepare1([])

    what = act.what
    act.binaries = []

    def debug_b(m):
        debug('* <%s:%s> %s' % (act.kind, act.what, m))

    if act.kind == 'dsc':
        dsc = act.arg
        dsc_tb = os.path.join(testbed.scratch, what, os.path.basename(dsc))

        # copy .dsc file itself
        TestbedPath(testbed, dsc, dsc_tb).copydown()

        # parse source file parts from the .dsc and copy them to testbed
        in_files = False
        fre = re.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
        with open(dsc) as f:
            for l in f:
                l = l.rstrip('\n')
                if l.startswith('Files:'):
                    in_files = True
                    continue
                elif l.startswith('#'):
                    pass
                elif not l.startswith(' '):
                    in_files = False
                    if l.startswith('Source:'):
                        act.blame = 'dsc:' + l[7:].strip()
                        testbed.blame(act.blame)
                if not in_files:
                    continue

                m = fre.match(l)
                if not m:
                    badpkg(".dsc contains unparseable line"
                           " in Files: `%s'" % l)
                leaf = m.groups(0)[0]

                part = TestbedPath(
                    testbed,
                    os.path.join(os.path.dirname(dsc), leaf),
                    os.path.join(testbed.scratch, what, leaf))
                part.copydown()

    if act.kind == 'ubtree':
        dsc = os.path.join(tmp, what + '-fakedsc')
        with open(dsc, 'w') as f_dsc:
            with open(os.path.join(act.arg, 'debian/control')) as f_control:
                for l in f_control:
                    if l == '\n':
                        break
                    f_dsc.write(l)
            f_dsc.write('Binary: none-so-this-is-not-a-package-name\n')
        atexit.register(lambda f: os.path.exists(f) and os.unlink(f), dsc)

    if act.kind in ['dsc', 'apt']:
        testbed.prepare2([])
        if testbed.execute('dpkg-source_check', ['which', 'dpkg-source']) != 0:
            debug('dpkg-source not available in testbed, installing dpkg-dev', 1)
            # Install dpkg-source for unpacking .dsc
            testbed.satisfy_dependencies_string('dpkg-dev',
                                                'install dpkg-dev')

    # we only use the testbed path from this
    work = os.path.join(testbed.scratch, what + '-build')

    if act.kind == 'ubtree':
        spec = '%s/real-tree' % work
        create_command = '''
    rm -rf "$spec"
    mkdir "$spec"
    cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
'''
        # copy unbuilt tree into testbed
        ubtree = TestbedPath(testbed, act.arg,
                             os.path.join(testbed.scratch, act.what + '-ubtree'))
        ubtree.copydown()
        initcwd = ubtree.tb

    if act.kind == 'dsc':
        spec = dsc_tb
        create_command = 'dpkg-source -x $spec\n'
        initcwd = work

    if act.kind == 'apt':
        spec = act.arg
        create_command = 'apt-get source $spec\n'
        initcwd = work

    script = [
        'spec="$2"',
        'origpwd=' + initcwd,
        'mkdir -p ' + work,
        'cd ' + work
    ]

    if opts.user:
        script += (['chown ' + opts.user + ' . ..'] +
                   ['spec="$spec" origpwd="$origpwd" '
                    + opts.user_wrap(create_command)])
    else:
        script += [create_command]

    script += [
        'cd [a-z0-9]*-*/.',
        'pwd >&3',
        'set +e; test -f debian/tests/control; echo $? >&3'
    ]
    (result_pwd, control_test_rc) = source_rules_command(
        script, what, 'extract',
        cwd=None, results_lines=2, xargs=['x', work, spec])

    filter = act.ah['dsc_filter']

    if control_test_rc == '1':
        act.missing_tests_control = True
        return

    # For optional builds:
    #
    # We might need to build the package because:
    #   - we want its binaries (filter isn't _ and at least one of the
    #	deb_... isn't ignore)
    #   - the test control file says so
    #       (assuming we have any tests)

    class NeedBuildException:
        pass

    def build_needed(m):
        debug_b('build needed for %s' % m)
        raise NeedBuildException()

    try:
        if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
                              act.ah['deb_fortests'] != 'ignore'):
            build_needed('binaries')

        # get test control file from testbed
        test_control = TestbedPath(testbed,
                                   os.path.join(tmp, what + '-testcontrol'),
                                   os.path.join(result_pwd, 'debian/tests/control'), False)
        test_control.copyup()
        stanzas = read_control(act, test_control.host)
        os.unlink(test_control.host)
        for stanza in stanzas:
            for t in stanza[' tests']:
                if 'build-needed' in t.restriction_names:
                    build_needed('test %s' % t.tname)

        debug_b('build not needed')
        built = False

    except NeedBuildException:
        testbed.needs_reset()

        if act.kind not in ['dsc', 'apt']:
            testbed.prepare2([])

        if act.kind == 'apt':
            # we need to get the downloaded debian/control from the testbed, so
            # that we can avoid calling "apt-get build-dep" and thus
            # introducing a second mechanism for installing build deps
            pkg_control = TestbedPath(testbed,
                                      os.path.join(tmp, what + '-control'),
                                      os.path.join(result_pwd, 'debian/control'), False)
            pkg_control.copyup()
            dsc = os.path.join(tmp, what + '-fakedsc')
            with open(dsc, 'w') as f_dsc:
                with open(pkg_control.host) as f_control:
                    for l in f_control:
                        if l == '\n':
                            break
                        f_dsc.write(l)
                f_dsc.write('Binary: none-so-this-is-not-a-package-name\n')
            atexit.register(lambda f: os.path.exists(f) and os.unlink(f), dsc)

        testbed.satisfy_dependencies_string(', '.join(build_essential),
                                            'install build-essential')
        testbed.satisfy_dependencies_dsc(dsc, 'build dependencies')

        script = [
            'cd "$2"',
            opts.user_wrap(dpkg_buildpackage),
        ]
        source_rules_command(script, what, 'build',
                             cwd=initcwd, xargs=['x', work, result_pwd])

        if os.path.dirname(result_pwd) != work:
            badpkg("results dir `%s' is not in expected parent"
                   " dir `%s'" % (result_pwd, work))

        built = True

    act.tests_tree = TestbedPath(
        testbed,
        os.path.join(tmp, what + '-tests-tree'),
        os.path.join(work, os.path.basename(result_pwd)),
        is_dir=True)
    # copy tests from testbed to hosts, we need that for parsing control files
    # and some runners like LXC run a fresh testbed after build for the tests
    act.tests_tree.copyup()
    atexit.register(rmtree, 'tests-tree', act.tests_tree.host)

    if not built:
        act.blamed = []
        return

    act.blamed = copy.copy(testbed.blamed)

    debug_b('filter=%s' % filter)
    if filter != '_':
        script = [
            'cd ' + work + '/[a-z0-9]*-*/.',
            opts.user_wrap(dpkg_buildpackage),
            'cd ..',
            'echo *.deb >&3',
        ]
        result_debs = source_rules_command(script, what,
                                           'binary', work,
                                           results_lines=1,
                                           xargs=['x'])
        if result_debs == '*.deb':
            debs = []
        else:
            debs = result_debs.split(' ')
        debug_b('debs=' + repr(debs))

        # determine built debs and copy them from testbed
        deb_re = re.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
        for deb in debs:
            m = deb_re.match(deb)
            if not m:
                badpkg("badly-named binary `%s'" % deb)
            pkg = m.groups()[0]
            debug_b(' deb=%s, pkg=%s' % (deb, pkg))
            for pat in filter.split(','):
                debug_b('  pat=%s' % pat)
                if not fnmatch.fnmatchcase(pkg, pat):
                    debug_b('   no match')
                    continue
                deb_what = pkg + '_' + what + '.deb'
                deb_path = TestbedPath(testbed,
                                       os.path.join(tmp, deb_what),
                                       os.path.join(work, deb),
                                       False)
                debug_b('  deb_what=%s, deb_path=%s' %
                        (deb_what, str(deb_path)))
                deb_path.copyup()
                binaries.register(act, pkg, deb_path.host,
                                  'forbuilds', testbed.blamed)
                act.binaries.append((pkg, deb_path.host))
                break
        debug_b('all done.')

#---------- main processing loop and main program


def process_actions():
    global binaries

    def debug_a1(m):
        debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ ' + m)

    def debug_a2(m):
        debug('@@@@@@@@@@@@@@@@@@@@ ' + m)

    def debug_a3(m):
        debug('@@@@@@@@@@ ' + m)

    binaries = Binaries(testbed)

    binaries.reset()

    debug_a1('builds ...')
    for act in opts.actions:
        debug_a2('%s %s' % (act.kind, act.what))

        if act.kind == 'deb':
            testbed.blame('arg:' + act.arg)
            determine_package(act)
            testbed.blame('deb:' + act.pkg)
            binaries.register(act, act.pkg, act.arg,
                              'forbuilds', testbed.blamed)
        if act.kind in ['dsc', 'ubtree', 'apt']:
            build_source(act)
        if act.kind == 'tree':
            act.binaries = []
        if act.kind == 'instantiate':
            pass

    debug_a1('builds done.')

    if act.missing_tests_control:
        report('*', 'SKIP no tests in this package')
        global errorcode
        errorcode |= 8
        return

    binaries.reset()

    debug_a1('tests ...')
    for act in opts.actions:
        debug_a2('test %s %s' % (act.kind, act.what))

        if act.kind == 'deb':
            binaries.register(act, act.pkg, act.arg, 'fortests',
                              ['deb:' + act.pkg])
        if act.kind == 'dsc' or act.kind == 'ubtree':
            for (pkg, path) in act.binaries:
                binaries.register(act, pkg, path, 'fortests',
                                  act.blamed)
        if act.kind in ['dsc', 'apt']:
            if act.ah['dsc_tests']:
                debug_a3('read control ...')
                stanzas = read_control(
                    act, os.path.join(act.tests_tree.host, 'debian/tests/control'))
                testbed.blamed += act.blamed
                debug_a3('run_tests ...')
                run_tests(stanzas, act.tests_tree)
        if act.kind == 'tree' or act.kind == 'ubtree':
            testbed.blame('arg:' + act.arg)
            stanzas = read_control(
                act, os.path.join(act.arg, 'debian/tests/control'))
            debug_a3('run_tests ...')
            if act.kind == 'ubtree':
                run_tests(stanzas, act.tests_tree)
            else:
                tree = TestbedPath(testbed, act.arg,
                                   os.path.join(testbed.scratch, act.what + '-tree'))
                run_tests(stanzas, tree)
        if act.kind == 'instantiate':
            testbed.prepare([])
        testbed.needs_reset()
    debug_a1('tests done.')


def main():
    global testbed
    try:
        parse_args()
    except SystemExit:
        # optparser exits with error 2 by default, but we have a different
        # meaning for that already
        sys.exit(20)

    try:
        setup_trace()
        testbed = Testbed()
        testbed.start()
        testbed.open()
        finalise_options()
        process_actions()
    except:
        ec = print_exception(sys.exc_info(), '')
        cleanup()
        sys.exit(ec)
    cleanup()
    sys.exit(errorcode)

main()