autopkgtest 3.6jessie1 / usr / bin / adt-run

#!/usr/bin/python3 -u
#
# adt-run is part of autopkgtest
# autopkgtest is a tool for testing Debian binary packages
#
# autopkgtest is Copyright (C) 2006-2007, 2013 Canonical Ltd.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
# See the file CREDITS for a full list of credits information (often
# installed as /usr/share/doc/autopkgtest/CREDITS).

import signal
import tempfile
import sys
import subprocess
import traceback
import re
import os
import errno
import shutil
import time
import atexit

from urllib.parse import quote as url_quote
from urllib.parse import unquote as url_unquote

from debian import deb822, debian_support

try:
    our_base = os.environ['AUTOPKGTEST_BASE'] + '/lib'
except KeyError:
    our_base = '/usr/share/autopkgtest/python'
sys.path.insert(1, our_base)
import VirtSubproc
import adtlog
import testdesc
import adt_run_args

# ---------- global variables

tmp = None		# pathstring on host
testbed = None		# Testbed
shared_downtmp = None   # testbed's downtmp on the host, if supported
opts = None             # argparse options
vserver_args = None     # everything after ---
actions = None          # list of (action_type, path)
errorcode = 0		# exit status that we are going to use
binaries = None		# Binaries (.debs we have registered)
build_essential = ['build-essential']
dpkg_buildpackage = 'dpkg-buildpackage -us -uc -b'

# do not consider using apt-listbugs
os.putenv('APT_LISTBUGS_FRONTEND', 'none')
# do not consider using apt-listchanges
os.putenv('APT_LISTCHANGES_FRONTEND', 'none')


# ---------- errors we define


class Quit(RuntimeError):

    def __init__(self, ec, m):
        self.ec = ec
        self.m = m


def bomb(m):
    raise Quit(20, 'unexpected error: %s' % m)


def badpkg(m):
    adtlog.preport('blame: ' + ' '.join(testbed.blamed))
    adtlog.preport('badpkg: ' + m)
    raise Quit(12, 'erroneous package: %s' % m)


class Unsupported(Exception):

    def __init__(self, lno, m):
        if lno >= 0:
            self.m = '%s (control line %d)' % (m, lno)
        else:
            self.m = m

    def report(self, tname):
        global errorcode
        errorcode |= 2
        adtlog.report(tname, 'SKIP %s' % self.m)

# ---------- convenience functions


def mkdir_okexist(pathname, mode=0o2755):
    try:
        os.mkdir(pathname, mode)
    except (IOError, OSError) as oe:
        if oe.errno != errno.EEXIST:
            raise


def rmtree(what, pathname):
    adtlog.debug('/ %s rmtree %s' % (what, pathname))
    try:
        shutil.rmtree(pathname)
    except (IOError, OSError) as oe:
        if oe.errno not in (errno.EEXIST, errno.ENOENT):
            raise


def script_out(argv, what=None, script=None, **kwargs):
    '''Call a script and get its return code, and optionally stdout.

    If what/script are given, log these for debugging.

    Return (exitcode, stdout). stdout will be a string if kwargs contains
    stdout=subprocess.PIPE, otherwise None.
    '''
    if what:
        adtlog.debug_subprocess(what, argv, script=script)

    process = subprocess.Popen(argv, universal_newlines=True, **kwargs)
    output = process.communicate()[0]
    return (process.returncode, output)


def files_from_dsc(dsc_path):
    '''Get files from a .dsc or a .changes

    Return list of files, including the directory of dsc_path.
    '''
    try:
        files = testdesc.parse_rfc822(dsc_path).__next__()['Files'].split()
    except (StopIteration, KeyError):
        badpkg('%s is invalid and does not contain Files:' % dsc_path)

    dsc_dir = os.path.dirname(dsc_path)

    return [os.path.join(dsc_dir, f) for f in files if '.' in f and '_' in f]


def setup_trace():
    global tmp

    if opts.output_dir is not None:
        rmtree('tmp(specified)', opts.output_dir)
        mkdir_okexist(opts.output_dir, 0o755)
        tmp = opts.output_dir
    else:
        assert(tmp is None)
        tmp = tempfile.mkdtemp(prefix='adt-run.output.')
        os.chmod(tmp, 0o755)

    if opts.logfile is None and opts.output_dir is not None:
        opts.logfile = opts.output_dir + '/log'

    if opts.logfile is not None:
        # tee stdout/err into log file
        fifo_log = os.path.join(tmp, 'fifo_log')
        os.mkfifo(fifo_log)
        atexit.register(os.unlink, fifo_log)
        out_tee = subprocess.Popen(['tee', fifo_log],
                                   stdin=subprocess.PIPE)
        err_tee = subprocess.Popen(['tee', fifo_log, '-a', '/dev/stderr'],
                                   stdin=subprocess.PIPE,
                                   stdout=open('/dev/null', 'wb'))
        log_cat = subprocess.Popen(['cat', fifo_log], stdout=open(opts.logfile, 'wb'))
        os.dup2(out_tee.stdin.fileno(), sys.stdout.fileno())
        os.dup2(err_tee.stdin.fileno(), sys.stderr.fileno())

        def cleanup():
            os.close(sys.stdout.fileno())
            os.close(out_tee.stdin.fileno())
            out_tee.wait()
            os.close(sys.stderr.fileno())
            os.close(err_tee.stdin.fileno())
            err_tee.wait()
            log_cat.wait()

        atexit.register(cleanup)

    if opts.summary is not None:
        adtlog.summary_stream = open(opts.summary, 'wb', 0)
    elif opts.output_dir is not None:
        adtlog.summary_stream = open(os.path.join(opts.output_dir, 'summary'), 'wb', 0)


def finalise_options():
    global opts, testbed, build_essential, dpkg_buildpackage

    if opts.user:
        if 'root-on-testbed' not in testbed.caps:
            adtlog.warning('virtualisation system does not offer root on '
                           'testbed but --user option specified: failure likely')
        opts.user_wrap = lambda x: "su --shell=/bin/sh %s -c 'set -e; %s'" % (opts.user, x)
    else:
        opts.user_wrap = lambda x: x

    if opts.gainroot is None:
        if opts.user or 'root-on-testbed' not in testbed.caps:
            opts.gainroot = 'fakeroot'
            build_essential += ['fakeroot']
    if opts.gainroot:
        dpkg_buildpackage += ' -r' + opts.gainroot


def child_ps(pid):
    '''Get all child processes of pid'''

    try:
        out = subprocess.check_output(['ps', '-o', 'pid=', '--ppid', str(pid)],
                                      universal_newlines=True)
        return [int(p) for p in out.split()]
    except subprocess.CalledProcessError:
        return []


def killtree(pid):
    '''Recursively kill pid and all of its children'''

    for child in child_ps(pid):
        killtree(child)
    try:
        os.kill(pid, signal.SIGTERM)
    except OSError:
        pass

# ---------- testbed management - the Testbed class


class Testbed:

    def __init__(self):
        self.sp = None
        self.lastsend = None
        self.scratch = None
        self.modified = False
        self.blamed = []
        self._need_reset_apt = False
        self.stop_sent = False
        self.dpkg_arch = None
        self.exec_cmd = None
        self.install_tmp_env = []
        adtlog.debug('testbed init')

    def start(self):
        # are we running from a checkout?
        root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
        if os.path.exists(os.path.join(root_dir, '.git')):
            try:
                head = subprocess.check_output(['git', 'show', '--no-patch', '--oneline'],
                                               cwd=root_dir, universal_newlines=True)
                head = head.strip()
            except subprocess.CalledProcessError:
                head = 'cannot determine current HEAD'
            adtlog.info('git checkout: %s' % head)
        else:
            adtlog.info('version 3.6jessie1')

        # vserver can be given without "adt-virt-" prefix
        if '/' not in vserver_args[0] and not vserver_args[0].startswith('adt-virt-'):
            vserver_args[0] = 'adt-virt-' + vserver_args[0]

        adtlog.debug_subprocess('vserver', vserver_args)
        self.sp = subprocess.Popen(vserver_args,
                                   stdin=subprocess.PIPE,
                                   stdout=subprocess.PIPE,
                                   universal_newlines=True)
        self.expect('ok', 0)

    def stop(self):
        adtlog.debug('testbed stop')
        if self.stop_sent:
            # avoid endless loop
            return
        self.stop_sent = True

        self.close()
        if self.sp is None:
            return
        ec = self.sp.returncode
        if ec is None:
            self.sp.stdout.close()
            self.send('quit')
            self.sp.stdin.close()
            ec = self.sp.wait()
        if ec:
            self.bomb('testbed gave exit status %d after quit' % ec)
        self.sp = None

    def open(self):
        adtlog.debug('testbed open, scratch=%s' % self.scratch)
        if self.scratch is not None:
            return
        pl = self.command('open', (), 1)
        self._opened(pl)

    def _opened(self, pl):
        global shared_downtmp

        self.scratch = pl[0]
        self.deps_installed = []
        self.recommends_installed = False
        self.exec_cmd = list(map(url_unquote, self.command('print-execute-command', (), 1)[0].split(',')))
        self.caps = self.command('capabilities', (), None)
        adtlog.debug('testbed capabilities: %s' % self.caps)
        for c in self.caps:
            if c.startswith('downtmp-host='):
                shared_downtmp = c.split('=', 1)[1]

        # provide a default for --user
        if opts.user is None and 'root-on-testbed' in self.caps:
            opts.user = ''
            for c in self.caps:
                if c.startswith('suggested-normal-user='):
                    opts.user = c.split('=', 1)[1]

        self.run_setup_commands()

        # determine testbed architecture
        self.dpkg_arch = self.check_exec(['dpkg', '--print-architecture'], True).strip()
        adtlog.info('testbed dpkg architecture: ' + self.dpkg_arch)

        # record package versions of pristine testbed
        if opts.output_dir:
            pkglist = TempTestbedPath(testbed, 'testbed-packages', autoclean=False)
            self.check_exec(['sh', '-ec', "dpkg-query --show -f '${Package}\\t${Version}\\n' > %s" % pkglist.tb])
            pkglist.copyup()

        # provide autopkgtest-reboot command, if reboot is supported
        if 'reboot' in self.caps and 'root-on-testbed' in self.caps:
            adtlog.debug('testbed supports reboot, creating /sbin/autopkgtest-reboot')
            rc = self.execute(['sh', '-ec', '''/bin/echo -e '#!/bin/sh -e\\n'''
                               '''[ -n "$1" ] || { echo "Usage: $0 <mark>" >&2; exit 1; }\\n'''
                               '''echo "$1" > /run/autopkgtest-reboot-mark\\n'''
                               '''kill -KILL $PPID\\n' > /sbin/autopkgtest-reboot;'''
                               '''chmod 755 /sbin/autopkgtest-reboot'''])[0]
            if rc != 0:
                adtlog.warning('testbed supports reboot and root-on-testbed, '
                               'but creating /sbin/autopkgtest-reboot failed;'
                               ' read-only fs?')

    def mungeing_apt(self):
        if 'revert' not in self.caps:
            self._need_reset_apt = True

    def reset_apt(self):
        if not self._need_reset_apt:
            return
        self._need_reset_apt = False
        self.check_exec(['rm', '-f', '/etc/apt/apt.conf.d/90autopkgtest',
                         '/etc/apt/sources.list.d/autopkgtest.list',
                         '/etc/apt/preferences.d/90autopkgtest'])
        self.check_exec(['sh', '-ec', '(apt-get --quiet update || (sleep 15; apt-get update)) 2>&1'],
                        kind='install')

    def close(self):
        global shared_downtmp

        adtlog.debug('testbed close, scratch=%s' % self.scratch)
        if self.scratch is None:
            return
        self.scratch = None
        if self.sp is None:
            return
        self.command('close')
        shared_downtmp = None

    def run_setup_commands(self):
        '''Run --setup-commmands and --copy'''

        if not opts.setup_commands and not opts.apt_pocket and not opts.copy:
            return

        adtlog.info('@@@@@@@@@@@@@@@@@@@@ test bed setup')
        for (host, tb) in opts.copy:
            adtlog.debug('Copying file %s to testbed %s' % (host, tb))
            TestbedPath(self, host, tb, os.path.isdir(host)).copydown()

        for p in opts.apt_pocket:
            script = '''awk '/^deb(-src)? .*(ubuntu.com|debian.org|ftpmaster)/ { if ($3 !~ /-/) { $3 = $3"-%s"; print }}' ''' \
                '''/etc/apt/sources.list `ls /etc/apt/sources.list.d/*.list 2>/dev/null|| true` ''' \
                ''' > /etc/apt/sources.list.d/%s.list''' % (p, p)
            self.check_exec(['sh', '-ec', script])

        # record the mtimes of dirs affecting the boot
        boot_dirs = '/boot /etc/init /etc/init.d /etc/systemd/system /lib/systemd/system'
        self.check_exec(['bash', '-ec',
                         'for d in %s; do [ ! -d $d ] || touch -r $d %s/${d//\//_}.stamp; done' % (
                             boot_dirs, self.scratch)])

        xenv = []
        if opts.user:
            xenv.append('ADT_NORMAL_USER=' + opts.user)

        for c in opts.setup_commands:
            rc = self.execute(['sh', '-ec', c], xenv=xenv, kind='install')[0]
            if rc:
                bomb('testbed setup commands failed with status %i' % rc)

        if opts.setup_commands and 'reboot' in self.caps:
            boot_affected = self.execute(
                ['bash', '-ec', '[ ! -e /run/autopkgtest_no_reboot.stamp ] || exit 0;'
                 'for d in %s; do s=%s/${d//\//_}.stamp;'
                 '  [ ! -d $d ] || [ `stat -c %%Y $d` = `stat -c %%Y $s` ]; done' % (
                     boot_dirs, self.scratch)])[0]
            if boot_affected:
                adtlog.info('rebooting testbed after setup commands that affected boot')
                self.command('reboot', ())

    def reset(self, deps_new, with_recommends):
        '''Reset the testbed, if possible and necessary'''

        adtlog.debug('testbed reset: modified=%s, deps_installed=%s(r: %s), deps_new=%s(r: %s)' %
                     (self.modified, self.deps_installed, self.recommends_installed,
                      deps_new, with_recommends))
        if 'revert' in self.caps and (
                self.modified or self.recommends_installed != with_recommends or
                [d for d in self.deps_installed if d not in deps_new]):
            adtlog.debug('testbed reset')
            pl = self.command('revert', (), 1)
            self._opened(pl)
        self.modified = False

    def install_deps(self, deps_new, recommends):
        '''Install dependencies into testbed

        Also publish the registered binaries.
        '''
        adtlog.debug('install_deps: deps_new=%s, recommends=%s' % (deps_new, recommends))
        binaries.publish()

        self.deps_installed = deps_new
        self.recommends_installed = recommends
        if not deps_new:
            return
        self.satisfy_dependencies_string(', '.join(deps_new), 'install-deps', recommends)

    def prepare(self, deps_new, recommends):
        '''Set up clean test bed with given dependencies'''

        self.reset(deps_new, recommends)
        self.install_deps(deps_new, recommends)

    def needs_reset(self):
        # show what caused a reset
        (fname, lineno, function, code) = traceback.extract_stack(limit=2)[0]
        adtlog.debug('needs_reset, previously=%s, requested by %s() line %i' %
                     (self.modified, function, lineno))
        self.modified = True

    def blame(self, m):
        adtlog.debug('blame += %s' % m)
        self.blamed.append(m)

    def bomb(self, m):
        adtlog.debug('bomb %s' % m)
        self.reset_apt()
        self.stop()
        raise Quit(16, 'testbed failed: %s' % m)

    def send(self, string):
        try:
            adtlog.debug('sending command to testbed: ' + string)
            self.sp.stdin.write(string)
            self.sp.stdin.write('\n')
            self.sp.stdin.flush()
            self.lastsend = string
        except:
            (type, value, dummy) = sys.exc_info()
            self.bomb('cannot send to testbed: %s' % traceback.
                      format_exception_only(type, value))

    def expect(self, keyword, nresults):
        l = self.sp.stdout.readline()
        if not l:
            self.bomb('unexpected eof from the testbed')
        if not l.endswith('\n'):
            self.bomb('unterminated line from the testbed')
        l = l.rstrip('\n')
        adtlog.debug('got reply from testbed: ' + l)
        ll = l.split()
        if not ll:
            self.bomb('unexpected whitespace-only line from the testbed')
        if ll[0] != keyword:
            if self.lastsend is None:
                self.bomb("got banner `%s', expected `%s...'"
                          (l, keyword))
            else:
                self.bomb("sent `%s', got `%s', expected `%s...'" %
                          (self.lastsend, l, keyword))
        ll = ll[1:]
        if nresults is not None and len(ll) != nresults:
            self.bomb("sent `%s', got `%s' (%d result parameters),"
                      " expected %d result parameters" %
                      (self.lastsend, l, len(ll), nresults))
        return ll

    def command(self, cmd, args=(), nresults=0, unquote=True):
        # pass args=[None,...] or =(None,...) to avoid more url quoting
        if type(cmd) is str:
            cmd = [cmd]
        if len(args) and args[0] is None:
            args = args[1:]
        else:
            args = list(map(url_quote, args))
        al = cmd + args
        self.send(' '.join(al))
        ll = self.expect('ok', nresults)
        if unquote:
            ll = list(map(url_unquote, ll))
        return ll

    def execute(self, argv, xenv=[], stdout=None, stderr=None, kind='short'):
        '''Run command in testbed.

        The commands stdout/err will be piped directly to adt-run and its log
        files, unless redirection happens with the stdout/stderr arguments
        (passed to Popen).

        Return (exit code, stdout, stderr). stdout/err will be None when output
        is not redirected.
        '''
        timeout = getattr(opts, 'timeout_' + kind)

        env = list(xenv)  # copy
        if kind == 'install':
            env.append('DEBIAN_FRONTEND=noninteractive')
            env.append('APT_LISTBUGS_FRONTEND=none')
        if opts.set_lang is not False:
            env.append('LANG=%s' % opts.set_lang)
        env += self.install_tmp_env

        adtlog.debug('testbed command %s, kind %s, sout %s, serr %s, env %s' %
                     (argv, kind, stdout and 'pipe' or 'raw',
                      stderr and 'pipe' or 'raw', env))

        if env:
            argv = ['env'] + env + argv

        VirtSubproc.timeout_start(timeout)
        try:
            proc = subprocess.Popen(self.exec_cmd + argv, stdout=stdout,
                                    stderr=stderr, universal_newlines=True)
            (out, err) = proc.communicate()
            VirtSubproc.timeout_stop()
        except VirtSubproc.Timeout:
            killtree(proc.pid)
            proc.wait()
            self.bomb('timed out on command "%s" (kind: %s)' % (' '.join(argv), kind))

        adtlog.debug('testbed command exited with code %i' % proc.returncode)

        return (proc.returncode, out, err)

    def check_exec(self, argv, stdout=False, kind='short'):
        '''Run argv in testbed.

        If stdout is True, capture stdout and return it. Otherwise, don't
        redirect and return None.

        argv must succeed and not print any stderr.
        '''
        (code, out, err) = self.execute(argv,
                                        stdout=(stdout and subprocess.PIPE or None),
                                        stderr=subprocess.PIPE, kind=kind)
        if err:
            bomb('"%s" failed with stderr "%s"' % (' '.join(argv), err))
        if code != 0:
            bomb('"%s" failed with status %i' % (' '.join(argv), code))
        return out

    def install_apt(self, deps, recommends=False):
        '''Install dependencies with apt-get into testbed

        This requires root privileges and a writable file system.
        '''
        # create a dummy deb with the deps
        pkgdir = tempfile.mkdtemp(prefix='adt-satdep.')
        debdir = os.path.join(pkgdir, 'DEBIAN')
        os.chmod(pkgdir, 0o755)
        os.mkdir(debdir)
        os.chmod(debdir, 0o755)
        with open(os.path.join(debdir, 'control'), 'w') as f:
            f.write('''Package: adt-satdep
Section: oldlibs
Priority: extra
Maintainer: autogenerated
Version: 0
Architecture: %s
Depends: %s
Description: satisfy autopkgtest test dependencies
''' % (self.dpkg_arch, deps))

        deb = TempTestbedPath(self, 'adt-satdep.deb')
        subprocess.check_call(['dpkg-deb', '-b', pkgdir, deb.host],
                              stdout=subprocess.PIPE)
        shutil.rmtree(pkgdir)
        deb.copydown()

        # install it and its dependencies in the tb
        self.check_exec(['dpkg', '--unpack', deb.tb], stdout=subprocess.PIPE)
        rc = self.execute(['apt-get', 'install', '--quiet', '--quiet', '--assume-yes', '--fix-broken',
                           '-o', 'APT::Install-Recommends=%s' % recommends,
                           '-o', 'Debug::pkgProblemResolver=true'],
                          kind='install')[0]
        if rc != 0:
            bomb('failed to run apt-get to satisfy adt-satdep.deb dependencies')
        rc = self.execute(['dpkg', '--status', 'adt-satdep'],
                          stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE)[0]
        if rc != 0:
            bomb('test dependencies are unsatisfiable')

    def install_tmp(self, deps, recommends=False):
        '''Unpack dependencies into temporary directory

        This is a fallback if the testbed does not have root privileges or a
        writable file system, and will only work for packages that can be
        used from a different directory with PATH, LD_LIBRARY_PATH, PYTHONPATH
        etc. set.

        Sets/updates self.install_tmp_env to necessary variables.
        '''
        unsupported = []
        pkg_constraints = {}  # pkg -> (relation, version)

        # parse deps into pkg_constraints
        dep_re = re.compile(
            r'(?P<p>[a-z0-9+-.]+)\s*'
            r'(\((?P<r><<|<=|>=|=|>>)\s*(?P<v>[^\)]*)\))?$')
        for dep in deps.split(','):
            dep = dep.strip()
            if not dep:
                continue  # trailing comma
            m = dep_re.match(dep)
            if not m:
                unsupported.append(dep)
                continue
            pkg_constraints[m.group('p')] = (m.group('r'), m.group('v'))

        adtlog.debug('install_tmp: "%s" -> %s, unsupported: %s' %
                     (deps, pkg_constraints, unsupported))

        if unsupported:
            adtlog.warning('The following dependencies cannot be handled in '
                           'reduced "unpack to temporary directory" mode: ' +
                           ', '.join(unsupported))

        # simulate installation, grab packages, and check constraints
        (rc, out, _) = self.execute(['apt-get', '--quiet', '--simulate', '--no-remove',
                                     '-o', 'Debug::pkgProblemResolver=true',
                                     '-o', 'Debug::NoLocking=true',
                                     '-o', 'APT::Install-Recommends=%s' % recommends,
                                     '-o', 'APT::Get::Show-User-Simulation-Note=False',
                                     'install'] + list(pkg_constraints),
                                    stdout=subprocess.PIPE)
        if rc != 0:
            bomb('test dependencies are unsatisfiable')

        def check_constraint(pkg, ver):
            constraint = pkg_constraints.get(pkg, (None, None))
            if constraint[0] is None:
                return True
            comp = debian_support.version_compare(ver, constraint[1])
            if constraint[0] == '<<':
                return comp < 0
            if constraint[0] == '<=':
                return comp <= 0
            if constraint[0] == '==':
                return comp == 0
            if constraint[0] == '>=':
                return comp >= 0
            if constraint[0] == '>>':
                return comp > 0
            raise ValueError('invalid dependency version relation %s' % constraint[0])

        to_install = []
        for line in out.splitlines():
            if not line.startswith('Inst '):
                continue
            fields = line.split()
            pkg = fields[1]
            ver = fields[2][1:]  # chop off '('
            # ignore Python 2 stuff, with PYTHONPATH we can only support one
            # Python major version (3)
            if pkg.startswith('python-') or pkg.startswith('libpython-') or \
               'python2.' in pkg or pkg == 'python':
                adtlog.warning('Ignoring Python 2.x dependency %s, not '
                               'supported in unpack only mode' % pkg)
                continue
            if not check_constraint(pkg, ver):
                bomb('test dependency %s (%s %s) is unsatisfiable: available version %s' %
                     (pkg, pkg_constraints[pkg][0], pkg_constraints[pkg][1], ver))
            to_install.append(pkg)

        adtlog.debug('install_tmp: packages to install: %s' % ' '.join(to_install))

        if not to_install:
            # we already have everything, all good
            return

        adtlog.warning('virtualisation system does not offer root or writable '
                       'testbed; unpacking dependencies to temporary dir, '
                       'which will only work for some packages')

        # download and unpack all debs
        script = '''d=%(t)s/deps
mkdir -p $d; cd $d
apt-get download %(pkgs)s >&2
for p in *.deb; do dpkg-deb --extract $p .; rm $p; done

# executables
echo PATH=$d/sbin:$d/bin:$d/usr/sbin:$d/usr/bin:$d/usr/games:$PATH

# shared libraries
l=""
for candidate in $(find $d -type d \( -name 'lib' -o -path '*/lib/*-linux-*' \)); do
    [ -z "$(ls $candidate/*.so $candidate/*.so.* 2>/dev/null)" ] || l="$candidate:$l"
done
[ -z "$l" ] || echo LD_LIBRARY_PATH=$l${LD_LIBRARY_PATH:-}

# Python modules
p=""
for candidate in $d/usr/lib/python3*/dist-packages; do
    [ ! -d $candidate ] || p="$candidate:$p"
done
[ -z "$p" ] || echo PYTHONPATH=$p${PYTHONPATH:-}

# gobject-introspection
l=""
if [ -d $d/usr/lib/girepository-1.0 ]; then
    l=$d/usr/lib/girepository-1.0
fi
for candidate in $(find $d -type d -path '*/usr/lib/*/girepository-*'); do
    [ -z "$(ls $candidate/*.typelib 2>/dev/null)" ] || l="$candidate:$l"
done
[ -z "$l" ] || echo GI_TYPELIB_PATH="$l:${GI_TYPELIB_PATH:-}"
''' % {'t': self.scratch, 'pkgs': ' '.join(to_install)}
        (rc, out, _) = self.execute(['sh', '-euc', script],
                                    stdout=subprocess.PIPE)
        if rc != 0:
            bomb('failed to download and unpack test dependencies')
        self.install_tmp_env = [l.strip() for l in out.splitlines() if l]
        adtlog.debug('install_tmp: env is now %s' % self.install_tmp_env)

    def install_click(self, clickpath):
        # copy click into testbed
        tp = TestbedPath(self, clickpath, os.path.join(
            self.scratch, os.path.basename(clickpath)))
        tp.copydown()
        # install it
        clickopts = ['--all-users']
        if 'ADT_CLICK_NO_FRAMEWORK_CHECK' in os.environ:
            # this is mostly for testing
            clickopts.append('--force-missing-framework')
        rc = self.execute(['click', 'install', '--allow-unauthenticated']
                          + clickopts + [tp.tb], kind='install')[0]
        if rc != 0:
            bomb('click install failed with status %i' % rc)

        # work around https://launchpad.net/bugs/1337253
        adtlog.info('Updating AppArmor rules to allow autopilot introspection')
        script = '[ -d /var/cache/apparmor ] || exit; ' \
            'echo "dbus (receive, send) bus=session path=/com/canonical/Autopilot/**,"' \
            ' > /var/cache/apparmor/click-ap.rules; ' \
            'info=$(click info %s %s); ' \
            '''name=$(echo "$info" | sed -rn '/"name"/ {s/^.*: *"([^"]+)",/\\1/; p}'); ''' \
            '''version=$(echo "$info" | sed -rn '/"version"/ {s/^.*: *"([^"]+)",/\\1/; p}'); ''' \
            'touch -h /var/lib/apparmor/clicks/${name}_*_${version}.json; '\
            'aa-clickhook --include=/var/cache/apparmor/click-ap.rules' % (
                opts.user and ('--user ' + opts.user) or '', tp.tb)
        self.execute(['sh', '-ec', script], kind='install')

        # work around https://launchpad.net/bugs/1333215
        self.check_exec(['sh', '-ec', opts.user_wrap(
            # we don't want su -l here which resets the environment from
            # self.execute(); so emulate the parts that we want
            # FIXME: move "run as user" as an argument of execute()/check_exec() and run with -l
            ('export USER=%s;' % opts.user) +
            '. /etc/profile >/dev/null 2>&1 || true; '
            ' . ~/.profile >/dev/null 2>&1 || true; '
            '[ -z "$UPSTART_SESSION" ] || /sbin/initctl --user start click-user-hooks')])

    def satisfy_dependencies_string(self, deps, what, recommends=False):
        '''Install dependencies from a string into the testbed'''

        adtlog.debug('%s: satisfying %s' % (what, deps))

        # ignore ":native" tags, apt cannot parse them and deps_parse() does
        # not seem to have an option to get rid of them; we always test on the
        # native platform
        deps = deps.replace(':native', '')

        # resolve arch specific dependencies; don't use universal_newlines
        # here, it's broken for stdin on Python 3.2
        perl = subprocess.Popen(['perl', '-'], stdin=subprocess.PIPE,
                                stdout=subprocess.PIPE)
        code = '''use Dpkg::Deps;
                  $dep = deps_parse('%s', reduce_arch => 1, host_arch => '%s');
                  print $dep->output(), "\\n";
                  ''' % (deps, self.dpkg_arch)
        deps = perl.communicate(code.encode('UTF-8'))[0].decode('UTF-8').strip()
        if perl.returncode != 0:
            bomb('failed to run perl for parsing dependencies')
        adtlog.debug('%s: architecture resolved: %s' % (what, deps))

        # check if we can use apt-get
        can_apt_get = False
        if 'root-on-testbed' in self.caps:
            rc = self.execute(['test', '-w', '/var/lib/dpkg/status'])[0]
            if rc == 0:
                can_apt_get = True
        adtlog.debug('can use apt-get on testbed: %s' % can_apt_get)

        if can_apt_get:
            self.install_apt(deps, recommends)
        else:
            self.install_tmp(deps, recommends)

    def run_shell(self, cwd=None):
        '''Run shell in testbed for debugging tests'''

        adtlog.info(' - - - - - - - - - - running shell - - - - - - - - - -')
        self.command('shell', [cwd or '/'] + self.install_tmp_env)

    def run_test(self, tree, test):
        '''Run given test in testbed

        tree (a TestbedPath) is the source tree root.
        '''
        def _info(m):
            adtlog.info('test %s: %s' % (test.name, m))

        _info('preparing')
        self.prepare(test.depends, 'needs-recommends' in test.restrictions)
        for c in test.clicks:
            testbed.install_click(c)

        # record installed package versions
        if opts.output_dir:
            pkglist = TempTestbedPath(self, test.name + '-packages.all', autoclean=False)
            self.check_exec([
                'sh', '-ec', "dpkg-query --show -f '${Package}\\t${Version}\\n' | grep -v '^adt-satdep' > %s" % pkglist.tb])
            pkglist.copyup()

            # filter out packages from the base system
            with open(pkglist.host[:-4], 'w') as out:
                rc = script_out(['join', '-v2', '-t\t',
                                 os.path.join(opts.output_dir, 'testbed-packages'),
                                 pkglist.host], stdout=out, env={})[0]
            if rc:
                badpkg('failed to call join for test specific package list, code %d' % rc)
            os.unlink(pkglist.host)

        # ensure our tests are in the testbed
        tree.copydown(check_existing=True)

        # stdout/err files in testbed
        so = TempTestbedPath(self, test.name + '-stdout', autoclean=False)
        se = TempTestbedPath(self, test.name + '-stderr', autoclean=False)

        # create script to run test
        test_artifacts = '%s/%s-artifacts' % (self.scratch, test.name)
        script = 'set -e; ' \
                 'export USER=`id -nu`; ' \
                 '. /etc/profile >/dev/null 2>&1 || true; ' \
                 ' . ~/.profile >/dev/null 2>&1 || true; ' \
                 'buildtree="%(t)s"; ' \
                 'mkdir -p -m 1777 -- "%(a)s"; ' \
                 'export ADT_ARTIFACTS="%(a)s"; ' \
                 'export ADTTMP=$(mktemp -d --tmpdir adttmp.XXXXXX); ' \
                 'trap "rm -rf $ADTTMP" EXIT INT QUIT PIPE; '\
                 'chmod 755 $ADTTMP; '\
                 'cd "$buildtree"; '\
                 % {'t': tree.tb, 'a': test_artifacts}

        if opts.user and 'rw-build-tree' in test.restrictions:
            script += 'chown -R %s "$buildtree"; ' % opts.user
        if opts.set_lang is not False:
            script += 'export LANG=%s; ' % opts.set_lang
        # there's no way to tell su to not reset $PATH, for install-tmp mode
        for e in self.install_tmp_env:
            if e.startswith('PATH='):
                script += 'export %s; ' % e
                break
        # if we have an user upstart session, poke the environment into it
        if self.install_tmp_env:
            script += 'if [ -n "$UPSTART_SESSION" ]; then '
            for e in self.install_tmp_env:
                script += ' initctl --user set-env "%s"; ' % e
            script += 'fi; '

        if test.path:
            test_cmd = os.path.join(tree.tb, test.path)
            script += 'chmod +x %s; ' % test_cmd
        else:
            test_cmd = "bash -ec '%s'" % test.command

        script += 'touch %(o)s %(e)s; ' \
                  '%(t)s 2> >(tee -a %(e)s >&2) > >(tee -a %(o)s); ' \
                  % {'t': test_cmd, 'o': so.tb, 'e': se.tb}

        if 'needs-root' not in test.restrictions and opts.user is not None:
            if 'root-on-testbed' not in self.caps:
                bomb('cannot change to user %s without root-on-testbed' % opts.user)
            # we don't want -l here which resets the environment from
            # self.execute(); so emulate the parts that we want
            # FIXME: move "run as user" as an argument of execute()/check_exec() and run with -l
            test_argv = ['su', '-s', '/bin/bash', opts.user, '-c']
        else:
            test_argv = ['bash', '-c']

        # run test script
        if test.command:
            _info(test.command)
        _info('[-----------------------')

        # tests may reboot, so we might need to run several times
        reboot_marker = None
        while True:
            if reboot_marker:
                script_prefix = 'export ADT_REBOOT_MARK="%s"; ' % reboot_marker
            else:
                script_prefix = ''
            rc = self.execute(test_argv + [script_prefix + script], kind='test')[0]

            # did the test invoke autopkgtest-reboot?
            if os.WIFSIGNALED(rc) and os.WTERMSIG(rc) == signal.SIGKILL and 'reboot' in self.caps:
                adtlog.debug('test process SIGKILLed, checking for reboot marker')
                (code, reboot_marker, err) = self.execute(
                    ['cat', '/run/autopkgtest-reboot-mark'],
                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                if code == 0:
                    reboot_marker = reboot_marker.strip()
                    adtlog.info('test process requested reboot with marker %s' % reboot_marker)
                    self.command('reboot', ())
                    continue
                else:
                    adtlog.debug('no reboot marker, considering a failure')
            break

        # give the setup_trace() cats some time to catch up
        sys.stdout.flush()
        sys.stderr.flush()
        time.sleep(0.3)
        _info('-----------------------]')
        adtlog.debug('testbed executing test finished with exit status %i' % rc)

        # copy stdout/err files to host
        so.copyup()
        se.copyup()
        se_size = os.path.getsize(se.host)

        # avoid mixing up stdout (from report) and stderr (from logging) in output
        sys.stdout.flush()
        sys.stderr.flush()
        time.sleep(0.1)

        _info(' - - - - - - - - - - results - - - - - - - - - -')

        global errorcode
        if rc != 0:
            test.failed('non-zero exit status %d' % rc)
            errorcode |= 4
        elif se_size != 0 and 'allow-stderr' not in test.restrictions:
            with open(se.host, encoding='UTF-8') as f:
                stderr_top = f.readline().rstrip('\n \t\r')
            test.failed('stderr: %s' % stderr_top)
            errorcode |= 4
        else:
            test.passed()

        sys.stdout.flush()
        sys.stderr.flush()

        if os.path.getsize(so.host) == 0:
            # don't produce empty -stdout files in --output-dir
            so.autoclean = True

        if se_size != 0 and 'allow-stderr' not in test.restrictions:
            # give tee processes some time to catch up, to avoid mis-ordered logs
            time.sleep(0.2)
            _info(' - - - - - - - - - - stderr - - - - - - - - - -')
            with open(se.host, 'rb') as f:
                while True:
                    block = f.read1(1000000)
                    if not block:
                        break
                    sys.stderr.buffer.write(block)
            sys.stderr.buffer.flush()
        else:
            # don't produce empty -stderr files in --output-dir
            if se_size == 0:
                se.autoclean = True

        # copy artifacts to host, if we have --output-dir
        if opts.output_dir:
            ap = TestbedPath(self,
                             os.path.join(opts.output_dir, 'artifacts'),
                             test_artifacts, is_dir=True)
            ap.copyup()
            # don't keep an empty artifacts dir around
            if not os.listdir(ap.host):
                os.rmdir(ap.host)

        # clean up artifacts dirs
        self.check_exec(['rm', '-rf', test_artifacts])

        if opts.shell or (opts.shell_fail and not test.result):
            self.run_shell(tree.tb)


class TestbedPath:
    '''Represent a file/dir with a host and a testbed path'''

    def __init__(self, testbed, host, tb, is_dir=None):
        '''Create a TestbedPath object.

        The object itself is just a pair of file names, nothing more. They do
        not need to exist until you call copyup() or copydown() on them.

        testbed: the Testbed object which this refers to
        host: path of the file on the host
        tb: path of the file in testbed
        is_dir: whether path is a directory; None for "unspecified" if you only
                need copydown()
        '''
        self.testbed = testbed
        self.host = host
        self.tb = tb
        self.is_dir = is_dir

    def copydown(self, check_existing=False):
        '''Copy file from the host to the testbed

        If check_existing is True, don't copy if the testbed path already
        exists.
        '''
        if check_existing and testbed.execute(['test', '-e', self.tb])[0] == 0:
            adtlog.debug('copydown: tb path %s already exists' % self.tb)
            return

        # create directory on testbed
        testbed.check_exec(['mkdir', '-p', os.path.dirname(self.tb)])

        if os.path.isdir(self.host):
            # directories need explicit '/' appended for VirtSubproc
            testbed.command('copydown', (self.host + '/', self.tb + '/'))
        else:
            testbed.command('copydown', (self.host, self.tb))

        # we usually want our files be readable for the non-root user
        if opts.user:
            rc = testbed.execute(['chown', '-R', opts.user, '--', self.tb],
                                 stderr=subprocess.PIPE)[0]
            if rc != 0:
                # chowning doesn't work on all shared downtmps, try to chmod
                # instead
                testbed.check_exec(['chmod', '-R', 'go+rwX', '--', self.tb])

    def copyup(self, check_existing=False):
        '''Copy file from the testbed to the host

        If check_existing is True, don't copy if the host path already
        exists.
        '''
        if check_existing and os.path.exists(self.host):
            adtlog.debug('copyup: host path %s already exists' % self.host)
            return

        mkdir_okexist(os.path.dirname(self.host))
        assert self.is_dir is not None
        if self.is_dir:
            testbed.command('copyup', (self.tb + '/', self.host + '/'))
        else:
            testbed.command('copyup', (self.tb, self.host))


class TempTestbedPath(TestbedPath):
    '''Represent a file in the hosts'/testbed's temporary directories

    These are only guaranteed to exit within one testbed run.
    '''
    def __init__(self, testbed, name, is_dir=False, autoclean=True):
        '''Create a temporary TestbedPath object.

        The object itself is just a pair of file names, nothing more. They do
        not need to exist until you call copyup() or copydown() on them.

        testbed: the Testbed object which this refers to
        name: name of the temporary file (without path); host and tb
              will then be derived from that
        is_dir: whether path is a directory; None for "unspecified" if you only
                need copydown()
        autoclean: If True (default), remove file when adt-run finishes. Should
                be set to False for files which you want to keep in the
                --output-dir which are useful for reporting results, like test
                stdout/err, log files, and binaries.
        '''
        # if the testbed supports a shared downtmp, use that to avoid
        # unnecessary copying, unless we want to permanently keep the file
        if shared_downtmp and (not opts.output_dir or autoclean):
            host = shared_downtmp
        else:
            host = tmp
        TestbedPath.__init__(self, testbed, os.path.join(host, name),
                             os.path.join(testbed.scratch, name),
                             is_dir)
        self.autoclean = autoclean

    def __del__(self):
        if self.autoclean:
            if os.path.exists(self.host):
                try:
                    os.unlink(self.host)
                except OSError as e:
                    if e.errno == errno.EPERM:
                        testbed.check_exec(['rm', '-rf', self.tb])
                    else:
                        raise


def run_tests(tests, tree):
    global errorcode, testbed

    if not tests:
        # if we have skipped tests, don't claim that we don't have any
        if not errorcode & 2:
            adtlog.report('*', 'SKIP no tests in this package')
            errorcode |= 8
        return

    for t in tests:
        testbed.run_test(tree, t)
        if 'breaks-testbed' in t.restrictions:
            testbed.needs_reset()

    testbed.needs_reset()


def print_exception(ei, msgprefix=''):
    if msgprefix:
        adtlog.error(msgprefix)
    (et, q, tb) = ei
    if et is Quit:
        adtlog.error(q.m)
        adtlog.psummary('quitting: ' + q.m)
        return q.ec
    else:
        adtlog.error('unexpected error:')
        adtlog.psummary('quitting: unexpected error, consult transcript')
        traceback.print_exc(None, sys.stderr)
        return 20


def cleanup():
    try:
        if testbed is not None:
            testbed.reset_apt()
            testbed.stop()
        if opts.output_dir is None and tmp is not None:
            rmtree('tmp', tmp)
    except:
        print_exception(sys.exc_info(),
                        '\nadt-run: error cleaning up:\n')
        sys.exit(20)


def signal_handler(signum, frame):
    adtlog.error('Received signal %i, cleaning up...' % signum)
    signal.signal(signum, signal.SIG_DFL)
    try:
        # don't call cleanup() here, resetting apt takes too long
        if testbed:
            testbed.stop()
    finally:
        os.kill(os.getpid(), signum)


# ---------- registration, installation etc. of .deb's: Binaries


class Binaries:

    def __init__(self, tb):
        # the binary dir must exist across tb reopenings, so don't use a
        # TempTestbedPath
        self.dir = TestbedPath(tb,
                               os.path.join(tmp, 'binaries'),
                               os.path.join(tb.scratch, 'binaries'),
                               is_dir=True)
        os.mkdir(self.dir.host)
        # clean up an empty binaries output dir
        atexit.register(lambda: os.path.exists(self.dir.host) and (
            os.listdir(self.dir.host) or os.rmdir(self.dir.host)))

        adtlog.debug('Binaries: initialising')
        self.apt_get_cmd = ['apt-get', '--quiet',
                            '-o', 'Debug::pkgProblemResolver=true ',
                            '-o', 'APT::Get::force-yes=true',
                            '-o', 'APT::Get::Assume-Yes=true']

    def genkey(self):
        adtlog.debug('Binaries: preparing for key generation')

        if not os.path.exists(os.path.dirname(opts.gnupghome)):
            os.makedirs(os.path.dirname(opts.gnupghome))
        mkdir_okexist(opts.gnupghome, 0o700)

        script = '''
  exec >&2
  cd "$1"
  cat <<"END" >key-gen-params
Key-Type: DSA
Key-Length: 1024
Key-Usage: sign
Name-Real: autopkgtest per-run key
Name-Comment: do not trust this key
Name-Email: autopkgtest@example.com
END
  set -x
  gpg --homedir="$1" --batch --no-random-seed-file --gen-key key-gen-params
'''
        cmdl = ['sh', '-ec', script, 'x', opts.gnupghome]
        rc = script_out(cmdl, what='genkey', script=script)[0]
        if rc:
            bomb('key generation failed, code %d' % rc)

    def _configure_apt(self, tb):
        prefs = TestbedPath(testbed, os.path.join(tmp, 'apt-prefs'),
                            '/etc/apt/preferences.d/90autopkgtest')
        with open(prefs.host, 'w') as f:
            f.write('''Package: *
Pin: origin ""
Pin-Priority: 1002
''')
        prefs.copydown()
        os.unlink(prefs.host)

    def reset(self):
        adtlog.debug('Binaries: reset')
        rmtree('binaries', self.dir.host)
        os.mkdir(self.dir.host)
        self.blamed = []
        self.registered = set()

    def register(self, path, pkgname):
        adtlog.debug('Binaries: register deb=%s pkgname=%s ' % (path, pkgname))
        self.blamed += testbed.blamed

        dest = os.path.join(self.dir.host, pkgname + '.deb')

        # link or copy to self.dir
        try:
            os.remove(dest)
        except (IOError, OSError) as oe:
            if oe.errno != errno.ENOENT:
                raise oe
        try:
            os.link(path, dest)
        except (IOError, OSError) as oe:
            if oe.errno != errno.EXDEV:
                raise oe
            shutil.copy(path, dest)
        # clean up locally built debs (what=ubtreeN) to keep a clean
        # --output-dir, but don't clean up --binary arguments
        if opts.output_dir and path.startswith(opts.output_dir):
            atexit.register(lambda f: os.path.exists(f) and os.unlink(f), path)
        self.registered.add(pkgname)

    def publish(self):
        adtlog.debug('Binaries: publish')
        if not self.registered:
            adtlog.debug('Binaries: no registered binaries, not publishing anything')
            return

        if os.path.exists(os.path.join(opts.gnupghome, 'secring.gpg')):
            adtlog.debug('Binaries: no key generation needed')
        else:
            self.genkey()

        self._configure_apt(testbed)

        script = '''
  exec >&2
  cd "$1"
  apt-ftparchive packages . >Packages
  gzip <Packages >Packages.gz
  apt-ftparchive release . >Release
  rm -f Release.gpg
  gpg --homedir="$2" --batch --no-random-seed-file --detach-sign --armour -o Release.gpg Release
  gpg --homedir="$2" --batch --no-random-seed-file --export >archive-key.pgp
'''
        cmdl = ['sh', '-ec', script, 'x', self.dir.host, opts.gnupghome]
        rc = script_out(cmdl, what='ftparchive', script=script)[0]
        if rc:
            bomb('apt-ftparchive or signature failed, code %d' % rc)

        # copy binaries directory to testbed; self.dir.tb might have changed
        # since last time due to a reset, so update it
        self.dir.tb = os.path.join(testbed.scratch, 'binaries')
        testbed.check_exec(['rm', '-rf', self.dir.tb])
        self.dir.copydown()

        aptkey_out = TempTestbedPath(testbed, 'apt-key.out')
        script = '''
  apt-key add %(d)s/archive-key.pgp
  echo "deb file://%(d)s /" >/etc/apt/sources.list.d/autopkgtest.list
  if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
    echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
  fi
  apt-get --quiet --no-list-cleanup -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/autopkgtest.list -o Dir::Etc::sourceparts=/dev/null update 2>&1
  cp /var/lib/dpkg/status %(o)s
  ''' % {'d': self.dir.tb, 'o': aptkey_out.tb}
        testbed.mungeing_apt()
        testbed.check_exec(['sh', '-ec', script], kind='install')
        testbed.blamed += self.blamed

        aptkey_out.copyup()

        adtlog.debug('Binaries: publish reinstall checking...')
        pkgs_reinstall = set()
        pkg = None
        for l in open(aptkey_out.host, encoding='UTF-8'):
            if l.startswith('Package: '):
                pkg = l[9:].rstrip()
            elif l.startswith('Status: install '):
                if pkg in self.registered:
                    pkgs_reinstall.add(pkg)
                    adtlog.debug('Binaries: publish reinstall needs ' + pkg)

        if pkgs_reinstall:
            for pkg in pkgs_reinstall:
                testbed.blame(pkg)
            rc = testbed.execute(
                self.apt_get_cmd + ['--reinstall', 'install'] + list(pkgs_reinstall),
                kind='install')[0]
            if rc:
                badpkg('installation of basic binaries failed, exit code %d' % rc)

        adtlog.debug('Binaries: publish done')

# ---------- processing of sources (building)


def deb_package_name(deb):
    '''Return package name from a .deb'''

    cmd = ['dpkg-deb', '--info', '--', deb, 'control']
    (rc, output) = script_out(cmd, stdout=subprocess.PIPE)
    if rc:
        badpkg('failed to parse binary package, code %d' % rc)
    pkg_re = re.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
    result = None
    for l in output.split('\n'):
        m = pkg_re.match(l)
        if not m:
            continue
        if result:
            badpkg('two Package: lines in control file')
        result = m.groups()[0]
    if not result:
        badpkg('no good Package: line in control file')
    return result


def source_rules_command(script, which, cwd=None, results_lines=0):
    if cwd is None:
        cwd = '.'
    if opts.verbosity > 1:
        script = ['exec 3>&1 >&2', 'set -x', 'cd ' + cwd] + script
    else:
        script = ['exec 3>&1 >&2', 'cd ' + cwd] + script
    script = opts.user_wrap('; '.join(script))
    (rc, out, _) = testbed.execute(['sh', '-ec', script],
                                   stdout=subprocess.PIPE,
                                   kind='build')
    results = out.rstrip('\n').splitlines()
    if rc:
        if opts.shell_fail:
            testbed.run_shell()
        badpkg('rules %s failed with exit code %d' % (which, rc))
    if results_lines is not None and len(results) != results_lines:
        badpkg('got %d lines of results from %s where %d expected'
               % (len(results), which, results_lines))
    if results_lines == 1:
        return results[0]
    return results


def build_source(kind, arg, built_binaries):
    '''Prepare action argument for testing

    This builds packages when necessary and registers their binaries, copies
    tests into the testbed, etc.

    Return a TestbedPath to the unpacked tests tree.
    '''
    testbed.blame(arg)
    testbed.reset([], testbed.recommends_installed)

    def debug_b(m):
        adtlog.debug('build_source: <%s:%s> %s' % (kind, arg, m))

    # copy necessary source files into testbed and set create_command for final unpacking
    if kind == 'source':
        dsc = arg
        dsc_tb = os.path.join(testbed.scratch, os.path.basename(dsc))

        # copy .dsc file itself
        TestbedPath(testbed, dsc, dsc_tb).copydown()
        # copy files from it
        for part in files_from_dsc(dsc):
            p = TestbedPath(testbed, part, os.path.join(testbed.scratch, os.path.basename(part)))
            p.copydown()

        create_command = 'dpkg-source -x "%s"' % dsc_tb

    elif kind == 'unbuilt-tree':
        dsc = os.path.join(tmp, 'fake.dsc')
        with open(dsc, 'w', encoding='UTF-8') as f_dsc:
            with open(os.path.join(arg, 'debian/control'), encoding='UTF-8') as f_control:
                for l in f_control:
                    if l == '\n':
                        break
                    f_dsc.write(l)
            f_dsc.write('Binary: none-so-this-is-not-a-package-name\n')
        atexit.register(lambda f: os.path.exists(f) and os.unlink(f), dsc)

        # copy unbuilt tree into testbed
        ubtree = TestbedPath(testbed, arg,
                             os.path.join(testbed.scratch, 'ubtree-' + os.path.basename(arg)))
        ubtree.copydown()
        create_command = 'cp -rd --preserve=timestamps -- "%s" real-tree' % ubtree.tb

    elif kind == 'built-tree':
        # this is a special case: we don't want to build, or even copy down
        # (and back up) the tree here for efficiency; so shortcut everything
        # below and just set the tests_tree and get the package version
        tests_tree = TestbedPath(testbed, arg, os.path.join(testbed.scratch, 'tree'), is_dir=True)

        with open(os.path.join(arg, 'debian', 'changelog'), encoding='UTF-8') as f:
            (testpkg_name, testpkg_version, _) = f.readline().split(' ', 2)
            testpkg_version = testpkg_version[1:-1]  # chop off parentheses

        adtlog.info('testing package %s version %s' % (testpkg_name, testpkg_version))
        if opts.output_dir:
            with open(os.path.join(tmp, 'testpkg-version'), 'w') as f:
                f.write('%s %s\n' % (testpkg_name, testpkg_version))
        return tests_tree

    elif kind == 'apt-source':
        # apt-get source is terribly noisy; only show what gets downloaded
        create_command = 'OUT=$(apt-get source -q --only-source %s); echo "$OUT" | grep ^Get: || true' % arg

    else:
        bomb('unknown action kind for build_source: ' + kind)

    if kind in ['source', 'apt-source']:
        testbed.install_deps([], False)
        if testbed.execute(['which', 'dpkg-source'],
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE) != 0:
            adtlog.debug('dpkg-source not available in testbed, installing dpkg-dev')
            # Install dpkg-source for unpacking .dsc
            testbed.satisfy_dependencies_string('dpkg-dev',
                                                'install dpkg-dev')

    # run create_command
    script = [
        'builddir=$(mktemp -d %s/build.XXX)' % testbed.scratch,
        'cd $builddir',
        create_command,
        'chmod -R a+rX .',
        'cd [a-z0-9]*/.',
        'pwd >&3',
        'sed -n "1 {s/).*//; s/ (/\\n/; p}" debian/changelog >&3',
        'set +e; grep -q "^Restrictions:.*\\bbuild-needed\\b" debian/tests/control 2>/dev/null; echo $? >&3'
    ]

    (result_pwd, testpkg_name, testpkg_version, build_needed_rc) = \
        source_rules_command(script, 'extract', results_lines=4)

    # record tested package version
    adtlog.info('testing package %s version %s' % (testpkg_name, testpkg_version))
    if opts.output_dir:
        with open(os.path.join(tmp, 'testpkg-version'), 'w') as f:
            f.write('%s %s\n' % (testpkg_name, testpkg_version))

    # For optional builds:
    #
    # We might need to build the package because:
    #   - we want its binaries
    #   - the test control file says so (assuming we have any tests)

    build_needed = False
    if built_binaries:
        adtlog.info('build needed for binaries')
        build_needed = True
    elif build_needed_rc == '0':
        adtlog.info('build needed for tests')
        build_needed = True
    else:
        adtlog.info('build not needed')

    if build_needed:
        testbed.needs_reset()
        if kind not in ['dsc', 'apt-source']:
            testbed.install_deps([], False)

        if kind == 'apt-source':
            # we need to get the downloaded debian/control from the testbed, so
            # that we can avoid calling "apt-get build-dep" and thus
            # introducing a second mechanism for installing build deps
            pkg_control = TestbedPath(testbed,
                                      os.path.join(tmp, 'apt-control'),
                                      os.path.join(result_pwd, 'debian/control'), False)
            pkg_control.copyup()
            dsc = pkg_control.host

        with open(dsc, encoding='UTF-8') as f:
            d = deb822.Deb822(sequence=f)
            bd = d.get('Build-Depends', '')
            bdi = d.get('Build-Depends-Indep', '')
        testbed.satisfy_dependencies_string(bd + ', ' + bdi + ', ' + ', '.join(build_essential), arg)

        source_rules_command([dpkg_buildpackage], 'build', cwd=result_pwd)

    # copy built tree from testbed to hosts
    tests_tree = TestbedPath(testbed, os.path.join(tmp, 'tests-tree'), result_pwd, is_dir=True)
    atexit.register(rmtree, 'tests-tree', tests_tree.host)
    tests_tree.copyup()

    if not build_needed:
        return tests_tree

    if built_binaries:
        debug_b('want built binaries, getting and registering built debs')
        script = [
            'cd ..',
            'echo *.deb >&3',
        ]
        result_debs = source_rules_command(script, 'binary', cwd=result_pwd,
                                           results_lines=1)
        if result_debs == '*.deb':
            debs = []
        else:
            debs = result_debs.split()
        debug_b('debs=' + repr(debs))

        # determine built debs and copy them from testbed
        deb_re = re.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
        for deb in debs:
            m = deb_re.match(deb)
            if not m:
                badpkg("badly-named binary `%s'" % deb)
            pkgname = m.groups()[0]
            debug_b(' deb=%s, pkgname=%s' % (deb, pkgname))
            deb_path = TestbedPath(testbed,
                                   os.path.join(tmp, os.path.basename(deb)),
                                   os.path.join(result_pwd, '..', deb),
                                   False)
            deb_path.copyup()
            binaries.register(deb_path.host, pkgname)
        debug_b('got all built binaries')

    return tests_tree


def process_actions():
    global actions, binaries, errorcode

    binaries = Binaries(testbed)
    binaries.reset()
    control_override = None
    pending_click_source = None

    for (kind, arg, built_binaries) in actions:
        # non-tests/build actions
        if kind == 'override-control':
            control_override = arg
            continue
        if kind == 'binary':
            testbed.blame('arg:' + arg)
            pkg = deb_package_name(arg)
            testbed.blame('deb:' + pkg)
            binaries.register(arg, pkg)
            continue
        if kind == 'click-source':
            if pending_click_source:
                adtlog.warning('Ignoring --click-source %s, no subsequent --click argument' % pending_click_source)
            pending_click_source = arg
            continue

        # tests/build actions
        assert kind in ('source', 'unbuilt-tree', 'built-tree', 'apt-source', 'click')
        adtlog.info('@@@@@@@@@@@@@@@@@@@@ %s %s' % (kind, arg))

        if kind == 'click':
            if control_override:
                # locally specified manifest
                with open(control_override) as f:
                    manifest = f.read()
                clicks = []
                if os.path.exists(arg):
                    clicks.append(arg)
                (srcdir, tests, skipped) = testdesc.parse_click_manifest(
                    manifest, testbed.caps, clicks, pending_click_source)

            elif os.path.exists(arg):
                # local .click package file
                (srcdir, tests, skipped) = testdesc.parse_click(
                    arg, testbed.caps, srcdir=pending_click_source)
            else:
                # already installed click package name
                if opts.user:
                    u = ['--user', opts.user]
                else:
                    u = []
                manifest = testbed.check_exec(['click', 'info'] + u + [arg], stdout=True)
                (srcdir, tests, skipped) = testdesc.parse_click_manifest(
                    manifest, testbed.caps, [], pending_click_source)

            if not srcdir:
                bomb('No click source available for %s' % arg)

            tests_tree = TestbedPath(
                testbed, srcdir, os.path.join(testbed.scratch, 'tree'),
                is_dir=True)
            pending_click_source = None
        else:
            tests_tree = build_source(kind, arg, built_binaries)
            (tests, skipped) = testdesc.parse_debian_source(
                tests_tree.host, testbed.caps, control_path=control_override)

        if skipped:
            errorcode |= 2
        control_override = None
        run_tests(tests, tests_tree)


def main():
    global testbed, opts, vserver_args, actions
    try:
        (opts, actions, vserver_args) = adt_run_args.parse_args()
    except SystemExit:
        # argparser exits with error 2 by default, but we have a different
        # meaning for that already
        sys.exit(20)

    # ensure proper cleanup on signals
    signal.signal(signal.SIGTERM, signal_handler)
    signal.signal(signal.SIGQUIT, signal_handler)

    try:
        setup_trace()
        testbed = Testbed()
        testbed.start()
        testbed.open()
        finalise_options()
        process_actions()
    except:
        ec = print_exception(sys.exc_info(), '')
        cleanup()
        sys.exit(ec)
    cleanup()
    sys.exit(errorcode)

main()