[reprotest] 03/05: Import autopkgtest 4.4

Ximin Luo infinity0 at debian.org
Wed Jul 19 15:13:12 UTC 2017


This is an automated email from the git hooks/post-receive script.

infinity0 pushed a commit to branch master
in repository reprotest.

commit b473281e8883dbf2efb7c637089bde81d3face34
Author: Ximin Luo <infinity0 at debian.org>
Date:   Wed Jul 19 17:07:50 2017 +0200

    Import autopkgtest 4.4
---
 reprotest/lib/VirtSubproc.py            |  25 +++--
 reprotest/lib/adt_testbed.py            | 159 +++++++++++++++++------------
 reprotest/virt/autopkgtest-virt-chroot  |   8 +-
 reprotest/virt/autopkgtest-virt-lxc     |  20 ++--
 reprotest/virt/autopkgtest-virt-lxd     |  48 +++++++--
 reprotest/virt/autopkgtest-virt-null    |   8 +-
 reprotest/virt/autopkgtest-virt-qemu    | 173 +++++++++++++++++---------------
 reprotest/virt/autopkgtest-virt-schroot |  14 +--
 reprotest/virt/autopkgtest-virt-ssh     |  71 +++++++++----
 9 files changed, 321 insertions(+), 205 deletions(-)

diff --git a/reprotest/lib/VirtSubproc.py b/reprotest/lib/VirtSubproc.py
index c37051d..ff1ef28 100644
--- a/reprotest/lib/VirtSubproc.py
+++ b/reprotest/lib/VirtSubproc.py
@@ -38,9 +38,9 @@ import shutil
 from reprotest.lib import adtlog
 
 progname = "<VirtSubproc>"
-devnull_read = open('/dev/null', 'r')
+devnull_read = open('/dev/null', 'rb')
 caller = __main__
-copy_timeout = int(os.getenv('ADT_VIRT_COPY_TIMEOUT', '300'))
+copy_timeout = int(os.getenv('AUTOPKGTEST_VIRT_COPY_TIMEOUT', '300'))
 
 downtmp_open = None  # downtmp after opening testbed
 downtmp = None  # current downtmp (None after close)
@@ -159,7 +159,7 @@ def execute_timeout(instr, timeout, *popenargs, **popenargsk):
     return (status, out, err)
 
 
-def check_exec(argv, downp=False, outp=False, timeout=0):
+def check_exec(argv, downp=False, outp=False, timeout=0, fail_on_stderr=True):
     '''Run successful command (argv list)
 
     Command must succeed (exit code 0) and not produce any stderr. If downp is
@@ -183,9 +183,9 @@ def check_exec(argv, downp=False, outp=False, timeout=0):
                                          stdout=stdout, stderr=subprocess.PIPE)
 
     if status:
-        bomb("%s%s failed (exit status %d)\n%s" %
+        bomb("%s%s failed (exit status %d, stderr %r)" %
              ((downp and "(down) " or ""), argv, status, err))
-    if err:
+    if fail_on_stderr and err:
         bomb("%s unexpectedly produced stderr output `%s'" %
              (argv, err))
 
@@ -230,7 +230,7 @@ def get_unix_socket(path):
 
 
 def expect(sock, search_bytes, timeout_sec, description=None, echo=False):
-    adtlog.debug('expect: "%s"' % search_bytes.decode())
+    adtlog.debug('expect: "%s"' % (search_bytes or b'<none>').decode())
     what = '"%s"' % (description or search_bytes or 'data')
     out = b''
     with timeout(timeout_sec,
@@ -512,9 +512,9 @@ def copyupdown_internal(wh, sd, upp):
     if not dirsp:
         rune = 'cat %s%s' % ('><'[upp], remfileq)
         if upp:
-            deststdout = open(sd[idst], 'w')
+            deststdout = open(sd[idst], 'wb')
         else:
-            srcstdin = open(sd[isrc], 'r')
+            srcstdin = open(sd[isrc], 'rb')
             status = os.fstat(srcstdin.fileno())
             if status.st_mode & 0o111:
                 rune += '; chmod +x -- %s' % (remfileq)
@@ -651,6 +651,7 @@ def command():
         r = fc.e
     print(' '.join(r))
 
+
 signal_list = [	signal.SIGHUP, signal.SIGTERM,
                 signal.SIGINT, signal.SIGPIPE]
 
@@ -699,6 +700,14 @@ def prepare():
     sethandlers(handler)
 
 
+def cmd_auxverb_debug_fail(c, ce):
+    cmdnumargs(c, ce)
+    try:
+        adtlog.info(caller.hook_debug_fail())
+    except AttributeError:
+        pass
+
+
 def mainloop():
     global in_mainloop
     in_mainloop = True
diff --git a/reprotest/lib/adt_testbed.py b/reprotest/lib/adt_testbed.py
index 7a55be3..61d120b 100644
--- a/reprotest/lib/adt_testbed.py
+++ b/reprotest/lib/adt_testbed.py
@@ -33,12 +33,7 @@ import tempfile
 import shutil
 import urllib.parse
 
-
-# TODO: removing this import disables install_tmp, may want to restore
-# it at some point if I'm improving support for building Debian packages in
-# particular.
-
-# from debian import debian_support
+from debian import debian_support
 
 from reprotest.lib import adtlog
 from reprotest.lib import VirtSubproc
@@ -50,7 +45,8 @@ timeouts = {'short': 100, 'copy': 300, 'install': 3000, 'test': 10000,
 
 class Testbed:
     def __init__(self, vserver_argv, output_dir, user,
-                 setup_commands=[], add_apt_pockets=[], copy_files=[]):
+                 setup_commands=[], setup_commands_boot=[], add_apt_pockets=[],
+                 copy_files=[]):
         self.sp = None
         self.lastsend = None
         self.scratch = None
@@ -65,6 +61,7 @@ class Testbed:
         self.install_tmp_env = []
         self.user = user
         self.setup_commands = setup_commands
+        self.setup_commands_boot = setup_commands_boot
         self.add_apt_pockets = add_apt_pockets
         self.copy_files = copy_files
         self.initial_kernel_version = None
@@ -148,7 +145,7 @@ class Testbed:
                           '''/bin/echo -e '#!/bin/sh -e\\n'''
                           '''[ -n "$1" ] || { echo "Usage: $0 <mark>" >&2; exit 1; }\\n'''
                           '''echo "$1" > /run/autopkgtest-reboot-mark\\n'''
-                          '''test_script_pid=$(cat /tmp/adt_test_script_pid)\\n'''
+                          '''test_script_pid=$(cat /tmp/autopkgtest_script_pid)\\n'''
                           '''p=$PPID; while true; do read _ c _ pp _ < /proc/$p/stat;'''
                           '''  [ $pp -ne $test_script_pid ] || break; p=$pp; done\\n'''
                           '''kill -KILL $p\\n' > /tmp/autopkgtest-reboot;'''
@@ -160,7 +157,7 @@ class Testbed:
                           '''/bin/echo -e '#!/bin/sh -e\\n'''
                           '''[ -n "$1" ] || { echo "Usage: $0 <mark>" >&2; exit 1; }\\n'''
                           '''echo "$1" > /run/autopkgtest-reboot-prepare-mark\\n'''
-                          '''test_script_pid=$(cat /tmp/adt_test_script_pid)\\n'''
+                          '''test_script_pid=$(cat /tmp/autopkgtest_script_pid)\\n'''
                           '''kill -KILL $test_script_pid\\n'''
                           '''while [ -e /run/autopkgtest-reboot-prepare-mark ]; do sleep 0.5; done\\n'''
                           ''' '> /tmp/autopkgtest-reboot-prepare;'''
@@ -191,6 +188,21 @@ class Testbed:
             if m:
                 self.cpu_flags = m.group(2)
 
+        xenv = ['AUTOPKGTEST_IS_SETUP_BOOT_COMMAND=1']
+        if self.user:
+            xenv.append('AUTOPKGTEST_NORMAL_USER=' + self.user)
+            xenv.append('ADT_NORMAL_USER=' + self.user)
+
+        for c in self.setup_commands_boot:
+            rc = self.execute(['sh', '-ec', c], xenv=xenv, kind='install')[0]
+            if rc:
+                # setup scripts should exit with 100 if it's the package's
+                # fault, otherwise it's considered a transient testbed failure
+                if rc == 100:
+                    self.badpkg('testbed boot setup commands failed with status 100')
+                else:
+                    self.bomb('testbed boot setup commands failed with status %i' % rc)
+
     def _opened(self, pl):
         self.scratch = pl[0]
         self.deps_installed = []
@@ -260,7 +272,7 @@ class Testbed:
         # create apt sources for --apt-pocket
         for pocket in self.add_apt_pockets:
             pocket = pocket.split('=', 1)[0]  # strip off package list
-            script = '''sed -rn 's/^(deb|deb-src) +(\[.*\] *)?([^ ]*(ubuntu.com|debian.org|ftpmaster|file:\/\/\/tmp\/testarchive)[^ ]*) +([^ -]+) +(.*)$/\\1 \\2\\3 \\5-%s \\6/p' /etc/apt/sources.list `ls /etc/apt/sources.list.d/*.list 2>/dev/null|| true)` > /etc/apt/sources.list.d/%s.list; for retry in 1 2 3; do apt-get --no-list-cleanup -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/%s.list -o Dir::Etc::sourceparts=/dev/null update 2>&1 && break || sleep 15; done''' % (pocket, pocket, pocket)
+            script = '''sed -rn 's/^(deb|deb-src) +(\[.*\] *)?([^ ]*(ubuntu.com|debian.org|ftpmaster|file:\/\/\/tmp\/testarchive)[^ ]*) +([^ -]+) +(.*)$/\\1 \\2\\3 \\5-%s \\6/p' /etc/apt/sources.list `ls /etc/apt/sources.list.d/*.list 2>/dev/null|| true` > /etc/apt/sources.list.d/%s.list; for retry in 1 2 3; do apt-get --no-list-cleanup -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/%s.list -o Dir::Etc::sourceparts=/dev/null update 2>&1 && break || sleep 15; done''' % (pocket, pocket, pocket)
             self.check_exec(['sh', '-ec', script])
 
         # create apt pinning for --apt-pocket with package list
@@ -278,14 +290,20 @@ class Testbed:
                          'for d in %s; do [ ! -d $d ] || touch -r $d %s/${d//\//_}.stamp; done' % (
                              boot_dirs, self.scratch)])
 
-        xenv = ['ADT_IS_SETUP_COMMAND=1']
+        xenv = ['AUTOPKGTEST_IS_SETUP_COMMAND=1']
         if self.user:
+            xenv.append('AUTOPKGTEST_NORMAL_USER=' + self.user)
             xenv.append('ADT_NORMAL_USER=' + self.user)
 
         for c in self.setup_commands:
             rc = self.execute(['sh', '-ec', c], xenv=xenv, kind='install')[0]
             if rc:
-                self.bomb('testbed setup commands failed with status %i' % rc)
+                # setup scripts should exit with 100 if it's the package's
+                # fault, otherwise it's considered a transient testbed failure
+                if rc == 100:
+                    self.badpkg('testbed setup commands failed with status 100')
+                else:
+                    self.bomb('testbed setup commands failed with status %i' % rc)
 
         # if the setup commands affected the boot, then reboot
         if self.setup_commands and 'reboot' in self.caps:
@@ -312,7 +330,7 @@ class Testbed:
             self._opened(pl)
         self.modified = False
 
-    def install_deps(self, deps_new, recommends):
+    def install_deps(self, deps_new, recommends, shell_on_failure=False):
         '''Install dependencies into testbed'''
         adtlog.debug('install_deps: deps_new=%s, recommends=%s' % (deps_new, recommends))
 
@@ -320,7 +338,7 @@ class Testbed:
         self.recommends_installed = recommends
         if not deps_new:
             return
-        self.satisfy_dependencies_string(', '.join(deps_new), 'install-deps', recommends)
+        self.satisfy_dependencies_string(', '.join(deps_new), 'install-deps', recommends, shell_on_failure=shell_on_failure)
 
     def needs_reset(self):
         # show what caused a reset
@@ -331,7 +349,7 @@ class Testbed:
 
     def bomb(self, m, _type=adtlog.TestbedFailure):
         adtlog.debug('%s %s' % (_type.__name__, m))
-        # self.stop()
+        self.stop()
         raise _type(m)
 
     def badpkg(self, m):
@@ -350,28 +368,28 @@ class Testbed:
                       format_exception_only(type, value))
 
     def expect(self, keyword, nresults):
-        l = self.sp.stdout.readline()
-        if not l:
+        line = self.sp.stdout.readline()
+        if not line:
             self.bomb('unexpected eof from the testbed')
-        if not l.endswith('\n'):
+        if not line.endswith('\n'):
             self.bomb('unterminated line from the testbed')
-        l = l.rstrip('\n')
-        adtlog.debug('got reply from testbed: ' + l)
-        ll = l.split()
+        line = line.rstrip('\n')
+        adtlog.debug('got reply from testbed: ' + line)
+        ll = line.split()
         if not ll:
             self.bomb('unexpected whitespace-only line from the testbed')
         if ll[0] != keyword:
             if self.lastsend is None:
                 self.bomb("got banner `%s', expected `%s...'" %
-                          (l, keyword))
+                          (line, keyword))
             else:
                 self.bomb("sent `%s', got `%s', expected `%s...'" %
-                          (self.lastsend, l, keyword))
+                          (self.lastsend, line, keyword))
         ll = ll[1:]
         if nresults is not None and len(ll) != nresults:
             self.bomb("sent `%s', got `%s' (%d result parameters),"
                       " expected %d result parameters" %
-                      (self.lastsend, l, len(ll), nresults))
+                      (self.lastsend, line, len(ll), nresults))
         return ll
 
     def command(self, cmd, args=(), nresults=0, unquote=True):
@@ -389,13 +407,10 @@ class Testbed:
             ll = list(map(urllib.parse.unquote, ll))
         return ll
 
-    # TODO: with stdout and stderr defaulting to None, this function
-    # eats all errors/output from its call, which is not the right
-    # thing.
     def execute(self, argv, xenv=[], stdout=None, stderr=None, kind='short'):
         '''Run command in testbed.
 
-        The commands stdout/err will be piped directly to adt-run and its log
+        The commands stdout/err will be piped directly to autopkgtest and its log
         files, unless redirection happens with the stdout/stderr arguments
         (passed to Popen).
 
@@ -416,7 +431,6 @@ class Testbed:
         if env:
             argv = ['env'] + env + argv
 
-        # import pdb; pdb.set_trace()
         VirtSubproc.timeout_start(timeouts[kind])
         try:
             proc = subprocess.Popen(self.exec_cmd + argv,
@@ -446,16 +460,12 @@ class Testbed:
         adtlog.debug('testbed command exited with code %i' % proc.returncode)
 
         if proc.returncode in (254, 255):
-            msg = 'testbed auxverb failed with exit code %i' % proc.returncode
-            if out:
-                msg += '\n---- stdout ----\n%s----------------\n' % out
-            if err:
-                msg += '\n---- stderr ----\n%s----------------\n' % err
-            self.bomb(msg)
+            self.command('auxverb_debug_fail')
+            self.bomb('testbed auxverb failed with exit code %i' % proc.returncode)
 
         return (proc.returncode, out, err)
 
-    def check_exec(self, argv, stdout=False, kind='short', xenv=[]):
+    def check_exec(self, argv, stdout=False, kind='short'):
         '''Run argv in testbed.
 
         If stdout is True, capture stdout and return it. Otherwise, don't
@@ -464,7 +474,6 @@ class Testbed:
         argv must succeed and not print any stderr.
         '''
         (code, out, err) = self.execute(argv,
-                                        xenv=xenv,
                                         stdout=(stdout and subprocess.PIPE or None),
                                         stderr=subprocess.PIPE, kind=kind)
         if err:
@@ -481,13 +490,13 @@ class Testbed:
         This requires root privileges and a writable file system.
         '''
         # create a dummy deb with the deps
-        pkgdir = tempfile.mkdtemp(prefix='adt-satdep.')
+        pkgdir = tempfile.mkdtemp(prefix='autopkgtest-satdep.')
         debdir = os.path.join(pkgdir, 'DEBIAN')
         os.chmod(pkgdir, 0o755)
         os.mkdir(debdir)
         os.chmod(debdir, 0o755)
         with open(os.path.join(debdir, 'control'), 'w') as f:
-            f.write('''Package: adt-satdep
+            f.write('''Package: autopkgtest-satdep
 Section: oldlibs
 Priority: extra
 Maintainer: autogenerated
@@ -497,7 +506,7 @@ Depends: %s
 Description: satisfy autopkgtest test dependencies
 ''' % (self.dpkg_arch, deps))
 
-        deb = TempPath(self, 'adt-satdep.deb')
+        deb = TempPath(self, 'autopkgtest-satdep.deb')
         subprocess.check_call(['dpkg-deb', '-b', pkgdir, deb.host],
                               stdout=subprocess.PIPE)
         shutil.rmtree(pkgdir)
@@ -514,6 +523,7 @@ Description: satisfy autopkgtest test dependencies
                                           '--assume-yes --fix-broken '
                                           '-o APT::Status-Fd=3 '
                                           '-o APT::Install-Recommends=%s '
+                                          '-o Dpkg::Options::=--force-confnew '
                                           '-o Debug::pkgProblemResolver=true 3>&2 2>&1' %
                                           (' '.join(self.eatmydata_prefix), recommends)],
                                          kind='install', stderr=subprocess.PIPE)
@@ -534,8 +544,8 @@ Description: satisfy autopkgtest test dependencies
                     self.run_shell()
             else:
                 # apt-get -f may succeed, but its solution might remove
-                # adt-satdep, which is still a failure
-                rc = self.execute(['dpkg', '--status', 'adt-satdep'],
+                # autopkgtest-satdep, which is still a failure
+                rc = self.execute(['dpkg', '--status', 'autopkgtest-satdep'],
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE)[0]
 
@@ -547,25 +557,27 @@ Description: satisfy autopkgtest test dependencies
                     self.check_exec(['/bin/sh', '-ec', 'rm /etc/apt/preferences.d/autopkgtest-*-' + pocket])
                     continue
 
+                if shell_on_failure:
+                    self.run_shell()
                 self.badpkg('Test dependencies are unsatisfiable. A common reason is '
                             'that your testbed is out of date with respect to the '
                             'archive, and you need to use a current testbed or run '
                             'apt-get update or use -U.')
             break
 
-        # remove adt-satdep to avoid confusing tests, but avoid marking our
+        # remove autopkgtest-satdep to avoid confusing tests, but avoid marking our
         # test dependencies for auto-removal
         out = self.check_exec(['apt-get', '--simulate', '--quiet',
                                '-o', 'APT::Get::Show-User-Simulation-Note=False',
                                '--auto-remove',
-                               'purge', 'adt-satdep'],
+                               'purge', 'autopkgtest-satdep'],
                               True)
         test_deps = []
         for line in out.splitlines():
             if not line.startswith('Purg '):
                 continue
             pkg = line.split()[1]
-            if pkg != 'adt-satdep':
+            if pkg != 'autopkgtest-satdep':
                 test_deps.append(pkg)
         if test_deps:
             adtlog.debug('Marking test dependencies as manually installed: %s' %
@@ -576,7 +588,7 @@ Description: satisfy autopkgtest test dependencies
                 self.check_exec(['apt-mark', 'manual', '-qq'] + test_deps[batch:batch + 20])
                 batch += 20
 
-        self.execute(['dpkg', '--purge', 'adt-satdep'])
+        self.execute(['dpkg', '--purge', 'autopkgtest-satdep'])
 
     def install_tmp(self, deps, recommends=False):
         '''Unpack dependencies into temporary directory
@@ -755,7 +767,7 @@ fi
         tp.copydown()
         # install it
         clickopts = ['--all-users']
-        if 'ADT_CLICK_NO_FRAMEWORK_CHECK' in os.environ:
+        if 'AUTOPKGTEST_CLICK_NO_FRAMEWORK_CHECK' in os.environ:
             # this is mostly for testing
             clickopts.append('--force-missing-framework')
         if 'root-on-testbed' in self.caps:
@@ -940,7 +952,7 @@ fi
 
         # create script to run test
         test_artifacts = '%s/%s-artifacts' % (self.scratch, test.name)
-        adttmp = '%s/adttmp' % (self.scratch)
+        autopkgtest_tmp = '%s/autopkgtest_tmp' % (self.scratch)
         assert self.nproc is not None
         script = 'set -e; ' \
                  'export USER=`id -nu`; ' \
@@ -948,20 +960,26 @@ fi
                  ' . ~/.profile >/dev/null 2>&1 || true; ' \
                  'buildtree="%(t)s"; ' \
                  'mkdir -p -m 1777 -- "%(a)s"; ' \
-                 'export ADT_ARTIFACTS="%(a)s"; ' \
-                 'mkdir -p -m 755 "%(tmp)s"; export ADTTMP="%(tmp)s" ' \
+                 'export AUTOPKGTEST_ARTIFACTS="%(a)s"; ' \
+                 'export ADT_ARTIFACTS="$AUTOPKGTEST_ARTIFACTS"; ' \
+                 'mkdir -p -m 755 "%(tmp)s"; export AUTOPKGTEST_TMP="%(tmp)s"; ' \
+                 'export ADTTMP="$AUTOPKGTEST_TMP"; ' \
                  'export DEBIAN_FRONTEND=noninteractive; ' \
                  'export LANG=C.UTF-8; ' \
                  '''export DEB_BUILD_OPTIONS=parallel=%(cpu)s; ''' \
                  'unset LANGUAGE LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE '\
                  '  LC_MONETARY LC_MESSAGES LC_PAPER LC_NAME LC_ADDRESS '\
                  '  LC_TELEPHONE LC_MEASUREMENT LC_IDENTIFICATION LC_ALL;' \
-                 'rm -f /tmp/adt_test_script_pid; set -C; echo $$ > /tmp/adt_test_script_pid; set +C; ' \
-                 'trap "rm -f /tmp/adt_test_script_pid" EXIT INT QUIT PIPE; '\
+                 'rm -f /tmp/autopkgtest_script_pid; set -C; echo $$ > /tmp/autopkgtest_script_pid; set +C; ' \
+                 'trap "rm -f /tmp/autopkgtest_script_pid" EXIT INT QUIT PIPE; '\
                  'cd "$buildtree"; '\
-                 % {'t': tree.tb, 'a': test_artifacts, 'tmp': adttmp,
+                 % {'t': tree.tb, 'a': test_artifacts, 'tmp': autopkgtest_tmp,
                     'cpu': build_parallel or self.nproc}
 
+        if 'needs-root' in test.restrictions and self.user is not None:
+            script += 'export AUTOPKGTEST_NORMAL_USER=%s; ' % self.user
+            script += 'export ADT_NORMAL_USER=%s; ' % self.user
+
         for e in extra_env:
             script += 'export \'%s\'; ' % e
         # there's no way to tell su to not reset $PATH, for install-tmp mode;
@@ -982,7 +1000,7 @@ fi
             test_cmd = "bash -ec '%s'" % test.command
 
         script += 'touch %(o)s %(e)s; ' \
-                  '%(t)s 2> >(tee -a %(e)s >&2) > >(tee -a %(o)s); ' \
+                  '%(t)s 2> >(tee -a %(e)s >&2) > >(tee -a %(o)s);' \
                   % {'t': test_cmd, 'o': so.tb, 'e': se.tb}
 
         if 'needs-root' not in test.restrictions and self.user is not None:
@@ -1015,7 +1033,7 @@ fi
         timeout = False
         while True:
             if self.last_reboot_marker:
-                script_prefix = 'export ADT_REBOOT_MARK="%s"; ' % self.last_reboot_marker
+                script_prefix = 'export AUTOPKGTEST_REBOOT_MARK="%s"; export ADT_REBOOT_MARK="$AUTOPKGTEST_REBOOT_MARK"; ' % self.last_reboot_marker
             else:
                 script_prefix = ''
             try:
@@ -1058,9 +1076,21 @@ fi
         adtlog.debug('testbed executing test finished with exit status %i' % rc)
 
         # copy stdout/err files to host
-        so.copyup()
-        se.copyup()
-        se_size = os.path.getsize(se.host)
+        try:
+            so.copyup()
+            se.copyup()
+            se_size = os.path.getsize(se.host)
+        except adtlog.TestbedFailure:
+            if timeout:
+                # if the test timed out, it's likely that the test destroyed
+                # the testbed, so ignore this and call it a failure
+                adtlog.warning('Copying up test output timed out, ignoring')
+                se_size = 0
+                so.host = None
+                se.host = None
+            else:
+                # smells like a tmpfail
+                raise
 
         # avoid mixing up stdout (from report) and stderr (from logging) in output
         sys.stdout.flush()
@@ -1083,6 +1113,11 @@ fi
         sys.stdout.flush()
         sys.stderr.flush()
 
+        # skip the remaining processing if the testbed got broken
+        if se.host is None:
+            adtlog.debug('Skipping remaining log processing and testbed restore after timeout')
+            return
+
         if os.path.getsize(so.host) == 0:
             # don't produce empty -stdout files in --output-dir
             so.autoclean = True
@@ -1113,11 +1148,11 @@ fi
                 os.rmdir(ap.host)
 
         if shell or (shell_on_failure and not test.result):
-            self.run_shell(tree.tb, ['ADT_ARTIFACTS="%s"' % test_artifacts,
-                                     'ADTTMP="%s"' % adttmp])
+            self.run_shell(tree.tb, ['AUTOPKGTEST_ARTIFACTS="%s"' % test_artifacts,
+                                     'AUTOPKGTEST_TMP="%s"' % autopkgtest_tmp])
 
-        # clean up artifacts and ADTTMP dirs
-        self.check_exec(['rm', '-rf', test_artifacts, adttmp])
+        # clean up artifacts and AUTOPKGTEST_TMP dirs
+        self.check_exec(['rm', '-rf', test_artifacts, autopkgtest_tmp])
 
         if need_click_restore:
             self.apparmor_restore_click(test.clicks, test.installed_clicks)
diff --git a/reprotest/virt/autopkgtest-virt-chroot b/reprotest/virt/autopkgtest-virt-chroot
index 8f3af3b..af41054 100755
--- a/reprotest/virt/autopkgtest-virt-chroot
+++ b/reprotest/virt/autopkgtest-virt-chroot
@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 #
-# adt-virt-chroot is part of autopkgtest
+# autopkgtest-virt-chroot is part of autopkgtest
 # autopkgtest is a tool for testing Debian binary packages
 #
 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
@@ -26,9 +26,9 @@ import sys
 import os
 import argparse
 
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
-    os.path.abspath(__file__)))))
+sys.path.insert(0, '/usr/share/autopkgtest/lib')
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__))), 'lib'))
 
 from reprotest.lib import VirtSubproc
 from reprotest.lib import adtlog
diff --git a/reprotest/virt/autopkgtest-virt-lxc b/reprotest/virt/autopkgtest-virt-lxc
index 5919eca..512b9b2 100755
--- a/reprotest/virt/autopkgtest-virt-lxc
+++ b/reprotest/virt/autopkgtest-virt-lxc
@@ -1,12 +1,12 @@
 #!/usr/bin/python3
 #
-# adt-virt-lxc is part of autopkgtest
+# autopkgtest-virt-lxc is part of autopkgtest
 # autopkgtest is a tool for testing Debian binary packages
 #
 # autopkgtest is Copyright (C) 2006-2014 Canonical Ltd.
 #
-# adt-virt-lxc was derived from adt-virt-schroot and modified to suit LXC by
-# Robie Basak <robie.basak at canonical.com>.
+# autopkgtest-virt-lxc was derived from autopkgtest-schroot and modified to suit LXC
+# by Robie Basak <robie.basak at canonical.com>.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -35,9 +35,9 @@ import tempfile
 import shutil
 import argparse
 
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
-    os.path.abspath(__file__)))))
+sys.path.insert(0, '/usr/share/autopkgtest/lib')
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__))), 'lib'))
 
 from reprotest.lib import VirtSubproc
 from reprotest.lib import adtlog
@@ -64,8 +64,8 @@ def parse_args():
                         'errors in some corner cases)')
     parser.add_argument('-s', '--sudo', action='store_true',
                         help='Run lxc-* commands with sudo; use if you run '
-                        'adt-run as normal user')
-    parser.add_argument('--name', help='container name (adt-virt-lxc-XXXXXX by default)')
+                        'autopkgtest as normal user')
+    parser.add_argument('--name', help='container name (autopkgtest-lxc-XXXXXX by default)')
     parser.add_argument('template', help='LXC container name that will be '
                         'used as a template')
     parser.add_argument('lxcargs', nargs=argparse.REMAINDER,
@@ -98,7 +98,7 @@ def get_available_lxc_container_name():
     while True:
         # generate random container name
         rnd = [random.choice(string.ascii_lowercase) for i in range(6)]
-        candidate = 'adt-virt-lxc-' + ''.join(rnd)
+        candidate = 'autopkgtest-lxc-' + ''.join(rnd)
 
         containers = VirtSubproc.check_exec(sudoify(['lxc-ls']), outp=True,
                                             timeout=10)
@@ -221,7 +221,7 @@ def hook_open():
         # shared bind mount works poorly for unprivileged containers due to
         # mapped UIDs
         if args.sudo or os.geteuid() == 0:
-            shared_dir = tempfile.mkdtemp(prefix='adt-virt-lxc.shared.')
+            shared_dir = tempfile.mkdtemp(prefix='autopkgtest-virt-lxc.shared.')
     else:
         # don't change the name between resets, to provide stable downtmp paths
         os.makedirs(shared_dir)
diff --git a/reprotest/virt/autopkgtest-virt-lxd b/reprotest/virt/autopkgtest-virt-lxd
index 45ea67e..ea83435 100755
--- a/reprotest/virt/autopkgtest-virt-lxd
+++ b/reprotest/virt/autopkgtest-virt-lxd
@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 #
-# adt-virt-lxd is part of autopkgtest
+# autopkgtest-virt-lxd is part of autopkgtest
 # autopkgtest is a tool for testing Debian binary packages
 #
 # autopkgtest is Copyright (C) 2006-2015 Canonical Ltd.
@@ -32,9 +32,9 @@ import subprocess
 import time
 import argparse
 
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
-    os.path.abspath(__file__)))))
+sys.path.insert(0, '/usr/share/autopkgtest/lib')
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__))), 'lib'))
 
 from reprotest.lib import VirtSubproc
 from reprotest.lib import adtlog
@@ -73,7 +73,7 @@ def get_available_container_name():
     while True:
         # generate random container name
         rnd = [random.choice(string.ascii_lowercase) for i in range(6)]
-        candidate = 'adt-virt-lxd-' + ''.join(rnd)
+        candidate = 'autopkgtest-lxd-' + ''.join(rnd)
 
         rc = VirtSubproc.execute_timeout(None, 10, ['lxc', 'info', candidate],
                                          stdout=subprocess.DEVNULL,
@@ -184,12 +184,46 @@ def hook_revert():
     hook_open()
 
 
+def get_uptime():
+    try:
+        (rc, out, _) = VirtSubproc.execute_timeout(
+            None, 10, ['lxc', 'exec', container_name, '--', 'cat', '/proc/uptime'],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+        if rc != 0:
+            return
+
+        return float(out.split()[0])
+    except IndexError:
+        return
+
+
 def hook_wait_reboot():
     adtlog.debug('hook_wait_reboot: waiting for container to shut down...')
     # "lxc exec" exits with 0 when the container stops, so just wait longer
     # than our timeout
-    VirtSubproc.execute_timeout(None, 300, ['lxc', 'exec', container_name,
-                                            'sleep', '3600'])
+    initial_uptime = get_uptime()
+
+    adtlog.debug('hook_wait_reboot: container up for %s, waiting for reboot' % initial_uptime)
+
+    for retry in range(20):
+        time.sleep(5)
+
+        current_uptime = get_uptime()
+
+        # container is probably in the very late stages of shutting down, just
+        # keep trying, if this persists we'll bomb out later on
+        if not current_uptime:
+            continue
+
+        if current_uptime < initial_uptime:
+            adtlog.debug('hook_wait_reboot: container now up for %s - has rebooted (initial uptime %s)' % (current_uptime, initial_uptime))
+            break
+        else:
+            adtlog.debug('hook_wait_reboot: container now up for %s - has not rebooted (initial uptime %s)' % (current_uptime, initial_uptime))
+    else:
+        VirtSubproc.bomb('timed out waiting for container %s to restart' % container_name)
+
     adtlog.debug('hook_wait_reboot: container restarted, waiting for boot to finish')
     wait_booted()
 
diff --git a/reprotest/virt/autopkgtest-virt-null b/reprotest/virt/autopkgtest-virt-null
index b23eb01..5ac2aeb 100755
--- a/reprotest/virt/autopkgtest-virt-null
+++ b/reprotest/virt/autopkgtest-virt-null
@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 #
-# adt-virt-null is part of autopkgtest
+# autopkgtest-virt-null is part of autopkgtest
 # autopkgtest is a tool for testing Debian binary packages
 #
 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
@@ -26,9 +26,9 @@ import sys
 import os
 import argparse
 
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
-    os.path.abspath(__file__)))))
+sys.path.insert(0, '/usr/share/autopkgtest/lib')
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__))), 'lib'))
 
 from reprotest.lib import VirtSubproc
 from reprotest.lib import adtlog
diff --git a/reprotest/virt/autopkgtest-virt-qemu b/reprotest/virt/autopkgtest-virt-qemu
index 2b49c93..4488c69 100755
--- a/reprotest/virt/autopkgtest-virt-qemu
+++ b/reprotest/virt/autopkgtest-virt-qemu
@@ -1,11 +1,11 @@
 #!/usr/bin/python3
 #
-# adt-virt-qemu is part of autopkgtest
+# autopkgtest-virt-qemu is part of autopkgtest
 # autopkgtest is a tool for testing Debian binary packages
 #
 # autopkgtest is Copyright (C) 2006-2014 Canonical Ltd.
 #
-# adt-virt-qemu was developed by
+# autopkgtest-virt-qemu was developed by
 # Martin Pitt <martin.pitt at ubuntu.com>
 #
 # This program is free software; you can redistribute it and/or modify
@@ -37,9 +37,9 @@ import fcntl
 import re
 import argparse
 
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
-    os.path.abspath(__file__)))))
+sys.path.insert(0, '/usr/share/autopkgtest/lib')
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__))), 'lib'))
 
 from reprotest.lib import VirtSubproc
 from reprotest.lib import adtlog
@@ -49,7 +49,6 @@ args = None
 workdir = None
 p_qemu = None
 ssh_port = None
-ssh_port_lock = None
 normal_user = None
 qemu_cmd_default = None
 
@@ -81,6 +80,8 @@ def parse_args():
                         help='Number of (virtual) CPUs in the VM (default: %(default)s)')
     parser.add_argument('--ram-size', type=int, default=1024,
                         help='VM RAM size in MiB (default: %(default)s)')
+    parser.add_argument('--timeout-reboot', type=int, metavar='SECONDS', default=60,
+                        help='timeout for waiting for reboot (default: %(default)ss)')
     parser.add_argument('--show-boot', action='store_true',
                         help='Show boot messages from serial console')
     parser.add_argument('-d', '--debug', action='store_true',
@@ -114,7 +115,7 @@ def prepare_overlay():
 
 def wait_boot():
     term = VirtSubproc.get_unix_socket(os.path.join(workdir, 'ttyS0'))
-    VirtSubproc.expect(term, b' login: ', 300, 'login prompt on ttyS0',
+    VirtSubproc.expect(term, b' login: ', args.timeout_reboot, 'login prompt on ttyS0',
                        echo=args.show_boot)
     # this is really ugly, but runlevel, "service status hwclock" etc. all
     # don't help to determine if the system is *really* booted; running
@@ -163,23 +164,27 @@ def login_tty_and_setup_shell():
     term = VirtSubproc.get_unix_socket(os.path.join(workdir, 'ttyS0'))
 
     # send user name
-    term.send(args.user.encode('UTF-8'))
-    term.send(b'\n')
-    # wait until we get some more data for the password prompt
-    VirtSubproc.expect(term, None, 10, 'password prompt')
+    term.send(args.user.encode('UTF-8') + b'\n')
+    VirtSubproc.expect(term, b'assword:', 10, 'password prompt')
     # send password
-    term.send(args.password.encode('UTF-8'))
-    term.send(b'\n')
-    adtlog.debug('login_tty: sent password')
+    passwd_b = args.password.encode('UTF-8')
+    term.send(passwd_b + b'\n')
+    VirtSubproc.expect(term, None, 10, 'acked password')
+    term.send(b'echo "LOG""IN""_"OK\n')
+    adtlog.debug('login_tty: logged in')
+    VirtSubproc.expect(term, b'LOGIN_OK', 120, 'logged in')
 
-    cmd = b'setsid sh </dev/ttyS1 >/dev/ttyS1 2>&1 &'
+    cmd = b'sh </dev/ttyS1 >/dev/ttyS1 2>&1'
 
     # if we are a non-root user, run through sudo
     if args.user != 'root':
-        cmd = b"sudo sh -c '" + cmd + "'"
+        cmd = b"echo '%s' | sudo --background --stdin sh -c '" % passwd_b + cmd + b"'"
+    else:
+        cmd = b'setsid ' + cmd
 
-    term.send(cmd)
-    term.send(b'\nexit\n')
+    term.send(cmd + b'\n')
+    VirtSubproc.expect(term, None, 10, 'accepted ttyS1 shell command')
+    term.send(b'exit\n')
     VirtSubproc.expect(term, b'\nlogout', 10)
 
 
@@ -190,11 +195,12 @@ def setup_baseimage():
 
     # Setup udev rules for /dev/baseimage; set link_priority to -1024 so
     # that the duplicate UUIDs of the partitions will have no effect.
-    term.send(b'''mkdir -p -m 0755 /run/udev/rules.d ; printf '# Created by adt-virt-qemu\\n%s\\n%s\\n' 'KERNEL=="vd*[!0-9]", ENV{ID_SERIAL}=="BASEIMAGE", OPTIONS+="link_priority=-1024", SYMLINK+="baseimage", MODE="0664"' 'KERNEL=="vd*[0-9]",  ENV{ID_SERIAL}=="BASEIMAGE", OPTIONS+="link_priority=-1024"' > /run/udev/rules.d/61-baseimage.rules\n''')
+    term.send(b'''mkdir -p -m 0755 /run/udev/rules.d ; printf '# Created by autopkgtest-virt-qemu\\n%s\\n%s\\n%s\\n' 'KERNEL=="vd*[!0-9]", ENV{ID_SERIAL}=="BASEIMAGE", OPTIONS+="link_priority=-1024", SYMLINK+="baseimage", MODE="0664"' 'KERNEL=="vd*[0-9]",  ENV{ID_SERIAL}=="BASEIMAGE", OPTIONS+="link_priority=-1024"' 'KERNEL=="vd*", ENV{ID_SERIAL}=="BASEIMAGE", ENV{ID_FS_TYPE}:="", ENV{ID_FS_USAGE}:="", ENV{ID_FS_UUID}:=""' > /run/udev/rules.d/61-baseimage.rules\n''')
     VirtSubproc.expect(term, b'#', 10)
     # Reload udev to make sure the rules take effect (udev only auto-
     # rereads rules every 3 seconds)
     term.send(b'udevadm control --reload\n')
+    VirtSubproc.expect(term, b'#', 10)
 
     # Add the base image as an additional drive
     monitor = VirtSubproc.get_unix_socket(os.path.join(workdir, 'monitor'))
@@ -212,10 +218,10 @@ def setup_shared(shared_dir):
 
     term = VirtSubproc.get_unix_socket(os.path.join(workdir, 'ttyS1'))
 
-    term.send(b'''mkdir -p -m 1777 /autopkgtest
-mount -t 9p -o trans=virtio,access=any autopkgtest /autopkgtest
-chmod 1777 /autopkgtest
-touch /autopkgtest/done_shared
+    term.send(b'''mkdir -p -m 1777 /run/autopkgtest/shared
+mount -t 9p -o trans=virtio,access=any autopkgtest /run/autopkgtest/shared
+chmod 1777 /run/autopkgtest/shared
+touch /run/autopkgtest/shared/done_shared
 ''')
 
     with VirtSubproc.timeout(10, 'timed out on client shared directory setup'):
@@ -228,48 +234,13 @@ touch /autopkgtest/done_shared
     term.send(b'[ -n "$HOME" ] || export HOME=`getent passwd root|cut -f6 -d:`\n')
     VirtSubproc.expect(term, b'#', 5)
 
-
-def setup_config(shared_dir):
-    '''Set up configuration files'''
-
-    term = VirtSubproc.get_unix_socket(os.path.join(workdir, 'ttyS1'))
-
-    # copy our timezone, to avoid time skews with the host
-    if os.path.exists('/etc/timezone'):
-        tz = None
-        with open('/etc/timezone', 'rb') as f:
-            for l in f:
-                if l.startswith(b'#'):
-                    continue
-                l = l.strip()
-                if l:
-                    tz = l
-                    break
-
-        if tz:
-            adtlog.debug('Copying host timezone %s to VM' % tz.decode())
-            term.send(b'echo ' + tz + b' > /etc/timezone; DEBIAN_FRONTEND=noninteractive dpkg-reconfigure tzdata\n')
-            VirtSubproc.expect(term, b'#', 30)
-        else:
-            adtlog.debug('Could not determine host timezone')
-
-    # ensure that we have Python for our the auxverb helpers
-    term.send(b'type python3 2>/dev/null || type python 2>/dev/null\n')
-    try:
-        out = VirtSubproc.expect(term, b'/python', 5)
-    except VirtSubproc.Timeout:
-        VirtSubproc.bomb('Neither python3 nor python is installed in the VM, '
-                         'one of them is required by autopkgtest')
-    if b'\n# ' not in out:
-        VirtSubproc.expect(term, b'# ', 5)
-
     # create helper for runcmd: cat data from its stdin (from a file) to stdout
     # eternally (like tail -f), but stop once either an "EOF" file exists and
     # we copied at least as many bytes as given in that EOF file (the first
     # arg), or an "exit flag" file exists.
-    # We don't run that from /autopkgtest/ as 9p from older QEMU versions is
-    # buggy and causes "invalid numeric result" errors on that.
-    term.send(b'''PYTHON=$(which python3) || PYTHON=$(which python); cat <<EOF > /bin/eofcat; chmod 755 /bin/eofcat
+    # We don't run that from /run/autopkgtest/shared as 9p from older QEMU
+    # versions is buggy and causes "invalid numeric result" errors on that.
+    term.send(b'''PYTHON=$(which python3) || PYTHON=$(which python); cat <<EOF > /tmp/eofcat; chmod 755 /tmp/eofcat
 #!$PYTHON
 import sys, os, fcntl, time, errno
 (feof, fexit) = sys.argv[1:]
@@ -302,6 +273,41 @@ EOF
     VirtSubproc.expect(term, b'# ', 5)
 
 
+def setup_config(shared_dir):
+    '''Set up configuration files'''
+
+    term = VirtSubproc.get_unix_socket(os.path.join(workdir, 'ttyS1'))
+
+    # copy our timezone, to avoid time skews with the host
+    if os.path.exists('/etc/timezone'):
+        tz = None
+        with open('/etc/timezone', 'rb') as f:
+            for line in f:
+                if line.startswith(b'#'):
+                    continue
+                line = line.strip()
+                if line:
+                    tz = line
+                    break
+
+        if tz:
+            adtlog.debug('Copying host timezone %s to VM' % tz.decode())
+            term.send(b'echo ' + tz + b' > /etc/timezone; DEBIAN_FRONTEND=noninteractive dpkg-reconfigure tzdata\n')
+            VirtSubproc.expect(term, b'#', 30)
+        else:
+            adtlog.debug('Could not determine host timezone')
+
+    # ensure that we have Python for our the auxverb helpers
+    term.send(b'type python3 2>/dev/null || type python 2>/dev/null\n')
+    try:
+        out = VirtSubproc.expect(term, b'/python', 5)
+    except VirtSubproc.Timeout:
+        VirtSubproc.bomb('Neither python3 nor python is installed in the VM, '
+                         'one of them is required by autopkgtest')
+    if b'\n# ' not in out:
+        VirtSubproc.expect(term, b'# ', 5)
+
+
 def make_auxverb(shared_dir):
     '''Create auxverb script'''
 
@@ -315,7 +321,7 @@ dir_host = '%(dir)s'
 job_host = tempfile.mkdtemp(prefix='job.', dir=dir_host)
 atexit.register(shutil.rmtree, job_host)
 os.chmod(job_host, 0o755)
-job_guest = '/autopkgtest/' + os.path.basename(job_host)
+job_guest = '/run/autopkgtest/shared/' + os.path.basename(job_host)
 running = True
 
 def shovel(fin, fout, flagfile_on_eof=None):
@@ -370,7 +376,7 @@ t_stderr.start()
 # "real" stdin behaviour.
 s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
 s.connect('%(tty)s')
-cmd = 'PYTHONHASHSEED=0 /bin/eofcat %%(d)s/stdin_eof %%(d)s/exit.tmp < %%(d)s/stdin | ' \\
+cmd = 'PYTHONHASHSEED=0 /tmp/eofcat %%(d)s/stdin_eof %%(d)s/exit.tmp < %%(d)s/stdin | ' \\
       '(%%(c)s >> %%(d)s/stdout 2>> %%(d)s/stderr; echo $? > %%(d)s/exit.tmp);' \\
       'mv %%(d)s/exit.tmp %%(d)s/exit\\n' %% \\
        {'d': job_guest, 'c': ' '.join(map(pipes.quote, sys.argv[1:]))}
@@ -444,26 +450,25 @@ def get_cpuflag():
 def find_free_port(start):
     '''Find an unused port in the range [start, start+50)'''
 
-    global ssh_port_lock
-
     for p in range(start, start + 50):
         adtlog.debug('find_free_port: trying %i' % p)
         try:
+            lockfile = '/tmp/autopkgtest-virt-qemu.port.%i' % p
+            f = None
             try:
-                ssh_port_lock = open('/run/lock/adt-virt-qemu.port.%i' % p, 'w')
-                fcntl.flock(ssh_port_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                f = open(lockfile, 'x')
+                os.unlink(lockfile)
+                fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
             except (IOError, OSError):
                 adtlog.debug('find_free_port: %i is locked' % p)
-                if ssh_port_lock:
-                    ssh_port_lock.close()
-                ssh_port_lock = None
                 continue
+            finally:
+                if f:
+                    f.close()
 
             s = socket.create_connection(('127.0.0.1', p))
             # if that works, the port is taken
             s.close()
-            ssh_port_lock.close()
-            ssh_port_lock = None
             continue
         except socket.error as e:
             if e.errno == errno.ECONNREFUSED:
@@ -489,7 +494,7 @@ def determine_normal_user(shared_dir):
     term = VirtSubproc.get_unix_socket(os.path.join(workdir, 'ttyS1'))
     term.send(b"getent passwd | sort -t: -nk3 | "
               b"awk -F: '{if ($3 >= 500) { print $1; exit } }'"
-              b"> /autopkgtest/normal_user\n")
+              b"> /run/autopkgtest/shared/normal_user\n")
     with VirtSubproc.timeout(5, 'timed out on determining normal user'):
         outfile = os.path.join(shared_dir, 'normal_user')
         while not os.path.exists(outfile):
@@ -506,7 +511,7 @@ def determine_normal_user(shared_dir):
 def hook_open():
     global workdir, p_qemu, ssh_port
 
-    workdir = tempfile.mkdtemp(prefix='adt-virt-qemu.')
+    workdir = tempfile.mkdtemp(prefix='autopkgtest-virt-qemu.')
     os.chmod(workdir, 0o755)
 
     shareddir = os.path.join(workdir, 'shared')
@@ -514,13 +519,21 @@ def hook_open():
 
     overlay = prepare_overlay()
 
+    # find free port to forward VM port 22 (for SSH access)
+    ssh_port = find_free_port(10022)
+    if ssh_port:
+        adtlog.debug('Forwarding local port %i to VM ssh port 22' % ssh_port)
+        nic_opt = ',hostfwd=tcp::%i-:22' % ssh_port
+    else:
+        nic_opt = ''
+
     # start QEMU
     argv = [args.qemu_command,
             '-m', str(args.ram_size),
             '-smp', str(args.cpus),
             '-nographic',
-            '-net', 'user',
             '-net', 'nic,model=virtio',
+            '-net', 'user' + nic_opt,
             '-monitor', 'unix:%s/monitor,server,nowait' % workdir,
             '-serial', 'unix:%s/ttyS0,server,nowait' % workdir,
             '-serial', 'unix:%s/ttyS1,server,nowait' % workdir,
@@ -542,13 +555,6 @@ def hook_open():
     if args.qemu_options:
         argv.extend(args.qemu_options.split())
 
-    # find free port to forward VM port 22 (for SSH access)
-    ssh_port = find_free_port(10022)
-    if ssh_port:
-        adtlog.debug('Forwarding local port %i to VM ssh port 22' % ssh_port)
-        argv.append('-redir')
-        argv.append('tcp:%i::22' % ssh_port)
-
     p_qemu = subprocess.Popen(argv)
 
     try:
@@ -573,7 +579,7 @@ def hook_open():
 def hook_downtmp(path):
     # we would like to do this, but 9p is currently way too slow for big source
     # trees
-    # downtmp = '/autopkgtest/tmp'
+    # downtmp = '/run/autopkgtest/shared/tmp'
     # VirtSubproc.check_exec(['mkdir', '-m', '1777', downtmp], downp=True)
     return VirtSubproc.downtmp_mktemp(path)
 
@@ -610,6 +616,7 @@ def hook_wait_reboot():
     shareddir = os.path.join(workdir, 'shared')
     os.unlink(os.path.join(shareddir, 'done_shared'))
     wait_boot()
+    setup_shell()
     setup_shared(shareddir)
     setup_baseimage()
 
diff --git a/reprotest/virt/autopkgtest-virt-schroot b/reprotest/virt/autopkgtest-virt-schroot
index c24e11d..9a62ac3 100755
--- a/reprotest/virt/autopkgtest-virt-schroot
+++ b/reprotest/virt/autopkgtest-virt-schroot
@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 #
-# adt-virt-schroot is part of autopkgtest
+# autopkgtest-schroot is part of autopkgtest
 # autopkgtest is a tool for testing Debian binary packages
 #
 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
@@ -31,9 +31,9 @@ import subprocess
 import time
 import argparse
 
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
-    os.path.abspath(__file__)))))
+sys.path.insert(0, '/usr/share/autopkgtest/lib')
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__))), 'lib'))
 
 from reprotest.lib import VirtSubproc
 from reprotest.lib import adtlog
@@ -99,7 +99,7 @@ def parse_args():
     # we don't want to clobber non-ephemeral schroots
     if cfg['type'] == 'directory' and cfg.get('union-type', 'none') in ('', 'none'):
         VirtSubproc.bomb(
-            'adt-virt-schroot requires ephemeral schroot sessions. Set a '
+            'autopkgtest-schroot requires ephemeral schroot sessions. Set a '
             '"union-type" or use a tarball schroot')
 
     if (match(cfg['root-users'], [os.getuid()], pw_uid) or
@@ -118,7 +118,7 @@ def hook_open():
     sessid = VirtSubproc.check_exec(['schroot', '--quiet', '--begin-session',
                                      '--chroot', schroot] +
                                     (sessid and ['--session-name', sessid] or []),
-                                    outp=True)
+                                    outp=True, fail_on_stderr=False)
     VirtSubproc.auxverb = ['schroot', '--run-session', '--quiet',
                            '--directory=/', '--chroot', sessid]
     if 'root-on-testbed' in capabilities:
@@ -165,7 +165,7 @@ def hook_cleanup():
     retries = 10
     while retries > 0:
         if VirtSubproc.execute_timeout(
-                None, 30, ['schroot', '--quiet', '--end-session', '--chroot', sessid])[0] == 0:
+                None, 300, ['schroot', '--quiet', '--end-session', '--chroot', sessid])[0] == 0:
             break
         retries -= 1
         adtlog.info('schroot --end-session failed, retrying')
diff --git a/reprotest/virt/autopkgtest-virt-ssh b/reprotest/virt/autopkgtest-virt-ssh
index 9d33bf0..062fb19 100755
--- a/reprotest/virt/autopkgtest-virt-ssh
+++ b/reprotest/virt/autopkgtest-virt-ssh
@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 #
-# adt-virt-ssh is part of autopkgtest
+# autopkgtest-virt-ssh is part of autopkgtest
 # autopkgtest is a tool for testing Debian binary packages
 #
 # autopkgtest is Copyright (C) 2006-2015 Canonical Ltd.
@@ -36,9 +36,9 @@ import time
 import subprocess
 import socket
 
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
-    os.path.abspath(__file__)))))
+sys.path.insert(0, '/usr/share/autopkgtest/lib')
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__))), 'lib'))
 
 from reprotest.lib import VirtSubproc
 from reprotest.lib import adtlog
@@ -87,7 +87,7 @@ KILL=""
 for pid in $PS; do
     [ $pid -ne $$ ] && [ $pid -ne $PPID ] && [ $pid -ne $PPPID ] || continue
     [ -r /proc/$pid/comm ] && [ "$(< /proc/$pid/comm)" != sshd ] || continue
-    #echo "XXXadt-ssh-wrapper($$ $PPID $PPPID $myout $myerr): killing $pid (`cat /proc/$pid/cmdline`)" >&2
+    #echo "XXXautopkgtest-ssh-wrapper($$ $PPID $PPPID $myout $myerr): killing $pid (`cat /proc/$pid/cmdline`)" >&2
     KILL="$KILL $pid"
 done
 
@@ -101,7 +101,7 @@ cleanup_paths = []  # paths on the device which we created
 
 
 def parse_args():
-    global args
+    global args, capabilities
 
     parser = argparse.ArgumentParser()
 
@@ -126,6 +126,10 @@ def parse_args():
                         help='Passed verbatim to ssh; see man ssh_config')
     parser.add_argument('-r', '--reboot', action='store_true',
                         help='Indicate that testbed supports reboot')
+    parser.add_argument('--capability', action='append',
+                        help='Indicate that testbed supports given capability.'
+                        ' Can be specified multiple times. Never use this on '
+                        'precious testbeds!')
     parser.add_argument('-s', '--setup-script',
                         help='Setup script to prepare testbed and ssh connection')
     parser.add_argument('--timeout-ssh', metavar='SECS', type=int, default=300,
@@ -155,8 +159,11 @@ def parse_args():
     if args.reboot:
         capabilities.append('reboot')
 
+    if args.capability:
+        capabilities += args.capability
+
 
-def execute_setup_script(command, fail_ok=False):
+def execute_setup_script(command, fail_ok=False, print_stderr=True):
     '''Run the --setup-script, if given.
 
     Arguments passed after -- to the main program are passed verbatim to the
@@ -166,10 +173,13 @@ def execute_setup_script(command, fail_ok=False):
 
     :param command: Command to execute. The command must match a function in
                     the ssh script
-    :param fail_ok: If True, failures will not cause bombing, and this function
-                    returns the exit code instead.
+    :param fail_ok: If True, failures will not cause bombing
+    :return: A tuple (return code, stdout, stderr). stdout and stderr may be
+             None, for example if the script fails and fail_ok is True.
     '''
     global sshconfig, args
+    out = None
+    err = None
 
     if args.setup_script:
         fpath = args.setup_script
@@ -188,13 +198,20 @@ def execute_setup_script(command, fail_ok=False):
 
         adtlog.debug('Executing setup script: %s' % ' '.join(cmd))
         (status, out, err) = VirtSubproc.execute_timeout(
-            None, 1800, cmd, stdout=subprocess.PIPE)
+            None,
+            1800,
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)
+        if print_stderr:
+            # Keep outputting the error on stderr as well as capturing it
+            sys.stderr.write(err)
         if status != 0:
             err = 'setup script failed with code %i: %s' % (status,
                                                             ' '.join(cmd))
             if fail_ok:
                 adtlog.debug(err)
-                return status
+                return (status, out, err)
             else:
                 execute_setup_script('debug-failure', fail_ok=True)
                 VirtSubproc.bomb(err)
@@ -211,7 +228,7 @@ def execute_setup_script(command, fail_ok=False):
         if a is not None:
             sshconfig[param] = a
 
-    return 0
+    return (0, out, err)
 
 
 def host_setup(command):
@@ -229,7 +246,7 @@ def host_setup(command):
 
     try:
         if workdir is None:
-            workdir = tempfile.mkdtemp(prefix='adt-virt-ssh.')
+            workdir = tempfile.mkdtemp(prefix='autopkgtest-virt-ssh.')
             os.chmod(workdir, 0o755)
         execute_setup_script(command)
         build_sshcmd()
@@ -327,16 +344,16 @@ def build_auxverb():
     # create remote wrapper
     rc = VirtSubproc.execute_timeout(
         terminal_kill_wrapper % extra_cmd, 30, sshcmd +
-        ['rm -f /tmp/adt-run-wrapper; set -C; cat > /tmp/adt-run-wrapper; chmod 755 /tmp/adt-run-wrapper'],
+        ['rm -f /tmp/autopkgtest-run-wrapper; set -C; cat > /tmp/autopkgtest-run-wrapper; chmod 755 /tmp/autopkgtest-run-wrapper'],
         stdin=subprocess.PIPE)[0]
     if rc != 0:
-        VirtSubproc.bomb('Failed to create /tmp/adt-run-wrapper')
+        VirtSubproc.bomb('Failed to create /tmp/autopkgtest-run-wrapper')
 
     # create local auxverb script
     auxverb = os.path.join(workdir, 'runcmd')
     with open(auxverb, 'w') as f:
         f.write('''#!/bin/bash
-exec %s -- %s /tmp/adt-run-wrapper $(printf '%%q ' "$@")
+exec %s -- %s /tmp/autopkgtest-run-wrapper $(printf '%%q ' "${@%% }")
 ''' % (" ".join(sshcmd), sudocmd or ''))
     os.chmod(auxverb, 0o755)
     VirtSubproc.auxverb = [auxverb]
@@ -392,6 +409,15 @@ def can_sudo(ssh_cmd):
     return (None, None)
 
 
+def hook_debug_fail():
+    # Don't print stderr; if we're being called for the hook, we assume the
+    # caller is going to do that.
+    (status, out, err) = execute_setup_script('debug-failure',
+                                              fail_ok=True,
+                                              print_stderr=False)
+    return err
+
+
 def hook_open():
     host_setup('open')
 
@@ -433,7 +459,7 @@ def hook_wait_reboot():
     global sshcmd
 
     if args.setup_script:
-        rc = execute_setup_script('wait-reboot', fail_ok=True)
+        (rc, _, _) = execute_setup_script('wait-reboot', fail_ok=True)
     else:
         # if we don't have a setup script, use the fallback below
         rc = 1
@@ -461,9 +487,14 @@ def hook_wait_reboot():
 def hook_cleanup():
     global capabilities, workdir, cleanup_paths, sshcmd
 
-    VirtSubproc.downtmp_remove()
-    if cleanup_paths:
-        VirtSubproc.check_exec(['rm', '-rf'] + cleanup_paths, downp=True, timeout=10)
+    try:
+        VirtSubproc.downtmp_remove()
+        if cleanup_paths:
+            VirtSubproc.check_exec(['rm', '-rf'] + cleanup_paths, downp=True,
+                                   timeout=VirtSubproc.copy_timeout)
+    except VirtSubproc.Timeout:
+        adtlog.error('Removing temporary files on testbed timed out')
+        # still do the remaining cleanup
 
     execute_setup_script('cleanup')
 

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/reprotest.git



More information about the Reproducible-commits mailing list