[reprotest] 01/01: Various fixes to get the basic dsc/schroot example working

Ximin Luo infinity0 at debian.org
Tue Oct 3 16:58:41 UTC 2017


This is an automated email from the git hooks/post-receive script.

infinity0 pushed a commit to branch master
in repository reprotest.

commit 62416ab8a9e2696ded0df6140f6685d477dbd75c
Author: Ximin Luo <infinity0 at debian.org>
Date:   Tue Oct 3 18:58:33 2017 +0200

    Various fixes to get the basic dsc/schroot example working
---
 reprotest/__init__.py | 52 ++++++++++++++++++++++++++++-----------------------
 reprotest/build.py    |  7 +++++--
 reprotest/presets.py  | 34 +++++++++++++++++++++++++--------
 3 files changed, 60 insertions(+), 33 deletions(-)

diff --git a/reprotest/__init__.py b/reprotest/__init__.py
index 7fbe1e3..8e77fa3 100644
--- a/reprotest/__init__.py
+++ b/reprotest/__init__.py
@@ -166,15 +166,16 @@ class BuildContext(collections.namedtuple('_BuildContext',
         logging.info("copying %s back from virtual server's %s", self.testbed_dist, self.local_dist)
         testbed.command('copyup', (self.testbed_dist, os.path.join(self.local_dist, '')))
 
-    def run_build(self, testbed, build, artifact_pattern):
+    def run_build(self, testbed, build, artifact_pattern, testbed_build_pre):
         logging.info("starting build with source directory: %s, artifact pattern: %s",
             self.testbed_src, artifact_pattern)
-        # remove any existing artifact, in case the build script doesn't overwrite
-        # it e.g. like how make(1) sometimes works.
+        # we remove existing artifacts in case the build doesn't overwrite it
+        # e.g. like how make(1) sometimes works
         testbed.check_exec2(
-            ['sh', '-ec', 'cd "%s" && rm -rf %s' %
-            (self.testbed_src, artifact_pattern)])
-        # this dance is necessary because the cwd can't be cd'd into during the setup phase under some variations like user_group
+            ['sh', '-ec', 'cd "%s" && rm -rf %s && %s' %
+            (self.testbed_src, artifact_pattern, testbed_build_pre or "true")])
+        # this dance is necessary because the cwd can't be cd'd into during the
+        # setup phase under some variations like user_group
         _ = build
         _ = _.append_setup_exec_raw('export', 'REPROTEST_BUILD_PATH=%s' % build.tree)
         _ = _.append_setup_exec_raw('export', 'REPROTEST_UMASK=$(umask)')
@@ -217,10 +218,10 @@ def run_diff(dist_0, dist_1, diffoscope_args, store_dir):
 
 
 class TestbedArgs(collections.namedtuple('_TestbedArgs',
-    'virtual_server_args testbed_pre testbed_init host_distro')):
+    'virtual_server_args testbed_pre testbed_init testbed_build_pre host_distro')):
     @classmethod
-    def of(cls, virtual_server_args=[], testbed_pre=None, testbed_init=None, host_distro='debian'):
-        return cls(virtual_server_args, testbed_pre, testbed_init, host_distro)
+    def of(cls, virtual_server_args=[], testbed_pre=None, testbed_init=None, testbed_build_pre=None, host_distro='debian'):
+        return cls(virtual_server_args, testbed_pre, testbed_init, testbed_build_pre, host_distro)
 
 
 class TestArgs(collections.namedtuple('_Test',
@@ -247,7 +248,7 @@ class TestArgs(collections.namedtuple('_Test',
         .>>>     ...
         """
         build_command, source_root, artifact_pattern, result_dir, source_pattern, no_clean_on_error, diffoscope_args = self
-        virtual_server_args, testbed_pre, testbed_init, host_distro = testbed_args
+        virtual_server_args, testbed_pre, testbed_init, testbed_build_pre, host_distro = testbed_args
 
         if not source_root:
             raise ValueError("invalid source root: %s" % source_root)
@@ -271,6 +272,9 @@ class TestArgs(collections.namedtuple('_Test',
             # not sure if it's worth implementing at this stage, but perhaps in the future.
             with start_testbed(virtual_server_args, temp_dir, no_clean_on_error,
                                host_distro=host_distro) as testbed:
+                if testbed_init:
+                    testbed.check_exec2(["sh", "-ec", testbed_init])
+
                 name_variation = yield
                 names_seen = set()
                 while name_variation:
@@ -290,11 +294,8 @@ class TestArgs(collections.namedtuple('_Test',
                     build = bctx.plan_variations(build)
                     logging.log(5, "build %s: %r", name, build)
 
-                    if testbed_init:
-                        testbed.check_exec2(["sh", "-ec", testbed_init])
-
                     bctx.copydown(testbed)
-                    bctx.run_build(testbed, build, artifact_pattern)
+                    bctx.run_build(testbed, build, artifact_pattern, testbed_build_pre)
                     bctx.copyup(testbed)
 
                     name_variation = yield bctx.local_dist
@@ -374,7 +375,7 @@ def check_auto(test_args, testbed_args, build_variations=Variations.of(Variation
         var_cur = var_x0
         unreproducibles = []
 
-        varnames = VariationSpec.all_names()
+        varnames = [v for v in VariationSpec.all_names() if v in var_x1.spec]
         random.shuffle(varnames)
         for v in varnames:
             var_test = var_cur.replace.spec._replace(**{v: var_x1.spec[v]})
@@ -469,20 +470,20 @@ def cli_parser():
         help='Like --verbosity, but given multiple times without arguments.')
     group1.add_argument('--host-distro', default='debian',
         help='The distribution that will run the tests (Default: %(default)s)')
-    group1.add_argument('-s', '--source-root', default=None,
+    group1.add_argument('-s', '--source-root', default=None, metavar='PATH',
         help='Root of the source tree, that is copied to the virtual server '
         'and made available during the build. If a file is given here, then '
         'its parent directory is used instead. Default: "." (current working '
         'directory).')
-    group1.add_argument('--source-pattern', default=None,
+    group1.add_argument('--source-pattern', default=None, metavar='PATTERNS',
         help='Shell glob pattern to restrict the files in <source_root> that '
         'are made available during the build. Default: empty, i.e. copy the '
         'whole <source_root> directory with no restrictions.')
-    group1.add_argument('-c', '--build-command', default=None,
+    group1.add_argument('-c', '--build-command', default=None, metavar='COMMANDS',
         help='Build command to execute. If this is "auto" then reprotest will '
         'guess how to build the given source_root, in which case various other '
         'options may be automatically set-if-unset. Default: auto'),
-    group1.add_argument('--store-dir', default=None,
+    group1.add_argument('--store-dir', default=None, metavar='DIRECTORY',
         help='Save the artifacts in this directory, which must be empty or '
         'non-existent. Otherwise, the artifacts will be deleted and you only '
         'see their hashes (if reproducible) or the diff output (if not).')
@@ -511,7 +512,7 @@ def cli_parser():
     group1.add_argument('--dont-vary', default=[], action='append', help=argparse.SUPPRESS)
 
     group2 = parser.add_argument_group('diff options')
-    group2.add_argument('--diffoscope-arg', action='append',
+    group2.add_argument('--diffoscope-arg', action='append', metavar='ARG',
         help='Give extra arguments to diffoscope when running it. Default: '
         '%(default)s', default=['--exclude-directory-metadata'])
     group2.add_argument('--no-diffoscope', action='store_true', default=False,
@@ -527,8 +528,11 @@ def cli_parser():
         'compute information needed by the build, where the computation needs '
         'packages you don\'t want installed in the testbed itself.')
     group3.add_argument('--testbed-init', default=None, metavar='COMMANDS',
-        help='Shell commands to run after starting the test bed, but before '
-        'applying variations. Used to e.g. install disorderfs in a chroot.')
+        help='Shell commands to run after starting the test bed, before '
+        'running anything else. Used to e.g. install disorderfs in a chroot.')
+    group3.add_argument('--testbed-build-pre', default=None, metavar='COMMANDS',
+        help='Shell commands to run before each build, even before applying '
+        'variations for that build. Used to e.g. install build-dependencies.')
     group3.add_argument('--auto-preset-expr', default="_", metavar='PYTHON_EXPRESSION',
         help='This may be used to transform the presets returned by the '
         'auto-detection feature. The value should be a python expression '
@@ -641,6 +645,7 @@ def run(argv, dry_run=None):
     artifact_pattern = parsed_args.artifact_pattern
     testbed_pre = parsed_args.testbed_pre
     testbed_init = parsed_args.testbed_init
+    testbed_build_pre = parsed_args.testbed_build_pre
     diffoscope_args = parsed_args.diffoscope_arg
     source_pattern = parsed_args.source_pattern
     if verbosity >= 2:
@@ -656,6 +661,7 @@ def run(argv, dry_run=None):
         artifact_pattern = artifact_pattern or values.artifact_pattern
         testbed_pre = testbed_pre or values.testbed_pre
         testbed_init = testbed_init or values.testbed_init
+        testbed_build_pre = testbed_build_pre or values.testbed_build_pre
         if values.diffoscope_args is not None:
             diffoscope_args = values.diffoscope_args + diffoscope_args
         if values.source_pattern is not None:
@@ -702,7 +708,7 @@ def run(argv, dry_run=None):
         print("No <artifact> to test for differences provided. See --help for options.")
         sys.exit(2)
 
-    testbed_args = TestbedArgs.of(virtual_server_args, testbed_pre, testbed_init, host_distro)
+    testbed_args = TestbedArgs.of(virtual_server_args, testbed_pre, testbed_init, testbed_build_pre, host_distro)
     test_args = TestArgs.of(build_command, source_root, artifact_pattern, store_dir,
                             source_pattern, no_clean_on_error, diffoscope_args)
 
diff --git a/reprotest/build.py b/reprotest/build.py
index e9390af..b1487fa 100644
--- a/reprotest/build.py
+++ b/reprotest/build.py
@@ -273,7 +273,7 @@ def exec_path(ctx, build, vary):
 # affects all user shells, which would be bad.
 # # def shell(ctx, script, env, tree):
 #     return script, env, tree
-# TODO: also test differences with /bin/sh as bash vs dash
+# FIXME: also test differences with /bin/sh as bash vs dash
 
 def timezone(ctx, build, vary):
     # These time zones are theoretically in the POSIX time zone format
@@ -294,7 +294,7 @@ def faketime(ctx, build, vary):
     # FIXME: better way of choosing which faketime to use
     if lastmt.startswith("@") and int(lastmt[1:]) < now - 32253180:
         # if lastmt is far in the past, use that, it's a bit safer
-        faket = '@%s' % lastmt
+        faket = lastmt
     else:
         # otherwise use a date far in the future
         faket = '+373days+7hours+13minutes'
@@ -442,6 +442,9 @@ class VariationSpec(mdiffconf.ImmutableNamespace):
         one = self.default()
         return mdiffconf.parse_all(self, actions, one, one, self.aliases, sep=",")
 
+    def __contains__(self, k):
+        return k in self.__dict__
+
     def __getitem__(self, k):
         return self.__dict__[k]
 
diff --git a/reprotest/presets.py b/reprotest/presets.py
index ca2a0e5..1f3cbe1 100644
--- a/reprotest/presets.py
+++ b/reprotest/presets.py
@@ -3,6 +3,7 @@
 
 import collections
 import os
+import re
 import shlex
 import subprocess
 
@@ -10,7 +11,7 @@ from reprotest.utils import AttributeFunctor
 
 
 class ReprotestPreset(collections.namedtuple('_ReprotestPreset',
-    'build_command artifact_pattern testbed_pre testbed_init source_pattern diffoscope_args')):
+    'build_command artifact_pattern testbed_pre testbed_init testbed_build_pre source_pattern diffoscope_args')):
     """Named-tuple representing a reprotest command preset.
 
     You can manipulate it like this:
@@ -39,36 +40,53 @@ class ReprotestPreset(collections.namedtuple('_ReprotestPreset',
     def set(self):
         """Set the given attribute to the given value."""
         return AttributeFunctor(self, lambda x, y: y)
+
     @property
     def str_replace(self):
         """Do a substring-replace on the given attribute."""
         return AttributeFunctor(self, str.replace)
+
     @property
     def prepend(self):
         """Prepend the given value to the given attribute."""
         return AttributeFunctor(self, lambda a, b: b + a)
+
     @property
     def append(self):
         """Apppend the given value to the given attribute."""
         return AttributeFunctor(self, lambda a, b: a + b)
 
+    @property
+    def re_replace(self):
+        """Do a substring-replace on the given attribute."""
+        return AttributeFunctor(self, lambda s, pattern, repl, **kwargs: re.sub(pattern, repl, s, **kwargs))
+
 
 PRESET_DEB_DIR = ReprotestPreset(
     build_command = 'dpkg-buildpackage --no-sign -b',
     artifact_pattern = '../*.deb',
     testbed_pre = None,
     testbed_init = None,
+    testbed_build_pre = None,
     source_pattern = None,
     diffoscope_args = [],
 )
 
-def preset_deb_schroot(preset):
-    return preset.str_replace.build_command("dpkg-buildpackage",
-        'PATH=/sbin:/usr/sbin:$PATH apt-get -y --no-install-recommends build-dep ./; dpkg-buildpackage'
+def preset_deb_schroot(fn, preset):
+    return preset.re_replace.build_command("(.*)", lambda m: r"""
+        if [ "$(id -u)" = 0 ]; then
+            sudo -E -u "$LOGNAME" sh -ec {0};
+        else
+            sh -ec {0};
+        fi
+        """.format(shlex.quote(m.group(1)))
+        # schroot starts us off as root, we drop privs here to do the actual build
     ).set.testbed_init(
+        # need to symlink /etc/mtab to work around a fusermount(1) deficiency
         'apt-get -y --no-install-recommends install disorderfs faketime locales-all sudo util-linux; \
-        test -c /dev/fuse || mknod -m 666 /dev/fuse c 10 229'
-    )
+        test -c /dev/fuse || mknod -m 666 /dev/fuse c 10 229; test -f /etc/mtab || ln -s ../proc/self/mounts /etc/mtab'
+    ).set.testbed_build_pre(
+        'PATH=/sbin:/usr/sbin:$PATH apt-get -y --no-install-recommends build-dep ./"%s"' % fn)
 
 def parse_dsc_aux(path):
     dscfiles = subprocess.check_output(["egrep",
@@ -92,12 +110,12 @@ def get_presets(buildfile, virtual_server):
             if virtual_server == "null":
                 return PRESET_DEB_DIR
             else:
-                return preset_deb_schroot(PRESET_DEB_DIR)
+                return preset_deb_schroot(".", PRESET_DEB_DIR)
     elif os.path.isfile(buildfile):
         if parts[1] == '.dsc':
             if virtual_server == "null":
                 return preset_deb_dsc(fn, parse_dsc_aux(buildfile))
             else:
-                return preset_deb_schroot(preset_deb_dsc(fn, parse_dsc_aux(buildfile)))
+                return preset_deb_schroot(fn, preset_deb_dsc(fn, parse_dsc_aux(buildfile)))
     raise ValueError('unrecognised file type: "%s"; try giving '
                      'an appropriate --build-command' % buildfile)

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/reprotest.git



More information about the Reproducible-commits mailing list