[reprotest] 01/01: Various fixes to get the basic dsc/schroot example working
Ximin Luo
infinity0 at debian.org
Tue Oct 3 16:58:41 UTC 2017
This is an automated email from the git hooks/post-receive script.
infinity0 pushed a commit to branch master
in repository reprotest.
commit 62416ab8a9e2696ded0df6140f6685d477dbd75c
Author: Ximin Luo <infinity0 at debian.org>
Date: Tue Oct 3 18:58:33 2017 +0200
Various fixes to get the basic dsc/schroot example working
---
reprotest/__init__.py | 52 ++++++++++++++++++++++++++++-----------------------
reprotest/build.py | 7 +++++--
reprotest/presets.py | 34 +++++++++++++++++++++++++--------
3 files changed, 60 insertions(+), 33 deletions(-)
diff --git a/reprotest/__init__.py b/reprotest/__init__.py
index 7fbe1e3..8e77fa3 100644
--- a/reprotest/__init__.py
+++ b/reprotest/__init__.py
@@ -166,15 +166,16 @@ class BuildContext(collections.namedtuple('_BuildContext',
logging.info("copying %s back from virtual server's %s", self.testbed_dist, self.local_dist)
testbed.command('copyup', (self.testbed_dist, os.path.join(self.local_dist, '')))
- def run_build(self, testbed, build, artifact_pattern):
+ def run_build(self, testbed, build, artifact_pattern, testbed_build_pre):
logging.info("starting build with source directory: %s, artifact pattern: %s",
self.testbed_src, artifact_pattern)
- # remove any existing artifact, in case the build script doesn't overwrite
- # it e.g. like how make(1) sometimes works.
+ # we remove existing artifacts in case the build doesn't overwrite it
+ # e.g. like how make(1) sometimes works
testbed.check_exec2(
- ['sh', '-ec', 'cd "%s" && rm -rf %s' %
- (self.testbed_src, artifact_pattern)])
- # this dance is necessary because the cwd can't be cd'd into during the setup phase under some variations like user_group
+ ['sh', '-ec', 'cd "%s" && rm -rf %s && %s' %
+ (self.testbed_src, artifact_pattern, testbed_build_pre or "true")])
+ # this dance is necessary because the cwd can't be cd'd into during the
+ # setup phase under some variations like user_group
_ = build
_ = _.append_setup_exec_raw('export', 'REPROTEST_BUILD_PATH=%s' % build.tree)
_ = _.append_setup_exec_raw('export', 'REPROTEST_UMASK=$(umask)')
@@ -217,10 +218,10 @@ def run_diff(dist_0, dist_1, diffoscope_args, store_dir):
class TestbedArgs(collections.namedtuple('_TestbedArgs',
- 'virtual_server_args testbed_pre testbed_init host_distro')):
+ 'virtual_server_args testbed_pre testbed_init testbed_build_pre host_distro')):
@classmethod
- def of(cls, virtual_server_args=[], testbed_pre=None, testbed_init=None, host_distro='debian'):
- return cls(virtual_server_args, testbed_pre, testbed_init, host_distro)
+ def of(cls, virtual_server_args=[], testbed_pre=None, testbed_init=None, testbed_build_pre=None, host_distro='debian'):
+ return cls(virtual_server_args, testbed_pre, testbed_init, testbed_build_pre, host_distro)
class TestArgs(collections.namedtuple('_Test',
@@ -247,7 +248,7 @@ class TestArgs(collections.namedtuple('_Test',
.>>> ...
"""
build_command, source_root, artifact_pattern, result_dir, source_pattern, no_clean_on_error, diffoscope_args = self
- virtual_server_args, testbed_pre, testbed_init, host_distro = testbed_args
+ virtual_server_args, testbed_pre, testbed_init, testbed_build_pre, host_distro = testbed_args
if not source_root:
raise ValueError("invalid source root: %s" % source_root)
@@ -271,6 +272,9 @@ class TestArgs(collections.namedtuple('_Test',
# not sure if it's worth implementing at this stage, but perhaps in the future.
with start_testbed(virtual_server_args, temp_dir, no_clean_on_error,
host_distro=host_distro) as testbed:
+ if testbed_init:
+ testbed.check_exec2(["sh", "-ec", testbed_init])
+
name_variation = yield
names_seen = set()
while name_variation:
@@ -290,11 +294,8 @@ class TestArgs(collections.namedtuple('_Test',
build = bctx.plan_variations(build)
logging.log(5, "build %s: %r", name, build)
- if testbed_init:
- testbed.check_exec2(["sh", "-ec", testbed_init])
-
bctx.copydown(testbed)
- bctx.run_build(testbed, build, artifact_pattern)
+ bctx.run_build(testbed, build, artifact_pattern, testbed_build_pre)
bctx.copyup(testbed)
name_variation = yield bctx.local_dist
@@ -374,7 +375,7 @@ def check_auto(test_args, testbed_args, build_variations=Variations.of(Variation
var_cur = var_x0
unreproducibles = []
- varnames = VariationSpec.all_names()
+ varnames = [v for v in VariationSpec.all_names() if v in var_x1.spec]
random.shuffle(varnames)
for v in varnames:
var_test = var_cur.replace.spec._replace(**{v: var_x1.spec[v]})
@@ -469,20 +470,20 @@ def cli_parser():
help='Like --verbosity, but given multiple times without arguments.')
group1.add_argument('--host-distro', default='debian',
help='The distribution that will run the tests (Default: %(default)s)')
- group1.add_argument('-s', '--source-root', default=None,
+ group1.add_argument('-s', '--source-root', default=None, metavar='PATH',
help='Root of the source tree, that is copied to the virtual server '
'and made available during the build. If a file is given here, then '
'its parent directory is used instead. Default: "." (current working '
'directory).')
- group1.add_argument('--source-pattern', default=None,
+ group1.add_argument('--source-pattern', default=None, metavar='PATTERNS',
help='Shell glob pattern to restrict the files in <source_root> that '
'are made available during the build. Default: empty, i.e. copy the '
'whole <source_root> directory with no restrictions.')
- group1.add_argument('-c', '--build-command', default=None,
+ group1.add_argument('-c', '--build-command', default=None, metavar='COMMANDS',
help='Build command to execute. If this is "auto" then reprotest will '
'guess how to build the given source_root, in which case various other '
'options may be automatically set-if-unset. Default: auto'),
- group1.add_argument('--store-dir', default=None,
+ group1.add_argument('--store-dir', default=None, metavar='DIRECTORY',
help='Save the artifacts in this directory, which must be empty or '
'non-existent. Otherwise, the artifacts will be deleted and you only '
'see their hashes (if reproducible) or the diff output (if not).')
@@ -511,7 +512,7 @@ def cli_parser():
group1.add_argument('--dont-vary', default=[], action='append', help=argparse.SUPPRESS)
group2 = parser.add_argument_group('diff options')
- group2.add_argument('--diffoscope-arg', action='append',
+ group2.add_argument('--diffoscope-arg', action='append', metavar='ARG',
help='Give extra arguments to diffoscope when running it. Default: '
'%(default)s', default=['--exclude-directory-metadata'])
group2.add_argument('--no-diffoscope', action='store_true', default=False,
@@ -527,8 +528,11 @@ def cli_parser():
'compute information needed by the build, where the computation needs '
'packages you don\'t want installed in the testbed itself.')
group3.add_argument('--testbed-init', default=None, metavar='COMMANDS',
- help='Shell commands to run after starting the test bed, but before '
- 'applying variations. Used to e.g. install disorderfs in a chroot.')
+ help='Shell commands to run after starting the test bed, before '
+ 'running anything else. Used to e.g. install disorderfs in a chroot.')
+ group3.add_argument('--testbed-build-pre', default=None, metavar='COMMANDS',
+ help='Shell commands to run before each build, even before applying '
+ 'variations for that build. Used to e.g. install build-dependencies.')
group3.add_argument('--auto-preset-expr', default="_", metavar='PYTHON_EXPRESSION',
help='This may be used to transform the presets returned by the '
'auto-detection feature. The value should be a python expression '
@@ -641,6 +645,7 @@ def run(argv, dry_run=None):
artifact_pattern = parsed_args.artifact_pattern
testbed_pre = parsed_args.testbed_pre
testbed_init = parsed_args.testbed_init
+ testbed_build_pre = parsed_args.testbed_build_pre
diffoscope_args = parsed_args.diffoscope_arg
source_pattern = parsed_args.source_pattern
if verbosity >= 2:
@@ -656,6 +661,7 @@ def run(argv, dry_run=None):
artifact_pattern = artifact_pattern or values.artifact_pattern
testbed_pre = testbed_pre or values.testbed_pre
testbed_init = testbed_init or values.testbed_init
+ testbed_build_pre = testbed_build_pre or values.testbed_build_pre
if values.diffoscope_args is not None:
diffoscope_args = values.diffoscope_args + diffoscope_args
if values.source_pattern is not None:
@@ -702,7 +708,7 @@ def run(argv, dry_run=None):
print("No <artifact> to test for differences provided. See --help for options.")
sys.exit(2)
- testbed_args = TestbedArgs.of(virtual_server_args, testbed_pre, testbed_init, host_distro)
+ testbed_args = TestbedArgs.of(virtual_server_args, testbed_pre, testbed_init, testbed_build_pre, host_distro)
test_args = TestArgs.of(build_command, source_root, artifact_pattern, store_dir,
source_pattern, no_clean_on_error, diffoscope_args)
diff --git a/reprotest/build.py b/reprotest/build.py
index e9390af..b1487fa 100644
--- a/reprotest/build.py
+++ b/reprotest/build.py
@@ -273,7 +273,7 @@ def exec_path(ctx, build, vary):
# affects all user shells, which would be bad.
# # def shell(ctx, script, env, tree):
# return script, env, tree
-# TODO: also test differences with /bin/sh as bash vs dash
+# FIXME: also test differences with /bin/sh as bash vs dash
def timezone(ctx, build, vary):
# These time zones are theoretically in the POSIX time zone format
@@ -294,7 +294,7 @@ def faketime(ctx, build, vary):
# FIXME: better way of choosing which faketime to use
if lastmt.startswith("@") and int(lastmt[1:]) < now - 32253180:
# if lastmt is far in the past, use that, it's a bit safer
- faket = '@%s' % lastmt
+ faket = lastmt
else:
# otherwise use a date far in the future
faket = '+373days+7hours+13minutes'
@@ -442,6 +442,9 @@ class VariationSpec(mdiffconf.ImmutableNamespace):
one = self.default()
return mdiffconf.parse_all(self, actions, one, one, self.aliases, sep=",")
+ def __contains__(self, k):
+ return k in self.__dict__
+
def __getitem__(self, k):
return self.__dict__[k]
diff --git a/reprotest/presets.py b/reprotest/presets.py
index ca2a0e5..1f3cbe1 100644
--- a/reprotest/presets.py
+++ b/reprotest/presets.py
@@ -3,6 +3,7 @@
import collections
import os
+import re
import shlex
import subprocess
@@ -10,7 +11,7 @@ from reprotest.utils import AttributeFunctor
class ReprotestPreset(collections.namedtuple('_ReprotestPreset',
- 'build_command artifact_pattern testbed_pre testbed_init source_pattern diffoscope_args')):
+ 'build_command artifact_pattern testbed_pre testbed_init testbed_build_pre source_pattern diffoscope_args')):
"""Named-tuple representing a reprotest command preset.
You can manipulate it like this:
@@ -39,36 +40,53 @@ class ReprotestPreset(collections.namedtuple('_ReprotestPreset',
def set(self):
"""Set the given attribute to the given value."""
return AttributeFunctor(self, lambda x, y: y)
+
@property
def str_replace(self):
"""Do a substring-replace on the given attribute."""
return AttributeFunctor(self, str.replace)
+
@property
def prepend(self):
"""Prepend the given value to the given attribute."""
return AttributeFunctor(self, lambda a, b: b + a)
+
@property
def append(self):
"""Apppend the given value to the given attribute."""
return AttributeFunctor(self, lambda a, b: a + b)
+ @property
+ def re_replace(self):
+ """Do a substring-replace on the given attribute."""
+ return AttributeFunctor(self, lambda s, pattern, repl, **kwargs: re.sub(pattern, repl, s, **kwargs))
+
PRESET_DEB_DIR = ReprotestPreset(
build_command = 'dpkg-buildpackage --no-sign -b',
artifact_pattern = '../*.deb',
testbed_pre = None,
testbed_init = None,
+ testbed_build_pre = None,
source_pattern = None,
diffoscope_args = [],
)
-def preset_deb_schroot(preset):
- return preset.str_replace.build_command("dpkg-buildpackage",
- 'PATH=/sbin:/usr/sbin:$PATH apt-get -y --no-install-recommends build-dep ./; dpkg-buildpackage'
+def preset_deb_schroot(fn, preset):
+ return preset.re_replace.build_command("(.*)", lambda m: r"""
+ if [ "$(id -u)" = 0 ]; then
+ sudo -E -u "$LOGNAME" sh -ec {0};
+ else
+ sh -ec {0};
+ fi
+ """.format(shlex.quote(m.group(1)))
+ # schroot starts us off as root, we drop privs here to do the actual build
).set.testbed_init(
+ # need to symlink /etc/mtab to work around a fusermount(1) deficiency
'apt-get -y --no-install-recommends install disorderfs faketime locales-all sudo util-linux; \
- test -c /dev/fuse || mknod -m 666 /dev/fuse c 10 229'
- )
+ test -c /dev/fuse || mknod -m 666 /dev/fuse c 10 229; test -f /etc/mtab || ln -s ../proc/self/mounts /etc/mtab'
+ ).set.testbed_build_pre(
+ 'PATH=/sbin:/usr/sbin:$PATH apt-get -y --no-install-recommends build-dep ./"%s"' % fn)
def parse_dsc_aux(path):
dscfiles = subprocess.check_output(["egrep",
@@ -92,12 +110,12 @@ def get_presets(buildfile, virtual_server):
if virtual_server == "null":
return PRESET_DEB_DIR
else:
- return preset_deb_schroot(PRESET_DEB_DIR)
+ return preset_deb_schroot(".", PRESET_DEB_DIR)
elif os.path.isfile(buildfile):
if parts[1] == '.dsc':
if virtual_server == "null":
return preset_deb_dsc(fn, parse_dsc_aux(buildfile))
else:
- return preset_deb_schroot(preset_deb_dsc(fn, parse_dsc_aux(buildfile)))
+ return preset_deb_schroot(fn, preset_deb_dsc(fn, parse_dsc_aux(buildfile)))
raise ValueError('unrecognised file type: "%s"; try giving '
'an appropriate --build-command' % buildfile)
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/reprotest.git
More information about the Reproducible-commits
mailing list