[reprotest] 04/07: main, presets: Add a --source-pattern option to restrict copying of source_root
Ximin Luo
infinity0 at debian.org
Tue Sep 26 14:36:08 UTC 2017
This is an automated email from the git hooks/post-receive script.
infinity0 pushed a commit to branch master
in repository reprotest.
commit c73c5bb7db7e314bd01b9df1d5afd11aea50e2c0
Author: Ximin Luo <infinity0 at debian.org>
Date: Tue Sep 26 15:49:25 2017 +0200
main, presets: Add a --source-pattern option to restrict copying of source_root
---
README.rst | 11 -----------
debian/changelog | 2 ++
reprotest/__init__.py | 48 +++++++++++++++++++++++++++++++++---------------
reprotest/presets.py | 21 ++++++++++++++++-----
4 files changed, 51 insertions(+), 31 deletions(-)
diff --git a/README.rst b/README.rst
index 97621cc..e92cbe7 100644
--- a/README.rst
+++ b/README.rst
@@ -304,14 +304,3 @@ If you see a difference that you really think should not be there, try passing
``--variations=-time`` to reprotest, and/or check our results on
https://tests.reproducible-builds.org/ which use a different (more reliable)
mechanism to vary the system time.
-
-
-Known bugs
-==========
-
-If the first argument is a file then reprotest will copy its whole directory.
-Sometimes this is unsuitable, e.g. when it contains 3GB of other stuff.
-Also if this copying fails (e.g. because irrelevant files are not copyable due
-to permissions) then reprotest might give a confusing error message about that.
-
-These will be fixed soon.
diff --git a/debian/changelog b/debian/changelog
index 252a158..b5abbce 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,8 @@ reprotest (0.7.1) UNRELEASED; urgency=medium
* Improve error messages in some common scenarios:
- giving a source_root or build_command that doesn't exist
- using reprotest with default settings after not installing Recommends
+ * Add a --source-pattern option to restrict copying of source_root, and set
+ this automatically in our presets.
-- Ximin Luo <infinity0 at debian.org> Fri, 22 Sep 2017 17:57:31 +0200
diff --git a/reprotest/__init__.py b/reprotest/__init__.py
index b75e6e0..0cbb4c9 100644
--- a/reprotest/__init__.py
+++ b/reprotest/__init__.py
@@ -115,6 +115,14 @@ def empty_or_temp_dir(empty_dir, name):
yield temp_dir
+def shell_copy_pattern(dst, src, globs):
+ # assumes globs is already sanitized
+ # mkdir -p dst if it doesn't already exist
+ return ['sh', '-ec', """mkdir -p "{0}"
+cd "{1}" && cp --parents -a -t "{0}" {2}
+""".format(dst, src, globs)]
+
+
class BuildContext(collections.namedtuple('_BuildContext',
'testbed_root local_dist_root local_src build_name variations')):
"""
@@ -177,11 +185,10 @@ class BuildContext(collections.namedtuple('_BuildContext',
xenv=['%s=%s' % (k, v) for k, v in build.env.items()],
kind='build')
dist_base = os.path.join(self.testbed_dist, VSRC_DIR)
- testbed.check_exec2(
- ['sh', '-ec', """mkdir -p "{0}"
-cd "{1}" && cp --parents -a -t "{0}" {2}
-cd "{0}" && touch -d at 0 . .. {2}
-""".format(dist_base, self.testbed_src, artifact_pattern)])
+ testbed.check_exec2(shell_copy_pattern(dist_base, self.testbed_src, artifact_pattern))
+ # FIXME: this is needed because of the FIXME in build.faketime(). we can rm it after that is fixed
+ testbed.check_exec2(['sh', '-ec',
+ r"""cd "{0}" && touch -d at 0 . .. {1}""".format(dist_base, artifact_pattern)])
def run_or_tee(progargs, filename, store_dir, *args, **kwargs):
@@ -218,12 +225,12 @@ class TestbedArgs(collections.namedtuple('_TestbedArgs',
class TestArgs(collections.namedtuple('_Test',
- 'build_command source_root artifact_pattern result_dir no_clean_on_error diffoscope_args')):
+ 'build_command source_root artifact_pattern result_dir source_pattern no_clean_on_error diffoscope_args')):
@classmethod
def default(cls, build_command, source_root, artifact_pattern, result_dir=None,
- no_clean_on_error=False, diffoscope_args=[]):
+ source_pattern=None, no_clean_on_error=False, diffoscope_args=[]):
return cls(build_command, source_root, artifact_pattern, result_dir,
- no_clean_on_error, diffoscope_args)
+ source_pattern, no_clean_on_error, diffoscope_args)
@coroutine
@@ -235,7 +242,7 @@ def corun_builds(test_args, testbed_args):
.>>> local_dist = proc.send((name, var))
.>>> ...
"""
- build_command, source_root, artifact_pattern, result_dir, no_clean_on_error, diffoscope_args = test_args
+ build_command, source_root, artifact_pattern, result_dir, source_pattern, no_clean_on_error, diffoscope_args = test_args
virtual_server_args, testbed_pre, testbed_init, host_distro = testbed_args
if not source_root:
@@ -246,15 +253,19 @@ def corun_builds(test_args, testbed_args):
artifact_pattern = shell_syn.sanitize_globs(artifact_pattern)
logging.debug("artifact_pattern sanitized to: %s", artifact_pattern)
+ if source_pattern:
+ source_pattern = shell_syn.sanitize_globs(source_pattern)
+ logging.debug("source_pattern sanitized to: %s", source_pattern)
logging.debug("virtual_server_args: %r", virtual_server_args)
# TODO: if no_clean_on_error then this shouldn't be rm'd
with tempfile.TemporaryDirectory() as temp_dir:
- if testbed_pre:
+ if testbed_pre or source_pattern:
new_source_root = os.path.join(temp_dir, "testbed_pre")
- shutil.copytree(source_root, new_source_root, symlinks=True)
- subprocess.check_call(["sh", "-ec", testbed_pre], cwd=new_source_root)
+ subprocess.check_call(shell_copy_pattern(new_source_root, source_root, source_pattern or "."))
source_root = new_source_root
+ if testbed_pre:
+ subprocess.check_call(["sh", "-ec", testbed_pre], cwd=new_source_root)
logging.debug("source_root: %s", source_root)
# TODO: an alternative strategy is to run the testbed many times, one for each build
@@ -292,7 +303,7 @@ def corun_builds(test_args, testbed_args):
def check(test_args, testbed_args, build_variations=Variations.of(VariationSpec.default())):
# default argument [] is safe here because we never mutate it.
- _, _, artifact_pattern, store_dir, _, diffoscope_args = test_args
+ _, _, artifact_pattern, store_dir, _, _, diffoscope_args = test_args
with empty_or_temp_dir(store_dir, "store_dir") as result_dir:
assert store_dir == result_dir or store_dir is None
proc = corun_builds(test_args._replace(result_dir=result_dir), testbed_args)
@@ -329,7 +340,7 @@ def check(test_args, testbed_args, build_variations=Variations.of(VariationSpec.
def check_auto(test_args, testbed_args, build_variations=Variations.of(VariationSpec.default())):
# default argument [] is safe here because we never mutate it.
- _, _, _, store_dir, _, diffoscope_args = test_args
+ _, _, _, store_dir, _, _, diffoscope_args = test_args
with empty_or_temp_dir(store_dir, "store_dir") as result_dir:
assert store_dir == result_dir or store_dir is None
proc = corun_builds(test_args._replace(result_dir=result_dir), testbed_args)
@@ -459,6 +470,10 @@ def cli_parser():
'and made available during the build. If a file is given here, then '
'its parent directory is used instead. Default: "." (current working '
'directory).')
+ group1.add_argument('--source-pattern', default=None,
+ help='Shell glob pattern to restrict the files in <source_root> that '
+ 'are made available during the build. Default: empty, i.e. copy the '
+ 'whole <source_root> directory with no restrictions.')
group1.add_argument('-c', '--build-command', default=None,
help='Build command to execute. If this is "auto" then reprotest will '
'guess how to build the given source_root, in which case various other '
@@ -622,6 +637,7 @@ def run(argv, dry_run=None):
testbed_pre = parsed_args.testbed_pre
testbed_init = parsed_args.testbed_init
diffoscope_args = parsed_args.diffoscope_arg
+ source_pattern = parsed_args.source_pattern
# Do presets
if build_command == 'auto':
@@ -635,6 +651,8 @@ def run(argv, dry_run=None):
testbed_init = testbed_init or values.testbed_init
if values.diffoscope_args is not None:
diffoscope_args = values.diffoscope_args + diffoscope_args
+ if values.source_pattern is not None:
+ source_pattern = values.source_pattern + (" " + source_pattern if source_pattern else "")
# Variations args
variations = [parsed_args.variations] + parsed_args.vary
@@ -679,7 +697,7 @@ def run(argv, dry_run=None):
testbed_args = TestbedArgs(virtual_server_args, testbed_pre, testbed_init, host_distro)
test_args = TestArgs(build_command, source_root, artifact_pattern, store_dir,
- no_clean_on_error, diffoscope_args)
+ source_pattern, no_clean_on_error, diffoscope_args)
check_args = (test_args, testbed_args, build_variations)
if parsed_args.dry_run or dry_run:
diff --git a/reprotest/presets.py b/reprotest/presets.py
index 289724d..564debb 100644
--- a/reprotest/presets.py
+++ b/reprotest/presets.py
@@ -3,6 +3,8 @@
import collections
import os
+import shlex
+import subprocess
class AttributeFunctor(collections.namedtuple('_AttributeFunctor', 'x f')):
@@ -13,7 +15,7 @@ class AttributeFunctor(collections.namedtuple('_AttributeFunctor', 'x f')):
class ReprotestPreset(collections.namedtuple('_ReprotestPreset',
- 'build_command artifact_pattern testbed_pre testbed_init diffoscope_args')):
+ 'build_command artifact_pattern testbed_pre testbed_init source_pattern diffoscope_args')):
"""Named-tuple representing a reprotest command preset.
You can manipulate it like this:
@@ -61,6 +63,7 @@ PRESET_DEB_DIR = ReprotestPreset(
artifact_pattern = '../*.deb',
testbed_pre = None,
testbed_init = None,
+ source_pattern = None,
diffoscope_args = ["--exclude-directory-metadata"],
)
@@ -72,10 +75,18 @@ def preset_deb_schroot(preset):
test -c /dev/fuse || mknod -m 666 /dev/fuse c 10 229'
)
-def preset_deb_dsc(fn):
+def parse_dsc_aux(path):
+ dscfiles = subprocess.check_output(["egrep",
+ # this regex comes from dcmd(1) from the devscripts package
+ r"^ [0-9a-f]{32} [0-9]+ ((([a-zA-Z0-9_.-]+/)?[a-zA-Z0-9_.-]+|-) ([a-zA-Z]+|-) )?(.*)$",
+ path])
+ return [x.split()[-1].decode("utf-8") for x in dscfiles.splitlines()]
+
+def preset_deb_dsc(fn, aux):
return PRESET_DEB_DIR.prepend.build_command(
'dpkg-source -x "%s" build && cd build && ' % fn
- ).set.artifact_pattern("*.deb")
+ ).set.artifact_pattern("*.deb"
+ ).set.source_pattern(" ".join(shlex.quote(a) for a in [fn] + aux))
def get_presets(buildfile, virtual_server):
fn = os.path.basename(buildfile)
@@ -89,8 +100,8 @@ def get_presets(buildfile, virtual_server):
elif os.path.isfile(buildfile):
if parts[1] == '.dsc':
if virtual_server == "null":
- return preset_deb_dsc(fn)
+ return preset_deb_dsc(fn, parse_dsc_aux(buildfile))
else:
- return preset_deb_schroot(preset_deb_dsc(fn))
+ return preset_deb_schroot(preset_deb_dsc(fn, parse_dsc_aux(buildfile)))
raise ValueError('unrecognised file type: "%s"; try giving '
'an appropriate --build-command' % buildfile)
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/reprotest.git
More information about the Reproducible-commits
mailing list