[Reproducible-commits] [reprotest] 04/04: Turn build commands into shell scripts with the AST library
Ceridwen
ceridwen-guest at moszumanska.debian.org
Wed Jul 13 21:26:39 UTC 2016
This is an automated email from the git hooks/post-receive script.
ceridwen-guest pushed a commit to branch virtualization
in repository reprotest.
commit 5bdccaad952dbc856e29c72c31463824fca1fa5b
Author: Ceridwen <ceridwenv at gmail.com>
Date: Wed Jul 13 17:26:28 2016 -0400
Turn build commands into shell scripts with the AST library
---
reprotest/__init__.py | 148 +++++++++++++++++++++++++++---------------------
reprotest/_shell_ast.py | 64 ++++++++++++++++-----
tests/tests.py | 27 +++++----
tox.ini | 4 +-
4 files changed, 154 insertions(+), 89 deletions(-)
diff --git a/reprotest/__init__.py b/reprotest/__init__.py
index 94ded07..775650d 100644
--- a/reprotest/__init__.py
+++ b/reprotest/__init__.py
@@ -20,13 +20,11 @@ import pkg_resources
from reprotest.lib import adtlog
from reprotest.lib import adt_testbed
+from reprotest import _shell_ast
-# adtlog.verbosity = 2
+adtlog.verbosity = 2
-# time zone, locales, disorderfs, host name, user/group, shell, CPU
-# number, architecture for uname (using linux64), umask, HOME, see
-# also: https://tests.reproducible-builds.org/index_variations.html
# chroot is the only form of OS virtualization that's available on
# most POSIX OSes. Linux containers (lxc) and namespaces are specific
@@ -42,6 +40,7 @@ def virtual_server(args, temp_dir):
# path for the correct virt-server script.
server_path = pkg_resources.resource_filename(__name__, 'virt/' +
args[0])
+ print('VIRTUAL SERVER', [server_path] + args[1:])
virtual_server = adt_testbed.Testbed([server_path] + args[1:], temp_dir, None)
virtual_server.start()
virtual_server.open()
@@ -51,6 +50,10 @@ def virtual_server(args, temp_dir):
virtual_server.stop()
+# time zone, locales, disorderfs, host name, user/group, shell, CPU
+# number, architecture for uname (using linux64), umask, HOME, see
+# also: https://tests.reproducible-builds.org/index_variations.html
+
Pair = collections.namedtuple('Pair', 'control experiment')
def add(mapping, key, value):
@@ -58,41 +61,46 @@ def add(mapping, key, value):
new_mapping[key] = value
return types.MappingProxyType(new_mapping)
+
# TODO: relies on a pbuilder-specific command to parallelize
# @contextlib.contextmanager
# def cpu(env, tree, builder):
-# yield command, env, tree
+# yield script, env, tree
@contextlib.contextmanager
-def captures_environment(command, env, tree, builder):
+def captures_environment(script, env, tree, builder):
new_env = add(env.experiment, 'CAPTURE_ENVIRONMENT',
'i_capture_the_environment')
- yield command, Pair(env.control, new_env), tree
+ yield script, Pair(env.control, new_env), tree
# TODO: this requires superuser privileges.
@contextlib.contextmanager
-def domain_host(command, env, tree, builder):
- yield command, env, tree
+def domain_host(script, env, tree, builder):
+ yield script, env, tree
+
+# @contextlib.contextmanager
+# def fileordering(script, env, tree, builder):
+# new_tree = os.path.dirname(os.path.dirname(tree.control)) + '/disorderfs/'
+# builder.execute(['mkdir', '-p', new_tree])
+# # disorderfs = tree2.parent/'disorderfs'
+# # disorderfs.mkdir()
+# builder.execute(['disorderfs', '--shuffle-dirents=yes',
+# tree.experiment, new_tree])
+# try:
+# yield script, env, Pair(tree.control, new_tree)
+# finally:
+# # subprocess.check_call(['fusermount', '-u', str(disorderfs)])
+# builder.execute(['fusermount', '-u', new_tree])
@contextlib.contextmanager
-def fileordering(command, env, tree, builder):
- new_tree = os.path.dirname(os.path.dirname(tree.control)) + '/disorderfs/'
- builder.execute(['mkdir', '-p', new_tree])
- # disorderfs = tree2.parent/'disorderfs'
- # disorderfs.mkdir()
- builder.execute(['disorderfs', '--shuffle-dirents=yes',
- tree.experiment, new_tree])
- try:
- yield command, env, Pair(tree.control, new_tree)
- finally:
- # subprocess.check_call(['fusermount', '-u', str(disorderfs)])
- builder.execute(['fusermount', '-u', new_tree])
+def fileordering(script, env, tree, builder):
+ yield script, env, tree
@contextlib.contextmanager
-def home(command, env, tree, builder):
+def home(script, env, tree, builder):
control = add(env.control, 'HOME', '/nonexistent/first-build')
experiment = add(env.experiment, 'HOME', '/nonexistent/second-build')
- yield command, Pair(control, experiment), tree
+ yield script, Pair(control, experiment), tree
# TODO: uname is a POSIX standard. The related Linux command
# (setarch) only affects uname at the moment according to the docs.
@@ -100,9 +108,13 @@ def home(command, env, tree, builder):
# reference to a setname command on another Unix variant:
# https://en.wikipedia.org/wiki/Uname
@contextlib.contextmanager
-def kernel(command, env, tree, builder):
- new_command = ['linux64', '--uname-2.6'] + command.experiment
- yield Pair(command.control, new_command), env, tree
+def kernel(script, env, tree, builder):
+ setarch = _shell_ast.SimpleCommand(
+ '', 'linux64', _shell_ast.CmdSuffix(
+ ['--uname-2.6', script.experiment[0].command]))
+ new_script = (script.experiment[:-1] +
+ _shell_ast.List([_shell_ast.Term(setarch, ';')]))
+ yield Pair(script.control, new_script), env, tree
# TODO: if this locale doesn't exist on the system, Python's
# locales.getlocale() will return (None, None) rather than this
@@ -117,53 +129,57 @@ def kernel(command, env, tree, builder):
# TODO: what exact locales and how to many test is probably a mailing
# list question.
@contextlib.contextmanager
-def locales(command, env, tree, builder):
+def locales(script, env, tree, builder):
# env1['LANG'] = 'C'
new_env = add(add(env.experiment, 'LANG', 'fr_CH.UTF-8'),
'LC_ALL', 'fr_CH.UTF-8')
# env1['LANGUAGE'] = 'en_US:en'
# env2['LANGUAGE'] = 'fr_CH:fr'
- yield command, Pair(env.control, new_env), tree
+ yield script, Pair(env.control, new_env), tree
# TODO: Linux-specific. unshare --uts requires superuser privileges.
# How is this related to host/domainname?
-# def namespace(command, env, tree, builder):
+# def namespace(script, env, tree, builder):
# # command1 = ['unshare', '--uts'] + command1
# # command2 = ['unshare', '--uts'] + command2
-# yield command, env, tree
+# yield script, env, tree
@contextlib.contextmanager
-def path(command, env, tree, builder):
+def path(script, env, tree, builder):
new_env = add(env.experiment, 'PATH', env.control['PATH'] +
'/i_capture_the_path')
- yield command, Pair(env.control, new_env), tree
+ yield script, Pair(env.control, new_env), tree
# This doesn't require superuser privileges, but the chsh command
# affects all user shells, which would be bad.
@contextlib.contextmanager
-def shell(command, env, tree, builder):
- yield command, env, tree
+def shell(script, env, tree, builder):
+ yield script, env, tree
@contextlib.contextmanager
-def timezone(command, env, tree, builder):
+def timezone(script, env, tree, builder):
# These time zones are theoretically in the POSIX time zone format
# (http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08),
# so they should be cross-platform compatible.
control = add(env.control, 'TZ', 'GMT+12')
experiment = add(env.experiment, 'TZ', 'GMT-14')
- yield command, Pair(control, experiment), tree
-
-# TODO: figure out how to make this compatible with adt_testbed.
+ yield script, Pair(control, experiment), tree
-# def umask(command, env, tree, builder):
-# command2 = ['umask', '0002;'] + command2
-# yield command, env, tree
+ at contextlib.contextmanager
+def umask(script, env, tree, builder):
+ umask = _shell_ast.SimpleCommand('', 'umask', _shell_ast.CmdSuffix(['0002']))
+ new_script = (_shell_ast.List([_shell_ast.Term(umask, ';')])
+ + script.experiment)
+ yield Pair(script.control, new_script), env, tree
# TODO: This requires superuser privileges.
@contextlib.contextmanager
-def user_group(command, env, tree, builder):
- yield command, env, tree
+def user_group(script, env, tree, builder):
+ yield script, env, tree
+
+# The order of the variations *is* important, because the command to
+# be executed in the container needs to be built from the inside out.
VARIATIONS = types.MappingProxyType(collections.OrderedDict([
('captures_environment', captures_environment),
# 'cpu': cpu,
@@ -171,38 +187,42 @@ VARIATIONS = types.MappingProxyType(collections.OrderedDict([
('home', home), ('kernel', kernel), ('locales', locales),
# 'namespace': namespace,
('path', path), ('shell', shell),
- ('timezone', timezone), # ('umask', umask)
+ ('timezone', timezone), ('umask', umask),
('user_group', user_group)
]))
-def build(command, source_root, built_artifact, builder, artifact_store, env):
- # print(command)
- # print(source_root)
- # print(list(pathlib.Path(source_root).glob('*')))
- # print(kws)
- # print(subprocess.check_output(['ls'], cwd=source_root, **kws).decode('ascii'))
- # print(subprocess.check_output('python --version', cwd=source_root, **kws))
- builder.execute(command, xenv=[str(k) + '=' + str(v) for k, v in env.items()], cwd=source_root)
- # subprocess.check_call(command, cwd=source_root, **kws)
+
+def build(script, source_root, built_artifact, builder, artifact_store, env):
+ print(source_root)
+ builder.execute(['ls', '-l', source_root])
+ # builder.execute(['pwd'])
+ print(built_artifact)
+ cd = _shell_ast.SimpleCommand('', 'cd', _shell_ast.CmdSuffix([source_root]))
+ new_script = (_shell_ast.List([_shell_ast.Term(cd, ';')]) + script)
+ print(new_script)
+ exit_code, stdout, stderr = builder.execute(['sh', '-ec', str(new_script)], xenv=[str(k) + '=' + str(v) for k, v in env.items()], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ print(exit_code, stdout, stderr)
+ builder.execute(['ls', '-l', source_root])
+ # subprocess.check_call(script, cwd=source_root, **kws)
# with open(built_artifact, 'rb') as artifact:
# artifact_store.write(artifact.read())
# artifact_store.flush()
builder.command('copyup', (built_artifact, artifact_store))
+
def check(build_command, artifact_name, virtual_server_args, source_root,
variations=VARIATIONS):
# print(virtual_server_args)
with tempfile.TemporaryDirectory() as temp_dir, virtual_server(virtual_server_args, temp_dir) as builder:
- command = Pair(build_command, build_command)
+ ast = _shell_ast.List(
+ [_shell_ast.Term(build_command, ';')])
+ script = Pair(ast, ast)
env = Pair(types.MappingProxyType(os.environ.copy()),
types.MappingProxyType(os.environ.copy()))
# TODO, why?: directories need explicit '/' appended for VirtSubproc
- tree = Pair(temp_dir + '/control/', temp_dir + '/experiment/')
+ tree = Pair(builder.scratch + '/control/', builder.scratch + '/experiment/')
builder.command('copydown', (str(source_root) + '/', tree.control))
builder.command('copydown', (str(source_root) + '/', tree.experiment))
- # tree1 = pathlib.Path(shutil.copytree(str(source_root), temp_dir + '/tree1'))
- # tree2 = pathlib.Path(shutil.copytree(str(source_root), temp_dir + '/tree2'))
- # print(' '.join(command1))
# print(pathlib.Path.cwd())
# print(source_root)
try:
@@ -210,33 +230,33 @@ def check(build_command, artifact_name, virtual_server_args, source_root,
for variation in variations:
# print('START')
# print(variation)
- command, env, tree = stack.enter_context(VARIATIONS[variation](command, env, tree, builder))
- # print(command)
+ script, env, tree = stack.enter_context(VARIATIONS[variation](script, env, tree, builder))
+ # print(script)
# print(env)
# print(tree)
# I would prefer to use pathlib here but
# .resolve(), to eliminate ../ references, doesn't
# work on nonexistent paths.
# print(env)
- build(command.control, tree.control,
+ build(script.control, tree.control,
os.path.normpath(tree.control + artifact_name),
builder,
os.path.normpath(temp_dir + '/control_artifact'),
env=env.control)
- build(command.experiment, tree.experiment,
+ build(script.experiment, tree.experiment,
os.path.normpath(tree.experiment + artifact_name),
builder,
os.path.normpath(temp_dir + '/experiment_artifact'),
env=env.experiment)
except Exception:
- # traceback.print_exc()
+ traceback.print_exc()
sys.exit(2)
sys.exit(subprocess.call(['diffoscope', temp_dir + '/control_artifact', temp_dir + '/experiment_artifact']))
COMMAND_LINE_OPTIONS = types.MappingProxyType(collections.OrderedDict([
('build_command', types.MappingProxyType({
- 'type': str.split, 'default': None, 'nargs': '?',
+ 'default': None, 'nargs': '?', # 'type': str.split
'help': 'Build command to execute.'})),
('artifact', types.MappingProxyType({
'default': None, 'nargs': '?',
diff --git a/reprotest/_shell_ast.py b/reprotest/_shell_ast.py
index 197b160..b6fb6bc 100644
--- a/reprotest/_shell_ast.py
+++ b/reprotest/_shell_ast.py
@@ -23,12 +23,46 @@ which should transform the AST into valid shell code.
'''
import collections
+import itertools
import shlex
+class _SequenceNode(tuple):
+ '''Tuple subclass that returns appropriate types from methods.
+
+ This overloads tuple methods so they return the subclass's type
+ rather than tuple and provides a nicer __repr__.
+
+ '''
+
+ def __add__(self, other):
+ if self.__class__ is other.__class__:
+ return self.__class__(itertools.chain(self, other))
+ else:
+ raise TypeError('Cannot add two shell AST nodes of different types.')
+ __iadd__ = __add__
+
+ def __radd__(self, other):
+ if self.__class__ is other.__class__:
+ return self.__class__(itertools.chain(other, self))
+ else:
+ raise TypeError('Cannot add two shell AST nodes of different types.')
+
+ def __getitem__(self, index):
+ if isinstance(index, slice):
+ return self.__class__(super().__getitem__(index))
+ else:
+ return super().__getitem__(index)
+
+ def __repr__(self):
+ return self.__class__.__name__ + super().__repr__()
+
+
class BaseNode:
'''Abstract base class for all nodes. This class should never be
- instantiated.'''
+ instantiated.
+
+ '''
__slots__ = ()
def __str__(self):
@@ -54,7 +88,7 @@ class Command(BaseNode):
__slots__ = ()
-class List(BaseNode, tuple):
+class List(BaseNode, _SequenceNode):
'''The recursion in this rule is flatted into a sequence.
separator_op is a & or ;.
@@ -78,7 +112,7 @@ class List(BaseNode, tuple):
__slots__ = ()
-class Term(BaseNode, collections.namedtuple('_Term', 'list separator')):
+class Term(BaseNode, collections.namedtuple('_Term', 'command separator')):
'''This rule is recursive in the grammar, but its direct recursion is
handled in List in this AST.
@@ -87,16 +121,16 @@ class Term(BaseNode, collections.namedtuple('_Term', 'list separator')):
term separator and_or | and_or
Attributes:
- list (AndList, OrList, Command): A command or sequence of commands.
+ command (AndList, OrList, Command): A command or sequence of commands.
separator (str): & or ;.
'''
__slots__ = ()
def __str__(self):
- return str(self.list) + ' ' + self.separator + '\n'
+ return str(self.command) + ' ' + self.separator + '\n'
-class AndList(BaseNode, tuple):
+class AndList(BaseNode, _SequenceNode):
'''While the && and || operators are not associative with each other,
each is associative with itself, so this recursion can also be
flattened into a sequence.
@@ -119,7 +153,7 @@ class AndList(BaseNode, tuple):
return ' && '.join(str(field) for field in self)
-class OrList(BaseNode, tuple):
+class OrList(BaseNode, _SequenceNode):
'''While the && and || operators are not associative with each other,
each is associative with itself, so this recursion can also be
flattened into a sequence.
@@ -140,7 +174,7 @@ class OrList(BaseNode, tuple):
return ' || '.join(str(field) for field in self)
-class Pipeline(BaseNode, tuple):
+class Pipeline(BaseNode, _SequenceNode):
'''The recursion in this rule is flatted into a sequence. The option
to prepend the bang (!) to a pipeline is deliberately omitted. It
would require another class because the __str__ method would be
@@ -188,7 +222,7 @@ class SimpleCommand(Command,
str(self.cmd_name) +
(' ' + str(self.cmd_suffix) if self.cmd_suffix else ''))
-class CmdPrefix(BaseNode, tuple):
+class CmdPrefix(BaseNode, _SequenceNode):
'''The recursion in this rule is flatted into a sequence.
Grammar rule:
@@ -247,7 +281,7 @@ class IORedirect(BaseNode,
str(self.filename))
-class CmdSuffix(BaseNode, tuple):
+class CmdSuffix(BaseNode, _SequenceNode):
''''The recursion in this rule is flatted into a sequence. This node
represents the arguments passed to a simple command.
@@ -309,7 +343,7 @@ class ElsePart(BaseNode, collections.namedtuple('_IfClause', 'elifs then')):
return (str(self.elifs) +
('\nelse ' + str(self.then)) if self.then else '')
-class Elifs(BaseNode, tuple):
+class Elifs(BaseNode, _SequenceNode):
'''This node doesn't directly correspond to a grammar rule. It
flattens the recursion for elif statements in else_part.
@@ -371,14 +405,18 @@ class Subshell(Command, collections.namedtuple('_BraceGroup', 'list')):
return '( ' + str(self.list) + ' )'
-class Quote(BaseNode, collections.namedtuple('_Quote', 'command')):
+class Quote(Command, collections.namedtuple('_Quote', 'command')):
'''This is a special node that allows nesting of commands using shell
quoting. For example, to pass a script to a specific shell:
SimpleCommand('', 'bash', CmdSuffix([Quote(<script>)]))
+ This can also be used to insert shell code from other sources into
+ an AST in a proper way.
+
Attributes:
- command (List, Command): AST to quote.
+ command (List, Command, str): AST or string to quote.
+
'''
__slots__ = ()
diff --git a/tests/tests.py b/tests/tests.py
index 1e8d795..70fc2dc 100755
--- a/tests/tests.py
+++ b/tests/tests.py
@@ -1,6 +1,7 @@
# Licensed under the GPL: https://www.gnu.org/licenses/gpl-3.0.en.html
# For details: reprotest/debian/copyright
+# import pathlib
import subprocess
import pytest
@@ -9,27 +10,33 @@ import reprotest
def check_return_code(command, virtual_server, code):
try:
- reprotest.check(command, 'artifact', virtual_server, 'tests/')
+ reprotest.check(command, 'artifact', virtual_server, 'tests')
except SystemExit as system_exit:
assert(system_exit.args[0] == code)
- at pytest.fixture(scope='module', params=['null']) # , 'chroot'
+ at pytest.fixture(scope='module', params=['null', 'schroot'])
def virtual_server(request):
if request.param == 'null':
return [request.param]
+ elif request.param == 'schroot':
+ # TODO: Debian/Ubuntu-specific; this builds the schroot if it
+ # doesn't already exist, requires root.
+ # if not pathlib.Path('/var/lib/schroot/chroots/jessie-amd64/').exists():
+ # subprocess.call(['mk-sbuild', '--debootstrap-include=devscripts', 'stable'])
+ return [request.param, 'stable-amd64']
else:
- return request.param
+ raise ValueError(request.param)
def test_simple_builds(virtual_server):
- check_return_code(['python', 'mock_build.py'], virtual_server, 0)
- # check_return_code(['python', 'mock_failure.py'], virtual_server, 2)
- check_return_code(['python', 'mock_build.py', 'irreproducible'],
+ check_return_code('python3 mock_build.py', virtual_server, 0)
+ # check_return_code('python3 mock_failure.py', virtual_server, 2)
+ check_return_code('python3 mock_build.py irreproducible',
virtual_server, 1)
@pytest.mark.parametrize('variation', ['fileordering', 'home', 'kernel', 'locales', 'path', 'timezone']) #, 'umask'
def test_variations(virtual_server, variation):
- check_return_code(['python', 'mock_build.py', variation], virtual_server, 1)
+ check_return_code('python3 mock_build.py ' + variation, virtual_server, 1)
-def test_self_build():
- assert(subprocess.call(['reprotest', 'python setup.py bdist', 'dist/reprotest-0.1.linux-x86_64.tar.gz', 'null']) == 1)
- assert(subprocess.call(['reprotest', 'debuild -b -uc -us', '../reprotest_0.1_all.deb', 'null']) == 1)
+def test_self_build(virtual_server):
+ assert(subprocess.call(['reprotest', 'python3 setup.py bdist', 'dist/reprotest-0.1.linux-x86_64.tar.gz'] + virtual_server) == 1)
+ assert(subprocess.call(['reprotest', 'debuild -b -uc -us', '../reprotest_0.1_all.deb'] + virtual_server) == 1)
diff --git a/tox.ini b/tox.ini
index 6ac1e9e..8ff1b21 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,6 +1,6 @@
[tox]
-envlist = coverage-clean, py34, py35, coverage-stats
-# envlist = py34, py35
+envlist = coverage-clean, py35, coverage-stats
+# envlist = py35
skip_missing_interpreters = true
[testenv:coverage-clean]
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/reprotest.git
More information about the Reproducible-commits
mailing list