[reprotest] 03/05: Fix the logic of some tests, and don't vary fileordering on Debian buildds
Ximin Luo
infinity0 at debian.org
Fri Sep 23 18:32:50 UTC 2016
This is an automated email from the git hooks/post-receive script.
infinity0 pushed a commit to branch master
in repository reprotest.
commit 0b0e6dc2fd5a2fba00958385d8bf625940b1aad9
Author: Ximin Luo <infinity0 at debian.org>
Date: Fri Sep 23 20:20:43 2016 +0200
Fix the logic of some tests, and don't vary fileordering on Debian buildds
Rename "variation" to "capture" in mock_build.py to be clearer what happens:
we're not varying those things, we're capturing them in the build output.
Capture the whole of `locale`; python getlocale() sometimes isn't enough.
---
debian/control | 5 ++---
debian/rules | 7 +++++--
tests/mock_build.py | 23 +++++++++++++----------
tests/tests.py | 35 +++++++++++++++++++++++------------
tox.ini | 2 +-
5 files changed, 44 insertions(+), 28 deletions(-)
diff --git a/debian/control b/debian/control
index f3a99d7..607bf8f 100644
--- a/debian/control
+++ b/debian/control
@@ -11,14 +11,13 @@ Build-Depends: debhelper (>= 9),
# tests.py uses debuild from devscripts
devscripts <!nocheck>,
diffoscope <!nocheck>,
-# test this variation as well
- disorderfs <!nocheck>,
python3-coverage <!nocheck>,
python3-pytest <!nocheck>,
tox <!nocheck>,
# these below helps diffoscope produce nicer output in tests
python3-tlsh <!nocheck>,
- unzip <!nocheck>
+ unzip <!nocheck>,
+ xxd <!nocheck>
Vcs-Git: https://anonscm.debian.org/git/reproducible/reprotest.git
Vcs-Browser: https://anonscm.debian.org/git/reproducible/reprotest.git
X-Python3-Version: >= 3.5
diff --git a/debian/rules b/debian/rules
index a62c807..f9307b6 100755
--- a/debian/rules
+++ b/debian/rules
@@ -6,12 +6,15 @@ export PYBUILD_NAME = reprotest
%:
dh $@ --with python3 --buildsystem=pybuild
+# It's hard to make sure disorderfs works on buildds, so drop it.
+# The build user needs fuse access and this is not easily arranged.
+export REPROTEST_TEST_DONTVARY = fileordering
+
override_dh_auto_test:
ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
-# FIXME: The patch from #838663 is also required for this to work on pbuilder
VIRTUALENV_DOWNLOAD=no \
http_proxy=http://127.0.9.1:9 \
https_proxy=https://127.0.9.1:9 \
TOX_TESTENV_PASSENV=PYTHONIOENCODING PYTHONIOENCODING=utf-8 \
- REPROTEST_TEST_SERVERS=null tox -r --sitepackages -- -s
+ tox -r --sitepackages -- -s
endif
diff --git a/tests/mock_build.py b/tests/mock_build.py
index 8223fc6..06538ac 100755
--- a/tests/mock_build.py
+++ b/tests/mock_build.py
@@ -15,18 +15,18 @@ if __name__ == '__main__':
arg_parser = argparse.ArgumentParser(
description='Create binaries for testing reproducibility.',
formatter_class=argparse.RawDescriptionHelpFormatter)
- arg_parser.add_argument('variations', nargs='*',
+ arg_parser.add_argument('captures', nargs='*',
help='Reproducibility properties.')
- variations = set(arg_parser.parse_args().variations)
+ captures = set(arg_parser.parse_args().captures)
output = [b'']
# This test can theoretically fail by producing the same
# random bits in both runs, but it is extremely unlikely.
- if 'irreproducible' in variations:
+ if 'irreproducible' in captures:
output.append(os.urandom(1024))
# Like the above test, this test can theoretically fail by
# producing the same file order, but this is unlikely, if not
# as unlikely as in the above test.
- if 'fileordering' in variations:
+ if 'fileordering' in captures:
# Ensure this temporary directory is created in the disorders
# mount point by passing the dir argument.
with tempfile.TemporaryDirectory(dir=str(pathlib.Path.cwd())) as temp:
@@ -34,20 +34,23 @@ if __name__ == '__main__':
for i in range(20):
str((test_file_order/str(i)).touch())
output.extend(p.name.encode('ascii') for p in test_file_order.iterdir())
- if 'home' in variations:
+ if 'home' in captures:
output.append(os.path.expanduser('~').encode('ascii'))
- if 'kernel' in variations:
+ if 'kernel' in captures:
output.append(subprocess.check_output(['uname', '-r']))
- if 'locales' in variations:
+ if 'locales' in captures:
output.extend(l.encode('ascii') if l else b'(None)' for l in locale.getlocale())
- if 'path' in variations:
+ output.append(subprocess.check_output(['locale']))
+ if 'path' in captures:
output.extend(p.encode('ascii') for p in os.get_exec_path())
- if 'timezone' in variations:
+ if 'timezone' in captures:
output.append(str(time.timezone).encode('ascii'))
- if 'umask' in variations:
+ if 'umask' in captures:
with tempfile.TemporaryDirectory(dir=str(pathlib.Path.cwd())) as temp:
test_permissions = pathlib.Path(temp)/'test_permissions'
test_permissions.touch()
output.append(stat.filemode(test_permissions.stat().st_mode).encode('ascii'))
+ else:
+ os.umask(0o0022) # otherwise open() will capture the surrounding one in its file metadata
with open('artifact', 'wb') as artifact:
artifact.write(b''.join(output))
diff --git a/tests/tests.py b/tests/tests.py
index 5d81c5f..3f775b4 100755
--- a/tests/tests.py
+++ b/tests/tests.py
@@ -9,15 +9,25 @@ import pytest
import reprotest
REPROTEST = [sys.executable, "-m", "reprotest"]
+REPROTEST_TEST_SERVERS = os.getenv("REPROTEST_TEST_SERVERS", "null").split(",")
+REPROTEST_TEST_DONTVARY = os.getenv("REPROTEST_TEST_DONTVARY", "").split(",")
+
+if REPROTEST_TEST_DONTVARY:
+ REPROTEST += ["--dont-vary", ",".join(REPROTEST_TEST_DONTVARY)]
+
+TEST_VARIATIONS = frozenset(reprotest.VARIATIONS.keys()) - frozenset(REPROTEST_TEST_DONTVARY)
def check_return_code(command, virtual_server, code):
try:
- retcode = reprotest.check(command, 'artifact', virtual_server, 'tests')
- assert(code == retcode)
+ retcode = reprotest.check(command, 'artifact', virtual_server, 'tests', variations=TEST_VARIATIONS)
except SystemExit as system_exit:
- assert(system_exit.args[0] == code)
+ retcode = system_exit.args[0]
+ finally:
+ if isinstance(code, int):
+ assert(retcode == code)
+ else:
+ assert(retcode in code)
-REPROTEST_TEST_SERVERS = os.getenv("REPROTEST_TEST_SERVERS", "null").split(",")
@pytest.fixture(scope='module', params=REPROTEST_TEST_SERVERS)
def virtual_server(request):
if request.param == 'null':
@@ -30,20 +40,21 @@ def virtual_server(request):
raise ValueError(request.param)
def test_simple_builds(virtual_server):
- # mock_build is not expected to reproduce when disorderfs is active, though
- # we should probably change "1" to int(is_disorderfs_active)
- check_return_code('python3 mock_build.py', virtual_server, 1)
+ check_return_code('python3 mock_build.py', virtual_server, 0)
check_return_code('python3 mock_failure.py', virtual_server, 2)
check_return_code('python3 mock_build.py irreproducible', virtual_server, 1)
- at pytest.mark.parametrize('variation', ['fileordering', 'home', 'kernel', 'locales', 'path', 'timezone', 'umask'])
-def test_variations(virtual_server, variation):
- check_return_code('python3 mock_build.py ' + variation, virtual_server, 1)
+ at pytest.mark.parametrize('captures', ['fileordering', 'home', 'kernel', 'locales', 'path', 'timezone', 'umask'])
+def test_variations(virtual_server, captures):
+ expected = 1 if captures in TEST_VARIATIONS else 0
+ check_return_code('python3 mock_build.py ' + captures, virtual_server, expected)
def test_self_build(virtual_server):
- assert(1 == subprocess.call(REPROTEST + ['python3 setup.py bdist', 'dist/*.tar.gz'] + virtual_server))
# at time of writing (2016-09-23) these are not expected to reproduce;
- # strip-nondeterminism normalises them for Debian
+ # if these start failing then you should change 1 == to 0 == but please
+ # figure out which version of setuptools made things reproduce and add a
+ # versioned dependency on that one
+ assert(1 == subprocess.call(REPROTEST + ['python3 setup.py bdist', 'dist/*.tar.gz'] + virtual_server))
assert(1 == subprocess.call(REPROTEST + ['python3 setup.py sdist 2>/dev/null', 'dist/*.tar.gz'] + virtual_server))
assert(1 == subprocess.call(REPROTEST + ['python3 setup.py bdist_wheel', 'dist/*.whl'] + virtual_server))
diff --git a/tox.ini b/tox.ini
index 3c8aae1..feb0ba8 100644
--- a/tox.ini
+++ b/tox.ini
@@ -15,7 +15,7 @@ commands =
{envpython} -m coverage html
[testenv]
-passenv = REPROTEST_TEST_SERVERS VIRTUALENV_DOWNLOAD *_proxy
+passenv = REPROTEST_TEST_* VIRTUALENV_DOWNLOAD *_proxy
# usedevelop = True
deps =
coverage
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/reprotest.git
More information about the Reproducible-commits
mailing list