[reprotest] 03/05: Fix the logic of some tests, and don't vary fileordering on Debian buildds

Ximin Luo infinity0 at debian.org
Fri Sep 23 18:32:50 UTC 2016


This is an automated email from the git hooks/post-receive script.

infinity0 pushed a commit to branch master
in repository reprotest.

commit 0b0e6dc2fd5a2fba00958385d8bf625940b1aad9
Author: Ximin Luo <infinity0 at debian.org>
Date:   Fri Sep 23 20:20:43 2016 +0200

    Fix the logic of some tests, and don't vary fileordering on Debian buildds
    
    Rename "variation" to "capture" in mock_build.py to be clearer what happens:
    we're not varying those things, we're capturing them in the build output.
    
    Capture the whole of `locale`; python getlocale() sometimes isn't enough.
---
 debian/control      |  5 ++---
 debian/rules        |  7 +++++--
 tests/mock_build.py | 23 +++++++++++++----------
 tests/tests.py      | 35 +++++++++++++++++++++++------------
 tox.ini             |  2 +-
 5 files changed, 44 insertions(+), 28 deletions(-)

diff --git a/debian/control b/debian/control
index f3a99d7..607bf8f 100644
--- a/debian/control
+++ b/debian/control
@@ -11,14 +11,13 @@ Build-Depends: debhelper (>= 9),
 # tests.py uses debuild from devscripts
  devscripts <!nocheck>,
  diffoscope <!nocheck>,
-# test this variation as well
- disorderfs <!nocheck>,
  python3-coverage <!nocheck>,
  python3-pytest <!nocheck>,
  tox <!nocheck>,
 # these below helps diffoscope produce nicer output in tests
  python3-tlsh <!nocheck>,
- unzip <!nocheck>
+ unzip <!nocheck>,
+ xxd <!nocheck>
 Vcs-Git: https://anonscm.debian.org/git/reproducible/reprotest.git
 Vcs-Browser: https://anonscm.debian.org/git/reproducible/reprotest.git
 X-Python3-Version: >= 3.5
diff --git a/debian/rules b/debian/rules
index a62c807..f9307b6 100755
--- a/debian/rules
+++ b/debian/rules
@@ -6,12 +6,15 @@ export PYBUILD_NAME = reprotest
 %:
 	dh $@ --with python3 --buildsystem=pybuild
 
+# It's hard to make sure disorderfs works on buildds, so drop it.
+# The build user needs fuse access and this is not easily arranged.
+export REPROTEST_TEST_DONTVARY = fileordering
+
 override_dh_auto_test:
 ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
-# FIXME: The patch from #838663 is also required for this to work on pbuilder
 	VIRTUALENV_DOWNLOAD=no \
 	http_proxy=http://127.0.9.1:9 \
 	https_proxy=https://127.0.9.1:9 \
 	TOX_TESTENV_PASSENV=PYTHONIOENCODING PYTHONIOENCODING=utf-8 \
-	REPROTEST_TEST_SERVERS=null tox -r --sitepackages -- -s
+	tox -r --sitepackages -- -s
 endif
diff --git a/tests/mock_build.py b/tests/mock_build.py
index 8223fc6..06538ac 100755
--- a/tests/mock_build.py
+++ b/tests/mock_build.py
@@ -15,18 +15,18 @@ if __name__ == '__main__':
     arg_parser = argparse.ArgumentParser(
         description='Create binaries for testing reproducibility.',
         formatter_class=argparse.RawDescriptionHelpFormatter)
-    arg_parser.add_argument('variations', nargs='*',
+    arg_parser.add_argument('captures', nargs='*',
                             help='Reproducibility properties.')
-    variations = set(arg_parser.parse_args().variations)
+    captures = set(arg_parser.parse_args().captures)
     output = [b'']
     # This test can theoretically fail by producing the same
     # random bits in both runs, but it is extremely unlikely.
-    if 'irreproducible' in variations:
+    if 'irreproducible' in captures:
         output.append(os.urandom(1024))
     # Like the above test, this test can theoretically fail by
     # producing the same file order, but this is unlikely, if not
     # as unlikely as in the above test.
-    if 'fileordering' in variations:
+    if 'fileordering' in captures:
         # Ensure this temporary directory is created in the disorders
         # mount point by passing the dir argument.
         with tempfile.TemporaryDirectory(dir=str(pathlib.Path.cwd())) as temp:
@@ -34,20 +34,23 @@ if __name__ == '__main__':
             for i in range(20):
                 str((test_file_order/str(i)).touch())
             output.extend(p.name.encode('ascii') for p in test_file_order.iterdir())
-    if 'home' in variations:
+    if 'home' in captures:
         output.append(os.path.expanduser('~').encode('ascii'))
-    if 'kernel' in variations:
+    if 'kernel' in captures:
         output.append(subprocess.check_output(['uname', '-r']))
-    if 'locales' in variations:
+    if 'locales' in captures:
         output.extend(l.encode('ascii') if l else b'(None)' for l in locale.getlocale())
-    if 'path' in variations:
+        output.append(subprocess.check_output(['locale']))
+    if 'path' in captures:
         output.extend(p.encode('ascii') for p in os.get_exec_path())
-    if 'timezone' in variations:
+    if 'timezone' in captures:
         output.append(str(time.timezone).encode('ascii'))
-    if 'umask' in variations:
+    if 'umask' in captures:
         with tempfile.TemporaryDirectory(dir=str(pathlib.Path.cwd())) as temp:
             test_permissions = pathlib.Path(temp)/'test_permissions'
             test_permissions.touch()
             output.append(stat.filemode(test_permissions.stat().st_mode).encode('ascii'))
+    else:
+        os.umask(0o0022) # otherwise open() will capture the surrounding one in its file metadata
     with open('artifact', 'wb') as artifact:
         artifact.write(b''.join(output))
diff --git a/tests/tests.py b/tests/tests.py
index 5d81c5f..3f775b4 100755
--- a/tests/tests.py
+++ b/tests/tests.py
@@ -9,15 +9,25 @@ import pytest
 import reprotest
 
 REPROTEST = [sys.executable, "-m", "reprotest"]
+REPROTEST_TEST_SERVERS = os.getenv("REPROTEST_TEST_SERVERS", "null").split(",")
+REPROTEST_TEST_DONTVARY = os.getenv("REPROTEST_TEST_DONTVARY", "").split(",")
+
+if REPROTEST_TEST_DONTVARY:
+    REPROTEST += ["--dont-vary", ",".join(REPROTEST_TEST_DONTVARY)]
+
+TEST_VARIATIONS = frozenset(reprotest.VARIATIONS.keys()) - frozenset(REPROTEST_TEST_DONTVARY)
 
 def check_return_code(command, virtual_server, code):
     try:
-        retcode = reprotest.check(command, 'artifact', virtual_server, 'tests')
-        assert(code == retcode)
+        retcode = reprotest.check(command, 'artifact', virtual_server, 'tests', variations=TEST_VARIATIONS)
     except SystemExit as system_exit:
-        assert(system_exit.args[0] == code)
+        retcode = system_exit.args[0]
+    finally:
+        if isinstance(code, int):
+            assert(retcode == code)
+        else:
+            assert(retcode in code)
 
-REPROTEST_TEST_SERVERS = os.getenv("REPROTEST_TEST_SERVERS", "null").split(",")
 @pytest.fixture(scope='module', params=REPROTEST_TEST_SERVERS)
 def virtual_server(request):
     if request.param == 'null':
@@ -30,20 +40,21 @@ def virtual_server(request):
         raise ValueError(request.param)
 
 def test_simple_builds(virtual_server):
-    # mock_build is not expected to reproduce when disorderfs is active, though
-    # we should probably change "1" to int(is_disorderfs_active)
-    check_return_code('python3 mock_build.py', virtual_server, 1)
+    check_return_code('python3 mock_build.py', virtual_server, 0)
     check_return_code('python3 mock_failure.py', virtual_server, 2)
     check_return_code('python3 mock_build.py irreproducible', virtual_server, 1)
 
- at pytest.mark.parametrize('variation', ['fileordering', 'home', 'kernel', 'locales', 'path', 'timezone', 'umask'])
-def test_variations(virtual_server, variation):
-    check_return_code('python3 mock_build.py ' + variation, virtual_server, 1)
+ at pytest.mark.parametrize('captures', ['fileordering', 'home', 'kernel', 'locales', 'path', 'timezone', 'umask'])
+def test_variations(virtual_server, captures):
+    expected = 1 if captures in TEST_VARIATIONS else 0
+    check_return_code('python3 mock_build.py ' + captures, virtual_server, expected)
 
 def test_self_build(virtual_server):
-    assert(1 == subprocess.call(REPROTEST + ['python3 setup.py bdist', 'dist/*.tar.gz'] + virtual_server))
     # at time of writing (2016-09-23) these are not expected to reproduce;
-    # strip-nondeterminism normalises them for Debian
+    # if these start failing then you should change 1 == to 0 == but please
+    # figure out which version of setuptools made things reproduce and add a
+    # versioned dependency on that one
+    assert(1 == subprocess.call(REPROTEST + ['python3 setup.py bdist', 'dist/*.tar.gz'] + virtual_server))
     assert(1 == subprocess.call(REPROTEST + ['python3 setup.py sdist 2>/dev/null', 'dist/*.tar.gz'] + virtual_server))
     assert(1 == subprocess.call(REPROTEST + ['python3 setup.py bdist_wheel', 'dist/*.whl'] + virtual_server))
 
diff --git a/tox.ini b/tox.ini
index 3c8aae1..feb0ba8 100644
--- a/tox.ini
+++ b/tox.ini
@@ -15,7 +15,7 @@ commands =
   {envpython} -m coverage html
 
 [testenv]
-passenv = REPROTEST_TEST_SERVERS VIRTUALENV_DOWNLOAD *_proxy
+passenv = REPROTEST_TEST_* VIRTUALENV_DOWNLOAD *_proxy
 # usedevelop = True
 deps =
   coverage

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/reprotest.git



More information about the Reproducible-commits mailing list