[reprotest] 01/01: build: add num_cpu variation

[Ximin Luo]

This is an automated email from the git hooks/post-receive script.

infinity0 pushed a commit to branch master
in repository reprotest.

commit d5dd4dfe88a467bb65e9cbe6249a9d383eed9b1c
Author: Ximin Luo <infinity0 at debian.org>
Date:   Fri Nov 24 20:40:49 2017 +0100

    build: add num_cpu variation
---
 README.rst            |  8 ++++++++
 debian/changelog      |  2 +-
 reprotest/__init__.py |  4 +++-
 reprotest/build.py    | 25 ++++++++++++++++++++++---
 tests/mock_build.py   |  2 ++
 5 files changed, 36 insertions(+), 5 deletions(-)

diff --git a/README.rst b/README.rst
index 9e712e0..a9f571c 100644
--- a/README.rst
+++ b/README.rst
@@ -284,6 +284,14 @@ We will hopefully lift these assumptions for certain virtual_server contexts,
 in future. That would likely allow for smoother operation in those contexts.
 The assumptions will remain for the "null" (default) virtual_server however.
 
+Number of CPUs
+--------------
+
+The control build uses only 1 CPU in order to try to reduce nondeterminism that
+might exist due to multithreading or multiprocessing. If you are sure your
+build is not affected by this (and good builds ought not to be), you can give
+--min-cpus=99999 to use all available cores for both builds.
+
 Domain or host
 --------------
 
diff --git a/debian/changelog b/debian/changelog
index af0b9b1..962139e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,7 +1,7 @@
 reprotest (0.7.4) UNRELEASED; urgency=medium
 
   * Hopefully fix the autopkgtest tests.
-  * Add aslr and domain_host variations.
+  * Add aslr, domain_host, and num_cpu variations.
   * Add a --print-sudoers feature.
   * Properly drop privs when running the build. (Closes: #877813)
 
diff --git a/reprotest/__init__.py b/reprotest/__init__.py
index 4494529..3a165ea 100644
--- a/reprotest/__init__.py
+++ b/reprotest/__init__.py
@@ -581,6 +581,8 @@ def cli_parser():
         'whitelist and blacklist. You probably want to set --vary=-all as well '
         'when setting this flag; see the man page for details. Conflicts with '
         '--extra-build and --auto-build.')
+    group1_0.add_argument('--min-cpus', default=1, type=int, metavar='NUM',
+        help='Minimum CPUs to use when fixing num_cpus. Default: %(default)s')
     # TODO: remove after reprotest 0.8
     group1.add_argument('--dont-vary', default=[], action='append', help=argparse.SUPPRESS)
 
@@ -767,7 +769,7 @@ def run(argv, dry_run=None):
         for extra_build in parsed_args.extra_build:
             specs.append(spec.extend(extra_build))
         check_func = check
-    build_variations = Variations.of(*specs, verbosity=verbosity)
+    build_variations = Variations.of(*specs, verbosity=verbosity, min_cpus=parsed_args.min_cpus)
 
     # Warn about missing programs
     if virtual_server_args[0] == "null" and not dry_run:
diff --git a/reprotest/build.py b/reprotest/build.py
index 4c9eba8..70554dd 100644
--- a/reprotest/build.py
+++ b/reprotest/build.py
@@ -300,6 +300,24 @@ def aslr(ctx, build, vary):
         return build
     return build.append_setup_exec_raw('SETARCH_OPTS="$SETARCH_OPTS -R"')
 
+def num_cpus(ctx, build, vary):
+    _ = build
+    _ = _.append_setup_exec_raw('CPU_MAX=$(nproc --all)')
+    _ = _.append_setup_exec_raw('CPU_MIN=$({ echo $CPU_MAX; echo %s; } | sort -n | head -n1)' % ctx.min_cpus)
+    if ctx.min_cpus <= 0:
+        raise ValueError("--min-cpus must be a positive integer: " % ctx.min_cpus)
+    if not vary:
+        if ctx.min_cpus == 1:
+            logger.warn("The control build runs on 1 CPU by default, give --min-cpus to increase this.")
+        _ = _.append_setup_exec_raw('CPU_NUM=$CPU_MIN')
+    else:
+        # random number between min_cpus and $(nproc --all)
+        _ = _.append_setup_exec_raw('CPU_NUM=$(shuf -i$CPU_MIN-$CPU_MAX -n1)')
+
+    # select CPU_NUM random cpus from the range 0..$((CPU_MAX-1))
+    cpu_list = "$(echo $(shuf -i0-$((CPU_MAX - 1)) -n$CPU_NUM) | tr ' ' ,)"
+    return _.prepend_to_build_command_raw('taskset', '-a', '-c', cpu_list)
+
 # TODO: if this locale doesn't exist on the system, Python's
 # locales.getlocale() will return (None, None) rather than this
 # locale.  I imagine it will also probably cause false positives with
@@ -452,6 +470,7 @@ VARIATIONS = collections.OrderedDict([
     ('aslr', aslr), # needs to run after kernel which runs "setarch"
                     # but also as close to the build command as possible, (i.e. earlier in this list)
                     # otherwise other variations below can affect the address layout
+    ('num_cpus', num_cpus),
     ('user_group', user_group),
     # ('cpu', cpu),
     ('fileordering', fileordering),
@@ -560,10 +579,10 @@ class VariationSpec(mdiffconf.ImmutableNamespace):
         })
 
 
-class Variations(collections.namedtuple('_Variations', 'spec verbosity')):
+class Variations(collections.namedtuple('_Variations', 'spec verbosity min_cpus')):
     @classmethod
-    def of(cls, *specs, zero=VariationSpec.empty(), verbosity=0):
-        return [cls(spec, verbosity) for spec in [zero] + list(specs)]
+    def of(cls, *specs, zero=VariationSpec.empty(), verbosity=0, min_cpus=1):
+        return [cls(spec, verbosity, min_cpus) for spec in [zero] + list(specs)]
 
     @property
     def replace(self):
diff --git a/tests/mock_build.py b/tests/mock_build.py
index 5b2ea74..536ac5f 100755
--- a/tests/mock_build.py
+++ b/tests/mock_build.py
@@ -49,6 +49,8 @@ if __name__ == '__main__':
         output.append(os.path.expanduser('~'))
     if 'kernel' in captures:
         output.append(subprocess.check_output(['uname', '-r']).decode("utf-8"))
+    if 'num_cpus' in captures:
+        output.append(subprocess.check_output(['nproc']).decode("utf-8"))
     if 'locales' in captures:
         output.extend(l or '(None)' for l in locale.getlocale())
         output.append(subprocess.check_output(['locale']).decode("utf-8"))

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/reprotest.git