[debhelper-devel] [debhelper] 04/08: Dh_Lib: Add on_pkgs_in_parallel

[Niels Thykier]

This is an automated email from the git hooks/post-receive script.

nthykier pushed a commit to branch master
in repository debhelper.

commit 9165ecdcacc70aee5d7b53ca607ad93de2ae7122
Author: Niels Thykier <niels at thykier.net>
Date:   Sun Jun 4 18:58:42 2017 +0000

    Dh_Lib: Add on_pkgs_in_parallel
    
    Signed-off-by: Niels Thykier <niels at thykier.net>
    
    fixup! Dh_Lib: Add on_each_pkg_in_parallel
---
 Debian/Debhelper/Dh_Lib.pm | 55 ++++++++++++++++++++++++++++++++++++++++++++++
 doc/PROGRAMMING            | 18 +++++++++++++++
 2 files changed, 73 insertions(+)

diff --git a/Debian/Debhelper/Dh_Lib.pm b/Debian/Debhelper/Dh_Lib.pm
index 6de5d42..ac00fb0 100644
--- a/Debian/Debhelper/Dh_Lib.pm
+++ b/Debian/Debhelper/Dh_Lib.pm
@@ -52,11 +52,14 @@ use vars qw(@EXPORT %dh);
 	    &restore_file_on_clean &restore_all_files
 	    &open_gz &reset_perm_and_owner &deprecated_functionality
 	    &log_installed_files &buildarch &rename_path
+	    &on_each_pkg_in_parallel
 );
 
 # The Makefile changes this if debhelper is installed in a PREFIX.
 my $prefix="/usr";
 
+my $MAX_PROCS = get_buildoption("parallel") || 1;
+
 sub init {
 	my %params=@_;
 
@@ -1599,6 +1602,58 @@ sub log_installed_files {
 	return 1;
 }
 
+sub on_pkgs_in_parallel(&) {
+	my ($code) = @_;
+	my @pkgs = @{$dh{DOPACKAGES}};
+	my %pids;
+	my $parallel = $MAX_PROCS;
+	my $count_per_proc = int(scalar(@pkgs) / $parallel);
+	my $exit = 0;
+	if ($count_per_proc < 1) {
+		$count_per_proc = 1;
+		if (@pkgs > 3) {
+			# Forking has a considerable overhead, so bulk the number
+			# a bit.  We do not do this unconditionally, because we
+			# want parallel issues (if any) to appear already with 2
+			# packages and two procs (because people are lazy when
+			# testing).
+			#
+			# Same reason for also unconditionally forking with 1 pkg
+			# in 1 proc.
+			$count_per_proc = 2;
+		}
+	}
+	# Assertion, $count_per_proc * $parallel >= scalar(@pkgs)
+	while (@pkgs) {
+		my @batch = splice(@pkgs, 0, $count_per_proc);
+		my $pid = fork() // error("fork: $!");
+		if (not $pid) {
+			eval {
+				$code->(@batch);
+			};
+			if (my $err = $@) {
+				$err =~ s/\n$//;
+				print STDERR "$err\n";
+				exit(2);
+			}
+			exit(0);
+		}
+		$pids{$pid} = 1;
+	}
+	while (%pids) {
+		my $pid = wait;
+		error("wait() failed: $!") if $pid == -1;
+		delete($pids{$pid});
+		if ($? != 0) {
+			$exit = 1;
+		}
+	}
+	if ($exit) {
+		error("Aborting due to earlier error");
+	}
+	return;
+}
+
 1
 
 # Local Variables:
diff --git a/doc/PROGRAMMING b/doc/PROGRAMMING
index 0236280..1827811 100644
--- a/doc/PROGRAMMING
+++ b/doc/PROGRAMMING
@@ -357,6 +357,24 @@ log_installed_files($package, @paths)
 	(e.g. avoid "foo/../bar" or "foo/./bar")
 	If a directory is listed, it and all paths recursively beneath is
 	also considered installed.
+on_pkgs_in_parallel($code) - prototype: (&)
+	Splits all the packages in $dh{DOPACKAGES} into a number of groups
+	based on the max parallel (as decided by DEB_BUILD_OPTIONS)
+	A subprocess is forked for each group (minimum 1 process will be
+	forked) and each subprocess will be added a group of packages
+	to process.  Each group is passed to the $code sub, which will
+	then process it and return normally on success.
+	Example:
+		on_pkgs_in_parallel {
+			for my $package (@_) {
+				my $tmp=tmpdir($package);
+				my $pkgfile = pkgfile($package, 'foo');
+				...;
+			}
+		 }
+
+	Keep in mind that the sub will always be run in a subprocess,
+	so it cannot update global state.
 
 Sequence Addons:
 ---------------

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debhelper/debhelper.git