[kup] 06/18: Support separate compress/decompress commands
debian-kernel at lists.debian.org
debian-kernel at lists.debian.org
Sun Apr 9 02:47:29 UTC 2017
This is an automated email from the git hooks/post-receive script.
benh pushed a commit to branch master
in repository kup.
commit 6dbcf542d73e075e08a83af2310bd185eedc4b40
Author: Konstantin Ryabitsev <konstantin at linuxfoundation.org>
Date: Fri Mar 24 17:36:39 2017 -0400
Support separate compress/decompress commands
Change kup-server so that separate compress/decompress commands can be
specified for each compressor. This allows us to use parallelizing
compressors such as pigz and pixz without impacting our ability to
decompress incoming tarballs.
Signed-off-by: Konstantin Ryabitsev <konstantin at linuxfoundation.org>
---
ChangeLog | 5 +++++
kup-server | 54 ++++++++++++++++++++++++++++++++++++------------------
kup-server.1 | 24 ++++++++++++------------
kup-server.cfg | 19 +++++++++++--------
4 files changed, 64 insertions(+), 38 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index db969aa..8be93b9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -3,6 +3,11 @@
kup is used with another authZ system relying on ssh (specifically,
gitolite).
* Small typo fixes
+ * Change kup-server so that separate compress/decompress commands can be
+ specified for each compressor. This allows us to use parallelizing
+ compressors such as pigz and pixz without impacting our ability to
+ decompress incoming tarballs.
+
2012-12-10 Konstantin Ryabitsev <mricon at kernel.org> - 0.3.4
* Calculate and log sha256sums of all uploaded files for forensic
record-keeping.
diff --git a/kup-server b/kup-server
index 02dd502..5c7f40d 100755
--- a/kup-server
+++ b/kup-server
@@ -134,20 +134,41 @@ my %zformats;
if (defined($cfg->param('compressors.use'))) {
- foreach my $zformat ($cfg->param('compressors.use')) {
- # Do we have a path defined?
- if (!defined($cfg->param("compressors.${zformat}"))) {
- fatal("Compressor ${zformat} requested, but path not specified.");
- }
- my $ext = '.' . $zformat;
- $zformats{$ext} = $cfg->param("compressors.${zformat}");
- }
+ foreach my $zformat ($cfg->param('compressors.use')) {
+ my $ext = '.' . $zformat;
+ # Do we have a path defined?
+ if (!defined($cfg->param("compressors.${zformat}"))) {
+ # Do we have a section with compress/decompress commands?
+ if (!defined($cfg->param("${zformat}.compress_cmd"))) {
+ fatal("Compressor ${zformat} requested, but commands not specified.");
+ }
+ $zformats{$ext} = {
+ 'compress' => $cfg->param("${zformat}.compress_cmd"),
+ 'decompress' => $cfg->param("${zformat}.decompress_cmd"),
+ }
+ } else {
+ # We have old-style definition, so just add -9 and -cd to it
+ $zformats{$ext} = {
+ 'compress' => $cfg->param("compressors.${zformat}") . " -9",
+ 'decompress' => $cfg->param("compressors.${zformat}") . " -cd",
+ }
+ }
+ }
} else {
%zformats = (
- '.gz' => '/bin/gzip',
- '.bz2' => '/usr/bin/bzip2',
- '.xz' => '/usr/bin/xz'
+ '.gz' => {
+ 'compress' => '/bin/gzip -9',
+ 'decompress' => '/bin/gzip -cd',
+ },
+ '.bz2' => {
+ 'compress' => '/usr/bin/bzip2 -9',
+ 'decompress' => '/usr/bin/bzip2 -cd',
+ },
+ '.xz' => {
+ 'compress' => '/usr/bin/xz -9',
+ 'decompress' => '/usr/bin/xz -cd',
+ },
);
}
@@ -370,7 +391,7 @@ sub get_blob($$@)
if ($format eq '') {
undef $zcmd;
- } elsif (!defined($zcmd = $zformats{'.'.$format})) {
+ } elsif (!defined($zcmd = $zformats{'.'.$format}{'decompress'})) {
fatal("Unsupported compression format");
}
@@ -387,12 +408,13 @@ sub get_blob($$@)
binmode($outfd);
if (defined($zcmd)) {
+ my @c = split(/\s+/, $zcmd);
open($oldstdout, '>&', \*STDOUT) or die;
open(STDOUT, '>&', $outfd) or die;
close($outfd);
undef $outfd;
- open($outfd, '|-', $zcmd, '-cd') or die;
+ open($outfd, '|-', @c) or die;
binmode($outfd);
open(STDOUT, '>&', $oldstdout) or die;
@@ -615,11 +637,7 @@ sub make_compressed_data()
my $tarsize = -s $tmpdir.'/data';
foreach my $e (keys(%zformats)) {
- my @c = ($zformats{$e}, '-9');
- if ($zformats{$e} =~ /\s+/) {
- @c = split(/\s+/, $zformats{$e});
- push (@c, '-9');
- }
+ my @c = split(/\s+/, $zformats{$e}{'compress'});
sysopen($infds{$e}, $tmpdir.'/data', O_RDONLY) or
fatal("Failed to open uncompressed data file");
diff --git a/kup-server.1 b/kup-server.1
index 8888aee..2143090 100644
--- a/kup-server.1
+++ b/kup-server.1
@@ -98,23 +98,23 @@ Each compression command must take at most this long in CPU time.
This section allows specifying the compressors to use when creating
compressed versions of uploaded content.
.TP
-\fBuse\fP = \fIgz, bz2, xz\fP
-A comma-separated list of file extensions to create (minus the leading dot).
-For each extension specified, you will need to add an extra entry to this
-section with the path to the matching gzip-compatible utility (i.e. it
-must accept \fI-9\fP and \fI-cd\fP command-line arguments). E.g., if you
-specified "\fIgz, bz2, xz\fP" as values in \fBuse\fP, you must add the
-following entries as well:
+\fBuse\fP = \fIgz, xz\fP
+A comma-separated list of file extensions to create (minus the leading dot).
+For each extension specified, you will need to add a matching section specifying
+which command and flags to use for decompression and which for compression. Make
+sure to configure the decompress command to output to stdout. E.g.:
.PP
.RS
-.RS
.nf
-gz = /bin/gzip
-bz2 = /usr/bin/bzip2
-xz = /usr/bin/xz
+[gz]
+compress_command = /bin/pigz -9
+decompress_command = /bin/gzip -cd
+
+[xz]
+compress_command = /bin/xz -9 -T0
+decompress_command = /bin/xz -cd
.fi
.RE
-.RE
.SH AUTHOR
Written by H. Peter Anvin <hpa at zytor.com>.
.SH COPYRIGHT
diff --git a/kup-server.cfg b/kup-server.cfg
index 17b00e2..1ece762 100644
--- a/kup-server.cfg
+++ b/kup-server.cfg
@@ -55,11 +55,14 @@ timeout_compress_cpu = 900
[compressors]
; Specify which compressors to use, separated by comma. These must match the
; file extensions that will be added to the compressed file (after the dot).
-use = gz, bz2, xz
-;
-; Specify paths to each compressor listed above. Each of these must accept
-; "-9" as commandline parameter for compression and "-cd" for decompression
-; to stdout.
-gz = /bin/gzip
-bz2 = /usr/bin/bzip2
-xz = /usr/bin/xz
+use = gz, xz
+
+; Specify which commands should be used for compression and decompression of
+; each archival format. Make sure the decompression command outputs to stdout.
+[gz]
+compress_command = /bin/pigz -9
+decompress_command = /bin/gzip -cd
+
+[xz]
+compress_command = /bin/xz -9 -T0
+decompress_command = /bin/xz -cd
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/kernel/kup.git
More information about the Kernel-svn-changes
mailing list