[Debian-ports-devel] [PATCH] Add automatic by-hand processing of d-i's raw-installer images
James Clarke
jrtc27 at debian.org
Sat Feb 11 21:56:17 UTC 2017
---
Hi,
Since debian-installer has now been fixed for (some) ports, the next
upload will see the buildds uploading installer images. Currently
mini-dak does not process these; this patch intends to function
similarly to dak's auto by-hand processing of d-i images. The
archive-byhand-di was taken from scripts/debian/byhand-di and slightly
modified to use mini-dak's config.
I have tested this with a local mini-dak setup and it seems to work,
unpacking to the installer-$arch directory. However, I expect there are
things I have missed, and this may not be implemented in the way you
would like. Comments?
Regards,
James
bin/archive-byhand-di | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++
bin/archive-lib | 8 ++++
bin/archive-queue | 98 ++++++++++++++++++++++++++++++++++++++++++--
bin/archive-setup | 1 +
4 files changed, 215 insertions(+), 3 deletions(-)
diff --git a/bin/archive-byhand-di b/bin/archive-byhand-di
new file mode 100755
index 0000000..6443a4d
--- /dev/null
+++ b/bin/archive-byhand-di
@@ -0,0 +1,111 @@
+#!/bin/bash
+
+. archive-lib
+
+set -u
+set -e
+set -o pipefail
+
+if [ $# -lt 5 ]; then
+ echo "Usage: $0 filename version arch changes_file suite"
+ exit 1
+fi
+
+TARBALL="$1" # Tarball to read, compressed with gzip
+VERSION="$2"
+ARCH="$3"
+CHANGES="$4" # Changes file for the upload
+SUITE="$5"
+
+error() {
+ echo "$*"
+ exit 1
+}
+
+# Check validity of version number
+# Expected are: YYYYMMDD, YYYYMMDD.x, YYYYMMDD<suite>x, YYYYMMDD+<suite>x,
+# YYYYMMDD+debXuZ and the +b[0-9] on the end
+if ! echo "$VERSION" | grep -Eq "^[0-9]{8}((\.|\+?[a-z]+|\+deb[0-9]+u|\+kbsd[0-9]+u)[0-9]+)?(\+b[0-9])?$"; then
+ error "Invalid version: '$VERSION'"
+fi
+
+case $SUITE in
+ unstable|sid|*proposed-updates)
+ : # nothing to do
+ ;;
+ *)
+ SUITE="${SUITE}-proposed-updates"
+ ;;
+esac
+
+# This must end with /
+TARGET="$dists_dir/$SUITE/main/installer-$ARCH/"
+
+# Check validity of the target directory
+# This could fail, for example for new architectures; doing
+# a regular BYHAND is safer in that case
+if [ ! -d "$TARGET" ]; then
+ mkdir -p "$TARGET"
+fi
+# Check that there isn't already a directory for this version
+if [ -d "$TARGET/$VERSION" ]; then
+ error "Directory already exists: $TARGET/$VERSION"
+fi
+
+# We know the VERSION is sane by here, we just need to make sure we escape the + in +b1 (if any)
+# It needs 'g' as well as we may have +$DIST+b[0-9] or +debXuZ+bY
+VERSIONREGEXP="$(echo $VERSION | sed 's at +@\\\+ at g')"
+
+# We know all data to be in ./installer-<arch>/<version>; see if there's
+# anything else in the tarball except that and the 'current' symlink
+if tar tzf "$TARBALL" | \
+ grep -Eqv "^\./(installer-$ARCH/($VERSIONREGEXP/.*|current|)|)$"; then
+ error "Tarball contains unexpected contents"
+fi
+
+# Create a temporary directory where to store the images
+umask 002
+TMPDIR="$(mktemp -td byhand-di.XXXXXX)"
+
+# If we fail somewhere, cleanup the temporary directory
+cleanup() {
+ rm -rf "$TMPDIR"
+}
+trap cleanup EXIT
+
+# Extract the data into the temporary directory
+tar xzf "$TARBALL" --directory="$TMPDIR" "./installer-$ARCH/"
+
+# Check the 'current' symlink
+if [ ! -L $TMPDIR/installer-$ARCH/current ]; then
+ error "Missing 'current' symlink"
+elif [ X"$(readlink "$TMPDIR/installer-$ARCH/current")" != X"$VERSION" ]; then
+ error "Incorrect 'current' symlink"
+fi
+
+# We should have an MD5SUMS file; use that for a final check
+if [ -r "$TMPDIR/installer-$ARCH/$VERSION/images/MD5SUMS" ]; then
+ (
+ cd "$TMPDIR/installer-$ARCH/$VERSION/images"
+ md5sum -c --status MD5SUMS || error "Error while checking MD5SUMS"
+ )
+else
+ error "Missing MD5SUMS file"
+fi
+
+# Move the data to the final location
+mv "$TMPDIR/installer-$ARCH/$VERSION" "$TARGET"
+mv "$TMPDIR/installer-$ARCH/current" "$TARGET"
+
+# Fixup permissions
+find "$TARGET/$VERSION" -type d -exec chmod 755 {} +
+find "$TARGET/$VERSION" -type f -exec chmod 644 {} +
+
+# Make sure nothing symlinks outside of the ftpdir
+# Shouldnt happen, but better be sure.
+symlinks -d -r $archive_dir
+
+trap - EXIT
+cleanup
+
+exit 0
diff --git a/bin/archive-lib b/bin/archive-lib
index 297c025..88c0c1c 100755
--- a/bin/archive-lib
+++ b/bin/archive-lib
@@ -38,6 +38,7 @@ rejected_daily_dir=$rejected_dir/`date -I`
unchecked_dir=$queue_dir/unchecked
byhand_dir=$queue_dir/byhand
buildd_dir=$accepted_dir/buildd
+bin_dir=$(readlink -f .)
######
# file input functions
@@ -64,6 +65,13 @@ fetch_files ()
formail -xFiles: | cut -d' ' -f6
}
+fetch_file_section ()
+{
+ local file=$1
+
+ formail -xFiles: | awk '$5 == "'"$file"'"{print $3}'
+}
+
fetch_md5sums_changes ()
{
formail -xFiles: | cut -d' ' -f2,6 | sed -e 's/ / /'
diff --git a/bin/archive-queue b/bin/archive-queue
index 1268e68..f5eff0d 100755
--- a/bin/archive-queue
+++ b/bin/archive-queue
@@ -71,7 +71,9 @@ verify_gpg_signature ()
{
local changes_file=$1
local archive_file=$2
+ local source_name=
local arch=`strip_gpg < $changes_file | fetch_field "Architecture"`
+ local allowed_regex="_${arch}\.(buildinfo|deb|udeb)$|_[0-9]{8}T[0-9]{6}z-[0-9a-z]{8}\.buildinfo$"
local files=""
# First look for keys allowed to upload to any architecture
@@ -80,7 +82,7 @@ verify_gpg_signature ()
fi
# Then look for architecture specific key, which may only sign
- # .deb and .udeb files for its architecture.
+ # .deb and .udeb (and d-i) files for its architecture.
if test `echo $arch | wc -w` -ne 1 ; then
return 1
fi
@@ -91,7 +93,11 @@ verify_gpg_signature ()
return 1
fi
- files=`fetch_secure_files < $archive_file | egrep -v "_${arch}\.(buildinfo|deb|udeb)$|_[0-9]{8}T[0-9]{6}z-[0-9a-z]{8}\.buildinfo$"`
+ source_name=`fetch_source_name < $archive_file`
+ if [ "$source_name" == "debian-installer" ]; then
+ allowed_regex="$allowed_regex|^debian-installer-images_[0-9]{8}((\.|\+?[a-z]+|\+deb[0-9]+u|\+kbsd[0-9]+u)[0-9]+)?(\+b[0-9])?_${arch}\.tar\.gz"
+ fi
+ files=`fetch_secure_files < $archive_file | egrep -v "$allowed_regex"`
if [ -n "$files" ]; then
return 1
fi
@@ -102,7 +108,7 @@ verify_gpg_signature ()
verify_dpkg_signature ()
{
local archive_file=$1
- local files=`fetch_secure_files < $archive_file | grep '\.u\?deb$'`
+ local files=`fetch_secure_files < $archive_file | grep '\.u\?deb$|\.tar\.gz$'`
if [ "$use_dpkg_sig" != yes ]; then
return 0
@@ -202,6 +208,80 @@ verify_multiarch_changes ()
# file output
+queue_is_autobyhand_d_i_images ()
+{
+ local changes_file=$1
+ local archive_file=$2
+ local file=$3
+ local source_name=`fetch_source_name < $archive_file`
+ local section=`fetch_file_section "$file" < $archive_file`
+
+ if ! echo "$file" | egrep -q "debian-installer-images_[^_]+_[^_]+\.tar\.gz"; then
+ return 1
+ fi
+
+ if [ "$source_name" != "debian-installer" -o "$section" != "raw-installer" ]; then
+ return 1
+ fi
+
+ return 0
+}
+
+queue_is_autobyhand ()
+{
+ local changes_file=$1
+ local archive_file=$2
+ local file=$3
+
+ if ! queue_is_autobyhand_d_i_images $1 $2 $3; then
+ return 1
+ fi
+
+ return 0
+}
+
+queue_process_autobyhand_d_i_images ()
+{
+ local changes_file=$1
+ local archive_file=$2
+ # Get full file path, since archive-byhand-di runs with CWD as bin_dir
+ local file=$(readlink -f $3)
+ local version=`fetch_field "Version" < $archive_file`
+ local target_arches=`strip_gpg < $changes_file | fetch_field "Architecture"`
+ local arch=`filter_real_arches $target_arches`
+ local suite=`fetch_field "Distribution" < $archive_file`
+
+ if test `echo $arch | wc -w` -ne 1 ; then
+ log queue "queue_process_autobyhand_d_i_images ${archive_file##*/} $file changes has arches != 1: $arch"
+ return 1
+ fi
+
+ if ! (cd $bin_dir && ./archive-byhand-di $file $version $arch $changes_file $suite); then
+ return 1
+ fi
+
+ return 0
+}
+
+queue_process_autobyhand ()
+{
+ local changes_file=$1
+ local archive_file=$2
+ local file=$3
+
+ if queue_is_autobyhand_d_i_images $1 $2 $3; then
+ if ! queue_process_autobyhand_d_i_images $1 $2 $3; then
+ return 1
+ fi
+ else
+ script_error "queue_process_autobyhand" "$?"
+ log queue "queue_process_autobyhand unknown type (not d-i images) ${archive_file##*/} $file"
+ return 1
+ fi
+
+ return 0
+}
+
queue_accepted ()
{
local changes_file=$1
@@ -212,6 +292,18 @@ queue_accepted ()
local files_install="$files $changes_file $archive_file"
local suite=`fetch_field "Distribution" < $archive_file`
+ for file in $files; do
+ if queue_is_autobyhand $changes_file $archive_file $file; then
+ if queue_process_autobyhand $changes_file $archive_file $file; then
+ log queue "queue_process_autobyhand_success ${archive_file##*/} $file"
+ else
+ script_error "queue_accepted" "$?"
+ log queue "queue_process_autobyhand_failed ${archive_file##*/} $file"
+ return 1
+ fi
+ fi
+ done
+
files_owner_perms $files_install
if mv $files_install $accepted_dir; then
diff --git a/bin/archive-setup b/bin/archive-setup
index 768d0eb..32c71a5 100755
--- a/bin/archive-setup
+++ b/bin/archive-setup
@@ -149,6 +149,7 @@ create_repo ()
else
path_dir=$dists_dir/$suite/$section/binary-$arch
mkdir -p $dists_dir/$suite/$section/debian-installer/binary-$arch
+ mkdir -p $dists_dir/$suite/$section/installer-$arch
fi
mkdir -p $path_dir
cat > $path_dir/Release <<-HERE
--
2.11.0
More information about the Debian-ports-devel
mailing list