[Collab-qa-commits] r1196 - in debcluster: . scripts scripts/bin scripts/bin.local scripts/rebuild-archive
lucas at alioth.debian.org
lucas at alioth.debian.org
Tue Sep 2 09:04:24 UTC 2008
Author: lucas
Date: 2008-09-02 09:04:23 +0000 (Tue, 02 Sep 2008)
New Revision: 1196
Added:
debcluster/scripts/bin/dc-sn
Removed:
debcluster/Notes
debcluster/scripts/dc-startcluster
debcluster/scripts/dc-startnodes
debcluster/scripts/masternode4.rb
Modified:
debcluster/scripts/bin.local/rebuild-lenny-main-32.sh
debcluster/scripts/bin/dc-mountnfs
debcluster/scripts/bin/dc-startcluster
debcluster/scripts/bin/dc-startnodes
debcluster/scripts/rebuild-archive/gentasklist.rb
debcluster/scripts/rebuild-archive/pasfilter.rb
Log:
debcluster cleanup
Deleted: debcluster/Notes
===================================================================
--- debcluster/Notes 2008-09-02 08:01:58 UTC (rev 1195)
+++ debcluster/Notes 2008-09-02 09:04:23 UTC (rev 1196)
@@ -1,232 +0,0 @@
-### REBUILD
-
-Starting a batch rebuild:
-Preparation:
-on f38:
-debcluster/scripts/update-mirror-debian.bash
-
-To run in ~/collab-qa/debcluster/scripts
-
-ARCH=i386
-ARCH=amd64
-ID=amd64.20061123
-CHROOT=etch64
-ARCH=i386
-ID=i386.20070313
-CHROOT=etch32
-DIST=sid
-DIST=etch
-DIST=sarge
-# ubuntu
-# cat ~/debcluster-data/ubuntu-mirror/dists/$DIST/{main,restricted,universe}/source/Sources | rebuild-archive/grep-sources-arch.rb $ARCH | sort > /tmp/wb_$ID.1
-cat ~/debcluster-data/debian-mirror/dists/$DIST/main/source/Sources | rebuild-archive/grep-sources-arch.rb $ARCH | sort > /tmp/wb_$ID.1
-rebuild-archive/pasfilter.rb $ARCH < /tmp/wb_$ID.1 > /tmp/wb_$ID.2
-rebuild-archive/annotatewithpriority.rb rebuild-archive/buildtime.list < /tmp/wb_$ID.2 > /tmp/wb_$ID.3
-rebuild-archive/gentasklist.rb $CHROOT $ID < /tmp/wb_$ID.3 | bogosort -n > /tmp/tasklisttext_$ID
-echo "Tasklist (in text) is /tmp/tasklisttext_$ID"
-
-# vorlon build-arch
-sed -i "s/buildpackage.rb -p/buildpackage.rb --ba -p/" /tmp/tasklisttext_$ID
-sed -i "s/-c sid32 /-c sid32-ba /" /tmp/tasklisttext_$ID
-
-# BDFH
-sed -i "s/$/ --backup \/debcluster-data\/results\/bdfh.normal/" /tmp/tasklisttext_$ID
-sed -i "s/sid32.buildlog/sid32.normal.buildlog/" /tmp/tasklisttext_$ID
-
-sed -i "s/buildpackage.rb -p/buildpackage.rb --bdfh -p/" /tmp/tasklisttext_$ID
-sed -i "s/-c sid32 /-c sid32-bdfh /" /tmp/tasklisttext_$ID
-sed -i "s/$/ --backup \/debcluster-data\/results\/bdfh.bdfh/" /tmp/tasklisttext_$ID
-sed -i "s/sid32.buildlog/sid32.bdfh.buildlog/" /tmp/tasklisttext_$ID
-
-# buxy
-sed -i "s/$/ --backup \/debcluster-data\/results\/$ID/" /tmp/tasklisttext_$ID
-sed -i 's/-c sid32 /-c sid32-sym /' /tmp/tasklisttext_$ID
-
-# old, gzip vs bzip2
-cp /tmp/tasklisttext_$ID /tmp/tasklisttext_$ID.z
-sed -i 's/^b_/bz_/' /tmp/tasklisttext_$ID.z
-sed -i 's/-c feisty32 /-c feisty32-bzip2 /' /tmp/tasklisttext_$ID.z
-sed -i "s/logs\/rebuild.$ID\//logs\/rebuild.$ID.z\//" /tmp/tasklisttext_$ID.z
-sed -i "s/$/ --backup \/debcluster-data\/results\/$ID.z/" /tmp/tasklisttext_$ID.z
-sed -i "s/$/ --backup \/debcluster-data\/results\/$ID/" /tmp/tasklisttext_$ID
-
-./buildtasklist.rb </tmp/tasklisttext_$ID > /tmp/tasklist_$ID
-
-P=libtk-img
-V=1:1.3-15
-ID=2
-./submit.rb -h node-36.lyon.grid5000.fr -i b_${P}_${ID} -c "/debcluster/scripts/tasks/buildpackage.rb -p $P -v $V -c etch32_2 -l /debclusterlogs/tmplogs/${P}_${V}_${ID}.log"
-
-awk -F, '{print $1 " " $2}' buildres.csv > knownfails
-awk -F, '{print $1 " " $2 " " $3}' buildres.csv > knownfailsreason
-
-cat result.i386.20061028 |sort |grep -v " OK " > failslist
-awk '{print $1 " " $2}' failslist > allfails
-awk '{print $1 " " $2 " " $5}' failslist > allfailsreason
-
-##### Get packages in main/universe
-DIST=feisty
-for s in main restricted universe multiverse; do
- cat ~/debcluster-data/ubuntu-mirror/dists/$DIST/$s/source/Sources | rebuild-archive/grep-sources-arch.rb $ARCH | sort > /tmp/source.$s
-done
-
-for s in main restricted universe multiverse; do
- cat ~/debcluster-data/ubuntu-mirror/dists/$DIST/$s/binary-i386/Packages | grep "^Package:" |awk '{print $2}' | sort > /tmp/binary.$s
-done
-
-
-###### START ENVIRONMENT
-
-ENV=debian4all
-ENV=sid-amd64-base
-ENV=sid-x64-base-1.0
-NODES=50
-WALLTIME=8:30
-MINNODES=9
-
-TL=tl
-oarsub -t deploy -l nodes=${NODES},walltime=${WALLTIME} "/home/grenoble/lnussbaum/bin/katapult --deploy-env ${ENV} --copy-ssh-key /home/grenoble/lnussbaum/bin/dc-startcluster ${TL}"
-
-oarsub -t deploy -l nodes=1,walltime=${WALLTIME} "/home/grenoble/lnussbaum/bin/katapult --deploy-env ${ENV} --copy-ssh-key --sleep /home/grenoble/lnussbaum/bin/dc-startnodes"
-
-MASTER=node-11.lyon.grid5000.fr
-oarsub -t deploy -l nodes=${NODES},walltime=${WALLTIME} "/home/grenoble/lnussbaum/bin/katapult --deploy-env ${ENV} --copy-ssh-key --min-nodes 2 --sleep /home/grenoble/lnussbaum/bin/dc-addnodes ${MASTER}"
-
-ENV=sid-x64-base-1.0
-PART=sda3
-for p in 1 2 4 6 8; do
-p=6
-oarsub -p "cluster='grelon'" -q deploy -l nodes=1,walltime=10 "/home/grenoble/lnussbaum/bin/katapult --deploy-env ${ENV} --deploy-part ${PART} --copy-ssh-key --sleep /home/grenoble/lnussbaum/bin/dc-startnodes /debcluster-data/oo/buildoo.rb $p 1 o"
-
-cat ../res.sym.ok | while read n; do
-set $n
-if grep dpkg-shlib ${1}_${2}_sid32.buildlog | grep -q "warning"; then
-echo $n
-fi
-done
-##################### PIUPARTS
-piuparts -a -b /debcluster-data/chroots/etch32-piuparts.tgz -d etch -m 'http://idpot.grenoble.grid5000.fr/debian' feed2imap
-
-ARCH=i386
-ID=20070630.remove
-PIUSCRIPT=/debcluster-scripts/configs/piuparts-dump-nopanic
-# ubuntu
-# PIUARGS=" -e $PIUSCRIPT -c /debcluster-data/chroots/feisty32-piuparts.tgz -d feisty "
-PIUARGS=" -e $PIUSCRIPT -c /debcluster-data/chroots/sid32-piuparts.tgz -d sid "
-
-# ubuntu
-# cat ~/debcluster-data/ubuntu-mirror/dists/feisty/{main,restricted,universe}/binary-${ARCH}/Packages |grep "^Package:" | awk '{print $2}' | sort > /tmp/wp.${ID}.1
-cat ~/debcluster-data/debian-mirror/dists/sid/main/binary-${ARCH}/Packages |grep "^Package:" | awk '{print $2}' | sort > /tmp/wp.${ID}.1
-mdt list_sub /tmp/wp.${ID}.1 piuparts/piuparts.untestable > /tmp/wp.${ID}.2
-piuparts/gentasklist.rb "$PIUARGS" $ID < /tmp/wp.${ID}.2 > /tmp/tasklisttext.piu.${ID}.3
-echo "Tasklist (in text) is /tmp/tasklisttext.piu.${ID}.3"
-
-./buildtasklist.rb </tmp/tasklisttext.piu.$ID.3 > /tmp/tasklist_$ID
-
-scp /tmp/tasklist_$ID frontale.lyon:
-
-##################### VARIOUS NOTES
-------------------------------------------------------------------
-scripts/piuparts/parse-logs.rb logs/piuparts.i386.20061114/ | grep -v " BRKSYM$" |grep -v Unknown | grep -v " OK " | grep -v " LEFTFILES$" |grep -v " FILESMODIFIED/LEFTFILES$" | grep -v " BRKSYM/MYSQL$" |grep -v " FILESMODIFIED$" | grep -v " BRKSYM/LINUXIMAGE$" | grep -v " FILESMODIFIED/LEFTFILES/MYSQL$" | grep -v " PROCESSESRUNNING$"
-
-scripts/piuparts/parse-logs.rb logs/piuparts.i386.20061114/ |grep -v Unknown | grep -v " OK " | grep "CMDFAILED"
-
-rsync -avzP $(awk '{print $1}' ../result.piuparts.20061114 | sed 's/$/.log/') lucas at ox.blop.info:public_html/bazaar/buildlogs/20061114/
-
-ARCH=i386
-ID=${ARCH}.20061123
-PIUARGS=" -c /debcluster/chroots/etch32-http.tgz -d etch "
-
-cat debcluster/debian-mirror/dists/etch/main/binary-${ARCH}/Packages |grep "^Package:" | awk '{print $2}' | sort > /tmp/wp.${ID}.1
-mdt list_sub /tmp/wp.${ID}.1 debcluster/scripts/piuparts/piuparts.untestable > /tmp/wp.${ID}.2
-debcluster/scripts/piuparts/gentasklist.rb "$PIUARGS" $ID < /tmp/wp.${ID}.2 > /tmp/wp.${ID}.3
-
-debcluster/scripts/buildtasklist.rb </tmp/wp.$ID.3 > /tmp/tasklist_$ID
-
-scp /tmp/tasklist_$ID frontale.lyon:
-
-P=seq24
-V=0.8.6-1
-N=5
-C=etch32
-/debcluster/scripts/tasks/buildpackage.rb -p ${P} -v ${V} -c ${C} -t 14 -l /debcluster/logs/tmp/${P}.${N}.log
-
-
-------------------------------------------------------------------
-scripts/piuparts/parse-logs.rb logs/piuparts.i386.20061114/ | grep -v " BRKSYM$" |grep -v Unknown | grep -v " OK " | grep -v " LEFTFILES$" |grep -v " FILESMODIFIED/LEFTFILES$" | grep -v " BRKSYM/MYSQL$" |grep -v " FILESMODIFIED$" | grep -v " BRKSYM/LINUXIMAGE$" | grep -v " FILESMODIFIED/LEFTFILES/MYSQL$" | grep -v " PROCESSESRUNNING$"
-
-scripts/piuparts/parse-logs.rb logs/piuparts.i386.20061114/ |grep -v Unknown | grep -v " OK " | grep "CMDFAILED"
-
-/debcluster/scripts/tasks/piuparts-debconfucf.rb -c /debcluster/chroots/etch32-http.tgz -d etch -p zoph
-
-
-rsync -avzP $(awk '{print $1}' ../result.piuparts.20061114 | sed 's/$/.log/') lucas at ox.blop.info:public_html/bazaar/buildlogs/20061114/
-
-rm tty
-mknod tty c 1 3
-
-#####
-# Creating a build chroot
-adduser --home /home/user --uid 1001 --ingroup adm --disabled-password --gecos User user
-cp /debcluster-scripts/configs/dotsbuildrc /home/user/.sbuildrc
-adduser user sbuild
-
-# Inside the chroot:
-# edit /etc/sudoers
-user ALL=NOPASSWD: ALL
-Defaults:user env_keep+="APT_CONFIG DEBIAN_FRONTEND SHELL"
-
-mkdir -p /build/user /var/lib/sbuild/srcdep-lock
-chown -R root:sbuild /var/lib/sbuild /build
-chown user:sbuild /build/user
-
-
-### Split a list by section
-for i in main contrib non-free ; do cat ~/debcluster-data/debian-mirror/dists/$DIST/$i/source/Sources | rebuild-archive/grep-sources-arch.rb i386 | rebuild-archive/pasfilter.rb i386 > /tmp/pkg.$i ; done
-
-for i in main contrib non-free ; do
- for p in extra optional standard important required; do
- grep-dctrl -F priority -sPackage -n $p ~/debcluster-data/debian-mirror/dists/$DIST/$i/source/Sources > p.$i.$p
- done
-done
-
-LIST=failed.sid.i386.20070416
-cat $LIST | mdt filter <(cat p.contrib.*) > $LIST.contrib
-cat $LIST | mdt filter <(cat p.non-free.*) > $LIST.non-free
-cat $LIST | mdt filter <(cat p.main.optional p.main.extra) > $LIST.main.optional
-cat $LIST | mdt filter <(cat p.main.standard p.main.important p.main.required) > $LIST.main.base
-
-######
-# 8h / 45 noeuds
-export TARGET=oar.orsay
-export TARGET=sagittaire.lyon
-export PART=sda3
-export WALLTIME=8
-export NODES=45
-~/rebuild-sid-main-sid32.sh
-
-# Analyzing results
-
-ssh -A f38
-
-ID=sid-i386.2007-08-06
-DATE=2007/08/06
-
-ssh lucas at gluck.debian.org mkdir public_html/logs/$DATE
-
-cd debcluster-data/logs
-~/collab-qa/log-analysis/parse-logs-rebuild.rb rebuild.$ID | sort > res.$ID
-cd rebuild.$ID
-cat ../res.$ID | grep -v " OK " | awk '{print $1 "_" $2 "_sid32.buildlog"}' > /tmp/loglist
-
-rsync -avzP $(</tmp/loglist) lucas at gluck.debian.org:public_html/logs/$DATE
-
-ssh gluck chmod -R a+rX public_html/
-
-# sync bdfh
-rsync -avzP --delete . root at idpot.grenoble.grid5000.fr:/nfs-scratch/grenoble/lnussbaum/debcluster-data/root-bdfh/
-
-### BDFH
-TLT=/tmp/tasklisttext_sid32-i386.2008-01-08
-cat $TLT | sed 's/^b_sid32-i386/b_sid32-normal-i386/' | sed 's/-l \/debcluster-data\/logs\/rebuild.sid32-i386/-l \/debcluster-data\/logs\/rebuild.sid32-normal-i386/' | sed 's/$/ -b \/debcluster-data\/results\/bdfh\/normal\//' > ${TLT}.normal
-cat $TLT | sed 's/^b_sid32-i386/b_sid32-bdfh-i386/' | sed 's/-l \/debcluster-data\/logs\/rebuild.sid32-i386/-l \/debcluster-data\/logs\/rebuild.sid32-bdfh-i386/' | sed 's/$/ --bdfh -b \/debcluster-data\/results\/bdfh\/bdfh\//' | sed 's/ -c sid32 / -c sid32-bdfh /' > ${TLT}.bdfh
Modified: debcluster/scripts/bin/dc-mountnfs
===================================================================
--- debcluster/scripts/bin/dc-mountnfs 2008-09-02 08:01:58 UTC (rev 1195)
+++ debcluster/scripts/bin/dc-mountnfs 2008-09-02 09:04:23 UTC (rev 1196)
@@ -9,9 +9,8 @@
apt-get -y install nfs-common initscripts sysvinit-utils
# cleanup
-umount -l /idpotscratch &>/dev/null || true
-mkdir /idpotscratch &>/dev/null || true
-rm -f /debcluster-data /debcluster-scripts &>/dev/null || true
+umount -l /dc &>/dev/null || true
+mkdir /dc &>/dev/null || true
# setup bridge if neeeded
if [ "$1" = "bridge" ];then
@@ -55,8 +54,6 @@
fi
# do mount
-mount -t nfs -o rw,hard,intr,async,nfsvers=3,rsize=32768,wsize=32768,sec=sys idpot.grenoble.grid5000.fr:/nfs-scratch /idpotscratch
-ln -sf /idpotscratch/grenoble/lnussbaum/debcluster-data /debcluster-data
-ln -sf /idpotscratch/grenoble/lnussbaum/collab-qa/debcluster /debcluster-scripts
+mount -t nfs -o rw,hard,intr,async,nfsvers=3,rsize=32768,wsize=32768,sec=sys nfs.grenoble.grid5000.fr:/home/grenoble/lnussbaum/pub /dc
echo "Finishing dc-mountnfs -- $(date)"
Added: debcluster/scripts/bin/dc-sn
===================================================================
--- debcluster/scripts/bin/dc-sn (rev 0)
+++ debcluster/scripts/bin/dc-sn 2008-09-02 09:04:23 UTC (rev 1196)
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# time we wait for a prediction before giving up
+MAXWAIT=100
+# max time we wait for job to start
+MAXWAIT_START=600
+
+WALLTIME=4
+ENV=sid-x64-base-1.1
+f=`mktemp /tmp/oarsub.XXXXXX`
+oarsub -t deploy -l nodes=1,walltime=${WALLTIME} "/home/grenoble/lnussbaum/bin/katapult --deploy-env ${ENV} --copy-ssh-key --sleep /home/grenoble/lnussbaum/bin/dc-startnodes" &> $f
+cat $f
+eval $(grep "^OAR_JOB_ID=" $f)
+echo "Job ID: $OAR_JOB_ID"
+if [ $OAR_JOB_ID -le 0 ]; then
+ echo "negative Job ID, something wrong happened. exiting."
+fi
+
+start=$(date +%s)
+while true; do
+ oarstat -fj $OAR_JOB_ID > $f
+ state=$(grep "state = " $f | cut -d '=' -f 2 | cut -c 2-)
+ schedstart=$(grep "scheduledStart = " $f | cut -d '=' -f 2 | cut -c 2-)
+ if [ "$state" == 'Running' ]; then
+ echo "Job $OAR_JOB_ID is now running."
+ exit 0
+ fi
+ if [ "$schedstart" == 'no prediction' ]; then
+ now=$(date +%s)
+ if [ $(($now - $start)) -gt $MAXWAIT ];then
+ echo "OAR failed to schedule the job after $MAXWAIT sec. exiting."
+ oardel $OAR_JOB_ID
+ exit 1
+ fi
+ echo "No prediction... Waiting for OAR to schedule the job."
+ else
+ sched=$(date -d "$schedstart" +%s)
+ delay=$(($sched - $start))
+ if [ $delay -gt $MAXWAIT_START ]; then
+ echo "Job scheduled to start in $delay sec. We don't want to wait that long. Exiting."
+ oardel $OAR_JOB_ID
+ exit 1
+ fi
+ echo "Job scheduled to start in $delay sec. Waiting."
+ sleep 1
+ fi
+done
+
Property changes on: debcluster/scripts/bin/dc-sn
___________________________________________________________________
Name: svn:executable
+ *
Modified: debcluster/scripts/bin/dc-startcluster
===================================================================
--- debcluster/scripts/bin/dc-startcluster 2008-09-02 08:01:58 UTC (rev 1195)
+++ debcluster/scripts/bin/dc-startcluster 2008-09-02 09:04:23 UTC (rev 1196)
@@ -36,4 +36,4 @@
scp $SSHOPTS ~/bin/dc-mountnfs $LOGIN@$HNODE:/root/dc-mountnfs
ssh $SSHOPTS $LOGIN@$HNODE "/root/dc-mountnfs"
-exec ssh $LOGIN@$HNODE "cd /debcluster-scripts/scripts/ && ./masternode.rb -n $F1 -t $F2 $*"
+exec ssh $LOGIN@$HNODE "cd /dc/debcluster/scripts/ && ./masternode.rb -n $F1 -t $F2 $*"
Modified: debcluster/scripts/bin/dc-startnodes
===================================================================
--- debcluster/scripts/bin/dc-startnodes 2008-09-02 08:01:58 UTC (rev 1195)
+++ debcluster/scripts/bin/dc-startnodes 2008-09-02 09:04:23 UTC (rev 1196)
@@ -20,7 +20,7 @@
ths << Thread::new(n) do |node|
system("scp #{SSHOPTS.join(' ')} ~/bin/dc-mountnfs #{LOGIN}@#{node}:/root/dc-mountnfs")
system("ssh #{SSHOPTS.join(' ')} #{LOGIN}@#{node} /root/dc-mountnfs")
- system("ssh #{SSHOPTS.join(' ')} #{LOGIN}@#{node} \"/debcluster-scripts/scripts/tasks/preparenode.rb --log /debcluster-data/logs/preparenode/#{node}.$(date --iso)\"")
+ system("ssh #{SSHOPTS.join(' ')} #{LOGIN}@#{node} \"/dc/debcluster/scripts/tasks/preparenode.rb --log /dc/data/logs/preparenode/#{node}.$(date --iso)\"")
end
end
ths.each do |t|
Modified: debcluster/scripts/bin.local/rebuild-lenny-main-32.sh
===================================================================
--- debcluster/scripts/bin.local/rebuild-lenny-main-32.sh 2008-09-02 08:01:58 UTC (rev 1195)
+++ debcluster/scripts/bin.local/rebuild-lenny-main-32.sh 2008-09-02 09:04:23 UTC (rev 1196)
@@ -15,7 +15,7 @@
# NNN: docbook-xsl-saxon: FTBFS: build blocks
# #479923: libmodule-install-perl: FTBFS: test blocked
# #479935: autopsy: FTBFS: prompts for location of perl executable
-cat ~/debcluster-data/mirrors/debian/dists/lenny/main/source/Sources | rebuild-archive/grep-sources-arch.rb $ARCH $WALL |\
+cat ~/mirrors/debian/dists/lenny/main/source/Sources | rebuild-archive/grep-sources-arch.rb $ARCH $WALL |\
grep -v -w ardour |\
grep -v -w docbook-xsl-saxon |\
grep -v -w libmodule-install-perl |\
@@ -24,10 +24,10 @@
grep -v -w linux-modules-di-i386-2.6 |\
grep -v -w linux-kernel-di-i386-2.6\
> /tmp/wb_$ID.1
-#rebuild-archive/pasfilter.rb $ARCH < /tmp/wb_$ID.1 > /tmp/wb_$ID.2
+rebuild-archive/pasfilter.rb $ARCH < /tmp/wb_$ID.1 > /tmp/wb_$ID.2
cat < /tmp/wb_$ID.1 > /tmp/wb_$ID.2
if [ "$LIST" != "" ]; then
- mdt filter $LIST < /tmp/wb_$ID.2 > /tmp/wb_$ID.2.1
+ fcb -l /tmp/wb_$ID.2 and $LIST > /tmp/wb_$ID.2.1
mv /tmp/wb_$ID.2.1 /tmp/wb_$ID.2
fi
rebuild-archive/annotatewithpriority.rb rebuild-archive/buildtime.list < /tmp/wb_$ID.2 > /tmp/wb_$ID.3
Deleted: debcluster/scripts/dc-startcluster
===================================================================
--- debcluster/scripts/dc-startcluster 2008-09-02 08:01:58 UTC (rev 1195)
+++ debcluster/scripts/dc-startcluster 2008-09-02 09:04:23 UTC (rev 1196)
@@ -1,37 +0,0 @@
-#!/bin/bash
-
-# Script called by katapult to prepare the head node and start masternode.rb
-
-LOGIN=root
-
-if [ $# -lt 1 ]; then
- echo "Usage: startcluster.bash jobsfile"
- echo "GOOD_NODES or OAR_NODEFILE must point to the nodes list"
- echo "First node is used as head node."
- exit 1
-fi
-
-TASKLIST=$1
-LNFILE=
-if [ "$GOOD_NODES" ]; then
- LNFILE=$GOOD_NODES
-elif [ "$OAR_NODEFILE" ]; then
- LNFILE=$OAR_NODEFILE
-fi
-HNODE=$(head -1 $LNFILE)
-
-echo "Head node: $HNODE"
-
-F1=$(ssh root@$HNODE mktemp /tmp/nodeslist.XXXXXX)
-scp $LNFILE $LOGIN@$HNODE:$F1
-
-F2=$(ssh root@$HNODE mktemp /tmp/tasklist.XXXXXX)
-if [ -f $TASKLIST ]; then
- scp $TASKLIST $LOGIN@$HNODE:$F2
-fi
-
-SSHOPTS="-o BatchMode=yes -o StrictHostKeyChecking=no"
-ssh $SSHOPTS $LOGIN@$HNODE "umount -l /idpotscratch &>/dev/null; apt-get update ; apt-get -y install nfs-common; mkdir /idpotscratch &>/dev/null; rm -f /debcluster-data &>/dev/null; rm -f /debcluster-scripts &>/dev/null ; mount -t nfs -o rw,hard,intr,async,nfsvers=3,rsize=32768,wsize=32768 idpot.grenoble.grid5000.fr:/nfs-scratch /idpotscratch ; ln -sf /idpotscratch/grenoble/lnussbaum/debcluster-data /debcluster-data ; ln -sf /idpotscratch/grenoble/lnussbaum/collab-qa/debcluster /debcluster-scripts ; mkdir ~/.ssh 2>/dev/null ; sed -i 's/.*http_proxy.*//' /etc/environment"
-scp $SSHOPTS -r .ssh/* $LOGIN@$HNODE:.ssh/
-
-exec ssh $LOGIN@$HNODE "cd /debcluster-scripts/scripts/ && ./masternode.rb $F1 $F2"
Deleted: debcluster/scripts/dc-startnodes
===================================================================
--- debcluster/scripts/dc-startnodes 2008-09-02 08:01:58 UTC (rev 1195)
+++ debcluster/scripts/dc-startnodes 2008-09-02 09:04:23 UTC (rev 1196)
@@ -1,32 +0,0 @@
-#!/usr/bin/ruby -w
-
-require 'thread'
-
-LOGIN='root'
-SSHOPTS=['-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no']
-
-if ENV['GOOD_NODES']
- LNFILE = ENV['GOOD_NODES']
-elsif ENV['OAR_NODEFILE']
- LNFILE = ENV['GOOD_NODES']
-else
- puts "Could not find node list"
- exit(1)
-end
-
-ths = []
-IO::read(LNFILE).split.sort.uniq.each do |n|
- ths << Thread::new(n) do |node|
- system("ssh #{SSHOPTS.join(' ')} #{LOGIN}@#{node} \"umount -l /idpotscratch &>/dev/null; apt-get update ; export DEBIAN_FRONTEND=noninteractive ; apt-get -y install nfs-common ; mkdir /idpotscratch &>/dev/null; rm -f /debcluster-data &>/dev/null; rm -f /debcluster-scripts &>/dev/null ; mount -t nfs -o rw,hard,intr,async,nfsvers=3,rsize=32768,wsize=32768 idpot.grenoble.grid5000.fr:/nfs-scratch /idpotscratch ; ln -sf /idpotscratch/grenoble/lnussbaum/debcluster-data /debcluster-data ; ln -sf /idpotscratch/grenoble/lnussbaum/collab-qa/debcluster /debcluster-scripts ; mkdir ~/.ssh 2>/dev/null ; sed -i 's/.*http_proxy.*//' /etc/environment\"")
- system("scp #{SSHOPTS.join(' ')} -qr .ssh/* #{LOGIN}@#{node}:.ssh/")
- system("ssh #{SSHOPTS.join(' ')} #{LOGIN}@#{node} \"/debcluster-scripts/scripts/tasks/preparenode.rb --log /debcluster-data/logs/preparenode/#{node}.$(date --iso)\"")
- if ARGV.length > 0
- c = ['/usr/bin/ssh'] + SSHOPTS + [ "#{LOGIN}@#{node}" ] + ARGV
- fork { exec(*c) }
- Process::wait
- end
- end
-end
-ths.each do |t|
- t.join
-end
Deleted: debcluster/scripts/masternode4.rb
===================================================================
--- debcluster/scripts/masternode4.rb 2008-09-02 08:01:58 UTC (rev 1195)
+++ debcluster/scripts/masternode4.rb 2008-09-02 09:04:23 UTC (rev 1196)
@@ -1,335 +0,0 @@
-#!/usr/bin/ruby1.8 -w
-
-require 'thread'
-require 'thwait'
-require 'debtask'
-require 'socket'
-require 'time'
-
-Thread::abort_on_exception = true
-
-DEFAULT_TASKS_PER_NODE = 4
-
-SSHOPTS="-o BatchMode=yes -o StrictHostKeyChecking=no"
-DEBUG_PREP = true
-SLEEPTIME = 10
-TCPPORT=36363
-
-# Convenient for testing: set false
-SHORTERLAST=true
-
-$nodes = {}
-$wtasks = []
-$nbfinished = 0
-$tottime = 0.0
-$mutex = Mutex::new
-$ths = []
-$masternode = "Unknown"
-$started = false
-$ready = false
-$interactive = false
-
-def preparenode(n)
- puts "Preparing #{n}..." if DEBUG_PREP
- system("ssh #{SSHOPTS} #{n} exit 42")
- if $?.exitstatus != 42
- puts "E:Public key auth on #{n} failed."
- return false
- end
- # remove proxy
- system("ssh #{SSHOPTS} #{n} \"sed -i 's/.*http_proxy.*//' /etc/environment\"")
- # mount /debcluster
- system("ssh #{SSHOPTS} #{n} \"apt-get update && apt-get install -y nfs-common initscripts sysvinit-utils\" &>/dev/null")
- if `ssh #{SSHOPTS} #{n} \"mount | grep /idpotscratch | wc -l\"`.to_i == 0
- system("ssh #{SSHOPTS} #{n} \"mkdir /idpotscratch &>/dev/null; rm -f /debcluster-data /debcluster-scripts &>/dev/null\"")
- ret = system("ssh #{SSHOPTS} #{n} mount -t nfs -o rw,hard,intr,async,nfsvers=3,rsize=32768,wsize=32768 idpot.grenoble.grid5000.fr:/nfs-scratch /idpotscratch")
- if not ret
- puts "E:Failed to mount /idpotscratch on #{n} failed."
- return false
- end
- end
- system("ssh #{SSHOPTS} #{n} \"rm -f /debcluster-data /debcluster-scripts ; ln -sf /idpotscratch/grenoble/lnussbaum/debcluster-data /debcluster-data ; ln -sf /idpotscratch/grenoble/lnussbaum/collab-qa/debcluster /debcluster-scripts\"")
- ret = system("ssh #{SSHOPTS} #{n} \"/debcluster-scripts/scripts/tasks/preparenode.rb --log /debcluster-data/logs/preparenode/#{n}.$(date --iso)\"")
- if not ret
- puts "E:preparenode.rb failed on #{n}."
- return false
- else
- return true
- end
-end
-
-def tcpserver
- s = TCPServer::new(TCPPORT)
- while true do
- sc = s.accept
- m = Marshal.load(sc)
- case m.type
- when :add_task
- $mutex.synchronize do
- if m.param.class == Array
- m.param.each do |t|
- $wtasks << t
- end
- else
- $wtasks << m.param
- end
- end
- when :sort_tasks
- $mutex.synchronize do
- $wtasks.sort!
- $wtasks.reverse! if not SHORTERLAST
- end
- when :list_tasks
- $mutex.synchronize do
- Marshal::dump($wtasks, sc)
- end
- when :del_task
- $mutex.synchronize do
- $wtasks.delete_if { |t| t.id =~ m.param }
- end
- when :add_node
- $mutex.synchronize do
- $ths << Thread::new(m.param) { |n| managenode(n) }
- end
- end
- sc.close
- end
-end
-
-def showtop
- while true do
- sleep SLEEPTIME
- $mutex.synchronize do
- if $nbfinished > 0
- avg = $tottime / $nbfinished
- else
- avg = 0
- end
- puts "*" * 75
- puts " Time: #{Time::now.iso8601.gsub(/T/,' ').gsub(/\+.*/,'')}"
- puts " Masternode: #{$masternode}"
- puts " Tasks finished: #{$nbfinished} (tot: #{$tottime}s ; avg: #{avg}s)"
- puts " Nodes: #{$nodes.length}"
- puts " Tasks waiting: #{$wtasks.length} (total ETA: #{$wtasks.length * avg}s, realtime ETA: #{$wtasks.length * avg.to_f / $nodes.length.to_f})"
- n = $nodes.to_a.sort do |a,b|
- ret = nil
- if a[1] == b[1] and a[1] == :free
- ret = 0
- elsif a[1] == :free
- ret = 1
- elsif b[1] == :free
- ret = -1
- else
- ret = a[1].starttime <=> b[1].starttime
- end
- ret
- end
- n.each do |i|
- if i[1] == :free
- puts " #{i[0]}: Free"
- else
- puts " #{i[0]}: #{i[1]} since #{Time::now - i[1].starttime}"
- end
- end
- STDOUT.flush
- end
- end
-end
-
-def runtask(node, command)
- ret = nil
- thtask = Thread::new(node, command) do |node, command|
- cmd = "ssh #{SSHOPTS} #{node} -- #{command} &> /tmp/log.ssh.#{node}"
- ret = system(cmd)
- if ret != true
- puts "#{node}: system() returned #{ret} / $? == #{$?}"
- end
- end
- thmonitor = Thread::new(node) do |node|
- ok = true
- while ok
- s = `ssh #{SSHOPTS} #{node} "(cd /debcluster-scripts/scripts/ && hostname) 2>&1"`
- if s.chomp != node
- p s.chomp
- p node
- ok = false
- else
- sleep SLEEPTIME
- end
- end
- puts "Monitoring thread for #{node} exiting."
- end
- thw = ThreadsWait::new(thtask, thmonitor)
- fin = thw.next_wait
- if fin == thtask and ret == true
- thmonitor.kill
- # dernieres verifications...
- s = `ssh #{SSHOPTS} #{node} "(cd /debcluster-scripts/scripts/ && hostname) 2>&1"`
- if s.chomp == node
- s = `ssh #{SSHOPTS} #{node} "runlevel | cut -d ' ' -f 2 2>&1"`.chomp
- if s == "2"
- return true
- else
- puts "Wrong runlevel"
- p s
- return false
- end
- else
- puts "Wrong hostname"
- p s
- return false
- end
- elsif fin == thtask
- thmonitor.kill
- return false
- else
- thtask.kill
- return false
- end
-end
-
-class Node
- attr_accessor :hostname, :ident
-
- def initialize(hostname, ident)
- @hostname = hostname
- @ident = ident
- end
-
- def to_s
- "#{@hostname}-#{@ident}"
- end
-end
-
-NBNODES = [
- [ /^helios.*/, 4 ]
-]
-def managenode(nodehost)
- # nodehost is just the node name in text.
- return if not preparenode(nodehost)
- nbsubnodes = DEFAULT_TASKS_PER_NODE # default value
- NBNODES.each do |t|
- if nodehost =~ t[0]
- nbsubnodes = t[1]
- end
- end
- (1..nbsubnodes).each do |i|
- node = Node::new(nodehost, i)
- Thread::new(node) { |node| manageworker(node) }
- end
-end
-
-def manageworker(node)
- # node is a Node object
- $mutex.synchronize do
- $nodes[node] = :free
- end
- while not $ready
- puts "#{node}: We are not ready yet."
- sleep SLEEPTIME
- end
- asleep = false
- running = true
- while running do
- t = nil
- $mutex.synchronize do
- # Take the last element, that is, the bigger.
- t = $wtasks.pop
- end
- if t.nil?
- puts "#{node}: no task, waiting for #{SLEEPTIME} seconds" if not asleep
- asleep = true
- sleep SLEEPTIME
- else
- $mutex.synchronize do
- $nodes[node] = t
- t.starttime = Time::now
- puts "#{node}: running #{t}"
- end
- ret = runtask(node.hostname, t.command)
- $mutex.synchronize do
- if ret
- puts "#{node}: finished #{t}"
- $nodes[node] = :free
- $nbfinished += 1
- $tottime += Time::now - t.starttime
- else
- puts "Something really wrong happend with #{node}. Removing it."
- $nodes.delete(node)
- running = false
- $wtasks.unshift(t)
- end
- end
- end
- end
-end
-
-Thread::new { tcpserver }
-Thread::new { showtop }
-
-# read and start nodes
-nodes = IO::read(ARGV[0]).split(/\n/).map { |l| l.chomp }
-$masternode = nodes[0]
-nodes = (nodes.sort.uniq - [nodes[0]]).sort.uniq # removing head node.
-nodes.each do |n|
- $ths << Thread::new(n) { |n| managenode(n) }
-end
-
-# Tasks
-if File::exists?(ARGV[1])
- begin
- tasks = Marshal::load(IO::read(ARGV[1]))
- rescue
- puts "Something really wrong happend while reading tasks list, setting interactive mode."
- $interactive = true
- tasks = []
- end
- $mutex.synchronize do
- tasks.each do |t|
- $wtasks << t
- end
- $wtasks.sort!
- $wtasks.reverse! if not SHORTERLAST
- end
-else
- puts "Task list not found, ignoring."
- $interactive = true
-end
-
-# Preparation of master node + chroots
-ret = system("/debcluster-scripts/scripts/tasks/preparenode.rb --log /debcluster-data/logs/preparenode/MASTER-#{$masternode}.$(date --iso)")
-if not ret
- puts "preparenode.rb FAILED on masternode. exiting."
- exit(1)
-end
-# run apt-get update inside chroots
-# [ 'etch32-source', 'etch64-source', 'etch32-piuparts-source', 'sid32-source', 'sid64-source' ].each do |c|
-[ ].each do |c|
- puts "Updating #{c}:"
- ret = system("cd / && schroot -c #{c} -- sh -c \"apt-get update && apt-get -y dist-upgrade && apt-get clean\"")
- if not ret
- puts "Update of #{c} FAILED. exiting."
- exit(1)
- end
-end
-$ready = true
-
-continue = true
-while continue
- $mutex.synchronize do
- if $wtasks.length > 0 and not $started
- $started = true
- end
- if $started and $wtasks.length == 0 and (not $interactive)
- # no tasks remaining.
- busy = $nodes.keys.inject(0) { |a, b| a + ($nodes[b] == :free ? 0 : 1) }
- if busy == 0
- puts "### No tasks, no nodes busy. Exiting."
- exit 0
- else
- puts "### No tasks, but #{busy} nodes busy."
- end
- end
- end
- sleep SLEEPTIME
-end
Modified: debcluster/scripts/rebuild-archive/gentasklist.rb
===================================================================
--- debcluster/scripts/rebuild-archive/gentasklist.rb 2008-09-02 08:01:58 UTC (rev 1195)
+++ debcluster/scripts/rebuild-archive/gentasklist.rb 2008-09-02 09:04:23 UTC (rev 1196)
@@ -23,5 +23,5 @@
prio = 0
nice = 19
end
- puts "b_#{ID}_#{p}_#{v} [#{prio},#{t}] /debcluster-scripts/scripts/tasks/buildpackage.rb -n #{nice} -p #{p} -v #{v} -c #{CHROOT} -t #{t} -l /debcluster-data/logs/rebuild.#{ID}/#{p}_#{v}_#{CHROOT}.buildlog"
+ puts "b_#{ID}_#{p}_#{v} [#{prio},#{t}] /dc/debcluster/scripts/tasks/buildpackage.rb -n #{nice} -p #{p} -v #{v} -c #{CHROOT} -t #{t} -l /dc/data/logs/rebuild.#{ID}/#{p}_#{v}_#{CHROOT}.buildlog"
end
Modified: debcluster/scripts/rebuild-archive/pasfilter.rb
===================================================================
--- debcluster/scripts/rebuild-archive/pasfilter.rb 2008-09-02 08:01:58 UTC (rev 1195)
+++ debcluster/scripts/rebuild-archive/pasfilter.rb 2008-09-02 09:04:23 UTC (rev 1196)
@@ -8,7 +8,7 @@
# calculate $archs
$archs = {}
-pas = `dog "http://cvs.debian.org/srcdep/Packages-arch-specific?cvsroot=dak&rev=HEAD"`
+pas = `dog "http://localhost:12003/srcdep/Packages-arch-specific?cvsroot=dak&rev=HEAD"`
arr = pas.split(/\n/).grep(/^%?[a-z0-9]/).map { |l| l.gsub(/\s*#.*$/,'') }
allarchs=['alpha','amd64','arm','armeb','freebsd-i386','hppa','hurd-i386','i386','ia64','kfreebsd-i386','m32r','m68k','mips','mipsel','netbsd-i386','powerpc','s390','s390x','sh','sparc']
arr.each do |l|
More information about the Collab-qa-commits
mailing list