[Collab-qa-commits] r406 - in debcluster/scripts: . cronjobs piuparts tasks
lucas at alioth.debian.org
lucas at alioth.debian.org
Wed Sep 5 09:00:13 UTC 2007
Author: lucas
Date: 2007-09-05 09:00:13 +0000 (Wed, 05 Sep 2007)
New Revision: 406
Added:
debcluster/scripts/dc-addnodes
debcluster/scripts/dc-startcluster
debcluster/scripts/dc-startnodes
Removed:
debcluster/scripts/addnodes.rb
debcluster/scripts/startcluster.bash
debcluster/scripts/startnodes.rb
Modified:
debcluster/scripts/cronjobs/update-env.bash
debcluster/scripts/piuparts/piuparts.untestable
debcluster/scripts/tasks/piuparts.rb
debcluster/scripts/tasks/preparenode.rb
debcluster/scripts/update-mirror-ubuntu.bash
Log:
batch commit of debcluster stuff
Deleted: debcluster/scripts/addnodes.rb
===================================================================
--- debcluster/scripts/addnodes.rb 2007-09-05 08:58:03 UTC (rev 405)
+++ debcluster/scripts/addnodes.rb 2007-09-05 09:00:13 UTC (rev 406)
@@ -1,30 +0,0 @@
-#!/usr/bin/ruby1.8 -w
-
-require 'socket'
-
-TCPPORT=36363
-
-class DebMessage
- attr_accessor :type, :param
-end
-
-if ENV['GOOD_NODES'] != nil
- f = ENV['GOOD_NODES']
-elsif ENV['OAR_NODEFILE']
- f = ENV['OAR_NODEFILE']
-else
- puts "No nodes file."
- exit(1)
-end
-
-hostname = ARGV[0]
-nodes = IO::read(f).split(/\n/).sort.uniq
-
-nodes.each do |node|
- s = TCPSocket::new(hostname, TCPPORT)
- m = DebMessage::new
- m.type = :add_node
- m.param = node
- Marshal::dump(m, s)
- s.close
-end
Modified: debcluster/scripts/cronjobs/update-env.bash
===================================================================
--- debcluster/scripts/cronjobs/update-env.bash 2007-09-05 08:58:03 UTC (rev 405)
+++ debcluster/scripts/cronjobs/update-env.bash 2007-09-05 09:00:13 UTC (rev 406)
@@ -3,4 +3,4 @@
ENV=sid-x64-base-1.0
~/collab-qa/debcluster/scripts/update-mirror-debian.bash
~/collab-qa/debcluster/scripts/update-mirror-ubuntu.bash
-ssh gdx "oarsub -q deploy -l nodes=1,walltime=0:20 \"/home/grenoble/lnussbaum/katapult --deploy-env ${ENV} --copy-ssh-key /home/grenoble/lnussbaum/startnodes.rb /debcluster-scripts/scripts/tasks/updateschroots.rb sid32-source sid32-piuparts-source\""
+ssh oar.orsay "oarsub -q deploy -l nodes=1,walltime=0:20 \"/home/grenoble/lnussbaum/bin/katapult --deploy-env ${ENV} --copy-ssh-key /home/grenoble/lnussbaum/bin/dc-startnodes /debcluster-scripts/scripts/tasks/updateschroots.rb sid32-source sid32-piuparts-source\""
Copied: debcluster/scripts/dc-addnodes (from rev 403, debcluster/scripts/addnodes.rb)
===================================================================
--- debcluster/scripts/dc-addnodes (rev 0)
+++ debcluster/scripts/dc-addnodes 2007-09-05 09:00:13 UTC (rev 406)
@@ -0,0 +1,30 @@
+#!/usr/bin/ruby1.8 -w
+
+require 'socket'
+
+TCPPORT=36363
+
+class DebMessage
+ attr_accessor :type, :param
+end
+
+if ENV['GOOD_NODES'] != nil
+ f = ENV['GOOD_NODES']
+elsif ENV['OAR_NODEFILE']
+ f = ENV['OAR_NODEFILE']
+else
+ puts "No nodes file."
+ exit(1)
+end
+
+hostname = ARGV[0]
+nodes = IO::read(f).split(/\n/).sort.uniq
+
+nodes.each do |node|
+ s = TCPSocket::new(hostname, TCPPORT)
+ m = DebMessage::new
+ m.type = :add_node
+ m.param = node
+ Marshal::dump(m, s)
+ s.close
+end
Copied: debcluster/scripts/dc-startcluster (from rev 403, debcluster/scripts/startcluster.bash)
===================================================================
--- debcluster/scripts/dc-startcluster (rev 0)
+++ debcluster/scripts/dc-startcluster 2007-09-05 09:00:13 UTC (rev 406)
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Script called by katapult to prepare the head node and start masternode.rb
+
+LOGIN=root
+
+if [ $# -lt 1 ]; then
+ echo "Usage: startcluster.bash jobsfile"
+ echo "GOOD_NODES or OAR_NODEFILE must point to the nodes list"
+ echo "First node is used as head node."
+ exit 1
+fi
+
+TASKLIST=$1
+LNFILE=
+if [ "$GOOD_NODES" ]; then
+ LNFILE=$GOOD_NODES
+elif [ "$OAR_NODEFILE" ]; then
+ LNFILE=$OAR_NODEFILE
+fi
+HNODE=$(head -1 $LNFILE)
+
+echo "Head node: $HNODE"
+
+F1=$(ssh root@$HNODE mktemp /tmp/nodeslist.XXXXXX)
+scp $LNFILE $LOGIN@$HNODE:$F1
+
+F2=$(ssh root@$HNODE mktemp /tmp/tasklist.XXXXXX)
+if [ -f $TASKLIST ]; then
+ scp $TASKLIST $LOGIN@$HNODE:$F2
+fi
+
+SSHOPTS="-o BatchMode=yes -o StrictHostKeyChecking=no"
+ssh $SSHOPTS $LOGIN@$HNODE "umount -l /idpotscratch &>/dev/null; apt-get update ; apt-get -y install nfs-common; mkdir /idpotscratch &>/dev/null; rm -f /debcluster-data &>/dev/null; rm -f /debcluster-scripts &>/dev/null ; mount -t nfs -o rw,hard,intr,async,nfsvers=3,rsize=32768,wsize=32768 idpot.grenoble.grid5000.fr:/scratch /idpotscratch ; ln -sf /idpotscratch/grenoble/lnussbaum/debcluster-data /debcluster-data ; ln -sf /idpotscratch/grenoble/lnussbaum/collab-qa/debcluster /debcluster-scripts ; mkdir ~/.ssh 2>/dev/null ; sed -i 's/.*http_proxy.*//' /etc/environment"
+scp $SSHOPTS -r .ssh/* $LOGIN@$HNODE:.ssh/
+
+exec ssh $LOGIN@$HNODE "cd /debcluster-scripts/scripts/ && ./masternode.rb $F1 $F2"
Copied: debcluster/scripts/dc-startnodes (from rev 405, debcluster/scripts/startnodes.rb)
===================================================================
--- debcluster/scripts/dc-startnodes (rev 0)
+++ debcluster/scripts/dc-startnodes 2007-09-05 09:00:13 UTC (rev 406)
@@ -0,0 +1,32 @@
+#!/usr/bin/ruby -w
+
+require 'thread'
+
+LOGIN='root'
+SSHOPTS=['-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no']
+
+if ENV['GOOD_NODES']
+ LNFILE = ENV['GOOD_NODES']
+elsif ENV['OAR_NODEFILE']
+ LNFILE = ENV['GOOD_NODES']
+else
+ puts "Could not find node list"
+ exit(1)
+end
+
+ths = []
+IO::read(LNFILE).split.sort.uniq.each do |n|
+ ths << Thread::new(n) do |node|
+ system("ssh #{SSHOPTS.join(' ')} #{LOGIN}@#{node} \"umount -l /idpotscratch &>/dev/null; apt-get update ; export DEBIAN_FRONTEND=noninteractive ; apt-get -y install nfs-common ; mkdir /idpotscratch &>/dev/null; rm -f /debcluster-data &>/dev/null; rm -f /debcluster-scripts &>/dev/null ; mount -t nfs -o rw,hard,intr,async,nfsvers=3,rsize=32768,wsize=32768 idpot.grenoble.grid5000.fr:/scratch /idpotscratch ; ln -sf /idpotscratch/grenoble/lnussbaum/debcluster-data /debcluster-data ; ln -sf /idpotscratch/grenoble/lnussbaum/collab-qa/debcluster /debcluster-scripts ; mkdir ~/.ssh 2>/dev/null ; sed -i 's/.*http_proxy.*//' /etc/environment\"")
+ system("scp #{SSHOPTS.join(' ')} -qr .ssh/* #{LOGIN}@#{node}:.ssh/")
+ system("ssh #{SSHOPTS.join(' ')} #{LOGIN}@#{node} \"/debcluster-scripts/scripts/tasks/preparenode.rb --log /debcluster-data/logs/preparenode/#{node}.$(date --iso)\"")
+ if ARGV.length > 0
+ c = ['/usr/bin/ssh'] + SSHOPTS + [ "#{LOGIN}@#{node}" ] + ARGV
+ fork { exec(*c) }
+ Process::wait
+ end
+ end
+end
+ths.each do |t|
+ t.join
+end
Modified: debcluster/scripts/piuparts/piuparts.untestable
===================================================================
--- debcluster/scripts/piuparts/piuparts.untestable 2007-09-05 08:58:03 UTC (rev 405)
+++ debcluster/scripts/piuparts/piuparts.untestable 2007-09-05 09:00:13 UTC (rev 406)
@@ -163,3 +163,4 @@
vm-bonus-el # depends on vm
vm # does some emacs bytecode compilation and prompts about it. #408281
websimba # mysql
+zephyr-clients # hangs on startup if no connectivity #433602 etch only
Deleted: debcluster/scripts/startcluster.bash
===================================================================
--- debcluster/scripts/startcluster.bash 2007-09-05 08:58:03 UTC (rev 405)
+++ debcluster/scripts/startcluster.bash 2007-09-05 09:00:13 UTC (rev 406)
@@ -1,37 +0,0 @@
-#!/bin/bash
-
-# Script called by katapult to prepare the head node and start masternode.rb
-
-LOGIN=root
-
-if [ $# -lt 1 ]; then
- echo "Usage: startcluster.bash jobsfile"
- echo "GOOD_NODES or OAR_NODEFILE must point to the nodes list"
- echo "First node is used as head node."
- exit 1
-fi
-
-TASKLIST=$1
-LNFILE=
-if [ "$GOOD_NODES" ]; then
- LNFILE=$GOOD_NODES
-elif [ "$OAR_NODEFILE" ]; then
- LNFILE=$OAR_NODEFILE
-fi
-HNODE=$(head -1 $LNFILE)
-
-echo "Head node: $HNODE"
-
-F1=$(ssh root@$HNODE mktemp /tmp/nodeslist.XXXXXX)
-scp $LNFILE $LOGIN@$HNODE:$F1
-
-F2=$(ssh root@$HNODE mktemp /tmp/tasklist.XXXXXX)
-if [ -f $TASKLIST ]; then
- scp $TASKLIST $LOGIN@$HNODE:$F2
-fi
-
-SSHOPTS="-o BatchMode=yes -o StrictHostKeyChecking=no"
-ssh $SSHOPTS $LOGIN@$HNODE "umount -l /idpotscratch &>/dev/null; apt-get update ; apt-get -y install nfs-common; mkdir /idpotscratch &>/dev/null; rm -f /debcluster-data &>/dev/null; rm -f /debcluster-scripts &>/dev/null ; mount -t nfs -o rw,hard,intr,async,nfsvers=3,rsize=32768,wsize=32768 idpot.grenoble.grid5000.fr:/scratch /idpotscratch ; ln -sf /idpotscratch/grenoble/lnussbaum/debcluster-data /debcluster-data ; ln -sf /idpotscratch/grenoble/lnussbaum/collab-qa/debcluster /debcluster-scripts ; mkdir ~/.ssh 2>/dev/null ; sed -i 's/.*http_proxy.*//' /etc/environment"
-scp $SSHOPTS -r .ssh/* $LOGIN@$HNODE:.ssh/
-
-exec ssh $LOGIN@$HNODE "cd /debcluster-scripts/scripts/ && ./masternode.rb $F1 $F2"
Deleted: debcluster/scripts/startnodes.rb
===================================================================
--- debcluster/scripts/startnodes.rb 2007-09-05 08:58:03 UTC (rev 405)
+++ debcluster/scripts/startnodes.rb 2007-09-05 09:00:13 UTC (rev 406)
@@ -1,32 +0,0 @@
-#!/usr/bin/ruby -w
-
-require 'thread'
-
-LOGIN='root'
-SSHOPTS=['-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no']
-
-if ENV['GOOD_NODES']
- LNFILE = ENV['GOOD_NODES']
-elsif ENV['OAR_NODEFILE']
- LNFILE = ENV['GOOD_NODES']
-else
- puts "Could not find node list"
- exit(1)
-end
-
-ths = []
-IO::read(LNFILE).split.sort.uniq.each do |n|
- ths << Thread::new(n) do |node|
- system("ssh #{SSHOPTS.join(' ')} #{LOGIN}@#{node} \"umount -l /idpotscratch &>/dev/null; apt-get update ; export DEBIAN_FRONTEND=noninteractive ; apt-get -y install nfs-common ; mkdir /idpotscratch &>/dev/null; rm -f /debcluster-data &>/dev/null; rm -f /debcluster-scripts &>/dev/null ; mount -t nfs -o rw,hard,intr,async,nfsvers=3,rsize=32768,wsize=32768 idpot.grenoble.grid5000.fr:/scratch /idpotscratch ; ln -sf /idpotscratch/grenoble/lnussbaum/debcluster-data /debcluster-data ; ln -sf /idpotscratch/grenoble/lnussbaum/collab-qa/debcluster /debcluster-scripts ; mkdir ~/.ssh 2>/dev/null ; sed -i 's/.*http_proxy.*//' /etc/environment\"")
- system("scp #{SSHOPTS.join(' ')} -qr .ssh/* #{LOGIN}@#{node}:.ssh/")
- system("ssh #{SSHOPTS.join(' ')} #{LOGIN}@#{node} \"/debcluster-scripts/scripts/tasks/preparenode.rb --log /debcluster-data/logs/preparenode/#{node}.$(date --iso)\"")
- if ARGV.length > 0
- c = ['/usr/bin/ssh'] + SSHOPTS + [ "#{LOGIN}@#{node}" ] + ARGV
- fork { exec(*c) }
- Process::wait
- end
- end
-end
-ths.each do |t|
- t.join
-end
Modified: debcluster/scripts/tasks/piuparts.rb
===================================================================
--- debcluster/scripts/tasks/piuparts.rb 2007-09-05 08:58:03 UTC (rev 405)
+++ debcluster/scripts/tasks/piuparts.rb 2007-09-05 09:00:13 UTC (rev 406)
@@ -6,7 +6,7 @@
require 'thread'
# Timer after which the run is declared as timed out
-SLEEPTIMEOUT=1800
+SLEEPTIMEOUT=3600
mutex = Mutex::new
package = nil
@@ -15,6 +15,7 @@
logfile = nil
time = 0
piuparts = '/usr/sbin/piuparts'
+piupartsargs = ''
opts = OptionParser::new do |opts|
opts.banner = "Usage: ./piuparts.rb [options]"
@@ -22,6 +23,7 @@
opts.separator "Options:"
opts.on("-p", "--package PACKAGE", "") { |p| package = p }
opts.on("-e", "--piuparts EXEC", "") { |p| piuparts = p }
+ opts.on("-a", "--piuparts-ags ARGS", "") { |a| piupartsargs = a }
opts.on("-d", "--dist DISTRIB", "") { |d| dist << d }
opts.on("-c", "--chroot CHROOT", "") { |c| chroot = c }
opts.on("-t", "--time TIME", "") { |t| time = t.to_i }
@@ -57,7 +59,7 @@
exit 0
end
end
-b = system("#{piuparts} -a -b #{lch} -d #{dist.join(' -d ')} -m 'http://idpot.grenoble.grid5000.fr/debian' #{package}")
+b = system("#{piuparts} #{piupartsargs} -a -b #{lch} -d #{dist.join(' -d ')} -m 'http://idpot.grenoble.grid5000.fr/debian' #{package}")
#b = system("#{piuparts} -a -b #{lch} -d #{dist.join(' -d ')} -m 'http://idpot.grenoble.grid5000.fr/ubuntu main restricted universe multiverse' #{package}")
ret = $?
realtime = Time::now - ts
Modified: debcluster/scripts/tasks/preparenode.rb
===================================================================
--- debcluster/scripts/tasks/preparenode.rb 2007-09-05 08:58:03 UTC (rev 405)
+++ debcluster/scripts/tasks/preparenode.rb 2007-09-05 09:00:13 UTC (rev 406)
@@ -27,8 +27,11 @@
end
system("echo \"debconf debconf/frontend select noninteractive\" | debconf-set-selections")
system("echo \"debconf debconf/priority select critical\" | debconf-set-selections")
+ENV['DEBIAN_FRONTEND'] = 'noninteractive'
system("apt-get update")
system("apt-get install -y pbuilder nullmailer sbuild schroot debootstrap fakeroot piuparts python nfs-common apache2 linux32")
+system("dpkg -i /debcluster-data/piuparts/piuparts*deb")
+system("apt-get install -y sbuild")
# configure apache2 as proxy
system("cp /debcluster-scripts/configs/{disk_cache.conf,mem_cache.conf,proxy.conf} /etc/apache2/mods-available/")
@@ -38,6 +41,7 @@
system("/etc/init.d/apache2 restart")
system("htcacheclean -d15 -n -l1500M -p/var/cache/apache2/mod_disk_cache")
system("echo 'http_proxy=http://localhost:81/' > /etc/environment")
+system("echo 'DEBIAN_FRONTEND=noninteractive' >> /etc/environment")
ENV['http_proxy'] = 'http://localhost:81/'
# do rest of config
@@ -58,7 +62,7 @@
system("mkdir -p /var/lib/schroot/mount /var/lib/schroot/session")
-system("cp /debcluster-scripts/configs/piuparts /usr/sbin/piuparts")
+#system("cp /debcluster-scripts/configs/piuparts /usr/sbin/piuparts")
system("cp /debcluster-scripts/configs/etcsbuild.conf /etc/sbuild/sbuild.conf")
system("cp /debcluster-scripts/configs/dotsbuildrc /root/.sbuildrc")
Modified: debcluster/scripts/update-mirror-ubuntu.bash
===================================================================
--- debcluster/scripts/update-mirror-ubuntu.bash 2007-09-05 08:58:03 UTC (rev 405)
+++ debcluster/scripts/update-mirror-ubuntu.bash 2007-09-05 09:00:13 UTC (rev 406)
@@ -1,9 +1,9 @@
#!/bin/bash
LPORT=12346
-SERV=fr.archive.ubuntu.com
+SERV=archive.ubuntu.com
ROOT=ubuntu
-DIST=dapper,dapper-updates,edgy,edgy-updates,feisty
+DIST=feisty,feisty-updates,gutsy
SECT=main,restricted,universe,multiverse
ARCH=amd64,i386
TARGET=/home/grenoble/lnussbaum/debcluster-data/ubuntu-mirror
More information about the Collab-qa-commits
mailing list