[Collab-qa-commits] r383 - svnbuildstat/script
goneri-guest at alioth.debian.org
goneri-guest at alioth.debian.org
Mon Aug 20 22:22:02 UTC 2007
Author: goneri-guest
Date: 2007-08-20 22:22:02 +0000 (Mon, 20 Aug 2007)
New Revision: 383
Modified:
svnbuildstat/script/svnbuildstat_agent.pl
Log:
now I can have more than one process at the same time. The father process is
used to detected freeze build and to take a look on the disk space usage.
With these changes, I will probably avoid 95% of the current false positive.
Modified: svnbuildstat/script/svnbuildstat_agent.pl
===================================================================
--- svnbuildstat/script/svnbuildstat_agent.pl 2007-08-13 21:07:17 UTC (rev 382)
+++ svnbuildstat/script/svnbuildstat_agent.pl 2007-08-20 22:22:02 UTC (rev 383)
@@ -1,4 +1,5 @@
#!/usr/bin/perl -w
+
use strict;
use lib ".";
@@ -10,6 +11,7 @@
use SvnBuildStat::Config;
use LWP::Simple;
+
###########################################################################
###########################################################################
###########################################################################
@@ -18,8 +20,13 @@
my $RELEASE = "0.0.1";
my $hostname = hostname();
my $distro = "sid";
-my $isnative;
+my $maxjobs = $config->agent_maxjobs;
+chomp (my $arch = `dpkg-architecture -qDEB_HOST_ARCH`);
+my %job;
+
+die unless $maxjobs =~ /^\d+$/;
+
if ($config->agent_vardir !~ /\/.+/) {
die "vardir not correclty set in the [agent] section";
}
@@ -36,6 +43,7 @@
die "please create: `".$config->agent_vardir unless -d
$config->agent_vardir;
+# TODO check for running script in background
# Purge
if (-d $workplace) {`rm -r $workplace`;}
if (-d $buildarea) {`rm -r $buildarea`;}
@@ -63,85 +71,106 @@
print CCACHECFG "BUILDRESULT=\"$tmpdir\"\n";
close CCACHECFG;
-my %report;
-chomp ($report{'arch'} = `dpkg-architecture -qDEB_HOST_ARCH`);
-$report{'buildadminaddr'} = $config->agent_buildadminaddr;
-$report{hostname} = $hostname;
-$report{distro} = $distro;
+sub isFull {
+ my $path = shift;
-$report{'agent_release'} = $RELEASE;
-$report{'svnbp_release'} = `dpkg-query -W -f='\${Version}' svn-buildpackage`;
-$report{'linda_release'} = `dpkg-query -W -f='\${Version}' linda`;
-$report{'lintian_release'} = `dpkg-query -W -f='\${Version}' lintian`;
-$report{'pbuilder_release'} = `dpkg-query -W -f='\${Version}' pbuilder`;
-$report{'subversion_release'} = `dpkg-query -W -f='\${Version}' subversion`;
-$report{'piuparts_release'} = `dpkg-query -W -f='\${Version}' piuparts`;
-###########################################################################
-###########################################################################
-###########################################################################
+ foreach (`df -P $path`) {
+ if (/^\S+\s+\d+\s+\d+\s+(\d+)\s+\S+/) {
+ return ($1 < 300000)?1:0;
+ }
+ }
-my $tobuild = LWP::Simple::get($config->agent_source."/".$report{'arch'});
-if (!$tobuild) {
- print "Failed to contact the serveur to get a package to build\n";
- exit 1;
+ return;
}
-my @tobuild = split /;/, $tobuild;
-my $package = $tobuild[0];
-$report{'source'} = $package;
-my $tarballuri = $tobuild[1];
-my $tarball = $tobuild[2];
-my $svnbase = $tobuild[3];
-$isnative = 1 unless ($tarballuri || $tarball);
-if (!$package||!$svnbase) {
- print "Failed to parse the package information from ".
- $config->agent_source."/".$report{'arch'}."\n";
- exit 1
-}
-my $t = $config->agent_debmirror;
-my $pdebuildparam = "--mirror ".$config->agent_debmirror." --buildplace $pbuilderplace --othermirror 'deb ".$config->agent_debmirror." $distro main contrib non-free' --distribution $distro --basetgz $pbuilderplace/$distro.tar.gz";
-`/usr/sbin/pbuilder update $pdebuildparam >$reportarea/pbuilder-update.log 2>&1`;
-if (($? >> 8)!=0){
- print "`/usr/sbin/pbuilder create $pdebuildparam >$reportarea/pbuilder-create.log 2>&1`";
- `/usr/sbin/pbuilder create $pdebuildparam >$reportarea/pbuilder-create.log 2>&1`;
- if (($? >> 8)!=0) {
- print "Failed to create pbuilder image\n";
- exit 1
+
+
+sub howxsnext {
+ my $p;
+
+ my $tobuild = LWP::Simple::get($config->agent_source."/".$arch);
+ if (!$tobuild) {
+ print "Failed to contact the serveur to get a package to build\n";
+ exit 1;
}
+ my @tobuild = split /;/, $tobuild;
+ $p->{svnbase} = $tobuild[0];
+ $p->{tarballuri} = $tobuild[1];
+
+ if (!$p->{packageuri}||!$p->{svnbase}) {
+ print "Failed to parse the package information from ".
+ $config->agent_source."/".$arch."\n";
+ return;
+ }
}
-foreach (`LC_ALL=C svn info $svnbase`) {
- $report{'svn_rev'} = $1 if /Last Changed Rev:\ (\d+)/;
-}
-`svn co $svnbase $workplace >$reportarea/svn.log 2>&1`;
-my @tmp = `cd $workplace ; svn log -q --limit 1`;
-foreach (`LC_ALL=C svn info $svnbase`) {
- if (/Last Changed Rev:\ (\d+)/) {
- if( $report{'svn_rev'} ne $1 ) {
- print "HACK: svn info> rev changed during checkout\n";
- exit;
- }
- }
-}
+sub build {
+ my $p = shift;
-my $ftp = Net::FTP->new($config->agent_ftphost, Debug => 0) or die "Cannot connect
-to ".$config->agent_ftphost.": $@";
+ my $threadworkplace = $workplace .'/'. $$;
+ my $threadreportarea = $reportarea .'/'. $$;
+ my $threadbuildarea = $buildarea .'/'. $$;
-$ftp->login($config->agent_ftplogin,$config->agent_ftppassword) or die "Cannot login ", $ftp->message;
+ die if (!-d $threadworkplace && !mkdir $threadworkplace);
+ die if (!-d $threadreportarea && !mkdir $threadreportarea);
+ die if (!-d $threadbuildarea && !mkdir $threadbuildarea);
-print "package: $package\n";
-if (!$ftp->appe($package."_".$report{'svn_rev'}."_".$report{'arch'}."_".$distro.".lock")) {
- print "package: $package already locked by a buildbot\n";
- exit 1;
-}
-$ftp->close;
+ my %report;
+ $report{'arch'} = $arch;
+ $report{'buildadminaddr'} = $config->agent_buildadminaddr;
+ $report{hostname} = $hostname;
+ $report{distro} = $distro;
-if (!open CTRL, "<$workplace/debian/control") {
- $report{checkout} = 'nok';
-} else {
- $report{checkout} = 'ok';
- close CTRL;
+ $report{'agent_release'} = $RELEASE;
+ $report{'svnbp_release'} = `dpkg-query -W -f='\${Version}' svn-buildpackage`;
+ $report{'linda_release'} = `dpkg-query -W -f='\${Version}' linda`;
+ $report{'lintian_release'} = `dpkg-query -W -f='\${Version}' lintian`;
+ $report{'pbuilder_release'} = `dpkg-query -W -f='\${Version}' pbuilder`;
+ $report{'subversion_release'} = `dpkg-query -W -f='\${Version}' subversion`;
+ $report{'piuparts_release'} = `dpkg-query -W -f='\${Version}' piuparts`;
+###########################################################################
+###########################################################################
+###########################################################################
- foreach (`cd $workplace ; dpkg-parsechangelog 2>&1`) {
+
+ my $pdebuildparam = "--debootstrap debootstrap --mirror ".$config->agent_debmirror." --buildplace $pbuilderplace --othermirror 'deb ".$config->agent_debmirror." $distro main contrib non-free' --distribution $distro --basetgz $pbuilderplace/$distro.tar.gz";
+ `/usr/sbin/pbuilder update $pdebuildparam >$threadreportarea/pbuilder-update.log 2>&1`;
+ if (($? >> 8)!=0){
+ `/usr/sbin/pbuilder create $pdebuildparam >$threadreportarea/pbuilder-create.log 1>&2`;
+ if (($? >> 8)!=0) {
+ print "Failed to create pbuilder image\n";
+ exit 1
+ }
+ }
+ foreach (`LC_ALL=C svn info $p->{svnbase}`) {
+ $report{'svn_rev'} = $1 if /Last Changed Rev:\ (\d+)/;
+ }
+ `svn co $p->{svnbase} $threadworkplace >$threadreportarea/svn.log 2>&1`;
+ my @tmp = `cd $threadworkplace ; svn log -q --limit 1`;
+ foreach (`LC_ALL=C svn info $p->{svnbase}`) {
+ if (/Last Changed Rev:\ (\d+)/) {
+ if( $report{'svn_rev'} ne $1 ) {
+ print "HACK: svn info> rev changed during checkout\n";
+ }
+ }
+ }
+
+
+ if (!open CTRL, "<$threadworkplace/debian/control") {
+ #$report{checkout} = 'nok';
+ return;
+ } else {
+
+ foreach (<CTRL>) {
+ if (/^Source:\s+(\S+)/) {
+ $report{'source'} = $1;
+ last;
+ }
+ }
+ $report{checkout} = 'ok';
+ close CTRL;
+ }
+
+ foreach (`cd $threadworkplace ; dpkg-parsechangelog 2>&1`) {
if (/Source: (.+)/) {
$report{'source'} = $1;
} elsif (/Version: (.+?)(-.+|$)/) {
@@ -150,19 +179,23 @@
}
- if ($isnative) {
+ if (!$p->{tarballuri}) {
$report{'download'} = "native";
} else {
- $tarballuri =~ s/\@DEBMIRROR\@/$t/;
- `cd $tarballsplace ; wget -O $tarball -c \"$tarballuri\" >$reportarea/wget.log 2>&1`;
- `tar tf $tarballsplace/$tarball`;
- if (($? >> 8)!=0) {
- print "download nok\n";
- $report{'download'} = "nok";
- unlink $tarballsplace."/".$tarball;
- } else {
- print "download ok\n";
- $report{'download'} = "ok";
+ my @t = split /\//, $p->{tarballuri};
+ my $tarball = pop @t;
+
+ my $t = $config->agent_debmirror;
+ $p->{tarballuri} =~ s/\@DEBMIRROR\@/$t/;
+ `cd $tarballsplace ; wget -O $tarball -c \"$p->{tarballuri}\" >$threadreportarea/wget.log 2>&1`;
+ `tar tf $tarballsplace/$tarball`;
+ if (($? >> 8)!=0) {
+ print "download nok\n";
+ $report{'download'} = "nok";
+ unlink $tarballsplace."/".$tarball;
+ } else {
+ print "download ok\n";
+ $report{'download'} = "ok";
}
}
@@ -171,52 +204,94 @@
$report{'stamp_build-start'} = time;
# If the there is just a debian directory I assum it's a mergeWithUpstream layout and I add the property
- if (!grep (!/^debian$/, `/bin/ls $workplace`)) {
- `cd $workplace; svn propset mergeWithUpstream 1 debian`;
+ if (!grep (!/^debian$/, `/bin/ls $threadworkplace`)) {
+ `cd $threadworkplace; svn propset mergeWithUpstream 1 debian`;
}
- `cd $workplace ; svn-buildpackage --svn-ignore-new --svn-builder "pdebuild --use-pdebuild-internal --configfile $ccachedir/ccache.cfg -- --buildplace $pbuilderplace --aptcache $aptcachedir --distribution $distro --basetgz $pbuilderplace/$distro.tar.gz" --svn-override=origDir=$tarballsplace --svn-override=buildArea=$buildarea --svn-noninteractive >$reportarea/build.log.tmp 2>&1`;
+ `cd $threadworkplace ; svn-buildpackage --svn-ignore-new --svn-builder "pdebuild --use-pdebuild-internal --configfile $ccachedir/ccache.cfg -- --buildplace $pbuilderplace --aptcache $aptcachedir --distribution $distro --basetgz $pbuilderplace/$distro.tar.gz" --svn-override=origDir=$tarballsplace --svn-override=buildArea=$threadbuildarea --svn-noninteractive >$threadreportarea/build.log.tmp 2>&1`;
$report{'stamp_build-end'} = time;
# To avoid breakage with tar on the service side, I do some clean up in the logs
- open BUILDLOGTMP, "<$reportarea/build.log.tmp" or die;
- open BUILDLOG, ">$reportarea/build.log" or die;
+ open BUILDLOGTMP, "<$threadreportarea/build.log.tmp" or die;
+ open BUILDLOG, ">$threadreportarea/build.log" or die;
foreach (<BUILDLOGTMP>) {
- s/[[:cntrl:]]//g;
- print BUILDLOG $_."\n";
+ s/[[:cntrl:]]//g;
+ print BUILDLOG $_."\n";
}
close BUILDLOGTMP;
close BUILDLOG;
- unlink "$reportarea/build.log.tmp";
+ unlink "$threadreportarea/build.log.tmp";
-
- if (<$buildarea/*.deb>) {
+ my $debfiles;
+ $debfiles .= $_." " foreach (<$threadbuildarea/*.{,u}deb>);
+ if ($debfiles) {
$report{build} = "ok";
- `cd $buildarea ; lintian --allow-root *.deb *.dsc > $reportarea/lintian.log`;
- `cd $buildarea ; linda --show-tag *.deb *.dsc 2>/dev/null > $reportarea/linda.log`;
- `cd $buildarea ; /usr/sbin/piuparts *.deb | grep -v DEBUG > $reportarea/piuparts.log`;
+ `cd $threadbuildarea ; lintian --allow-root $debfiles *.dsc > $threadreportarea/lintian.log`;
+ `cd $threadbuildarea ; linda --show-tag $debfiles *.dsc 2>/dev/null > $threadreportarea/linda.log`;
+ `cd $threadbuildarea ; /usr/sbin/piuparts $debfiles | grep -v DEBUG > $threadreportarea/piuparts.log`;
$report{piuparts} = (($? >> 8)==0)?'ok':'nok';
} else {
$report{build} = "nok";
}
}
-}
# Prepare and send the report
-open BUILDREPORT,">".$reportarea."/info";
-foreach (sort keys %report) {
- chomp $report{$_};
+ open BUILDREPORT,">".$threadreportarea."/info";
+ foreach (sort keys %report) {
+ chomp $report{$_};
- print BUILDREPORT $_."=".$report{$_}."\n";
+ print BUILDREPORT $_."=".$report{$_}."\n";
+ }
+ close BUILDREPORT;
+
+ my $ftp = Net::FTP->new($config->agent_ftphost, Debug => 0) or die "Cannot connect
+ to ".$config->agent_ftphost.": $@";
+
+ my $reporttarball = "$report{'source'}_$report{'svn_rev'}_".$arch."$hostname.tar";
+ $ftp->login($config->agent_ftplogin,$config->agent_ftppassword) or die "Cannot login ", $ftp->message;
+ `cd $tmpdir ; cp -r $threadreportarea . ; tar cf $reporttarball report 2>&1`;
+ $ftp->put($tmpdir."/".$reporttarball) or warn "Failed to send the report\n";
+ $ftp->close or warn "Failed to close the ftp connection to the server\n";
+
}
-close BUILDREPORT;
-$ftp = Net::FTP->new($config->agent_ftphost, Debug => 0) or die "Cannot connect
-to ".$config->agent_ftphost.": $@";
+$SIG{CHLD} = 'IGNORE';
+while (sleep 1) {
-my $reporttarball = "$report{'source'}_$report{'svn_rev'}_$report{'arch'}_$hostname.tar";
-$ftp->login($config->agent_ftplogin,$config->agent_ftppassword) or die "Cannot login ", $ftp->message;
-`cd $tmpdir ; cp -r $reportarea . ; tar cf $reporttarball report 2>&1`;
-$ftp->put($tmpdir."/".$reporttarball) or warn "Failed to send the report\n";
-$ftp->close or warn "Failed to close the ftp connection to the server\n";
+ if (isFull($workplace)||isFull($reportarea)||isFull($buildarea)) {
+ `rm -f $tarballsplace/* $aptcachedir/*`;
+ print "disk is full\n";
+ if (%job&&(isFull($workplace)||isFull($reportarea)||isFull($buildarea))) {
+ print "Emergency clean up\n";
+ kill 9, $_ foreach (keys %job);
+ }
+ }
+
+ foreach my $pid (keys %job) {
+ if (time > $job{$pid} + 3600*3) {
+ print "kill\n";
+ kill 9, $pid; # timeout
+ sleep 5;
+ }
+ if (! -d "/proc/".$pid) {
+ `rm -rf $workplace/$pid $reportarea/$pid $buildarea/$pid`;
+ delete ($job{$pid});
+ }
+ }
+
+ if (keys %job < $maxjobs) {
+ my $pid = fork;
+ if ($pid) {
+ print "$$ papa $pid\n";
+ $job{$pid} = time;
+ } else {
+ print "fille $$\n";
+ my $p;# = howxsnext();
+ $p = {svn => "svn://svn.debian.org/svn/pkg-perl/trunk/libtest-object-perl", tarballuri => "http://backpan.perl.org/authors/id/A/AD/ADAMK/Test-Object-0.07.tar.gz;libtest-object-perl_0.07.orig.tar.gz"};
+ build($p);
+ last;
+ }
+ }
+
+}
More information about the Collab-qa-commits
mailing list