[Collab-qa-commits] r614 - svnbuildstat/trunk/script
goneri-guest at alioth.debian.org
goneri-guest at alioth.debian.org
Wed Dec 19 15:36:00 UTC 2007
Author: goneri-guest
Date: 2007-12-19 15:36:00 +0000 (Wed, 19 Dec 2007)
New Revision: 614
Modified:
svnbuildstat/trunk/script/svnbuildstat_tarball.pl
svnbuildstat/trunk/script/svnbuildstat_update-bugs.pl
svnbuildstat/trunk/script/svnbuildstat_update-db.pl
svnbuildstat/trunk/script/svnbuildstat_update-repository.pl
Log:
changes for the new DB layout and the ini config file
Modified: svnbuildstat/trunk/script/svnbuildstat_tarball.pl
===================================================================
--- svnbuildstat/trunk/script/svnbuildstat_tarball.pl 2007-12-19 14:21:21 UTC (rev 613)
+++ svnbuildstat/trunk/script/svnbuildstat_tarball.pl 2007-12-19 15:36:00 UTC (rev 614)
@@ -8,6 +8,8 @@
use SvnBuildStat::Schema;
use SvnBuildStat::Common;
use Logger::Syslog;
+use LWP::Simple;
+use File::Copy;
my $cfg = Config::IniFiles->new( -file => "../svnbuildstat.ini" ) or die "Can't load config file";
my $schema = SvnBuildStat::Schema->connect(
@@ -17,37 +19,60 @@
{AutoCommit => 1, debug => 1}
);
+my $tarballdir = $cfg->val('path', 'tarball');
+
+if (-f $tarballdir) {
+ print "the tarball directory (`$tarballdir') doesn't exit";
+ exit 1;
+}
+
my $begin = time;
debug ("begin");
-my $repositoryentry_rs = $schema->resultset('Repositoryentry')->search();
-while (my $repositoryentry = $repositoryentry_rs->next) {
- my $dataFromDebianFtp = getDataFromDebianFtp(\$repositoryentry);
+my $changelogentry_rs = $schema->resultset('Changelogentry')->search({ tarball_id => undef , version => { '!=', undef}, isnative => 'false' });
+while (my $changelogentry = $changelogentry_rs->next) {
+ my $tarball;
+ my $filename = mkTarballFromChangelogentry(\$changelogentry);
+ if (-f $tarballdir.'/'.$filename) {
+ $tarball = $schema->resultset('Tarball')->find_or_create({ name => $filename });
+ $changelogentry->tarball_id($tarball->id);
+ $changelogentry->update();
+ next;
+ }
+
+ my $repositoryentry = $changelogentry->repositoryentry_id;
+ my $dataFromDebianFtp = getDataFromDebianFtp(\$changelogentry);
+
+ my $tarballuri;
if ($dataFromDebianFtp->{isindebian} eq 't') {
- $repositoryentry->isindebian(1);
- $repositoryentry->tarballuri($dataFromDebianFtp->{tarballuri});
+ $changelogentry->isindebian(1);
+ $tarballuri = $dataFromDebianFtp->{tarballuri};
} else {
- $repositoryentry->isindebian(0);
+ $changelogentry->isindebian(0);
+ }
- if ($repositoryentry->repository_id->tarballlayout) {
- my $tarballurlfromtarballlayout = createTarballUrlFromTarballlayout(\$repositoryentry);
+ if (!$tarballuri) {
+ if ($changelogentry->repositoryentry_id->repository_id->tarballlayout) {
+ my $tarballurlfromtarballlayout = createTarballUrlFromChangeogentry(\$changelogentry);
if ($tarballurlfromtarballlayout && testUrl($tarballurlfromtarballlayout)) {
- $repositoryentry->tarballuri($tarballurlfromtarballlayout);
+ $tarballuri = $tarballurlfromtarballlayout;
}
- } elsif ($repositoryentry->tarballuri && !testUrl($repositoryentry->tarballuri)) {
-#Probably filled by the watch file, let's check the url
- $repositoryentry->tarballuri(undef);
}
}
- # this is useless but I prefere to be sure there is no non null empty string
- if ($repositoryentry->tarballuri) {
- $repositoryentry->tarballuri(undef);
+ if (!$tarballuri) {
+ $tarballuri = getTarballURLWithWatchfile(\$changelogentry);
}
-
- $repositoryentry->update();
+ next unless $tarballuri;
+ if (getstore ($tarballuri, $tarballdir.'/'.$filename.'.part') && move($tarballdir.'/'.$filename.'.part', $tarballdir.'/'.$filename)) {
+ $tarball = $schema->resultset('Tarball')->find_or_create({ name => $filename });
+ $changelogentry->tarball_id($tarball->id);
+ } else {
+ unlink $tarballdir.'/'.$filename.'.part';
+ }
+
# TODO, check for svnbuildpackage OrigUrl here
# TODO deal with tarballs that are one the repository
Modified: svnbuildstat/trunk/script/svnbuildstat_update-bugs.pl
===================================================================
--- svnbuildstat/trunk/script/svnbuildstat_update-bugs.pl 2007-12-19 14:21:21 UTC (rev 613)
+++ svnbuildstat/trunk/script/svnbuildstat_update-bugs.pl 2007-12-19 15:36:00 UTC (rev 614)
@@ -3,21 +3,20 @@
use strict;
use LWP::Simple;
-use Data::Dumper;
+use Config::IniFiles;
use File::Temp qw/ tempfile /;
-use lib '/home/sites/svnbuildstat.debian.net/svnbuildstat/lib';
+use lib '../lib';
#"http://qa.debian.org/data/bts2ldap/fullindex";
use SvnBuildStat::Schema;
-use SvnBuildStat::Config;
-my $config = new SvnBuildStat::Config();
+my $cfg = Config::IniFiles->new( -file => $ENV{HOME}."/.svnbuildstat.ini" ) or die "Can't load config file";
my $schema = SvnBuildStat::Schema->connect(
- $config->db_dsn,
- $config->db_user,
- $config->db_password,
+ $cfg->val('db', 'dsn'),
+ $cfg->val('db', 'user'),
+ $cfg->val('db', 'password'),
{AutoCommit => 1, debug => 1}
);
@@ -31,7 +30,7 @@
unlink $tmpfile;
die if (!is_success( getstore("http://qa.debian.org/data/bts2ldap/fullindex", $tmpfile)));
print "download ok\n";
-my %registredPackage;
+my %keepThisBug;
open BTS2LDAP, "<$tmpfile" or die;
print "updating\n";
@@ -42,15 +41,16 @@
if (!($h{debbugsTag} && $h{debbugsTag} =~ /(pending|wontfix|unreproducible|moreinfo)/)&& $h{debbugsState} =~ /open/) {
my $sourcepackage = $schema->resultset('Sourcepackage')->search({name => $h{debbugsSourcePackage}})->first;
+ print $sourcepackage->id."\n";
my $severity = $schema->resultset('Severity')->find_or_create({name=>$h{debbugsSeverity}});
- my $bug = $schema->resultset('Bug')->find_or_create({bug_id => $h{debbugsID}});
+ my $bug = $schema->resultset('Bug')->find_or_create({id => $h{debbugsID}});
$bug->severity_id ($severity->id);
$bug->sourcepackage_id ($sourcepackage->id);
$bug->name ($h{debbugsTitle});
$bug->update;
- $registredPackage{$h{debbugsID}} = 1;
+ $keepThisBug{$h{debbugsID}} = 1;
}
}
%h = ();
@@ -64,5 +64,5 @@
unlink $tmpfile;
my $bug_rs = $schema->resultset('Bug');
while( my $bug = $bug_rs->next) {
- $bug->delete unless exists ($registredPackage{$bug->id});
+ $bug->delete unless exists ($keepThisBug{$bug->id});
}
Modified: svnbuildstat/trunk/script/svnbuildstat_update-db.pl
===================================================================
--- svnbuildstat/trunk/script/svnbuildstat_update-db.pl 2007-12-19 14:21:21 UTC (rev 613)
+++ svnbuildstat/trunk/script/svnbuildstat_update-db.pl 2007-12-19 15:36:00 UTC (rev 614)
@@ -102,10 +102,11 @@
my $repositoryentry = shift;
my $vcscache = $cfg->val('path', 'vcscache');
- my $dir = $vcscache.'/'.$$repositoryentry->repository_id->id.$$repositoryentry->subdir;
+ my $localdir = $vcscache.'/'.$$repositoryentry->repository_id->id.$$repositoryentry->subdir;
- my $control = parseControl($dir.'/debian/control');
- my $changelog = parseChangelog($dir.'/debian/changelog');
+ my $control = parseControl($localdir);
+ my $changelog = parseChangelog($localdir);
+ my $todo = getTODO($localdir);
if (!$control || !$changelog) {
debug ("repositoryentry ".$$repositoryentry->id.": failed to parse the debian files");
return;
@@ -114,16 +115,18 @@
print $control->{packagesrc}."\n";
my $sourcepackage = $schema->resultset('Sourcepackage')->find_or_create({name => $control->{packagesrc}});
$$repositoryentry->sourcepackage_id($sourcepackage->id);
- $$repositoryentry->dscuri(undef);
- $$repositoryentry->realsvndebrelease($changelog->{realsvndebrelease});
- $$repositoryentry->svndebrelease ($changelog->{svndebrelease});
- if ($changelog->{svndebrelease} !~ /-[\d+\.]+$/) {
- $$repositoryentry->isnative('t');
- } else {
- $$repositoryentry->isnative('f');
- }
- $$repositoryentry->currentchangelogentry($changelog->{currentchangelogentry});
- $$repositoryentry->currentpendingbug($changelog->{currentpendingbug});
+# TODO
+# $$repositoryentry->dscuri(undef);
+# $$repositoryentry->realsvndebrelease($changelog->{realsvndebrelease});
+# $$repositoryentry->svndebrelease ($changelog->{svndebrelease});
+# if ($changelog->{svndebrelease} !~ /-[\d+\.]+$/) {
+# $$repositoryentry->isnative('t');
+# } else {
+# $$repositoryentry->isnative('f');
+# }
+ $$repositoryentry->debchangelogentry($changelog->{debchangelogentry});
+# $$repositoryentry->currentpendingbug($changelog->{currentpendingbug});
+ $$repositoryentry->todo($todo);
$$repositoryentry->i386($control->{i386});
$$repositoryentry->powerpc($control->{powerpc});
$$repositoryentry->sparc($control->{sparc});
@@ -155,6 +158,13 @@
my $vcscache = $cfg->val('path', 'vcscache');
my $localdir = $vcscache.'/'.$$repositoryentry->repository_id->id.$$repositoryentry->subdir;
+ my $control = parseControl($localdir);
+ my $changelog = parseChangelog($localdir);
+ if (!$control || !$changelog) {
+ debug ("repositoryentry ".$$repositoryentry->id.": failed to parse the debian files");
+ return;
+ }
+
my $vcs = new SvnBuildStat::Vcs({
vcstype => $$repositoryentry->repository_id->vcs_id->name,
localdir => $localdir,
@@ -162,6 +172,7 @@
my $vcschanges = $vcs->getLastVcsChanges(5);
my @added_entry_id;
+ my $first = 1;
foreach (@$vcschanges) {
# TODO Git users put there email address in there login. With it we can link the vcs user
# to the package maintainer
@@ -172,8 +183,33 @@
date => $_->{date},
rev => $_->{rev},
});
-# print "->".$entry->id."\n";
$entry->text($_->{text});
+
+ if ($first) {
+ if ($changelog->{debrevision}) {
+ $entry->isnative('f');
+ } else {
+ $entry->isnative('t');
+ }
+ $entry->debrevision($changelog->{debrevision});
+ $entry->version($changelog->{version});
+ $entry->epoch($changelog->{epoch});
+
+ foreach my $bugnum (@{$changelog->{currentpendingbug}}) {
+ my $bug = $schema->resultset('Bug')->find_or_create({
+ id => $bugnum
+ });
+
+ $schema->resultset('ChangelogentryBug')->find_or_create({
+ bug_id => $bug->id,
+ changelogentry_id => $entry->id,
+ });
+ }
+ $$repositoryentry->changelogentry_id($entry->id);
+ $$repositoryentry->update();
+ $first = undef;
+ }
+
$entry->update;
push @added_entry_id, $entry->id;
}
@@ -207,8 +243,8 @@
if ($repository->vcs_id->name eq "pool") {
next
}
- my $currentvcsrev = refreshRepository(\$repository);
- if ($repository->vcsrev && ($repository->vcsrev eq $currentvcsrev)) {
+ my $currentrev = refreshRepository(\$repository);
+ if ($repository->rev && ($repository->rev eq $currentrev)) {
debug ($repository->name." didn't change since last check");
next;
}
@@ -222,7 +258,7 @@
}
# TODO uncomment
- $repository->vcsrev($currentvcsrev);
+ $repository->rev($currentrev);
$repository->lastcheck(DateTime::Format::Pg->format_timestamp_with_time_zone($begin));
$repository->update();
debug ($repository->name." updated in ".(time-$begin->epoch)." seconde(s)");
Modified: svnbuildstat/trunk/script/svnbuildstat_update-repository.pl
===================================================================
--- svnbuildstat/trunk/script/svnbuildstat_update-repository.pl 2007-12-19 14:21:21 UTC (rev 613)
+++ svnbuildstat/trunk/script/svnbuildstat_update-repository.pl 2007-12-19 15:36:00 UTC (rev 614)
@@ -20,9 +20,13 @@
my $debmirror = $cfg->val('path', 'debmirror');
my $repositorydir = $cfg->val('path', 'repositorydir');
+my $tarballdir = $cfg->val('path', 'tarball');
if (!-d $repositorydir) {
- die "repositorydir `'".$repositorydir."' doesn't exist\n";
+ die "repositorydir `'".$repositorydir."' doesn't exist\n";
}
+if (!-d $tarballdir) {
+ die "tarballdir `'".$tarballdir."' doesn't exist\n";
+}
chdir $repositorydir or die;
@@ -30,132 +34,135 @@
sub addSubRevInChangelog {
- my ($rootdir, $subrev) = @_;
-
- open ORIG, "<$rootdir/debian/changelog" or return;
- my @orig = <ORIG>;
- close ORIG;
+ my ($rootdir, $subrev) = @_;
- return unless ($orig[0] =~ s/\((.*)\)/($1~$subrev)/);
+ open ORIG, "<$rootdir/debian/changelog" or return;
+ my @orig = <ORIG>;
+ close ORIG;
- open DEST, ">$rootdir/debian/changelog" or return;
- foreach (@orig) {
- print DEST;
- }
- close DEST;
+ return unless ($orig[0] =~ s/\((.*)\)/($1~$subrev)/);
+ open DEST, ">$rootdir/debian/changelog" or return;
+ foreach (@orig) {
+ print DEST;
+ }
+ close DEST;
+
}
sub fetchAndPrepareDirectoryForMergeWithUpstream {
- my ($repositoryentry, $workdir) = @_;
+ my ($repositoryentry, $workdir) = @_;
- my $tarball = SvnBuildStat::Common::mkTarballFromRepositoryentry($repositoryentry);
+ my $tarball = $$repositoryentry->changelogentry_id->tarball_id->name;
- return unless $tarball;
- debug("looks for the tarball");
+ debug("looks for the tarball");
- # TODO Will always fails because I should check the
- # package repository directly
- if (!-f "$workdir/$tarball") {
- my $tarballuri = $$repositoryentry->tarballuri;
- if (!$tarballuri) {
- debug("can't find the tarball");
- return;
+# TODO Will always fails because I should check the
+# package repository directly
+ if (!-f "$tarballdir/$tarball") {
+ $$repositoryentry->changelogentry_id->tarball_id(undef);
}
- $tarballuri =~ s/\@DEBMIRROR@/$debmirror/;
- if (is_error(getstore($tarballuri,"$workdir/$tarball"))) {
- debug ("Failed to download the tarball");
-# debug("failed to download ".$tarballuri);
- return;
+ my $cmd = "cp -l $tarballdir/$tarball $workdir/$tarball";
+ `$cmd`;
+ if (-f "$workdir/$tarball") {
+ debug ("Failed to copy $tarballdir/$tarball to $workdir/$tarball");
+ return;
}
- }
- chdir $workdir or die;
- my $rootdirectory;
- foreach (`tar tf $tarball 2>&1`) {
- if (/^(.\/|)(.*?)\//) {
- if ($rootdirectory && $rootdirectory ne $2) {
- info("tarball has more than one root directory! I ignore it");
- return;
- }
- $rootdirectory = $2;
+
+ chdir $workdir or die;
+ my $pkgrootdirectory;
+ foreach (`tar tf $tarball 2>&1`) {
+ print;
+ if (/^(.\/|)(.*?)\//) {
+ if ($pkgrootdirectory && $pkgrootdirectory ne $2) {
+ info("tarball has more than one root directory! I ignore it");
+ return;
+ }
+ $pkgrootdirectory = $2;
}
- }
- `tar xf $tarball 2>&1`;
- if (($? >> 8)!=0) {
- info("failed to untar $tarball");
- return;
- }
- if (!move($rootdirectory, SvnBuildStat::Common::mkRootdirectoryFromRepositoryentry($repositoryentry))) {
- return;
- }
+ }
+ `tar xf $tarball 2>&1`;
+ if (($? >> 8)!=0) {
+ info("failed to untar $tarball");
+ return;
+ }
+# if (!move($pkgrootdirectory, SvnBuildStat::Common::mkRootdirectoryFromRepositoryentry($repositoryentry))) {
+# return;
+# }
+ return $pkgrootdirectory;
- 1;#OK
+# 1;#OK
}
sub prepare {
- my ($repositoryentry) = @_;
- debug("prepareFromSvn: ".$$repositoryentry->sourcepackage_id->name);
+ my ($repositoryentry) = @_;
+ debug("prepareFromSvn: ".$$repositoryentry->sourcepackage_id->name);
- my $vcscache = $cfg->val('path', 'vcscache');
- my $localdir = $vcscache.'/'.$$repositoryentry->repository_id->id.$$repositoryentry->subdir;
- my $vcsname = $$repositoryentry->repository_id->vcs_id->name;
+ my $vcscache = $cfg->val('path', 'vcscache');
+ my $localdir = $vcscache.'/'.$$repositoryentry->repository_id->id.$$repositoryentry->subdir;
+ my $vcsname = $$repositoryentry->repository_id->vcs_id->name;
- my $vcs = new SvnBuildStat::Vcs({
- vcstype => $$repositoryentry->repository_id->vcs_id->name,
- localdir => $localdir,
- });
+ my $vcs = new SvnBuildStat::Vcs({
+ vcstype => $$repositoryentry->repository_id->vcs_id->name,
+ localdir => $localdir,
+ });
- my $repo_shortname = $$repositoryentry->repository_id->shortname;
- my $directory = $$repositoryentry->sourcepackage_id->name;
- # TODO replace repo_shortname/ by team short name
- #my $workdir = $config->server_repositorydir."/$repo_shortname/".$$package->name."/tmp";
- my $workdir = $repositorydir."/tmp";
-
- `rm -Rf $workdir; mkdir -p $workdir`;
- chdir $workdir or die;
+ my $directory = $$repositoryentry->sourcepackage_id->name;
+ my $workdir = $repositorydir."/tmp";
- if (!$$repositoryentry->isnative) {
- return unless fetchAndPrepareDirectoryForMergeWithUpstream($repositoryentry, $workdir);
- }
+ `rm -Rf $workdir; mkdir -p $workdir`;
+ chdir $workdir or die;
- my $packagerootdir = SvnBuildStat::Common::mkRootdirectoryFromRepositoryentry($repositoryentry);
- my $vcsrev = $vcs->currentRev;
- $vcs->exportIn($workdir."/".$packagerootdir);
+ my $pkgrootdirectory;
+ if (!$$repositoryentry->changelogentry_id->isnative) {
+ $pkgrootdirectory = fetchAndPrepareDirectoryForMergeWithUpstream($repositoryentry, $workdir);
+ print "rootdirectoy: `".$pkgrootdirectory."'\n";
+ return unless $pkgrootdirectory;
+ print "rootdirectoy is ok\n";
+ } else {
+ $pkgrootdirectory = $$repositoryentry->sourcepackage_id->name.'-'.$$repositoryentry->changelogentry_id->version;
+ `mkdir -p $workdir."/".$pkgrootdirectory`;
+ }
- return unless addSubRevInChangelog($packagerootdir, $vcsname.$vcsrev);
+ my $vcsrev = $vcs->currentRev;
+ print "exportIn begin".$workdir."/".$pkgrootdirectory."\n";
+ $vcs->exportIn($workdir."/".$pkgrootdirectory);
+ print "exportIn end".$workdir."/".$pkgrootdirectory."\n";
- `dpkg-source -b -W $packagerootdir 2>&1`;
- if (($? >> 8)!=0) {
- info ("failed to create .dsc");
- return;
- }
+ return unless addSubRevInChangelog($workdir."/".$pkgrootdirectory, $vcsname.$vcsrev);
- my $destdir = $repositorydir.'/'.
- $$repositoryentry->repository_id->team_id->shortname.'/'.
- $$repositoryentry->sourcepackage_id->name;
- `mkdir -p $destdir` unless -d $destdir;
+ `dpkg-source -b -W $pkgrootdirectory 2>&1`;
+ if (($? >> 8)!=0) {
+ info ("failed to create .dsc");
+ return;
+ }
- my $dscfile;
- foreach my $file (<$workdir/*>) {
- next unless -f $file;
- $dscfile = basename($file) if $file =~ /\.dsc/;
- move ($file, $destdir);
- }
- return unless $dscfile;
+ my $destdir = $repositorydir.'/'.
+ $$repositoryentry->repository_id->team_id->shortname.'/'.
+ $$repositoryentry->sourcepackage_id->name;
+ `mkdir -p $destdir` unless -d $destdir;
- $$repositoryentry->dscuri('@REPOSITORY_URL@/'.$$repositoryentry->repository_id->team_id->shortname.'/'.$$repositoryentry->sourcepackage_id->name.'/'.$dscfile);
- $$repositoryentry->issrcinmypool(1);#TODO deprecated, we use dscuri instead
- $$repositoryentry->update;
- `rm -Rf $workdir`;
- 1;
-}
+ my $dscfile;
+ foreach my $file (<$workdir/*>) {
+ next unless -f $file;
+ $dscfile = basename($file) if $file =~ /\.dsc/;
+ move ($file, $destdir);
+ }
+ return unless $dscfile;
+ $$repositoryentry->changelogentry_id->dscuri('@REPOSITORY_URL@/'.$$repositoryentry->repository_id->team_id->shortname.'/'.$$repositoryentry->sourcepackage_id->name.'/'.$dscfile);
+ $$repositoryentry->issrcinmypool(1);#TODO deprecated, we use dscuri instead
+ $$repositoryentry->update;
+ `rm -Rf $workdir`;
+ 1;
+ }
+
##########################################
##########################################
################## MAIN ##################
@@ -176,10 +183,10 @@
info ("starting");
$schema = SvnBuildStat::Schema->connect(
- $cfg->val('db', 'dsn'),
- $cfg->val('db', 'user'),
- $cfg->val('db', 'password'),
- {AutoCommit => 1, debug => 1}
+$cfg->val('db', 'dsn'),
+$cfg->val('db', 'user'),
+$cfg->val('db', 'password'),
+{AutoCommit => 1, debug => 1}
);
# It's not a joke since I'll do rm -Rf in this directory and don't want to trash
# the system yet :)
@@ -187,9 +194,12 @@
#purgeOutDated();
-my $repositoryentry_rs = $schema->resultset('Repositoryentry')->search({ dscuri => undef, tarballuri => { '!=' => undef} });
+my $repositoryentry_rs = $schema->resultset('Repositoryentry');
while (my $repositoryentry = $repositoryentry_rs->next) {
- prepare(\$repositoryentry);
+ if ($repositoryentry->changelogentry_id && !$repositoryentry->changelogentry_id->dscuri && $repositoryentry->changelogentry_id->tarball_id) {
+ print "prepare ".$repositoryentry->sourcepackage_id->name."\n";
+ prepare(\$repositoryentry);
+ }
}
More information about the Collab-qa-commits
mailing list