[Collab-qa-commits] r794 - in svnbuildstat/trunk: . lib/SvnBuildStat lib/SvnBuildStat/Controller lib/SvnBuildStat/Model root/src/packages script
goneri-guest at alioth.debian.org
goneri-guest at alioth.debian.org
Thu Apr 17 20:29:18 UTC 2008
Author: goneri-guest
Date: 2008-04-17 20:29:18 +0000 (Thu, 17 Apr 2008)
New Revision: 794
Modified:
svnbuildstat/trunk/lib/SvnBuildStat/Common.pm
svnbuildstat/trunk/lib/SvnBuildStat/Controller/Packages.pm
svnbuildstat/trunk/lib/SvnBuildStat/Model/DB.pm
svnbuildstat/trunk/root/src/packages/main.tt2
svnbuildstat/trunk/script/svnbuildstat_update-bugs.pl
svnbuildstat/trunk/script/svnbuildstat_update-db-slow.pl
svnbuildstat/trunk/script/svnbuildstat_update-db.pl
svnbuildstat/trunk/svnbuildstat.ini
Log:
fix most of the scripts for the new DB schema, still some changes to do
Modified: svnbuildstat/trunk/lib/SvnBuildStat/Common.pm
===================================================================
--- svnbuildstat/trunk/lib/SvnBuildStat/Common.pm 2008-04-17 12:13:32 UTC (rev 793)
+++ svnbuildstat/trunk/lib/SvnBuildStat/Common.pm 2008-04-17 20:29:18 UTC (rev 794)
@@ -5,11 +5,13 @@
use LWP::UserAgent;
use Config::IniFiles;
+use LWP::Simple;
+use Logger::Syslog;
require Exporter;
our @ISA = "Exporter";
-our @EXPORT = qw(mkTarballFromChangelogentry checkRepositoryentryWatchfile testUrl mkRootdirectoryFromRepositoryentry getDataFromDebianFtp parseControl parseChangelog createTarballUrlFromTarballlayout getTODO getTarballURLWithWatchfile);
+our @EXPORT = qw(mkTarballFromChangelogentry updateRepositoryentryDebianversion checkRepositoryentryWatchfile testUrl mkRootdirectoryFromRepositoryentry getDataFromDebianFtp parseControl parseChangelog createTarballUrlFromTarballlayout getTODO getTarballURLWithWatchfile);
sub mkTarballFromChangelogentry {
my $changelogentry = shift;
@@ -17,25 +19,76 @@
$$changelogentry->repositoryentry_id->sourcepackage_id->name.'_'.$$changelogentry->version.".orig.tar.gz";
}
+sub updateRepositoryentryDebianversion {
+
+ my ($repositoryentry) = shift;
+
+ if (!$$repositoryentry->changelogentry_id || !$$repositoryentry->changelogentry_id->version) {
+ # The changelogentry are created by reading the vcs changelog, so sometime I may have no
+ # version
+ return;
+ }
+
+ my $package = $$repositoryentry->sourcepackage_id->name;
+ my $content = get("http://qa.debian.org/madison.php?package=".$package."&text=on&s=unstable");
+ my $isindebian = $content?'t':'f';
+
+ if (!$content) {
+ $$repositoryentry->changelogentry_id->isindebian('f');
+ $$repositoryentry->changelogentry_id->update;
+ } else {
+ if ($content !~ /\S+\s+\|\s+(\S+?)(\+\w+|)\s+\|\s+\S+\s+\|[-\w\d,\s]*$/) {
+ debug ("failed to parse madison for $package");
+ print ("failed to parse madison for $package\n");
+ return;
+ }
+ my $sidversion = $1;
+ my $currentversion = $$repositoryentry->changelogentry_id->version;
+ if ($$repositoryentry->changelogentry_id->debrevision) {
+ $currentversion .= '-'.$$repositoryentry->changelogentry_id->debrevision;
+ }
+ if ($$repositoryentry->changelogentry_id->epoch) {
+ $currentversion = $$repositoryentry->changelogentry_id->epoch.':'.$currentversion;
+ }
+
+
+ my $isindebian = ($sidversion eq $currentversion)?'t':'f';
+ # TODO deals with testing, stable, ... Not only with Sid
+ $$repositoryentry->sourcepackage_id->versioninsid($sidversion);
+ $$repositoryentry->sourcepackage_id->update;
+ $$repositoryentry->changelogentry_id->isindebian($isindebian);
+ $$repositoryentry->changelogentry_id->update;
+ $$repositoryentry->update;
+ }
+
+}
+
+
# The name of this function sucks
sub checkRepositoryentryWatchfile {
my ($repositoryentry) = shift;
my $cmd;
+ if (!$$repositoryentry->changelogentry_id || !$$repositoryentry->changelogentry_id->version) {
+ return;
+ }
# TODO, no ini file access in a shared function
my $cfg = Config::IniFiles->new( -file => "../svnbuildstat.ini" ) or die "Can't load config file";
my $vcscache = $cfg->val('path', 'vcscache');
my $watchfile = $vcscache.'/'.$$repositoryentry->repository_id->id.$$repositoryentry->subdir.'/debian/watch';
return unless -f $watchfile;
- my $majorrelease = $$repositoryentry->svndebrelease;
- return unless $majorrelease;
+ my $version = $$repositoryentry->changelogentry_id->version;
+ if (!$version) {
+ debug ($$repositoryentry->changelogentry_id->id." as no version!");
+ return;
+ }
- $majorrelease =~ s/^\d+://;
- $majorrelease =~ s/-[0-9A-Za-z\.~]*$//;
- $majorrelease =~ s/dfsg.*//;
- $majorrelease =~ s/\d+://;
- $cmd = "uscan --package ".$$repositoryentry->sourcepackage_id->name." --dehs --upstream-version ".$majorrelease." --watchfile ".$watchfile;
+# $majorrelease =~ s/^\d+://;
+# $majorrelease =~ s/-[0-9A-Za-z\.~]*$//;
+# $majorrelease =~ s/dfsg.*//;
+# $majorrelease =~ s/\d+://;
+ $cmd = "uscan --package ".$$repositoryentry->sourcepackage_id->name." --dehs --upstream-version ".$version." --watchfile ".$watchfile;
my @uscan = `$cmd`;
return unless @uscan > 2; # empty output
@@ -50,14 +103,13 @@
$iswatchfilebroken = 't' if (/^<errors>/);
}
$iswatchfilebroken = 't' unless $upstreamrelease;
+# print "\nversion: ".$version."\n";
+# print "upstreamrelease: ".$upstreamrelease."\n";
+# print "is up to date: $isuptodate\n";
- $$repositoryentry->upstreamrelease($upstreamrelease);
- $$repositoryentry->isuptodate($isuptodate);
+ $$repositoryentry->upstreamversion($upstreamrelease);
+ $$repositoryentry->issyncedwithupstream($isuptodate);
$$repositoryentry->iswatchfilebroken($iswatchfilebroken);
- if ($isuptodate eq 't' && !$$repositoryentry->isindebian && $tarballuri =~ /(\.tar\.gz|tgz)$/i) {
- $$repositoryentry->tarballuri($tarballuri);
- }
- $$repositoryentry->lastwatchcheck('now');
$$repositoryentry->update();
}
@@ -94,53 +146,6 @@
return;
}
-
-## The name of this function sucks
-#sub checkrepositoryentrywatchfile {
-# my ($repositoryentry) = shift;
-# my $cmd;
-#
-# # todo, no ini file access in a shared function
-# my $cfg = config::inifiles->new( -file => "../svnbuildstat.ini" ) or die "can't load config file";
-# my $vcscache = $cfg->val('path', 'vcscache');
-# my $watchfile = $vcscache.'/'.$$repositoryentry->repository_id->id.$$repositoryentry->subdir.'/debian/watch';
-# return unless -f $watchfile;
-#
-# my $majorrelease = $$repositoryentry->svndebrelease;
-# return unless $majorrelease;
-#
-# $majorrelease =~ s/^\d+://;
-# $majorrelease =~ s/-[0-9a-za-z\.~]*$//;
-# $majorrelease =~ s/dfsg.*//;
-# $majorrelease =~ s/\d+://;
-# $cmd = "uscan --package ".$$repositoryentry->sourcepackage_id->name." --dehs --upstream-version ".$majorrelease." --watchfile ".$watchfile;
-# my @uscan = `$cmd`;
-# return unless @uscan > 2; # empty output
-#
-# my $tarballuri;
-# my $isuptodate = 'f';
-# my $iswatchfilebroken = 'f';
-# my $upstreamrelease;
-# foreach (@uscan) {
-# $tarballuri = $1 if (/^<upstream-url>(.+tar\.gz)<\/upstream-url>$/i);
-# $isuptodate = 't' if (/^<status>up to date<\/status>$/);
-# $upstreamrelease = $1 if (/^<upstream-version>(.+)<\/upstream-version>$/);
-# $iswatchfilebroken = 't' if (/^<errors>/);
-# }
-# $iswatchfilebroken = 't' unless $upstreamrelease;
-#
-# $$repositoryentry->upstreamrelease($upstreamrelease);
-# $$repositoryentry->isuptodate($isuptodate);
-# $$repositoryentry->iswatchfilebroken($iswatchfilebroken);
-# if ($isuptodate eq 't' && !$$repositoryentry->isindebian && $tarballuri =~ /(\.tar\.gz|tgz)$/i) {
-# $$repositoryentry->tarballuri($tarballuri);
-# }
-# $$repositoryentry->lastwatchcheck('now');
-#
-# $$repositoryentry->update();
-#}
-
-
# Return true if the file pointed by the URL exists
sub testUrl {
my $url = shift;
Modified: svnbuildstat/trunk/lib/SvnBuildStat/Controller/Packages.pm
===================================================================
--- svnbuildstat/trunk/lib/SvnBuildStat/Controller/Packages.pm 2008-04-17 12:13:32 UTC (rev 793)
+++ svnbuildstat/trunk/lib/SvnBuildStat/Controller/Packages.pm 2008-04-17 20:29:18 UTC (rev 794)
@@ -46,7 +46,6 @@
my $package_rs;
my $maintainer;
-
if (!$param) {
$c->response->redirect($c->uri_for('/repositorys/list'));
return;
@@ -58,21 +57,21 @@
$maintainer = $c->model('SvnBuildStat::Model::DB::Maintainer')->search({email => $param})->first;
$c->stash->{maintainer} = $maintainer;
- my $packagemaintainer_rs = $c->model('SvnBuildStat::Model::DB::PackageMaintainer')->search({maintainer_id => $maintainer->id});
+ my $repositoryentry_maintainer_rs = $c->model('SvnBuildStat::Model::DB::RepositoryentryMaintainer')->search({maintainer_id => $maintainer->id});
- my @package_id;
- while (my $packagemaintainer = $packagemaintainer_rs->next) {
- push @package_id, $packagemaintainer->package_id->id;
+ my @repositoryentry_id;
+ while (my $repositoryentry_maintainer = $repositoryentry_maintainer_rs->next) {
+ push @repositoryentry_id, $repositoryentry_maintainer->repositoryentry_id;
}
- $c->stash->{packages} = [ $c->model('SvnBuildStat::Model::DB::Viewpackage')->search({ package_id => { '-in' => \@package_id } },{order_by=>"name"}) ];
+# $c->stash->{packages} = [ $c->model('SvnBuildStat::Model::DB::Viewpackage')->search({ package_id => { '-in' => \@repository_id } },{order_by=>"name"}) ];
##
# team view
} elsif ($param =~ /^\d+$/) {
$c->stash->{maintainer} = $c->model('SvnBuildStat::Model::DB::Repository')->search({id => $param})->first;
$c->stash->{packages} = [ $c->model('SvnBuildStat::Model::DB::Viewpackage')->search({ repository_id => $param },{order_by=>"name"}) ];
} else { # I don't want to kill my server with too much request
- $c->response->redirect($c->uri_for('/repositorys/list'));
- return;
+# $c->response->redirect($c->uri_for('/repositorys/list'));
+# return;
}
$c->stash->{template} = 'packages/main.tt2';
Modified: svnbuildstat/trunk/lib/SvnBuildStat/Model/DB.pm
===================================================================
--- svnbuildstat/trunk/lib/SvnBuildStat/Model/DB.pm 2008-04-17 12:13:32 UTC (rev 793)
+++ svnbuildstat/trunk/lib/SvnBuildStat/Model/DB.pm 2008-04-17 20:29:18 UTC (rev 794)
@@ -16,6 +16,15 @@
{AutoCommit => 1, debug => 1}
]
);
+
+
+sub count_users {
+ my $self = shift;
+ my $dbh = $self->dbh;
+ my $rows = $dbh->
+ selectall_arrayref('SELECT COUNT(id) FROM maintainer');
+ return $rows->[0]->[0]; # first row, then the first column
+}
=head1 NAME
SvnBuildStat::odel::DB - Catalyst DBIC Schema Model
Modified: svnbuildstat/trunk/root/src/packages/main.tt2
===================================================================
--- svnbuildstat/trunk/root/src/packages/main.tt2 2008-04-17 12:13:32 UTC (rev 793)
+++ svnbuildstat/trunk/root/src/packages/main.tt2 2008-04-17 20:29:18 UTC (rev 794)
@@ -7,9 +7,11 @@
[% META title = 'Package list' -%]
<h1>
+[% IF maintainer.email %]<a href="http://qa.debian.org/developer.php?login=[% maintainer.email %]">[% END %]
[% IF maintainer.name %]
[% maintainer.name %]
[% END %] packages overview
+[% IF maintainer.email %]</a>[% END %]
</h1>
<table>
@@ -65,7 +67,7 @@
[% END %]
</td>
- <td [% IF package.iswatchfilebroken || package.isuptodate == 0 %]class="error"[% ELSE %][% IF package.isuptodate == 1 || package.isnative == 1 %]class="ok"[% ELSE %]class="warning"[% END %][% END %]>
+ <td [% IF package.iswatchfilebroken || package.issyncedwithupstream == 0 %]class="error"[% ELSE %][% IF package.issyncedwithupstream == 1 || package.isnative == 1 %]class="ok"[% ELSE %]class="warning"[% END %][% END %]>
<a href="[% Catalyst.uri_for('info/') _ package.name %]">
[% IF package.isnative == 1 %]
Native
@@ -73,8 +75,8 @@
[% IF package.iswatchfilebroken == 1 %]
Broken watch file
[% ELSE%]
- [% IF package.isuptodate == 0 %]New upstream release[% END %]
- [% IF package.isuptodate == "" %]No watch file[% END %]
+ [% IF package.issyncedwithupstream == 0 %]New upstream release[% END %]
+ [% IF package.issyncedwithupstream == "" %]No watch file[% END %]
[% END %]
[% END %]
</a>
Modified: svnbuildstat/trunk/script/svnbuildstat_update-bugs.pl
===================================================================
--- svnbuildstat/trunk/script/svnbuildstat_update-bugs.pl 2008-04-17 12:13:32 UTC (rev 793)
+++ svnbuildstat/trunk/script/svnbuildstat_update-bugs.pl 2008-04-17 20:29:18 UTC (rev 794)
@@ -25,12 +25,14 @@
while( my $sourcepackage = $sourcepackage_rs->next) {
$currentPackage{$sourcepackage->name} = 1;
}
-my $tmpfile = "/tmp/fullindex".$$;
-
-unlink $tmpfile;
-die if (!is_success( getstore("http://qa.debian.org/data/bts2ldap/fullindex", $tmpfile)));
-print "download ok\n";
+#my $tmpfile = "/tmp/fullindex".$$;
+#
+#die if (!is_success( getstore("http://qa.debian.org/data/bts2ldap/fullindex", $tmpfile)));
+#print "download ok\n";
+my $tmpfile = "/tmp/fullindex.save";
my %keepThisBug;
+my %keepThisBugmerged;
+my %keepThisBugbugtag;
open BTS2LDAP, "<$tmpfile" or die;
print "updating\n";
@@ -38,30 +40,67 @@
LINE: while (my $line = readline BTS2LDAP) {
if ($line =~ /^$/) {
if ($h{debbugsSourcePackage} && exists ($currentPackage{$h{debbugsSourcePackage}})) {
- if (!($h{debbugsTag} && $h{debbugsTag} =~ /(pending|wontfix|unreproducible|moreinfo)/)&& $h{debbugsState} =~ /open/) {
+# if (!($h{debbugsTag} && $h{debbugsTag} =~ /(pending|wontfix|unreproducible|moreinfo)/)&& $h{debbugsState} =~ /open/) {
- my $sourcepackage = $schema->resultset('Sourcepackage')->search({name => $h{debbugsSourcePackage}})->first;
- my $severity = $schema->resultset('Severity')->find_or_create({name=>$h{debbugsSeverity}});
- my $bug = $schema->resultset('Bug')->find_or_create({id => $h{debbugsID}});
+ my $sourcepackage = $schema->resultset('Sourcepackage')->search({name => $h{debbugsSourcePackage}})->first;
+ my $severity = $schema->resultset('Severity')->find_or_create({name=>$h{debbugsSeverity}});
+ my $bug = $schema->resultset('Bug')->find_or_create({id => $h{debbugsID}});
- $bug->severity_id ($severity->id);
- $bug->sourcepackage_id ($sourcepackage->id);
- $bug->name ($h{debbugsTitle});
- $bug->update;
+ $bug->severity_id ($severity->id);
+ $bug->sourcepackage_id ($sourcepackage->id);
+ $bug->name ($h{debbugsTitle});
+ $bug->update;
- $keepThisBug{$h{debbugsID}} = 1;
+ $keepThisBug{$h{debbugsID}} = 1;
+
+ foreach (@{$h{debbugsMergedWith}}) {
+ $schema->resultset('Bug')->find_or_create({id => $_}); # In case the debbugsMergedWith referes to an old
+ # bug, I create an empty entry for it
+ my $bugmerged;
+ if ($_ > $h{debbugsID}) {
+ # bug1_id is the smallest bugnum
+ $bugmerged = $schema->resultset('Mergedbug')->find_or_create(bug1_id => $h{debbugsID}, bug2_id => $_);
+ } else {
+ $bugmerged = $schema->resultset('Mergedbug')->find_or_create(bug1_id => $_, bug2_id => $h{debbugsID});
+ }
+ $keepThisBugmerged{$bugmerged->id} = 1;
}
+ foreach (@{$h{debbugsMergedWith}}) {
+ my $bugtag = $schema->resultset('Bugtag')->find_or_create({name => $_});
+ my $bugbugtag = $schema->resultset('BugBugtag')->find_or_create(bug_id => $h{debbugsID}, bugtag_id => $bugtag->id);
+ $keepThisBugbugtag{$bugbugtag->id} = 1;
+ }
+
+# }
}
%h = ();
}
- next unless $line =~ /^(debbugsID|debbugsSourcePackage|debbugsState|debbugsTitle|debbugsTag|debbugsSeverity):/;
+ next unless $line =~ /^(debbugsID|debbugsSourcePackage|debbugsState|debbugsTitle|debbugsTag|debbugsSeverity|debbugsMergedWith):/;
chomp($line);
- $h{$1} = $2 if ($line =~ /(\w+): (.+)$/);
+ if ($line =~ /(\w+): (.+)$/) {
+ my $key = $1;
+ my $val = $2;
+ if ($key =~ /(debbugsMergedWith|debbugsTag)/) {
+# print $line."\n";
+ $h{$key} = [] if (!$h{$key});;
+ push @{$h{$key}}, $val;
+ } else {
+ $h{$key} = $val;
+ }
+ }
}
close BTS2LDAP;
-unlink $tmpfile;
+#unlink $tmpfile;
my $bug_rs = $schema->resultset('Bug');
while( my $bug = $bug_rs->next) {
$bug->delete unless exists ($keepThisBug{$bug->id});
}
+my $bugmerged_rs = $schema->resultset('Mergedbug');
+while( my $bugmerged = $bugmerged_rs->next) {
+ $bugmerged->delete unless exists ($keepThisBugmerged{$bugmerged->id});
+}
+my $bugbugtag_rs = $schema->resultset('BugBugtag');
+while( my $bugbugtag = $bugbugtag_rs->next) {
+ $bugbugtag->delete unless exists ($keepThisBugbugtag{$bugbugtag->id});
+}
Modified: svnbuildstat/trunk/script/svnbuildstat_update-db-slow.pl
===================================================================
--- svnbuildstat/trunk/script/svnbuildstat_update-db-slow.pl 2008-04-17 12:13:32 UTC (rev 793)
+++ svnbuildstat/trunk/script/svnbuildstat_update-db-slow.pl 2008-04-17 20:29:18 UTC (rev 794)
@@ -31,7 +31,9 @@
});
while (my $repositoryentry = $repositoryentry_rs->next) {
checkRepositoryentryWatchfile(\$repositoryentry);
-
+ updateRepositoryentryDebianversion(\$repositoryentry);
+ $repositoryentry->lastwatchcheck('now');
+ $repositoryentry->update();
}
debug ("end (".(time - $begin)." secondes)");
Modified: svnbuildstat/trunk/script/svnbuildstat_update-db.pl
===================================================================
--- svnbuildstat/trunk/script/svnbuildstat_update-db.pl 2008-04-17 12:13:32 UTC (rev 793)
+++ svnbuildstat/trunk/script/svnbuildstat_update-db.pl 2008-04-17 20:29:18 UTC (rev 794)
@@ -35,7 +35,7 @@
});
my $newrev = $vcs->refresh();
- print "newrev: $newrev\n";
+ debug ("new VCS revision: $newrev\n");
if (!$newrev) {
info ("Failed to refresh repository");
@@ -68,6 +68,7 @@
next unless -f $dirname.'/rules';
my $lastchange = DateTime->from_epoch(epoch => findLastChangeDate($dirname));
# no change since the last check?
+ # I turn off this to for the fullscan of the repository
return if $lastcheck && DateTime->compare($lastcheck, $lastchange);
my $subdir = $dirname; # subdir is NOT NULL in DB so I need a last the '/'
@@ -112,7 +113,6 @@
return;
}
- print $control->{packagesrc}."\n";
my $sourcepackage = $schema->resultset('Sourcepackage')->find_or_create({name => $control->{packagesrc}});
$$repositoryentry->sourcepackage_id($sourcepackage->id);
# $$repositoryentry->dscuri(undef);
@@ -261,7 +261,6 @@
$$repositoryentry->update();
}
- # TODO uncomment
$repository->rev($currentrev);
$repository->lastcheck(DateTime::Format::Pg->format_timestamp_with_time_zone($begin));
$repository->update();
Modified: svnbuildstat/trunk/svnbuildstat.ini
===================================================================
--- svnbuildstat/trunk/svnbuildstat.ini 2008-04-17 12:13:32 UTC (rev 793)
+++ svnbuildstat/trunk/svnbuildstat.ini 2008-04-17 20:29:18 UTC (rev 794)
@@ -1,7 +1,7 @@
[db]
dsn = DBI:Pg:host=127.0.0.1;dbname=svnbuildstat
user = svnbuildstat
-password = XXXXXX
+password = XXXXX
[path]
debmirror = http://ftp.fr.debian.org
@@ -10,3 +10,4 @@
uploaddir = /home/goneri/cache/svnbuildstat/uploads
vcscache = /home/goneri/cache/vcscache
tarballdir = /home/goneri/cache/tarball
+webroot = /home/goneri/public_html/svnbuildstat
More information about the Collab-qa-commits
mailing list