[Collab-qa-commits] r571 - svnbuildstat/trunk/script
goneri-guest at alioth.debian.org
goneri-guest at alioth.debian.org
Wed Dec 12 16:57:17 UTC 2007
Author: goneri-guest
Date: 2007-12-12 16:57:17 +0000 (Wed, 12 Dec 2007)
New Revision: 571
Modified:
svnbuildstat/trunk/script/svnbuildstat_update-db.pl
Log:
new revision of the tool
Modified: svnbuildstat/trunk/script/svnbuildstat_update-db.pl
===================================================================
--- svnbuildstat/trunk/script/svnbuildstat_update-db.pl 2007-12-11 08:22:50 UTC (rev 570)
+++ svnbuildstat/trunk/script/svnbuildstat_update-db.pl 2007-12-12 16:57:17 UTC (rev 571)
@@ -1,332 +1,234 @@
#!/usr/bin/perl -w
-use File::Temp qw/ tempfile /;
+
use strict;
-use LWP::UserAgent;
-use Data::Dumper;
+use lib '/home/sites/svnbuildstat.debian.net/svnbuildstat/lib';
+
+use Config::IniFiles;
+use File::Find;
use File::Basename;
-use Time::Local 'timelocal_nocheck';
+use DateTime::Format::Pg;
+use DateTime;
+use File::stat;
+use Logger::Syslog;
-use lib '/home/sites/svnbuildstat.debian.net/svnbuildstat/lib';
use SvnBuildStat::Schema;
-use SvnBuildStat::Config;
-use Logger::Syslog;
use SvnBuildStat::Common;
use SvnBuildStat::Vcs;
-my $config;
my $schema;
+my $cfg;
+info ("starting");
-sub createTarballUrlFromTarballlayout {
- my ($repository, $repositoryentry) = @_;
+sub refreshRepository {
+ my $repository = shift;
- return unless $$repository->tarballlayout;
- my $packagename = $$repositoryentry->sourcepackage_id->name;
- my $tarball = SvnBuildStat::Common::mkTarballFromPackage($repositoryentry);
- my $tarballuri = $$repository->tarballlayout;
- $tarballuri =~ s/\@TARBALL@/$tarball/;
- $tarballuri =~ s/\@PACKAGE@/$packagename/;
-# print "\n--->".$tarballuri."\n";
+ my $vcscache = $cfg->val('path', 'vcscache');
+ my $localcache = $vcscache.'/'.$$repository->id;
- return $tarballuri;
-}
+ my $vcs = new SvnBuildStat::Vcs({
+ vcstype => $$repository->vcs_id->name,
+ uri => $$repository->uri,
+ localdir => $localcache,
+ });
+ my $newrev = $vcs->refresh();
+ print "newrev: $newrev\n";
-sub getUscanData {
- my ($repositoryentry, $watchfile) = shift;
- my $cmd;
+ if (!$newrev) {
+ info ("Failed to refresh repository");
+ return;
+ }
- return unless $watchfile;
- return unless $$repositoryentry->svndebrelease;
+ $newrev;
+}
- my ($fh, $tmpfile) = tempfile(SUFFIX => '.uscan');
- print $fh @$watchfile;
- close ($fh);
+sub scanRepositoryForChangedEntry {
+ my $repository = shift;
- my $majorrelease = $$repositoryentry->svndebrelease;
- $majorrelease =~ s/^\d+://;
- $majorrelease =~ s/-[0-9A-Za-z\.~]*$//;
- $majorrelease =~ s/dfsg.*//;
- $majorrelease =~ s/\d+://;
- $cmd = "uscan --package ".$$repositoryentry->sourcepackage_id->name." --dehs --upstream-version ".$majorrelease." --watchfile ".$tmpfile;
-# print $cmd."\n";
- my @uscan = `$cmd`;
- unlink $tmpfile or warn;
- return unless @uscan > 2; # empty output
-
- my $tarballuri;
- my $isuptodate = 'f';
- my $iswatchfilebroken = 'f';
- my $upstreamrelease;
- foreach (@uscan) {
-# print;
- $tarballuri = $1 if (/^<upstream-url>(.+tar\.gz)<\/upstream-url>$/i);
- $isuptodate = 't' if (/^<status>up to date<\/status>$/);
- $upstreamrelease = $1 if (/^<upstream-version>(.+)<\/upstream-version>$/);
- $iswatchfilebroken = 't' if (/^<errors>/);
+ my $vcscache = $cfg->val('path', 'vcscache');
+ my $localcache = $vcscache.'/'.$$repository->id;
+ my $lastcheck;
+ if ($$repository->lastcheck) {
+ $lastcheck = DateTime::Format::Pg->parse_datetime($$repository->lastcheck);
}
- $iswatchfilebroken = 't' unless $upstreamrelease;
- $tarballuri = '' unless $isuptodate eq 't';
- return {tarballuri => $tarballuri, isuptodate => $isuptodate, upstreamrelease => $upstreamrelease, iswatchfilebroken => $iswatchfilebroken};
-}
+ my $vcs = new SvnBuildStat::Vcs({
+ vcstype => $$repository->vcs_id->name,
+ uri => $$repository->uri
+ });
-sub updateVcsChanges {
- my ($repositoryentry, $vcschanges) = @_;
+ my $ret;
+ foreach (`find $localcache -type f -name 'control'`) {
+ my $dirname = dirname($_);
+ next if /\/tags\// || /\/branches\//;
+ next unless -f $dirname.'/changelog';
+ next unless -f $dirname.'/rules';
+ my $lastchange = DateTime->from_epoch(epoch => findLastChangeDate($dirname));
+ # no change since the last check?
+ return if $lastcheck && DateTime->compare($lastcheck, $lastchange);
- my @added_entry_id;
- foreach (@$vcschanges) {
- # TODO Git users put there email address in there login. With it we can link the vcs user
- # to the package maintainer
- use Data::Dumper;
- print Dumper($_);
- my $userlogin = $schema->resultset('Userlogin')->find_or_create({name=>$_->{login}});
- my $entry = $schema->resultset('Changelogentry')->create({
- userlogin_id =>$userlogin->id,
- repositoryentry_id => $$repositoryentry->id,
- date => $_->{date},
- rev => $_->{rev},
+ my $subdir = $dirname; # subdir is NOT NULL in DB so I need a last the '/'
+ $subdir =~ s/^$localcache//;
+ $subdir =~ s/\/debian$//;
+
+ my $repositoryentry = $schema->resultset('Repositoryentry')->find_or_create({
+ repository_id => $$repository->id,
+ subdir => $subdir,
});
- $entry->log($_->{text});
- $entry->update;
- push @added_entry_id, $entry->id;
- }
- # I remove the old
- if (@added_entry_id) {
- $schema->resultset('Changelogentry')->search({
- id => { '!=', \@added_entry_id },
- repositoryentry_id => $$repositoryentry->id
- })->delete;
+ push @$ret, \$repositoryentry;
}
-
+
+ return $ret;
}
-sub importRepositoryEntry {
- my( $repository, $uri, $tarballonrepository) = @_;
+sub findLastChangeDate {
+ my $dir = shift;
- my @maintainer;
- my $tarballuri;
- my $todo;
- my $currentpendingbug;
-# arch
- my $i386 = 'f';
- my $powerpc = 'f';
- my $sparc = 'f';
- my $amd64 = 'f';
+ my $lastchange = 0;
+ find(sub {
+ my $st = stat($File::Find::name);
+ $lastchange = $st->ctime if $st->ctime > $lastchange;
+
+ }, $dir);
- my $vcs;
+ return $lastchange;
+}
- my $vcsname = $$repository->vcs_id->name;
- $vcs = new SvnBuildStat::Vcs($vcsname, $uri);
+sub updateRepositoryentry {
+ my $repositoryentry = shift;
- my $rev = $vcs->getCurrentRev();
+ my $vcscache = $cfg->val('path', 'vcscache');
+ my $dir = $vcscache.'/'.$$repositoryentry->repository_id->id.$$repositoryentry->subdir;
-
- if (!$rev) {
- info ("Can't get current svn revision");
+ my $control = parseControl($dir.'/debian/control');
+ my $changelog = parseChangelog($dir.'/debian/changelog');
+ use Data::Dumper;
+ if (!$control || !$changelog) {
+ debug ("repositoryentry ".$$repositoryentry->id.": failed to parse the debian files");
return;
}
- my $control = SvnBuildStat::Common::parseControl($vcs->getControl());
- if (!$control) {
- info ("Failed to parse $uri/debian/control");
- return;
- }
-
my $sourcepackage = $schema->resultset('Sourcepackage')->find_or_create({name => $control->{packagesrc}});
- my $repositoryentry = $schema->resultset('Repositoryentry')->find_or_create({
- sourcepackage_id => $sourcepackage->id,
- repository_id => $$repository->id
- });
-
- if (defined($rev) && defined ($repositoryentry->vcsrev) && $rev eq $repositoryentry->vcsrev) {
- debug ("no need to refresh");
- return;
+ $$repositoryentry->sourcepackage_id($sourcepackage->id);
+ $$repositoryentry->dscuri(undef);
+ $$repositoryentry->realsvndebrelease($changelog->{realsvndebrelease});
+ $$repositoryentry->svndebrelease ($changelog->{svndebrelease});
+ if ($changelog->svndebrelease !~ /-[\d+\.]+$/) {
+ $$repositoryentry->isnative('t');
+ } else {
+ $$repositoryentry->isnative('f');
}
- print "GO ".$sourcepackage->name."\n";
+ $$repositoryentry->currentchangelogentry($changelog->{currentchangelogentry});
+ $$repositoryentry->currentpendingbug($changelog->{currentpendingbug});
+ $$repositoryentry->i386($control->{i386});
+ $$repositoryentry->powerpc($control->{powerpc});
+ $$repositoryentry->sparc($control->{sparc});
+ $$repositoryentry->amd64($control->{amd64});
- $repositoryentry->issrcinmypool (0); # deprecated, replaced by dscuri
- $repositoryentry->dscuri(undef);
- my $changelog = SvnBuildStat::Common::parseChangelog($vcs->getChangelog);
- $repositoryentry->realsvndebrelease($changelog->{realsvndebrelease});
- $repositoryentry->svndebrelease ($changelog->{svndebrelease});
- $repositoryentry->currentchangelogentry($changelog->{currentchangelogentry});
- $repositoryentry->currentpendingbug($changelog->{currentpendingbug});
-
- my @todo = $vcs->getTODO();
- if (@todo) {
- $todo .= $_ foreach @todo;
- }
- $repositoryentry->update_from_related('repository_id',$$repository);
-
- $repositoryentry->vcsuri($uri);
- $repositoryentry->repository_id($$repository);
-
- # Delete the current relation betwen the maintainers and the repositoryentry
+# Delete the current relation betwen the maintainers and the repositoryentry
$schema->resultset('RepositoryentryMaintainer')->search({
- repositoryentry_id => $repositoryentry->id
+ repositoryentry_id => $$repositoryentry->id
})->delete;
- # And recreate them
+# And recreate them
foreach (@{$control->{maintainers}}) {
my $maintainer = $schema->resultset('Maintainer')->find_or_create({email=>$_->{email}});
$maintainer->name($_->{name});
$maintainer->update();
my $repositoryentry_maintainer =
- $repositoryentry->find_or_create_related('repositoryentry_maintainers', {'maintainer_id' => $maintainer->id});
+ $$repositoryentry->find_or_create_related('repositoryentry_maintainers', {'maintainer_id' => $maintainer->id});
}
- # I keep the changelog entry since the oldest build, si I need.. ...to
- # find the oldest build
- my $vcschangelog_rs = $schema->resultset('Changelogentry')->search({
- repositoryentry_id => $repositoryentry->id,
- }, {order_by => "rev DESC"});
- my $last_rev = $vcschangelog_rs->first->rev if ($vcschangelog_rs->first);
- updateVcsChanges(\$repositoryentry, $vcs->getLastVcsChanges(5));
+ $$repositoryentry->update();
-
- # Search for the tarball
- my $tarball = SvnBuildStat::Common::mkTarballFromRepositoryentry(\$repositoryentry);
- my $uscandata = getUscanData(\$repositoryentry, $vcs->getWatchfile);
- my $dataFromDebianFtp = SvnBuildStat::Common::getDataFromDebianFtp(\$repositoryentry);
- use Data::Dumper;
- print Dumper($dataFromDebianFtp);
- my $tarballurlfromtarballlayout = createTarballUrlFromTarballlayout($repository,\$repositoryentry);
- $repositoryentry->iswatchfilebroken($uscandata->{iswatchfilebroken});
- # Is the tarball on a Debian mirror?
- # TODO, isnative shouldn't be returned by dataFromDebianFtp
- if ($dataFromDebianFtp->{isnative} eq 't') {
- $repositoryentry->isnative(1);
- $repositoryentry->tarballuri('');
- $repositoryentry->istarballpresent(0);
- } else {
- $repositoryentry->isnative(0);
- if ($dataFromDebianFtp->{tarballuri}) {
- $repositoryentry->tarballuri($dataFromDebianFtp->{tarballuri});
- $repositoryentry->istarballpresent(1);
- }
- elsif ($uscandata->{tarballuri}) {
- $repositoryentry->tarballuri($uscandata->{tarballuri});
- $repositoryentry->istarballpresent(1);
- }
- # Or on a HTTP/FTP space is a tarball layout exists
- elsif (SvnBuildStat::Common::testUrl($tarballurlfromtarballlayout)) {
- $repositoryentry->tarballuri($tarballurlfromtarballlayout);
- $repositoryentry->istarballpresent(1);
- # Or on the same repository
- } elsif($tarballonrepository && exists ($tarballonrepository->{$tarball})) {
- my $t = $tarballonrepository->{$tarball};
- $t =~ s!svn://svn.debian.org/svn/(.*)!http://svn.debian.org/wsvn/$1?op=file&rev=0&sc=0!;
- } else {
- $repositoryentry->tarballuri('');
- $repositoryentry->istarballpresent(0);
- }
- }
-
- #
- $repositoryentry->isuptodate($uscandata->{isuptodate});
- $repositoryentry->upstreamrelease($uscandata->{upstreamrelease});
- $repositoryentry->isindebian($dataFromDebianFtp->{isindebian});
-
- $repositoryentry->i386($control->{i386});
- $repositoryentry->powerpc($control->{powerpc});
- $repositoryentry->sparc($control->{sparc});
- $repositoryentry->amd64($control->{amd64});
- $repositoryentry->todo($todo);
- $repositoryentry->lastcheck('now');
- $repositoryentry->vcsrev($rev); # at the end since it marks in the new status of the repositoryentry ins the DB
- $repositoryentry->update();
}
+sub updateChangelogentry {
+ my $repositoryentry = shift;
-sub importRepository {
- my $repository = shift;
+ my $vcscache = $cfg->val('path', 'vcscache');
+ my $localdir = $vcscache.'/'.$$repositoryentry->repository_id->id.$$repositoryentry->subdir;
- my $vcs = new SvnBuildStat::Vcs::Svn($$repository->uri);
- print "Repository: ".$$repository->name."\n";
- debug ("importing ".$$repository->name);
- my $rev = $vcs->getCurrentRev();
- if(! $rev) {
- print "Failed to get the current revision of ".$$repository->name."\n";
- return;
- }
+ my $vcs = new SvnBuildStat::Vcs({
+ vcstype => $$repositoryentry->repository_id->vcs_id->name,
+ localdir => $localdir,
+ });
- if ($$repository->vcsrev && $$repository->vcsrev eq $rev) {
- debug ("no need to refresh ".$$repository->name);
- return;
+ my $vcschanges = $vcs->getLastVcsChanges(5);
+ my @added_entry_id;
+ foreach (@$vcschanges) {
+ # TODO Git users put there email address in there login. With it we can link the vcs user
+ # to the package maintainer
+ use Data::Dumper;
+ print Dumper($_);
+ my $userlogin = $schema->resultset('Userlogin')->find_or_create({name=>$_->{login}});
+ my $entry = $schema->resultset('Changelogentry')->find_or_create({
+ userlogin_id =>$userlogin->id,
+ repositoryentry_id => $$repositoryentry->id,
+ date => $_->{date},
+ rev => $_->{rev},
+ });
+ print "->".$entry->id."\n";
+ $entry->text($_->{text});
+ $entry->update;
+ push @added_entry_id, $entry->id;
}
- my ($uri, $tarballonrepository) = $vcs->scan();
-
- foreach (@$uri) {
- # look for package on repository
- importRepositoryEntry($repository,$_,$tarballonrepository);
- }
-
- $$repository->vcsrev($rev);
- $$repository->lastcheck('now');
- $$repository->update();
-
- debug ($$repository->name." imported");
+ # I remove the old entry TODO
+# if (@added_entry_id) {
+# $schema->resultset('Changelogentry')->search({
+# id => { '!=', \@added_entry_id },
+# repositoryentry_id => $$repositoryentry->id
+# })->delete;
+# }
}
-###################################################
-###################################################
-###################################################
-###################################################
-###################################################
-############### MAIN ##############################
-###################################################
-###################################################
-info ("starting");
-$config = new SvnBuildStat::Config();
+$cfg = Config::IniFiles->new( -file => "../svnbuildstat.ini" ) or die "Can't load config file";
$schema = SvnBuildStat::Schema->connect(
- $config->db_dsn,
- $config->db_user,
- $config->db_password,
+ $cfg->val('db', 'dsn'),
+ $cfg->val('db', 'user'),
+ $cfg->val('db', 'password'),
{AutoCommit => 1, debug => 1}
);
-
########
+
my $repository_rs = $schema->resultset('Repository')->search({enabled => 'true'});
while (my $repository = $repository_rs->next) {
- my $vcsname = $repository->vcs_id->name;
- my $uri = $repository->uri;
+ my $begin = DateTime->now;
- # one repo per team
- if ($vcsname =~ /^(svn)$/) {
- print "import Repository". $repository->name."\n";
- importRepository(\$repository);
- } else { # or one repo per package
- # TODO put this in importRepository()
- my $vcs = new SvnBuildStat::Vcs($vcsname, $uri);
- my $beginrev = $vcs->getCurrentRev();
- next if $repository->vcsrev eq $beginrev;
- print "import Repository as package ". $repository->name."\n";
- importRepositoryEntry (\$repository, $uri);
- my $endrev = $vcs->getCurrentRev();
- print "$beginrev eq $endrev\n";
- if ($beginrev eq $endrev) {
- $repository->vcsrev($endrev);
- $repository->lastcheck('now');
- }
- $repository->update();
+ # Pool repository are not scanned by this script
+ if ($repository->vcs_id->name eq "pool") {
+ next
}
-}
+ my $currentvcsrev = refreshRepository(\$repository);
+ if ($repository->vcsrev && ($repository->vcsrev eq $currentvcsrev)) {
+ debug ($repository->name." didn't change since last check");
+ next;
+ }
-# Purge the removed packages
-# This semems to be fucked?
-#my $interval = "< repository_id.lastcheck - interval '1 day'";
-#my $package_rs = $schema->resultset('Package')->search({'me.lastcheck' => \$interval} , {join => => 'repository_id'});
-#$package_rs->delete_all;
+ my $repositoryentrys = scanRepositoryForChangedEntry(\$repository);
+ foreach my $repositoryentry (@$repositoryentrys) {
+ updateRepositoryentry($repositoryentry);
+ updateChangelogentry($repositoryentry);
+ $$repositoryentry->lastcheck('now');
+ $$repositoryentry->update();
+ }
+ # TODO uncomment
+ $repository->vcsrev($currentvcsrev);
+ $repository->lastcheck(DateTime::Format::Pg->format_timestamp_with_time_zone($begin));
+ $repository->update();
+ debug ($repository->name." updated in ".(time-$begin->epoch)." seconde(s)");
+}
+
info ("stopping");
More information about the Collab-qa-commits
mailing list