[Collab-qa-commits] r794 - in svnbuildstat/trunk: . lib/SvnBuildStat lib/SvnBuildStat/Controller lib/SvnBuildStat/Model root/src/packages script

goneri-guest at alioth.debian.org goneri-guest at alioth.debian.org
Thu Apr 17 20:29:18 UTC 2008


Author: goneri-guest
Date: 2008-04-17 20:29:18 +0000 (Thu, 17 Apr 2008)
New Revision: 794

Modified:
   svnbuildstat/trunk/lib/SvnBuildStat/Common.pm
   svnbuildstat/trunk/lib/SvnBuildStat/Controller/Packages.pm
   svnbuildstat/trunk/lib/SvnBuildStat/Model/DB.pm
   svnbuildstat/trunk/root/src/packages/main.tt2
   svnbuildstat/trunk/script/svnbuildstat_update-bugs.pl
   svnbuildstat/trunk/script/svnbuildstat_update-db-slow.pl
   svnbuildstat/trunk/script/svnbuildstat_update-db.pl
   svnbuildstat/trunk/svnbuildstat.ini
Log:
fix most of the scripts for the new DB schema, still some changes to do


Modified: svnbuildstat/trunk/lib/SvnBuildStat/Common.pm
===================================================================
--- svnbuildstat/trunk/lib/SvnBuildStat/Common.pm	2008-04-17 12:13:32 UTC (rev 793)
+++ svnbuildstat/trunk/lib/SvnBuildStat/Common.pm	2008-04-17 20:29:18 UTC (rev 794)
@@ -5,11 +5,13 @@
 
 use LWP::UserAgent;
 use Config::IniFiles;
+use LWP::Simple;
+use Logger::Syslog;
 
 require Exporter;
 
 our @ISA = "Exporter";
-our @EXPORT = qw(mkTarballFromChangelogentry checkRepositoryentryWatchfile testUrl mkRootdirectoryFromRepositoryentry getDataFromDebianFtp parseControl parseChangelog createTarballUrlFromTarballlayout getTODO getTarballURLWithWatchfile);
+our @EXPORT = qw(mkTarballFromChangelogentry updateRepositoryentryDebianversion checkRepositoryentryWatchfile testUrl mkRootdirectoryFromRepositoryentry getDataFromDebianFtp parseControl parseChangelog createTarballUrlFromTarballlayout getTODO getTarballURLWithWatchfile);
 
 sub mkTarballFromChangelogentry {
   my $changelogentry = shift;
@@ -17,25 +19,76 @@
   $$changelogentry->repositoryentry_id->sourcepackage_id->name.'_'.$$changelogentry->version.".orig.tar.gz";
 }
 
+sub updateRepositoryentryDebianversion {
+  
+  my ($repositoryentry) = shift;
+
+  if (!$$repositoryentry->changelogentry_id || !$$repositoryentry->changelogentry_id->version) {
+    # The changelogentry are created by reading the vcs changelog, so sometime I may have no
+    # version
+    return;
+  }
+
+  my $package = $$repositoryentry->sourcepackage_id->name;
+  my $content = get("http://qa.debian.org/madison.php?package=".$package."&text=on&s=unstable");
+  my $isindebian = $content?'t':'f';
+ 
+  if (!$content) {
+    $$repositoryentry->changelogentry_id->isindebian('f');
+    $$repositoryentry->changelogentry_id->update;
+  } else {
+    if ($content !~ /\S+\s+\|\s+(\S+?)(\+\w+|)\s+\|\s+\S+\s+\|[-\w\d,\s]*$/) {
+      debug ("failed to parse madison for $package");
+      print ("failed to parse madison for $package\n");
+      return;
+    }
+    my $sidversion = $1;
+    my $currentversion = $$repositoryentry->changelogentry_id->version;
+    if ($$repositoryentry->changelogentry_id->debrevision) {
+      $currentversion .= '-'.$$repositoryentry->changelogentry_id->debrevision;
+    }
+    if ($$repositoryentry->changelogentry_id->epoch) {
+      $currentversion = $$repositoryentry->changelogentry_id->epoch.':'.$currentversion;
+    }
+  
+ 
+    my $isindebian = ($sidversion eq $currentversion)?'t':'f';
+    # TODO deals with testing, stable, ... Not only with Sid
+    $$repositoryentry->sourcepackage_id->versioninsid($sidversion);
+    $$repositoryentry->sourcepackage_id->update;
+    $$repositoryentry->changelogentry_id->isindebian($isindebian);
+    $$repositoryentry->changelogentry_id->update;
+    $$repositoryentry->update;
+  }
+
+}
+
+
 # The name of this function sucks
 sub checkRepositoryentryWatchfile {
   my ($repositoryentry) = shift;
   my $cmd;
 
+  if (!$$repositoryentry->changelogentry_id || !$$repositoryentry->changelogentry_id->version) {
+    return;
+  }
   # TODO, no ini file access in a shared function
   my $cfg = Config::IniFiles->new( -file => "../svnbuildstat.ini" ) or die "Can't load config file";
   my $vcscache = $cfg->val('path', 'vcscache');
   my $watchfile = $vcscache.'/'.$$repositoryentry->repository_id->id.$$repositoryentry->subdir.'/debian/watch';
   return unless -f $watchfile;
 
-  my $majorrelease = $$repositoryentry->svndebrelease;
-  return unless $majorrelease;
+  my $version = $$repositoryentry->changelogentry_id->version;
+  if (!$version) {
+    debug ($$repositoryentry->changelogentry_id->id." as no version!");
+    return;
+  }
 
-  $majorrelease =~ s/^\d+://;
-  $majorrelease =~ s/-[0-9A-Za-z\.~]*$//;
-  $majorrelease =~ s/dfsg.*//;
-  $majorrelease =~ s/\d+://;
-  $cmd = "uscan --package ".$$repositoryentry->sourcepackage_id->name." --dehs --upstream-version ".$majorrelease." --watchfile ".$watchfile;
+#  $majorrelease =~ s/^\d+://;
+#  $majorrelease =~ s/-[0-9A-Za-z\.~]*$//;
+#  $majorrelease =~ s/dfsg.*//;
+#  $majorrelease =~ s/\d+://;
+  $cmd = "uscan --package ".$$repositoryentry->sourcepackage_id->name." --dehs --upstream-version ".$version." --watchfile ".$watchfile;
   my @uscan = `$cmd`;
   return unless @uscan > 2; # empty output
 
@@ -50,14 +103,13 @@
     $iswatchfilebroken = 't' if (/^<errors>/);
   }
   $iswatchfilebroken = 't' unless $upstreamrelease;
+#  print "\nversion: ".$version."\n";
+#  print "upstreamrelease: ".$upstreamrelease."\n";
+#  print "is up to date: $isuptodate\n";
   
-  $$repositoryentry->upstreamrelease($upstreamrelease);
-  $$repositoryentry->isuptodate($isuptodate);
+  $$repositoryentry->upstreamversion($upstreamrelease);
+  $$repositoryentry->issyncedwithupstream($isuptodate);
   $$repositoryentry->iswatchfilebroken($iswatchfilebroken);
-  if ($isuptodate eq 't' && !$$repositoryentry->isindebian && $tarballuri =~ /(\.tar\.gz|tgz)$/i) {
-    $$repositoryentry->tarballuri($tarballuri);
-  }
-  $$repositoryentry->lastwatchcheck('now');
 
   $$repositoryentry->update();
 }
@@ -94,53 +146,6 @@
     return;
 }
 
-
-## The name of this function sucks
-#sub checkrepositoryentrywatchfile {
-#  my ($repositoryentry) = shift;
-#  my $cmd;
-#
-#  # todo, no ini file access in a shared function
-#  my $cfg = config::inifiles->new( -file => "../svnbuildstat.ini" ) or die "can't load config file";
-#  my $vcscache = $cfg->val('path', 'vcscache');
-#  my $watchfile = $vcscache.'/'.$$repositoryentry->repository_id->id.$$repositoryentry->subdir.'/debian/watch';
-#  return unless -f $watchfile;
-#
-#  my $majorrelease = $$repositoryentry->svndebrelease;
-#  return unless $majorrelease;
-#
-#  $majorrelease =~ s/^\d+://;
-#  $majorrelease =~ s/-[0-9a-za-z\.~]*$//;
-#  $majorrelease =~ s/dfsg.*//;
-#  $majorrelease =~ s/\d+://;
-#  $cmd = "uscan --package ".$$repositoryentry->sourcepackage_id->name." --dehs --upstream-version ".$majorrelease." --watchfile ".$watchfile;
-#  my @uscan = `$cmd`;
-#  return unless @uscan > 2; # empty output
-#
-#  my $tarballuri;
-#  my $isuptodate = 'f';
-#  my $iswatchfilebroken = 'f';
-#  my $upstreamrelease;
-#  foreach (@uscan) {
-#    $tarballuri = $1 if (/^<upstream-url>(.+tar\.gz)<\/upstream-url>$/i);
-#    $isuptodate = 't' if (/^<status>up to date<\/status>$/);
-#    $upstreamrelease = $1 if (/^<upstream-version>(.+)<\/upstream-version>$/);
-#    $iswatchfilebroken = 't' if (/^<errors>/);
-#  }
-#  $iswatchfilebroken = 't' unless $upstreamrelease;
-#  
-#  $$repositoryentry->upstreamrelease($upstreamrelease);
-#  $$repositoryentry->isuptodate($isuptodate);
-#  $$repositoryentry->iswatchfilebroken($iswatchfilebroken);
-#  if ($isuptodate eq 't' && !$$repositoryentry->isindebian && $tarballuri =~ /(\.tar\.gz|tgz)$/i) {
-#    $$repositoryentry->tarballuri($tarballuri);
-#  }
-#  $$repositoryentry->lastwatchcheck('now');
-#
-#  $$repositoryentry->update();
-#}
-
-
 # Return true if the file pointed by the URL exists
 sub testUrl {
   my $url = shift;

Modified: svnbuildstat/trunk/lib/SvnBuildStat/Controller/Packages.pm
===================================================================
--- svnbuildstat/trunk/lib/SvnBuildStat/Controller/Packages.pm	2008-04-17 12:13:32 UTC (rev 793)
+++ svnbuildstat/trunk/lib/SvnBuildStat/Controller/Packages.pm	2008-04-17 20:29:18 UTC (rev 794)
@@ -46,7 +46,6 @@
 
   my $package_rs;
   my $maintainer;
-
   if (!$param) {
     $c->response->redirect($c->uri_for('/repositorys/list'));
     return;
@@ -58,21 +57,21 @@
     $maintainer = $c->model('SvnBuildStat::Model::DB::Maintainer')->search({email => $param})->first;
     $c->stash->{maintainer} = $maintainer;
 
-    my $packagemaintainer_rs = $c->model('SvnBuildStat::Model::DB::PackageMaintainer')->search({maintainer_id => $maintainer->id});
+    my $repositoryentry_maintainer_rs = $c->model('SvnBuildStat::Model::DB::RepositoryentryMaintainer')->search({maintainer_id => $maintainer->id});
 
-    my @package_id;
-    while (my $packagemaintainer = $packagemaintainer_rs->next) {
-      push @package_id, $packagemaintainer->package_id->id;
+    my @repositoryentry_id;
+    while (my $repositoryentry_maintainer = $repositoryentry_maintainer_rs->next) {
+      push @repositoryentry_id, $repositoryentry_maintainer->repositoryentry_id;
     }
-    $c->stash->{packages} = [ $c->model('SvnBuildStat::Model::DB::Viewpackage')->search({ package_id => { '-in' => \@package_id } },{order_by=>"name"}) ];
+#    $c->stash->{packages} = [ $c->model('SvnBuildStat::Model::DB::Viewpackage')->search({ package_id => { '-in' => \@repository_id } },{order_by=>"name"}) ];
     ##
     #  team view
   } elsif ($param =~ /^\d+$/) {
     $c->stash->{maintainer} = $c->model('SvnBuildStat::Model::DB::Repository')->search({id => $param})->first;
     $c->stash->{packages} = [ $c->model('SvnBuildStat::Model::DB::Viewpackage')->search({ repository_id => $param },{order_by=>"name"}) ];
   } else { # I don't want to kill my server with too much request
-    $c->response->redirect($c->uri_for('/repositorys/list'));
-    return;
+#    $c->response->redirect($c->uri_for('/repositorys/list'));
+#    return;
   }
 
   $c->stash->{template} = 'packages/main.tt2';

Modified: svnbuildstat/trunk/lib/SvnBuildStat/Model/DB.pm
===================================================================
--- svnbuildstat/trunk/lib/SvnBuildStat/Model/DB.pm	2008-04-17 12:13:32 UTC (rev 793)
+++ svnbuildstat/trunk/lib/SvnBuildStat/Model/DB.pm	2008-04-17 20:29:18 UTC (rev 794)
@@ -16,6 +16,15 @@
   {AutoCommit => 1, debug => 1}
   ]
 );
+
+
+sub count_users {
+  my $self = shift;
+  my $dbh = $self->dbh;
+  my $rows = $dbh->
+  selectall_arrayref('SELECT COUNT(id) FROM maintainer');
+  return $rows->[0]->[0]; # first row, then the first column
+}
 =head1 NAME
 
 SvnBuildStat::odel::DB - Catalyst DBIC Schema Model

Modified: svnbuildstat/trunk/root/src/packages/main.tt2
===================================================================
--- svnbuildstat/trunk/root/src/packages/main.tt2	2008-04-17 12:13:32 UTC (rev 793)
+++ svnbuildstat/trunk/root/src/packages/main.tt2	2008-04-17 20:29:18 UTC (rev 794)
@@ -7,9 +7,11 @@
 [% META title = 'Package list' -%]
 
 <h1>
+[% IF maintainer.email %]<a href="http://qa.debian.org/developer.php?login=[% maintainer.email %]">[% END %]
 [% IF maintainer.name %]
 [% maintainer.name %]
 [% END %] packages overview
+[% IF maintainer.email %]</a>[% END %]
 </h1>
 
 <table>
@@ -65,7 +67,7 @@
 
 [% END %]
     </td>
-    <td [% IF package.iswatchfilebroken || package.isuptodate == 0 %]class="error"[% ELSE %][% IF package.isuptodate == 1 || package.isnative == 1 %]class="ok"[% ELSE %]class="warning"[% END %][% END %]>
+    <td [% IF package.iswatchfilebroken || package.issyncedwithupstream == 0 %]class="error"[% ELSE %][% IF package.issyncedwithupstream == 1 || package.isnative == 1 %]class="ok"[% ELSE %]class="warning"[% END %][% END %]>
     <a href="[% Catalyst.uri_for('info/') _ package.name %]">
         [% IF package.isnative == 1 %]
 	  Native
@@ -73,8 +75,8 @@
 	  [% IF package.iswatchfilebroken == 1 %]
 	    Broken watch file
 	  [% ELSE%]
-            [% IF package.isuptodate == 0 %]New upstream release[% END %]
-            [% IF package.isuptodate == "" %]No watch file[% END %]
+            [% IF package.issyncedwithupstream == 0 %]New upstream release[% END %]
+            [% IF package.issyncedwithupstream == "" %]No watch file[% END %]
           [% END %]
         [% END %]
       </a>

Modified: svnbuildstat/trunk/script/svnbuildstat_update-bugs.pl
===================================================================
--- svnbuildstat/trunk/script/svnbuildstat_update-bugs.pl	2008-04-17 12:13:32 UTC (rev 793)
+++ svnbuildstat/trunk/script/svnbuildstat_update-bugs.pl	2008-04-17 20:29:18 UTC (rev 794)
@@ -25,12 +25,14 @@
 while( my $sourcepackage = $sourcepackage_rs->next) {
   $currentPackage{$sourcepackage->name} = 1;
 }
-my $tmpfile = "/tmp/fullindex".$$;
-
-unlink $tmpfile;
-die if (!is_success( getstore("http://qa.debian.org/data/bts2ldap/fullindex", $tmpfile)));
-print "download ok\n";
+#my $tmpfile = "/tmp/fullindex".$$;
+#
+#die if (!is_success( getstore("http://qa.debian.org/data/bts2ldap/fullindex", $tmpfile)));
+#print "download ok\n";
+my $tmpfile = "/tmp/fullindex.save";
 my %keepThisBug;
+my %keepThisBugmerged;
+my %keepThisBugbugtag;
 
 open BTS2LDAP, "<$tmpfile" or die;
 print "updating\n";
@@ -38,30 +40,67 @@
 LINE: while (my $line = readline BTS2LDAP) {
   if ($line =~ /^$/) {
     if ($h{debbugsSourcePackage} && exists ($currentPackage{$h{debbugsSourcePackage}})) {
-      if (!($h{debbugsTag} && $h{debbugsTag} =~ /(pending|wontfix|unreproducible|moreinfo)/)&& $h{debbugsState} =~ /open/) {
+#      if (!($h{debbugsTag} && $h{debbugsTag} =~ /(pending|wontfix|unreproducible|moreinfo)/)&& $h{debbugsState} =~ /open/) {
 
-	my $sourcepackage = $schema->resultset('Sourcepackage')->search({name => $h{debbugsSourcePackage}})->first;
-	my $severity = $schema->resultset('Severity')->find_or_create({name=>$h{debbugsSeverity}});
-	my $bug = $schema->resultset('Bug')->find_or_create({id => $h{debbugsID}});
+      my $sourcepackage = $schema->resultset('Sourcepackage')->search({name => $h{debbugsSourcePackage}})->first;
+      my $severity = $schema->resultset('Severity')->find_or_create({name=>$h{debbugsSeverity}});
+      my $bug = $schema->resultset('Bug')->find_or_create({id => $h{debbugsID}});
 
-	$bug->severity_id ($severity->id);
-	$bug->sourcepackage_id ($sourcepackage->id);
-	$bug->name ($h{debbugsTitle});
-	$bug->update;
+      $bug->severity_id ($severity->id);
+      $bug->sourcepackage_id ($sourcepackage->id);
+      $bug->name ($h{debbugsTitle});
+      $bug->update;
 
-	$keepThisBug{$h{debbugsID}} = 1;
+      $keepThisBug{$h{debbugsID}} = 1;
+
+      foreach (@{$h{debbugsMergedWith}}) {
+        $schema->resultset('Bug')->find_or_create({id => $_}); # In case the debbugsMergedWith referes to an old
+        # bug, I create an empty entry for it
+        my $bugmerged;
+        if ($_ > $h{debbugsID}) {
+          # bug1_id is the smallest bugnum 
+          $bugmerged = $schema->resultset('Mergedbug')->find_or_create(bug1_id => $h{debbugsID}, bug2_id => $_);
+        } else {
+          $bugmerged = $schema->resultset('Mergedbug')->find_or_create(bug1_id => $_, bug2_id => $h{debbugsID});
+        }
+        $keepThisBugmerged{$bugmerged->id} = 1;
       }
+      foreach (@{$h{debbugsMergedWith}}) {
+        my $bugtag = $schema->resultset('Bugtag')->find_or_create({name => $_});
+        my $bugbugtag = $schema->resultset('BugBugtag')->find_or_create(bug_id => $h{debbugsID}, bugtag_id => $bugtag->id);
+        $keepThisBugbugtag{$bugbugtag->id} = 1;
+      }
+
+#      }
     }
     %h = ();
   }
 
-  next unless $line =~ /^(debbugsID|debbugsSourcePackage|debbugsState|debbugsTitle|debbugsTag|debbugsSeverity):/;
+  next unless $line =~ /^(debbugsID|debbugsSourcePackage|debbugsState|debbugsTitle|debbugsTag|debbugsSeverity|debbugsMergedWith):/;
   chomp($line);
-  $h{$1} = $2 if ($line =~ /(\w+): (.+)$/);
+  if ($line =~ /(\w+): (.+)$/) {
+    my $key = $1;
+    my $val = $2;
+    if ($key =~  /(debbugsMergedWith|debbugsTag)/) {
+#      print $line."\n";
+      $h{$key} = [] if (!$h{$key});;
+      push @{$h{$key}}, $val;
+    } else {
+      $h{$key} = $val;
+    }
+  }
 }
 close BTS2LDAP;
-unlink $tmpfile;
+#unlink $tmpfile;
 my $bug_rs = $schema->resultset('Bug');
 while( my $bug = $bug_rs->next) {
   $bug->delete unless exists ($keepThisBug{$bug->id});
 }
+my $bugmerged_rs = $schema->resultset('Mergedbug');
+while( my $bugmerged = $bugmerged_rs->next) {
+  $bugmerged->delete unless exists ($keepThisBugmerged{$bugmerged->id});
+}
+my $bugbugtag_rs = $schema->resultset('BugBugtag');
+while( my $bugbugtag = $bugbugtag_rs->next) {
+  $bugbugtag->delete unless exists ($keepThisBugbugtag{$bugbugtag->id});
+}

Modified: svnbuildstat/trunk/script/svnbuildstat_update-db-slow.pl
===================================================================
--- svnbuildstat/trunk/script/svnbuildstat_update-db-slow.pl	2008-04-17 12:13:32 UTC (rev 793)
+++ svnbuildstat/trunk/script/svnbuildstat_update-db-slow.pl	2008-04-17 20:29:18 UTC (rev 794)
@@ -31,7 +31,9 @@
   });
 while (my $repositoryentry = $repositoryentry_rs->next) {
   checkRepositoryentryWatchfile(\$repositoryentry);
-
+  updateRepositoryentryDebianversion(\$repositoryentry);
+  $repositoryentry->lastwatchcheck('now');
+  $repositoryentry->update();
 }
 
 debug ("end (".(time - $begin)." secondes)");

Modified: svnbuildstat/trunk/script/svnbuildstat_update-db.pl
===================================================================
--- svnbuildstat/trunk/script/svnbuildstat_update-db.pl	2008-04-17 12:13:32 UTC (rev 793)
+++ svnbuildstat/trunk/script/svnbuildstat_update-db.pl	2008-04-17 20:29:18 UTC (rev 794)
@@ -35,7 +35,7 @@
   });
 
   my $newrev = $vcs->refresh();
-  print "newrev: $newrev\n";
+  debug ("new VCS revision: $newrev\n");
 
   if (!$newrev) {
     info ("Failed to refresh repository");
@@ -68,6 +68,7 @@
     next unless -f $dirname.'/rules';
     my $lastchange = DateTime->from_epoch(epoch => findLastChangeDate($dirname));
     # no change since the last check?
+    # I turn off this to for the fullscan of the repository
     return if $lastcheck && DateTime->compare($lastcheck, $lastchange);
 
     my $subdir = $dirname; # subdir is NOT NULL in DB so I need a last the '/'
@@ -112,7 +113,6 @@
     return;
   }
 
-  print $control->{packagesrc}."\n";
   my $sourcepackage = $schema->resultset('Sourcepackage')->find_or_create({name => $control->{packagesrc}});
   $$repositoryentry->sourcepackage_id($sourcepackage->id);
 #  $$repositoryentry->dscuri(undef);
@@ -261,7 +261,6 @@
     $$repositoryentry->update();
   }
 
-  # TODO uncomment
   $repository->rev($currentrev);
   $repository->lastcheck(DateTime::Format::Pg->format_timestamp_with_time_zone($begin));
   $repository->update();

Modified: svnbuildstat/trunk/svnbuildstat.ini
===================================================================
--- svnbuildstat/trunk/svnbuildstat.ini	2008-04-17 12:13:32 UTC (rev 793)
+++ svnbuildstat/trunk/svnbuildstat.ini	2008-04-17 20:29:18 UTC (rev 794)
@@ -1,7 +1,7 @@
 [db]
 dsn = DBI:Pg:host=127.0.0.1;dbname=svnbuildstat
 user = svnbuildstat
-password = XXXXXX 
+password = XXXXX
 
 [path]
 debmirror = http://ftp.fr.debian.org
@@ -10,3 +10,4 @@
 uploaddir = /home/goneri/cache/svnbuildstat/uploads
 vcscache = /home/goneri/cache/vcscache
 tarballdir = /home/goneri/cache/tarball
+webroot = /home/goneri/public_html/svnbuildstat




More information about the Collab-qa-commits mailing list