r3040 - people/goneri/SvnBuildStat-WWW/script

[Gonéri Le Bouder]

Author: goneri-guest
Date: 2007-06-20 22:16:28 +0000 (Wed, 20 Jun 2007)
New Revision: 3040

Modified:
   people/goneri/SvnBuildStat-WWW/script/svnbuildstat_update-db.pl
Log:
-use the repository->rev to store the last revision of the repository to avoid useless refresh of an unchanged directory
-remove the majority of the debug print
-increment package->ismissin every time the package is mising, I'll remove them with a purge script
-fix the use of Thread::Pool::Simple


Modified: people/goneri/SvnBuildStat-WWW/script/svnbuildstat_update-db.pl
===================================================================
--- people/goneri/SvnBuildStat-WWW/script/svnbuildstat_update-db.pl	2007-06-20 21:36:06 UTC (rev 3039)
+++ people/goneri/SvnBuildStat-WWW/script/svnbuildstat_update-db.pl	2007-06-20 22:16:28 UTC (rev 3040)
@@ -22,7 +22,6 @@
 use SvnBuildStat::Schema;
 use SvnBuildStat::Config;
 
-my $maxThreadPerPool=10;
 
 my $config;
 my $schema;
@@ -30,6 +29,16 @@
 #my $poolImportPkg;
 my $poolImportRepository;
 
+sub getRev {
+  my $uri = shift;
+
+  foreach (`LC_ALL=C svn info $uri`) {
+    return $1 if /Last Changed Rev:\ (\d+)/;
+  }
+
+  return;
+}
+
 sub mkTarballFromPackage {
   my $package = shift;
 
@@ -61,7 +70,7 @@
   my $tarballuri = $$repository->tarballlayout;
   $tarballuri =~ s/\@TARBALL@/$tarball/;
   $tarballuri =~ s/\@PACKAGE@/$packagename/;
-  print  "\n--->".$tarballuri."\n";
+#  print  "\n--->".$tarballuri."\n";
 
   return $tarballuri;
 }
@@ -79,7 +88,7 @@
   $ua->agent("SvnBuildStat/0.1 ");
 
   if ($$package->svndebrelease !~ /-[\d\.]+$/) {
-    print $$package->name."\n";
+#    print $$package->name."\n";
     $isnative = 't';
   } else {
     my $debmirror = 'http://ftp.debian.org/debian';
@@ -91,7 +100,7 @@
 	my $tmp = "$1/".$$package->name if $$package->name =~ /^(lib.|.)/;
 	my $debdiffuri = $debmirror.'/pool/'.$section.'/'.$tmp.'/'.$debdiff;
 	my $tmp_tarballuri = $debmirror.'/pool/'.$section.'/'.$tmp.'/'.$tarball;
-	print $tmp_tarballuri."\n";
+#	print $tmp_tarballuri."\n";
 	if (testUrl($debdiffuri)) {
 	  $isindebian = 't';
 	}
@@ -102,7 +111,7 @@
       }
     }
   }
-  print "tarballuri -> ".$tarballuri."\n";
+#  print "tarballuri -> ".$tarballuri."\n";
   return { tarballuri => $tarballuri, isindebian => $isindebian, isnative => $isnative };
 }
 
@@ -124,7 +133,7 @@
   $majorrelease =~ s/-[0-9A-Za-z\.~]*$//;
   $majorrelease =~ s/dfsg.*//;
   $cmd = "uscan --package ".$$package->name." --dehs --upstream-version ".$majorrelease." --watchfile ".$watchfile;
-  print $cmd."\n";
+#  print $cmd."\n";
   my @uscan = `$cmd`;
   unlink $watchfile or warn;
   return unless @uscan > 2; # empty output
@@ -133,7 +142,7 @@
   my $isuptodate = 't';
   my $upstreamrelease;
   foreach (@uscan) {
-    print;
+#    print;
     $tarballuri = $1 if (/^<upstream-url>(.+)<\/upstream-url>$/);
     if (/^<status>/) {
       $isuptodate = 'f' unless (/^<status>up to date<\/status>$/);
@@ -147,18 +156,15 @@
 
 
 sub importPkg {
-  print "ImportPkg\n";
+#  print "ImportPkg\n";
   my( $repository, $uri, $tarballonrepository) = @_;
 
   my @maintainer;
   my $packagesrc;
   my $svndebrelease;
   my $tarballuri;
-  my $rev;
+  my $rev = getRev($uri);
 
-  foreach (`LC_ALL=C svn info $uri`) {
-    $rev = $1 if /Last Changed Rev:\ (\d+)/;
-  }
   return unless $rev;
 
   my @control = `svn cat $uri/debian/control`;
@@ -205,7 +211,7 @@
   $package->update_from_related('repository_id',$$repository);
   $package->svndebrelease ($svndebrelease);
   $package->uri($uri);
-  $package->rev($rev) if $rev;
+  $package->rev($rev);
   $package->repository_id($$repository);
   foreach my $maintainer (@maintainer) {
     my $package_maintainer =
@@ -252,6 +258,7 @@
   $package->upstreamrelease($uscandata->{upstreamrelease});
   $package->isindebian($ondebiandata->{isindebian});
   $package->update();
+  return $package->id;
 }
 
 
@@ -266,9 +273,20 @@
 sub importRepository {
   my $repository = shift;
 
+  my %present_packages;
+
   my $tarballonrepository;
   print "Repository: ".$$repository->name."\n";
   my $t = 'svn ls -R '.$$repository->uri;
+  my $rev = getRev($$repository->uri);
+  return unless $rev;
+  if ($$repository->rev eq $rev && $$repository->lastcheck) {
+    # If he repository is up to date, a still do a refresh every 48h
+    # this because the tarball check
+    my ($year, $mon, $day, $hour, $min, $sec) = ($$repository->lastcheck =~ /(\d{4})-(\d{2})-(\d{2})\ (\d{1,2}):(\d{2}):(\d{2})/);
+    my $lastbuild = timelocal($sec, $min, $hour, $day, $mon, $year);
+    return if $lastbuild > time - 3600*48;
+  }
 
   my @uri;
 
@@ -286,10 +304,28 @@
 
   foreach my $uri (@uri) {
     # look for packages
-    print "->".$uri."\n";
+#    print "->".$uri."\n";
     #my $pid = $poolImportPkg->add($repository,$uri,$tarballonrepository) or die "Fucked!\n";
-    importPkg($repository,$uri,$tarballonrepository);
+    my $package_id = importPkg($repository,$uri,$tarballonrepository);
+    $present_packages{$package_id} = 1;
   }
+
+
+  my $package_rs = $schema->resultset('Package')->search({repository_id => $$repository->id});
+
+  while (my $package = $package_rs->next) {
+    if (!exists ($present_packages{$package->id})) {
+      $package->ismissing($package->ismissing + 1);
+    } else {
+      $package->ismissing(0);
+    }
+      $package->update();
+  }
+
+  $repository->rev($rev);
+  $repository->lastcheck('now');
+  $repository->update();
+
   print "end import Repo\n";
 }
 
@@ -299,15 +335,18 @@
 #   do => [\&importPkg],
 # );
 $poolImportRepository = Thread::Pool::Simple->new(
-   max => $maxThreadPerPool,
+   min => 5,
+   max => 7,
+   load => 15,
    do => [\&importRepository],
+   lifespan => 1
  );
 ########
 
 # Import packages
 my $repository_rs = $schema->resultset('Repository')->search({enabled => 'true'});
 while (my $repository = $repository_rs->next) {
-  # importRepository(\$repository);
   $poolImportRepository->add(\$repository) or die "Fucked\n";
+  sleep 60;
 }#
 $poolImportRepository->join;