[Debian-l10n-commits] r1207 - in /ddtp-web: Packages2db.sh Packages2packages_tb.pl Packages2packages_tb.sh packages_tb2Packages.pl
grisu at users.alioth.debian.org
grisu at users.alioth.debian.org
Thu Aug 14 23:51:23 UTC 2008
Author: grisu
Date: Thu Aug 14 23:51:22 2008
New Revision: 1207
URL: http://svn.debian.org/wsvn/?sc=1&rev=1207
Log:
* Packages2packages_tb.sh
* packages_tb2Packages.pl
* Packages2packages_tb.pl
* write a general Package file to db and after this to a file
(speedup)
* Packages2db.sh
* use the general Package file
Added:
ddtp-web/Packages2packages_tb.pl (with props)
ddtp-web/Packages2packages_tb.sh (with props)
ddtp-web/packages_tb2Packages.pl (with props)
Modified:
ddtp-web/Packages2db.sh
Modified: ddtp-web/Packages2db.sh
URL: http://svn.debian.org/wsvn/ddtp-web/Packages2db.sh?rev=1207&op=diff
==============================================================================
--- ddtp-web/Packages2db.sh (original)
+++ ddtp-web/Packages2db.sh Thu Aug 14 23:51:22 2008
@@ -4,58 +4,41 @@
cd ~ddtp/Packages
-#DISTS="alpha amd64 arm hppa hurd-i386 i386 ia64 m68k mips mipsel powerpc s390 sparc"
-DISTS="alpha amd64 arm hppa i386 ia64 mips mipsel powerpc s390 sparc"
PART="main contrib"
distribution="etch"
for part in $PART
do
- for arch in $DISTS
- do
- file="Packages_${distribution}_${part}_${arch}"
+ file="Packages_${distribution}_${part}"
- echo `date`: ${distribution}/$arch
- [ -s $file.bz2 ] && mv $file.bz2 Packages.bz2
- wget -q -m -nd http://ftp.de.debian.org/debian/dists/${distribution}/${part}/binary-$arch/Packages.bz2
- mv Packages.bz2 $file.bz2
- echo `date`: Packages file downloaded
+ echo `date`: ${distribution}/${part}
bunzip2 -k -f $file.bz2
echo `date`: Packages bunzip2
- $INPORT2DB $file $distribution $arch
+ $INPORT2DB $file $distribution
echo `date`: data in db
rm -f $file
- done
done
# Regular vacuum to cut disk usage
psql ddtp -c "VACUUM"
-DISTS="alpha amd64 arm hppa i386 ia64 mips mipsel powerpc s390 sparc"
distribution="lenny"
for part in $PART
do
- for arch in $DISTS
- do
- file="Packages_${distribution}_${part}_${arch}"
+ file="Packages_${distribution}_${part}"
- echo `date`: ${distribution}/$arch
- [ -s $file.bz2 ] && mv $file.bz2 Packages.bz2
- wget -q -m -nd http://ftp.de.debian.org/debian/dists/${distribution}/${part}/binary-$arch/Packages.bz2
- mv Packages.bz2 $file.bz2
- echo `date`: Packages file downloaded
+ echo `date`: ${distribution}/${part}
bunzip2 -k -f $file.bz2
echo `date`: Packages bunzip2
- $INPORT2DB $file $distribution $arch
+ $INPORT2DB $file $distribution
echo `date`: data in db
rm -f $file
- done
done
# Regular vacuum to cut disk usage
psql ddtp -c "VACUUM"
@@ -64,29 +47,21 @@
psql ddtp -c "TRUNCATE active_tb"
psql ddtp -c "TRUNCATE part_description_tb"
-DISTS="alpha amd64 arm hppa i386 ia64 m68k mips mipsel powerpc s390 sparc"
distribution="sid"
for part in $PART
do
- for arch in $DISTS
- do
- file="Packages_${distribution}_${part}_${arch}"
+ file="Packages_${distribution}_${part}"
- echo `date`: ${distribution}/$arch
- [ -s $file.bz2 ] && mv $file.bz2 Packages.bz2
- wget -q -m -nd http://ftp.de.debian.org/debian/dists/${distribution}/${part}/binary-$arch/Packages.bz2
- mv Packages.bz2 $file.bz2
- echo `date`: Packages file downloaded
+ echo `date`: ${distribution}/${part}
bunzip2 -k -f $file.bz2
echo `date`: Packages bunzip2
- $INPORT2DB $file $distribution $arch
+ $INPORT2DB $file $distribution
echo `date`: data in db
rm -f $file
- done
done
# Regular vacuum to cut disk usage
psql ddtp -c "VACUUM"
@@ -97,5 +72,3 @@
chmod 644 pg_dump/pg_ddts.dump.gz www/source/pg_ddts_current.dump.gz
-# rotate the pg-dump
-/usr/sbin/logrotate --state /org/ddtp.debian.net/lib/logrotate.state /org/ddtp.debian.net/logrotate.config
Added: ddtp-web/Packages2packages_tb.pl
URL: http://svn.debian.org/wsvn/ddtp-web/Packages2packages_tb.pl?rev=1207&op=file
==============================================================================
--- ddtp-web/Packages2packages_tb.pl (added)
+++ ddtp-web/Packages2packages_tb.pl Thu Aug 14 23:51:22 2008
@@ -1,0 +1,115 @@
+#!/usr/bin/perl -w
+use strict;
+use LWP::Simple;
+use POSIX qw(strftime);
+use Digest::MD5 qw(md5_hex);
+use DBI;
+my @DSN = ("DBI:Pg:dbname=ddtp", "", "");
+
+my $dbh = DBI->connect(@DSN,
+ { PrintError => 0,
+ RaiseError => 1,
+ AutoCommit => 0,
+ });
+
+die $DBI::errstr unless $dbh;
+
+my $data = "/org/ddtp.debian.net/Packages/";
+
+my $DIST = shift || "etch";
+my $SECTION = shift || "main";
+my $ARCH = shift || "i386";
+
+my %descrmd5; # $descrmd5{$md5} = $desc_id, represents all known descriptions
+my %descrlist; # $descrlist{$package}{$md5} exists for each package in package file
+ # $descrlist{$package}{priority} = package priority
+my %total_counts; # $total_counts{$priority} = number of packages with that priority
+my %important_packages; # $important_packages{$package}{$md5} exists for packages+description of priority standard or higher
+
+load_packages(); # Read packages file
+exit;
+
+sub load_packages
+{
+ print STDERR "Loading package file Packages_${DIST}_${SECTION}_${ARCH}.bz2";
+ my $fh = open_bz2_file( "$data/Packages_${DIST}_${SECTION}_${ARCH}.bz2" );
+ parse_header_format( $fh, \&process_package );
+ close $fh;
+ my $sth = $dbh->prepare("SELECT count(packages_id) FROM packages_tb");
+ $sth->execute;
+ my ($counter) = $sth->fetchrow_array;
+ print STDERR " -> $counter packages in packages_tb\n";
+}
+
+# Helper for load_packages
+sub process_package
+{
+ my $hash = shift;
+ my $md5 = md5_hex( $hash->{Description}."\n" );
+# print "$hash->{Package} : $md5\n";
+ if (not $hash->{Source}) {
+ $hash->{Source} = $hash->{Package};
+ }
+
+ eval {
+ $dbh->do("INSERT INTO packages_tb (package,source,version,tag,priority,maintainer,task,section,description,description_md5)
+ VALUES (?,?,?,?,?,?,?,?,?,?);", undef,
+ $hash->{Package},
+ $hash->{Source},
+ $hash->{Version},
+ $hash->{Tag},
+ $hash->{Priority},
+ $hash->{Maintainer},
+ $hash->{Task},
+ $hash->{Section},
+ $hash->{Description},
+ $md5
+ );
+ $dbh->commit; # commit the changes if we get this far
+ };
+ if ($@) {
+ $dbh->rollback; # undo the incomplete changes
+ }
+}
+
+sub open_bz2_file
+{
+ my $file = shift;
+
+ my $fh;
+
+ open $fh, "bzcat $file |" or die "Couldn't open $file ($!)\n";
+
+ return $fh;
+}
+
+sub parse_header_format
+{
+ my $fh = shift;
+ my $sub = shift;
+
+ my $lastfield = undef;
+ my %hash;
+ while(<$fh>)
+ {
+ chomp;
+ if( /^([\w.-]+): (.*)/ )
+ {
+ $lastfield = $1;
+ $hash{$1} = $2;
+ }
+ elsif( /^( .*)/ )
+ {
+ $hash{$lastfield} .= "\n$_";
+ }
+ elsif( /^$/ )
+ {
+ $sub->( \%hash );
+ %hash = ();
+ $lastfield = undef;
+ }
+ }
+}
+
+
+
Propchange: ddtp-web/Packages2packages_tb.pl
------------------------------------------------------------------------------
svn:executable = *
Added: ddtp-web/Packages2packages_tb.sh
URL: http://svn.debian.org/wsvn/ddtp-web/Packages2packages_tb.sh?rev=1207&op=file
==============================================================================
--- ddtp-web/Packages2packages_tb.sh (added)
+++ ddtp-web/Packages2packages_tb.sh Thu Aug 14 23:51:22 2008
@@ -1,0 +1,50 @@
+#!/bin/bash -e
+
+INPORT2DB="./Packages2packages_tb.pl"
+DB2FILE="./packages_tb2Packages.pl"
+
+cd ~ddtp
+
+DISTS="alpha amd64 arm armel hppa hurd-i386 i386 ia64 m68k mips mipsel powerpc s390 sparc"
+#DISTS="i386"
+PART="main contrib"
+DISTRIBUTION="sarge etch lenny sid"
+DISTRIBUTION="etch lenny sid"
+
+for distribution in $DISTRIBUTION
+do
+ for part in $PART
+ do
+ for arch in $DISTS
+ do
+ file="Packages/Packages_${distribution}_${part}_${arch}"
+
+ echo `date`: ${distribution}/${part}/$arch
+ [ -s $file.bz2 ] && mv $file.bz2 Packages/Packages.bz2
+ wget -P Packages -q -m -nd http://ftp.de.debian.org/debian/dists/${distribution}/${part}/binary-$arch/Packages.bz2
+ [ -s Packages/Packages.bz2 ] && mv Packages/Packages.bz2 $file.bz2
+ echo `date`: Packages file downloaded
+ done
+ done
+done
+
+for distribution in $DISTRIBUTION
+do
+ for part in $PART
+ do
+ # Clear active before loading sid (which is what counts as active)
+ psql ddtp -c "TRUNCATE packages_tb"
+
+ for arch in $DISTS
+ do
+ [ -f Packages/Packages_${distribution}_${part}_${arch}.bz2 ] && echo -n `date` :
+ [ -f Packages/Packages_${distribution}_${part}_${arch}.bz2 ] && $INPORT2DB $distribution $part $arch
+ [ -f Packages/Packages_${distribution}_${part}_${arch}.bz2 ] || echo no Packages_${distribution}_${part}_${arch}.bz2
+ done
+
+ $DB2FILE $distribution $part
+ done
+done
+# Regular vacuum to cut disk usage
+psql ddtp -c "VACUUM"
+
Propchange: ddtp-web/Packages2packages_tb.sh
------------------------------------------------------------------------------
svn:executable = *
Added: ddtp-web/packages_tb2Packages.pl
URL: http://svn.debian.org/wsvn/ddtp-web/packages_tb2Packages.pl?rev=1207&op=file
==============================================================================
--- ddtp-web/packages_tb2Packages.pl (added)
+++ ddtp-web/packages_tb2Packages.pl Thu Aug 14 23:51:22 2008
@@ -1,0 +1,96 @@
+#!/usr/bin/perl -w
+use strict;
+use LWP::Simple;
+use POSIX qw(strftime);
+use Digest::MD5 qw(md5_hex);
+use DBI;
+my @DSN = ("DBI:Pg:dbname=ddtp", "", "");
+
+my $dbh = DBI->connect(@DSN,
+ { PrintError => 0,
+ RaiseError => 1,
+ AutoCommit => 0,
+ });
+
+die $DBI::errstr unless $dbh;
+
+my $data = "/org/ddtp.debian.net/Packages/";
+
+my $DIST = shift || "etch";
+my $SECTION = shift || "main";
+
+my %descrmd5; # $descrmd5{$md5} = $desc_id, represents all known descriptions
+my %descrlist; # $descrlist{$package}{$md5} exists for each package in package file
+ # $descrlist{$package}{priority} = package priority
+my %total_counts; # $total_counts{$priority} = number of packages with that priority
+my %important_packages; # $important_packages{$package}{$md5} exists for packages+description of priority standard or higher
+
+export_packages(); # Read packages file
+exit;
+
+sub export_packages
+{
+ my ($package,$source,$version,$tag,$priority,$maintainer,$task,$section,$description);
+
+ print STDERR "Export package file Packages_${DIST}_${SECTION}.bz2\n";
+ my $fh = open_bz2_file( "$data/Packages_${DIST}_${SECTION}.bz2" );
+
+ my $sth = $dbh->prepare("SELECT package,source,version,tag,priority,maintainer,task,section,description FROM packages_tb ORDER BY package");
+ $sth->execute;
+ while ( ($package,$source,$version,$tag,$priority,$maintainer,$task,$section,$description) = $sth->fetchrow_array ) {
+ print $fh "Package: $package\n";
+ print $fh "Source: $source\n";
+ print $fh "Version: $version\n";
+ print $fh "Tag: $tag\n" if $tag;
+ print $fh "Priority: $priority\n";
+ print $fh "Maintainer: $maintainer\n";
+ print $fh "Task: $task\n" if $task;
+ print $fh "Section: $section\n";
+ print $fh "Description: $description\n";
+ print $fh "\n";
+ }
+ close $fh;
+}
+
+# Helper for load_packages
+sub open_bz2_file
+{
+ my $file = shift;
+
+ my $fh;
+
+ open $fh, "| bzip2 > $file " or die "Couldn't open $file ($!)\n";
+
+ return $fh;
+}
+
+sub parse_header_format
+{
+ my $fh = shift;
+ my $sub = shift;
+
+ my $lastfield = undef;
+ my %hash;
+ while(<$fh>)
+ {
+ chomp;
+ if( /^([\w.-]+): (.*)/ )
+ {
+ $lastfield = $1;
+ $hash{$1} = $2;
+ }
+ elsif( /^( .*)/ )
+ {
+ $hash{$lastfield} .= "\n$_";
+ }
+ elsif( /^$/ )
+ {
+ $sub->( \%hash );
+ %hash = ();
+ $lastfield = undef;
+ }
+ }
+}
+
+
+
Propchange: ddtp-web/packages_tb2Packages.pl
------------------------------------------------------------------------------
svn:executable = *
More information about the Debian-l10n-commits
mailing list