[Debian-l10n-commits] [SCM] DDTP - Debian Descriptions Translation Project branch, master, updated. 20080812.0-117-gd16e661
Martijn van Oosterhout
kleptog at svana.org
Sun Jun 17 14:01:06 UTC 2012
The following commit has been merged in the master branch:
commit d16e661cc340224930cb86a458cad48f147ec684
Author: Martijn van Oosterhout <kleptog at svana.org>
Date: Sun Jun 17 15:59:46 2012 +0200
Add more files to bring git up-to-date with churro.
Not sure how these got missed the first time around.
diff --git a/Packages2db.pl b/Packages2db.pl
index 3062619..5ea6a12 100755
--- a/Packages2db.pl
+++ b/Packages2db.pl
@@ -5,6 +5,7 @@ use strict;
my $packagefile= shift(@ARGV);
my $distribution= shift(@ARGV);
+my $description_en_file= shift(@ARGV); # Optional
my $description_id;
my $description_tag_id;
@@ -52,6 +53,7 @@ sub desc_to_parts ($) {
sub scan_packages_file {
my $filename= shift(@_);
my $distribution= shift(@_);
+ my $description_en= shift(@_);
my $package;
my $prioritize;
@@ -61,6 +63,8 @@ sub scan_packages_file {
my $source;
my $priority;
my $section;
+ my @task;
+ my @tag;
my $version;
sub get_old_description_id {
@@ -68,7 +72,10 @@ sub scan_packages_file {
my $old_description_id;
- my $sth = $dbh->prepare("SELECT description_id FROM description_tb WHERE package=?");
+ #my $sth = $dbh->prepare("SELECT description_id FROM description_tb WHERE package=?");
+ #my $sth = $dbh->prepare("SELECT description_id from package_version_tb where package=?");
+ # better without LIMIT ?
+ my $sth = $dbh->prepare("SELECT description_id from package_version_tb where package=? order by description_id DESC LIMIT 1");
$sth->execute($package);
($old_description_id) = $sth->fetchrow_array;
return $old_description_id;
@@ -125,6 +132,7 @@ sub scan_packages_file {
my $description_id= shift(@_);
my $version= shift(@_);
my $package= shift(@_);
+ my $source= shift(@_);
my $package_version_id;
@@ -134,7 +142,7 @@ sub scan_packages_file {
if (not $package_version_id) {
eval {
- $dbh->do("INSERT INTO package_version_tb (description_id,package,version) VALUES (?,?,?);", undef, $description_id, $package, $version);
+ $dbh->do("INSERT INTO package_version_tb (description_id,package,version,source) VALUES (?,?,?,?);", undef, $description_id, $package, $version, $source);
$dbh->commit; # commit the changes if we get this far
};
if ($@) {
@@ -142,20 +150,97 @@ sub scan_packages_file {
$dbh->rollback; # undo the incomplete changes
}
}
+ }
+
+ sub save_milestone_to_db {
+ my $description_id= shift(@_);
+ my $priority= shift(@_);
+ my $section= shift(@_);
+ my @task= @_;
+
+ my $sth = $dbh->prepare("DELETE FROM description_milestone_tb WHERE description_id=? and
+ not milestone like 'rtrn:%' and
+ not milestone like 'popc:%' and
+ not milestone like 'part:%'");
+ $sth->execute($description_id);
- my $version_id;
+ eval {
+ $dbh->do("INSERT INTO description_milestone_tb (description_id,milestone) VALUES (?,?);", undef, $description_id, "prio:$priority");
+ $dbh->commit; # commit the changes if we get this far
+ };
+ if ($@) {
+ warn "Packages2db.pl: failed to INSERT Package '$description_id', milestone 'prio:$priority' into description_milestone_tb: $@\n";
+ $dbh->rollback; # undo the incomplete changes
+ }
- $sth = $dbh->prepare("SELECT version_id FROM version_tb WHERE description_id=? and version=?");
- $sth->execute($description_id, $version);
- ($version_id) = $sth->fetchrow_array;
+ eval {
+ $dbh->do("INSERT INTO description_milestone_tb (description_id,milestone) VALUES (?,?);", undef, $description_id, "sect:$section");
+ $dbh->commit; # commit the changes if we get this far
+ };
+ if ($@) {
+ warn "Packages2db.pl: failed to INSERT Package '$description_id', milestone 'sect:$section' into description_milestone_tb: $@\n";
+ $dbh->rollback; # undo the incomplete changes
+ }
- if (not $version_id) {
+ foreach (@task) {
eval {
- $dbh->do("INSERT INTO version_tb (description_id,version) VALUES (?,?);", undef, $description_id, $version);
+ $dbh->do("INSERT INTO description_milestone_tb (description_id,milestone) VALUES (?,?);", undef, $description_id, "task:$_");
$dbh->commit; # commit the changes if we get this far
};
if ($@) {
- warn "Packages2db.pl: failed to INSERT description_id '$description_id', version '$version' into version_tb: $@\n";
+ warn "Packages2db.pl: failed to INSERT Package '$description_id', milestone 'task:$_' into description_milestone_tb: $@\n";
+ $dbh->rollback; # undo the incomplete changes
+ }
+ }
+ }
+
+ sub save_tag_milestone_to_db {
+ my $description_id= shift(@_);
+ my @tags= @_;
+
+ my @splittags;
+
+ foreach my $in (@tags) {
+ #print " $description_id task: $in\n";
+ if ($in =~ m/^([^{]*)\{([^}]*)\}(.*)$/) {
+ my $prefix = $1;
+ my $postfix = $3;
+ my @list = split /,/,$2;
+ foreach my $o (@list) {
+ push @splittags, $prefix.$o.$postfix;
+ }
+ } else {
+ push @splittags, $in;
+ }
+ }
+
+ #print "$description_id task: " . join(" ", @splittags) . "\n";
+
+ foreach (@splittags) {
+ eval {
+ $dbh->do("INSERT INTO description_milestone_tb (description_id,milestone) VALUES (?,?);", undef, $description_id, "tags:$_");
+ $dbh->commit; # commit the changes if we get this far
+ };
+ if ($@) {
+ warn "Packages2db.pl: failed to INSERT Package '$description_id', milestone 'tags:$_' into description_milestone_tb: $@\n";
+ $dbh->rollback; # undo the incomplete changes
+ }
+ }
+ }
+
+ sub save_oldtranslang_milestone_to_db {
+ my $description_id= shift(@_);
+ my @oldtranslangs = @_;
+
+ print " save_oldtranslang_milestone_to_db: $description_id task: " . join(" ", @oldtranslangs) . " #:$#oldtranslangs\n";
+
+ foreach (@oldtranslangs) {
+ eval {
+ $dbh->do("INSERT INTO description_milestone_tb (description_id,milestone) VALUES (?,?);", undef, $description_id, "rtrn:$_");
+ $dbh->commit; # commit the changes if we get this far
+ };
+ if ($@) {
+ warn "Packages2db.pl: failed to INSERT Package '$description_id', milestone 'oldtranslangs:$_' into description_milestone_tb: $@\n";
$dbh->rollback; # undo the incomplete changes
}
}
@@ -176,9 +261,11 @@ sub scan_packages_file {
}
open (PACKAGES, "$filename") or die "open packagefile failed";
+ my $in_descr = 0;
while (<PACKAGES>) {
if ($_=~/^$/) {
my $description_orig=$description;
+ my @oldtranslang;
eval {
$description_id=get_description_id($description_orig);
if ($description_id) {
@@ -197,12 +284,21 @@ sub scan_packages_file {
my $old_description_id=get_old_description_id($package);
if ($old_description_id) {
print " changed description from $package ($source)\n" ;
+ # search for translations of the old description:
+ # SELECT language FROM translation_tb where description_id=
+ my $lang;
+ my $sth = $dbh->prepare("SELECT language FROM translation_tb where description_id=?");
+ $sth->execute($old_description_id);
+ while(($lang) = $sth->fetchrow_array) {
+ #print " old description was translated in $lang\n" ;
+ push @oldtranslang,$lang;
+ }
}
my $md5=md5_hex($description_orig);
$dbh->do("INSERT INTO description_tb (description_md5, description, package, source, prioritize) VALUES (?,?,?,?,?);", undef, $md5,$description,$package,$source,$prioritize);
$description_id=get_description_id($description_orig);
$dbh->do("INSERT INTO description_tag_tb (description_id, tag, date_begin, date_end) VALUES (?,?,CURRENT_DATE,CURRENT_DATE);", undef, $description_id,$distribution);
- print " add new description from $package ($source) with $prioritize\n" ;
+ print " add new description from $package ($source) with prio $prioritize\n" ;
}
$dbh->commit; # commit the changes if we get this far
};
@@ -211,7 +307,14 @@ sub scan_packages_file {
$dbh->rollback; # undo the incomplete changes
}
if (($description_id)) {
- save_version_to_db($description_id,$version,$package);
+ save_version_to_db($description_id,$version,$package,$source);
+ }
+ if (($description_id)) {
+ save_milestone_to_db($description_id,$priority, $section, @task);
+ save_tag_milestone_to_db($description_id, at tag);
+ if ($#oldtranslang>=0) {
+ save_oldtranslang_milestone_to_db($description_id, at oldtranslang);
+ }
}
if (($description_id) and ($distribution eq 'sid')) {
if (! is_description_id_active($description_id)) {
@@ -255,6 +358,7 @@ sub scan_packages_file {
$version=$1;
}
if (/^Tag: (.+)/) { # new item
+ @tag=split(',? +',$1);
$tag=$1;
$prioritize += 1;
$prioritize += 2 if $tag =~ /role[^ ]+program/i;
@@ -263,6 +367,9 @@ sub scan_packages_file {
$prioritize -= 2 if $tag =~ /role[^ ]+source/i;
$prioritize -= 1 if $tag =~ /role[^ ]+shared-lib/i;
$prioritize -= 1 if $tag =~ /role[^ ]+data/i;
+ # In newer packages files this can be split over multiple lines
+ # This is a hack to prevent these lines being added to the description
+ $in_descr = 0;
}
if (/^Priority: (\w+)/) { # new item
$priority=$1;
@@ -287,7 +394,7 @@ sub scan_packages_file {
if (/^Maintainer: (.*)/) { # new item
}
if (/^Task: (.*)/) { # new item
- $priority="task";
+ @task=split('[, ]+',$1);
$prioritize+=2;
}
if (/^Section: (\w+)/) { # new item
@@ -304,15 +411,75 @@ sub scan_packages_file {
}
if (/^Description: (.*)/) { # new item
$description=$1 . "\n";
+ # Following lines are part of this
+ $in_descr = 1;
}
- if (/^ /) {
+ if (/^ / and $in_descr) {
+ # in_descr is because tag can also be multiple lines
$description.=$_;
}
+ if (/^Description-md5: (.*)/) { # new item
+ my $md5 = $1;
+ if (defined $description_en->{$md5}) {
+ $description = $description_en->{$md5}{'Description-en'} . "\n";
+ }
+ }
}
close PACKAGES or die "packagefile failed";
}
+sub parse_header_format
+{
+ my $fh = shift;
+ my $sub = shift;
+
+ my $lastfield = undef;
+ my $hash;
+ while(<$fh>)
+ {
+ chomp;
+ if( /^([\w.-]+): (.*)/ )
+ {
+ $lastfield = $1;
+ $hash->{$1} = $2;
+ }
+ elsif( /^( .*)/ )
+ {
+ $hash->{$lastfield} .= "\n$_";
+ }
+ elsif( /^$/ )
+ {
+ $sub->( $hash );
+ $hash = {};
+ $lastfield = undef;
+ }
+ }
+}
+
+# Loads the english translation file, indexed by md5
+sub load_english_translations
+{
+ my $filename = shift;
+ my $description_en = {};
+
+ my $process_descr = sub {
+ my $hash = shift;
+ if( not exists $hash->{'Description-en'} ) {
+ die "Bad description $hash->{'Description-md5'}";
+ }
+ $description_en->{$hash->{'Description-md5'}} = $hash;
+ };
+ open my $fh, "<", "$filename" or die "open description_en file $filename failed";
+ parse_header_format($fh, $process_descr);
+ return $description_en;
+}
+
+my $description_en = {};
+if ( $description_en_file ) {
+ $description_en = load_english_translations($description_en_file);
+}
+
if ( -r $packagefile ) {
- scan_packages_file($packagefile,$distribution)
+ scan_packages_file($packagefile,$distribution,$description_en)
}
diff --git a/Packages2db.sh b/Packages2db.sh
index 63755a0..1fe574c 100755
--- a/Packages2db.sh
+++ b/Packages2db.sh
@@ -11,17 +11,23 @@ load_distribution ()
for part in $parts
do
file="Packages_${distribution}_${part}"
+ transfile="Translation-en_${distribution}_${part}"
echo `date`: ${distribution}/${part}
+ if [ -r "$transfile.bz2" ] ; then
+ bunzip2 -k -f $transfile.bz2
+ else
+ transfile=""
+ fi
bunzip2 -k -f $file.bz2
echo `date`: Packages bunzip2
# [ "$distribution" = "lliurex" ] && distribution="gaia"
- $INPORT2DB $file $distribution
+ $INPORT2DB $file $distribution $transfile
echo `date`: data in db
- rm -f $file
+ rm -f $file $transfile
done
}
diff --git a/Packages2packages_tb.pl b/Packages2packages_tb.pl
index 274c489..a06e542 100755
--- a/Packages2packages_tb.pl
+++ b/Packages2packages_tb.pl
@@ -48,7 +48,7 @@ sub process_package
$hash->{Task},
$hash->{Section},
$hash->{Description},
- "0" # This is set as NOT NULL, but should be removed from the sql layout.
+ $hash->{"Description-md5"} || "0",
);
$dbh->commit; # commit the changes if we get this far
};
diff --git a/Packages2packages_tb.sh b/Packages2packages_tb.sh
index cbe5f49..00f03ea 100755
--- a/Packages2packages_tb.sh
+++ b/Packages2packages_tb.sh
@@ -13,6 +13,19 @@ for distribution in $DISTRIBUTION
do
for part in $PART
do
+ # Download Translation-en file if present
+ file="Packages/Translation-en_${distribution}_${part}"
+ echo `date`: Translation-en ${distribution}/${part}
+
+ [ -s $file.bz2 ] && mv $file.bz2 Packages/Translation_en.bz2
+ wget -P Packages -q -m -nd \
+ http://ftp.de.debian.org/debian/dists/${distribution}/${part}/i18n/Translation-en.bz2 && {
+ echo `date`: Translation-en file downloaded
+ } || {
+ echo `date`: Failed to download Translation-en ${distribution}/${part} 1>&2
+ }
+ [ -s Packages/Translation-en.bz2 ] && mv Packages/Translation-en.bz2 $file.bz2
+
for arch in $DISTS
do
file="Packages/Packages_${distribution}_${part}_${arch}"
diff --git a/build_user_milestones.pl b/build_user_milestones.pl
new file mode 100755
index 0000000..0f29140
--- /dev/null
+++ b/build_user_milestones.pl
@@ -0,0 +1,73 @@
+#!/usr/bin/perl -w
+use strict;
+use LWP::Simple;
+use POSIX qw(strftime);
+use DBI;
+my @DSN = ("DBI:Pg:dbname=ddtp", "", "");
+
+my $dbh = DBI->connect(@DSN,
+ { PrintError => 0,
+ RaiseError => 1,
+ AutoCommit => 0,
+ });
+
+die $DBI::errstr unless $dbh;
+
+my $sth;
+
+# build user milestones
+
+$sth = $dbh->prepare("DELETE FROM description_milestone_tb WHERE milestone like 'user:%' or milestone like 'lang:%'");
+$sth->execute();
+
+$sth = $dbh->prepare("INSERT INTO description_milestone_tb (description_id,milestone) VALUES ('999','user:jjjj')");
+$sth->execute();
+
+$sth = $dbh->prepare("SELECT collection,name,nametype FROM collection_milestone_tb");
+$sth->execute();
+while(my ($collection,$name,$nametype) = $sth->fetchrow_array) {
+ my ($collectiontype,$collectionname) = ($collection =~ m/([^:]+):(.*)/);
+ #print " name : $name\n";
+ #print " nametype: $nametype\n";
+ #print "collectiontype: $collectiontype\n";
+ #print "collectionname: $collectionname\n";
+
+ my $milestone;
+ my $sql;
+
+ if ($nametype == 1) {
+ $milestone="user:$name";
+ } else {
+ $milestone="lang:$name";
+ }
+
+ if ($collectiontype eq "mile") {
+ print "-->found mile -> $milestone\n";
+ $sql="INSERT INTO description_milestone_tb (description_id,milestone) SELECT DISTINCT description_id,'$milestone' FROM description_milestone_tb WHERE milestone='$collectionname' and description_id not in (SELECT DISTINCT description_id FROM description_milestone_tb WHERE milestone='$milestone')";
+ } elsif ( $collectiontype eq "pkg" ) {
+ print "-->found pkg -> $milestone\n";
+ $sql="INSERT INTO description_milestone_tb (description_id,milestone) SELECT DISTINCT description_id,'$milestone' FROM package_version_tb WHERE package='$collectionname' and description_id not in (SELECT DISTINCT description_id FROM description_milestone_tb WHERE milestone='$milestone')";
+ } elsif ( $collectiontype eq "source" ) {
+ print "-->found source -> $milestone\n";
+ $sql="INSERT INTO description_milestone_tb (description_id,milestone) SELECT DISTINCT description_id,'$milestone' FROM package_version_tb WHERE package in (SELECT DISTINCT package FROM description_tb WHERE source='$collectionname') and description_id not in (SELECT DISTINCT description_id FROM description_milestone_tb WHERE milestone='$milestone')";
+ } elsif ( $collectiontype eq "d_id" ) {
+ print "-->found d_id -> $milestone\n";
+ $sql="INSERT INTO description_milestone_tb (description_id,milestone) SELECT DISTINCT description_id,'$milestone' FROM description_tb WHERE description_id='$collectionname' and description_id not in (SELECT DISTINCT description_id FROM description_milestone_tb WHERE milestone='$milestone')";
+ } else {
+ print "-->found unknown '$collectiontype'\n";
+ }
+
+ eval {
+ #print "---->> $sql\n";
+ my $sth2 = $dbh->prepare($sql);
+ $sth2->execute();
+ };
+ if ($@) {
+ warn "build_user_milestones.pl: failed to INSERT in description_milestone_tb: $@\n";
+ $dbh->rollback; # undo the incomplete changes
+ }
+
+ #print "\n";
+}
+$dbh->disconnect();
+
diff --git a/completeTranslations.pl b/completeTranslations.pl
index e8dbbd8..b2a9050 100755
--- a/completeTranslations.pl
+++ b/completeTranslations.pl
@@ -5,6 +5,8 @@ use strict;
my $count=0;
my $count_trans=0;
+my $count_part=0;
+my $count_1part=0;
use DBI;
use Digest::MD5 qw(md5_hex);
@@ -52,7 +54,12 @@ sub get_descrition_ids {
my $lang=shift ;
#my $sth = $dbh->prepare("SELECT description,description_id FROM description_tb WHERE description_id in (SELECT description_id FROM active_tb) and description_id not in (SELECT description_id FROM translation_tb WHERE description_id in (SELECT description_id FROM active_tb) and language=?) and package in (SELECT package FROM description_tb WHERE description_id in (SELECT description_id FROM translation_tb WHERE language=?) GROUP BY package)");
- my $sth = $dbh->prepare("SELECT description,description_id FROM description_tb WHERE description_id in (SELECT description_id FROM active_tb) and description_id not in (SELECT description_id FROM translation_tb WHERE description_id in (SELECT description_id FROM active_tb) and language=?)");
+ #my $sth = $dbh->prepare("SELECT description,description_id FROM description_tb WHERE description_id in (SELECT description_id FROM active_tb) and description_id not in (SELECT description_id FROM translation_tb WHERE description_id in (SELECT description_id FROM active_tb) and language=?)");
+
+ # check all descriptions...
+ #my $sth = $dbh->prepare("SELECT description,description_id FROM description_tb WHERE description_id not in (SELECT description_id FROM translation_tb WHERE language=?)");
+ # only the last descriptions from today
+ my $sth = $dbh->prepare("SELECT description,description_id FROM description_tb WHERE description_id in (SELECT description_id from description_tag_tb where date_end=current_date group by description_id) and description_id not in (SELECT description_id FROM translation_tb WHERE language=?)");
#$sth->execute($lang,$lang);
$sth->execute($lang);
@@ -91,7 +98,7 @@ sub get_descrition_ids {
}
if ($untranslated==0) {
- #print " $d_id is full translated\n";
+ print " $d_id is full translated\n";
$count_trans++;
#print "$translation";
eval {
@@ -104,11 +111,31 @@ sub get_descrition_ids {
$dbh->rollback; # undo the incomplete changes
}
+ } else {
+ if ($translated>0) {
+ $count_part++;
+ if ($untranslated==1) {
+ $count_1part++;
+
+ #print " save_part_milestone_to_db: $d_id milestone: part:$lang \n";
+
+ eval {
+ $dbh->do("INSERT INTO description_milestone_tb (description_id,milestone) VALUES (?,?);", undef, $d_id, "part:$untranslated-$lang");
+ $dbh->commit; # commit the changes if we get this far
+ };
+ if ($@) {
+ # warn "Packages2db.pl: failed to INSERT Package '$d_id', milestone 'part:$lang' into description_milestone_tb: $@\n";
+ $dbh->rollback; # undo the incomplete changes
+ }
+ }
+ }
}
}
}
-get_descrition_ids(shift);
+my $lang=shift ;
+
+get_descrition_ids($lang);
-print "#p_trans: $count #full_trans: $count_trans\n";
+print "#checked description:$count #partly translation:$count_part #one missing part:$count_1part #full_trans:$count_trans\n";
diff --git a/completeTranslations.sh b/completeTranslations.sh
index b6893ab..452db19 100755
--- a/completeTranslations.sh
+++ b/completeTranslations.sh
@@ -3,10 +3,11 @@
cd ~ddtp
# Fetch active langs from database
-LANGS=`psql ddtp -q -A -t -c "select distinct language from translation_tb"`
+LANGS=`psql ddtp -q -A -t -c "select distinct language from translation_tb where description_id>1"`
for lang in $LANGS
do
- echo -n "$lang: "
+ DATE=`date`
+ echo -n "$DATE $lang: "
./completeTranslations.pl $lang
done
diff --git a/db2Translation.pl b/db2Translation.pl
index eae016c..529a09b 100755
--- a/db2Translation.pl
+++ b/db2Translation.pl
@@ -22,54 +22,11 @@ my $dbh = DBI->connect(@DSN,
die $DBI::errstr unless $dbh;
-
-my $package;
-my $description_md5;
-
-sub get_description_ids {
- my $tag= shift(@_);
-
- my @description_ids;
-
- my $sth = $dbh->prepare("SELECT description_id FROM description_tag_tb WHERE tag=? and date_end=CURRENT_DATE ORDER BY description_id");
- $sth->execute($tag);
- while(($description_id) = $sth->fetchrow_array) {
- push @description_ids,$description_id;
- }
- return @description_ids;
+my $sth = $dbh->prepare("SELECT package,description_md5,translation FROM description_tag_tb join translation_tb ON description_tag_tb.description_id=translation_tb.description_id join description_tb ON description_tb.description_id=translation_tb.description_id WHERE tag=? and date_end=CURRENT_DATE and language=? order by translation_tb.description_id");
+$sth->execute($dists,$lang);
+while(my ($package,$description_md5,$translation) = $sth->fetchrow_array) {
+ print "Package: $package\n";
+ print "Description-md5: $description_md5\n";
+ print "Description-$lang: $translation\n";
}
-sub get_translation {
- my $description_id= shift(@_);
- my $lang= shift(@_);
-
- my $translation;
-
- my $sth = $dbh->prepare("SELECT translation FROM translation_tb WHERE description_id=? and language=?");
- $sth->execute($description_id,$lang);
- ($translation) = $sth->fetchrow_array;
- return $translation;
-}
-
-sub get_packageinfos {
- my $description_id= shift(@_);
-
- my $package;
- my $description_md5;
-
- my $sth = $dbh->prepare("SELECT package,description_md5 FROM description_tb WHERE description_id=?");
- $sth->execute($description_id);
- ($package,$description_md5) = $sth->fetchrow_array;
- return ($package,$description_md5);
-}
-
-foreach (get_description_ids($dists)) {
- $description_id=$_;
- $translation=get_translation($description_id,$lang);
- if ($translation) {
- ($package,$description_md5)=get_packageinfos($description_id);
- print "Package: $package\n";
- print "Description-md5: $description_md5\n";
- print "Description-$lang: $translation\n";
- }
-}
diff --git a/db2Translation.sh b/db2Translation.sh
index 18a22f9..3d5db4b 100755
--- a/db2Translation.sh
+++ b/db2Translation.sh
@@ -3,7 +3,7 @@
cd ~ddtp
# Fetch active langs from database
-LANGS=`psql ddtp -q -A -t -c "select distinct language from translation_tb"`
+LANGS=`psql ddtp -q -A -t -c "select distinct language from translation_tb where description_id>1"`
for distribution in etch lenny sid
do
diff --git a/db2po.pl b/db2po.pl
index 6901c4f..9303d92 100755
--- a/db2po.pl
+++ b/db2po.pl
@@ -125,6 +125,7 @@ foreach (get_description_ids($dists)) {
#print "Source: $source\n";
#print "Package: $package\n";
#print "Dir: $dir\n";
+ if (defined $dir) {
mkdir "pos";
mkdir "pos/$lang";
mkdir "pos/$lang/$dists";
@@ -138,7 +139,7 @@ foreach (get_description_ids($dists)) {
print FILE "\"POT-Creation-Date: \\n\"\n";
print FILE "\"PO-Revision-Date: \\n\"\n";
print FILE "\"Last-Translator: \\n\"\n";
- print FILE "\"Language-Team: German \\n\"\n";
+ print FILE "\"Language-Team: \\n\"\n";
print FILE "\"MIME-Version: 1.0\\n\"\n";
print FILE "\"Content-Type: text/plain; charset=UTF-8\\n\"\n";
print FILE "\"Content-Transfer-Encoding: 8bit\\n\"\n";
@@ -157,6 +158,12 @@ foreach (get_description_ids($dists)) {
$tparts[$index] =~ s/^/\"/mg;
$tparts[$index] =~ s/$/\\n\"/mg;
}
+ print FILE "#. Translators: This is the short description. \n" if ($index == 0);
+ print FILE "#.\n" if ($index == 0);
+ print FILE "#: short\n" if ($index == 0);
+ print FILE "#. Translators: This is the long description part $index. \n" if ($index > 0);
+ print FILE "#.\n" if ($index > 0);
+ print FILE "#: long part $index\n" if ($index > 0);
print FILE "msgid \"\"\n";
print FILE "$parts[$index]\n";
print FILE "msgstr \"\"\n";
@@ -164,4 +171,12 @@ foreach (get_description_ids($dists)) {
print FILE "\n";
}
close (FILE);
+ } else {
+ print "dir is not defined!\n";
+ print " for debug\n";
+ print " description_id: $description_id\n";
+ print " package: $package\n";
+ print " source: $source\n";
+ print "\n";
+ }
}
diff --git a/db2po.sh b/db2po.sh
index 536998c..1e2c39a 100755
--- a/db2po.sh
+++ b/db2po.sh
@@ -7,7 +7,7 @@ mkdir -p pos
if [ "x$1" = "x" ]
then
- LANGS=`psql ddtp -q -A -t -c "select distinct language from translation_tb"`
+ LANGS=`psql ddtp -q -A -t -c "select distinct language from translation_tb where description_id>1"`
else
LANGS=$1
fi
diff --git a/db2web.sh b/db2web.sh
index 350780a..73fc9ff 100755
--- a/db2web.sh
+++ b/db2web.sh
@@ -42,7 +42,7 @@ echo "set xdata time" >> lib/sid-stat.gnuplot
#echo "set logscale y" >> lib/sid-stat.gnuplot
echo "set format x \"%d.%m\\n%Y\"" >> lib/sid-stat.gnuplot
-LANGS=`psql ddtp -q -A -t -c "select distinct language from translation_tb"`
+LANGS=`psql ddtp -q -A -t -c "select distinct language from translation_tb where description_id>1"`
for lang in $LANGS
do
echo "<h3>$lang in sid</h3>" >> www/index.html.new
diff --git a/ddtp-dinstall/ddtp_dinstall.sh b/ddtp-dinstall/ddtp_dinstall.sh
new file mode 120000
index 0000000..b81e9c2
--- /dev/null
+++ b/ddtp-dinstall/ddtp_dinstall.sh
@@ -0,0 +1 @@
+checks/ddtp_dinstall.sh
\ No newline at end of file
diff --git a/ddtp-dinstall/ddtp_i18n_check.sh b/ddtp-dinstall/ddtp_i18n_check.sh
new file mode 120000
index 0000000..3facc0d
--- /dev/null
+++ b/ddtp-dinstall/ddtp_i18n_check.sh
@@ -0,0 +1 @@
+checks/ddtp_i18n_check.sh
\ No newline at end of file
diff --git a/ddts-stats b/ddts-stats
index 0d690f9..604857c 100755
--- a/ddts-stats
+++ b/ddts-stats
@@ -16,6 +16,7 @@ die $DBI::errstr unless $dbh;
my @langs = map { $_->[0] } @{ $dbh->selectall_arrayref("SELECT DISTINCT language FROM translation_tb") };
my %longnames = (
+ bg => 'Bulgarian',
da => 'Danish',
de => 'German',
ca => 'Catalan',
@@ -31,6 +32,7 @@ my %longnames = (
pt => 'Portugese',
ru => 'Russian',
sk => 'Slovak',
+ sr => 'Serbian',
sv => 'Swedish',
uk => 'Ukrainian',
es => 'Spanish',
@@ -96,7 +98,12 @@ sub load_packages
sub process_package
{
my $hash = shift;
- my $md5 = md5_hex( $hash->{Description}."\n" );
+ my $md5;
+ if( defined $hash->{"Description-md5"} ) {
+ $md5 = $hash->{"Description-md5"};
+ } else {
+ $md5 = md5_hex( $hash->{Description}."\n" );
+ }
# print "$hash->{Package} : $md5\n";
my $package = $hash->{Package};
$descrlist{$package}{$md5} = 0;
@@ -104,9 +111,6 @@ sub process_package
$descrlist{$package}{priority} = $hash->{Priority};
$descrlist{$package}{md5} = $md5;
- print "[".$hash->{Description}."]\n" if $package eq "kuvert";
- print "$package: $hash->{Priority} ($md5)\n" if $package eq "kuvert";
-
if( $hash->{Priority} =~ /required|important|standard/ )
{ $important_packages{$package}{$md5} = 0 }
}
@@ -128,7 +132,7 @@ sub count_packages
if( not defined $descrmd5{$md5} )
{
- warn "Can't find md5 ($md5), package $package\n";
+ print "Can't find md5 ($md5), package $package in $DIST\n";
next;
}
my $desc_id = $descrmd5{$md5};
diff --git a/file2Translation.pl b/file2Translation.pl
index bf29853..237708e 100755
--- a/file2Translation.pl
+++ b/file2Translation.pl
@@ -2,6 +2,7 @@
use diagnostics;
use strict;
+use Encode qw(decode);
# If --with-version is provided as first argument, add a Version field in
# the Translation file.
@@ -53,6 +54,11 @@ sub get_packageinfos {
return ($description_md5);
}
+sub check_utf8 {
+ my $str = shift;
+ return eval { decode('UTF-8', $str, Encode::FB_CROAK); 1 };
+}
+
sub make_translation_file {
my $tag= shift(@_);
@@ -74,6 +80,10 @@ sub make_translation_file {
#print " -> $description_id\n";
unless ($seen_package_and_description_ids{"$package $description_id"}) {
$translation=get_translation($description_id,$lang);
+ if (!check_utf8($translation)) {
+ print STDERR "Translation $lang:$package:$description_id:$description_md5 is not valid UTF-8, skipped\n";
+ next;
+ }
if ($translation) {
($description_md5)=get_packageinfos($description_id);
print "Package: $package\n";
diff --git a/file2Translation.sh b/file2Translation.sh
index 93d3a5a..8580e97 100755
--- a/file2Translation.sh
+++ b/file2Translation.sh
@@ -3,7 +3,7 @@
cd ~ddtp
# Fetch active langs from database
-LANGS=`psql ddtp -q -A -t -c "select distinct language from translation_tb"`
+LANGS=`psql ddtp -q -A -t -c "select distinct language from translation_tb where description_id>1"`
#DISTS="lenny sid"
#DISTS="squeeze sid"
diff --git a/file2Translation_udd.sh b/file2Translation_udd.sh
index 465d73d..13d634a 100755
--- a/file2Translation_udd.sh
+++ b/file2Translation_udd.sh
@@ -3,7 +3,7 @@
cd ~ddtp
# Fetch active langs from database
-LANGS=`psql ddtp -q -A -t -c "select distinct language from translation_tb"`
+LANGS=`psql ddtp -q -A -t -c "select distinct language from translation_tb where description_id>1"`
#DISTS="lenny squeeze wheezy sid"
DISTS="squeeze wheezy sid"
diff --git a/fill_package_version.pl b/fill_package_version.pl
new file mode 100755
index 0000000..136e795
--- /dev/null
+++ b/fill_package_version.pl
@@ -0,0 +1,33 @@
+#!/usr/bin/perl -w
+use strict;
+use LWP::Simple;
+use POSIX qw(strftime);
+use DBI;
+my @DSN = ("DBI:Pg:dbname=ddtp", "", "");
+
+my $dbh = DBI->connect(@DSN,
+ { PrintError => 0,
+ RaiseError => 1,
+ AutoCommit => 0,
+ });
+
+die $DBI::errstr unless $dbh;
+
+my $sth;
+
+#SELECT package_version_tb.description_id,description_tb.source from package_version_tb join description_tb ON package_version_tb.description_id = description_tb.description_id
+
+
+$sth = $dbh->prepare("SELECT package_version_tb.description_id,description_tb.source from package_version_tb join description_tb ON package_version_tb.description_id = description_tb.description_id");
+$sth->execute();
+while(my ($desc_id,$source) = $sth->fetchrow_array) {
+ eval {
+ $dbh->do("Update package_version_tb SET source='$source' WHERE description_id='$desc_id'");
+ $dbh->commit; # commit the changes if we get this far
+ };
+ if ($@) {
+ warn "can't update package_version_tb source='$source' for id='$desc_id': $@\n";
+ $dbh->rollback; # undo the incomplete changes
+ }
+}
+
diff --git a/fill_statistic.pl b/fill_statistic.pl
new file mode 100755
index 0000000..0fa6c2c
--- /dev/null
+++ b/fill_statistic.pl
@@ -0,0 +1,202 @@
+#!/usr/bin/perl -w
+use strict;
+use LWP::Simple;
+use POSIX qw(strftime);
+use DBI;
+my @DSN = ("DBI:Pg:dbname=ddtp", "", "");
+
+my $dbh = DBI->connect(@DSN,
+ { PrintError => 0,
+ RaiseError => 1,
+ AutoCommit => 0,
+ });
+
+die $DBI::errstr unless $dbh;
+
+my $sth;
+
+$sth = $dbh->prepare("DELETE FROM statistic_tb WHERE date=CURRENT_DATE");
+$sth->execute();
+
+
+# global
+
+$sth = $dbh->prepare("SELECT count(description_id),CURRENT_DATE from description_tb");
+$sth->execute();
+while(my ($count,$date) = $sth->fetchrow_array) {
+ eval {
+ $dbh->do("INSERT INTO statistic_tb (value,date,stat) VALUES (?,?,?);", undef, $count, $date, "all");
+ $dbh->commit; # commit the changes if we get this far
+ };
+ if ($@) {
+ warn "fill_statistic.pl: failed to INSERT stat 'all': $@\n";
+ $dbh->rollback; # undo the incomplete changes
+ }
+}
+
+$sth = $dbh->prepare("SELECT language,count(description_id),CURRENT_DATE from translation_tb group by language");
+$sth->execute();
+while(my ($lang,$count,$date) = $sth->fetchrow_array) {
+ if ($lang) {
+ eval {
+ $dbh->do("INSERT INTO statistic_tb (value,date,stat) VALUES (?,?,?);", undef, $count, $date, "all:trans-$lang");
+ $dbh->commit; # commit the changes if we get this far
+ };
+ if ($@) {
+ warn "fill_statistic.pl: failed to INSERT stat 'all:trans-$lang': $@\n";
+ $dbh->rollback; # undo the incomplete changes
+ }
+ }
+}
+
+# FIXME all in review/pending
+
+
+# Dists (sid, testing, stable)
+
+$sth = $dbh->prepare("SELECT tag,count(description_id),CURRENT_DATE from description_tag_tb where date_end=CURRENT_DATE group by tag");
+$sth->execute();
+while(my ($dist,$count,$date) = $sth->fetchrow_array) {
+ eval {
+ $dbh->do("INSERT INTO statistic_tb (value,date,stat) VALUES (?,?,?);", undef, $count, $date, "dist:$dist");
+ $dbh->commit; # commit the changes if we get this far
+ };
+ if ($@) {
+ warn "fill_statistic.pl: failed to INSERT stat 'dist:$dist': $@\n";
+ $dbh->rollback; # undo the incomplete changes
+ }
+}
+
+$sth = $dbh->prepare("SELECT tag,translation_tb.language,count(translation_tb.description_id),CURRENT_DATE from description_tag_tb LEFT JOIN translation_tb ON translation_tb.description_id=description_tag_tb.description_id where date_end=CURRENT_DATE group by tag, translation_tb.language");
+$sth->execute();
+while(my ($dist,$lang,$count,$date) = $sth->fetchrow_array) {
+ if ($lang) {
+ eval {
+ $dbh->do("INSERT INTO statistic_tb (value,date,stat) VALUES (?,?,?);", undef, $count, $date, "dist:$dist:trans-$lang");
+ $dbh->commit; # commit the changes if we get this far
+ };
+ if ($@) {
+ warn "fill_statistic.pl: failed to INSERT stat 'dist:$dist:trans-$lang': $@\n";
+ $dbh->rollback; # undo the incomplete changes
+ }
+ }
+}
+
+# FIXME dists in review/pending
+
+
+# Milestones
+
+$sth = $dbh->prepare("SELECT milestone,count(milestone),CURRENT_DATE from description_milestone_tb group by milestone");
+$sth->execute();
+while(my ($milestone,$count,$date) = $sth->fetchrow_array) {
+ eval {
+ $dbh->do("INSERT INTO statistic_tb (value,date,stat) VALUES (?,?,?);", undef, $count, $date, "mile:$milestone");
+ $dbh->commit; # commit the changes if we get this far
+ };
+ if ($@) {
+ warn "fill_statistic.pl: failed to INSERT stat 'mile:$milestone': $@\n";
+ $dbh->rollback; # undo the incomplete changes
+ }
+}
+
+$sth = $dbh->prepare("SELECT description_milestone_tb.milestone,translation_tb.language,count(description_milestone_tb.description_id),CURRENT_DATE from description_milestone_tb LEFT OUTER JOIN translation_tb ON (description_milestone_tb.description_id=translation_tb.description_id) group by description_milestone_tb.milestone, translation_tb.language order by translation_tb.language,description_milestone_tb.milestone");
+$sth->execute();
+
+while(my ($milestone,$lang,$count,$date) = $sth->fetchrow_array) {
+ if ($lang) {
+ eval {
+ $dbh->do("INSERT INTO statistic_tb (value,date,stat) VALUES (?,?,?);", undef, $count, $date, "mile:$milestone:trans-$lang");
+ $dbh->commit; # commit the changes if we get this far
+ };
+ if ($@) {
+ warn "fill_statistic.pl: failed to INSERT stat 'mile:$milestone:trans-$lang': $@\n";
+ $dbh->rollback; # undo the incomplete changes
+ }
+ }
+}
+
+# FIXME Milestones in review/pending
+
+
+# Users
+
+# $sth = $dbh->prepare("SELECT username,counttranslations,CURRENT_DATE from users_tb;");
+$sth = $dbh->prepare("SELECT *,CURRENT_DATE from ddtss where key like 'aliases/%/counttranslations'");
+$sth->execute();
+# while(my ($user,$count,$date) = $sth->fetchrow_array) {
+while(my ($key,$count,$date) = $sth->fetchrow_array) {
+ if ( $key =~ /aliases\/([a-zA-Z0-9_]*)\/counttranslations/ ) {
+ my $user = $1;
+ eval {
+ $dbh->do("INSERT INTO statistic_tb (value,date,stat) VALUES (?,?,?);", undef, $count, $date, "user:translations-$user");
+ $dbh->commit; # commit the changes if we get this far
+ };
+ if ($@) {
+ warn "fill_statistic.pl: failed to INSERT stat 'user:translations-$user': $@\n";
+ $dbh->rollback; # undo the incomplete changes
+ }
+ }
+
+}
+
+# $sth = $dbh->prepare("SELECT username,countreviews,CURRENT_DATE from users_tb;");
+$sth = $dbh->prepare("SELECT *,CURRENT_DATE from ddtss where key like 'aliases/%/countreviews'");
+$sth->execute();
+# while(my ($user,$count,$date) = $sth->fetchrow_array) {
+while(my ($key,$count,$date) = $sth->fetchrow_array) {
+ if ( $key =~ /aliases\/([a-zA-Z0-9_]*)\/countreviews/ ) {
+ my $user = $1;
+ eval {
+ $dbh->do("INSERT INTO statistic_tb (value,date,stat) VALUES (?,?,?);", undef, $count, $date, "user:reviews-$user");
+ $dbh->commit; # commit the changes if we get this far
+ };
+ if ($@) {
+ warn "fill_statistic.pl: failed to INSERT stat 'user:reviews-$user': $@\n";
+ $dbh->rollback; # undo the incomplete changes
+ }
+ }
+
+}
+
+
+
+# lang pending
+
+my @lang;
+
+$sth = $dbh->prepare("select distinct language from translation_tb where description_id>1");
+$sth->execute();
+while(my ($lang) = $sth->fetchrow_array) {
+ push (@lang,$lang);
+}
+
+foreach my $lang (@lang) {
+ $sth = $dbh->prepare("SELECT count(*),CURRENT_DATE from ddtss where value='forreview' and key like ?");
+ $sth->execute("$lang%");
+ while(my ($count,$date) = $sth->fetchrow_array) {
+ eval {
+ $dbh->do("INSERT INTO statistic_tb (value,date,stat) VALUES (?,?,?);", undef, $count, $date, "lang:pendingreview-$lang");
+ $dbh->commit; # commit the changes if we get this far
+ };
+ if ($@) {
+ warn "fill_statistic.pl: failed to INSERT stat 'lang:pendingreview-$lang': $@\n";
+ $dbh->rollback; # undo the incomplete changes
+ }
+ }
+
+ $sth = $dbh->prepare("SELECT count(*),CURRENT_DATE from ddtss where value like 'untranslated,%' and key like ?");
+ $sth->execute("$lang%");
+ while(my ($count,$date) = $sth->fetchrow_array) {
+ eval {
+ $dbh->do("INSERT INTO statistic_tb (value,date,stat) VALUES (?,?,?);", undef, $count, $date, "lang:pendingtranslation-$lang");
+ $dbh->commit; # commit the changes if we get this far
+ };
+ if ($@) {
+ warn "fill_statistic.pl: failed to INSERT stat 'lang:pendingtranslation-$lang': $@\n";
+ $dbh->rollback; # undo the incomplete changes
+ }
+
+ }
+
+}
diff --git a/fill_user_statistic.pl b/fill_user_statistic.pl
new file mode 100755
index 0000000..b9e3d65
--- /dev/null
+++ b/fill_user_statistic.pl
@@ -0,0 +1,59 @@
+#!/usr/bin/perl -w
+use strict;
+use LWP::Simple;
+use POSIX qw(strftime);
+use DBI;
+my @DSN = ("DBI:Pg:dbname=ddtp", "", "");
+
+my $dbh = DBI->connect(@DSN,
+ { PrintError => 0,
+ RaiseError => 1,
+ AutoCommit => 0,
+ });
+
+die $DBI::errstr unless $dbh;
+
+my $sth;
+
+$sth = $dbh->prepare("DELETE FROM statistic_tb WHERE stat like 'lang:pendingreview-%' or stat like 'lang:pendingtranslation-%'");
+$sth->execute();
+
+# lang pending
+
+my @lang;
+
+$sth = $dbh->prepare("select distinct language from translation_tb where description_id>1");
+$sth->execute();
+while(my ($lang) = $sth->fetchrow_array) {
+ push (@lang,$lang);
+}
+
+foreach my $lang (@lang) {
+ $sth = $dbh->prepare("SELECT count(*),CURRENT_DATE from ddtss where value='forreview' and key like ?");
+ $sth->execute("$lang%");
+ while(my ($count,$date) = $sth->fetchrow_array) {
+ eval {
+ $dbh->do("INSERT INTO statistic_tb (value,date,stat) VALUES (?,?,?);", undef, $count, $date, "lang:pendingreview-$lang");
+ $dbh->commit; # commit the changes if we get this far
+ };
+ if ($@) {
+ warn "fill_statistic.pl: failed to INSERT stat 'lang:pendingreview-$lang': $@\n";
+ $dbh->rollback; # undo the incomplete changes
+ }
+ }
+
+ $sth = $dbh->prepare("SELECT count(*),CURRENT_DATE from ddtss where value like 'untranslated,%' and key like ?");
+ $sth->execute("$lang%");
+ while(my ($count,$date) = $sth->fetchrow_array) {
+ eval {
+ $dbh->do("INSERT INTO statistic_tb (value,date,stat) VALUES (?,?,?);", undef, $count, $date, "lang:pendingtranslation-$lang");
+ $dbh->commit; # commit the changes if we get this far
+ };
+ if ($@) {
+ warn "fill_statistic.pl: failed to INSERT stat 'lang:pendingtranslation-$lang': $@\n";
+ $dbh->rollback; # undo the incomplete changes
+ }
+
+ }
+
+}
diff --git a/packages_tb2Packages.pl b/packages_tb2Packages.pl
index 6c894a0..95fe22c 100755
--- a/packages_tb2Packages.pl
+++ b/packages_tb2Packages.pl
@@ -21,15 +21,15 @@ exit;
sub export_packages
{
- my ($package,$source,$version,$tag,$priority,$maintainer,$task,$section,$description);
+ my ($package,$source,$version,$tag,$priority,$maintainer,$task,$section,$description,$description_md5);
print " Export package file \n";
my $fh;
open $fh, '>', "$file" or die "Couldn't write to $file ($!)\n";
- my $sth = $dbh->prepare("SELECT package,source,version,tag,priority,maintainer,task,section,description FROM packages_tb ORDER BY package");
+ my $sth = $dbh->prepare("SELECT package,source,version,tag,priority,maintainer,task,section,description,description_md5 FROM packages_tb ORDER BY package");
$sth->execute;
- while ( ($package,$source,$version,$tag,$priority,$maintainer,$task,$section,$description) = $sth->fetchrow_array ) {
+ while ( ($package,$source,$version,$tag,$priority,$maintainer,$task,$section,$description,$description_md5) = $sth->fetchrow_array ) {
print $fh "Package: $package\n";
print $fh "Source: $source\n";
print $fh "Version: $version\n";
@@ -39,6 +39,7 @@ sub export_packages
print $fh "Task: $task\n" if $task;
print $fh "Section: $section\n";
print $fh "Description: $description\n";
+ print $fh "Description-md5: $description_md5\n" if length($description_md5) == 32;
print $fh "\n";
}
close $fh;
diff --git a/popcon2db.pl b/popcon2db.pl
new file mode 100755
index 0000000..9e140d9
--- /dev/null
+++ b/popcon2db.pl
@@ -0,0 +1,75 @@
+#!/usr/bin/perl -w
+use strict;
+use LWP::Simple;
+use POSIX qw(strftime);
+use DBI;
+my @DSN = ("DBI:Pg:dbname=ddtp", "", "");
+
+my $dbh = DBI->connect(@DSN,
+ { PrintError => 0,
+ RaiseError => 1,
+ AutoCommit => 0,
+ });
+
+die $DBI::errstr unless $dbh;
+
+my $data = "/srv/ddtp.debian.net/Packages/";
+
+my $POPCON = "http://popcon.debian.org/by_vote";
+
+my $POPCON_COUNT = 500;
+
+fetch_data();
+load_popcon(); # Read popcon data
+exit;
+
+sub fetch_data
+{
+ my $code = mirror( $POPCON, "$data/popcon.txt" );
+ warn "$POPCON: $code\n" if ($code != 200 and $code != 304);
+}
+
+# This goes through the package list and compares it against the DDTP.
+# Firstly to count the number of each priority, secondly to detect missing
+# package entries.
+
+sub load_popcon
+{
+ my $sth = $dbh->prepare("DELETE FROM description_milestone_tb WHERE milestone like 'popc:%'");
+ $sth->execute();
+
+ my $fh;
+ open $fh, "$data/popcon.txt" or die "Couldn't read popcon data ($!)\n";
+
+ my $count = 0;
+ while(<$fh>)
+ {
+ next if /^#/;
+ next unless /^\d+\s+/;
+ my @F = split /\s+/;
+ $count++;
+ if( $count <= 8000 ) {
+ $sth = $dbh->prepare("SELECT package_version_tb.description_id,tag from package_version_tb join description_tag_tb ON description_tag_tb.description_id=package_version_tb.description_id where date_end=CURRENT_DATE and package=? group by package_version_tb.description_id, tag");
+ #$sth = $dbh->prepare("SELECT package_version_tb.description_id,tag from package_version_tb join description_tag_tb ON description_tag_tb.description_id=package_version_tb.description_id where date_end='2011-08-16' and package=? group by package_version_tb.description_id, tag");
+ $sth->execute($F[1]);
+
+
+ while(my ($description_id,$tag) = $sth->fetchrow_array) {
+ foreach my $maxcount (500,1000,2000,4000,8000) {
+ if ($count <= $maxcount) {
+ eval {
+ $dbh->do("INSERT INTO description_milestone_tb (description_id,milestone) VALUES (?,?);", undef, $description_id, "popc:$tag-$maxcount");
+ $dbh->commit; # commit the changes if we get this far
+ };
+ if ($@) {
+ #warn "popcon2db.pl: failed to INSERT Package '$description_id', milestone 'popc:500' into description_milestone_tb: $@\n";
+ $dbh->rollback; # undo the incomplete changes
+ }
+ }
+ }
+ }
+ }
+ }
+
+ close $fh;
+}
diff --git a/sql_layout.txt b/sql_layout.txt
index a1a35e6..ab2e086 100644
--- a/sql_layout.txt
+++ b/sql_layout.txt
@@ -4,7 +4,6 @@ DROP TABLE suggestion_tb;
DROP TABLE packages_tb;
DROP TABLE package_version_tb;
-DROP TABLE version_tb;
DROP TABLE owner_tb;
@@ -101,19 +100,11 @@ CREATE INDEX owner_tb_1_IDX ON owner_tb (language);
CREATE INDEX owner_tb_2_IDX ON owner_tb (owner);
CREATE UNIQUE INDEX owner_tb_3_IDX ON owner_tb (description_id,language);
-CREATE TABLE version_tb
-(
- version_id serial PRIMARY KEY ,
- version text NOT NULL,
- description_id integer NOT NULL REFERENCES description_tb
-);
-CREATE INDEX version_tb_1_IDX ON version_tb (description_id);
-CREATE UNIQUE INDEX version_tb_3_IDX ON version_tb (description_id,version);
-
CREATE TABLE package_version_tb
(
package_version_id serial PRIMARY KEY ,
package text NOT NULL,
+ source text NOT NULL,
version text NOT NULL,
description_id integer NOT NULL REFERENCES description_tb
);
@@ -121,6 +112,7 @@ CREATE INDEX package_version_tb_1_IDX ON package_version_tb (description_id);
CREATE INDEX package_version_tb_2_IDX ON package_version_tb (package);
CREATE UNIQUE INDEX package_version_tb_3_IDX ON package_version_tb (description_id,package,version);
CREATE INDEX package_version_tb_4_IDX ON package_version_tb (package,version);
+CREATE INDEX package_version_tb_5_IDX ON package_version_tb (source);
CREATE TABLE packages_tb
(
@@ -152,3 +144,44 @@ CREATE TABLE suggestion_tb
);
CREATE INDEX suggestion_tb_1_IDX ON suggestion_tb (package);
CREATE INDEX suggestion_tb_2_IDX ON suggestion_tb (package,language,description_md5);
+
+
+
+DROP TABLE description_milestone_tb;
+
+CREATE TABLE description_milestone_tb (
+ description_milestone_id serial PRIMARY KEY,
+ description_id integer NOT NULL,
+ milestone text NOT NULL
+);
+
+CREATE INDEX description_milestone_tb1_IDX ON description_milestone_tb (description_id);
+CREATE INDEX description_milestone_tb2_IDX ON description_milestone_tb (milestone);
+CREATE UNIQUE INDEX description_milestone_tb3_IDX ON description_milestone_tb (description_id,milestone);
+
+DROP TABLE collection_milestone_tb;
+
+CREATE TABLE collection_milestone_tb (
+ collection_milestone_id serial PRIMARY KEY,
+ collection text NOT NULL,
+ name text NOT NULL,
+ nametype integer NOT NULL
+);
+
+CREATE INDEX collection_milestone_tb1_IDX ON collection_milestone_tb (name,nametype);
+CREATE UNIQUE INDEX collection_milestone_tb2_IDX ON collection_milestone_tb (name,nametype,collection);
+
+
+DROP TABLE statistic_tb;
+
+CREATE TABLE statistic_tb (
+ statistic_id serial PRIMARY KEY,
+ value integer NOT NULL,
+ date date NOT NULL,
+ stat text NOT NULL
+);
+
+CREATE INDEX statistic_tb1_IDX ON statistic_tb (stat);
+CREATE INDEX statistic_tb2_IDX ON statistic_tb (date);
+CREATE UNIQUE INDEX statistic_tb13IDX ON statistic_tb (stat,date);
+
diff --git a/update.sh b/update.sh
index 0b1482d..32cb285 100755
--- a/update.sh
+++ b/update.sh
@@ -7,9 +7,6 @@ LOGPREFIX=$LOGDIR/update.cron.$(date "+%Y%m%d-%H%M")
[ ! -d "$LOGDIR" ] && mkdir "$LOGDIR"
-# This should be removed later -- Nekral
-date
-
date >> $LOGPREFIX.log
./Packages2packages_tb.sh >> $LOGPREFIX.log 2>> $LOGPREFIX.err
./Packages2db.sh >> $LOGPREFIX.log 2>> $LOGPREFIX.err
@@ -22,6 +19,9 @@ date >> $LOGPREFIX.log
./ddts-stats wheezy >> $LOGPREFIX.log 2>> $LOGPREFIX.err
./ddts-stats squeeze >> $LOGPREFIX.log 2>> $LOGPREFIX.err
+./popcon2db.pl >> $LOGPREFIX.log 2>> $LOGPREFIX.err
+./fill_statistic.pl >> $LOGPREFIX.log 2>> $LOGPREFIX.err
+
#cp -a /home/grisu/public_html/ddtp/* /var/www/ddtp/
echo -n "Rotating (/srv/ddtp.debian.net/logrotate.config)" ... >> $LOGPREFIX.log
@@ -31,6 +31,3 @@ echo "OK" >> $LOGPREFIX.log
date >> $LOGPREFIX.log
cat $LOGPREFIX.err
-
-# This should be removed later -- Nekral
-date
diff --git a/update0.5.sh b/update0.5.sh
index e6163f7..04200df 100755
--- a/update0.5.sh
+++ b/update0.5.sh
@@ -7,16 +7,14 @@ LOGPREFIX=$LOGDIR/update.cron.$(date "+%Y%m%d-%H%M")
[ ! -d "$LOGDIR" ] && mkdir "$LOGDIR"
-# This should be removed later -- Nekral
-date
-
date >> $LOGPREFIX.log
#./Packages2packages_tb.sh >> $LOGPREFIX.log 2>> $LOGPREFIX.err
#./Packages2db.sh >> $LOGPREFIX.log 2>> $LOGPREFIX.err
#./completeTranslations.sh >> $LOGPREFIX.log 2>> $LOGPREFIX.err
./db2web.sh >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-./file2Translation.sh >> $LOGPREFIX.log 2>> $LOGPREFIX.err
+#./file2Translation.sh >> $LOGPREFIX.log 2>> $LOGPREFIX.err
./file2Translation_udd.sh >> $LOGPREFIX.log 2>> $LOGPREFIX.err
+./db2po.sh zh_CN sid >> $LOGPREFIX.log 2>> $LOGPREFIX.err
# Regenerate the stats files
./ddts-stats sid >> $LOGPREFIX.log 2>> $LOGPREFIX.err
@@ -29,5 +27,3 @@ date >> $LOGPREFIX.log
cat $LOGPREFIX.err
-# This should be removed later -- Nekral
-date
--
DDTP - Debian Descriptions Translation Project
More information about the Debian-l10n-commits
mailing list