[Collab-qa-commits] r1068 - in udd/src: . schema udd
neronus-guest at alioth.debian.org
neronus-guest at alioth.debian.org
Mon Aug 11 13:33:26 UTC 2008
Author: neronus-guest
Date: 2008-08-11 13:33:24 +0000 (Mon, 11 Aug 2008)
New Revision: 1068
Added:
udd/src/schema/
udd/src/schema/bugs
udd/src/schema/carnivore
udd/src/schema/lintian
udd/src/schema/orphaned_packages
udd/src/schema/packages
udd/src/schema/popcon
udd/src/schema/sources
udd/src/schema/testing_migrations
udd/src/schema/upload_history
Modified:
udd/src/db_manager.py
udd/src/test.yaml
udd/src/udd-dispatch.py
udd/src/udd/bugs_gatherer.pl
udd/src/udd/carnivore_gatherer.py
udd/src/udd/gatherer.py
udd/src/udd/lintian_gatherer.py
udd/src/udd/orphaned_packages_gatherer.py
udd/src/udd/packages_gatherer.py
udd/src/udd/popcon_gatherer.py
udd/src/udd/sources_gatherer.py
udd/src/udd/src_and_pkg_gatherer.py
udd/src/udd/testing_migrations_gatherer.py
udd/src/udd/upload_history_gatherer.py
Log:
Decentralized setup and drop scheme
bugs_gatherer.pl now has prepared SQL statements
for archived bugs, affects_* is now false
Modified: udd/src/db_manager.py
===================================================================
--- udd/src/db_manager.py 2008-08-11 04:51:35 UTC (rev 1067)
+++ udd/src/db_manager.py 2008-08-11 13:33:24 UTC (rev 1068)
@@ -9,9 +9,10 @@
TABLES = ('popcon_src_average', 'popcon_src', 'popcon',
'ubuntu_popcon_src_average', 'ubuntu_popcon_src', 'ubuntu_popcon',
'sources', 'packages', 'ubuntu_sources', 'ubuntu_packages', 'migrations',
- 'bugs_archived', 'bugs', 'bug_merged_with', 'bug_user_tags',
- 'bug_found_in', 'bug_fixed_in', 'bug_tags', 'upload_history', 'carnivore_emails', 'carnivore_keys', 'carnivore_login', 'carnivore_names', 'lintian', 'orphaned_packages', 'packages_summary', 'ubuntu_packages_summary')
-VIEWS = ('bugs_both',)
+ 'bugs_archived', 'bugs_unarchived', 'bug_merged_with', 'bug_user_tags',
+ 'bug_found_in', 'bug_fixed_in', 'bug_tags', 'upload_history',
+ 'carnivore_emails', 'carnivore_names', 'carnivore_keys', 'carnivore_login' )
+VIEWS = ('bugs', 'bug_rt_affects_stable', 'bug_rt_affects_testing', 'bug_rt_affects_unstable', 'bug_rt_affects_testing_and_unstable')
def print_help():
print "Usage: %s <config> <delete|setup>" % sys.argv[0]
Added: udd/src/schema/bugs
===================================================================
--- udd/src/schema/bugs (rev 0)
+++ udd/src/schema/bugs 2008-08-11 13:33:24 UTC (rev 1068)
@@ -0,0 +1,96 @@
+CREATE TABLE %(table)s
+ (id int PRIMARY KEY, package text, source text, arrival timestamp, status text,
+ severity text, submitter text, owner text, title text,
+ last_modified timestamp, affects_stable boolean,
+ affects_testing boolean, affects_unstable boolean);
+
+CREATE TABLE %(table)s_merged_with
+ (id int, merged_with int,
+PRIMARY KEY(id, merged_with));
+
+CREATE TABLE %(table)s_found_in
+ (id int, version text,
+ PRIMARY KEY(id, version));
+
+CREATE TABLE %(table)s_fixed_in
+ (id int, version text,
+ PRIMARY KEY(id, version));
+
+CREATE TABLE %(table)s_tags
+ (id int, tag text, PRIMARY KEY (id, tag));
+
+CREATE TABLE %(archived-table)s
+ (id int PRIMARY KEY, package text, source text, arrival timestamp, status text,
+ severity text, submitter text, owner text, title text,
+ last_modified timestamp, affects_sarchived_table boolean,
+ affects_testing boolean, affects_unsarchived_table boolean);
+
+CREATE TABLE %(archived-table)s_merged_with
+ (id int, merged_with int,
+PRIMARY KEY(id, merged_with));
+
+CREATE TABLE %(archived-table)s_found_in
+ (id int, version text,
+ PRIMARY KEY(id, version));
+
+CREATE TABLE %(archived-table)s_fixed_in
+ (id int, version text,
+ PRIMARY KEY(id, version));
+
+CREATE TABLE %(archived-table)s_tags
+ (id int, tag text, PRIMARY KEY (id, tag));
+
+CREATE VIEW bugs_rt_affects_stable AS
+SELECT id, package, source FROM %(table)s
+WHERE affects_stable
+AND (id NOT IN (SELECT id FROM %(table)s_tags WHERE tag IN ('sid', 'sarge', 'lenny', 'experimental'))
+OR id IN (SELECT id FROM %(table)s_tags WHERE tag = 'etch'))
+AND id IN (select id FROM %(table)s_tags WHERE tag = 'etch-ignore')
+AND ( package IN (SELECT DISTINCT package FROM %(packages-table)s p WHERE release = 'etch')
+OR source IN (SELECT DISTINCT package FROM %(sources-table)s WHERE release = 'etch'));
+
+CREATE VIEW bugs_rt_affects_testing AS
+SELECT id, package, source FROM %(table)s
+WHERE affects_testing
+AND (id NOT IN (SELECT id FROM %(table)s_tags WHERE tag IN ('sid', 'sarge', 'etch', 'experimental'))
+OR id IN (SELECT id FROM %(table)s_tags WHERE tag = 'lenny'))
+AND id IN (select id FROM %(table)s_tags WHERE tag = 'lenny-ignore')
+AND ( package IN (SELECT DISTINCT package FROM %(packages-table)s p WHERE release = 'lenny')
+OR source IN (SELECT DISTINCT package FROM %(sources-table)s WHERE release = 'lenny'));
+
+CREATE VIEW bugs_rt_affects_unstable AS
+SELECT id, package, source FROM %(table)s
+WHERE affects_unstable
+AND (id NOT IN (SELECT id FROM %(table)s_tags WHERE tag IN ('lenny', 'sarge', 'etch', 'experimental'))
+OR id IN (SELECT id FROM %(table)s_tags WHERE tag = 'sid'))
+AND ( package IN (SELECT DISTINCT package FROM %(packages-table)s p WHERE release = 'sid')
+OR source IN (SELECT DISTINCT package FROM %(sources-table)s WHERE release = 'sid'));
+
+CREATE VIEW bugs_rt_affects_testing_and_unstable AS
+SELECT id, package, source FROM %(table)s
+WHERE affects_unstable AND affects_testing
+AND (id NOT IN (SELECT id FROM %(table)s_tags WHERE tag IN ('sarge', 'etch', 'experimental'))
+OR (id IN (SELECT id FROM %(table)s_tags WHERE tag = 'sid') AND id IN (SELECT id FROM %(table)s_tags WHERE tag = 'lenny')))
+AND ( package IN (SELECT DISTINCT package FROM %(packages-table)s p WHERE release = 'sid')
+OR source IN (SELECT DISTINCT package FROM %(sources-table)s WHERE release = 'sid'))
+AND ( package IN (SELECT DISTINCT package FROM %(packages-table)s p WHERE release = 'lenny')
+OR source IN (SELECT DISTINCT package FROM %(sources-table)s WHERE release = 'lenny'));
+
+CREATE TABLE %(usertags-table)s
+ (email text, tag text, id int);
+
+GRANT SELECT ON %(table)s TO PUBLIC;
+GRANT SELECT ON %(table)s_merged_with TO PUBLIC;
+GRANT SELECT ON %(table)s_found_in TO PUBLIC;
+GRANT SELECT ON %(table)s_fixed_in TO PUBLIC;
+GRANT SELECT ON %(table)s_tags TO PUBLIC;
+GRANT SELECT ON %(archived-table)s TO PUBLIC;
+GRANT SELECT ON %(archived-table)s_merged_with TO PUBLIC;
+GRANT SELECT ON %(archived-table)s_found_in TO PUBLIC;
+GRANT SELECT ON %(archived-table)s_fixed_in TO PUBLIC;
+GRANT SELECT ON %(archived-table)s_tags TO PUBLIC;
+GRANT SELECT ON bugs_rt_affects_stable TO PUBLIC;
+GRANT SELECT ON bugs_rt_affects_testing_and_unstable TO PUBLIC;
+GRANT SELECT ON bugs_rt_affects_unstable TO PUBLIC;
+GRANT SELECT ON bugs_rt_affects_testing TO PUBLIC;
+GRANT SELECT ON %(usertags-table)s TO PUBLIC;
Added: udd/src/schema/carnivore
===================================================================
--- udd/src/schema/carnivore (rev 0)
+++ udd/src/schema/carnivore 2008-08-11 13:33:24 UTC (rev 1068)
@@ -0,0 +1,17 @@
+CREATE TABLE %(emails-table)s
+ (id int, email text);
+
+CREATE TABLE %(names-table)s
+ (id int, name text);
+
+CREATE TABLE %(keys-table)s
+ (id int, key text, key_type text);
+
+CREATE TABLE %(login-table)s
+ (id int, login text,
+ PRIMARY KEY(id));
+
+GRANT SELECT on %(emails-table)s TO PUBLIC;
+GRANT SELECT on %(names-table)s TO PUBLIC;
+GRANT SELECT on %(keys-table)s TO PUBLIC;
+GRANT SELECT on %(login-table)s TO PUBLIC;
Added: udd/src/schema/lintian
===================================================================
--- udd/src/schema/lintian (rev 0)
+++ udd/src/schema/lintian 2008-08-11 13:33:24 UTC (rev 1068)
@@ -0,0 +1,18 @@
+CREATE DOMAIN lintian_tag_type AS TEXT
+NOT NULL
+CHECK(
+ VALUE = 'error'
+ OR VALUE = 'warning'
+ OR VALUE = 'information'
+ OR VALUE = 'experimental'
+ OR VALUE = 'overriden'
+);
+
+CREATE TABLE %(table)s (
+ package TEXT NOT NULL,
+ tag_type lintian_tag_type,
+ package_type TEXT,
+ tag TEXT NOT NULL
+);
+
+GRANT SELECT ON %(table)s TO PUBLIC;
Added: udd/src/schema/orphaned_packages
===================================================================
--- udd/src/schema/orphaned_packages (rev 0)
+++ udd/src/schema/orphaned_packages 2008-08-11 13:33:24 UTC (rev 1068)
@@ -0,0 +1,9 @@
+CREATE TABLE %(table)s (
+ source TEXT PRIMARY KEY,
+ type TEXT,
+ bug INT,
+ description TEXT,
+ orphaned_time TIMESTAMP
+);
+
+GRANT SELECT ON %(table)s TO PUBLIC;
Added: udd/src/schema/packages
===================================================================
--- udd/src/schema/packages (rev 0)
+++ udd/src/schema/packages 2008-08-11 13:33:24 UTC (rev 1068)
@@ -0,0 +1,18 @@
+CREATE TABLE %(packages-table)s
+ (package text, version text, architecture text, maintainer text, description
+ text, source text, source_version text, essential text, depends text,
+ recommends text, suggests text, enhances text, pre_depends text,
+ installed_size int, homepage text, size int,
+ build_essential text, origin text, sha1 text, replaces text, section text,
+ md5sum text, bugs text, priority text, tag text, task text, python_version text,
+ provides text, conflicts text, sha256 text, original_maintainer text,
+ distribution text, release text, component text,
+ PRIMARY KEY (package, version, architecture, distribution, release, component));
+
+GRANT SELECT ON %(packages-table)s TO PUBLIC;
+
+CREATE INDEX %(packages-table)s_source_idx on %(packages-table)s(source);
+CREATE INDEX %(packages-table)s_distrelcomp_idx on %(packages-table)s(distribution, release, component);
+CREATE INDEX %(packages-table)s_distribution_idx on %(packages-table)s(distribution);
+CREATE INDEX %(packages-table)s_release_idx on %(packages-table)s(release);
+CREATE INDEX %(packages-table)s_component_idx on %(packages-table)s(component);
Added: udd/src/schema/popcon
===================================================================
--- udd/src/schema/popcon (rev 0)
+++ udd/src/schema/popcon 2008-08-11 13:33:24 UTC (rev 1068)
@@ -0,0 +1,15 @@
+CREATE TABLE %(table)s (
+ package text, insts int, vote int, olde int, recent int, nofiles int,
+ PRIMARY KEY (package));
+
+CREATE TABLE %(table)s_src (
+ source text, insts int, vote int, olde int, recent int, nofiles int,
+ PRIMARY KEY (source));
+
+CREATE TABLE %(table)s_src_average (
+ source text, insts int, vote int, olde int, recent int, nofiles int,
+ PRIMARY KEY (source));
+
+GRANT SELECT ON %(table)s TO PUBLIC;
+GRANT SELECT ON %(table)s_src_average TO PUBLIC;
+GRANT SELECT ON %(table)s_src TO PUBLIC;
Added: udd/src/schema/sources
===================================================================
--- udd/src/schema/sources (rev 0)
+++ udd/src/schema/sources 2008-08-11 13:33:24 UTC (rev 1068)
@@ -0,0 +1,17 @@
+CREATE TABLE %(sources-table)s
+ (source text, version text, maintainer text, format text, files text,
+ uploaders text, bin text, architecture text, standards_version text,
+ homepage text, build_depends text, build_depends_indep text,
+ build_conflicts text, build_conflicts_indep text, priority text, section
+ text, distribution text, release text, component text, vcs_type text,
+ vcs_url text, vcs_browser text,
+ python_version text, checksums_sha1 text, checksums_sha256 text,
+ original_maintainer text, dm_upload_allowed text,
+ PRIMARY KEY (source, version, distribution, release, component));
+
+GRANT SELECT ON %(sources-table)s TO PUBLIC;
+
+CREATE INDEX %(sources-table)s_distrelcomp_idx on %(sources-table)s(distribution, release, component);
+CREATE INDEX %(sources-table)s_distribution_idx on %(sources-table)s(distribution);
+CREATE INDEX %(sources-table)s_release_idx on %(sources-table)s(release);
+CREATE INDEX %(sources-table)s_component_idx on %(sources-table)s(component);
Added: udd/src/schema/testing_migrations
===================================================================
--- udd/src/schema/testing_migrations (rev 0)
+++ udd/src/schema/testing_migrations 2008-08-11 13:33:24 UTC (rev 1068)
@@ -0,0 +1,4 @@
+CREATE TABLE %(table)s
+ (source text PRIMARY KEY, in_testing date, testing_version text, in_unstable date, unstable_version text, sync date, sync_version text, first_seen date);
+
+GRANT SELECT ON %(table)s TO PUBLIC;
Added: udd/src/schema/upload_history
===================================================================
--- udd/src/schema/upload_history (rev 0)
+++ udd/src/schema/upload_history 2008-08-11 13:33:24 UTC (rev 1068)
@@ -0,0 +1,5 @@
+CREATE TABLE %(table)s
+ (package text, version text, date timestamp with time zone, changed_by text,
+ maintainer text, nmu boolean, signed_by text, key_id text);
+
+GRANT SELECT ON %(table)s TO PUBLIC;
Modified: udd/src/test.yaml
===================================================================
--- udd/src/test.yaml 2008-08-11 04:51:35 UTC (rev 1067)
+++ udd/src/test.yaml 2008-08-11 13:33:24 UTC (rev 1068)
@@ -12,9 +12,11 @@
bugs: exec DEBBUGS_CONFIG_FILE=/org/udd.debian.net/mirrors/bugs.debian.org/etc/config perl ./udd/bugs_gatherer.pl
carnivore: module udd.carnivore_gatherer
lintian: module udd.lintian_gatherer
+ #src-pkg: module udd.packages_gatherer
#src-pkg: python sources_gatherer.py
debug: 1
update-timestamp-folder: ./timestamps/
+ schema-dir: ./schema/
archs:
[alpha, amd64, arm, armeb, armel, hppa, hurd-i386,
@@ -38,6 +40,8 @@
mipsel, powerpc, s390, sparc]
directory: /org/ftp.debian.org/dists/lenny/
components: [main, contrib, non-free]
+ packages-schema: packages
+ sources-schema: sources
packages-table: packages
sources-table: sources
distribution: debian
@@ -50,6 +54,8 @@
mipsel, powerpc, ppc64, s390, sparc]
directory: /org/ftp.debian.org/dists/sid/
components: [main, contrib, non-free]
+ packages-schema: packages
+ sources-schema: sources
packages-table: packages
sources-table: sources
distribution: debian
@@ -63,6 +69,8 @@
directory: /org/ftp.debian.org/dists/etch/
components: [main, contrib, non-free]
distribution: debian
+ packages-schema: packages
+ sources-schema: sources
packages-table: packages
sources-table: sources
release: etch
@@ -75,6 +83,8 @@
directory: /org/ftp.backports.org/dists/etch-backports/
components: [main, contrib, non-free]
distribution: debian-backports
+ packages-schema: packages
+ sources-schema: sources
packages-table: packages
sources-table: sources
release: etch
@@ -86,6 +96,8 @@
mipsel, powerpc, ppc64, s390, sparc]
directory: /org/volatile.debian.org/dists/etch/volatile/
components: [main, contrib, non-free]
+ packages-schema: packages
+ sources-schema: sources
packages-table: packages
sources-table: sources
distribution: debian-volatile
@@ -98,6 +110,8 @@
components: [main, multiverse, universe, restricted]
distribution: ubuntu
release: intrepid
+ packages-schema: packages
+ sources-schema: sources
packages-table: ubuntu_packages
sources-table: ubuntu_sources
update-command: ./ubuntu-rsync intrepid ./ubuntu
@@ -109,6 +123,8 @@
components: [main, multiverse, universe, restricted]
distribution: ubuntu
release: hardy
+ packages-schema: packages
+ sources-schema: sources
packages-table: ubuntu_packages
sources-table: ubuntu_sources
update-command: ./ubuntu-rsync hardy ./ubuntu
@@ -137,6 +153,7 @@
path: ./all-popcon-results.txt.gz
update-command: wget -O ./all-popcon-results.txt.gz http://popcon.debian.org/all-popcon-results.txt.gz
table: popcon
+ schema: popcon
packages-table: packages
ubuntu-popcon:
@@ -144,27 +161,43 @@
path: /tmp/udd-test/ubuntu-popcon-results.txt.gz
update-command: wget -O /tmp/udd-test/ubuntu-popcon-results.txt.gz http://popcon.ubuntu.com/all-popcon-results.txt.gz
table: ubuntu_popcon
+ schema: popcon
packages-table: ubuntu_packages
testing-migrations:
type: testing-migrations
path: /tmp/migrations
+ schema: testing_migrations
+ table: migrations
update-command: wget -O /tmp/migrations 'http://qa.debian.org/~lucas/testing-status.raw'
upload-history:
type: upload-history
path: /tmp/upload-history
- #update-command: if [ ! -e /tmp/upload-history/ ]; then mkdir /tmp/upload-history/; fi; rm -rf /tmp/upload-history/*; wget -r --no-parent -nd -A debian-devel-* -P /tmp/upload-history http://qa.debian.org/~filippo/ddc/
+ schema: upload_history
+ table: upload_history
update-command: if [ ! -e /tmp/upload-history/ ]; then mkdir /tmp/upload-history/; fi; rm -rf /tmp/upload-history/*; wget -r --no-parent -nd -A debian-devel-* -P /tmp/upload-history http://giunched.web.cs.unibo.it/nokeys/
bugs:
type: bugs
archived: false
+ table: bugs
+ schema: bugs
+ archived-table: archived_bugs
+ packages-table: packages
+ usertags-table: bugs_usertags
+ sources-table: sources
update-command: /org/udd.debian.net/mirrors/sync-bugs.debian.org
bugs-archive:
type: bugs
archived: true
+ table: bugs
+ schema: bugs
+ archived-table: archived_bugs
+ packages-table: packages
+ sources-table: sources
+ usertags-table: bugs_usertags
update-command: /org/udd.debian.net/mirrors/sync-bugs.debian.org
carnivore:
@@ -175,9 +208,11 @@
names-table: carnivore_names
keys-table: carnivore_keys
login-table: carnivore_login
+ schema: carnivore
lintian:
type: lintian
update-command: rm -f /org/udd.debian.net/mirrors/lintian.log && wget http://lintian.debian.org/lintian.log -O /org/udd.debian.net/mirrors/lintian.log
path: /org/udd.debian.net/mirrors/lintian.log
+ schema: lintian
table: lintian
Modified: udd/src/udd/bugs_gatherer.pl
===================================================================
--- udd/src/udd/bugs_gatherer.pl 2008-08-11 04:51:35 UTC (rev 1067)
+++ udd/src/udd/bugs_gatherer.pl 2008-08-11 13:33:24 UTC (rev 1068)
@@ -1,5 +1,5 @@
#!/usr/bin/perl
-# Last-Modified: <Sun Aug 10 10:31:57 2008>
+# Last-Modified: <Mon Aug 11 13:35:58 2008>
use strict;
use warnings;
@@ -10,6 +10,7 @@
use lib $Bin, qw{/org/udd.debian.net/mirrors/bugs.debian.org/perl};
use DBI;
+use DBI qw{:sql_types};
use YAML::Syck;
use Time::Local;
@@ -22,6 +23,10 @@
$YAML::Syck::ImplicitTyping = 1;
+#Used for measuring time
+my $t;
+my $timing = 1;
+
# Return the list of usernames
sub get_bugs_users {
my $topdir = "$gSpoolDir/user";
@@ -87,29 +92,43 @@
return (grep { ($h{$_}++ == 0) || 0 } @_);
}
-sub main {
- if(@ARGV != 2) {
- print STDERR "Usage: $0 <config> <source>";
- exit 1;
+sub setup {
+ my ($config, $source, $dbh) = @_;
+ my $schema = $config->{general}->{'schema-dir'} . '/' . $config->{$source}->{schema};
+ open SQL, "<", $schema or die $!;
+ my $command = join "", <SQL>;
+ close SQL;
+ $command =~ s/%\(([^)]+)\)s/$config->{$source}->{$1}/g;
+ $dbh->prepare($command)->execute() or die $!;
+}
+
+sub drop {
+ my ($config, $source, $dbh) = @_;
+ map {
+ $dbh->prepare("DROP VIEW $_")->execute() or die $!;
}
+ qw{bugs_rt_affects_stable bugs_rt_affects_testing_and_unstable bugs_rt_affects_unstable bugs_rt_affects_testing};
- my $config = LoadFile($ARGV[0]) or die "Could not load configuration: $!";
- my $source = $ARGV[1];
+ foreach my $prefix ($config->{$source}->{table}, $config->{$source}->{'archived-table'}) {
+ foreach my $postfix ('', qw{_merged_with _found_in _fixed_in _tags}) {
+ $dbh->prepare("DROP TABLE $prefix$postfix")->execute() or die $!;
+ }
+ }
+ $dbh->prepare("DROP TABLE " . $config->{$source}->{'usertags-table'})->execute() or die $!;
+}
+
+sub run_usertags {
+ my ($config, $source, $dbh) = @_;
my %src_config = %{$config->{$source}};
+ my $table = $src_config{'usertags-table'} or die "usertags-table not specified for source $source";
+ our $timing;
+ our $t;
- my $dbname = $config->{general}->{dbname};
- # Connection to DB
- my $dbh = DBI->connect("dbi:Pg:dbname=$dbname");
- # We want to commit the transaction as a hole at the end
- $dbh->{AutoCommit} = 0;
- my $timing = 1;
- my $t;
-
$t = time();
# Free usertags table
- $dbh->do("DELETE FROM bug_user_tags") or die
- "Couldn't empty bug_user_tags: $!";
+ $dbh->do("DELETE FROM $table") or die
+ "Couldn't empty $table: $!";
print "Deleting usertags: ",(time() - $t),"s\n" if $timing;
$t = time();
# read and insert user tags
@@ -121,12 +140,41 @@
$user = $dbh->quote($user);
foreach my $tag (keys %tags) {
my $qtag = $dbh->quote($tag);
- map { $dbh->do("INSERT INTO bug_user_tags VALUES ($user, $qtag, $_)") or die $! } @{$tags{$tag}};
+ map { $dbh->do("INSERT INTO $table VALUES ($user, $qtag, $_)") or die $! } @{$tags{$tag}};
}
}
+}
+
+sub run {
+ my ($config, $source, $dbh) = @_;
+
+ our $t;
+ our $timing;
print "Inserting usertags: ",(time() - $t),"s\n" if $timing;
$t = time();
+ run_usertags($config, $source, $dbh);
+
+ my %src_config = %{$config->{$source}};
+ my $table = $src_config{table};
+ my $archived_table = $src_config{'archived-table'};
+
my @modified_bugs;
+ ####### XXX EXPERIMENT
+ ####### XXX What to do with bugs both archived and unarchived
+ #my $max_last_modified = get_db_max_last_modified($dbh);
+ #my @modified_bugs;
+ #if($max_last_modified) {
+ # @modified_bugs = @{get_modified_bugs($max_last_modified)};
+ # Delete modified bugs
+ # for my $bug (@modified_bugs) {
+ # map {
+ # $dbh->prepare("DELETE FROM $_ WHERE id = $bug")->execute()
+ # } qw{bugs bug_merged_with bug_found_in bug_fixed_in};
+ # }
+ #} else {
+ # @modified_bugs = get_bugs(archive => 'both');
+ #}
+ #@modified_bugs = without_duplicates(@modified_bugs);
if($src_config{archived}) {
@modified_bugs = get_bugs(archive => 1);
} else {
@@ -144,10 +192,13 @@
print "Fetching list of ",scalar(@modified_bugs), " bugs to insert: ",(time() - $t),"s\n" if $timing;
$t = time();
- # Delete all bugs we are going to import
- my $modbugs = join(", ", @modified_bugs);
- for my $table qw{bugs_archived bugs bug_merged_with bug_found_in bug_fixed_in bug_tags} {
- $dbh->do("DELETE FROM $table WHERE id IN ($modbugs)") or die $!
+ foreach my $prefix ($table, $archived_table) {
+ foreach my $postfix ('', qw{_merged_with _found_in _fixed_in _tags}) {
+ my $sth = $dbh->prepare("DELETE FROM $prefix$postfix WHERE id = \$1");
+ map {
+ $sth->execute($_) or die $!;
+ } @modified_bugs;
+ }
}
print "Deleting bugs: ",(time() - $t),"s\n" if $timing;
$t = time();
@@ -156,8 +207,19 @@
my %binarytosource = ();
# XXX What if a bug is in location 'db' (which currently doesn't exist)
my $location = $src_config{archived} ? 'archive' : 'db_h';
- my $table = $src_config{archived} ? 'bugs_archived' : 'bugs';
+ #my $table = $src_config{archived} ? 'bugs_archived' : 'bugs';
+ $table = $src_config{archived} ? $archived_table : $table;
# Read all bugs
+ my $insert_bugs_handle = $dbh->prepare("INSERT INTO $table VALUES (\$1, \$2, \$3, \$4::abstime, \$5, \$6, \$7, \$8, \$9, \$10::abstime, \$11, \$12, \$13)");
+ my $insert_bugs_found_handle = $dbh->prepare("INSERT INTO ${table}_found_in VALUES (\$1, \$2)");
+ my $insert_bugs_fixed_handle = $dbh->prepare("INSERT INTO ${table}_fixed_in VALUES (\$1, \$2)");
+ my $insert_bugs_merged_handle = $dbh->prepare("INSERT INTO ${table}_merged_with VALUES (\$1, \$2)");
+ my $insert_bugs_tags_handle = $dbh->prepare("INSERT INTO ${table}_tags VALUES (\$1, \$2)");
+ $insert_bugs_handle->bind_param(4, undef, SQL_INTEGER);
+ $insert_bugs_handle->bind_param(10, undef, SQL_INTEGER);
+
+ print "Inserting bugs: ",(time() - $t),"s\n" if $timing;
+ $t = time();
foreach my $bug_nr (@modified_bugs) {
# Fetch bug using Debbugs
# Bugs which were once archived and have been unarchived again will appear in get_bugs(archive => 1).
@@ -176,9 +238,10 @@
# are assumed to be epoch (i.e. bug #4170)
map {
if($bug{$_}) {
- $bug{$_} = "$bug{$_}::abstime";
+ #$bug{$_} = "$bug{$_}::abstime";
+ $bug{$_} = int($bug{$_});
} else {
- $bug{$_} = '0::abstime';
+ $bug{$_} = 0;
}
} qw{date log_modified};
@@ -198,63 +261,88 @@
}
#Calculate bug presence in distributions
- my $present_in_stable =
- bug_presence(bug => $bug_nr, status => \%bug,
- dist => 'stable');
- my $present_in_testing =
- bug_presence(bug => $bug_nr, status => \%bug,
- dist => 'testing');
- my $present_in_unstable =
- bug_presence(bug => $bug_nr, status => \%bug,
- dist => 'unstable');
- if(!defined($present_in_stable) or !defined($present_in_unstable) or !defined($present_in_testing)) {
- print "NUMBER: $bug_nr\n";
- }
- if(defined($present_in_stable) and ($present_in_stable eq 'absent' or $present_in_stable eq 'fixed')) {
- $present_in_stable = 'FALSE';
+ my ($present_in_stable, $present_in_testing, $present_in_unstable);
+ if($src_config{archived}) {
+ $present_in_stable = $present_in_testing = $present_in_unstable = 'FALSE';
} else {
- $present_in_stable = 'TRUE';
+ $present_in_stable =
+ bug_presence(bug => $bug_nr, status => \%bug,
+ dist => 'stable');
+ $present_in_testing =
+ bug_presence(bug => $bug_nr, status => \%bug,
+ dist => 'testing');
+ $present_in_unstable =
+ bug_presence(bug => $bug_nr, status => \%bug,
+ dist => 'unstable');
+ if(!defined($present_in_stable) or !defined($present_in_unstable) or !defined($present_in_testing)) {
+ print "NUMBER: $bug_nr\n";
+ }
+
+ if(defined($present_in_stable) and ($present_in_stable eq 'absent' or $present_in_stable eq 'fixed')) {
+ $present_in_stable = 'FALSE';
+ } else {
+ $present_in_stable = 'TRUE';
+ }
+ if(defined($present_in_testing) and ($present_in_testing eq 'absent' or $present_in_testing eq 'fixed')) {
+ $present_in_testing = 'FALSE';
+ } else {
+ $present_in_testing = 'TRUE';
+ }
+ if(defined($present_in_unstable) and ($present_in_unstable eq 'absent' or $present_in_unstable eq 'fixed')) {
+ $present_in_unstable = 'FALSE';
+ } else {
+ $present_in_unstable = 'TRUE';
+ }
}
- if(defined($present_in_testing) and ($present_in_testing eq 'absent' or $present_in_testing eq 'fixed')) {
- $present_in_testing = 'FALSE';
- } else {
- $present_in_testing = 'TRUE';
- }
- if(defined($present_in_unstable) and ($present_in_unstable eq 'absent' or $present_in_unstable eq 'fixed')) {
- $present_in_unstable = 'FALSE';
- } else {
- $present_in_unstable = 'TRUE';
- }
# Insert data into bugs table
- my $query = "INSERT INTO $table VALUES ($bug_nr, '$bug{package}', $source, $bug{date}, \
- E$bug{pending}, '$bug{severity}', E$bug{originator}, E$bug{owner}, \
- E$bug{subject}, $bug{log_modified}, $present_in_stable,
- $present_in_testing, $present_in_unstable)";
- # Execute insertion
- $dbh->do($query) or die $!;
+ $insert_bugs_handle->execute($bug_nr, $bug{package}, $source, $bug{date}, $bug{pending},
+ $bug{severity}, $bug{originator}, $bug{owner}, $bug{subject}, $bug{log_modified},
+ $present_in_stable, $present_in_testing, $present_in_unstable) or die $!;
# insert data into bug_fixed_in and bug_found_in tables
foreach my $version (without_duplicates(@found_versions)) {
- $query = "INSERT INTO bug_found_in VALUES ($bug_nr, $version)";
- $dbh->do($query) or die $!;
+ $insert_bugs_found_handle->execute($bug_nr, $version) or die $!;
}
foreach my $version (without_duplicates(@fixed_versions)) {
- $query = "INSERT INTO bug_fixed_in VALUES ($bug_nr, $version)";
- $dbh->do($query) or die $!;
+ $insert_bugs_fixed_handle->execute($bug_nr, $version) or die $!;
}
foreach my $mergee (without_duplicates(split / /, $bug{mergedwith})) {
- $query = "INSERT INTO bug_merged_with VALUES ($bug_nr, $mergee)";
- $dbh->do($query) or die $!;
+ $insert_bugs_merged_handle->execute($bug_nr, $mergee) or die $!;
}
foreach my $tag (without_duplicates(@tags)) {
- $query = "INSERT INTO bug_tags VALUES ($bug_nr, $tag)";
- $dbh->do($query) or die $!;
+ $insert_bugs_tags_handle->execute($bug_nr, $tag) or die $!;
}
}
+}
- print "Inserting bugs: ",(time() - $t),"s\n" if $timing;
- $t = time();
+sub main {
+ if(@ARGV != 3) {
+ print STDERR "Usage: $0 <config> <command> <source>\n";
+ exit 1;
+ }
+
+ my $config = LoadFile($ARGV[0]) or die "Could not load configuration: $!";
+ my $command = $ARGV[1];
+ my $source = $ARGV[2];
+
+ my $dbname = $config->{general}->{dbname};
+ # Connection to DB
+ my $dbh = DBI->connect("dbi:Pg:dbname=$dbname");
+ # We want to commit the transaction as a hole at the end
+ $dbh->{AutoCommit} = 0;
+
+ if($command eq 'run') {
+ run($config, $source, $dbh);
+ } elsif ($command eq 'setup') {
+ setup($config, $source, $dbh);
+ } elsif ($command eq 'drop') {
+ drop($config, $source, $dbh);
+ } else {
+ print STDERR "<command> has to be one of run, drop and setup\n";
+ exit(1)
+ }
+
$dbh->commit();
print "Committing bugs: ",(time() - $t),"s\n" if $timing;
}
Modified: udd/src/udd/carnivore_gatherer.py
===================================================================
--- udd/src/udd/carnivore_gatherer.py 2008-08-11 04:51:35 UTC (rev 1067)
+++ udd/src/udd/carnivore_gatherer.py 2008-08-11 13:33:24 UTC (rev 1068)
@@ -11,8 +11,8 @@
from gatherer import gatherer
import re
-def get_gatherer(connection, config):
- return carnivore_gatherer(connection, config)
+def get_gatherer(connection, config, source):
+ return carnivore_gatherer(connection, config, source)
class carnivore_gatherer(gatherer):
field_ignores = ["Packages", "X-MIA", "X-Warning"]
@@ -26,19 +26,17 @@
"Key in removed": {"name": "removed_key", "content-type": "multiple entries"},
}
- def __init__(self, connection, config):
- gatherer.__init__(self, connection, config)
+ def __init__(self, connection, config, source):
+ gatherer.__init__(self, connection, config, source)
+ self.assert_my_config('path', 'emails-table', 'names-table', 'keys-table', 'login-table')
- def run(self, source):
- try:
- my_config = self.config[source]
- except:
- raise
+ def drop(self):
+ cur = self.cursor()
+ for table in ['emails', 'names', 'keys', 'login']:
+ cur.execute("DROP TABLE %s" % self.my_config["%s-table" % table])
- #check that the config contains everything we need:
- for key in ['path', 'emails-table', 'names-table', 'keys-table', 'login-table']:
- if not key in my_config:
- raise aux.ConfigException, "%s not configured for source %s" % (key, source)
+ def run(self):
+ my_config = self.my_config
#start harassing the DB, preparing the final inserts and making place
#for the new data:
Modified: udd/src/udd/gatherer.py
===================================================================
--- udd/src/udd/gatherer.py 2008-08-11 04:51:35 UTC (rev 1067)
+++ udd/src/udd/gatherer.py 2008-08-11 13:33:24 UTC (rev 1068)
@@ -1,4 +1,7 @@
# This file is part of the Ultimate Debian Database project
+
+import aux
+
class gatherer:
"""
This is the base class of all gatherers which want to use the python
@@ -7,14 +10,47 @@
Attributes:
connection: The connection to the SQL database
config: The hashmap representing the configuration"""
- def __init__(self, connection, config):
+ def __init__(self, connection, config, source):
self.connection = connection
self.config = config
+ self.source = source
+ self.my_config = config[source]
- def run(self, source):
+ def run(self):
"""Called by the dispatcher for a source"""
raise NotImplementedError
def cursor(self):
"""Return the cursor for the current connection"""
return self.connection.cursor()
+
+ def setup(self):
+ if 'schema-dir' in self.config['general']:
+ schema_dir = self.config['general']['schema-dir']
+ if 'schema' in self.my_config:
+ schema = schema_dir + '/' + self.my_config['schema']
+ self.eval_sql_file(schema, self.my_config)
+ else:
+ raise Exception("'schema' not specified for source " + self.source)
+ else:
+ raise Exception("'schema-dir' not specified")
+
+ def drop(self):
+ if 'table' in self.my_config:
+ self.cursor().execute("DROP TABLE " + self.my_config['table'])
+
+ def eval_sql_file(self, path, d = None):
+ """Load the SQL code from the file specified by <path>. Use pythons string
+ formating for the dictionary <d> if it is not None
+ Warning: No quoting for the elements of d is done"""
+ c = file(path).read()
+ if d is not None:
+ c = c % d
+
+ cur = self.cursor()
+ cur.execute(c)
+
+ def assert_my_config(self, *keywords):
+ for k in keywords:
+ if not k in self.my_config:
+ raise aux.ConfigException("%s not specified for source %s" % (k, self.source))
Modified: udd/src/udd/lintian_gatherer.py
===================================================================
--- udd/src/udd/lintian_gatherer.py 2008-08-11 04:51:35 UTC (rev 1067)
+++ udd/src/udd/lintian_gatherer.py 2008-08-11 13:33:24 UTC (rev 1068)
@@ -9,8 +9,8 @@
from gatherer import gatherer
import re
-def get_gatherer(connection, config):
- return lintian_gatherer(connection, config)
+def get_gatherer(connection, config, source):
+ return lintian_gatherer(connection, config, source)
class lintian_gatherer(gatherer):
#RE to parse lintian output, pushing the tag code to $1, package name
@@ -26,20 +26,13 @@
"O": "overriden",
}
- def __init__(self, connection, config):
- gatherer.__init__(self, connection, config)
+ def __init__(self, connection, config, source):
+ gatherer.__init__(self, connection, config, source)
+ self.assert_my_config('path', 'table')
- def run(self, source):
- try:
- my_config = self.config[source]
- except:
- raise
+ def run(self):
+ my_config = self.my_config
- #check that the config contains everything we need:
- for key in ['path', 'table']:
- if not key in my_config:
- raise aux.ConfigException, "%s not configured for source %s" % (key, source)
-
#start harassing the DB, preparing the final inserts and making place
#for the new data:
cur = self.cursor()
Modified: udd/src/udd/orphaned_packages_gatherer.py
===================================================================
--- udd/src/udd/orphaned_packages_gatherer.py 2008-08-11 04:51:35 UTC (rev 1067)
+++ udd/src/udd/orphaned_packages_gatherer.py 2008-08-11 13:33:24 UTC (rev 1068)
@@ -8,12 +8,13 @@
from gatherer import gatherer
import re
-def get_gatherer(connection, config):
- return orphaned_packages_gatherer(connection, config)
+def get_gatherer(connection, config, source):
+ return orphaned_packages_gatherer(connection, config, source)
class orphaned_packages_gatherer(gatherer):
- def __init__(self, connection, config):
- gatherer.__init__(self, connection, config)
+ def __init__(self, connection, config, source):
+ gatherer.__init__(self, connection, config, source)
+ self.assert_my_config('bugs-path', 'table', 'unarchived-table')
title_re = re.compile('^(ITA|RFA|O): ([^\s]*) [-]+ (.*)$')
otime_re = re.compile('^<!-- time:([0-9]+) ')
@@ -33,8 +34,7 @@
otime = int(m.group(1))
return None
- def run(self, source):
- self.my_config = self.config[source]
+ def run(self):
#check that the config contains everything we need:
for key in ['bugs-path']:
if not key in self.my_config:
@@ -44,11 +44,11 @@
#for the new data:
cur = self.cursor()
cur2 = self.cursor()
- cur.execute("SELECT id, title, arrival FROM bugs WHERE package = 'wnpp' AND status != 'done' AND title ~* '^(ITA|RFA|O):' AND id NOT IN (SELECT id from bug_merged_with WHERE id > merged_with)")
+ cur.execute("SELECT id, title, arrival FROM %s WHERE package = 'wnpp' AND status != 'done' AND title ~* '^(ITA|RFA|O):' AND id NOT IN (SELECT id from bug_merged_with WHERE id > merged_with)" % self.my_config['unarchived-table'])
rows = cur.fetchall()
- cur2.execute("DELETE FROM orphaned_packages")
- cur2.execute("PREPARE opkgs_insert AS INSERT INTO orphaned_packages VALUES ($1, $2, $3, $4, $5)")
+ cur2.execute("DELETE FROM %s" % self.my_config['table'])
+ cur2.execute("PREPARE opkgs_insert AS INSERT INTO %s VALUES ($1, $2, $3, $4, $5)" % self.my_config['table'])
for row in rows:
m = self.title_re.match(row[1])
Modified: udd/src/udd/packages_gatherer.py
===================================================================
--- udd/src/udd/packages_gatherer.py 2008-08-11 04:51:35 UTC (rev 1067)
+++ udd/src/udd/packages_gatherer.py 2008-08-11 13:33:24 UTC (rev 1068)
@@ -1,5 +1,5 @@
# /usr/bin/env python
-# Last-Modified: <Sat Aug 9 17:28:45 2008>
+# Last-Modified: <Sun Aug 10 12:11:38 2008>
# This file is a part of the Ultimate Debian Database project
import debian_bundle.deb822
@@ -12,8 +12,8 @@
import psycopg2
from gatherer import gatherer
-def get_gatherer(connection, config):
- return packages_gatherer(connection, config)
+def get_gatherer(connection, config, source):
+ return packages_gatherer(connection, config, source)
class packages_gatherer(gatherer):
"This class imports the data from Packages.gz files into the database"
@@ -40,10 +40,11 @@
# with the same version, and we don't want these duplicate entries
imported_all_pkgs = {}
- def __init__(self, connection, config):
- gatherer.__init__(self, connection, config)
+ def __init__(self, connection, config, source):
+ gatherer.__init__(self, connection, config, source)
# The ID for the distribution we want to include
self._distr = None
+ self.assert_my_config('directory', 'archs', 'release', 'components', 'distribution', 'packages-table', 'packages-schema')
def build_dict(self, control):
"""Build a dictionary from the control dictionary.
@@ -123,15 +124,23 @@
print query
raise
- def run(self, source):
- if not source in self.config:
- raise ConfigException, "Source %s not specified" %(source)
- src_cfg = self.config[source]
+ def setup(self):
+ if 'schema-dir' in self.config['general']:
+ schema_dir = self.config['general']['schema-dir']
+ if 'packages-schema' in self.my_config:
+ schema = schema_dir + '/' + self.my_config['packages-schema']
+ self.eval_sql_file(schema, self.my_config)
+ else:
+ raise Exception("'packages-schema' not specified for source " + self.source)
+ else:
+ raise Exception("'schema-dir' not specified")
- for k in ['directory', 'archs', 'release', 'components', 'distribution', 'packages-table']:
- if not k in src_cfg:
- raise ConfigException(k + ' not sepcified for source ' + source)
+ def drop(self):
+ self.cursor().execute("DROP TABLE " + self.my_config['packages-table'])
+ def run(self):
+ src_cfg = self.my_config
+
aux.debug = self.config['general']['debug']
table = src_cfg['packages-table']
Modified: udd/src/udd/popcon_gatherer.py
===================================================================
--- udd/src/udd/popcon_gatherer.py 2008-08-11 04:51:35 UTC (rev 1067)
+++ udd/src/udd/popcon_gatherer.py 2008-08-11 13:33:24 UTC (rev 1068)
@@ -11,23 +11,23 @@
from gatherer import gatherer
import re
-def get_gatherer(connection, config):
- return popcon_gatherer(connection, config)
+def get_gatherer(connection, config, source):
+ return popcon_gatherer(connection, config, source)
class popcon_gatherer(gatherer):
- def __init__(self, connection, config):
- gatherer.__init__(self, connection, config)
+ def __init__(self, connection, config, source):
+ gatherer.__init__(self, connection, config, source)
- def run(self, source):
- try:
- my_config = self.config[source]
- except:
- raise
+ self.assert_my_config('path', 'table', 'packages-table', 'schema')
- for k in ['path', 'table', 'packages-table']:
- if k not in my_config:
- raise aux.ConfigException(k + ' not specified in ' + source)
+ def drop(self):
+ cur = self.cursor()
+ for sub in ('', '_src', '_src_average'):
+ cur.execute("DROP TABLE %s%s" % (self.my_config['table'], sub))
+ def run(self):
+ my_config = self.my_config
+
table = my_config['table']
table_src = table + "_src"
table_src_average = table + "_src_average"
Modified: udd/src/udd/sources_gatherer.py
===================================================================
--- udd/src/udd/sources_gatherer.py 2008-08-11 04:51:35 UTC (rev 1067)
+++ udd/src/udd/sources_gatherer.py 2008-08-11 13:33:24 UTC (rev 1068)
@@ -1,5 +1,5 @@
#/usr/bin/env python
-# Last-Modified: <Sat Aug 9 17:28:33 2008>
+# Last-Modified: <Sun Aug 10 12:11:44 2008>
# This file is a part of the Ultimate Debian Database project
import debian_bundle.deb822
@@ -12,8 +12,8 @@
from aux import null_or_quote, quote
from gatherer import gatherer
-def get_gatherer(connection, config):
- return sources_gatherer(connection, config)
+def get_gatherer(connection, config, source):
+ return sources_gatherer(connection, config, source)
class sources_gatherer(gatherer):
"This class imports the data from Sources.gz files into the database"
@@ -31,9 +31,10 @@
warned_about = {}
- def __init__(self, connection, config):
- gatherer.__init__(self, connection, config)
+ def __init__(self, connection, config, source):
+ gatherer.__init__(self, connection, config, source)
self._distr = None
+ self.assert_my_config('directory', 'components', 'distribution', 'release', 'sources-table', 'sources-schema')
def build_dict(self, control):
"""Build a dictionary from the control dictionary.
@@ -96,17 +97,13 @@
"""
cur.execute(query, d)
- def run(self, source):
- if not source in self.config:
- raise ConfigException, "Source %s not specified" %(src_name)
- src_cfg = self.config[source]
+ def drop(self):
+ self.cursor().execute("DROP TABLE " + self.my_config['sources-table'])
- for k in ['directory', 'components', 'distribution', 'release', 'sources-table']:
- if not k in src_cfg:
- raise ConfigException(k + ' not specified for source ' + source)
-
+ def run(self):
+ src_cfg = self.my_config
+
table = src_cfg['sources-table']
-
aux.debug = self.config['general']['debug']
@@ -148,6 +145,17 @@
self.print_warnings()
+ def setup(self):
+ if 'schema-dir' in self.config['general']:
+ schema_dir = self.config['general']['schema-dir']
+ if 'sources-schema' in self.my_config:
+ schema = schema_dir + '/' + self.my_config['sources-schema']
+ self.eval_sql_file(schema, self.my_config)
+ else:
+ raise Exception("'packages-schema' not specified for source " + self.source)
+ else:
+ raise Exception("'schema-dir' not specified")
+
def print_warnings(self):
for key in sources_gatherer.warned_about:
print "Unknown key %s appeared %d times" % (key, sources_gatherer.warned_about[key])
Modified: udd/src/udd/src_and_pkg_gatherer.py
===================================================================
--- udd/src/udd/src_and_pkg_gatherer.py 2008-08-11 04:51:35 UTC (rev 1067)
+++ udd/src/udd/src_and_pkg_gatherer.py 2008-08-11 13:33:24 UTC (rev 1068)
@@ -3,15 +3,23 @@
import packages_gatherer
-def get_gatherer(connection, config):
- return src_and_pkg_gatherer(connection, config)
+def get_gatherer(connection, config, source):
+ return src_and_pkg_gatherer(connection, config, source)
class src_and_pkg_gatherer(gatherer.gatherer):
- def __init__(self, connection, config):
- gatherer.gatherer.__init__(self, connection, config)
- self.src = sources_gatherer.sources_gatherer(connection, config)
- self.pkg = packages_gatherer.packages_gatherer(connection, config)
+ def __init__(self, connection, config, source):
+ gatherer.gatherer.__init__(self, connection, config, source)
+ self.src = sources_gatherer.sources_gatherer(connection, config, source)
+ self.pkg = packages_gatherer.packages_gatherer(connection, config, source)
- def run(self, source):
- self.src.run(source)
- self.pkg.run(source)
+ def run(self):
+ self.src.run()
+ self.pkg.run()
+
+ def drop(self):
+ self.src.drop()
+ self.pkg.drop()
+
+ def setup(self):
+ self.src.setup()
+ self.pkg.setup()
Modified: udd/src/udd/testing_migrations_gatherer.py
===================================================================
--- udd/src/udd/testing_migrations_gatherer.py 2008-08-11 04:51:35 UTC (rev 1067)
+++ udd/src/udd/testing_migrations_gatherer.py 2008-08-11 13:33:24 UTC (rev 1068)
@@ -1,4 +1,4 @@
-# Last-Modified: <Sat 09 Aug 2008 18:19:40 CEST>
+# Last-Modified: <Sun Aug 10 12:16:12 2008>
# This file is a part of the Ultimate Debian Database Project
@@ -8,26 +8,21 @@
ZERO_DATE = '0000-01-01'
-def get_gatherer(config, connection):
- return testing_migrations_gatherer(config, connection)
+def get_gatherer(config, connection, source):
+ return testing_migrations_gatherer(config, connection, source)
class testing_migrations_gatherer(gatherer):
"""This class imports testing migrations data into the database.
For the files, see http://qa.debian.org/~lucas/testing-status.raw"""
- def __init__(self, connection, config):
- gatherer.__init__(self, connection, config)
+ def __init__(self, connection, config, source):
+ gatherer.__init__(self, connection, config, source)
+ self.assert_my_config('path')
- def run(self, source):
- if not source in self.config:
- raise ConfigException('Source %s was not specified' % source)
+ def run(self):
+ src_cfg = self.my_config
- src_cfg = self.config[source]
-
- if not 'path' in src_cfg:
- raise ConfigException('path not specified for source %s' % source)
-
c = self.connection.cursor()
c.execute("DELETE FROM migrations")
Modified: udd/src/udd/upload_history_gatherer.py
===================================================================
--- udd/src/udd/upload_history_gatherer.py 2008-08-11 04:51:35 UTC (rev 1067)
+++ udd/src/udd/upload_history_gatherer.py 2008-08-11 13:33:24 UTC (rev 1068)
@@ -1,4 +1,4 @@
-# Last-Modified: <Sat 09 Aug 2008 18:32:29 CEST>
+# Last-Modified: <Sun Aug 10 15:21:06 2008>
# This file is part of the Ultimate Debian Database Project
from gatherer import gatherer
@@ -8,29 +8,24 @@
import psycopg2
import sys
-date_translation = {
- 'Deb': 'Feb',
- 'Augl': 'Aug',
- 'Fev': 'Feb' }
+def get_gatherer(config, connection, source):
+ return upload_history_gatherer(config, connection, source)
-def get_gatherer(config, connection):
- return upload_history_gatherer(config, connection)
-
class upload_history_gatherer(gatherer):
- def __init__(self, connection, config):
- gatherer.__init__(self, connection, config)
-
- def run(self, source):
- if not 'path' in self.config[source]:
+ def __init__(self, connection, config, source):
+ gatherer.__init__(self, connection, config, source)
+ if not 'path' in self.my_config:
raise aux.ConfigException('path not specified for source ' + source)
- path = self.config[source]['path']
+ def run(self):
+ path = self.my_config['path']
+
cursor = self.cursor()
- cursor.execute("DELETE FROM upload_history")
+ cursor.execute("DELETE FROM " + self.my_config['table'])
- cursor.execute("PREPARE uh_insert AS INSERT INTO upload_history VALUES \
- ($1, $2, $3, $4, $5, $6, $7, $8)")
+ cursor.execute("PREPARE uh_insert AS INSERT INTO %s VALUES \
+ ($1, $2, $3, $4, $5, $6, $7, $8)" % self.my_config['table'])
for name in glob(path + '/debian-devel-*'):
print name
Modified: udd/src/udd-dispatch.py
===================================================================
--- udd/src/udd-dispatch.py 2008-08-11 04:51:35 UTC (rev 1067)
+++ udd/src/udd-dispatch.py 2008-08-11 13:33:24 UTC (rev 1068)
@@ -1,5 +1,5 @@
#!/usr/bin/env python
-# Last-Modified: <Sun Jun 29 11:47:21 2008>
+# Last-Modified: <Sun Aug 10 11:44:14 2008>
"""Dispatch udd gatherers
@@ -12,26 +12,31 @@
import os.path
def print_help():
- print "Usage: " + sys.argv[0] + " <configuration> <source1> [source2 source3 ...]"
+ print "Usage: " + sys.argv[0] + " <configuration> <command> <source1> [source2 source3 ...]"
if __name__ == '__main__':
- if len(sys.argv) < 3:
+ if len(sys.argv) < 4:
print_help()
sys.exit(1)
+ command = sys.argv[2]
+ if command not in ('run', 'setup', 'drop'):
+ sys.stderr.write("command has to be one of 'run', 'setup', 'drop'\n")
+ sys.ext(1)
+
# Check the configuration
config = udd.aux.load_config(open(sys.argv[1]).read())
types = config['general']['types']
- for src in sys.argv[2:]:
+ for src in sys.argv[3:]:
if not src in config:
raise udd.aux.ConfigException("%s is not specified in %s" % (src, sys.argv[1]))
connection = udd.aux.open_connection(config)
# Process the sources
- for src in sys.argv[2:]:
+ for src in sys.argv[3:]:
src_config = config[src]
type = src_config['type']
if not type in types:
@@ -39,17 +44,16 @@
sys.exit(1)
- (command,rest) = types[type].split(None, 1)
+ (src_command,rest) = types[type].split(None, 1)
- if command == "exec":
- system(rest + " " + sys.argv[1] + " " + src)
- elif command == "module":
+ if src_command == "exec":
+ system(rest + " " + sys.argv[1] + " " + sys.argv[2] + " " + src)
+ elif src_command == "module":
exec("import " + rest)
- exec "gatherer = " + rest + ".get_gatherer(connection, config)"
- gatherer.run(src)
- connection.commit()
+ exec "gatherer = " + rest + ".get_gatherer(connection, config, src)"
+ exec "gatherer.%s()" % command
if 'timestamp-folder' in config['general']:
f = open(os.path.join(config['general']['timestamp-folder'], src+".dispatch"), "w")
f.write(asctime())
f.close()
-
+ connection.commit()
More information about the Collab-qa-commits
mailing list