[Collab-qa-commits] r998 - in udd/src: . udd
neronus-guest at alioth.debian.org
neronus-guest at alioth.debian.org
Fri Aug 8 13:51:14 UTC 2008
Author: neronus-guest
Date: 2008-08-08 13:51:14 +0000 (Fri, 08 Aug 2008)
New Revision: 998
Modified:
udd/src/db_manager.py
udd/src/setup-db.sql
udd/src/test.yaml
udd/src/udd/packages_gatherer.py
udd/src/udd/popcon_gatherer.py
udd/src/udd/sources_gatherer.py
Log:
debian und ubuntu each get a table of its own for packages, sources and popcon
made table names configurable for the above tables
Modified: udd/src/db_manager.py
===================================================================
--- udd/src/db_manager.py 2008-08-08 13:35:09 UTC (rev 997)
+++ udd/src/db_manager.py 2008-08-08 13:51:14 UTC (rev 998)
@@ -6,10 +6,12 @@
"""This scripts sets up and deletes the tables of the database"""
-TABLES = ('sources', 'packages', 'popcon', 'migrations', 'bugs_archived',
- 'bugs_unarchived', 'bug_merged_with', 'bug_user_tags', 'bug_found_in',
- 'bug_fixed_in', 'bug_tags', 'upload_history')
-VIEWS = ('popcon_src_average', 'popcon_src_max', 'bugs')
+TABLES = ('popcon_src_average', 'popcon_src_max', 'popcon',
+ 'ubuntu_popcon_src_average', 'ubuntu_popcon_src_max', 'ubuntu_popcon',
+ 'sources', 'packages', 'ubuntu_sources', 'ubuntu_packages', 'migrations',
+ 'bugs_archived', 'bugs_unarchived', 'bug_merged_with', 'bug_user_tags',
+ 'bug_found_in', 'bug_fixed_in', 'bug_tags', 'upload_history')
+VIEWS = ('bugs', )
def print_help():
print "Usage: %s <config> <delete|setup>" % sys.argv[0]
Modified: udd/src/setup-db.sql
===================================================================
--- udd/src/setup-db.sql 2008-08-08 13:35:09 UTC (rev 997)
+++ udd/src/setup-db.sql 2008-08-08 13:51:14 UTC (rev 998)
@@ -6,6 +6,14 @@
text, Release text, Component text,
PRIMARY KEY (Package, Version, Architecture, Distribution, Release, Component));
+CREATE TABLE ubuntu_packages
+ (Package text, Version text, Architecture text, Maintainer text, Description
+ text, Source text, Source_version text, Essential text, Depends text,
+ Recommends text, Suggests text, Enhances text, Pre_Depends text,
+ Installed_Size int, Homepage text, Size int, MD5Sum text, Distribution
+ text, Release text, Component text,
+ PRIMARY KEY (Package, Version, Architecture, Distribution, Release, Component));
+
CREATE TABLE sources
(Package text, Version text, Maintainer text, Format text, Files text,
Uploaders text, Bin text, Architecture text, Standards_Version text,
@@ -17,14 +25,44 @@
text, X_Vcs_Svn text,
PRIMARY KEY (package, version, distribution, release, component));
+CREATE TABLE ubuntu_sources
+ (Package text, Version text, Maintainer text, Format text, Files text,
+ Uploaders text, Bin text, Architecture text, Standards_Version text,
+ Homepage text, Build_Depends text, Build_Depends_Indep text,
+ Build_Conflicts text, Build_Conflicts_Indep text, Priority text, Section
+ text, Distribution text, Release text, Component text, Vcs_Arch text,
+ Vcs_Browser text, Vcs_Bzr text, Vcs_Cvs text, Vcs_Darcs text, Vcs_Git text,
+ Vcs_Hg text, Vcs_Svn text, X_Vcs_Browser text, X_Vcs_Bzr text, X_Vcs_Darcs
+ text, X_Vcs_Svn text,
+ PRIMARY KEY (package, version, distribution, release, component));
+
CREATE TABLE migrations
(source text PRIMARY KEY, in_testing date, testing_version text, in_unstable date, unstable_version text, sync date, sync_version text, first_seen date);
CREATE TABLE popcon (
package text, insts int, vote int, olde int, recent int, nofiles int,
- distribution text,
- PRIMARY KEY (package, distribution));
+ PRIMARY KEY (package));
+
+CREATE TABLE popcon_src_max (
+ package text, insts int, vote int, olde int, recent int, nofiles int,
+ PRIMARY KEY (package));
+CREATE TABLE popcon_src_average (
+ package text, insts int, vote int, olde int, recent int, nofiles int,
+ PRIMARY KEY (package));
+
+CREATE TABLE ubuntu_popcon (
+ package text, insts int, vote int, olde int, recent int, nofiles int,
+ PRIMARY KEY (package));
+
+CREATE TABLE ubuntu_popcon_src_max (
+ package text, insts int, vote int, olde int, recent int, nofiles int,
+ PRIMARY KEY (package));
+
+CREATE TABLE ubuntu_popcon_src_average (
+ package text, insts int, vote int, olde int, recent int, nofiles int,
+ PRIMARY KEY (package));
+
CREATE TABLE bugs_unarchived
(id int PRIMARY KEY, package text, source text, arrival timestamp, status text,
severity text, submitter text, owner text, title text,
@@ -69,24 +107,6 @@
(package text, version text, date timestamp with time zone, changed_by text,
maintainer text, nmu boolean, signed_by text, key_id text);
-CREATE VIEW popcon_src_average AS
- SELECT packages.source, avg(insts) AS insts, avg(vote) AS vote, avg(olde) AS old, avg(recent) AS recent, avg(nofiles) as nofiles, packages.distribution
- FROM popcon,
- (SELECT DISTINCT packages.package, packages.source, packages.distribution FROM packages) as packages
- WHERE
- popcon.package = packages.package AND
- popcon.distribution = packages.distribution
- GROUP BY packages.source, packages.distribution;
-
-CREATE VIEW popcon_src_max AS
- SELECT packages.source, max(insts) AS insts, max(vote) AS vote, max(olde) AS old, max(recent) AS recent, max(nofiles) as nofiles, packages.distribution
- FROM popcon,
- (SELECT DISTINCT packages.package, packages.source, packages.distribution FROM packages) as packages
- WHERE
- popcon.package = packages.package AND
- popcon.distribution = packages.distribution
- GROUP BY packages.source, packages.distribution;
-
CREATE INDEX packages_source_idx on packages(source);
CREATE INDEX sources_distribution_idx on sources(distribution);
CREATE INDEX sources_release_idx on sources(release);
Modified: udd/src/test.yaml
===================================================================
--- udd/src/test.yaml 2008-08-08 13:35:09 UTC (rev 997)
+++ udd/src/test.yaml 2008-08-08 13:51:14 UTC (rev 998)
@@ -36,6 +36,8 @@
mipsel, powerpc, s390, sparc]
directory: /org/ftp.debian.org/dists/lenny/
components: [main, contrib, non-free]
+ packages-table: packages
+ sources-table: sources
distribution: debian
release: lenny
@@ -46,6 +48,8 @@
mipsel, powerpc, ppc64, s390, sparc]
directory: /org/ftp.debian.org/dists/sid/
components: [main, contrib, non-free]
+ packages-table: packages
+ sources-table: sources
distribution: debian
release: sid
@@ -57,6 +61,8 @@
directory: /org/ftp.debian.org/dists/etch/
components: [main, contrib, non-free]
distribution: debian
+ packages-table: packages
+ sources-table: sources
release: etch
debian-backports-etch:
@@ -67,6 +73,8 @@
directory: /org/ftp.backports.org/dists/etch-backports/
components: [main, contrib, non-free]
distribution: debian-backports
+ packages-table: packages
+ sources-table: sources
release: etch
debian-volatile-etch:
@@ -76,6 +84,8 @@
mipsel, powerpc, ppc64, s390, sparc]
directory: /org/volatile.debian.org/dists/etch/volatile/
components: [main, contrib, non-free]
+ packages-table: packages
+ sources-table: sources
distribution: debian-volatile
release: etch
@@ -86,6 +96,8 @@
components: [main, multiverse, universe, restricted]
distribution: ubuntu
release: intrepid
+ packages-table: ubuntu_packages
+ sources-table: ubuntu_sources
update-command: ./ubuntu-rsync intrepid ./ubuntu
ubuntu-hardy:
@@ -95,6 +107,8 @@
components: [main, multiverse, universe, restricted]
distribution: ubuntu
release: hardy
+ packages-table: ubuntu_packages
+ sources-table: ubuntu_sources
update-command: ./ubuntu-rsync hardy ./ubuntu
@@ -120,13 +134,15 @@
type: popcon
path: ./all-popcon-results.txt.gz
update-command: wget -O ./all-popcon-results.txt.gz http://popcon.debian.org/all-popcon-results.txt.gz
- distribution: debian
+ table: popcon
+ packages-table: packages
ubuntu-popcon:
type: popcon
path: /tmp/udd-test/ubuntu-popcon-results.txt.gz
update-command: wget -O /tmp/udd-test/ubuntu-popcon-results.txt.gz http://popcon.ubuntu.com/all-popcon-results.txt.gz
- distribution: ubuntu
+ table: ubuntu_popcon
+ packages-table: ubuntu_packages
testing-migrations:
type: testing-migrations
Modified: udd/src/udd/packages_gatherer.py
===================================================================
--- udd/src/udd/packages_gatherer.py 2008-08-08 13:35:09 UTC (rev 997)
+++ udd/src/udd/packages_gatherer.py 2008-08-08 13:51:14 UTC (rev 998)
@@ -1,5 +1,5 @@
# /usr/bin/env python
-# Last-Modified: <Sat Jul 26 12:58:43 2008>
+# Last-Modified: <Fri Aug 8 12:42:28 2008>
# This file is a part of the Ultimate Debian Database project
import debian_bundle.deb822
@@ -144,7 +144,11 @@
raise ConfigException('distribution not specified for source %s' %
(source))
+ if not 'packages-table' in src_cfg:
+ raise ConfigException('packages-table not specified for source %s' % (source))
+
aux.debug = self.config['general']['debug']
+ table = src_cfg['packages-table']
# Get distribution ID
self._distr = src_cfg['distribution']
@@ -153,12 +157,12 @@
# For every part and every architecture, import the packages into the DB
for comp in src_cfg['components']:
- cur.execute("DELETE FROM packages WHERE distribution = '%s' AND release = '%s' AND component = '%s'" %\
- (self._distr, src_cfg['release'], comp))
+ cur.execute("DELETE FROM %s WHERE distribution = '%s' AND release = '%s' AND component = '%s'" %\
+ (table, self._distr, src_cfg['release'], comp))
for arch in src_cfg['archs']:
path = os.path.join(src_cfg['directory'], comp, 'binary-' + arch, 'Packages.gz')
try:
- cur.execute("""PREPARE package_insert AS INSERT INTO Packages
+ cur.execute("""PREPARE package_insert AS INSERT INTO %s
(Package, Version, Architecture, Maintainer, Description, Source,
Source_Version, Essential, Depends, Recommends, Suggests, Enhances,
Pre_Depends, Installed_Size, Homepage, Size, MD5Sum, Distribution,
@@ -166,7 +170,7 @@
VALUES
( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15,
$16, $17, '%s', '%s', '%s')
- """ % (self._distr, src_cfg['release'], comp))
+ """ % (table, self._distr, src_cfg['release'], comp))
aux.print_debug("Reading file " + path)
# Copy content from gzipped file to temporary file, so that apt_pkg is
# used by debian_bundle
Modified: udd/src/udd/popcon_gatherer.py
===================================================================
--- udd/src/udd/popcon_gatherer.py 2008-08-08 13:35:09 UTC (rev 997)
+++ udd/src/udd/popcon_gatherer.py 2008-08-08 13:51:14 UTC (rev 998)
@@ -27,16 +27,25 @@
if not 'path' in my_config:
raise aux.ConfigException, "path not configured for source " + source
- if not 'distribution' in my_config:
- raise aux.ConfigException, "distribution not configured for source " + source
+ if not 'table' in my_config:
+ raise aux.ConfigException, "table not configured for source " + source
+ if not 'packages-table' in my_config:
+ raise aux.ConfigException, "packages-table not configured for source " + source
+
+ table = my_config['table']
+ table_src_max = table + "_src_max"
+ table_src_average = table + "_src_average"
+
cur = self.cursor()
- cur.execute("PREPARE pop_insert AS INSERT INTO popcon (package, insts, vote, olde, recent, nofiles, distribution) VALUES ($1, $2, $3, $4, $5, $6, '%s')" % my_config['distribution'])
+ cur.execute("PREPARE pop_insert AS INSERT INTO %s (package, insts, vote, olde, recent, nofiles) VALUES ($1, $2, $3, $4, $5, $6)" % (table))
popcon = gzip.open(my_config['path'])
- cur.execute("DELETE FROM popcon WHERE distribution = '%s'" % my_config['distribution'])
+ cur.execute("DELETE FROM " + table)
+ cur.execute("DELETE FROM " + table_src_max)
+ cur.execute("DELETE FROM " + table_src_average)
# used for ignoring ubuntu's broken popcon lines
ascii_match = re.compile("^[A-Za-z0-9-.+_]+$")
@@ -46,7 +55,7 @@
linenr += 1
name, data = line.split(None, 1)
if name == "Submissions:":
- cur.execute("INSERT INTO popcon (package, vote, distribution) VALUES ('_submissions', %s, '%s')" % (data, my_config['distribution']))
+ cur.execute("INSERT INTO %s (package, vote) VALUES ('_submissions', %s)" % (table, data))
try:
(name, vote, old, recent, nofiles) = data.split()
if ascii_match.match(name) == None:
@@ -60,5 +69,35 @@
cur.execute("DEALLOCATE pop_insert")
+ #calculate _src_max and _src_avg
+ cur.execute("PREPARE pop_insert AS INSERT INTO %s VALUES ($1, $2, $3, $4, $5, $6)" % table_src_max)
+ cur.execute("""
+ SELECT packages.source, max(insts) AS insts, max(vote) AS vote, max(olde) AS old,
+ max(recent) AS recent, max(nofiles) as nofiles
+ FROM %(table)s,
+ (SELECT DISTINCT %(packages-table)s.package, %(packages-table)s.source FROM %(packages-table)s)
+ as packages
+ WHERE
+ %(table)s.package = packages.package
+ GROUP BY packages.source;
+ """ % my_config)
+ for line in cur.fetchall():
+ cur.execute("EXECUTE pop_insert ('%s', %s, %s, %s, %s, %s)" % line)
+ cur.execute("DEALLOCATE pop_insert");
+ cur.execute("PREPARE pop_insert AS INSERT INTO %s VALUES ($1, $2, $3, $4, $5, $6)" % table_src_average)
+ cur.execute("""
+ SELECT packages.source, avg(insts) AS insts, avg(vote) AS vote, avg(olde) AS old,
+ avg(recent) AS recent, avg(nofiles) as nofiles
+ FROM %(table)s,
+ (SELECT DISTINCT %(packages-table)s.package, %(packages-table)s.source FROM %(packages-table)s)
+ as packages
+ WHERE
+ %(table)s.package = packages.package
+ GROUP BY packages.source;
+ """ % my_config)
+ for line in cur.fetchall():
+ cur.execute("EXECUTE pop_insert ('%s', %s, %s, %s, %s, %s)" % line)
+ cur.execute("DEALLOCATE pop_insert");
+
if __name__ == '__main__':
main()
Modified: udd/src/udd/sources_gatherer.py
===================================================================
--- udd/src/udd/sources_gatherer.py 2008-08-08 13:35:09 UTC (rev 997)
+++ udd/src/udd/sources_gatherer.py 2008-08-08 13:51:14 UTC (rev 998)
@@ -1,5 +1,5 @@
#/usr/bin/env python
-# Last-Modified: <Sat Jul 26 12:59:30 2008>
+# Last-Modified: <Fri Aug 8 12:45:43 2008>
# This file is a part of the Ultimate Debian Database project
import debian_bundle.deb822
@@ -92,17 +92,24 @@
raise ConfigException('release not specified for source %s' %
(src_name))
+ if not 'sources-table' in src_cfg:
+ raise ConfigException('sources-table not specifed for source %s' %
+ (src_name))
+
+ table = src_cfg['sources-table']
+
+
aux.debug = self.config['general']['debug']
cur = self.cursor()
for comp in src_cfg['components']:
path = os.path.join(src_cfg['directory'], comp, 'source', 'Sources.gz')
- cur.execute("DELETE from sources WHERE Distribution = '%s' AND\
+ cur.execute("DELETE from %s WHERE Distribution = '%s' AND\
release = '%s' AND component = '%s'"\
- % (src_cfg['distribution'], src_cfg['release'], comp))
+ % (table, src_cfg['distribution'], src_cfg['release'], comp))
try:
- query = """PREPARE source_insert as INSERT INTO sources
+ query = """PREPARE source_insert as INSERT INTO %s
(Package, Version, Maintainer, Format, Files, Uploaders, Bin,
Architecture, Standards_Version, Homepage, Build_Depends,
Build_Depends_Indep, Build_Conflicts, Build_Conflicts_Indep, Priority,
@@ -113,7 +120,7 @@
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16,
$17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, '%s', '%s',
'%s')"""\
- % (src_cfg['distribution'], src_cfg['release'], comp)
+ % (table, src_cfg['distribution'], src_cfg['release'], comp)
cur.execute(query)
aux.print_debug("Reading file " + path)
More information about the Collab-qa-commits
mailing list