[Collab-qa-commits] r865 - udd/src
neronus-guest at alioth.debian.org
neronus-guest at alioth.debian.org
Fri May 30 11:47:38 UTC 2008
Author: neronus-guest
Date: 2008-05-30 11:47:37 +0000 (Fri, 30 May 2008)
New Revision: 865
Added:
udd/src/db_manager.py
udd/src/packages_gatherer.py
Removed:
udd/src/destroy-db.sh
udd/src/import-pkgs.py
udd/src/packages-gatherer.py
Modified:
udd/src/aux.py
udd/src/delete-tables.py
udd/src/setup-db.py
udd/src/test.yaml
udd/src/udd-dispatch.py
Log:
* rearanged types in test.yaml
* introduced general section in test.yaml
* added open_connection to aux.py
* merged setup-db.py and delete-tables.py into db_manger.py
* removed redundant import-packages.py
* adapted the scripts to the changes in test.yaml
Modified: udd/src/aux.py
===================================================================
--- udd/src/aux.py 2008-05-29 11:38:57 UTC (rev 864)
+++ udd/src/aux.py 2008-05-30 11:47:37 UTC (rev 865)
@@ -2,6 +2,7 @@
import syck
import sys
+import psycopg2
# If debug is something that evaluates to True, then print_debug actually prints something
debug = 0
@@ -36,32 +37,39 @@
def __str__(self):
return "ConfigException: " + self.message
-def load_config(seq):
- """Load and check configuration from seq.
+def open_connection(config):
+ """Open the connection to the database and return it"""
+ return psycopg2.connect("dbname=" + config['general']['dbname'])
- seq has to be a sequence type, containings strings (see syck.load)"""
- config = syck.load(seq)
- if not 'dbname' in config:
+def load_config(str):
+ """Load and check configuration from the string"""
+ config = syck.load(str)
+ if not 'general' in config:
+ raise ConfigException('general section not specified')
+
+ general = config['general']
+
+ if not 'dbname' in general:
raise ConfigException('dbname not specified')
- if not 'archs' in config:
+ if not 'archs' in general:
raise ConfigException('archs not specified')
- if not 'types' in config:
+ if not 'types' in general:
raise ConfigException('types not specified')
- if not 'debug' in config:
- config['debug'] = 0
+ if not 'debug' in general:
+ general['debug'] = 0
# Check that the source-entries are well-formed
for name in config:
- if name in ('dbname', 'archs', 'types', 'debug'):
+ if name == 'general':
continue
src = config[name]
if not 'type' in src:
raise ConfigException('type not specified for "%s"' % name)
- if src['type'] not in config['types']:
+ if src['type'] not in general['types']:
raise ConfigException('Type of %s not specified in types' % name)
return config
Added: udd/src/db_manager.py
===================================================================
--- udd/src/db_manager.py (rev 0)
+++ udd/src/db_manager.py 2008-05-30 11:47:37 UTC (rev 865)
@@ -0,0 +1,54 @@
+#!/usr/bin/python
+
+import aux
+import sys
+
+"""This scripts sets up and deletes the tables of the database"""
+
+TABLES = ('sources', 'pkgs', 'distr_ids', 'arch_ids')
+
+def print_help():
+ print "Usage: %s <config> <delete|setup>" % sys.argv[0]
+
+def delete(conn):
+ c = conn.cursor()
+
+ for t in TABLES:
+ c.execute("DROP TABLE " + t)
+
+def setup(conn, config):
+ c = conn.cursor()
+ c.execute("CREATE TABLE pkgs (pkg_id serial, name text, distr_id int, arch_id int, version text, src_id int);")
+ c.execute("CREATE TABLE sources (src_id serial, name text, upload_date timestamp, uploader_key int, maintainer int, build_archs int, version text, distr_id int);")
+ c.execute("CREATE TABLE distr_ids (distr_id serial, name text);")
+ c.execute("CREATE TABLE arch_ids (arch_id serial, name text);")
+
+ #Setup architecture table
+ for arch in config['general']['archs']:
+ c.execute("INSERT INTO arch_ids (name) VALUES ('%s');" % (arch))
+
+def main():
+ if len(sys.argv) != 3:
+ print_help()
+ sys.exit(1)
+
+ command = sys.argv[2]
+ config_path = sys.argv[1]
+
+ config = aux.load_config(open(config_path).read())
+ conn = aux.open_connection(config)
+
+ if command == 'setup':
+ setup(conn, config)
+ elif command == 'delete':
+ delete(conn)
+ else:
+ print "Unknown command: " + command
+ sys.exit(1)
+
+ conn.commit()
+
+if __name__ == '__main__':
+ main()
+
+
Modified: udd/src/delete-tables.py
===================================================================
--- udd/src/delete-tables.py 2008-05-29 11:38:57 UTC (rev 864)
+++ udd/src/delete-tables.py 2008-05-30 11:47:37 UTC (rev 865)
@@ -5,17 +5,17 @@
import sys
def main():
- if len(sys.argv) != 2:
- print "Usage: %s <config>" % sys,argv[0]
+ if len(sys.argv) < 2:
+ print "Usage: %s <config>" % sys.argv[0]
sys.exit(1)
- config = aux.load_config(sys.argv[1])
+ config = aux.load_config(open(sys.argv[1]).read())
- conn = psycopg2.connect(config['dbname'])
+ conn = psycopg2.connect("dbname=" + config['dbname'])
c = conn.cursor()
for table in ('sources', 'pkgs', 'distr_ids', 'arch_ids'):
- c.execute("DELETE * FROM " + table)
+ c.execute("DROP TABLE " + table)
conn.commit()
conn.close()
Deleted: udd/src/destroy-db.sh
===================================================================
--- udd/src/destroy-db.sh 2008-05-29 11:38:57 UTC (rev 864)
+++ udd/src/destroy-db.sh 2008-05-30 11:47:37 UTC (rev 865)
@@ -1,5 +0,0 @@
-#!/bin/sh
-
-SUDO="sudo -u postgres"
-$SUDO dropdb udd
-$SUDO createdb -O christian udd
Deleted: udd/src/import-pkgs.py
===================================================================
--- udd/src/import-pkgs.py 2008-05-29 11:38:57 UTC (rev 864)
+++ udd/src/import-pkgs.py 2008-05-30 11:47:37 UTC (rev 865)
@@ -1,42 +0,0 @@
-#!/usr/bin/env python
-# Last-Modified: <Fri 23 May 2008 14:52:25 CEST>
-
-import debian_bundle
-from debian_bundle import debfile
-import psycopg2
-import sys
-
-def get_archs(conn):
- c = conn.cursor();
- c.execute("SELECT * from arch_ids")
- result = {}
- for row in c.fetchall():
- result[row[1]] = row[0]
- return result
-
-def get_distrs(conn):
- c = conn.cursor()
- c.execute("SELECT * from distr_ids")
- result = {}
- for row in c.fetchall():
- result[row[1]] = row[0]
- return result
-
-if __name__ == '__main__':
- conn = psycopg2.connect("dbname=udd")
- archs = get_archs(conn)
- distr = get_distrs(conn)
- n = 0
- for file in sys.argv[1:]:
- try:
- control = debfile.DebFile(file).debcontrol()
- except Exception:
- print "Could not parse " + file
- continue
- c = conn.cursor()
- c.execute("INSERT INTO pkgs (name, distr_id, arch_id, version, src_id) VALUES ('%s', 0, %d, '%s', 0)" % (control["Package"], archs[control["Architecture"]], control["Version"]))
- n += 1
- if n % 100 == 0:
- print n
- conn.commit()
-
Deleted: udd/src/packages-gatherer.py
===================================================================
--- udd/src/packages-gatherer.py 2008-05-29 11:38:57 UTC (rev 864)
+++ udd/src/packages-gatherer.py 2008-05-30 11:47:37 UTC (rev 865)
@@ -1,112 +0,0 @@
-#/usr/bin/env python
-# Last-Modified: <Sat May 24 11:34:28 2008>
-
-from psycopg2 import connect
-import debian_bundle.deb822
-import gzip
-import os
-import sys
-import aux
-from aux import ConfigException
-
-# A mapping from the architecture names to architecture IDs
-archs = {}
-# A mapping from <package-name><version> to 1
-# If <package-name><version> is included in this dictionary, this means,
-# that we've already added this package with this version for architecture 'all'
-# to the database. Needed because different architectures include packages
-# for architecture 'all' with the same version, and we don't want these duplicate
-# entries
-imported_all_pkgs = {}
-# The ID for the distribution we want to include
-distr_id = None
-
-def import_packages(conn, sequence):
- """Import the packages from the sequence into the database-connection conn.
-
- Sequence has to have an iterator interface, that yields a line every time it
- is called.The Format of the sequence is expected to be that of a debian
- packages file."""
- global imported_all_pkgs
- # The fields that are to be read. Other fields are ignored
- fields = ('Architecture', 'Package', 'Version')
- for control in debian_bundle.deb822.Packages.iter_paragraphs(sequence, fields):
- # Check whether packages with architectue 'all' have already been
- # imported
- if control['Architecture'] == 'all':
- t = control['Package'] + control['Version']
- if t in imported_all_pkgs:
- continue
- imported_all_pkgs[t] = 1
-
- cur = conn.cursor()
- query = "INSERT INTO pkgs (name, distr_id, arch_id, version, src_id)\
- VALUES ('%s', %d, %d, '%s', 0)" % (control["Package"], distr_id, archs[control["Architecture"]], control["Version"])
- cur.execute(query)
-
-def main():
- global distr_id
- global archs
- if len(sys.argv) != 3:
- print "Usage: %s <config> <source>" % sys.argv[0]
- sys.exit(1)
-
- src_name = sys.argv[2]
- cfg_path = sys.argv[1]
- config = None
- try:
- config = aux.load_config(open(cfg_path).read())
- except ConfigException, e:
- raise ConfigException, "Configuration error in " + cfg_path +": " + e.message
-
- if not src_name in config:
- raise ConfigException, "Source %s not specified in " + cfg_path
- src_cfg = config[src_name]
-
- if not 'directory' in src_cfg:
- raise ConfigException('directory not specified for source %s in file %s' %
- (src_name, cfg_path))
-
- if not 'archs' in src_cfg:
- raise ConfigException('archs not specified for source %s in file %s' %
- (src_name, cfg_path))
-
- if not 'parts' in src_cfg:
- raise ConfigException('parts not specified for source %s in file %s' %
- (src_name, cfg_path))
-
- if not 'distribution' in src_cfg:
- raise ConfigException('distribution not specified for source %s in file %s' %
- (src_name, cfg_path))
-
- aux.debug = config['debug']
-
- conn = connect('dbname=' + config['dbname'])
-
- # Get distribution ID. If it does not exist, create it
- distr_ids = aux.get_distrs(conn)
- if src_cfg['distribution'] not in distr_ids:
- aux.insert_distr(conn, src_cfg['distribution'])
- distr_ids = aux.get_distrs(conn)
- distr_id = distr_ids[src_cfg['distribution']]
-
- archs = aux.get_archs(conn)
-
- # For every part and every architecture, import the packages into the DB
- for part in src_cfg['parts']:
- for arch in src_cfg['archs']:
- path = os.path.join(src_cfg['directory'], part, 'binary-' + arch, 'Packages.gz')
- try:
- aux.print_debug("Reading file " + path)
- file = gzip.open(path)
- lines = aux.BufferedLineReader(file, 1024*1024*4)
- aux.print_debug("Importing from " + path)
- import_packages(conn, lines)
- file.close()
- except IOError, (e, message):
- print "Could not read packages from %s: %s" % (path, message)
-
- conn.commit()
-
-if __name__ == '__main__':
- main()
Copied: udd/src/packages_gatherer.py (from rev 858, udd/src/packages-gatherer.py)
===================================================================
--- udd/src/packages_gatherer.py (rev 0)
+++ udd/src/packages_gatherer.py 2008-05-30 11:47:37 UTC (rev 865)
@@ -0,0 +1,115 @@
+#/usr/bin/env python
+# Last-Modified: <Thu May 29 21:00:51 2008>
+
+import debian_bundle.deb822
+import gzip
+import os
+import sys
+import aux
+from aux import ConfigException
+
+# A mapping from the architecture names to architecture IDs
+archs = {}
+# A mapping from <package-name><version> to 1
+# If <package-name><version> is included in this dictionary, this means,
+# that we've already added this package with this version for architecture 'all'
+# to the database. Needed because different architectures include packages
+# for architecture 'all' with the same version, and we don't want these duplicate
+# entries
+imported_all_pkgs = {}
+# The ID for the distribution we want to include
+distr_id = None
+
+def import_packages(conn, sequence):
+ """Import the packages from the sequence into the database-connection conn.
+
+ Sequence has to have an iterator interface, that yields a line every time it
+ is called.The Format of the sequence is expected to be that of a debian
+ packages file."""
+ global imported_all_pkgs
+ # The fields that are to be read. Other fields are ignored
+ fields = ('Architecture', 'Package', 'Version')
+ for control in debian_bundle.deb822.Packages.iter_paragraphs(sequence, fields):
+ # Check whether packages with architectue 'all' have already been
+ # imported
+ if control['Architecture'] == 'all':
+ t = control['Package'] + control['Version']
+ if t in imported_all_pkgs:
+ continue
+ imported_all_pkgs[t] = 1
+
+ cur = conn.cursor()
+ query = "INSERT INTO pkgs (name, distr_id, arch_id, version, src_id)\
+ VALUES ('%s', %d, %d, '%s', 0)" % (control["Package"], distr_id, archs[control["Architecture"]], control["Version"])
+ cur.execute(query)
+
+def main():
+ global distr_id
+ global archs
+ if len(sys.argv) != 3:
+ print "Usage: %s <config> <source>" % sys.argv[0]
+ sys.exit(1)
+
+ src_name = sys.argv[2]
+ cfg_path = sys.argv[1]
+ config = None
+ try:
+ config = aux.load_config(open(cfg_path).read())
+ except ConfigException, e:
+ raise ConfigException, "Configuration error in " + cfg_path +": " + e.message
+
+ if not src_name in config:
+ raise ConfigException, "Source %s not specified in " + cfg_path
+ src_cfg = config[src_name]
+
+ if not 'directory' in src_cfg:
+ raise ConfigException('directory not specified for source %s in file %s' %
+ (src_name, cfg_path))
+
+ if not 'archs' in src_cfg:
+ raise ConfigException('archs not specified for source %s in file %s' %
+ (src_name, cfg_path))
+
+ if not 'parts' in src_cfg:
+ raise ConfigException('parts not specified for source %s in file %s' %
+ (src_name, cfg_path))
+
+ if not 'distribution' in src_cfg:
+ raise ConfigException('distribution not specified for source %s in file %s' %
+ (src_name, cfg_path))
+
+ #if not 'release' in src_cfg:
+ # raise ConfigException('release not specified for source %s in file %s' %
+ # (src_name, cfg_path))
+
+ aux.debug = config['general']['debug']
+
+ conn = aux.open_connection(config)
+
+ # Get distribution ID. If it does not exist, create it
+ distr_ids = aux.get_distrs(conn)
+ if src_cfg['distribution'] not in distr_ids:
+ aux.insert_distr(conn, src_cfg['distribution'])
+ distr_ids = aux.get_distrs(conn)
+ distr_id = distr_ids[src_cfg['distribution']]
+
+ archs = aux.get_archs(conn)
+
+ # For every part and every architecture, import the packages into the DB
+ for part in src_cfg['parts']:
+ for arch in src_cfg['archs']:
+ path = os.path.join(src_cfg['directory'], part, 'binary-' + arch, 'Packages.gz')
+ try:
+ aux.print_debug("Reading file " + path)
+ file = gzip.open(path)
+ lines = aux.BufferedLineReader(file, 1024*1024*4)
+ aux.print_debug("Importing from " + path)
+ import_packages(conn, lines)
+ file.close()
+ except IOError, (e, message):
+ print "Could not read packages from %s: %s" % (path, message)
+
+ conn.commit()
+
+if __name__ == '__main__':
+ main()
Modified: udd/src/setup-db.py
===================================================================
--- udd/src/setup-db.py 2008-05-29 11:38:57 UTC (rev 864)
+++ udd/src/setup-db.py 2008-05-30 11:47:37 UTC (rev 865)
@@ -1,5 +1,5 @@
#!/usr/bin/env python
-# Last-Modified: <Sat May 24 11:29:22 2008>
+# Last-Modified: <Thu May 29 20:47:58 2008>
# Starting from an empty database, create the necessary tables
from psycopg2 import connect
@@ -7,7 +7,7 @@
import sys
if __name__ == '__main__':
- if len(sys.argv) != 2:
+ if len(sys.argv) < 2:
print "Usage: %s <udd config file>" % (sys.argv[0])
sys.exit(1)
Modified: udd/src/test.yaml
===================================================================
--- udd/src/test.yaml 2008-05-29 11:38:57 UTC (rev 864)
+++ udd/src/test.yaml 2008-05-30 11:47:37 UTC (rev 865)
@@ -1,73 +1,82 @@
-dbname: udd
-types:
- sources: echo
- packages: python packages-gatherer.py
-debug: 1
+general:
+ dbname: udd
+ types:
+ sources: echo
+ packages: python packages_gatherer.py
+ setup: python db_manager.py
+ delete: python db_manager.py
+ debug: 1
-archs:
- [alpha, amd64, arm, armeb, armel, hppa, hurd-i386,
- i386, i486, ia64, kfreebsd-amd64, kfreebsd-i386, m68k, mips,
- mipsel, powerpc, ppc64, s390, sparc, all, any]
+ archs:
+ [alpha, amd64, arm, armeb, armel, hppa, hurd-i386,
+ i386, i486, ia64, kfreebsd-amd64, kfreebsd-i386, m68k, mips,
+ mipsel, powerpc, ppc64, s390, sparc, all, any]
+delete:
+ type: delete
+
+setup:
+ type: setup
+
debian-lenny:
- archs: [alpha, amd64, arm, armel, hppa,
- i386, ia64, mips,
- mipsel, powerpc, s390, sparc]
- directory: /org/ftp.debian.org/dists/lenny/
- parts: [main, contrib, non-free]
- distribution: debian-lenny
- type: packages
+ type: packages
+ archs: [alpha, amd64, arm, armel, hppa,
+ i386, ia64, mips,
+ mipsel, powerpc, s390, sparc]
+ directory: /org/ftp.debian.org/dists/lenny/
+ parts: [main, contrib, non-free]
+ distribution: debian-lenny
debian-sid:
- archs: [alpha, amd64, arm, armel, hppa, hurd-i386,
- i386, i486, ia64, kfreebsd-amd64, kfreebsd-i386, m68k, mips,
- mipsel, powerpc, ppc64, s390, sparc]
- directory: /org/ftp.debian.org/dists/sid/
- parts: [main, contrib, non-free]
- distribution: debian-sid
- type: packages
+ type: packages
+ archs: [alpha, amd64, arm, armel, hppa, hurd-i386,
+ i386, i486, ia64, kfreebsd-amd64, kfreebsd-i386, m68k, mips,
+ mipsel, powerpc, ppc64, s390, sparc]
+ directory: /org/ftp.debian.org/dists/sid/
+ parts: [main, contrib, non-free]
+ distribution: debian-sid
debian-sarge:
- archs: [alpha, amd64, arm, armel, hppa, hurd-i386,
- i386, i486, ia64, kfreebsd-amd64, kfreebsd-i386, m68k, mips,
- mipsel, powerpc, ppc64, s390, sparc]
- directory: /org/ftp.debian.org/dists/sarge/
- parts: [main, contrib, non-free]
- distribution: debian-sarge
- type: packages
+ type: packages
+ archs: [alpha, amd64, arm, armel, hppa, hurd-i386,
+ i386, i486, ia64, kfreebsd-amd64, kfreebsd-i386, m68k, mips,
+ mipsel, powerpc, ppc64, s390, sparc]
+ directory: /org/ftp.debian.org/dists/sarge/
+ parts: [main, contrib, non-free]
+ distribution: debian-sarge
debian-backports-etch:
- archs: [alpha, amd64, arm, armel, hppa, hurd-i386,
- i386, i486, ia64, kfreebsd-amd64, kfreebsd-i386, m68k, mips,
- mipsel, powerpc, ppc64, s390, sparc]
- directory: /org/ftp.backports.org/dists/etch-backports/
- parts: [main, contrib, non-free]
- distribution: debian-backports-etch
- type: packages
+ type: packages
+ archs: [alpha, amd64, arm, armel, hppa, hurd-i386,
+ i386, i486, ia64, kfreebsd-amd64, kfreebsd-i386, m68k, mips,
+ mipsel, powerpc, ppc64, s390, sparc]
+ directory: /org/ftp.backports.org/dists/etch-backports/
+ parts: [main, contrib, non-free]
+ distribution: debian-backports-etch
debian-backports-sarge:
- archs: [alpha, amd64, arm, armel, hppa, hurd-i386,
- i386, i486, ia64, kfreebsd-amd64, kfreebsd-i386, m68k, mips,
- mipsel, powerpc, ppc64, s390, sparc]
- directory: /org/ftp.backports.org/dists/sarge-backports/
- parts: [main, contrib, non-free]
- distribution: debian-backports-sarge
- type: packages
+ type: packages
+ archs: [alpha, amd64, arm, armel, hppa, hurd-i386,
+ i386, i486, ia64, kfreebsd-amd64, kfreebsd-i386, m68k, mips,
+ mipsel, powerpc, ppc64, s390, sparc]
+ directory: /org/ftp.backports.org/dists/sarge-backports/
+ parts: [main, contrib, non-free]
+ distribution: debian-backports-sarge
debian-volatile-etch:
- archs: [alpha, amd64, arm, armel, hppa, hurd-i386,
- i386, i486, ia64, kfreebsd-amd64, kfreebsd-i386, m68k, mips,
- mipsel, powerpc, ppc64, s390, sparc]
- directory: /org/volatile.debian.org/dists/etch/volatile/
- parts: [main, contrib, non-free]
- distribution: debian-volatile-etch
- type: packages
+ type: packages
+ archs: [alpha, amd64, arm, armel, hppa, hurd-i386,
+ i386, i486, ia64, kfreebsd-amd64, kfreebsd-i386, m68k, mips,
+ mipsel, powerpc, ppc64, s390, sparc]
+ directory: /org/volatile.debian.org/dists/etch/volatile/
+ parts: [main, contrib, non-free]
+ distribution: debian-volatile-etch
debian-volatile-sarge:
- archs: [alpha, amd64, arm, armel, hppa, hurd-i386,
- i386, i486, ia64, kfreebsd-amd64, kfreebsd-i386, m68k, mips,
- mipsel, powerpc, ppc64, s390, sparc]
- directory: /org/volatile.debian.org/dists/sarge/volatile/
- parts: [main, contrib, non-free]
- distribution: debian-volatile-sarge
- type: packages
+ type: packages
+ archs: [alpha, amd64, arm, armel, hppa, hurd-i386,
+ i386, i486, ia64, kfreebsd-amd64, kfreebsd-i386, m68k, mips,
+ mipsel, powerpc, ppc64, s390, sparc]
+ directory: /org/volatile.debian.org/dists/sarge/volatile/
+ parts: [main, contrib, non-free]
+ distribution: debian-volatile-sarge
Modified: udd/src/udd-dispatch.py
===================================================================
--- udd/src/udd-dispatch.py 2008-05-29 11:38:57 UTC (rev 864)
+++ udd/src/udd-dispatch.py 2008-05-30 11:47:37 UTC (rev 865)
@@ -1,5 +1,5 @@
#!/usr/bin/env python
-# Last-Modified: <Sat May 24 11:57:17 2008>
+# Last-Modified: <Thu May 29 21:02:10 2008>
"""Dispatch udd gatherers
@@ -20,7 +20,7 @@
# Check the configuration
config = aux.load_config(open(sys.argv[1]).read())
- types = config['types']
+ types = config['general']['types']
for src in sys.argv[2:]:
if not src in config:
More information about the Collab-qa-commits
mailing list