[Collab-qa-commits] r885 - udd/src
neronus-guest at alioth.debian.org
neronus-guest at alioth.debian.org
Fri Jun 6 10:42:32 UTC 2008
Author: neronus-guest
Date: 2008-06-06 10:42:31 +0000 (Fri, 06 Jun 2008)
New Revision: 885
Added:
udd/src/sources_gatherer.py
Modified:
udd/src/db_manager.py
udd/src/packages_gatherer.py
udd/src/setup-db.sql
udd/src/test.yaml
Log:
* Added build_archs table
* Added code to suck in Sources
* Added indice for DB
* src_id is now set for packages
Modified: udd/src/db_manager.py
===================================================================
--- udd/src/db_manager.py 2008-06-05 12:55:41 UTC (rev 884)
+++ udd/src/db_manager.py 2008-06-06 10:42:31 UTC (rev 885)
@@ -5,7 +5,7 @@
"""This scripts sets up and deletes the tables of the database"""
-TABLES = ('sources', 'pkgs', 'distr_ids', 'arch_ids')
+TABLES = ('sources', 'pkgs', 'distr_ids', 'arch_ids', 'build_archs')
def print_help():
print "Usage: %s <config> <delete|setup>" % sys.argv[0]
Modified: udd/src/packages_gatherer.py
===================================================================
--- udd/src/packages_gatherer.py 2008-06-05 12:55:41 UTC (rev 884)
+++ udd/src/packages_gatherer.py 2008-06-06 10:42:31 UTC (rev 885)
@@ -1,5 +1,5 @@
#/usr/bin/env python
-# Last-Modified: <Fri May 30 15:03:14 2008>
+# Last-Modified: <Fri Jun 6 09:23:59 2008>
import debian_bundle.deb822
import gzip
@@ -30,6 +30,7 @@
global imported_all_pkgs
# The fields that are to be read. Other fields are ignored
fields = ('Architecture', 'Package', 'Version')
+ cur = conn.cursor()
for control in debian_bundle.deb822.Packages.iter_paragraphs(sequence, fields):
# Check whether packages with architectue 'all' have already been
# imported
@@ -39,9 +40,9 @@
continue
imported_all_pkgs[t] = 1
- cur = conn.cursor()
- query = "INSERT INTO pkgs (name, distr_id, arch_id, version, src_id)\
- VALUES ('%s', %d, %d, '%s', 0)" % (control["Package"], distr_id, archs[control["Architecture"]], control["Version"])
+ #query = "INSERT INTO pkgs (name, distr_id, arch_id, version, src_id)\
+ #VALUES ('%s', %d, %d, '%s', 0);" % (control["Package"], distr_id, archs[control["Architecture"]], control["Version"])
+ query = "EXECUTE pkg_insert('%s', %d, %d, '%s')" % (control["Package"], distr_id, archs[control["Architecture"]], control["Version"])
cur.execute(query)
def main():
@@ -96,6 +97,9 @@
archs = aux.get_archs(conn)
+ cur = conn.cursor()
+ cur.execute("PREPARE pkg_insert AS INSERT INTO pkgs (name, distr_id, arch_id, version) VALUES ($1, $2, $3, $4);")
+
# For every part and every architecture, import the packages into the DB
for part in src_cfg['parts']:
for arch in src_cfg['archs']:
@@ -115,6 +119,7 @@
except IOError, (e, message):
print "Could not read packages from %s: %s" % (path, message)
+ cur.execute("DEALLOCATE pkg_insert")
conn.commit()
if __name__ == '__main__':
Modified: udd/src/setup-db.sql
===================================================================
--- udd/src/setup-db.sql 2008-06-05 12:55:41 UTC (rev 884)
+++ udd/src/setup-db.sql 2008-06-06 10:42:31 UTC (rev 885)
@@ -1,9 +1,16 @@
CREATE TABLE pkgs (pkg_id serial, name text, distr_id int, arch_id int, version text, src_id int);
-CREATE TABLE sources (src_id serial, name text, upload_date timestamp, uploader_key int, maintainer int, build_archs int, version text, distr_id int);
+CREATE TABLE sources (src_id serial, name text, upload_date timestamp, uploader_key int, maintainer text, version text, distr_id int);
CREATE TABLE distr_ids (distr_id serial, name text);
CREATE TABLE arch_ids (arch_id serial, name text);
+CREATE TABLE build_archs (src_id int, arch_id int);
+CREATE INDEX pkgs_id_idx ON pkgs (pkg_id);
+CREATE INDEX pkgs_name_idx ON pkgs (name);
+CREATE INDEX sources_id_idx ON sources (src_id);
+CREATE INDEX sources_name_idx ON sources (name);
+
GRANT SELECT ON pkgs TO PUBLIC;
GRANT SELECT ON sources TO PUBLIC;
GRANT SELECT ON distr_ids TO PUBLIC;
GRANT SELECT ON arch_ids TO PUBLIC;
+GRANT SELECT ON build_archs TO PUBLIC;
Added: udd/src/sources_gatherer.py
===================================================================
--- udd/src/sources_gatherer.py (rev 0)
+++ udd/src/sources_gatherer.py 2008-06-06 10:42:31 UTC (rev 885)
@@ -0,0 +1,123 @@
+#/usr/bin/env python
+# Last-Modified: <Fri Jun 6 10:07:50 2008>
+
+import debian_bundle.deb822
+import gzip
+import os
+import sys
+import aux
+import tempfile
+from aux import ConfigException
+
+# A mapping from the architecture names to architecture IDs
+archs = {}
+# The ID for the distribution we want to include
+distr_id = None
+
+def import_sources(conn, file):
+ """Import the sources from the file into the database-connection conn.
+
+ Sequence has to have an iterator interface, that yields a line every time it
+ is called.The Format of the file is expected to be that of a debian
+ source file."""
+ # The fields that are to be read. Other fields are ignored
+ fields = ('Package', 'Version', 'Architecture', 'Maintainer', 'Uploaders', 'Binary')
+ cur = conn.cursor()
+ for control in debian_bundle.deb822.Packages.iter_paragraphs(file, fields):
+ # Put the source package into the DB
+ query = "EXECUTE source_insert('%s', '%s', '%s', %d)" % (control["Package"], control['Maintainer'].replace("'", "\\'"), control["Version"],
+ distr_id)
+ cur.execute(query)
+ # Get the src_id of the source
+ #cur.execute("SELECT src_id FROM sources WHERE name = '%(Package)s' AND version = '%(Version)s'" % control)
+ cur.execute("EXECUTE select_src_id('%(Package)s', '%(Version)s')" % control)
+ src_id = int(cur.fetchone()[0])
+ # Fill the build_archs table for this source package
+ if control['Architecture'] == 'all' or control['Architecture'] == 'any':
+ query = "EXECUTE build_archs_insert(%d, %d)" % (src_id, archs[control['Architecture']])
+ cur.execute(query)
+ else:
+ for arch in control['Architecture'].split():
+ query = "EXECUTE build_archs_insert(%d, %d)" % (src_id, archs[arch])
+ cur.execute(query)
+
+ # Set the source_ids for the binaries
+ for binary in control['Binary'].split(", "):
+ query = "EXECUTE pkgs_update_src_id(%d, '%s', %d)" % (src_id, binary, distr_id)
+ cur.execute(query)
+
+def main():
+ global distr_id
+ global archs
+ if len(sys.argv) != 3:
+ print "Usage: %s <config> <source>" % sys.argv[0]
+ sys.exit(1)
+
+ src_name = sys.argv[2]
+ cfg_path = sys.argv[1]
+ config = None
+ try:
+ config = aux.load_config(open(cfg_path).read())
+ except ConfigException, e:
+ raise ConfigException, "Configuration error in " + cfg_path +": " + e.message
+
+ if not src_name in config:
+ raise ConfigException, "Source %s not specified in %s" %(src_name, cfg_path)
+ src_cfg = config[src_name]
+
+ if not 'directory' in src_cfg:
+ raise ConfigException('directory not specified for source %s in file %s' %
+ (src_name, cfg_path))
+
+ if not 'parts' in src_cfg:
+ raise ConfigException('parts not specified for source %s in file %s' %
+ (src_name, cfg_path))
+
+ if not 'distribution' in src_cfg:
+ raise ConfigException('distribution not specified for source %s in file %s' %
+ (src_name, cfg_path))
+
+ aux.debug = config['general']['debug']
+
+ conn = aux.open_connection(config)
+
+ # Get distribution ID. If it does not exist, create it
+ distr_ids = aux.get_distrs(conn)
+ if src_cfg['distribution'] not in distr_ids:
+ aux.insert_distr(conn, src_cfg['distribution'])
+ distr_ids = aux.get_distrs(conn)
+ distr_id = distr_ids[src_cfg['distribution']]
+
+ archs = aux.get_archs(conn)
+
+ cur = conn.cursor()
+ cur.execute("PREPARE source_insert AS INSERT INTO sources (name, maintainer, version, distr_id) VALUES ($1,$2,$3,$4)")
+ cur.execute("PREPARE build_archs_insert AS INSERT INTO build_archs (src_id, arch_id) VALUES ($1,$2)")
+ cur.execute("PREPARE pkgs_update_src_id AS UPDATE pkgs SET src_id = $1 WHERE name = $2 AND distr_id = $3")
+ cur.execute("PREPARE select_src_id AS SELECT src_id FROM sources WHERE name = $1 AND version = $2")
+
+ for part in src_cfg['parts']:
+ path = os.path.join(src_cfg['directory'], part, 'source', 'Sources.gz')
+ try:
+ aux.print_debug("Reading file " + path)
+ # Copy content from gzipped file to temporary file, so that apt_pkg is
+ # used by debian_bundle
+ tmp = tempfile.NamedTemporaryFile()
+ file = gzip.open(path)
+ tmp.write(file.read())
+ file.close()
+ tmp.seek(0)
+ aux.print_debug("Importing from " + path)
+ import_sources(conn, open(tmp.name))
+ tmp.close()
+ except IOError, (e, message):
+ print "Could not read packages from %s: %s" % (path, message)
+
+ cur.execute("DEALLOCATE source_insert")
+ cur.execute("DEALLOCATE build_archs_insert")
+ cur.execute("DEALLOCATE select_src_id")
+ cur.execute("DEALLOCATE pkgs_update_src_id")
+ conn.commit()
+
+if __name__ == '__main__':
+ main()
Modified: udd/src/test.yaml
===================================================================
--- udd/src/test.yaml 2008-06-05 12:55:41 UTC (rev 884)
+++ udd/src/test.yaml 2008-06-06 10:42:31 UTC (rev 885)
@@ -1,7 +1,7 @@
general:
dbname: udd
types:
- sources: echo
+ sources: python sources_gatherer.py
packages: python packages_gatherer.py
setup: python db_manager.py
delete: python db_manager.py
@@ -10,8 +10,12 @@
archs:
[alpha, amd64, arm, armeb, armel, hppa, hurd-i386,
i386, i486, ia64, kfreebsd-amd64, kfreebsd-i386, m68k, mips,
- mipsel, powerpc, ppc64, s390, sparc, all, any]
+ mipsel, powerpc, ppc64, s390, sparc, all, any, lpia, m32r, s390x, sh3,
+ sh3eb, sh4, sh4eb, sh, knetbsd-i386, netbsd-alpha, sparc64,
+ netbsd-i386, hurd-powerpc, kfreebsd-powerpc, netbsd-powerpc, hurd-sparc,
+ kfreebsd-sparc, netbsd-sparc]
+
delete:
type: delete
@@ -81,3 +85,19 @@
directory: /org/volatile.debian.org/dists/sarge/volatile/
parts: [main, contrib, non-free]
distribution: debian-volatile-sarge
+
+test-src:
+ type: sources
+ directory: /org/ftp.debian.org/dists/lenny/
+ parts: [main, contrib, non-free]
+ distribution: test
+
+test-pkg:
+ type: packages
+ directory: /org/ftp.debian.org/dists/lenny/
+ parts: [main, contrib, non-free]
+ distribution: test
+ archs:
+ [alpha, amd64, arm, armeb, armel, hppa, hurd-i386,
+ i386, i486, ia64, kfreebsd-amd64, kfreebsd-i386, m68k, mips,
+ mipsel, powerpc, ppc64, s390, sparc, all, any]
More information about the Collab-qa-commits
mailing list