[Collab-qa-commits] r1525 - udd/udd
Lucas Nussbaum
lucas at alioth.debian.org
Fri Jul 17 02:59:27 UTC 2009
Author: lucas
Date: 2009-07-17 02:59:27 +0000 (Fri, 17 Jul 2009)
New Revision: 1525
Modified:
udd/udd/packages_gatherer.py
udd/udd/sources_gatherer.py
Log:
optimize import using executemany()
Modified: udd/udd/packages_gatherer.py
===================================================================
--- udd/udd/packages_gatherer.py 2009-07-17 02:58:42 UTC (rev 1524)
+++ udd/udd/packages_gatherer.py 2009-07-17 02:59:27 UTC (rev 1525)
@@ -78,6 +78,16 @@
Sequence has to have an iterator interface, that yields a line every time
it is called.The Format of the sequence is expected to be that of a
debian packages file."""
+ pkgs = ()
+ query = """EXECUTE package_insert
+ (%(Package)s, %(Version)s, %(Architecture)s, %(Maintainer)s,
+ %(Description)s, %(Long_Description)s, %(Source)s, %(Source_Version)s, %(Essential)s,
+ %(Depends)s, %(Recommends)s, %(Suggests)s, %(Enhances)s,
+ %(Pre-Depends)s, %(Breaks)s, %(Installed-Size)s, %(Homepage)s, %(Size)s,
+ %(Build-Essential)s, %(Origin)s, %(SHA1)s,
+ %(Replaces)s, %(Section)s, %(MD5sum)s, %(Bugs)s, %(Priority)s,
+ %(Tag)s, %(Task)s, %(Python-Version)s, %(Provides)s,
+ %(Conflicts)s, %(SHA256)s, %(Original-Maintainer)s)"""
# The fields that are to be read. Other fields are ignored
for control in debian_bundle.deb822.Packages.iter_paragraphs(sequence):
# Check whether packages with architectue 'all' have already been
@@ -114,21 +124,13 @@
else:
d['Source'] = split[0]
d['Source_Version'] = split[1].strip("()")
+ pkgs += (d,)
- query = """EXECUTE package_insert
- (%(Package)s, %(Version)s, %(Architecture)s, %(Maintainer)s,
- %(Description)s, %(Long_Description)s, %(Source)s, %(Source_Version)s, %(Essential)s,
- %(Depends)s, %(Recommends)s, %(Suggests)s, %(Enhances)s,
- %(Pre-Depends)s, %(Breaks)s, %(Installed-Size)s, %(Homepage)s, %(Size)s,
- %(Build-Essential)s, %(Origin)s, %(SHA1)s,
- %(Replaces)s, %(Section)s, %(MD5sum)s, %(Bugs)s, %(Priority)s,
- %(Tag)s, %(Task)s, %(Python-Version)s, %(Provides)s,
- %(Conflicts)s, %(SHA256)s, %(Original-Maintainer)s)"""
- try:
- cur.execute(query, d)
- except psycopg2.ProgrammingError:
- print query
- raise
+ try:
+ cur.executemany(query, pkgs)
+ except psycopg2.ProgrammingError:
+ print query
+ raise
def setup(self):
if 'schema-dir' in self.config['general']:
@@ -201,11 +203,11 @@
package, version, source, source_version, maintainer, distribution,
release, component
FROM %s""" % (table + '_summary', table));
- cur.execute("DELETE FROM all_packages_distrelcomparch");
- cur.execute("""INSERT INTO all_packages_distrelcomparch
+ cur.execute("DELETE FROM %s" % (table + '_distrelcomparch'));
+ cur.execute("""INSERT INTO %s
(distribution, release, component, architecture)
SELECT DISTINCT distribution, release, component, architecture
- FROM all_packages""")
+ FROM %s""" % (table + '_distrelcomparch', table))
self.print_warnings()
Modified: udd/udd/sources_gatherer.py
===================================================================
--- udd/udd/sources_gatherer.py 2009-07-17 02:58:42 UTC (rev 1524)
+++ udd/udd/sources_gatherer.py 2009-07-17 02:59:27 UTC (rev 1525)
@@ -92,20 +92,23 @@
is called.The Format of the file is expected to be that of a debian
source file."""
cur = self.cursor()
+ pkgs = ()
+ query = """EXECUTE source_insert
+ (%(Package)s, %(Version)s, %(Maintainer)s,
+ %(maintainer_name)s, %(maintainer_email)s, %(Format)s, %(Files)s,
+ %(Uploaders)s, %(Binary)s, %(Architecture)s, %(Standards-Version)s,
+ %(Homepage)s, %(Build-Depends)s, %(Build-Depends-Indep)s,
+ %(Build-Conflicts)s, %(Build-Conflicts-Indep)s, %(Priority)s,
+ %(Section)s, %(Vcs-Type)s, %(Vcs-Url)s, %(Vcs-Browser)s,
+ %(Python-Version)s, %(Checksums-Sha1)s, %(Checksums-Sha256)s,
+ %(Original-Maintainer)s, %(Dm-Upload-Allowed)s)"""
+ query_uploaders = """EXECUTE uploader_insert (%(Package)s, %(Version)s, %(Name)s, %(Email)s)"""
+ uploaders = ()
for control in debian_bundle.deb822.Packages.iter_paragraphs(file):
d = self.build_dict(control)
d['maintainer_name'], d['maintainer_email'] = email.Utils.parseaddr(d['Maintainer'])
- query = """EXECUTE source_insert
- (%(Package)s, %(Version)s, %(Maintainer)s,
- %(maintainer_name)s, %(maintainer_email)s, %(Format)s, %(Files)s,
- %(Uploaders)s, %(Binary)s, %(Architecture)s, %(Standards-Version)s,
- %(Homepage)s, %(Build-Depends)s, %(Build-Depends-Indep)s,
- %(Build-Conflicts)s, %(Build-Conflicts-Indep)s, %(Priority)s,
- %(Section)s, %(Vcs-Type)s, %(Vcs-Url)s, %(Vcs-Browser)s,
- %(Python-Version)s, %(Checksums-Sha1)s, %(Checksums-Sha256)s,
- %(Original-Maintainer)s, %(Dm-Upload-Allowed)s)
- """
- cur.execute(query, d)
+ pkgs += (d,)
+
ud = {}
ud['Package'] = d['Package']
ud['Version'] = d['Version']
@@ -113,9 +116,9 @@
for uploader in email.Utils.getaddresses([d['Uploaders']]):
ud['Name'] = uploader[0]
ud['Email'] = uploader[1]
- query = """EXECUTE uploader_insert (%(Package)s, %(Version)s, %(Name)s,
- %(Email)s)"""
- cur.execute(query, ud)
+ uploaders += (ud,)
+ cur.executemany(query, pkgs)
+ cur.executemany(query_uploaders, uploaders)
def tables(self):
return [self.my_config['sources-table']]
More information about the Collab-qa-commits
mailing list