[Collab-qa-commits] r2141 - in udd: . sql udd
Andreas Tille
tille at alioth.debian.org
Sat Feb 11 21:13:47 UTC 2012
Author: tille
Date: 2012-02-11 21:13:46 +0000 (Sat, 11 Feb 2012)
New Revision: 2141
Modified:
udd/config-org.yaml
udd/sql/setup.sql
udd/sql/upgrade.sql
udd/udd/packages_gatherer.py
Log:
Commit changes by Stuart Prescott to enable me commiting ddtp changes based on this
Modified: udd/config-org.yaml
===================================================================
--- udd/config-org.yaml 2012-02-09 21:01:20 UTC (rev 2140)
+++ udd/config-org.yaml 2012-02-11 21:13:46 UTC (rev 2141)
@@ -168,6 +168,7 @@
packages-schema: packages
sources-schema: sources
release: squeeze
+ descriptions-table: descriptions
debian-squeeze-security:
type: src-pkg
@@ -183,6 +184,7 @@
packages-schema: packages
sources-schema: sources
release: squeeze-security
+ descriptions-table: descriptions
debian-squeeze-proposed-updates:
type: src-pkg
@@ -196,6 +198,7 @@
packages-schema: packages
sources-schema: sources
release: squeeze-proposed-updates
+ descriptions-table: descriptions
debian-squeeze-updates:
type: src-pkg
@@ -209,6 +212,7 @@
packages-schema: packages
sources-schema: sources
release: squeeze-updates
+ descriptions-table: descriptions
debian-backports-squeeze:
type: src-pkg
@@ -222,6 +226,7 @@
packages-schema: packages
sources-schema: sources
release: squeeze
+ descriptions-table: descriptions
#debian-backports-squeeze-sloppy:
# type: src-pkg
@@ -235,6 +240,7 @@
# packages-schema: packages
# sources-schema: sources
# release: squeeze
+# descriptions-table: descriptions
debian-wheezy:
type: src-pkg
@@ -450,6 +456,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-natty-security:
type: src-pkg
@@ -463,6 +470,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-natty-updates:
type: src-pkg
@@ -476,6 +484,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-natty-proposed:
type: src-pkg
@@ -489,6 +498,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-natty-backports:
type: src-pkg
@@ -502,6 +512,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-maverick:
@@ -516,6 +527,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-maverick-security:
type: src-pkg
@@ -529,6 +541,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-maverick-updates:
type: src-pkg
@@ -542,6 +555,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-maverick-proposed:
type: src-pkg
@@ -555,6 +569,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-maverick-backports:
type: src-pkg
@@ -568,6 +583,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-lucid:
@@ -582,6 +598,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-lucid-security:
type: src-pkg
@@ -595,6 +612,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-lucid-updates:
type: src-pkg
@@ -608,6 +626,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-lucid-proposed:
type: src-pkg
@@ -621,6 +640,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-lucid-backports:
type: src-pkg
@@ -634,6 +654,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-karmic:
@@ -648,6 +669,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-karmic-security:
type: src-pkg
@@ -661,6 +683,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-karmic-updates:
type: src-pkg
@@ -674,6 +697,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-karmic-proposed:
type: src-pkg
@@ -687,6 +711,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-karmic-backports:
type: src-pkg
@@ -700,6 +725,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
#ubuntu-jaunty:
@@ -741,6 +767,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-hardy-security:
type: src-pkg
@@ -754,6 +781,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-hardy-updates:
type: src-pkg
@@ -767,6 +795,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-hardy-proposed:
type: src-pkg
@@ -780,6 +809,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
ubuntu-hardy-backports:
type: src-pkg
@@ -793,6 +823,7 @@
uploaders-table: ubuntu_uploaders
packages-schema: packages
sources-schema: sources
+ descriptions-table: ubuntu_descriptions
#ubuntu-dapper:
@@ -965,14 +996,11 @@
ddtp:
type: ddtp
- xxx-update-command: /org/udd.debian.org/udd/scripts/fetch_ddtp_translations.sh /org/udd.debian.org/mirrors/ddtp ddtp.debian.net/Translation_udd sid lenny squeeze
- update-command: /org/udd.debian.org/udd/scripts/fetch_ddtp_translations.sh /org/udd.debian.org/mirrors/ddtp ftp.debian.org/debian sid lenny squeeze
+ update-command: /org/udd.debian.org/udd/scripts/fetch_ddtp_translations.sh /org/udd.debian.org/mirrors/ddtp /org/mirrors/ftp.debian.org/ftp/dists
path: /org/udd.debian.org/mirrors/ddtp
- xxx-mirror: ddtp.debian.net/Translation_udd
- mirror: ftp.debian.org/debian
files: Translation-.*\.gz
- releases: sid lenny squeeze
- table: ddtp
+ mirrorpath: /org/mirrors/ftp.debian.org/ftp/dists
+ table: descriptions
schema: ddtp
ftpnew:
Modified: udd/sql/setup.sql
===================================================================
--- udd/sql/setup.sql 2012-02-09 21:01:20 UTC (rev 2140)
+++ udd/sql/setup.sql 2012-02-11 21:13:46 UTC (rev 2141)
@@ -601,7 +601,7 @@
GRANT SELECT ON all_packages_distrelcomparch TO PUBLIC;
GRANT SELECT ON all_bugs TO PUBLIC;
-CREATE TABLE ddtp (
+CREATE TABLE descriptions (
package text,
release text,
language text,
@@ -610,9 +610,30 @@
description_md5 text, -- md5 sum of the original English description
PRIMARY KEY (package, release, language, description, description_md5)
);
+GRANT SELECT ON descriptions TO PUBLIC;
-GRANT SELECT ON ddtp TO PUBLIC;
+CREATE TABLE ubuntu_descriptions (
+ package text,
+ release text,
+ language text,
+ description text,
+ long_description text,
+ description_md5 text, -- md5 sum of the original English description
+ PRIMARY KEY (package, release, language, description, description_md5)
+);
+GRANT SELECT ON ubuntu_descriptions TO PUBLIC;
+CREATE TABLE derivatives_descriptions (
+ package text,
+ release text,
+ language text,
+ description text,
+ long_description text,
+ description_md5 text, -- md5 sum of the original English description
+ PRIMARY KEY (package, release, language, description, description_md5)
+);
+GRANT SELECT ON derivatives_descriptions TO PUBLIC;
+
-- active_dds view
CREATE VIEW active_dds AS
SELECT DISTINCT carnivore_login.id, login
Modified: udd/sql/upgrade.sql
===================================================================
--- udd/sql/upgrade.sql 2012-02-09 21:01:20 UTC (rev 2140)
+++ udd/sql/upgrade.sql 2012-02-11 21:13:46 UTC (rev 2141)
@@ -365,3 +365,38 @@
PRIMARY KEY (package, release, language, description, description_md5)
);
GRANT SELECT ON ddtp TO PUBLIC;
+-- 2012-02-09
+-- Add description tables for ubuntu and derivatives; rename ddtp table
+ALTER TABLE ddtp RENAME TO descriptions;
+GRANT SELECT ON descriptions TO PUBLIC;
+
+CREATE TABLE ubuntu_descriptions (
+ package text,
+ release text,
+ language text,
+ description text,
+ long_description text,
+ description_md5 text, -- md5 sum of the original English description
+ PRIMARY KEY (package, release, language, description, description_md5)
+);
+GRANT SELECT ON ubuntu_descriptions TO PUBLIC;
+
+CREATE TABLE derivatives_descriptions (
+ package text,
+ release text,
+ language text,
+ description text,
+ long_description text,
+ description_md5 text, -- md5 sum of the original English description
+ PRIMARY KEY (package, release, language, description, description_md5)
+);
+GRANT SELECT ON derivatives_descriptions TO PUBLIC;
+
+DROP VIEW all_packages;
+ALTER TABLE packages DROP COLUMN long_description;
+ALTER TABLE ubuntu_packages DROP COLUMN long_description;
+ALTER TABLE derivatives_packages DROP COLUMN long_description;
+CREATE VIEW all_packages AS
+SELECT * FROM packages
+UNION ALL SELECT * FROM ubuntu_packages
+UNION ALL SELECT * FROM derivatives_packages;
Modified: udd/udd/packages_gatherer.py
===================================================================
--- udd/udd/packages_gatherer.py 2012-02-09 21:01:20 UTC (rev 2140)
+++ udd/udd/packages_gatherer.py 2012-02-11 21:13:46 UTC (rev 2141)
@@ -14,6 +14,7 @@
from time import time
import email.Utils
import re
+import hashlib
def get_gatherer(connection, config, source):
return packages_gatherer(connection, config, source)
@@ -38,7 +39,7 @@
pkgquery = """EXECUTE package_insert
(%(Package)s, %(Version)s, %(Architecture)s, %(Maintainer)s, %(maintainer_name)s, %(maintainer_email)s,
- %(Description)s, %(Long_Description)s, %(Description-md5)s, %(Source)s, %(Source_Version)s, %(Essential)s,
+ %(Description)s, %(Description-md5)s, %(Source)s, %(Source_Version)s, %(Essential)s,
%(Depends)s, %(Recommends)s, %(Suggests)s, %(Enhances)s,
%(Pre-Depends)s, %(Breaks)s, %(Installed-Size)s, %(Homepage)s, %(Size)s,
%(Build-Essential)s, %(Origin)s, %(SHA1)s,
@@ -46,6 +47,10 @@
%(Tag)s, %(Task)s, %(Python-Version)s, %(Ruby-Versions)s, %(Provides)s,
%(Conflicts)s, %(SHA256)s, %(Original-Maintainer)s)"""
+ descriptionquery = """EXECUTE description_insert
+ (%(Package)s, %(Language)s,
+ %(Description)s, %(Long_Description)s, %(Description-md5)s)"""
+
def __init__(self, connection, config, source):
gatherer.__init__(self, connection, config, source)
# The ID for the distribution we want to include
@@ -58,6 +63,7 @@
# because different architectures include packages for architecture 'all'
# with the same version, and we don't want these duplicate entries
self.imported_all_pkgs = {}
+ self.add_descriptions = False
def build_dict(self, control):
"""Build a dictionary from the control dictionary.
@@ -89,6 +95,7 @@
it is called.The Format of the sequence is expected to be that of a
debian packages file."""
pkgs = []
+ pkgdescs = []
# The fields that are to be read. Other fields are ignored
for control in debian.deb822.Packages.iter_paragraphs(sequence):
@@ -103,11 +110,20 @@
# We split the description
if 'Description' in d:
- if len(d['Description'].split("\n",1)) > 1:
- d['Long_Description'] = d['Description'].split("\n",1)[1]
- else:
- d['Long_Description'] = ''
- d['Description'] = d['Description'].split("\n",1)[0]
+ if self.add_descriptions and \
+ ('Description-md5' not in d or not d['Description-md5']):
+ try:
+ d['Description-md5'] = hashlib.md5((d['Description']+"\n").encode('utf-8')).hexdigest()
+ d['Language'] = 'en'
+ pkgdescs.append(d)
+ except UnicodeEncodeError:
+ self.warned_about['%s description encoding' % d['Package']] = 1
+ if len(d['Description'].split("\n",1)) > 1:
+ d['Long_Description'] = d['Description'].split("\n",1)[1]
+ else:
+ d['Long_Description'] = ''
+ d['Description'] = d['Description'].split("\n",1)[0]
+ # Calculate Description-md5 for releases that don't include it
# Convert numbers to numbers
for f in ['Installed-Size', 'Size']:
@@ -133,6 +149,12 @@
except psycopg2.ProgrammingError:
print "Error while inserting packages"
raise
+ try:
+ if self.add_descriptions:
+ cur.executemany(self.descriptionquery, pkgdescs)
+ except psycopg2.ProgrammingError:
+ print "Error while inserting descriptions"
+ raise
def setup(self):
if 'schema-dir' in self.config['general']:
@@ -159,6 +181,9 @@
# Get distribution ID
self._distr = src_cfg['distribution']
+ self.add_descriptions = ('descriptions-table' in self.my_config
+ and self.my_config['descriptions-table'])
+
cur = self.cursor()
# defer constraints checking until the end of the transaction
cur.execute("SET CONSTRAINTS ALL DEFERRED")
@@ -166,12 +191,17 @@
# For every part and every architecture, import the packages into the DB
for comp in src_cfg['components']:
cur.execute("DELETE FROM %s WHERE distribution = '%s' AND release = '%s' AND component = '%s'" %\
- (table, self._distr, src_cfg['release'], comp))
+ (table, self._distr, src_cfg['release'], comp))
+ # For releases that have long descriptions in Packages and not in Translation,
+ # add the description to the configured descriptions table.
+ if self.add_descriptions:
+ cur.execute("DELETE FROM %s WHERE release = '%s' AND language = '%s'" %\
+ (src_cfg['descriptions-table'], src_cfg['release'], 'en'))
for arch in src_cfg['archs']:
path = os.path.join(src_cfg['directory'], comp, 'binary-' + arch, 'Packages.gz')
try:
cur.execute("""PREPARE package_insert AS INSERT INTO %s
- (Package, Version, Architecture, Maintainer, maintainer_name, maintainer_email, Description, Long_Description, description_md5, Source,
+ (Package, Version, Architecture, Maintainer, maintainer_name, maintainer_email, Description, description_md5, Source,
Source_Version, Essential, Depends, Recommends, Suggests, Enhances,
Pre_Depends, Breaks, Installed_Size, Homepage, Size,
build_essential, origin, sha1, replaces, section,
@@ -181,8 +211,22 @@
VALUES
( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15,
$16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28,
- $29, $30, $31, $32, $33, $34, $35, $36, $37, '%s', '%s', '%s')
+ $29, $30, $31, $32, $33, $34, $35, $36, '%s', '%s', '%s')
""" % (table, self._distr, src_cfg['release'], comp))
+ if self.add_descriptions:
+ cur.execute("""PREPARE description_insert AS
+ INSERT INTO %s
+ (package, release, language, description, long_description, description_md5)
+ (SELECT $1 AS package, '%s' AS release, $2 AS language,
+ $3 AS description, $4 AS long_description, $5 AS description_md5
+ WHERE NOT EXISTS
+ (SELECT 1
+ FROM %s
+ WHERE package=$1 AND release='%s' AND language=$2 AND
+ description=$3 AND long_description=$4 AND description_md5=$5))
+
+ """ % (src_cfg['descriptions-table'], src_cfg['release'],
+ src_cfg['descriptions-table'], src_cfg['release']))
# aux.print_debug("Reading file " + path)
# Copy content from gzipped file to temporary file, so that apt_pkg is
# used by debian
@@ -197,7 +241,9 @@
except IOError, (e, message):
print "Could not read packages from %s: %s" % (path, message)
sys.exit(1)
- cur.execute("DEALLOCATE package_insert")
+ cur.execute("DEALLOCATE package_insert")
+ if self.add_descriptions:
+ cur.execute("DEALLOCATE description_insert")
# Fill the summary tables
cur.execute("DELETE FROM %s" % (table + '_summary'));
cur.execute("""INSERT INTO %s (package, version, source, source_version,
More information about the Collab-qa-commits
mailing list