[Collab-qa-commits] r1770 - in udd: . sql udd
Lucas Nussbaum
lucas at alioth.debian.org
Thu Aug 5 15:50:35 UTC 2010
Author: lucas
Date: 2010-08-05 15:50:35 +0000 (Thu, 05 Aug 2010)
New Revision: 1770
Modified:
udd/config-org.yaml
udd/sql/setup.sql
udd/udd/upload_history_gatherer.py
Log:
upload-history partial rewrite
Modified: udd/config-org.yaml
===================================================================
--- udd/config-org.yaml 2010-08-05 15:48:02 UTC (rev 1769)
+++ udd/config-org.yaml 2010-08-05 15:50:35 UTC (rev 1770)
@@ -375,7 +375,7 @@
upload-history:
type: upload-history
path: /org/udd.debian.org/tmp/upload-history
- update-command: if [ ! -e /org/udd.debian.org/tmp/upload-history/ ]; then mkdir /org/udd.debian.org/tmp/upload-history/; fi; rm -rf /org/udd.debian.org/tmp/upload-history/*; wget -q -r --no-parent -nd -A debian-devel-* -P /org/udd.debian.org/tmp/upload-history http://qa.debian.org/~filippo/ddc/
+ update-command: if [ ! -e /org/udd.debian.org/tmp/upload-history/ ]; then mkdir /org/udd.debian.org/tmp/upload-history/; fi; lftp -c 'mirror -e -P http://master.debian.org/~lucas/ddc-parser/ /org/udd.debian.org/tmp/upload-history'
schema: upload_history
table: upload_history
Modified: udd/sql/setup.sql
===================================================================
--- udd/sql/setup.sql 2010-08-05 15:48:02 UTC (rev 1769)
+++ udd/sql/setup.sql 2010-08-05 15:50:35 UTC (rev 1770)
@@ -435,17 +435,17 @@
CREATE TABLE upload_history
(source text, version debversion, date timestamp with time zone,
- changed_by text, changed_by_name text, changed_by_email text, maintainer text, maintainer_name text, maintainer_email text, nmu boolean, signed_by text, signed_by_name text, signed_by_email text, key_id text,
+ changed_by text, changed_by_name text, changed_by_email text, maintainer text, maintainer_name text, maintainer_email text, nmu boolean, signed_by text, signed_by_name text, signed_by_email text, key_id text, file text,
fingerprint text,
PRIMARY KEY (source, version));
CREATE TABLE upload_history_architecture
- (source text, version debversion, architecture text,
+ (source text, version debversion, architecture text, file text,
PRIMARY KEY (source, version, architecture),
FOREIGN KEY (source, version) REFERENCES upload_history DEFERRABLE);
CREATE TABLE upload_history_closes
- (source text, version debversion, bug int,
+ (source text, version debversion, bug int, file text,
PRIMARY KEY (source, version, bug),
FOREIGN KEY (source, version) REFERENCES upload_history DEFERRABLE);
Modified: udd/udd/upload_history_gatherer.py
===================================================================
--- udd/udd/upload_history_gatherer.py 2010-08-05 15:48:02 UTC (rev 1769)
+++ udd/udd/upload_history_gatherer.py 2010-08-05 15:50:35 UTC (rev 1770)
@@ -8,6 +8,7 @@
import psycopg2
import sys
import email.Utils
+import os.path
def get_gatherer(config, connection, source):
return upload_history_gatherer(config, connection, source)
@@ -30,28 +31,33 @@
cursor = self.cursor()
- cursor.execute("DELETE FROM " + self.my_config['table'] + '_architecture')
- cursor.execute("DELETE FROM " + self.my_config['table'] + '_closes')
- cursor.execute("DELETE FROM " + self.my_config['table'])
-
cursor.execute("PREPARE uh_insert AS INSERT INTO %s (source, \
- version, date, changed_by, changed_by_name, changed_by_email, maintainer, maintainer_name, maintainer_email, nmu, signed_by, signed_by_name, signed_by_email, key_id, fingerprint) VALUES \
- ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)" % self.my_config['table'])
+ version, date, changed_by, changed_by_name, changed_by_email, maintainer, maintainer_name, maintainer_email, nmu, signed_by, signed_by_name, signed_by_email, key_id, fingerprint, distribution, file) VALUES \
+ ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)" % self.my_config['table'])
cursor.execute("PREPARE uh_arch_insert AS INSERT INTO %s (source, \
- version, architecture) VALUES \
- ($1, $2, $3)" % (self.my_config['table'] + '_architecture'))
- cursor.execute("PREPARE uh_close_insert AS INSERT INTO %s (source, version, bug) \
- VALUES ($1, $2, $3)" % (self.my_config['table'] + '_closes'))
+ version, architecture, file) VALUES \
+ ($1, $2, $3, $4)" % (self.my_config['table'] + '_architecture'))
+ cursor.execute("PREPARE uh_close_insert AS INSERT INTO %s (source, version, bug, file) \
+ VALUES ($1, $2, $3, $4)" % (self.my_config['table'] + '_closes'))
query = "EXECUTE uh_insert(%(Source)s, %(Version)s, %(Message-Date)s, \
%(Changed-By)s, %(Changed-By_name)s, %(Changed-By_email)s, \
%(Maintainer)s, %(Maintainer_name)s, %(Maintainer_email)s, %(NMU)s, \
%(Signed-By)s, %(Signed-By_name)s, %(Signed-By_email)s, %(Key)s, \
- %(Fingerprint)s)"
- query_archs = "EXECUTE uh_arch_insert(%(Source)s, %(Version)s, %(arch)s)"
- query_closes = "EXECUTE uh_close_insert(%(Source)s, %(Version)s, %(closes)s)"
+ %(Fingerprint)s, %(Distribution)s, %(File)s)"
+ query_archs = "EXECUTE uh_arch_insert(%(Source)s, %(Version)s, %(arch)s, %(File)s)"
+ query_closes = "EXECUTE uh_close_insert(%(Source)s, %(Version)s, %(closes)s, %(File)s)"
added = {}
- for name in glob(path + '/debian-devel-changes.*'):
+ files = glob(path + '/debian-devel-changes.*')
+ files.sort()
+# for name in files:
+ for name in files[-2:]:
+ bname = os.path.basename(name)
+# print bname
+ cursor.execute("DELETE FROM " + self.my_config['table'] + "_architecture where file='%s'" % (bname))
+ cursor.execute("DELETE FROM " + self.my_config['table'] + "_closes where file='%s'" % (bname))
+ cursor.execute("DELETE FROM " + self.my_config['table'] + " where file='%s'" % (bname))
+
f = None
if name.endswith(".gz"):
f = gzip.open(name)
@@ -59,6 +65,7 @@
f = open(name)
current = {}
current['Fingerprint'] = 'N/A' # hack: some entries don't have fp
+ current['File'] = bname
last_field = None
line_count = 0
uploads = []
@@ -81,21 +88,23 @@
print "Skipping upload: "+current['Source']+" "+current['Version']+" "+current['Date']
current = {}
current['Fingerprint'] = 'N/A' # hack: some entries don't have fp
+ current['File'] = bname
last_field = None
continue
added[(current['Source'], current['Version'])] = True
uploads.append(current)
for arch in set(current['Architecture'].split()):
- current_arch = {'Source': current['Source'], 'Version': current['Version']}
+ current_arch = {'Source': current['Source'], 'Version': current['Version'], 'File': bname}
current_arch['arch'] = arch
uploads_archs.append(current_arch)
if current['Closes'] != 'N/A':
for closes in set(current['Closes'].split()):
- current_closes = {'Source': current['Source'], 'Version': current['Version']}
+ current_closes = {'Source': current['Source'], 'Version': current['Version'], 'File': bname}
current_closes['closes'] = closes
uploads_closes.append(current_closes)
current = {}
current['Fingerprint'] = 'N/A' # hack: some entries don't have fp
+ current['File'] = bname
last_field = None
continue
More information about the Collab-qa-commits
mailing list