[Collab-qa-commits] r1961 - in udd: . scripts sql udd
Lucas Nussbaum
lucas at alioth.debian.org
Thu Jun 30 07:02:38 UTC 2011
Author: lucas
Date: 2011-06-30 07:02:38 +0000 (Thu, 30 Jun 2011)
New Revision: 1961
Modified:
udd/config-org.yaml
udd/scripts/check_timestamps
udd/scripts/monitor_locks
udd/sql/setup.sql
udd/udd.py
udd/udd/aux.py
udd/udd/bugs_gatherer.pl
udd/udd/carnivore_gatherer.py
udd/udd/sources_gatherer.py
udd/udd/upload_history_gatherer.py
Log:
oops, mass-commit of uncommitted changes, including the addition of ubuntu-upload-history
Modified: udd/config-org.yaml
===================================================================
--- udd/config-org.yaml 2011-06-09 09:06:03 UTC (rev 1960)
+++ udd/config-org.yaml 2011-06-30 07:02:38 UTC (rev 1961)
@@ -277,7 +277,7 @@
debian-sid:
type: src-pkg
- archs: [alpha, amd64, armel, hppa, hurd-i386,
+ archs: [amd64, armel, hurd-i386,
i386, ia64, mips,
mipsel, powerpc, s390, sparc, kfreebsd-amd64, kfreebsd-i386]
directory: /org/mirrors/ftp.debian.org/ftp/dists/sid/
@@ -292,7 +292,7 @@
debian-experimental:
type: src-pkg
- archs: [alpha, amd64, armel, hppa, hurd-i386,
+ archs: [amd64, armel, hurd-i386,
i386, ia64, mips,
mipsel, powerpc, s390, sparc, kfreebsd-amd64, kfreebsd-i386]
directory: /org/mirrors/ftp.debian.org/ftp/dists/experimental/
@@ -494,7 +494,16 @@
update-command: if [ ! -e /org/udd.debian.org/tmp/upload-history/ ]; then mkdir /org/udd.debian.org/tmp/upload-history/; fi; cd /org/udd.debian.org/tmp/upload-history ; lftp -c 'open http://master.debian.org/~lucas/ddc-parser/ ; mirror -e -P .'
schema: upload_history
table: upload_history
+ #only-recent: False
+ubuntu-upload-history:
+ type: upload-history
+ path: /home/laney/ubuntu-udd/ubuntu-changes/
+ schema: upload_history
+ table: ubuntu_upload_history
+ #only-recent: False
+
+
hints:
type: hints
path: /org/udd.debian.org/tmp/hints
Modified: udd/scripts/check_timestamps
===================================================================
--- udd/scripts/check_timestamps 2011-06-09 09:06:03 UTC (rev 1960)
+++ udd/scripts/check_timestamps 2011-06-30 07:02:38 UTC (rev 1961)
@@ -1,7 +1,7 @@
echo "select source, max(end_time) from timestamps
where command = 'run' and source not in (
-'debian-etch', 'debian-backports-etch', 'debian-volatile-etch', 'debian-volatile-sloppy-etch', 'debian-volatile-proposed-etch', 'debian-etch-proposed-updates', 'debian-etch-security', 'ddtp'
+'debian-etch', 'debian-backports-etch', 'debian-volatile-etch', 'debian-volatile-sloppy-etch', 'debian-volatile-proposed-etch', 'debian-etch-proposed-updates', 'debian-etch-security', 'ddtp', 'ubuntu-maverick'
) and source not in (
-select source from timestamps where command = 'run' and end_time > (current_timestamp - interval '36 hours')) group by source;" | psql -q -t udd | grep -v "^$"
+select source from timestamps where command = 'run' and end_time > (current_timestamp - interval '36 hours')) group by source;" | psql -p 5441 -q -t udd | grep -v "^$"
ls -l --time-style=full-iso /org/bugs.debian.org/www/project/trace/ |sort -n -k 6 | grep -v $(date --iso) | grep -v $(date --iso -d '1 day ago') | grep -v "^total"
Modified: udd/scripts/monitor_locks
===================================================================
--- udd/scripts/monitor_locks 2011-06-09 09:06:03 UTC (rev 1960)
+++ udd/scripts/monitor_locks 2011-06-30 07:02:38 UTC (rev 1961)
@@ -1,15 +1,15 @@
#!/bin/bash
set -e
-ungranted=$(echo "select locktype, database, relation, pid, mode, granted from pg_locks where granted='f' and mode !='ShareUpdateExclusiveLock';" | psql -A -q udd | grep row | awk '{print $1}' | cut -c 2-)
+ungranted=$(echo "select locktype, database, relation, pid, mode, granted from pg_locks where granted='f' and mode !='ShareUpdateExclusiveLock';" | psql -p 5441 -A -q udd | grep row | awk '{print $1}' | cut -c 2-)
if [ -z "$ungranted" ]; then
-echo "select locktype, database, relation, pid, mode, granted from pg_locks where granted='f';" | psql -A -q udd
+echo "select locktype, database, relation, pid, mode, granted from pg_locks where granted='f';" | psql -p 5441 -A -q udd
fi
-[ "$ungranted" -eq 0 ] || echo "select locktype, database, relation, pid, mode, granted from pg_locks where granted='f';" | psql -A -q udd
+[ "$ungranted" -eq 0 ] || echo "select locktype, database, relation, pid, mode, granted from pg_locks where granted='f';" | psql -p 5441 -A -q udd
if [ "$ungranted" -ne 0 ]; then
- echo "select locktype, database, relation, pid, mode, granted from pg_locks;" | psql -q udd
+ echo "select locktype, database, relation, pid, mode, granted from pg_locks;" | psql -p 5441 -q udd
echo "##########################"
ps fx
echo "##########################"
Modified: udd/sql/setup.sql
===================================================================
--- udd/sql/setup.sql 2011-06-09 09:06:03 UTC (rev 1960)
+++ udd/sql/setup.sql 2011-06-30 07:02:38 UTC (rev 1961)
@@ -205,6 +205,11 @@
(id int REFERENCES bugs, package text, source text,
PRIMARY KEY (id, package));
+CREATE INDEX bugs_packages_source_idx ON bugs_packages (source);
+CREATE INDEX bugs_packages_package_idx ON bugs_packages (package);
+CREATE INDEX bugs_source_idx ON bugs (source);
+CREATE INDEX bugs_package_idx ON bugs (package);
+
CREATE INDEX sources_release_idx ON sources(release);
CREATE TABLE bugs_merged_with
@@ -474,6 +479,35 @@
group by uh1.source;
GRANT SELECT ON upload_history_nmus TO PUBLIC;
+CREATE TABLE ubuntu_upload_history
+ (source text, version debversion, date timestamp with time zone,
+ changed_by text, changed_by_name text, changed_by_email text, maintainer text, maintainer_name text, maintainer_email text, nmu boolean, signed_by text, signed_by_name text, signed_by_email text, key_id text, distribution text, file text,
+ fingerprint text,
+ PRIMARY KEY (source, version));
+
+CREATE TABLE ubuntu_upload_history_architecture
+ (source text, version debversion, architecture text, file text,
+ PRIMARY KEY (source, version, architecture),
+FOREIGN KEY (source, version) REFERENCES ubuntu_upload_history DEFERRABLE);
+
+CREATE TABLE ubuntu_upload_history_closes
+ (source text, version debversion, bug int, file text,
+ PRIMARY KEY (source, version, bug),
+FOREIGN KEY (source, version) REFERENCES ubuntu_upload_history DEFERRABLE);
+
+GRANT SELECT ON ubuntu_upload_history TO PUBLIC;
+GRANT SELECT ON ubuntu_upload_history_architecture TO PUBLIC;
+GRANT SELECT ON ubuntu_upload_history_closes TO PUBLIC;
+
+CREATE VIEW ubuntu_upload_history_nmus AS
+select uh1.source, count(*) AS nmus
+from ubuntu_upload_history uh1, (select source, max(date) as date from upload_history where nmu = false group by source) uh2
+where uh1.nmu = true
+and uh1.source = uh2.source
+and uh1.date > uh2.date
+group by uh1.source;
+GRANT SELECT ON ubuntu_upload_history_nmus TO PUBLIC;
+
-- Ubuntu bugs
CREATE TABLE ubuntu_bugs (
bug int,
Modified: udd/udd/aux.py
===================================================================
--- udd/udd/aux.py 2011-06-09 09:06:03 UTC (rev 1960)
+++ udd/udd/aux.py 2011-06-30 07:02:38 UTC (rev 1961)
@@ -103,3 +103,10 @@
name = re.sub('^[^\w]*([^<]+[.\w\)\]]) *[<\(][.\w]+@[.\w]+[>\)].*', '\\1', str)
print_debug("parse_email: %s ---> %s <%s>" % (str, name, email))
return name, email
+
+def validutf8(str):
+ try:
+ str.decode('utf-8')
+ return True
+ except UnicodeDecodeError:
+ return False
Modified: udd/udd/bugs_gatherer.pl
===================================================================
--- udd/udd/bugs_gatherer.pl 2011-06-09 09:06:03 UTC (rev 1960)
+++ udd/udd/bugs_gatherer.pl 2011-06-30 07:02:38 UTC (rev 1961)
@@ -166,12 +166,11 @@
print "Fetching list of ",scalar(@modified_bugs), " bugs to insert: ",(time() - $t),"s\n" if $timing;
$t = time();
+ my $modbugs = join ',', @modified_bugs;
+
foreach my $prefix ($table, $archived_table) {
foreach my $postfix (qw{_packages _merged_with _found_in _fixed_in _tags _blocks _blockedby}, '') {
- my $sth = $dbh->prepare("DELETE FROM $prefix$postfix WHERE id = \$1");
- map {
- $sth->execute($_) or die $!;
- } @modified_bugs;
+ $dbh->do("DELETE FROM $prefix$postfix where id in ($modbugs)") or die
}
}
print "Deleting bugs: ",(time() - $t),"s\n" if $timing;
@@ -189,7 +188,8 @@
my $insert_bugs_blocks_handle = $dbh->prepare("INSERT INTO ${table}_blocks (id, blocked) VALUES (\$1, \$2)");
my $insert_bugs_blockedby_handle = $dbh->prepare("INSERT INTO ${table}_blockedby (id, blocker) VALUES (\$1, \$2)");
$insert_bugs_handle->bind_param(4, undef, SQL_INTEGER);
- $insert_bugs_handle->bind_param(18, undef, SQL_INTEGER);
+ $insert_bugs_handle->bind_param(16, undef, SQL_INTEGER);
+ $insert_bugs_handle->bind_param(19, undef, SQL_INTEGER);
$t = time();
foreach my $bug_nr (@modified_bugs) {
Modified: udd/udd/carnivore_gatherer.py
===================================================================
--- udd/udd/carnivore_gatherer.py 2011-06-09 09:06:03 UTC (rev 1960)
+++ udd/udd/carnivore_gatherer.py 2011-06-30 07:02:38 UTC (rev 1961)
@@ -5,7 +5,7 @@
See merkel.debian.org:/org/qa.debian.org/carnivore/
"""
-from aux import quote
+from aux import quote, validutf8
import sys
import gzip
from gatherer import gatherer
@@ -73,8 +73,17 @@
qs = []
for email in record["emails"]:
qs.append("EXECUTE carnivore_email_insert (%d, %s)" % (record_number, quote(email)))
+ added = {}
for name in record["names"]:
- qs.append("EXECUTE carnivore_name_insert (%d, %s)" % (record_number, quote(name)))
+ if not validutf8(name):
+ try:
+ name = name.decode('latin1').encode('utf-8')
+ except:
+ print "Error while decoding: " + name
+ continue
+ if not name in added:
+ qs.append("EXECUTE carnivore_name_insert (%d, %s)" % (record_number, quote(name)))
+ added[name] = True
if "login" in record:
qs.append("EXECUTE carnivore_login_insert (%d, %s)" % (record_number, quote(record["login"])))
for key_type in ['keyring', 'ldap', 'emeritus', 'removed', 'dm']:
Modified: udd/udd/sources_gatherer.py
===================================================================
--- udd/udd/sources_gatherer.py 2011-06-09 09:06:03 UTC (rev 1960)
+++ udd/udd/sources_gatherer.py 2011-06-30 07:02:38 UTC (rev 1961)
@@ -30,7 +30,7 @@
'Vcs-Cvs': 0, 'Vcs-Darcs': 0, 'Vcs-Git': 0, 'Vcs-Hg': 0, 'Vcs-Svn': 0,
'Vcs-Mtn':0,
'X-Vcs-Browser': 0, 'Vcs-Browser': 0, 'X-Vcs-Bzr': 0, 'X-Vcs-Darcs': 0, 'X-Vcs-Svn': 0, 'X-Vcs-Hg':0, 'X-Vcs-Git':0, 'Vcs-Browse':0,
- 'Directory':0, 'Comment':0, 'Origin':0, 'Url':0, 'X-Collab-Maint':0, 'Autobuild':0, 'Vcs-Cvs:':0, 'Python-Standards-Version':0, 'url':0, 'originalmaintainer':0, 'Originalmaintainer':0, 'Build-Recommends':0, 'Maintainer-Homepage': 0, 'Python3-Version': 0}
+ 'Directory':0, 'Comment':0, 'Origin':0, 'Url':0, 'X-Collab-Maint':0, 'Autobuild':0, 'Vcs-Cvs:':0, 'Python-Standards-Version':0, 'url':0, 'originalmaintainer':0, 'Originalmaintainer':0, 'Build-Recommends':0, 'Maintainer-Homepage': 0, 'Python3-Version': 0, 'Package-List':0}
#Vcs-Cvs: is caused by a bug in python-debian, apparently.
ignorable_re = re.compile("^(Orig-|Original-|Origianl-|Orginal-|Orignal-|Orgiinal-|Orginial-|Debian-|X-Original-|Upstream-)")
vcs = [ 'Svn', 'Git', 'Arch', 'Bzr', 'Cvs', 'Darcs', 'Hg', 'Mtn']
Modified: udd/udd/upload_history_gatherer.py
===================================================================
--- udd/udd/upload_history_gatherer.py 2011-06-09 09:06:03 UTC (rev 1960)
+++ udd/udd/upload_history_gatherer.py 2011-06-30 07:02:38 UTC (rev 1961)
@@ -51,14 +51,18 @@
query_archs = "EXECUTE uh_arch_insert(%(Source)s, %(Version)s, %(arch)s, %(File)s)"
query_closes = "EXECUTE uh_close_insert(%(Source)s, %(Version)s, %(closes)s, %(File)s)"
added = {}
- files = glob(path + '/debian-devel-changes.*')
+ files = glob(path + '/*-changes*mbox*')
files.sort()
if onlyrecent:
files = files[-2:]
+ print files
+ else:
+ print "Doing full import!"
+ cursor.execute("delete from " + self.my_config['table'] + "_architecture")
+ cursor.execute("delete from " + self.my_config['table'] + "_closes")
+ cursor.execute("delete from " + self.my_config['table'])
for name in files:
-# for name in files:
bname = os.path.basename(name).replace(".gz","").replace(".out","")
-# print bname
cursor.execute("DELETE FROM " + self.my_config['table'] + "_architecture where file='%s'" % (bname))
cursor.execute("DELETE FROM " + self.my_config['table'] + "_closes where file='%s'" % (bname))
cursor.execute("DELETE FROM " + self.my_config['table'] + " where file='%s'" % (bname))
@@ -70,6 +74,8 @@
f = open(name)
current = {}
current['Fingerprint'] = 'N/A' # hack: some entries don't have fp
+ current['NMU'] = False
+ current['Key'] = ''
current['File'] = bname
last_field = None
line_count = 0
@@ -93,10 +99,13 @@
if (current['Source'], current['Version']) in added or \
(current['Source'], current['Version']) == ('libapache-authznetldap-perl', '0.07-4') or \
(current['Source'], current['Version']) == ('knj10font', '1.01-1') or \
+ (current['Source'], current['Version']) == ('xmorph', '1:20010421') or \
current['Message-Date'] == 'None':
print "Skipping upload: "+current['Source']+" "+current['Version']+" "+current['Date']
current = {}
current['Fingerprint'] = 'N/A' # hack: some entries don't have fp
+ current['NMU'] = False
+ current['Key'] = ''
current['File'] = bname
last_field = None
continue
@@ -113,6 +122,8 @@
uploads_closes.append(current_closes)
current = {}
current['Fingerprint'] = 'N/A' # hack: some entries don't have fp
+ current['NMU'] = False
+ current['Key'] = ''
current['File'] = bname
last_field = None
continue
@@ -129,6 +140,10 @@
last_field = field
+ #print uploads
+ #for u in uploads:
+ # print u
+ # cursor.execute(query, u)
cursor.executemany(query, uploads)
cursor.executemany(query_archs, uploads_archs)
cursor.executemany(query_closes, uploads_closes)
Modified: udd/udd.py
===================================================================
--- udd/udd.py 2011-06-09 09:06:03 UTC (rev 1960)
+++ udd/udd.py 2011-06-30 07:02:38 UTC (rev 1961)
@@ -23,16 +23,12 @@
def insert_timestamps(config, source, command, start_time, end_time):
connection = udd.aux.open_connection(config)
cur = connection.cursor()
- values = { 'source' : source,
- 'command' : command,
- 'start_time' : start_time,
- 'end_time' : end_time }
cur.execute("""INSERT INTO timestamps
(source, command, start_time, end_time)
- VALUES (%(source)s, %(command)s, %(start_time)s,
- %(end_time)s)""",
- values)
+ VALUES ('%s', '%s', '%s', '%s')""" %
+ (source, command, start_time, end_time))
connection.commit()
+ connection.close()
def get_timestamp():
return time.strftime('%Y-%m-%d %H:%M:%S')
@@ -88,7 +84,9 @@
exec "gatherer.%s()" % command
connection.commit()
end_time = get_timestamp()
+# print "lalala"
insert_timestamps(config, src, command, start_time, end_time)
+# print "lalala2"
except:
udd.aux.unlock(config, src)
raise
More information about the Collab-qa-commits
mailing list