[Collab-qa-commits] r1834 - in udd: . sql udd

[Lucas Nussbaum]

Author: lucas
Date: 2010-11-01 10:02:47 +0000 (Mon, 01 Nov 2010)
New Revision: 1834

Added:
   udd/udd/deferred_gatherer.py
Modified:
   udd/config-org.yaml
   udd/sql/setup.sql
Log:
deferred gatherer

Modified: udd/config-org.yaml
===================================================================
--- udd/config-org.yaml	2010-10-31 20:18:47 UTC (rev 1833)
+++ udd/config-org.yaml	2010-11-01 10:02:47 UTC (rev 1834)
@@ -28,6 +28,7 @@
     history-daily: module udd.history_daily_gatherer
     i18n-apps: module udd.i18n_apps_gatherer
     hints: module udd.hints_gatherer
+    deferred: module udd.deferred_gatherer
   timestamp-dir: /org/udd.debian.org/timestamps
   lock-dir: /org/udd.debian.org/locks
   archs:
@@ -588,3 +589,7 @@
   release: experimental
   update-command: /org/udd.debian.org/udd/scripts/sync-dist.sh rsync.debian-ports.org::debian/dists/experimental/ /org/udd.debian.org/mirrors/debian-ports/experimental
   no-sources: true
+
+deferred:
+  type: deferred
+  url: http://ftp-master.debian.org/deferred/status

Modified: udd/sql/setup.sql
===================================================================
--- udd/sql/setup.sql	2010-10-31 20:18:47 UTC (rev 1833)
+++ udd/sql/setup.sql	2010-11-01 10:02:47 UTC (rev 1834)
@@ -750,3 +750,28 @@
   OR (type IN ('unblock', 'age-days', 'hint', 'easy') AND (source, version) IN (select source, version from sources where release='sid'))
   OR (type IN ('remove') AND (source, version) IN (select source, version from sources where release='squeeze')) ;
 GRANT SELECT ON relevant_hints TO public;
+
+CREATE TABLE deferred
+ (source text, version debversion, distribution text, urgency text, date timestamp with time zone, delayed_until timestamp, delay_remaining interval,
+ changed_by text, changed_by_name text, changed_by_email text, maintainer text, maintainer_name text, maintainer_email text, changes text,
+ PRIMARY KEY (source, version));
+
+CREATE TABLE deferred_architecture
+ (source text, version debversion, architecture text,
+ PRIMARY KEY (source, version, architecture),
+FOREIGN KEY (source, version) REFERENCES deferred DEFERRABLE);
+
+CREATE TABLE deferred_binary
+ (source text, version debversion, package text,
+ PRIMARY KEY (source, version, package),
+FOREIGN KEY (source, version) REFERENCES deferred DEFERRABLE);
+   
+CREATE TABLE deferred_closes
+ (source text, version debversion, bug int,
+ PRIMARY KEY (source, version, bug),
+FOREIGN KEY (source, version) REFERENCES deferred DEFERRABLE);
+
+GRANT SELECT ON deferred TO public;
+GRANT SELECT ON deferred_architecture TO public;
+GRANT SELECT ON deferred_binary TO public;
+GRANT SELECT ON deferred_closes TO public;

Added: udd/udd/deferred_gatherer.py
===================================================================
--- udd/udd/deferred_gatherer.py	                        (rev 0)
+++ udd/udd/deferred_gatherer.py	2010-11-01 10:02:47 UTC (rev 1834)
@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+
+"""
+This script imports the deferred queue status from
+http://ftp-master.debian.org/deferred/status
+"""
+
+from debian_bundle import deb822
+from os import access, mkdir, unlink, W_OK
+from sys import stderr
+import aux
+from aux import quote
+from gatherer import gatherer
+import email.Utils
+import re
+from time import ctime
+from psycopg2 import IntegrityError, ProgrammingError
+import urllib
+
+def get_gatherer(connection, config, source):
+  return deferred_gatherer(connection, config, source)
+
+DEBUG=0
+def to_unicode(value, encoding='utf-8'):
+  if isinstance(value, str):
+    return value.decode(encoding)
+  else:
+    return unicode(value)
+
+class deferred_gatherer(gatherer):
+  "This class imports the data from Deferred queue into the database"
+
+  def __init__(self, connection, config, source):
+    gatherer.__init__(self, connection, config, source)
+    self.assert_my_config('url')
+
+  def run(self):
+    my_config = self.my_config
+
+    cur = self.cursor()
+
+    cur.execute("PREPARE d_insert AS INSERT INTO deferred (source, version, distribution, urgency, date, delayed_until, delay_remaining, changed_by, changed_by_name, changed_by_email, maintainer, maintainer_name, maintainer_email, changes) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)")
+    q_deferred = "EXECUTE d_insert(%(Source)s, %(Version)s, %(Distribution)s, \
+            %(Urgency)s, %(Date)s, %(Delayed-Until)s, %(Delay-Remaining)s, \
+            %(Changed-By)s, %(Changed-By_name)s, %(Changed-By_email)s, \
+            %(Maintainer)s, %(Maintainer_name)s, %(Maintainer_email)s, \
+            %(Changes)s)"
+    cur.execute("PREPARE da_insert AS INSERT INTO deferred_architecture (source, version, architecture) VALUES ($1, $2, $3)")
+    q_defarch = "EXECUTE da_insert(%(Source)s, %(Version)s, %(Architecture)s)"
+    cur.execute("PREPARE da_binary AS INSERT INTO deferred_binary (source, version, package) VALUES ($1, $2, $3)")
+    q_defbin = "EXECUTE da_binary(%(Source)s, %(Version)s, %(Package)s)"
+    cur.execute("PREPARE da_closes AS INSERT INTO deferred_closes (source, version, bug) VALUES ($1, $2, $3)")
+    q_defcloses = "EXECUTE da_closes(%(Source)s, %(Version)s, %(Bug)s)"
+
+    cur.execute("DELETE FROM deferred_closes")
+    cur.execute("DELETE FROM deferred_binary")
+    cur.execute("DELETE FROM deferred_architecture")
+    cur.execute("DELETE FROM deferred")
+
+    d_list = []
+    da_list = []
+    db_list = []
+    dc_list = []
+    for current in deb822.Deb822.iter_paragraphs(urllib.urlopen(my_config['url'])):
+      current['Changed-By_name'], current['Changed-By_email'] = email.Utils.parseaddr(current['Changed-By'])
+      current['Maintainer_name'], current['Maintainer_email'] = email.Utils.parseaddr(current['Maintainer'])
+      d_list.append(current)
+      for arch in set(current['Architecture'].split()):
+        current_arch = {'Source': current['Source'], 'Version': current['Version']} 
+        current_arch['Architecture'] = arch
+        da_list.append(current_arch)
+      for binary in set(current['Binary'].split()):
+        current_binary = {'Source': current['Source'], 'Version': current['Version']} 
+        current_binary['Package'] = binary
+        db_list.append(current_binary)
+      for bug in set(current['Closes'].split()):
+        current_c = {'Source': current['Source'], 'Version': current['Version']} 
+        current_c['Bug'] = bug
+        dc_list.append(current_c)
+
+    cur.executemany(q_deferred, d_list)
+    cur.executemany(q_defarch, da_list)
+    cur.executemany(q_defbin, db_list)
+    cur.executemany(q_defcloses, dc_list)
+    cur.execute("ANALYZE deferred")
+    cur.execute("ANALYZE deferred_architecture")
+    cur.execute("ANALYZE deferred_binary")
+    cur.execute("ANALYZE deferred_closes")
+
+# vim:set et tabstop=2:0