[Collab-qa-commits] r1026 - in udd/src: . udd

[he at alioth.debian.org]

Author: he
Date: 2008-08-09 01:31:01 +0000 (Sat, 09 Aug 2008)
New Revision: 1026

Added:
   udd/src/udd/lintian_gatherer.py
Modified:
   udd/src/setup-db.sql
   udd/src/test.yaml
Log:
First draft for a lintian.log UDD import


Modified: udd/src/setup-db.sql
===================================================================
--- udd/src/setup-db.sql	2008-08-09 00:36:31 UTC (rev 1025)
+++ udd/src/setup-db.sql	2008-08-09 01:31:01 UTC (rev 1026)
@@ -126,6 +126,23 @@
  (id int, login text,
    PRIMARY KEY(id));
 
+CREATE DOMAIN lintian_tag_type AS TEXT
+NOT NULL
+CHECK(
+     VALUE = 'error'
+  OR VALUE = 'warning'
+  OR VALUE = 'information'
+  OR VALUE = 'experimental'
+  OR VALUE = 'overriden'
+);
+
+CREATE TABLE lintian (
+  package TEXT NOT NULL,
+  tag_type lintian_tag_type,
+  package_type TEXT,
+  tag TEXT NOT NULL
+);
+
 CREATE INDEX packages_source_idx on packages(source);
 CREATE INDEX sources_distribution_idx on sources(distribution);
 CREATE INDEX sources_release_idx on sources(release);
@@ -156,4 +173,5 @@
 GRANT SELECT ON carnivore_names TO PUBLIC;
 GRANT SELECT ON carnivore_keys TO PUBLIC;
 GRANT SELECT ON carnivore_login TO PUBLIC;
+GRANT SELECT ON lintian TO PUBLIC;
 

Modified: udd/src/test.yaml
===================================================================
--- udd/src/test.yaml	2008-08-09 00:36:31 UTC (rev 1025)
+++ udd/src/test.yaml	2008-08-09 01:31:01 UTC (rev 1026)
@@ -11,6 +11,7 @@
     upload-history: module udd.upload_history_gatherer
     bugs: exec DEBBUGS_CONFIG_FILE=/org/udd.debian.net/mirrors/bugs.debian.org/etc/config perl ./udd/bugs_gatherer.pl 
     carnivore: module udd.carnivore_gatherer
+    lintian: module udd.lintian_gatherer
     #src-pkg: python sources_gatherer.py
   debug: 1
   update-timestamp-folder: ./timestamps/
@@ -174,3 +175,9 @@
   names-table: carnivore_names
   keys-table: carnivore_keys
   login-table: carnivore_login
+
+lintian:
+  type: lintian
+  update-command: rm -f /org/udd.debian.net/mirrors/lintian.log && wget http://lintian.debian.org/lintian.log -O /org/udd.debian.net/mirrors/lintian.log
+  path: /org/udd.debian.net/mirrors/lintian.log
+  table: lintian

Added: udd/src/udd/lintian_gatherer.py
===================================================================
--- udd/src/udd/lintian_gatherer.py	                        (rev 0)
+++ udd/src/udd/lintian_gatherer.py	2008-08-09 01:31:01 UTC (rev 1026)
@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+
+"""
+This script imports lintian run results into the database
+See lintian.debian.org
+"""
+
+from aux import quote
+from gatherer import gatherer
+import re
+
+def get_gatherer(connection, config):
+  return carnivore_gatherer(connection, config)
+
+class lintian_gatherer(gatherer):
+  #RE to parse lintian output, pushing the tag code to $1, package name
+  #to $2, pkg type to $3, tag name to $4 and extra info to $5
+  # (stolen from Russ Allbery, thanks dude)
+  output_re = re.compile("([EWIXO]): (\S+)(?: (\S+))?: (\S+)(?:\s+(.*))?/");
+
+  code_to_tag_type_map = {
+    "E": "error",
+    "W": "warning",
+    "I": "information",
+    "X": "experimental",
+    "O": "overriden",
+  }
+
+  def __init__(self, connection, config):
+    gatherer.__init__(self, connection, config)
+
+  def run(self, source):
+    try:
+      my_config = self.config[source]
+    except:
+      raise
+
+    #check that the config contains everything we need:
+    for key in ['path', 'table']:
+      if not key in my_config:
+        raise aux.ConfigException, "%s not configured for source %s" % (key, source)
+
+    #start harassing the DB, preparing the final inserts and making place
+    #for the new data:
+    cur = self.cursor()
+
+    cur.execute("DELETE FROM %s" % my_config["table"])
+
+    cur.execute("""PREPARE lintian_insert 
+      AS INSERT INTO %s (package, package_type, tag, tag_type)
+      VALUES ($1, $2, $3, $4)""" % (my_config['table']))
+
+    lintian_data = open(my_config['path'])
+    line_number = 0
+    for line in lintian_data:
+      line_number += 1
+
+      #ignore information and verbose output:
+      if line.startswith("N:"):
+        continue
+
+      match = lintian_gatherer.output_re.match(line)
+      if match:
+        (code, pkg, pkg_type, tag, extra) = match.groups();
+
+        cur.execute("EXECUTE lintian_insert (%s, %s, %s, %s)" % \
+          (pkg, pkg_type, tag, lintian_gatherer.code_to_tag_type_map[code]));
+      else:
+        print "Can't parse line %d: %s" % (line_number, line)
+
+if __name__ == '__main__':
+  main()
+
+# vim:set et tabstop=2: