[Collab-qa-commits] r1024 - in udd/src: . udd

[he at alioth.debian.org]

Author: he
Date: 2008-08-09 00:26:03 +0000 (Sat, 09 Aug 2008)
New Revision: 1024

Modified:
   udd/src/setup-db.sql
   udd/src/test.yaml
   udd/src/udd/carnivore_gatherer.py
Log:
Actually tested and working carnivore importer.


Modified: udd/src/setup-db.sql
===================================================================
--- udd/src/setup-db.sql	2008-08-08 23:58:51 UTC (rev 1023)
+++ udd/src/setup-db.sql	2008-08-09 00:26:03 UTC (rev 1024)
@@ -114,16 +114,13 @@
   maintainer text, nmu boolean, signed_by text, key_id text);
 
 CREATE TABLE carnivore_emails
- (id int, email text,
-  PRIMARY KEY(id));
+ (id int, email text);
 
 CREATE TABLE carnivore_names
- (id int, name text,
-   PRIMARY KEY(id));
+ (id int, name text);
 
 CREATE TABLE carnivore_keys
- (id int, key text, key_type text,
-   PRIMARY KEY(id));
+ (id int, key text, key_type text);
 
 CREATE TABLE carnivore_login
  (id int, login text,

Modified: udd/src/test.yaml
===================================================================
--- udd/src/test.yaml	2008-08-08 23:58:51 UTC (rev 1023)
+++ udd/src/test.yaml	2008-08-09 00:26:03 UTC (rev 1024)
@@ -10,6 +10,7 @@
     testing-migrations: module udd.testing_migrations_gatherer
     upload-history: module udd.upload_history_gatherer
     bugs: exec DEBBUGS_CONFIG_FILE=/org/udd.debian.net/mirrors/bugs.debian.org/etc/config perl ./udd/bugs_gatherer.pl 
+    carnivore: module udd.carnivore_gatherer
     #src-pkg: python sources_gatherer.py
   debug: 1
   update-timestamp-folder: ./timestamps/
@@ -164,3 +165,12 @@
   type: bugs
   archived: true
   update-command: /org/udd.debian.net/mirrors/sync-bugs.debian.org
+
+carnivore:
+  type: carnivore
+  update-command: /org/udd.debian.net/mirrors/sync-carnivore
+  path: /org/udd.debian.net/mirrors/qa.debian.org/carnivore/report
+  emails-table: carnivore_emails
+  names-table: carnivore_names
+  keys-table: carnivore_keys
+  login-table: carnivore_login

Modified: udd/src/udd/carnivore_gatherer.py
===================================================================
--- udd/src/udd/carnivore_gatherer.py	2008-08-08 23:58:51 UTC (rev 1023)
+++ udd/src/udd/carnivore_gatherer.py	2008-08-09 00:26:03 UTC (rev 1024)
@@ -5,7 +5,7 @@
 See merkel.debian.org:/org/qa.debian.org/carnivore/
 """
 
-import aux
+from aux import quote
 import sys
 import gzip
 from gatherer import gatherer
@@ -15,8 +15,8 @@
   return carnivore_gatherer(connection, config)
 
 class carnivore_gatherer(gatherer):
-  carnivore_field_ignores = ["Packages", "X-MIA", "X-Warning"]
-  carnivore_field_to_DB_map = {
+  field_ignores = ["Packages", "X-MIA", "X-Warning"]
+  field_to_DB_map = {
     "Using emails":    {"name": "emails", "content-type": "comma-separated"},
     "Known as":        {"name": "names", "content-type": "comma-separated"},
     "DD":              {"name": "login", "content-type": "unique-login"},
@@ -52,7 +52,7 @@
       VALUES ($1, $2)""" % (my_config['emails-table']))
     cur.execute("""PREPARE carnivore_name_insert
       AS INSERT INTO %s (id, name)
-      VALUES ($1, $2)""" % (my_config['name-table']))
+      VALUES ($1, $2)""" % (my_config['names-table']))
     cur.execute("""PREPARE carnivore_keys_insert
       AS INSERT INTO %s (id, key, key_type)
       VALUES ($1, $2, $3)""" % (my_config['keys-table']))
@@ -60,7 +60,7 @@
       AS INSERT INTO %s (id, login)
       VALUES ($1, $2)""" % (my_config['login-table']))
 
-    carnivore_data = file.open(my_config['path'])
+    carnivore_data = open(my_config['path'])
     (line_number, record_number) = (0, 1);
     record = {}
     for line in carnivore_data:
@@ -70,16 +70,16 @@
         if 'emails' in record and 'names' in record:
           #collect all queries:
           qs = []
-          for email in record[emails]:
-            qs.append("EXECUTE carnivore_email_insert (%d, '%s')" % (record_number, email))
-          for name in record[names]:
-            qs.append("EXECUTE carnivore_name_insert (%d, '%s')" % (record_number, name))
-          if record[login]:
-            qs.append("EXECUTE carnivore_login_insert (%d, '%s')" % (record_number, record[login]))
-          for key_type in ['keyring', 'ldap', 'emeritus', 'removed']
-            if record["%s_key" % key_type]:
+          for email in record["emails"]:
+            qs.append("EXECUTE carnivore_email_insert (%d, %s)" % (record_number, quote(email)))
+          for name in record["names"]:
+            qs.append("EXECUTE carnivore_name_insert (%d, %s)" % (record_number, quote(name)))
+          if "login" in record:
+            qs.append("EXECUTE carnivore_login_insert (%d, %s)" % (record_number, quote(record["login"])))
+          for key_type in ['keyring', 'ldap', 'emeritus', 'removed']:
+            if ("%s_key" % key_type) in record:
               for key in record["%s_key" % key_type]:
-                qs.append("EXECUTE carnivore_keys_insert (%d, '%s', '%s')" % (record_number, key, key_type))
+                qs.append("EXECUTE carnivore_keys_insert (%d, %s, '%s')" % (record_number, quote(key), key_type))
           for query in qs:
             cur.execute(query)
         record_number += 1
@@ -90,10 +90,10 @@
           print "Couldn't parse line %d: %s" % (line_number, line)
         else:
           field_info = {}
-          if field in carnivore_field_ignores:
+          if field in carnivore_gatherer.field_ignores:
             continue
-          elif carnivore_field_to_DB_map[field]:
-            info = carnivore_field_to_DB_map[field]
+          elif carnivore_gatherer.field_to_DB_map[field]:
+            info = carnivore_gatherer.field_to_DB_map[field]
           else:
             print "Unknown field in line %d: %s" % (line_number, field)
             continue