[Collab-qa-commits] r1087 - in udd/src: . schema udd

neronus-guest at alioth.debian.org neronus-guest at alioth.debian.org
Tue Aug 12 16:38:59 UTC 2008


Author: neronus-guest
Date: 2008-08-12 16:38:58 +0000 (Tue, 12 Aug 2008)
New Revision: 1087

Modified:
   udd/src/schema/upload_history
   udd/src/test.yaml
   udd/src/udd-dispatch.py
   udd/src/udd-update.py
   udd/src/udd/aux.py
   udd/src/udd/gatherer.py
   udd/src/udd/upload_history_gatherer.py
Log:
Locking is implemented
upload_histories has now an extra cloeses table


Modified: udd/src/schema/upload_history
===================================================================
--- udd/src/schema/upload_history	2008-08-12 15:13:01 UTC (rev 1086)
+++ udd/src/schema/upload_history	2008-08-12 16:38:58 UTC (rev 1087)
@@ -1,12 +1,15 @@
 CREATE TABLE %(table)s
  (id serial, package text, version text, date timestamp with time zone,
- changed_by text, maintainer text, nmu boolean, signed_by text, key_id text,
- closes int);
+ changed_by text, maintainer text, nmu boolean, signed_by text, key_id text);
 
 CREATE TABLE %(table)s_architecture
  (id int, architecture text,
  PRIMARY KEY (id, architecture));
   
+CREATE TABLE %(table)s_closes
+ (id int, bug int,
+ PRIMARY KEY (id, bug));
 
 GRANT SELECT ON %(table)s TO PUBLIC;
 GRANT SELECT ON %(table)s_architecture TO PUBLIC;
+GRANT SELECT ON %(table)s_closes TO PUBLIC;

Modified: udd/src/test.yaml
===================================================================
--- udd/src/test.yaml	2008-08-12 15:13:01 UTC (rev 1086)
+++ udd/src/test.yaml	2008-08-12 16:38:58 UTC (rev 1087)
@@ -17,6 +17,7 @@
   debug: 1
   update-timestamp-folder: ./timestamps/
   schema-dir: ./schema/
+  lock-dir: ./locks/
 
   archs:
    [alpha, amd64, arm, armeb, armel, hppa, hurd-i386,

Modified: udd/src/udd/aux.py
===================================================================
--- udd/src/udd/aux.py	2008-08-12 15:13:01 UTC (rev 1086)
+++ udd/src/udd/aux.py	2008-08-12 16:38:58 UTC (rev 1087)
@@ -3,6 +3,8 @@
 import syck
 import sys
 import psycopg2
+from os import path
+import fcntl
 
 # If debug is something that evaluates to True, then print_debug actually prints something
 debug = 0
@@ -30,6 +32,22 @@
   """Open the connection to the database and return it"""
   return psycopg2.connect("dbname=" + config['general']['dbname'])
 
+__locks = {}
+def lock(config, source):
+  lock_dir = config['general']['lock-dir']
+  lock_path = path.join(lock_dir, source)
+  f = file(lock_path, "w+")
+  __locks[lock_path] = f
+  fcntl.flock(f.fileno(), fcntl.LOCK_EX)
+
+def unlock(config, source):
+  lock_dir = config['general']['lock-dir']
+  lock_path = path.join(lock_dir, source)
+  if lock_path in __locks:
+    f = file(lock_path)
+    fcntl.flock(f.fileno(), fcntl.LOCK_UN)
+    del __locks[lock_path]
+
 def load_config(str):
   """Load and check configuration from the string"""
   config = syck.load(str)
@@ -37,7 +55,7 @@
     raise ConfigException('general section not specified')
   
   general = config['general']
-  for k in ['dbname', 'archs', 'types']:
+  for k in ['dbname', 'archs', 'types', 'lock-dir']:
     if not k in general:
       raise ConfigException(k + ' not specified in node "general"')
   if not 'debug' in general:

Modified: udd/src/udd/gatherer.py
===================================================================
--- udd/src/udd/gatherer.py	2008-08-12 15:13:01 UTC (rev 1086)
+++ udd/src/udd/gatherer.py	2008-08-12 16:38:58 UTC (rev 1087)
@@ -1,6 +1,7 @@
 # This file is part of the Ultimate Debian Database project
 
 import aux
+import sys
 
 class gatherer:
   """

Modified: udd/src/udd/upload_history_gatherer.py
===================================================================
--- udd/src/udd/upload_history_gatherer.py	2008-08-12 15:13:01 UTC (rev 1086)
+++ udd/src/udd/upload_history_gatherer.py	2008-08-12 16:38:58 UTC (rev 1087)
@@ -1,4 +1,4 @@
-# Last-Modified: <Tue Aug 12 15:09:13 2008>
+# Last-Modified: <Tue Aug 12 16:01:29 2008>
 # This file is part of the Ultimate Debian Database Project
 
 from gatherer import gatherer
@@ -21,6 +21,7 @@
     cur = self.cursor()
     cur.execute("DROP TABLE %s" % self.my_config['table'])
     cur.execute("DROP TABLE %s" % self.my_config['table'] + '_architecture')
+    cur.execute("DROP TABLE %s" % self.my_config['table'] + '_closes')
 
 
   def run(self):
@@ -30,11 +31,14 @@
 
     cursor.execute("DELETE FROM " + self.my_config['table'])
     cursor.execute("DELETE FROM " + self.my_config['table'] + '_architecture')
+    cursor.execute("DELETE FROM " + self.my_config['table'] + '_closes')
 
     cursor.execute("PREPARE uh_insert AS INSERT INTO %s VALUES \
 	($1, $2, $3, $4, $5, $6, $7, $8, $9)" % self.my_config['table'])
     cursor.execute("PREPARE uh_arch_insert AS INSERT INTO %s VALUES \
 	($1, $2)" % (self.my_config['table'] + '_architecture'))
+    cursor.execute("PREPARE uh_close_insert AS INSERT INTO %s VALUES \
+	($1, $2)" % (self.my_config['table'] + '_closes'))
 
     id = 0
     for name in glob(path + '/debian-devel-*'):
@@ -53,17 +57,23 @@
 	line = line.strip()
 	# Stupid multi-line maintainer fields *grml*
 	if line == '':
-	  for arch in set(current['Architecture'].split()):
-	    current['arch'] = arch
-	    query = "EXECUTE uh_arch_insert(%(id)s, %(arch)s)"
-	    cursor.execute(query, current)
-	  query = "EXECUTE uh_insert(%(id)s, %(Source)s, %(Version)s, %(Date)s, %(Changed-By)s, \
-	      %(Maintainer)s, %(NMU)s, %(Key)s, %(Signed-By)s)"
 	  try:
+	    for arch in set(current['Architecture'].split()):
+	      current['arch'] = arch
+	      query = "EXECUTE uh_arch_insert(%(id)s, %(arch)s)"
+	      cursor.execute(query, current)
+	    if current['Closes'] != 'N/A':
+	      for closes in set(current['Closes'].split()):
+		current['closes'] = closes
+		query = "EXECUTE uh_close_insert(%(id)s, %(closes)s)"
+		cursor.execute(query, current)
+	    query = "EXECUTE uh_insert(%(id)s, %(Source)s, %(Version)s, %(Date)s, %(Changed-By)s, \
+		%(Maintainer)s, %(NMU)s, %(Key)s, %(Signed-By)s)"
 	    cursor.execute(query, current)
 	  except psycopg2.ProgrammingError, s:
 	    print "Error at line %d of file %s" % (line_count, name)
-	    raise
+	    continue
+	    #raise
 	  id += 1
 	  current = {'id': id}
 	  last_field = None
@@ -78,11 +88,7 @@
 
 	(field, data) = line.split(':', 1)
 	data = data.strip()
-
-	if field != 'NMU':
-	  current[field] = aux.quote(data)
-	else:
-	  current[field] = data
+	current[field] = data
 	
 	last_field = field
     

Modified: udd/src/udd-dispatch.py
===================================================================
--- udd/src/udd-dispatch.py	2008-08-12 15:13:01 UTC (rev 1086)
+++ udd/src/udd-dispatch.py	2008-08-12 16:38:58 UTC (rev 1087)
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Last-Modified: <Mon Aug 11 13:49:35 2008>
+# Last-Modified: <Tue Aug 12 16:20:44 2008>
 
 """Dispatch udd gatherers
 
@@ -46,14 +46,20 @@
 
     (src_command,rest) = types[type].split(None, 1)
     
-    if src_command == "exec":
-      system(rest + " " + sys.argv[1] + " " + sys.argv[2] + " " + src)
-    elif src_command == "module":
-      exec("import " + rest)
-      exec "gatherer = " + rest + ".get_gatherer(connection, config, src)"
-      exec "gatherer.%s()" % command
-    if 'timestamp-folder' in config['general']:
-      f = open(os.path.join(config['general']['timestamp-folder'], src+".dispatch"), "w")
-      f.write(asctime())
-      f.close()
+    udd.aux.lock(config, src)
+    try:
+      if src_command == "exec":
+	system(rest + " " + sys.argv[1] + " " + sys.argv[2] + " " + src)
+      elif src_command == "module":
+	exec("import " + rest)
+	exec "gatherer = " + rest + ".get_gatherer(connection, config, src)"
+	exec "gatherer.%s()" % command
+      if 'timestamp-folder' in config['general']:
+	f = open(os.path.join(config['general']['timestamp-folder'], src+".dispatch"), "w")
+	f.write(asctime())
+	f.close()
+    except:
+      udd.aux.unlock(config, src)
+      raise
   connection.commit()
+

Modified: udd/src/udd-update.py
===================================================================
--- udd/src/udd-update.py	2008-08-12 15:13:01 UTC (rev 1086)
+++ udd/src/udd-update.py	2008-08-12 16:38:58 UTC (rev 1087)
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Last-Modified: <Thu Aug  7 19:07:48 2008>
+# Last-Modified: <Tue Aug 12 16:19:09 2008>
 
 """
 This script executes the update statements for selected sources
@@ -28,11 +28,16 @@
   for src in sys.argv[2:]:
     src_cfg = config[src]
     if "update-command" in src_cfg:
-      result = system(src_cfg['update-command']) 
-      if result != 0:
-	sys.exit(result)
-      if 'timestamp-folder' in config['general']:
-	f = open(os.path.join(config['general']['timestamp-folder'], src+".update"), "w")
-	f.write(asctime())
-	f.close()
+      udd.aux.lock(config, src)
+      try:
+	result = system(src_cfg['update-command']) 
+	if result != 0:
+	  sys.exit(result)
+	if 'timestamp-folder' in config['general']:
+	  f = open(os.path.join(config['general']['timestamp-folder'], src+".update"), "w")
+	  f.write(asctime())
+	  f.close()
+      except:
+	udd.aux.unlock(config, src)
+	raise
 




More information about the Collab-qa-commits mailing list