[Collab-qa-commits] r1087 - in udd/src: . schema udd
neronus-guest at alioth.debian.org
neronus-guest at alioth.debian.org
Tue Aug 12 16:38:59 UTC 2008
Author: neronus-guest
Date: 2008-08-12 16:38:58 +0000 (Tue, 12 Aug 2008)
New Revision: 1087
Modified:
udd/src/schema/upload_history
udd/src/test.yaml
udd/src/udd-dispatch.py
udd/src/udd-update.py
udd/src/udd/aux.py
udd/src/udd/gatherer.py
udd/src/udd/upload_history_gatherer.py
Log:
Locking is implemented
upload_histories has now an extra cloeses table
Modified: udd/src/schema/upload_history
===================================================================
--- udd/src/schema/upload_history 2008-08-12 15:13:01 UTC (rev 1086)
+++ udd/src/schema/upload_history 2008-08-12 16:38:58 UTC (rev 1087)
@@ -1,12 +1,15 @@
CREATE TABLE %(table)s
(id serial, package text, version text, date timestamp with time zone,
- changed_by text, maintainer text, nmu boolean, signed_by text, key_id text,
- closes int);
+ changed_by text, maintainer text, nmu boolean, signed_by text, key_id text);
CREATE TABLE %(table)s_architecture
(id int, architecture text,
PRIMARY KEY (id, architecture));
+CREATE TABLE %(table)s_closes
+ (id int, bug int,
+ PRIMARY KEY (id, bug));
GRANT SELECT ON %(table)s TO PUBLIC;
GRANT SELECT ON %(table)s_architecture TO PUBLIC;
+GRANT SELECT ON %(table)s_closes TO PUBLIC;
Modified: udd/src/test.yaml
===================================================================
--- udd/src/test.yaml 2008-08-12 15:13:01 UTC (rev 1086)
+++ udd/src/test.yaml 2008-08-12 16:38:58 UTC (rev 1087)
@@ -17,6 +17,7 @@
debug: 1
update-timestamp-folder: ./timestamps/
schema-dir: ./schema/
+ lock-dir: ./locks/
archs:
[alpha, amd64, arm, armeb, armel, hppa, hurd-i386,
Modified: udd/src/udd/aux.py
===================================================================
--- udd/src/udd/aux.py 2008-08-12 15:13:01 UTC (rev 1086)
+++ udd/src/udd/aux.py 2008-08-12 16:38:58 UTC (rev 1087)
@@ -3,6 +3,8 @@
import syck
import sys
import psycopg2
+from os import path
+import fcntl
# If debug is something that evaluates to True, then print_debug actually prints something
debug = 0
@@ -30,6 +32,22 @@
"""Open the connection to the database and return it"""
return psycopg2.connect("dbname=" + config['general']['dbname'])
+__locks = {}
+def lock(config, source):
+ lock_dir = config['general']['lock-dir']
+ lock_path = path.join(lock_dir, source)
+ f = file(lock_path, "w+")
+ __locks[lock_path] = f
+ fcntl.flock(f.fileno(), fcntl.LOCK_EX)
+
+def unlock(config, source):
+ lock_dir = config['general']['lock-dir']
+ lock_path = path.join(lock_dir, source)
+ if lock_path in __locks:
+ f = file(lock_path)
+ fcntl.flock(f.fileno(), fcntl.LOCK_UN)
+ del __locks[lock_path]
+
def load_config(str):
"""Load and check configuration from the string"""
config = syck.load(str)
@@ -37,7 +55,7 @@
raise ConfigException('general section not specified')
general = config['general']
- for k in ['dbname', 'archs', 'types']:
+ for k in ['dbname', 'archs', 'types', 'lock-dir']:
if not k in general:
raise ConfigException(k + ' not specified in node "general"')
if not 'debug' in general:
Modified: udd/src/udd/gatherer.py
===================================================================
--- udd/src/udd/gatherer.py 2008-08-12 15:13:01 UTC (rev 1086)
+++ udd/src/udd/gatherer.py 2008-08-12 16:38:58 UTC (rev 1087)
@@ -1,6 +1,7 @@
# This file is part of the Ultimate Debian Database project
import aux
+import sys
class gatherer:
"""
Modified: udd/src/udd/upload_history_gatherer.py
===================================================================
--- udd/src/udd/upload_history_gatherer.py 2008-08-12 15:13:01 UTC (rev 1086)
+++ udd/src/udd/upload_history_gatherer.py 2008-08-12 16:38:58 UTC (rev 1087)
@@ -1,4 +1,4 @@
-# Last-Modified: <Tue Aug 12 15:09:13 2008>
+# Last-Modified: <Tue Aug 12 16:01:29 2008>
# This file is part of the Ultimate Debian Database Project
from gatherer import gatherer
@@ -21,6 +21,7 @@
cur = self.cursor()
cur.execute("DROP TABLE %s" % self.my_config['table'])
cur.execute("DROP TABLE %s" % self.my_config['table'] + '_architecture')
+ cur.execute("DROP TABLE %s" % self.my_config['table'] + '_closes')
def run(self):
@@ -30,11 +31,14 @@
cursor.execute("DELETE FROM " + self.my_config['table'])
cursor.execute("DELETE FROM " + self.my_config['table'] + '_architecture')
+ cursor.execute("DELETE FROM " + self.my_config['table'] + '_closes')
cursor.execute("PREPARE uh_insert AS INSERT INTO %s VALUES \
($1, $2, $3, $4, $5, $6, $7, $8, $9)" % self.my_config['table'])
cursor.execute("PREPARE uh_arch_insert AS INSERT INTO %s VALUES \
($1, $2)" % (self.my_config['table'] + '_architecture'))
+ cursor.execute("PREPARE uh_close_insert AS INSERT INTO %s VALUES \
+ ($1, $2)" % (self.my_config['table'] + '_closes'))
id = 0
for name in glob(path + '/debian-devel-*'):
@@ -53,17 +57,23 @@
line = line.strip()
# Stupid multi-line maintainer fields *grml*
if line == '':
- for arch in set(current['Architecture'].split()):
- current['arch'] = arch
- query = "EXECUTE uh_arch_insert(%(id)s, %(arch)s)"
- cursor.execute(query, current)
- query = "EXECUTE uh_insert(%(id)s, %(Source)s, %(Version)s, %(Date)s, %(Changed-By)s, \
- %(Maintainer)s, %(NMU)s, %(Key)s, %(Signed-By)s)"
try:
+ for arch in set(current['Architecture'].split()):
+ current['arch'] = arch
+ query = "EXECUTE uh_arch_insert(%(id)s, %(arch)s)"
+ cursor.execute(query, current)
+ if current['Closes'] != 'N/A':
+ for closes in set(current['Closes'].split()):
+ current['closes'] = closes
+ query = "EXECUTE uh_close_insert(%(id)s, %(closes)s)"
+ cursor.execute(query, current)
+ query = "EXECUTE uh_insert(%(id)s, %(Source)s, %(Version)s, %(Date)s, %(Changed-By)s, \
+ %(Maintainer)s, %(NMU)s, %(Key)s, %(Signed-By)s)"
cursor.execute(query, current)
except psycopg2.ProgrammingError, s:
print "Error at line %d of file %s" % (line_count, name)
- raise
+ continue
+ #raise
id += 1
current = {'id': id}
last_field = None
@@ -78,11 +88,7 @@
(field, data) = line.split(':', 1)
data = data.strip()
-
- if field != 'NMU':
- current[field] = aux.quote(data)
- else:
- current[field] = data
+ current[field] = data
last_field = field
Modified: udd/src/udd-dispatch.py
===================================================================
--- udd/src/udd-dispatch.py 2008-08-12 15:13:01 UTC (rev 1086)
+++ udd/src/udd-dispatch.py 2008-08-12 16:38:58 UTC (rev 1087)
@@ -1,5 +1,5 @@
#!/usr/bin/env python
-# Last-Modified: <Mon Aug 11 13:49:35 2008>
+# Last-Modified: <Tue Aug 12 16:20:44 2008>
"""Dispatch udd gatherers
@@ -46,14 +46,20 @@
(src_command,rest) = types[type].split(None, 1)
- if src_command == "exec":
- system(rest + " " + sys.argv[1] + " " + sys.argv[2] + " " + src)
- elif src_command == "module":
- exec("import " + rest)
- exec "gatherer = " + rest + ".get_gatherer(connection, config, src)"
- exec "gatherer.%s()" % command
- if 'timestamp-folder' in config['general']:
- f = open(os.path.join(config['general']['timestamp-folder'], src+".dispatch"), "w")
- f.write(asctime())
- f.close()
+ udd.aux.lock(config, src)
+ try:
+ if src_command == "exec":
+ system(rest + " " + sys.argv[1] + " " + sys.argv[2] + " " + src)
+ elif src_command == "module":
+ exec("import " + rest)
+ exec "gatherer = " + rest + ".get_gatherer(connection, config, src)"
+ exec "gatherer.%s()" % command
+ if 'timestamp-folder' in config['general']:
+ f = open(os.path.join(config['general']['timestamp-folder'], src+".dispatch"), "w")
+ f.write(asctime())
+ f.close()
+ except:
+ udd.aux.unlock(config, src)
+ raise
connection.commit()
+
Modified: udd/src/udd-update.py
===================================================================
--- udd/src/udd-update.py 2008-08-12 15:13:01 UTC (rev 1086)
+++ udd/src/udd-update.py 2008-08-12 16:38:58 UTC (rev 1087)
@@ -1,5 +1,5 @@
#!/usr/bin/env python
-# Last-Modified: <Thu Aug 7 19:07:48 2008>
+# Last-Modified: <Tue Aug 12 16:19:09 2008>
"""
This script executes the update statements for selected sources
@@ -28,11 +28,16 @@
for src in sys.argv[2:]:
src_cfg = config[src]
if "update-command" in src_cfg:
- result = system(src_cfg['update-command'])
- if result != 0:
- sys.exit(result)
- if 'timestamp-folder' in config['general']:
- f = open(os.path.join(config['general']['timestamp-folder'], src+".update"), "w")
- f.write(asctime())
- f.close()
+ udd.aux.lock(config, src)
+ try:
+ result = system(src_cfg['update-command'])
+ if result != 0:
+ sys.exit(result)
+ if 'timestamp-folder' in config['general']:
+ f = open(os.path.join(config['general']['timestamp-folder'], src+".update"), "w")
+ f.write(asctime())
+ f.close()
+ except:
+ udd.aux.unlock(config, src)
+ raise
More information about the Collab-qa-commits
mailing list