[Collab-qa-commits] r895 - udd/src
neronus-guest at alioth.debian.org
neronus-guest at alioth.debian.org
Sun Jun 22 21:03:39 UTC 2008
Author: neronus-guest
Date: 2008-06-22 21:03:38 +0000 (Sun, 22 Jun 2008)
New Revision: 895
Added:
udd/src/popcon_gatherer.py
Modified:
udd/src/db_manager.py
udd/src/packages_gatherer.py
udd/src/setup-db.sql
udd/src/sources_gatherer.py
udd/src/test.yaml
Log:
Added Code to gather Popcon sources
Modified: udd/src/db_manager.py
===================================================================
--- udd/src/db_manager.py 2008-06-20 08:21:36 UTC (rev 894)
+++ udd/src/db_manager.py 2008-06-22 21:03:38 UTC (rev 895)
@@ -6,7 +6,7 @@
"""This scripts sets up and deletes the tables of the database"""
-TABLES = ('sources', 'packages')
+TABLES = ('sources', 'packages', 'popcon')
def print_help():
print "Usage: %s <config> <delete|setup>" % sys.argv[0]
Modified: udd/src/packages_gatherer.py
===================================================================
--- udd/src/packages_gatherer.py 2008-06-20 08:21:36 UTC (rev 894)
+++ udd/src/packages_gatherer.py 2008-06-22 21:03:38 UTC (rev 895)
@@ -1,5 +1,5 @@
#/usr/bin/env python
-# Last-Modified: <Tue Jun 17 10:58:01 2008>
+# Last-Modified: <Sun Jun 22 17:16:49 2008>
import debian_bundle.deb822
import gzip
@@ -20,10 +20,12 @@
# The ID for the distribution we want to include
distr = None
-mandatory = ('Package', 'Version', 'Architecture', 'Maintainer',
- 'Description')
-non_mandatory = ('Source', 'Essential', 'Depends', 'Recommends', 'Suggests',
- 'Enhances', 'Pre-Depends', 'Installed-Size', 'Homepage', 'Size', 'MD5Sum')
+# For efficiency, these are dictionaries
+mandatory = {'Package': 0, 'Version': 0, 'Architecture': 0, 'Maintainer': 0,
+ 'Description': 0}
+non_mandatory = {'Source': 0, 'Essential': 0, 'Depends': 0, 'Recommends': 0,
+ 'Suggests': 0, 'Enhances': 0, 'Pre-Depends': 0, 'Installed-Size': 0,
+ 'Homepage': 0, 'Size': 0, 'MD5Sum': 0}
ignorable = ()
def quote(s):
@@ -35,7 +37,7 @@
else:
return 'NULL'
-warned_about = []
+warned_about = {}
def build_dict(control):
"""Build a dictionary from the control dictionary.
@@ -52,8 +54,9 @@
for k in control.keys():
if k not in mandatory and k not in non_mandatory and k not in ignorable:
if k not in warned_about:
- print("Unknown key: " + k)
- warned_about.append(k)
+ warned_about[k] = 1
+ else:
+ warned_about[k] += 1
return d
def import_packages(conn, sequence):
@@ -84,12 +87,13 @@
# We just use the first line of the description
if d['Description'] != "NULL":
- d['Description'] = d['Description'].split("\n")[0]
- # This problem appears, if the description was a one-liner
+ d['Description'] = d['Description'].split("\n",1)[0]
+ # If the description was a one-liner only, we don't need to add
+ # a quote
if d['Description'][-1] != "'" or d['Description'][-2] == '\\':
d['Description'] += "'"
- # Source is non-mandatory
+ # Source is non-mandatory, but we don't want it to be zero
if d['Source'] == "NULL":
d['Source'] = d['Package']
d['Source_Version'] = d['Version']
@@ -196,5 +200,8 @@
conn.commit()
+ for key in warned_about:
+ print("Unknown key: %s appeared %d times" % (key, warned_about[key]))
+
if __name__ == '__main__':
main()
Added: udd/src/popcon_gatherer.py
===================================================================
--- udd/src/popcon_gatherer.py (rev 0)
+++ udd/src/popcon_gatherer.py 2008-06-22 21:03:38 UTC (rev 895)
@@ -0,0 +1,44 @@
+import aux
+import sys
+import gzip
+
+def main():
+ if len(sys.argv) != 3:
+ print 'Usage: %s <config-file> <source>' % sys.argv[0]
+ sys.exit(1)
+
+ config = aux.load_config(open(sys.argv[1]).read())
+ source = sys.argv[2]
+
+ try:
+ my_config = config[source]
+ except:
+ raise
+
+ if not 'path' in my_config:
+ raise aux.ConfigException, "path not configured for source " % source
+
+ conn = aux.open_connection(config)
+
+ cur = conn.cursor()
+
+ cur.execute("PREPARE pop_insert AS INSERT INTO popcon (name, vote, olde, recent, nofiles) VALUES ($1, $2, $3, $4, $5)")
+
+ popcon = gzip.open(my_config['path'])
+
+ for line in popcon.readlines():
+ name, data = line.split(None, 1)
+ if name == "Submissions:":
+ cur.execute("INSERT INTO popcon (name, vote) VALUES ('_submissions', %s)" % (data))
+ try:
+ (name, vote, old, recent, nofiles) = data.split()
+ cur.execute("EXECUTE pop_insert('%s', %s, %s, %s, %s)" %\
+ (name, vote, old, recent, nofiles))
+ except ValueError:
+ continue
+
+ cur.execute("DEALLOCATE pop_insert")
+ conn.commit()
+
+if __name__ == '__main__':
+ main()
Modified: udd/src/setup-db.sql
===================================================================
--- udd/src/setup-db.sql 2008-06-20 08:21:36 UTC (rev 894)
+++ udd/src/setup-db.sql 2008-06-22 21:03:38 UTC (rev 895)
@@ -17,9 +17,13 @@
text, X_Vcs_Svn text,
UNIQUE (package, version, distribution, release, component));
+CREATE TABLE popcon
+ (Name text, vote int, olde int, recent int, nofiles int, UNIQUE (Name));
+
CREATE INDEX pkgs_name_idx ON Packages (Package);
CREATE INDEX sources_id_idx ON sources (Package);
CREATE INDEX pkgs_src_id_idx ON Packages USING btree (Source);
GRANT SELECT ON Packages TO PUBLIC;
GRANT SELECT ON sources TO PUBLIC;
+GRANT SELECT ON popcon TO PUBLIC;
Modified: udd/src/sources_gatherer.py
===================================================================
--- udd/src/sources_gatherer.py 2008-06-20 08:21:36 UTC (rev 894)
+++ udd/src/sources_gatherer.py 2008-06-22 21:03:38 UTC (rev 895)
@@ -1,5 +1,5 @@
#/usr/bin/env python
-# Last-Modified: <Sun Jun 15 13:15:10 2008>
+# Last-Modified: <Sun Jun 22 20:43:14 2008>
import debian_bundle.deb822
import gzip
@@ -11,15 +11,15 @@
distr = None
-mandatory = ('Format', 'Maintainer', 'Package', 'Version', 'Files')
-non_mandatory = ('Uploaders', 'Binary', 'Architecture', 'Standards-Version',
- 'Homepage', 'Build-Depends', 'Build-Depends-Indep',
- 'Build-Conflicts', 'Build-Conflicts-Indep', 'Priority',
- 'Section', 'Vcs-Arch', 'Vcs-Browser', 'Vcs-Bzr', 'Vcs-Cvs',
- 'Vcs-Darcs', 'Vcs-Git', 'Vcs-Hg', 'Vcs-Svn', 'X-Vcs-Browser',
- 'X-Vcs-Bzr', 'X-Vcs-Darcs', 'X-Vcs-Svn')
+mandatory = {'Format': 0, 'Maintainer': 0, 'Package': 0, 'Version': 0, 'Files': 0}
+non_mandatory = {'Uploaders': 0, 'Binary': 0, 'Architecture': 0,
+ 'Standards-Version': 0, 'Homepage': 0, 'Build-Depends': 0,
+ 'Build-Depends-Indep': 0, 'Build-Conflicts': 0, 'Build-Conflicts-Indep': 0,
+ 'Priority': 0, 'Section': 0, 'Vcs-Arch': 0, 'Vcs-Browser': 0, 'Vcs-Bzr': 0,
+ 'Vcs-Cvs': 0, 'Vcs-Darcs': 0, 'Vcs-Git': 0, 'Vcs-Hg': 0, 'Vcs-Svn': 0,
+ 'X-Vcs-Browser': 0, 'X-Vcs-Bzr': 0, 'X-Vcs-Darcs': 0, 'X-Vcs-Svn': 0}
-ignorable = ()
+ignorable = {}
def null_or_quote(dict, key):
if key in dict:
@@ -27,7 +27,7 @@
else:
return 'NULL'
-warned_about = []
+warned_about = {}
def build_dict(control):
"""Build a dictionary from the control dictionary.
@@ -43,8 +43,9 @@
for k in control.keys():
if k not in mandatory and k not in non_mandatory and k not in ignorable:
if k not in warned_about:
- print("Unknown key: " + k)
- warned_about.append(k)
+ warned_about[k] = 1
+ else:
+ warned_about[k] += 1
return d
def import_sources(conn, file):
@@ -141,5 +142,8 @@
conn.commit()
+ for key in warned_about:
+ print "Unknowen key %s appeared %d times" % (key, warned_about[key])
+
if __name__ == '__main__':
main()
Modified: udd/src/test.yaml
===================================================================
--- udd/src/test.yaml 2008-06-20 08:21:36 UTC (rev 894)
+++ udd/src/test.yaml 2008-06-22 21:03:38 UTC (rev 895)
@@ -6,6 +6,7 @@
setup: python db_manager.py
delete: python db_manager.py
src-pkg: python srcs_and_pkgs.py
+ popcon: python popcon_gatherer.py
# src-pkg: python sources_gatherer.py
debug: 1
@@ -91,3 +92,7 @@
[alpha, amd64, arm, armeb, armel, hppa, hurd-i386,
i386, i486, ia64, kfreebsd-amd64, kfreebsd-i386, m68k, mips,
mipsel, powerpc, ppc64, s390, sparc, all, any]
+
+popcon:
+ type: popcon
+ path: /tmp/all-popcon-results.txt.gz
More information about the Collab-qa-commits
mailing list