[Collab-qa-commits] r1489 - in udd: . scripts sql udd
Andreas Tille
tille at alioth.debian.org
Tue Jun 16 18:38:36 UTC 2009
Author: tille
Date: 2009-06-16 18:38:35 +0000 (Tue, 16 Jun 2009)
New Revision: 1489
Added:
udd/scripts/fetch_screenshots.sh
udd/sql/screenshots.sql
udd/udd/screenshot_gatherer.py
Removed:
udd/config_ddtp.yaml
udd/config_ftpnew.yaml
Modified:
udd/config-org.yaml
Log:
Add gatherer for screenshots.debian.net
Modified: udd/config-org.yaml
===================================================================
--- udd/config-org.yaml 2009-06-15 21:05:40 UTC (rev 1488)
+++ udd/config-org.yaml 2009-06-16 18:38:35 UTC (rev 1489)
@@ -18,6 +18,7 @@
ubuntu-bugs: module udd.ubuntu_bugs_gatherer
ddtp: module udd.ddtp_gatherer
ftpnew: module udd.ftpnew_gatherer
+ screenshots: module udd.screenshot_gatherer
debug: 1
timestamp-dir: /org/udd.debian.org/timestamps
lock-dir: /org/udd.debian.org/locks
@@ -406,3 +407,10 @@
table_packages: new_packages
releases_ignore: "'etch'"
+screenshots:
+ type: screenshots
+ update-command: /org/udd.debian.org/udd/scripts/fetch_screenshots.sh
+ path: /org/udd.debian.org/mirrors/screenshots
+ cache: /org/udd.debian.org/mirrors/cache
+ table: screenshots
+ screenshots_json: /org/udd.debian.org/mirrors/screenshots/screenshots.json
Deleted: udd/config_ddtp.yaml
===================================================================
--- udd/config_ddtp.yaml 2009-06-15 21:05:40 UTC (rev 1488)
+++ udd/config_ddtp.yaml 2009-06-16 18:38:35 UTC (rev 1489)
@@ -1,19 +0,0 @@
-general:
- dbname: udd
- types:
- ddtp: module udd.ddtp_gatherer
- debug: 1
- lock-dir: /org/udd.debian.net/locks
-
- archs:
- [i386]
-
-ddtp:
- type: ddtp
- update-command: /org/udd.debian.net/udd/scripts/fetch_ddtp_translations.sh
- path: /org/udd.debian.net/mirrors/ddtp
- mirror: ddtp.debian.net/Translation_udd
- files: Translation-.*\.gz
- releases: sid lenny squeeze
- table: ddtp
- schema: ddtp
Deleted: udd/config_ftpnew.yaml
===================================================================
--- udd/config_ftpnew.yaml 2009-06-15 21:05:40 UTC (rev 1488)
+++ udd/config_ftpnew.yaml 2009-06-16 18:38:35 UTC (rev 1489)
@@ -1,19 +0,0 @@
-general:
- dbname: udd
- types:
- ftpnew: module udd.ftpnew_gatherer
- debug: 1
- lock-dir: /org/udd.debian.org/locks
-
- archs:
- [i386]
-
-ftpnew:
- type: ftpnew
- update-command: /org/udd.debian.org/udd/scripts/fetch_ftpnew.sh
- path: /org/udd.debian.org/mirrors/ftpnew
- cache: /org/udd.debian.org/mirrors/cache
- ftpmasterURL: http://ftp-master.debian.org/new/
- table_sources: new_sources
- table_packages: new_packages
- releases_ignore: "'etch'"
Added: udd/scripts/fetch_screenshots.sh
===================================================================
--- udd/scripts/fetch_screenshots.sh (rev 0)
+++ udd/scripts/fetch_screenshots.sh 2009-06-16 18:38:35 UTC (rev 1489)
@@ -0,0 +1,7 @@
+#!/bin/sh
+TARGETDIR=/org/udd.debian.org/mirrors/screenshots
+mkdir -p $TARGETDIR
+rm -rf $TARGETDIR/*
+wget -q http://screenshots.debian.net/json/screenshots -O ${TARGETDIR}/screenshots.json
+# packages.json just contains less information - but we want it all ...
+# wget -q http://screenshots.debian.net/json/packages -O ${TARGETDIR}/packages.json
Property changes on: udd/scripts/fetch_screenshots.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: udd/sql/screenshots.sql
===================================================================
--- udd/sql/screenshots.sql (rev 0)
+++ udd/sql/screenshots.sql 2009-06-16 18:38:35 UTC (rev 1489)
@@ -0,0 +1,34 @@
+-- http://screenshots.debian.net/json/screenshots
+
+BEGIN;
+
+DROP TABLE IF EXISTS screenshots CASCADE;
+
+CREATE TABLE screenshots (
+ package text NOT NULL,
+ version text,
+ homepage text,
+ maintainer_name text,
+ maintainer_email text,
+ description text,
+ section text,
+ screenshot_url text NOT NULL,
+ large_image_url text NOT NULL,
+ small_image_url text NOT NULL,
+ PRIMARY KEY (small_image_url)
+);
+
+GRANT SELECT ON screenshots TO PUBLIC;
+
+COMMIT;
+
+-- 'name' --> 'package'
+-- 'section'
+-- 'maintainer' --> 'maintainer_name'
+-- 'maintainer_email'
+-- 'version'
+-- 'homepage'
+-- 'description'
+-- 'url' --> 'screenshot_url'
+-- 'large_image_url'
+-- 'small_image_url'
Added: udd/udd/screenshot_gatherer.py
===================================================================
--- udd/udd/screenshot_gatherer.py (rev 0)
+++ udd/udd/screenshot_gatherer.py 2009-06-16 18:38:35 UTC (rev 1489)
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+
+"""
+This script imports screenshot URLs from screenshots.debian.net
+"""
+
+from aux import quote
+from gatherer import gatherer
+import json
+from sys import stderr, exit
+
+from psycopg2 import IntegrityError, InternalError
+
+online=0
+
+def get_gatherer(connection, config, source):
+ return screenshot_gatherer(connection, config, source)
+
+class screenshot_gatherer(gatherer):
+ # Screenshots from screnshots.debian.net
+
+ def __init__(self, connection, config, source):
+ gatherer.__init__(self, connection, config, source)
+ self.assert_my_config('table')
+ my_config = self.my_config
+
+ cur = self.cursor()
+ query = "TRUNCATE %s" % my_config['table']
+ cur.execute(query)
+ query = """PREPARE screenshots_insert (text, text, text, text, text, text, text, text, text, text) AS
+ INSERT INTO %s
+ (package, version, homepage, maintainer_name, maintainer_email, description,
+ section, screenshot_url, large_image_url, small_image_url)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)""" % (my_config['table'])
+ cur.execute(query)
+
+ pkg = None
+
+ def run(self):
+ my_config = self.my_config
+ #start harassing the DB, preparing the final inserts and making place
+ #for the new data:
+ cur = self.cursor()
+
+ screenshot_file = my_config['screenshots_json']
+ fp = open(screenshot_file, 'r')
+ result = json.read(fp.read())
+ fp.close()
+
+ for res in result['screenshots']:
+ # print res
+ try:
+ res['maintainer'] = res['maintainer'].encode('utf-8')
+ except AttributeError, err:
+ print >>stderr, "Missing maintainer for screenshot of package %s" % res['name']
+ query = """EXECUTE screenshots_insert
+ (%(name)s, %(version)s, %(homepage)s, %(maintainer)s, %(maintainer_email)s,
+ %(description)s, %(section)s, %(url)s, %(large_image_url)s, %(small_image_url)s)"""
+ try:
+ cur.execute(query, res)
+ except UnicodeEncodeError, err:
+ print >>stderr, "Unable to inject data for package %s. %s" % (res['name'], err)
+ print >>stderr, "-->", res
+ cur.execute("DEALLOCATE screenshots_insert")
+
+if __name__ == '__main__':
+ main()
+
+# vim:set et tabstop=2:
+
More information about the Collab-qa-commits
mailing list