[Collab-qa-commits] r1637 - in udd: . scripts sql udd

[Lucas Nussbaum]

Author: lucas
Date: 2009-12-09 20:26:14 +0000 (Wed, 09 Dec 2009)
New Revision: 1637

Added:
   udd/udd/history_daily_gatherer.py
Modified:
   udd/config-org.yaml
   udd/scripts/dump-db.sh
   udd/sql/setup.sql
Log:
store historical data about sources into history.sources_count

Modified: udd/config-org.yaml
===================================================================
--- udd/config-org.yaml	2009-12-09 20:20:05 UTC (rev 1636)
+++ udd/config-org.yaml	2009-12-09 20:26:14 UTC (rev 1637)
@@ -23,6 +23,7 @@
     ldap: module udd.ldap_gatherer
     wannabuild: module udd.wannabuild_gatherer
     removals: module udd.removals_gatherer
+    history-daily: module udd.history_daily_gatherer
   timestamp-dir: /org/udd.debian.org/timestamps
   lock-dir: /org/udd.debian.org/locks
   archs:
@@ -448,7 +449,10 @@
 
 removals:
   type: removals
-  update-command: wget -q http://ftp-master.debian.org/removals-full.txt -O - | scripts/fix-removal-timestamps.py > /org/udd.debian.org/mirrors/removals-full.txt
+  update-command: wget -q http://ftp-master.debian.org/removals-full.txt -O - | /org/udd.debian.org/udd/scripts/fix-removal-timestamps.py > /org/udd.debian.org/mirrors/removals-full.txt
   path: /org/udd.debian.org/mirrors/removals-full.txt
   table: package_removal
   schema: package_removal
+
+history-daily:
+  type: history-daily

Modified: udd/scripts/dump-db.sh
===================================================================
--- udd/scripts/dump-db.sh	2009-12-09 20:20:05 UTC (rev 1636)
+++ udd/scripts/dump-db.sh	2009-12-09 20:26:14 UTC (rev 1637)
@@ -3,5 +3,7 @@
 set -e
 cd /org/udd.debian.org/udd/web/
 umask 022
+pg_dump --no-owner -p 5441 -n history udd | gzip > udd-history.sql.gz.new
+mv udd-history.sql.gz.new udd-history.sql.gz
 pg_dump --no-owner -p 5441 -T ldap -T really_active_dds udd | gzip > udd.sql.gz.new
 mv udd.sql.gz.new udd.sql.gz

Modified: udd/sql/setup.sql
===================================================================
--- udd/sql/setup.sql	2009-12-09 20:20:05 UTC (rev 1636)
+++ udd/sql/setup.sql	2009-12-09 20:26:14 UTC (rev 1637)
@@ -612,4 +612,15 @@
     SELECT DISTINCT carnivore_login.id, carnivore_login.login FROM carnivore_login, carnivore_keys, ldap WHERE ((((carnivore_keys.id = carnivore_login.id) AND (carnivore_keys.key_type = 'keyring'::text)) AND (carnivore_login.login = ldap.login)) AND (ldap.activity_pgp > '2009-01-01 00:00:00+00'::timestamp with time zone));
 GRANT SELECT ON TABLE really_active_dds TO guestdd;
 
+-- HISTORICAL DATA
+CREATE SCHEMA history;
+CREATE TABLE history.sources_count (
+  ts timestamp,
+  total_sid_main int, total_sid_contrib int, total_sid_nonfree int,
+  vcstype_arch int, vcstype_bzr int, vcstype_cvs int, vcstype_darcs int, vcstype_git int, vcstype_hg  int, vcstype_mtn int, vcstype_svn int,
+format_3native int, format_3quilt int,
+  PRIMARY KEY (time)
+);
+GRANT SELECT ON history.sources_count TO public;
 
+

Added: udd/udd/history_daily_gatherer.py
===================================================================
--- udd/udd/history_daily_gatherer.py	                        (rev 0)
+++ udd/udd/history_daily_gatherer.py	2009-12-09 20:26:14 UTC (rev 1637)
@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+
+"""
+This script imports some historical data on a daily basis into UDD
+"""
+
+from aux import quote
+from gatherer import gatherer
+import re
+
+def get_gatherer(connection, config, source):
+  return history_daily_gatherer(connection, config, source)
+
+class history_daily_gatherer(gatherer):
+
+  def __init__(self, connection, config, source):
+    gatherer.__init__(self, connection, config, source)
+    self.assert_my_config()
+
+  def run(self):
+    #start harassing the DB, preparing the final inserts and making place
+    #for the new data:
+    cur = self.cursor()
+    cur2 = self.cursor()
+
+    # source format
+    data = {}
+    cur.execute("""SELECT format, COUNT(DISTINCT source) AS cnt FROM sources
+    WHERE distribution='debian' and release='sid' GROUP BY format""")
+    rows = {}
+    for r in cur.fetchall():
+        rows[r[0]] = r[1]
+    data['format_3native'] = rows['3.0 (native)']
+    data['format_3quilt'] = rows['3.0 (quilt)']
+    cur.execute("""SELECT vcs_type, COUNT(DISTINCT source) AS cnt FROM sources
+    WHERE distribution='debian' and release='sid' GROUP BY vcs_type""")
+    rows = {}
+    for r in cur.fetchall():
+        rows[r[0]] = r[1]
+    data['vcstype_arch'] = rows.get('Arch',0)
+    data['vcstype_bzr'] = rows.get('Bzr',0)
+    data['vcstype_cvs'] = rows.get('Cvs',0)
+    data['vcstype_darcs'] = rows.get('Darcs',0)
+    data['vcstype_git'] = rows.get('Git',0)
+    data['vcstype_hg'] = rows.get('Hg',0)
+    data['vcstype_mtn'] = rows.get('Mtn',0)
+    data['vcstype_svn'] = rows.get('Svn',0)
+
+    cur2.execute("""INSERT INTO history.sources_count (ts, 
+        format_3native, format_3quilt,
+        vcstype_arch, vcstype_bzr, vcstype_cvs, vcstype_darcs, vcstype_git, vcstype_hg, vcstype_mtn, vcstype_svn
+        ) VALUES (NOW(),
+        %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""",
+            (data['format_3native'], data['format_3quilt'],
+             data['vcstype_arch'], data['vcstype_bzr'], data['vcstype_cvs'], data['vcstype_darcs'], data['vcstype_git'], data['vcstype_hg'], data['vcstype_mtn'], data['vcstype_svn']))
+
+if __name__ == '__main__':
+  main()
+
+# vim:set et tabstop=2: