[Collab-qa-commits] r546 - / upload-history
filippo at alioth.debian.org
filippo at alioth.debian.org
Tue Dec 4 21:22:22 UTC 2007
Author: filippo
Date: 2007-12-04 21:22:22 +0000 (Tue, 04 Dec 2007)
New Revision: 546
Added:
upload-history/
upload-history/Notes
upload-history/changes_to_csv.py
upload-history/changes_to_sql.py
upload-history/csv_to_sql.sh
upload-history/update.sh
upload-history/upload_maintainer.sh
upload-history/upload_package.sh
upload-history/uploads.schema
Log:
first import to collab-qa
Added: upload-history/Notes
===================================================================
--- upload-history/Notes (rev 0)
+++ upload-history/Notes 2007-12-04 21:22:22 UTC (rev 546)
@@ -0,0 +1,12 @@
+upload_history: extract data from changes files
+===============================================
+
+These scripts are meant to be run on merkel and will go through queue/done to
+extract data from processed changes files.
+
+A SQLite script to extract useful data (package/last MU upload/last (MU | NMU)
+upload) for bapase:
+
+.separator " "
+.output file.out
+select MU.package, date(max(MU.date), "unixepoch"), date(max(uploads.date), "unixepoch") from MU left join uploads on mu.package = uploads.package group by MU.package;
Added: upload-history/changes_to_csv.py
===================================================================
--- upload-history/changes_to_csv.py (rev 0)
+++ upload-history/changes_to_csv.py 2007-12-04 21:22:22 UTC (rev 546)
@@ -0,0 +1,91 @@
+#!/usr/bin/python
+import os
+import sys
+import re
+import rfc822
+# if python-debian is not installed
+#sys.path.insert(0, "/path/to/python-debian/")
+from debian_bundle import deb822
+
+SCHEMANAME = "uploads.schema"
+DBNAME = "uploads.db"
+
+nmu_version_RE = re.compile("-\S+\.\d+$")
+nmu_changelog_RE = re.compile("\s+\*\s+.*(NMU|non[- ]maintainer)", re.IGNORECASE + re.MULTILINE)
+
+logfile = open(sys.argv[0] + ".log", "a")
+donefile = open(sys.argv[0] + ".done", "a")
+
+a = open(donefile, "r")
+alreadydone = [ x.strip() for x in a.readlines()]
+a.close()
+
+def changes_to_csv(fname):
+ """input: filename to parse
+ output: parsed data |-separated
+
+ returns True on success, False otherwise"""
+
+ is_nmu = False
+
+ c = deb822.Changes(file(fname))
+
+ try:
+ if not 'source' in c['Architecture']:
+ return False
+ except KeyError:
+ return False
+
+ if os.path.basename(fname) in alreadydone:
+# logfile.write(fname + ": already done, skipping\n")
+ return False
+
+# print "processing " + fname
+
+ try:
+ nmu_version = nmu_version_RE.search(c['Version'])
+ except KeyError:
+ logfile.write(fname + " no version, skipping\n")
+ return False
+
+ try:
+ nmu_changes = nmu_changelog_RE.search(c['Changes'])
+ except KeyError:
+ logfile.write(fname + " no changes NMU not detected\n")
+
+# XXX gives false positives for example with -x+y.z.w
+ if is_nmu is not False:
+ is_nmu = ((nmu_version is not None) and (nmu_changes is not None))
+
+ try:
+ unixtime = int(rfc822.mktime_tz(rfc822.parsedate_tz(c['Date'])))
+ except:
+ logfile.write(fname + " no date, skipping\n")
+ return False
+
+ if not c.has_key('Changed-By'):
+ logfile.write(fname + " no changed-by, setting null\n")
+ c['Changed-By'] = ""
+
+ try:
+ print "%s|%s|%s|%s|%s|%s" % (c['Source'], c['Version'], unixtime, c['Maintainer'], c['Changed-By'], is_nmu)
+ except KeyError:
+ return False
+
+ donefile.write(os.path.basename(fname) + "\n")
+
+ return True
+
+
+if __name__ == '__main__':
+ if len(sys.argv) < 2:
+ filelist = sys.stdin
+ else:
+ filelist = file(sys.argv[1])
+
+ for l in filelist:
+ l = l.strip()
+ if not l.endswith(".changes"):
+ continue
+
+ changes_to_csv(l)
Property changes on: upload-history/changes_to_csv.py
___________________________________________________________________
Name: svn:executable
+ *
Added: upload-history/changes_to_sql.py
===================================================================
--- upload-history/changes_to_sql.py (rev 0)
+++ upload-history/changes_to_sql.py 2007-12-04 21:22:22 UTC (rev 546)
@@ -0,0 +1,77 @@
+#!/usr/bin/python
+import os
+import sys
+import re
+import rfc822
+sys.path.insert(0, "/home/godog/projects/debian/python-debian/")
+from debian_bundle import deb822
+from debian_bundle import pysqlite2.dbapi2 as sqlite
+
+SCHEMANAME = "uploads.schema"
+DBNAME = "uploads.db"
+
+nmu_version_RE = re.compile("-\S+\.\d+$")
+nmu_changelog_RE = re.compile("\s+\*\s+.*(NMU|non[- ]maintainer)", re.IGNORECASE + re.MULTILINE)
+
+def changes_to_sqlite(fname):
+ c = deb822.Changes(file(fname))
+
+ try:
+ if not 'source' in c['Architecture']:
+ return None
+ except KeyError:
+ return None
+
+ print "processing " + fname
+
+ nmu_version = nmu_version_RE.search(c['Version'])
+ nmu_changes = nmu_changelog_RE.search(c['Changes'])
+#print nmu_version
+#print nmu_changes
+
+# XXX gives false positives for example with -x+y.z.w
+ is_nmu = ((nmu_version is not None) and (nmu_changes is not None))
+
+ if not os.path.exists(DBNAME):
+ (f, g) = os.popen2("sqlite3 "+ DBNAME)
+# f.write("CREATE TABLE uploads (id INTEGER PRIMARY KEY AUTOINCREMENT, package TEXT, version TEXT, date TEXT, maintainer TEXT, changedby TEXT, NMU BOOL);")
+ f.writelines(file(SCHEMANAME))
+ f.close()
+ g.close()
+
+ try:
+ unixtime = int(rfc822.mktime_tz(rfc822.parsedate_tz(c['Date'])))
+ except:
+ print "unparsable date for " + fname
+ return None
+
+ # check if it is alread there
+ (f, g) = os.popen2("sqlite3 " + DBNAME)
+ f.write('SELECT COUNT(package) FROM uploads WHERE package = "%s" AND version = "%s"' % (c['Source'], c['Version']))
+ if int(g.read()) == 0:
+ print "skipping " + fname
+ return None
+ f.close()
+ g.close()
+
+ query = 'INSERT INTO uploads (package, version, date, maintainer, changedby, NMU) VALUES ("%s", "%s", "%s", "%s", "%s", "%s");' % (c['Source'], c['Version'], unixtime, c['Maintainer'], c['Changed-By'], is_nmu)
+
+ (f, g) = os.popen2("sqlite3 " + DBNAME)
+ f.write(query)
+ f.close()
+ g.close()
+
+ return query
+
+
+if __name__ == '__main__':
+ if len(sys.argv) < 2:
+ print "directory name required"
+ sys.exit(1)
+
+ for d in sys.argv[1:]:
+ for f in os.listdir(d):
+ if not f.endswith(".changes"):
+ continue
+
+ changes_to_sqlite(os.path.join(d,f))
Property changes on: upload-history/changes_to_sql.py
___________________________________________________________________
Name: svn:executable
+ *
Added: upload-history/csv_to_sql.sh
===================================================================
--- upload-history/csv_to_sql.sh (rev 0)
+++ upload-history/csv_to_sql.sh 2007-12-04 21:22:22 UTC (rev 546)
@@ -0,0 +1,9 @@
+#!/bin/bash
+basedir=/org/scratch/filippo/
+
+sqlite3 uploads_new.db < uploads.schema
+for f in $basedir/*.csv; do
+ echo ".import $f uploads" | sqlite3 uploads_new.db
+done
+
+mv uploads_new.db uploads.db
Property changes on: upload-history/csv_to_sql.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: upload-history/update.sh
===================================================================
--- upload-history/update.sh (rev 0)
+++ upload-history/update.sh 2007-12-04 21:22:22 UTC (rev 546)
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+year=$(date +%Y)
+basedir=/org/ftp.debian.org/queue/done/$year/
+scratchdir=/org/scratch/filippo/
+
+workfile=$scratchdir/changes_working.csv
+yearfile=$scratchdir/changes_$year.csv
+tmpfile=$scratchdir/changes_tmp.csv
+
+months=$( cd $basedir && ls -1 | tail -2 )
+
+for m in $months; do
+ find $basedir/$m -type f | nice -n19 ./changes_to_csv.py >> $workfile
+done
+
+find /org/ftp.debian.org/queue/done/ -maxdepth 1 -type f | nice -n19 ./changes_to_csv.py >> $workfile
+
+cat $yearfile $workfile > $tmpfile
+
+mv $tmpfile $yearfile
+rm $workfile
Property changes on: upload-history/update.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: upload-history/upload_maintainer.sh
===================================================================
--- upload-history/upload_maintainer.sh (rev 0)
+++ upload-history/upload_maintainer.sh 2007-12-04 21:22:22 UTC (rev 546)
@@ -0,0 +1,23 @@
+#!/bin/bash
+# shows upload history by maintainer/uploader or by package
+db=${DBNAME:-uploads.db}
+
+if [ ! -r "$db" ]; then echo "unable to read $db"; exit 1; fi
+if [ -z "$1" ]; then echo "argument required" ; exit 1; fi
+
+me=$(basename "$0")
+
+case "$me" in
+ upload_maintainer.sh)
+ q="SELECT package, version, date(date, \"unixepoch\"), maintainer FROM uploads where maintainer like '%$1%' or changedby like '%$1%' GROUP BY package ORDER BY date ;"
+ ;;
+ upload_package.sh)
+ q="SELECT package, version, date(date, \"unixepoch\") FROM uploads WHERE package LIKE '%$1%' ORDER BY date ;"
+ ;;
+ *)
+ echo "no such action"
+ exit 1
+ ;;
+esac
+
+echo $q | sqlite3 uploads.db
Property changes on: upload-history/upload_maintainer.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: upload-history/upload_package.sh
===================================================================
--- upload-history/upload_package.sh (rev 0)
+++ upload-history/upload_package.sh 2007-12-04 21:22:22 UTC (rev 546)
@@ -0,0 +1 @@
+link upload_maintainer.sh
\ No newline at end of file
Property changes on: upload-history/upload_package.sh
___________________________________________________________________
Name: svn:special
+ *
Added: upload-history/uploads.schema
===================================================================
--- upload-history/uploads.schema (rev 0)
+++ upload-history/uploads.schema 2007-12-04 21:22:22 UTC (rev 546)
@@ -0,0 +1,5 @@
+CREATE TABLE uploads (package TEXT, version TEXT, date TEXT, maintainer TEXT, changedby TEXT, NMU BOOL);
+CREATE VIEW MU as select * from uploads where NMU = "False";
+CREATE VIEW NMU as select * from uploads where NMU = "True";
+CREATE INDEX maint on uploads ( maintainer );
+CREATE INDEX pkg on uploads ( package );
More information about the Collab-qa-commits
mailing list