[Collab-qa-commits] r1314 - in upload-history: . legacy
filippo at alioth.debian.org
filippo at alioth.debian.org
Sat Oct 4 14:50:25 UTC 2008
Author: filippo
Date: 2008-10-04 14:50:25 +0000 (Sat, 04 Oct 2008)
New Revision: 1314
Added:
upload-history/legacy/
upload-history/legacy/changes_to_csv.py
upload-history/legacy/changes_to_sql.py
upload-history/legacy/crontab.sh
upload-history/legacy/mbox_to_csv.py
upload-history/legacy/update.sh
Removed:
upload-history/changes_to_csv.py
upload-history/changes_to_sql.py
upload-history/crontab.sh
upload-history/mbox_to_csv.py
upload-history/update.sh
Log:
move old stuff out of the way
Deleted: upload-history/changes_to_csv.py
===================================================================
--- upload-history/changes_to_csv.py 2008-10-04 14:18:48 UTC (rev 1313)
+++ upload-history/changes_to_csv.py 2008-10-04 14:50:25 UTC (rev 1314)
@@ -1,93 +0,0 @@
-#!/usr/bin/python
-import os
-import sys
-import re
-import rfc822
-# if python-debian is not installed
-#sys.path.insert(0, "/path/to/python-debian/")
-from debian_bundle import deb822
-
-SCHEMANAME = "uploads.schema"
-DBNAME = "uploads.db"
-
-nmu_version_RE = re.compile("-\S+\.\d+$")
-nmu_changelog_RE = re.compile("\s+\*\s+.*(NMU|non[- ]maintainer)", re.IGNORECASE + re.MULTILINE)
-
-donename = sys.argv[0] + ".done"
-
-logfile = open(sys.argv[0] + ".log", "a")
-donefile = open(donename, "a")
-
-a = open(donename, "r")
-alreadydone = [ x.strip() for x in a.readlines()]
-a.close()
-
-def changes_to_csv(fname):
- """input: filename to parse
- output: parsed data |-separated
-
- returns True on success, False otherwise"""
-
- is_nmu = False
-
- c = deb822.Changes(file(fname))
-
- try:
- if not 'source' in c['Architecture']:
- return False
- except KeyError:
- return False
-
- if os.path.basename(fname) in alreadydone:
-# logfile.write(fname + " already done, skipping\n")
- return False
-
-# print "processing " + fname
-
- try:
- nmu_version = nmu_version_RE.search(c['Version'])
- except KeyError:
- logfile.write(fname + " no version, skipping\n")
- return False
-
- try:
- nmu_changes = nmu_changelog_RE.search(c['Changes'])
- except KeyError:
- logfile.write(fname + " no changes NMU not detected\n")
-
-# XXX gives false positives for example with -x+y.z.w
- if is_nmu is not False:
- is_nmu = ((nmu_version is not None) and (nmu_changes is not None))
-
- try:
- unixtime = int(rfc822.mktime_tz(rfc822.parsedate_tz(c['Date'])))
- except:
- logfile.write(fname + " no date, skipping\n")
- return False
-
- if not c.has_key('Changed-By'):
- logfile.write(fname + " no changed-by, setting null\n")
- c['Changed-By'] = ""
-
- try:
- print "%s|%s|%s|%s|%s|%s" % (c['Source'], c['Version'], unixtime, c['Maintainer'], c['Changed-By'], is_nmu)
- except KeyError:
- return False
-
- donefile.write(os.path.basename(fname) + "\n")
-
- return True
-
-
-if __name__ == '__main__':
- if len(sys.argv) < 2:
- filelist = sys.stdin
- else:
- filelist = file(sys.argv[1])
-
- for l in filelist:
- l = l.strip()
- if not l.endswith(".changes"):
- continue
-
- changes_to_csv(l)
Deleted: upload-history/changes_to_sql.py
===================================================================
--- upload-history/changes_to_sql.py 2008-10-04 14:18:48 UTC (rev 1313)
+++ upload-history/changes_to_sql.py 2008-10-04 14:50:25 UTC (rev 1314)
@@ -1,77 +0,0 @@
-#!/usr/bin/python
-import os
-import sys
-import re
-import rfc822
-sys.path.insert(0, "/home/godog/projects/debian/python-debian/")
-from debian_bundle import deb822
-from debian_bundle import pysqlite2.dbapi2 as sqlite
-
-SCHEMANAME = "uploads.schema"
-DBNAME = "uploads.db"
-
-nmu_version_RE = re.compile("-\S+\.\d+$")
-nmu_changelog_RE = re.compile("\s+\*\s+.*(NMU|non[- ]maintainer)", re.IGNORECASE + re.MULTILINE)
-
-def changes_to_sqlite(fname):
- c = deb822.Changes(file(fname))
-
- try:
- if not 'source' in c['Architecture']:
- return None
- except KeyError:
- return None
-
- print "processing " + fname
-
- nmu_version = nmu_version_RE.search(c['Version'])
- nmu_changes = nmu_changelog_RE.search(c['Changes'])
-#print nmu_version
-#print nmu_changes
-
-# XXX gives false positives for example with -x+y.z.w
- is_nmu = ((nmu_version is not None) and (nmu_changes is not None))
-
- if not os.path.exists(DBNAME):
- (f, g) = os.popen2("sqlite3 "+ DBNAME)
-# f.write("CREATE TABLE uploads (id INTEGER PRIMARY KEY AUTOINCREMENT, package TEXT, version TEXT, date TEXT, maintainer TEXT, changedby TEXT, NMU BOOL);")
- f.writelines(file(SCHEMANAME))
- f.close()
- g.close()
-
- try:
- unixtime = int(rfc822.mktime_tz(rfc822.parsedate_tz(c['Date'])))
- except:
- print "unparsable date for " + fname
- return None
-
- # check if it is alread there
- (f, g) = os.popen2("sqlite3 " + DBNAME)
- f.write('SELECT COUNT(package) FROM uploads WHERE package = "%s" AND version = "%s"' % (c['Source'], c['Version']))
- if int(g.read()) == 0:
- print "skipping " + fname
- return None
- f.close()
- g.close()
-
- query = 'INSERT INTO uploads (package, version, date, maintainer, changedby, NMU) VALUES ("%s", "%s", "%s", "%s", "%s", "%s");' % (c['Source'], c['Version'], unixtime, c['Maintainer'], c['Changed-By'], is_nmu)
-
- (f, g) = os.popen2("sqlite3 " + DBNAME)
- f.write(query)
- f.close()
- g.close()
-
- return query
-
-
-if __name__ == '__main__':
- if len(sys.argv) < 2:
- print "directory name required"
- sys.exit(1)
-
- for d in sys.argv[1:]:
- for f in os.listdir(d):
- if not f.endswith(".changes"):
- continue
-
- changes_to_sqlite(os.path.join(d,f))
Deleted: upload-history/crontab.sh
===================================================================
--- upload-history/crontab.sh 2008-10-04 14:18:48 UTC (rev 1313)
+++ upload-history/crontab.sh 2008-10-04 14:50:25 UTC (rev 1314)
@@ -1,8 +0,0 @@
-#!/bin/bash
-./update.sh
-./csv_to_sql.sh
-
-cat << EOF | sqlite3 uploads.db > ~/public_html/upload-history
-.separator " "
-select MU.package, date(max(MU.date), "unixepoch"), date(max(uploads.date), "unixepoch") from MU left join uploads on mu.package = uploads.package group by MU.package;
-EOF
Copied: upload-history/legacy/changes_to_csv.py (from rev 1311, upload-history/changes_to_csv.py)
===================================================================
--- upload-history/legacy/changes_to_csv.py (rev 0)
+++ upload-history/legacy/changes_to_csv.py 2008-10-04 14:50:25 UTC (rev 1314)
@@ -0,0 +1,93 @@
+#!/usr/bin/python
+import os
+import sys
+import re
+import rfc822
+# if python-debian is not installed
+#sys.path.insert(0, "/path/to/python-debian/")
+from debian_bundle import deb822
+
+SCHEMANAME = "uploads.schema"
+DBNAME = "uploads.db"
+
+nmu_version_RE = re.compile("-\S+\.\d+$")
+nmu_changelog_RE = re.compile("\s+\*\s+.*(NMU|non[- ]maintainer)", re.IGNORECASE + re.MULTILINE)
+
+donename = sys.argv[0] + ".done"
+
+logfile = open(sys.argv[0] + ".log", "a")
+donefile = open(donename, "a")
+
+a = open(donename, "r")
+alreadydone = [ x.strip() for x in a.readlines()]
+a.close()
+
+def changes_to_csv(fname):
+ """input: filename to parse
+ output: parsed data |-separated
+
+ returns True on success, False otherwise"""
+
+ is_nmu = False
+
+ c = deb822.Changes(file(fname))
+
+ try:
+ if not 'source' in c['Architecture']:
+ return False
+ except KeyError:
+ return False
+
+ if os.path.basename(fname) in alreadydone:
+# logfile.write(fname + " already done, skipping\n")
+ return False
+
+# print "processing " + fname
+
+ try:
+ nmu_version = nmu_version_RE.search(c['Version'])
+ except KeyError:
+ logfile.write(fname + " no version, skipping\n")
+ return False
+
+ try:
+ nmu_changes = nmu_changelog_RE.search(c['Changes'])
+ except KeyError:
+ logfile.write(fname + " no changes NMU not detected\n")
+
+# XXX gives false positives for example with -x+y.z.w
+ if is_nmu is not False:
+ is_nmu = ((nmu_version is not None) and (nmu_changes is not None))
+
+ try:
+ unixtime = int(rfc822.mktime_tz(rfc822.parsedate_tz(c['Date'])))
+ except:
+ logfile.write(fname + " no date, skipping\n")
+ return False
+
+ if not c.has_key('Changed-By'):
+ logfile.write(fname + " no changed-by, setting null\n")
+ c['Changed-By'] = ""
+
+ try:
+ print "%s|%s|%s|%s|%s|%s" % (c['Source'], c['Version'], unixtime, c['Maintainer'], c['Changed-By'], is_nmu)
+ except KeyError:
+ return False
+
+ donefile.write(os.path.basename(fname) + "\n")
+
+ return True
+
+
+if __name__ == '__main__':
+ if len(sys.argv) < 2:
+ filelist = sys.stdin
+ else:
+ filelist = file(sys.argv[1])
+
+ for l in filelist:
+ l = l.strip()
+ if not l.endswith(".changes"):
+ continue
+
+ changes_to_csv(l)
Property changes on: upload-history/legacy/changes_to_csv.py
___________________________________________________________________
Name: svn:executable
+ *
Name: svn:mergeinfo
+
Copied: upload-history/legacy/changes_to_sql.py (from rev 1311, upload-history/changes_to_sql.py)
===================================================================
--- upload-history/legacy/changes_to_sql.py (rev 0)
+++ upload-history/legacy/changes_to_sql.py 2008-10-04 14:50:25 UTC (rev 1314)
@@ -0,0 +1,77 @@
+#!/usr/bin/python
+import os
+import sys
+import re
+import rfc822
+sys.path.insert(0, "/home/godog/projects/debian/python-debian/")
+from debian_bundle import deb822
+from debian_bundle import pysqlite2.dbapi2 as sqlite
+
+SCHEMANAME = "uploads.schema"
+DBNAME = "uploads.db"
+
+nmu_version_RE = re.compile("-\S+\.\d+$")
+nmu_changelog_RE = re.compile("\s+\*\s+.*(NMU|non[- ]maintainer)", re.IGNORECASE + re.MULTILINE)
+
+def changes_to_sqlite(fname):
+ c = deb822.Changes(file(fname))
+
+ try:
+ if not 'source' in c['Architecture']:
+ return None
+ except KeyError:
+ return None
+
+ print "processing " + fname
+
+ nmu_version = nmu_version_RE.search(c['Version'])
+ nmu_changes = nmu_changelog_RE.search(c['Changes'])
+#print nmu_version
+#print nmu_changes
+
+# XXX gives false positives for example with -x+y.z.w
+ is_nmu = ((nmu_version is not None) and (nmu_changes is not None))
+
+ if not os.path.exists(DBNAME):
+ (f, g) = os.popen2("sqlite3 "+ DBNAME)
+# f.write("CREATE TABLE uploads (id INTEGER PRIMARY KEY AUTOINCREMENT, package TEXT, version TEXT, date TEXT, maintainer TEXT, changedby TEXT, NMU BOOL);")
+ f.writelines(file(SCHEMANAME))
+ f.close()
+ g.close()
+
+ try:
+ unixtime = int(rfc822.mktime_tz(rfc822.parsedate_tz(c['Date'])))
+ except:
+ print "unparsable date for " + fname
+ return None
+
+ # check if it is alread there
+ (f, g) = os.popen2("sqlite3 " + DBNAME)
+ f.write('SELECT COUNT(package) FROM uploads WHERE package = "%s" AND version = "%s"' % (c['Source'], c['Version']))
+ if int(g.read()) == 0:
+ print "skipping " + fname
+ return None
+ f.close()
+ g.close()
+
+ query = 'INSERT INTO uploads (package, version, date, maintainer, changedby, NMU) VALUES ("%s", "%s", "%s", "%s", "%s", "%s");' % (c['Source'], c['Version'], unixtime, c['Maintainer'], c['Changed-By'], is_nmu)
+
+ (f, g) = os.popen2("sqlite3 " + DBNAME)
+ f.write(query)
+ f.close()
+ g.close()
+
+ return query
+
+
+if __name__ == '__main__':
+ if len(sys.argv) < 2:
+ print "directory name required"
+ sys.exit(1)
+
+ for d in sys.argv[1:]:
+ for f in os.listdir(d):
+ if not f.endswith(".changes"):
+ continue
+
+ changes_to_sqlite(os.path.join(d,f))
Property changes on: upload-history/legacy/changes_to_sql.py
___________________________________________________________________
Name: svn:executable
+ *
Name: svn:mergeinfo
+
Copied: upload-history/legacy/crontab.sh (from rev 1311, upload-history/crontab.sh)
===================================================================
--- upload-history/legacy/crontab.sh (rev 0)
+++ upload-history/legacy/crontab.sh 2008-10-04 14:50:25 UTC (rev 1314)
@@ -0,0 +1,8 @@
+#!/bin/bash
+./update.sh
+./csv_to_sql.sh
+
+cat << EOF | sqlite3 uploads.db > ~/public_html/upload-history
+.separator " "
+select MU.package, date(max(MU.date), "unixepoch"), date(max(uploads.date), "unixepoch") from MU left join uploads on mu.package = uploads.package group by MU.package;
+EOF
Property changes on: upload-history/legacy/crontab.sh
___________________________________________________________________
Name: svn:executable
+ *
Name: svn:mergeinfo
+
Copied: upload-history/legacy/mbox_to_csv.py (from rev 1311, upload-history/mbox_to_csv.py)
===================================================================
--- upload-history/legacy/mbox_to_csv.py (rev 0)
+++ upload-history/legacy/mbox_to_csv.py 2008-10-04 14:50:25 UTC (rev 1314)
@@ -0,0 +1,83 @@
+#!/usr/bin/python2.4
+
+import email
+import sys
+import mailbox
+import re
+import rfc822
+
+from debian_bundle import deb822
+
+# XXX gives false positives for example with -x+y.z.w, mitigated by checking Changes also
+nmu_version_RE = re.compile("-\S+\.\d+$")
+nmu_changelog_RE = re.compile("\s+\*\s+.*(NMU|non[- ]maintainer)", re.IGNORECASE + re.MULTILINE)
+
+def mbox_to_csv( mb_file ):
+ outfile = file(mb_file + ".csv", 'w')
+
+ mb = mailbox.PortableUnixMailbox(file(mb_file), factory=email.message_from_file)
+
+ for msg in mb:
+ if msg.is_multipart():
+ continue
+
+ body = msg.get_payload().split('\n')
+
+ is_nmu = False
+ c = deb822.Changes(body)
+
+ try:
+ sys.stdout.write("processing: " + c['Source'] + "\n")
+ except KeyError:
+ print "unable to find source"
+ continue
+
+ pkg = c['Source']
+
+ try:
+ if not 'source' in c['Architecture']:
+ sys.stderr.write("%s: source not in architecture\n" % pkg)
+ continue
+ except KeyError:
+ sys.stderr.write("%s: architecture not found\n" % pkg)
+ continue
+
+ try:
+ nmu_version = nmu_version_RE.search(c['Version'])
+ except KeyError:
+ sys.stderr.write("%s: version not found\n" % pkg)
+ continue
+
+ try:
+ nmu_changes = nmu_changelog_RE.search(c['Changes'])
+ except KeyError:
+ sys.stderr.write("%s: changes not found\n" % pkg)
+ pass
+ # logfile.write(fname + " no changes NMU not detected\n")
+
+ is_nmu = ((nmu_version is not None) and (nmu_changes is not None))
+
+ try:
+ unixtime = int(rfc822.mktime_tz(rfc822.parsedate_tz(c['Date'])))
+ except KeyError:
+ sys.stderr.write("%s: date not found\n" % pkg)
+ continue
+ except TypeError:
+ sys.stderr.write("%s: date not parsable\n" % pkg)
+ continue
+
+ if not c.has_key('Changed-By'):
+ c['Changed-By'] = ""
+
+ try:
+ outfile.write("%s|%s|%s|%s|%s|%s\n" % (c['Source'], c['Version'], unixtime, c['Maintainer'], c['Changed-By'], is_nmu))
+ except KeyError:
+ sys.stderr.write("%s: error printing\n" % pkg)
+ continue
+
+if __name__ == '__main__':
+ if len(sys.argv) < 2:
+ print "Filename argument required"
+ sys.exit(1)
+
+ mbox_to_csv(sys.argv[1])
Property changes on: upload-history/legacy/mbox_to_csv.py
___________________________________________________________________
Name: svn:executable
+ *
Name: svn:mergeinfo
+
Copied: upload-history/legacy/update.sh (from rev 1311, upload-history/update.sh)
===================================================================
--- upload-history/legacy/update.sh (rev 0)
+++ upload-history/legacy/update.sh 2008-10-04 14:50:25 UTC (rev 1314)
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+year=$(date +%Y)
+basedir=/org/ftp.debian.org/queue/done/$year/
+scratchdir=/org/scratch/filippo/
+
+workfile=$scratchdir/last_update.csv
+yearfile=$scratchdir/changes_$year.csv
+tmpfile=$scratchdir/changes_tmp.csv
+
+months=$( cd $basedir && ls -1 | tail -2 )
+
+rm -f $workfile
+
+for m in $months; do
+ find $basedir/$m -type f | nice -n19 ./changes_to_csv.py >> $workfile
+done
+
+find /org/ftp.debian.org/queue/done/ -maxdepth 1 -type f | nice -n19 ./changes_to_csv.py >> $workfile
+
+cat $yearfile $workfile > $tmpfile
+
+mv $tmpfile $yearfile
+#rm $workfile
Property changes on: upload-history/legacy/update.sh
___________________________________________________________________
Name: svn:executable
+ *
Name: svn:mergeinfo
+
Deleted: upload-history/mbox_to_csv.py
===================================================================
--- upload-history/mbox_to_csv.py 2008-10-04 14:18:48 UTC (rev 1313)
+++ upload-history/mbox_to_csv.py 2008-10-04 14:50:25 UTC (rev 1314)
@@ -1,83 +0,0 @@
-#!/usr/bin/python2.4
-
-import email
-import sys
-import mailbox
-import re
-import rfc822
-
-from debian_bundle import deb822
-
-# XXX gives false positives for example with -x+y.z.w, mitigated by checking Changes also
-nmu_version_RE = re.compile("-\S+\.\d+$")
-nmu_changelog_RE = re.compile("\s+\*\s+.*(NMU|non[- ]maintainer)", re.IGNORECASE + re.MULTILINE)
-
-def mbox_to_csv( mb_file ):
- outfile = file(mb_file + ".csv", 'w')
-
- mb = mailbox.PortableUnixMailbox(file(mb_file), factory=email.message_from_file)
-
- for msg in mb:
- if msg.is_multipart():
- continue
-
- body = msg.get_payload().split('\n')
-
- is_nmu = False
- c = deb822.Changes(body)
-
- try:
- sys.stdout.write("processing: " + c['Source'] + "\n")
- except KeyError:
- print "unable to find source"
- continue
-
- pkg = c['Source']
-
- try:
- if not 'source' in c['Architecture']:
- sys.stderr.write("%s: source not in architecture\n" % pkg)
- continue
- except KeyError:
- sys.stderr.write("%s: architecture not found\n" % pkg)
- continue
-
- try:
- nmu_version = nmu_version_RE.search(c['Version'])
- except KeyError:
- sys.stderr.write("%s: version not found\n" % pkg)
- continue
-
- try:
- nmu_changes = nmu_changelog_RE.search(c['Changes'])
- except KeyError:
- sys.stderr.write("%s: changes not found\n" % pkg)
- pass
- # logfile.write(fname + " no changes NMU not detected\n")
-
- is_nmu = ((nmu_version is not None) and (nmu_changes is not None))
-
- try:
- unixtime = int(rfc822.mktime_tz(rfc822.parsedate_tz(c['Date'])))
- except KeyError:
- sys.stderr.write("%s: date not found\n" % pkg)
- continue
- except TypeError:
- sys.stderr.write("%s: date not parsable\n" % pkg)
- continue
-
- if not c.has_key('Changed-By'):
- c['Changed-By'] = ""
-
- try:
- outfile.write("%s|%s|%s|%s|%s|%s\n" % (c['Source'], c['Version'], unixtime, c['Maintainer'], c['Changed-By'], is_nmu))
- except KeyError:
- sys.stderr.write("%s: error printing\n" % pkg)
- continue
-
-if __name__ == '__main__':
- if len(sys.argv) < 2:
- print "Filename argument required"
- sys.exit(1)
-
- mbox_to_csv(sys.argv[1])
Deleted: upload-history/update.sh
===================================================================
--- upload-history/update.sh 2008-10-04 14:18:48 UTC (rev 1313)
+++ upload-history/update.sh 2008-10-04 14:50:25 UTC (rev 1314)
@@ -1,24 +0,0 @@
-#!/bin/bash
-
-year=$(date +%Y)
-basedir=/org/ftp.debian.org/queue/done/$year/
-scratchdir=/org/scratch/filippo/
-
-workfile=$scratchdir/last_update.csv
-yearfile=$scratchdir/changes_$year.csv
-tmpfile=$scratchdir/changes_tmp.csv
-
-months=$( cd $basedir && ls -1 | tail -2 )
-
-rm -f $workfile
-
-for m in $months; do
- find $basedir/$m -type f | nice -n19 ./changes_to_csv.py >> $workfile
-done
-
-find /org/ftp.debian.org/queue/done/ -maxdepth 1 -type f | nice -n19 ./changes_to_csv.py >> $workfile
-
-cat $yearfile $workfile > $tmpfile
-
-mv $tmpfile $yearfile
-#rm $workfile
More information about the Collab-qa-commits
mailing list