[Collab-qa-commits] r1683 - ddpo-by-mail
Lucas Nussbaum
lucas at alioth.debian.org
Sat Feb 13 17:45:51 UTC 2010
Author: lucas
Date: 2010-02-13 17:45:50 +0000 (Sat, 13 Feb 2010)
New Revision: 1683
Modified:
ddpo-by-mail/Makefile
ddpo-by-mail/bugs.rb
ddpo-by-mail/ddpo-config.rb
ddpo-by-mail/reminder-mail.rb
ddpo-by-mail/testingissues.rb
Log:
massive changes
- use UDD instead of ddpo_packages/ddpo_maintainers
- use UDD for buildd status
- offload most of the testing migrations work to UDD
- increase thresholds for testing migration, we had a problem on UDD causing missing data
- disable stable RC bugs
- disable security issues reporting
- disable gcc 4.5 RG
Modified: ddpo-by-mail/Makefile
===================================================================
--- ddpo-by-mail/Makefile 2010-02-11 21:15:28 UTC (rev 1682)
+++ ddpo-by-mail/Makefile 2010-02-13 17:45:50 UTC (rev 1683)
@@ -1,11 +1,10 @@
#!/usr/bin/make -f
-UPDATE_TARGETS:=ddpo_packages ddpo_maintainers testing-status.raw dehs.txt ood_excuses.txt pts_subscriptions.txt wnpp_rm lintian-list.txt debian-installer/all bugs_stable.txt bugs_unstable.txt security-packages.txt
+UPDATE_TARGETS:=testing-status.txt dehs.txt pts_subscriptions.txt wnpp_rm lintian-list.txt di-sources.txt bugs_stable.txt bugs_unstable.txt security-packages.txt maintainers.txt uploaders.txt wannabuild.txt
ARCHITECTURES:=alpha,amd64,armel,hppa,hurd-i386,i386,ia64,kfreebsd-amd64,kfreebsd-i386,mips,mipsel,powerpc,s390,sparc
#ARCHITECTURES:=$(shell dpkg-architecture -L | tr "\n" ",")
LOCAL_MIRROR:=/org/ftp.debian.org/ftp
MIRROR:=http://ftp.de.debian.org/debian
-DI_PATH:=dists/sid/main/debian-installer
WGET_OPTIONS:=-nv -N
WGET:=wget $(WGET_OPTIONS)
SSH_UDD_ACCESS_HOST := alioth.debian.org
@@ -16,18 +15,25 @@
all:
-$(DATA_DIR) debian-installer ood:
+$(DATA_DIR) ood:
mkdir $@
update: $(DATA_DIR)
@cd $(DATA_DIR) && $(MAKE) -f ../Makefile FORCE=phony $(UPDATE_TARGETS)
-ddpo_packages ddpo_maintainers: $(FORCE)
- $(WGET) http://qa.debian.org/data/ddpo/results/$@
+maintainers.txt:
+ ssh -oBatchMode=yes $(SSH_UDD_ACCESS_HOST) "psql -At -c \"select source, maintainer_email, maintainer_name from sources_uniq where distribution = 'debian' and release = 'sid';\" 'service=udd'" > $@
-testing-status.raw: $(FORCE)
- ssh -oBatchMode=yes $(SSH_UDD_ACCESS_HOST) "psql -At -c \"SELECT * FROM migrations;\" 'service=udd'" > $@
+uploaders.txt:
+ ssh -oBatchMode=yes $(SSH_UDD_ACCESS_HOST) "psql -At -c \"select source, email, name from uploaders where distribution = 'debian' and release = 'sid';\" 'service=udd'" > $@
+testing-status.txt: $(FORCE)
+ ssh -oBatchMode=yes $(SSH_UDD_ACCESS_HOST) "psql -At -c \"select source, in_testing, current_date - in_testing as in_testing_age, sync, current_date - sync as sync_age, current_date - first_seen as debian_age from migrations where current_date - in_unstable < 2 and (sync is null or current_date - sync > 1);\" 'service=udd'" > $@
+
+# FIXME temporarily disabled mips and mipsel
+wannabuild.txt:
+ ssh -oBatchMode=yes $(SSH_UDD_ACCESS_HOST) "psql -At -c \"select source, architecture from wannabuild where distribution='unstable' and state not in ('Installed', 'Needs-Build', 'Dep-Wait', 'Not-For-Us') and (state not in ('Built', 'Uploaded') or now() - state_change > interval '3 days') and architecture not in ('hurd-i386', 'mips', 'mipsel') and notes <> 'uncompiled';;\" 'service=udd'" > $@
+
ood/update_output.txt ood/update_excuses.html: ood $(FORCE)
cd ood/ && $(WGET) http://release.debian.org/britney/$(subst ood/,,$@)
@@ -53,20 +59,9 @@
security-packages.txt: $(FORCE)
$(WGET) -O$@ http://security-tracker.debian.org/tracker/data/pts/1
-debian-installer/binary-%: $(FORCE)
- mkdir -p `dirname "$@"`
- ( cd `dirname "$@"` && $(WGET) $(MIRROR)/$(DI_PATH)/binary-$* ; ) || touch $@
+di-sources.txt:
+ ssh -oBatchMode=yes $(SSH_UDD_ACCESS_HOST) "psql -At -c \"select source from sources_uniq where distribution = 'debian' and release = 'sid' and source not in (select distinct source from packages where distribution = 'debian' and release = 'sid' and component not like '%/debian-installer');\" 'service=udd'" > $@
-debian-installer/all: debian-installer
- if [ -d "$(LOCAL_MIRROR)/$(DI_PATH)" ]; then \
- [ -L debian-installer ] || ln -s debian-installer '$(LOCAL_MIRROR)/$(DI_PATH)';\
- else \
- $(MAKE) -f ../Makefile `echo "$(ARCHITECTURES)" | tr ',' '\n' | sed 's,^,debian-installer/binary-,g;s,$$,/Packages.gz,g'`; \
- fi
-
-clean-update::
- $(RM) -f $(DATA_DIR)/debian-installer/binary-*/Packages.gz
-
bugs_unstable.txt: $(FORCE)
ssh -oBatchMode=yes $(SSH_UDD_ACCESS_HOST) "psql -Atx -c \"SELECT b.id, b.source, b.title, b.arrival, b.severity, b.affects_unstable, b.affects_stable, bmw.merged_with, bt.tag, but.tag AS usertag, but.email FROM bugs_rt_affects_unstable, bugs AS b FULL JOIN bugs_merged_with AS bmw ON b.id = bmw.id FULL JOIN bugs_tags AS bt ON b.id = bt.id FULL JOIN bugs_usertags AS but ON b.id = but.id WHERE (bugs_rt_affects_unstable.id = b.id) AND b.status <> 'done' AND (b.severity > 'important' OR (bt.tag = '$(subst ;,' OR bt.tag = ',$(TAGS))') OR (but.email = '$(subst ;,' OR but.email = ',$(USERTAGS_EMAILS))'));\" 'service=udd'" > $@
Modified: ddpo-by-mail/bugs.rb
===================================================================
--- ddpo-by-mail/bugs.rb 2010-02-11 21:15:28 UTC (rev 1682)
+++ ddpo-by-mail/bugs.rb 2010-02-13 17:45:50 UTC (rev 1683)
@@ -20,7 +20,8 @@
bugutag = nil
buguemail = nil
- ['stable', 'unstable'].each do |release|
+ # Only report about unstable RC bugs.
+ ['unstable'].each do |release|
IO::read("data/bugs_#{release}.txt").each_line do |l|
l.chomp!
key, val = l.split('|', 2)
Modified: ddpo-by-mail/ddpo-config.rb
===================================================================
--- ddpo-by-mail/ddpo-config.rb 2010-02-11 21:15:28 UTC (rev 1682)
+++ ddpo-by-mail/ddpo-config.rb 2010-02-13 17:45:50 UTC (rev 1683)
@@ -82,10 +82,10 @@
EXC_EMAILS_PKGS['marcot at riseup.net'] = ['gtk2hs']
EXC_EMAILS_PKGS['pkern at debian.org'] = ['ekiga']
-BUG_AGE = 4
+BUG_AGE = 2
AGE_IN_DEBIAN = 30
-LAST_IN_SYNC = 20
-LAST_IN_SYNC_SERIOUS = 30
+LAST_IN_SYNC = 40
+LAST_IN_SYNC_SERIOUS = 50
# etch released on 2007-04-08. So
# (Date::today - Date::parse('2007-04-08')).to_i
@@ -123,6 +123,35 @@
'incorrect-boot-order' => 'dependency-based init system',
'incorrect-updatercd-order' => 'dependency-based init system',
'ftbfs-gcc-4.4' => 'support gcc 4.4 as the default compiler',
+ 'kfreebsd' => 'kfreebsd-* as release architectures'
+}
+
+=begin
+RGTAGS = {
+ 'goal-dash' => 'switch /bin/sh to dash',
+ 'qa-doublebuild' => 'double compilation support',
+ 'goal-recommends' => 'no unmet recommends',
+ 'debmake' => 'drop debmake from Debian',
+ 'utf8-control' => 'UTF-8 changelog and control',
+ 'piuparts-stable-upgrade' => 'piuparts-clean archive',
+ 'goal-python2.5' => 'python2.5 support',
+ 'gfortran' => 'g77 -> gfortran transition',
+ 'lfs' => 'Remove arbitrary file size limits',
+ 'ipv6' => 'IPv6 support',
+ 'nfs-v4' => 'Support version 4 of the NFS protocol',
+ 'ddtp' => 'I18n support for package descriptions',
+ 'not-using-po-debconf' => 'I18n support in all debconf-using packages',
+ 'goal-ddtp' => 'I18n support for package descriptions',
+ 'boot-dependencies' => 'dependency-based init system',
+ 'missing-dependency' => 'dependency-based init system',
+ 'obsolete-initscript' => 'dependency-based init system',
+ 'runtime-problem' => 'dependency-based init system',
+ 'incorrect-provides' => 'dependency-based init system',
+ 'incorrect-dependency' => 'dependency-based init system',
+ 'incorrect-boot-order' => 'dependency-based init system',
+ 'incorrect-updatercd-order' => 'dependency-based init system',
+ 'ftbfs-gcc-4.4' => 'support gcc 4.4 as the default compiler',
'ftbfs-gcc-4.5' => 'support gcc 4.5 as the default compiler',
'kfreebsd' => 'kfreebsd-* as release architectures'
}
+=end
Modified: ddpo-by-mail/reminder-mail.rb
===================================================================
--- ddpo-by-mail/reminder-mail.rb 2010-02-11 21:15:28 UTC (rev 1682)
+++ ddpo-by-mail/reminder-mail.rb 2010-02-13 17:45:50 UTC (rev 1683)
@@ -1,47 +1,60 @@
#!/usr/bin/ruby -w
+puts "#{Time::now} Starting and reading the data we need"
+
require 'ddpo-config'
require 'bugs'
require 'testingissues'
require 'dehs'
+# Data.
+# $dipkgs: list of packages that don't have binary packages in !d-i
+# $email_maintains: hash of email => maintained pkgs
+# $email_uploads: hash of email => "uploaders:" pkgs
+# $email_realname: hash of email => real name
+# $pkg_maintlist: hash of srcpkg => bool (is it maintained or comaint by a list?)
+# $pkg_maintainer: hash of pkg => maintainer
+
+
# parse debian-installer packages
-pkg = nil
-dipkgs = []
-`zcat data/debian-installer/binary-*/Packages.gz`.each_line do |l|
- if l =~ /^Package: /
- if pkg != nil
- dipkgs << pkg
- end
- pkg = l.split(' ')[1].chomp
- elsif l =~ /^Source: /
- dipkgs << l.split(' ')[1].chomp
- pkg = nil
+$dipkgs = IO::readlines('data/di-sources.txt').map { |l| l.chomp }
+
+# Parse maintainers.txt and uploaders.txt
+$email_maintains = {}
+$email_uploads = {}
+$email_realname = {}
+$pkg_maintlist = {}
+$pkg_maintainer = {}
+IO::readlines('data/maintainers.txt').each do |l|
+ l.chomp!
+ src, email, name = l.split('|')
+ if $email_maintains[email].nil?
+ $email_maintains[email] = [ src ]
+ else
+ $email_maintains[email] << src
end
+ $pkg_maintlist[src] = true if email =~ /@list.*\.debian\.org$/
+ $pkg_maintainer[src] = email
+ if name != nil
+ name.gsub!(/í/, 'i')
+ $email_realname[email] = name
+ end
end
-dipkgs.uniq!
-
-# Parse ddpo_packages
-pkgs = Hash::new { [] }
-f = IO::read("data/ddpo_packages")
-f.each_line do |l|
- next if l =~ /^\)/
- pkg = l.match(/^([^\(]*)\(/)[1]
- vers = l.match(/\[([^\]]*)\]/)[1].split(/,/)
- pkgs[pkg] = vers
+IO::readlines('data/uploaders.txt').each do |l|
+ l.chomp!
+ src, email, name = l.split('|')
+ if $email_uploads[email].nil?
+ $email_uploads[email] = [ src ]
+ else
+ $email_uploads[email] << src
+ end
+ $pkg_maintlist[src] = true if email =~ /@list.*\.debian\.org$/
+ if name != nil
+ name.gsub!(/í/, 'i')
+ $email_realname[email] = name
+ end
end
-# Parse ddpo_maintainers
-maintainers_pkg = Hash::new { [] }
-maintainers_name = Hash::new { "" }
-f = IO::read('data/ddpo_maintainers')
-f.each_line do |l|
- n, email, noid, realname, packages = l.split(';', 5)
- maintainers_pkg[email] = packages.tr_s(';', ' ').split(' ')
- realname.gsub!(/í/, 'i')
- maintainers_name[email] = realname
-end
-
# read PTS subscribers
subscribers = Hash::new { [] }
f = IO::read('data/pts_subscriptions.txt')
@@ -57,28 +70,24 @@
end
end
-# Get maintainer for each package
-haslist = {}
-pkg_maintainer = {}
-maintainers_pkg.keys.each do |email|
- maintainers_pkg[email].each do |pkg|
- if pkg =~ /#/
- pkg = pkg.chop
- else
- pkg_maintainer[pkg] = email
- end
- haslist[pkg] = true if email =~ /@list.*\.debian\.org$/
- end
-end
-
# Get missing builds
+=begin
builds = {}
IO::read('data/ood_excuses.txt').each_line do |l|
pkg, ver, archs = l.chomp.split(' ')
builds[pkg] = archs
end
+=end
+builds = {}
+IO::readlines('data/wannabuild.txt').each do |l|
+ src, arch = l.chomp.split('|')
+ if builds[src].nil?
+ builds[src] = [ arch ]
+ else
+ builds[src] << arch
+ end
+end
-
# get removals
rms = []
IO::read('data/wnpp_rm').each_line do |l|
@@ -97,20 +106,24 @@
# get security issues
security = {}
+=begin # temporarily disabled until we figure out a good process for them
IO::read('data/security-packages.txt').each_line do |l|
pkg, issues = l.chomp.split(':')
security[pkg] = issues.to_i
end
+=end
bugs = parse_bugs
-testingissues = testing_issues(dipkgs)
+testingissues = testing_issues($dipkgs)
dehs = get_dehs
-puts "Computing issues for each package ..."
+puts "#{Time::now} Computing issues for each package ..."
# compute issues for each package
serious = {}
issues = {}
(bugs.keys + testingissues.keys + dehs.keys + security.keys).sort.uniq.each do |pkg|
+ next if $pkg_maintainer[pkg].nil? # means the package wasn't in unstable Sources!
+
if EXC_PKGS.include?(pkg)
puts "Skipping #{pkg}, it is excluded."
next
@@ -129,7 +142,7 @@
pissues << [ :builds, builds[pkg] ]
pserious = true
end
- if testingissues[pkg] and not builds[pkg]
+ if testingissues[pkg]
pissues << testingissues[pkg]
pserious ||= testingissues[pkg][1]
end
@@ -139,11 +152,9 @@
end
if lintian[pkg]
pissues << [ :lintian, lintian[pkg][0], lintian[pkg][1] ]
- #pserious ||= lintian[pkg][0]
end
if security[pkg]
pissues << [ :security, security[pkg] ]
- #pserious = true
end
unless pissues.empty?
issues[pkg] = pissues
@@ -166,9 +177,9 @@
T_DEHS = "dehs"
T_SECURITY = "security"
-puts "Preparing mail for each maintainer..."
+puts "#{Time::now} Preparing mail for each maintainer..."
# prepare mail for each maintainer
-(maintainers_pkg.keys + subscribers.keys).sort.uniq.each do |email|
+($email_maintains.keys + $email_uploads.keys + subscribers.keys).sort.uniq.each do |email|
if EXC_EMAILS.include?(email)
puts "Skipping mail for #{email}, IS EXCLUDED"
next
@@ -183,25 +194,25 @@
subscribers[email].each do |pkg|
maintained[pkg] = false
end
- maintainers_pkg[email].each do |pkg|
- if pkg =~ /#/
- maintained[pkg.chop] = true
- mpkgs[pkg.chop] = false
- else
+ if $email_maintains[email]
+ $email_maintains[email].each do |pkg|
maintained[pkg] = true
mpkgs[pkg] = true
end
end
- name = email
- if maintainers_name[email].length > 0
- name = maintainers_name[email]
+ if $email_uploads[email]
+ $email_uploads[email].each do |pkg|
+ maintained[pkg] = true
+ mpkgs[pkg] = false
+ end
end
+ name = $email_realname[email] ? $email_realname[email] : email
gtext =<<-EOF
From: DDPOMail robot <#{OWNER_EMAIL}>
To: "#{name}" <#{email}>
X-Debian: ddpomail
EOF
- if not maintainers_pkg[email].empty?
+ if not ($email_maintains[email].nil? and $email_uploads[email].nil?)
gtext +=<<-EOF
Subject: Possible problems in your Debian packages
@@ -216,13 +227,6 @@
EOF
end
- gtext +=<<-EOF
-This is an automated mail. These mails are sent twice a month.
-For more information about these mails, refer to
-http://wiki.debian.org/qa.debian.org/DdpoByMail
-
-EOF
-
text = {}
text[T_DEHS] = ""
@@ -233,7 +237,7 @@
next if EXC_EMAILS_PKGS[email].include?(pkg)
if maintained[pkg] and not mpkgs[pkg]
comaint = ' (you co-maintain it)'
- only_uploader = false if haslist[pkg].nil? and serious[pkg]
+ only_uploader = false if not $pkg_maintlist[pkg] and serious[pkg]
elsif maintained[pkg]
comaint = ''
only_uploader = false if serious[pkg]
@@ -249,7 +253,7 @@
issues[pkg].each do |iss|
if iss[0] == :bugs
bugs = iss[1].reject { |b| EXC_EMAILS_BUGS[email].include?(b[0]) }
- next if bugs.length == 0
+ next if bugs.length == 0
text[pkg][T_INCLUDE] = true
text[pkg][T_BUGS] = "= #{bugs.length} bug(s) that should be fixed soon:\n"
bugs.each do |b|
@@ -259,16 +263,16 @@
if b[4]
text[pkg][T_BUGS] += " Appears to affect stable, you should fix it for the next point release\n"
end
- # add nothing
+ # add nothing
elsif b[2] == "PROP"
- # add nothing
+ # add nothing
else
text[pkg][T_BUGS] += " Part of release goal: #{b[2]}\n"
end
end
elsif iss[0] == :builds
text[pkg][T_INCLUDE] = true
- text[pkg][T_BUILDS] = "= Missing build(s) on #{iss[1]}\n"
+ text[pkg][T_BUILDS] = "= Missing build(s) on #{iss[1].join(' ')}\n"
text[pkg][T_BUILDS] += " This might need manual action from your side.\n"
text[pkg][T_BUILDS] += " See https://buildd.debian.org/status/package.php?p=#{pkg}\n"
elsif iss[0] == :not_testing
@@ -292,14 +296,13 @@
end
text[T_DEHS] += " #{pkg} #{iss[2]} (Debian: #{iss[1]})\n"
elsif iss[0] == :lintian
- next if (iss[1] < 1 and iss[2] < 2)
- maintainer = pkg_maintainer[pkg]
- if maintainer == nil or maintainer.empty?
+ next if iss[1] < 1
+ if $pkg_maintainer[pkg].nil?
puts "Ugh, pkg vs maintainer relationship missing for #{pkg}!"
exit
end
text[pkg][T_INCLUDE] = true if iss[1] > 0
- text[pkg][T_LINTIAN] = "= Lintian reports"
+ text[pkg][T_LINTIAN] = "= Lintian:"
if iss[1] > 0
text[pkg][T_LINTIAN] += " #{iss[1]} error(s)"
end
@@ -307,8 +310,8 @@
text[pkg][T_LINTIAN] += " and" if iss[1] > 0
text[pkg][T_LINTIAN] += " #{iss[2]} warning(s)"
end
- text[pkg][T_LINTIAN] += ", you should consider fixing them.\n"
- text[pkg][T_LINTIAN] += " See http://lintian.debian.org/maintainer/#{maintainer}.html##{pkg}\n"
+ text[pkg][T_LINTIAN] += "\n"
+ text[pkg][T_LINTIAN] += " See http://lintian.debian.org/maintainer/#{$pkg_maintainer[pkg]}.html##{pkg}\n"
elsif iss[0] == :security
text[pkg][T_INCLUDE] = true
text[pkg][T_SECURITY] = "= There "
@@ -350,6 +353,10 @@
end
gtext += <<-EOF
------------ interesting stuff probably ends here ------------
+This is an automated mail. These mails are sent at most twice a month.
+For more information about these mails, refer to
+http://wiki.debian.org/qa.debian.org/DdpoByMail
+
We are sorry if this mail was useless for you. If you think it was
avoidable (that we can detect easily that the problems weren't
actually problems), please reply to it and let us know.
@@ -382,10 +389,10 @@
--
DDPOMail,
run by #{OWNER}
- EOF
+EOF
- fname = email.gsub('/', '__')
- File::open(MAIL_DIR + "/mail_#{fname}.txt", 'w') do |f|
- f.puts gtext
- end
+fname = email.gsub('/', '__')
+File::open(MAIL_DIR + "/mail_#{fname}.txt", 'w') do |f|
+ f.puts gtext
end
+end
Modified: ddpo-by-mail/testingissues.rb
===================================================================
--- ddpo-by-mail/testingissues.rb 2010-02-11 21:15:28 UTC (rev 1682)
+++ ddpo-by-mail/testingissues.rb 2010-02-13 17:45:50 UTC (rev 1683)
@@ -1,28 +1,31 @@
# parse testing status
require 'ddpo-config'
-require 'date'
-require 'pkgtestingstatus'
def testing_issues(dipkgs)
- now = Date::today
# file = Dir::glob('../testing-status/data.*').sort[-1]
- file = 'data/testing-status.raw'
- pts = PkgTestingStatus::read(File::new(file,'r')).reject { |k,v| v.inunstable != now }
+ file = 'data/testing-status.txt'
testingissues = {}
- pts.each_pair do |pkg, val|
+ IO::readlines('data/testing-status.txt').each do |l|
+ l.chomp!
+ pkg, in_testing, in_testing_age, sync, sync_age, debian_age = l.split('|')
+ in_testing_age = (in_testing_age != "")?in_testing_age.to_i : nil
+ sync_age = (sync_age != "")?sync_age.to_i : nil
+ debian_age = (debian_age != "")?debian_age.to_i : nil
+# p [pkg, in_testing, in_testing_age, sync, sync_age, debian_age]
next if dipkgs.include?(pkg)
- ok, days = val.testing_ok?(AGE_IN_DEBIAN)
- if not ok
- testingissues[pkg] = [:not_testing, true, days ]
+ if in_testing_age.nil?
+ if debian_age > AGE_IN_DEBIAN
+ testingissues[pkg] = [:not_testing, true, debian_age ]
+ end
+ elsif in_testing_age > 1 # in case there's some incoherency in udd
+ testingissues[pkg] = [:not_testing, true, in_testing_age ]
else
- ok, days = val.sync_ok?(LAST_IN_SYNC_SERIOUS)
- if not ok
- testingissues[pkg] = [:no_migration, true, days ]
- else
- ok, days = val.sync_ok?(LAST_IN_SYNC)
- if not ok
- testingissues[pkg] = [:no_migration, false, days ]
- end
+ if sync_age.nil?
+# puts "Interesting buggy case with #{pkg}. Ignore."
+ elsif sync_age > LAST_IN_SYNC_SERIOUS
+ testingissues[pkg] = [:no_migration, true, sync_age ]
+ elsif sync_age > LAST_IN_SYNC
+ testingissues[pkg] = [:no_migration, false, sync_age ]
end
end
end
More information about the Collab-qa-commits
mailing list