[Collab-qa-commits] r247 - / archive-cleanup debcluster tools tools/debcmp
lucas at alioth.debian.org
lucas at alioth.debian.org
Mon Jun 25 12:44:02 UTC 2007
Author: lucas
Date: 2007-06-25 12:44:01 +0000 (Mon, 25 Jun 2007)
New Revision: 247
Added:
tools/
tools/debcmp/
tools/debcmp/Notes
tools/debcmp/debcmp.rb
Modified:
archive-cleanup/Template.proposed-orphan
archive-cleanup/Template.proposed-removal
debcluster/Notes
Log:
commited tools/debcmp: compares build results with what's in the archive
Modified: archive-cleanup/Template.proposed-orphan
===================================================================
--- archive-cleanup/Template.proposed-orphan 2007-06-22 07:52:19 UTC (rev 246)
+++ archive-cleanup/Template.proposed-orphan 2007-06-25 12:44:01 UTC (rev 247)
@@ -1,7 +1,8 @@
-Subject: <packagename>: should this package be orphaned?
+To: submit at bugs.debian.org
+Subject: XXX: should this package be orphaned?
-Package: <packagename>
-Version: <version>
+Package: XXX
+Version: XXX
Severity: serious
User: debian-qa at lists.debian.org
Usertags: proposed-orphan
@@ -12,7 +13,7 @@
came up as a package that should maybe be orphaned by its maintainer,
because:
-[ include as many reasons as possible here ]
+[ XXX include as many reasons as possible here ]
If you think that it should be removed from Debian instead of being
orphaned, please reply to this bug and tell so.
Modified: archive-cleanup/Template.proposed-removal
===================================================================
--- archive-cleanup/Template.proposed-removal 2007-06-22 07:52:19 UTC (rev 246)
+++ archive-cleanup/Template.proposed-removal 2007-06-25 12:44:01 UTC (rev 247)
@@ -1,7 +1,8 @@
-Subject: <packagename>: should this package be removed?
+To: submit at bugs.debian.org
+Subject: XXX: should this package be removed?
-Package: <packagename>
-Version: <version>
+Package: XXX
+Version: XXX
Severity: serious
User: debian-qa at lists.debian.org
Usertags: proposed-removal
@@ -11,7 +12,7 @@
While reviewing packages that were not included in Etch, your package
came up as a possible candidate for removal from Debian, because:
-[ include as many reasons as possible here ]
+[ XXX include as many reasons as possible here ]
If you think that it should be orphaned instead of being removed from
Debian, please reply to this bug and tell so.
Modified: debcluster/Notes
===================================================================
--- debcluster/Notes 2007-06-22 07:52:19 UTC (rev 246)
+++ debcluster/Notes 2007-06-25 12:44:01 UTC (rev 247)
@@ -25,6 +25,11 @@
rebuild-archive/gentasklist.rb $CHROOT $ID < /tmp/wb_$ID.3 | bogosort -n > /tmp/tasklisttext_$ID
echo "Tasklist (in text) is /tmp/tasklisttext_$ID"
+# buxy
+sed -i "s/$/ --backup \/debcluster-data\/results\/$ID/" /tmp/tasklisttext_$ID
+sed -i 's/-c sid32 /-c sid32-sym /' /tmp/tasklisttext_$ID
+
+# old, gzip vs bzip2
cp /tmp/tasklisttext_$ID /tmp/tasklisttext_$ID.z
sed -i 's/^b_/bz_/' /tmp/tasklisttext_$ID.z
sed -i 's/-c feisty32 /-c feisty32-bzip2 /' /tmp/tasklisttext_$ID.z
@@ -32,7 +37,6 @@
sed -i "s/$/ --backup \/debcluster-data\/results\/$ID.z/" /tmp/tasklisttext_$ID.z
sed -i "s/$/ --backup \/debcluster-data\/results\/$ID/" /tmp/tasklisttext_$ID
-
./buildtasklist.rb </tmp/tasklisttext_$ID > /tmp/tasklist_$ID
P=libtk-img
@@ -93,15 +97,15 @@
piuparts -a -b /debcluster-data/chroots/etch32-piuparts.tgz -d etch -m 'http://idpot.grenoble.grid5000.fr/debian' feed2imap
ARCH=i386
-ID=20070118.remove
+ID=20070618.remove
PIUSCRIPT=/debcluster-scripts/configs/piuparts-dump-nopanic
# ubuntu
# PIUARGS=" -e $PIUSCRIPT -c /debcluster-data/chroots/feisty32-piuparts.tgz -d feisty "
-PIUARGS=" -e $PIUSCRIPT -c /debcluster-data/chroots/etch32-piuparts.tgz -d etch "
+PIUARGS=" -e $PIUSCRIPT -c /debcluster-data/chroots/sid32-piuparts.tgz -d sid "
# ubuntu
# cat ~/debcluster-data/ubuntu-mirror/dists/feisty/{main,restricted,universe}/binary-${ARCH}/Packages |grep "^Package:" | awk '{print $2}' | sort > /tmp/wp.${ID}.1
-cat ~/debcluster-data/debian-mirror/dists/etch/main/binary-${ARCH}/Packages |grep "^Package:" | awk '{print $2}' | sort > /tmp/wp.${ID}.1
+cat ~/debcluster-data/debian-mirror/dists/sid/main/binary-${ARCH}/Packages |grep "^Package:" | awk '{print $2}' | sort > /tmp/wp.${ID}.1
mdt list_sub /tmp/wp.${ID}.1 piuparts/piuparts.untestable > /tmp/wp.${ID}.2
piuparts/gentasklist.rb "$PIUARGS" $ID < /tmp/wp.${ID}.2 > /tmp/tasklisttext.piu.${ID}.3
echo "Tasklist (in text) is /tmp/tasklisttext.piu.${ID}.3"
Added: tools/debcmp/Notes
===================================================================
--- tools/debcmp/Notes (rev 0)
+++ tools/debcmp/Notes 2007-06-25 12:44:01 UTC (rev 247)
@@ -0,0 +1,20 @@
+
+D1=/scratch/grenoble/lnussbaum/debcluster-data/debian-mirror/pool/main
+D2=/scratch/grenoble/lnussbaum/debcluster-data/logs/output.20070220
+
+for i in $(find $D2 -name '*.dsc' -exec basename {} \; |sed 's/.dsc$//' | sort); do
+ set $(echo $i | sed 's/_/ /')
+ echo $1 $2
+ echo "./debcmp.rb -p $1 -v $2 -a $D1 -b $D2 --pool" > ${1}_${2}.cmd
+ ./debcmp.rb -p $1 -v $2 -a $D1 -b $D2 --pool &> ${1}_${2}.out || echo "FAILED $1 $2"
+done &> debcmp.log
+
+# grep -E '^(E|W|I):'
+
+cat debcmp.log | grep FAILED | while read l; do
+set $l
+shift
+ echo "./debcmp.rb -p $1 -v $2 -a $D1 -b $D2 --pool" > ${1}_${2}.cmd
+ ./debcmp.rb -p $1 -v $2 -a $D1 -b $D2 --pool &> ${1}_${2}.out || echo "FAILED $1 $2"
+done
+
Added: tools/debcmp/debcmp.rb
===================================================================
--- tools/debcmp/debcmp.rb (rev 0)
+++ tools/debcmp/debcmp.rb 2007-06-25 12:44:01 UTC (rev 247)
@@ -0,0 +1,355 @@
+#!/usr/bin/ruby -w
+#
+# example:
+# ./debcmd.rb -p hebcal -v 3.5-1 -a ~/mir/mirror/h/hebcal -b ~/mir/built/h/hebcal
+#
+# read ./debcmp.rb --help
+
+require 'optparse'
+
+package = nil
+version = nil
+dir_a = nil
+dir_b = nil
+$verbose = false
+$arch = 'i386'
+pool = false
+tmpbase = '/scratch/grenoble/lnussbaum/tmp'
+
+progname = File::basename($PROGRAM_NAME)
+opts = OptionParser::new do |opts|
+ opts.program_name = progname
+ opts.banner = "Usage: #{progname} [options]"
+ opts.separator ""
+ opts.separator "Options:"
+
+ opts.on("-p", "--package PACKAGE", "Package") do |p|
+ package = p
+ end
+ opts.on("-v", "--version VERSION", "Version") do |v|
+ version = v
+ end
+
+ opts.on("-a", "--a-dir DIR", "Directory for a") do |v|
+ dir_a = v
+ end
+
+ opts.on("-b", "--b-dir DIR", "Directory for b") do |v|
+ dir_b = v
+ end
+
+ opts.on("", "--verbose", "Verbose mode") do
+ $verbose = true
+ end
+
+ opts.on("", "--arch ARCH", "Architecture") do |a|
+ $arch = a
+ end
+
+ opts.on("", "--pool", "Use pool dir structure. -a and -b then specify the base directory of the pool.") do
+ pool = true
+ end
+ opts.separator ""
+ opts.separator "Examples:"
+ opts.separator " ./debcmd.rb -p hebcal -v 3.5-1 -a ~/mirror/h/hebcal -b ~/built/h/hebcal"
+ opts.separator " ./debcmd.rb -p hebcal -v 3.5-1 --pool -a ~/mirror -b ~/built"
+end
+opts.parse!(ARGV)
+
+if [ package, version, dir_a, dir_b ].include?(nil)
+ puts "Missing mandatory argument"
+ exit(1)
+end
+
+# determine upstream version
+if version =~ /-/
+ sversion = version.gsub(/-[^-]*$/,'')
+else
+ sversion = version
+end
+
+# create the first working directories
+$wd = `mktemp -d #{tmpbase}/debcmp.XXXXXX`.chomp
+system("mkdir -p #{$wd}/a #{$wd}/b")
+
+# determine source dirs
+if pool
+ if package =~ /^lib/
+ sdir_a = dir_a + '/' + package[0..3] + '/' + package
+ sdir_b = dir_b + '/' + package[0..3] + '/' + package
+ else
+ sdir_a = dir_a + '/' + package[0..0] + '/' + package
+ sdir_b = dir_b + '/' + package[0..0] + '/' + package
+ end
+else
+ sdir_a = dir_a
+ sdir_b = dir_b
+end
+
+puts "Comparing #{package} #{version} (#{sversion})" if $verbose
+
+def systemr(cmd)
+ raise "Command failed: #{cmd}" if not system(cmd)
+end
+
+# copy files to local tmp dir
+if sversion == version
+ systemr("cp #{sdir_a}/*_#{sversion}.tar.gz #{sdir_a}/*_#{version}.dsc #{$wd}/a")
+ systemr("cp #{sdir_b}/*_#{sversion}.tar.gz #{sdir_b}/*_#{version}.dsc #{$wd}/b")
+ l = Dir::glob("#{sdir_a}/*_#{version}_*.deb").reject { |e| e !~ /_(#{$arch}|all).deb/ }
+ systemr("cp #{l.join(' ')} #{$wd}/a")
+ l = Dir::glob("#{sdir_b}/*_#{version}_*.deb").reject { |e| e !~ /_(#{$arch}|all).deb/ }
+ systemr("cp #{l.join(' ')} #{$wd}/b")
+else
+ systemr("cp #{sdir_a}/*_#{sversion}.orig.tar.gz #{sdir_a}/*_#{version}{.dsc,.diff.gz} #{$wd}/a")
+ l = Dir::glob("#{sdir_a}/*_#{version}_*.deb").reject { |e| e !~ /_(#{$arch}|all).deb/ }
+ systemr("cp #{l.join(' ')} #{$wd}/a")
+ systemr("cp #{sdir_b}/*_#{sversion}.orig.tar.gz #{sdir_b}/*_#{version}{.dsc,.diff.gz} #{$wd}/b")
+ l = Dir::glob("#{sdir_b}/*_#{version}_*.deb").reject { |e| e !~ /_(#{$arch}|all).deb/ }
+ systemr("cp #{l.join(' ')} #{$wd}/b")
+end
+
+#############################
+# compare orig.tar.gz files
+if sversion == version
+ if not system("cmp #{$wd}/a/*.tar.gz #{$wd}/b/*.tar.gz &>/dev/null")
+ puts "E: .tar.gz files differ (native package)."
+ end
+else
+ if not system("cmp #{$wd}/a/*.orig.tar.gz #{$wd}/b/*.orig.tar.gz")
+ puts "E: .orig.tar.gz files differ."
+ end
+end
+
+#############################
+# Compare .diff.gz files
+if sversion != version
+ # extract .diff.gz files and compare them
+ systemr("mkdir -p #{$wd}/diffs")
+ systemr("zcat #{$wd}/a/*.diff.gz > #{$wd}/diffs/a.diff")
+ systemr("zcat #{$wd}/b/*.diff.gz > #{$wd}/diffs/b.diff")
+
+ s = `bash -c "interdiff #{$wd}/diffs/{a,b}.diff 2>&1"`.chomp
+ if s.length != 0
+ puts "E: .diff.gz files differ. Interdiff:"
+ puts s
+ end
+end
+
+# compare list of .debs
+debs_a = Dir::glob("#{$wd}/a/*.deb").map { |f| File::basename(f) }.sort
+debs_b = Dir::glob("#{$wd}/b/*.deb").map { |f| File::basename(f) }.sort
+if debs_a != debs_b
+ puts "E: different debs in dirs:"
+ puts "E: A: #{debs_a.join(' ')}"
+ puts "E: B: #{debs_b.join(' ')}"
+end
+
+debstrip_a = debs_a.map { |d| d.gsub(/_.*/,'') }
+debstrip_b = debs_b.map { |d| d.gsub(/_.*/,'') }
+
+# Extract control files and packages
+systemr("mkdir -p #{$wd}/c_a #{$wd}/c_b #{$wd}/p_a #{$wd}/p_b")
+# dpkg-deb creates the target dir if needed
+debs_a.each do |d|
+ ds = d.gsub(/_.*/, '')
+ systemr("dpkg-deb -e #{$wd}/a/#{d} #{$wd}/c_a/#{ds}")
+ systemr("dpkg-deb -x #{$wd}/a/#{d} #{$wd}/p_a/#{ds}")
+end
+debs_b.each do |d|
+ ds = d.gsub(/_.*/, '')
+ systemr("dpkg-deb -e #{$wd}/b/#{d} #{$wd}/c_b/#{ds}")
+ systemr("dpkg-deb -x #{$wd}/b/#{d} #{$wd}/p_b/#{ds}")
+end
+
+# Compare control files
+# We ignore the Installed-Size line
+debstrip_a.each do |d|
+ s = `bash -c "diff -u <(grep -v -e '^Installed-Size: ' #{$wd}/c_a/#{d}/control) <(grep -v -e '^Installed-Size: ' #{$wd}/c_b/#{d}/control)"`.chomp
+ if s.length != 0
+ puts "E: control files for #{d} differ:"
+ puts s
+ end
+end
+
+# Get files lists
+debstrip_a.each do |d|
+ systemr("cd #{$wd}/p_a/#{d} && find . | sort > #{$wd}/a_#{d}.files")
+ systemr("cd #{$wd}/p_b/#{d} && find . | sort > #{$wd}/b_#{d}.files")
+ systemr("cd #{$wd}/p_a/#{d} && find . -exec file {} + | sort | sed -e 's/for GNU\\/Linux .\\..\\../for GNU\\/Linux X.X.X/g' > #{$wd}/a_#{d}.file_output")
+ systemr("cd #{$wd}/p_b/#{d} && find . -exec file {} + | sort | sed -e 's/for GNU\\/Linux .\\..\\../for GNU\\/Linux X.X.X/g' > #{$wd}/b_#{d}.file_output")
+ systemr("cd #{$wd}/p_a/#{d} && find . -exec file -i {} + | sort | sed -e 's/for GNU\\/Linux .\\..\\../for GNU\\/Linux X.X.X/g' > #{$wd}/a_#{d}.filei_output")
+ systemr("cd #{$wd}/p_b/#{d} && find . -exec file -i {} + | sort | sed -e 's/for GNU\\/Linux .\\..\\../for GNU\\/Linux X.X.X/g' > #{$wd}/b_#{d}.filei_output")
+end
+
+# compare files list
+debstrip_a.each do |d|
+ s = `bash -c "diff -u #{$wd}/a_#{d}.files #{$wd}/b_#{d}.files"`.chomp
+ if s.length != 0
+ puts "E: files list for #{d} differ:"
+ puts s
+ end
+end
+
+# compare file output list
+debstrip_a.each do |d|
+ s = `bash -c "diff -u <(sed 's/last modified: [^,]*,//' #{$wd}/a_#{d}.file_output) <(sed 's/last modified: [^,]*,//' #{$wd}/b_#{d}.file_output)"`.chomp
+ if s.length != 0
+ puts "E: file output list for #{d} differ:"
+ puts s
+ end
+end
+debstrip_a.each do |d|
+ s = `bash -c "diff -u #{$wd}/a_#{d}.filei_output #{$wd}/b_#{d}.filei_output"`.chomp
+ if s.length != 0
+ puts "E: file -i output list for #{d} differ:"
+ puts s
+ end
+end
+
+# function to compare file sizes
+ERROR_THR = 0.2
+WARNING_THR = 0.03
+def cmpsize(d, base_a, base_b, f)
+ s_a = File::size("#{base_a}/#{f}")
+ s_b = File::size("#{base_b}/#{f}")
+ diff = (s_a - s_b).abs / [ s_a, s_b ].min.to_f
+ if diff >= ERROR_THR
+ puts "E: #{d}: #{f} sizes differ (#{s_a} vs #{s_b}, #{(diff * 1000).to_i / 10.0}% diff)"
+ elsif diff >= WARNING_THR
+ puts "W: #{d}: #{f} sizes differ (#{s_a} vs #{s_b}, #{(diff * 1000).to_i / 10.0}% diff)"
+ end
+end
+
+def equalfiles?(d, base_a, base_b, f, verbose = true)
+ s = `diff -u #{base_a}/#{f} #{base_b}/#{f}`
+ if s.length != 0
+ if verbose
+ puts "E: #{d}: #{f} differs:"
+ puts s
+ end
+ return false
+ end
+ return true
+end
+
+# Compare two files
+def filecmp(d, base_a, base_b, f, t)
+ if t == 'application/x-not-regular-file'
+ # nothing to do
+ elsif t == 'application/octet-stream'
+ s = `file -b #{base_a}/#{f}`.chomp
+ if s =~ /gzip compressed data/
+ systemr("zcat #{base_a}/#{f} > #{base_a}/#{f}.ungzipped")
+ systemr("zcat #{base_b}/#{f} > #{base_b}/#{f}.ungzipped")
+ t = `file -ib #{base_a}/#{f}.ungzipped`.chomp
+ filecmp(d, base_a, base_b, "#{f}.ungzipped", t)
+ systemr("rm -f #{base_a}/#{f}.ungzipped #{base_b}/#{f}.ungzipped")
+ elsif [ 'data' ].include?(t)
+ cmpsize(d, base_a, base_b, f)
+ elsif s =~ /GNU message catalog/
+ equalfiles?(d, base_a, base_b, f, false)
+ else
+ if not equalfiles?(d, base_a, base_b, f)
+ puts "I: Filetype unknown, files differ (mime: application/octet-stream): #{f} / #{s}"
+ cmpsize(d, base_a, base_b, f)
+ end
+ end
+
+ elsif t =~ /^application\/x-sharedlib/
+ system("objdump -p #{base_a}/#{f} |grep NEEDED > #{base_a}/#{f}.objdumpp")
+ system("objdump -p #{base_b}/#{f} |grep NEEDED > #{base_b}/#{f}.objdumpp")
+ s = `diff -u #{base_a}/#{f}.objdumpp #{base_a}/#{f}.objdumpp`.chomp
+ if s.length != 0
+ puts "E: #{d}: #{f} NEEDED in headers differ:"
+ puts s
+ end
+ systemr("objdump -T #{base_a}/#{f} | grep '*UND*' | cut -c 34- > #{base_a}/#{f}.undef")
+ systemr("objdump -T #{base_b}/#{f} | grep '*UND*' | cut -c 34- > #{base_b}/#{f}.undef")
+ s = `diff -u #{base_b}/#{f}.undef #{base_b}/#{f}.undef`.chomp
+ if s.length != 0
+ puts "E: #{d}: #{f} undefined symbols differ:"
+ puts s
+ end
+ systemr("objdump -T #{base_a}/#{f} | grep '.text' | cut -c 34- > #{base_a}/#{f}.def")
+ systemr("objdump -T #{base_b}/#{f} | grep '.text' | cut -c 34- > #{base_b}/#{f}.def")
+ s = `diff -u #{base_b}/#{f}.def #{base_b}/#{f}.def`.chomp
+ if s.length != 0
+ puts "E: #{d}: #{f} Defined symbols differ:"
+ puts s
+ end
+ systemr("rm -f #{base_a}/#{f}.objdumpp #{base_b}/#{f}.objdumpp #{base_a}/#{f}.undef #{base_b}/#{f}.undef #{base_a}/#{f}.def #{base_b}/#{f}.def")
+
+ elsif t =~ /^application\/x-executable.*dynamically linked \(uses shared libs\)/
+ system("objdump -p #{base_a}/#{f} |grep NEEDED > #{base_a}/#{f}.objdumpp")
+ system("objdump -p #{base_b}/#{f} |grep NEEDED > #{base_b}/#{f}.objdumpp")
+ s = `diff -u #{base_a}/#{f}.objdumpp #{base_a}/#{f}.objdumpp`.chomp
+ if s.length != 0
+ puts "E: #{d}: #{f} NEEDED in headers differ:"
+ puts s
+ end
+ systemr("objdump -T #{base_a}/#{f} | grep '*UND*' | cut -c 34- > #{base_a}/#{f}.undef")
+ systemr("objdump -T #{base_b}/#{f} | grep '*UND*' | cut -c 34- > #{base_b}/#{f}.undef")
+ s = `diff -u #{base_b}/#{f}.undef #{base_b}/#{f}.undef`.chomp
+ if s.length != 0
+ puts "E: #{d}: #{f} undefined symbols differ:"
+ puts s
+ end
+ systemr("rm -f #{base_a}/#{f}.objdumpp #{base_b}/#{f}.objdumpp #{base_a}/#{f}.undef #{base_b}/#{f}.undef")
+ cmpsize(d, base_a, base_b, f)
+
+ elsif t =~ /^text\// or t == 'application/x-perl' or t == 'application/x-empty' or t == '' or t =~ /^image\// or t == 'application/x-shellscript' or t == 'application/pdf'
+ # files must be the sames
+ equalfiles?(d, base_a, base_b, f)
+ elsif t == 'application/x-archive'
+ # just compare size
+ cmpsize(d, base_a, base_b, f)
+ else
+ if not equalfiles?(d, base_a, base_b, f)
+ puts "I: #{d} Filetype unknown, files differ: #{f} / #{t}"
+ cmpsize(d, base_a, base_b, f)
+ end
+ end
+end
+
+# Compare by filetype
+debstrip_a.each do |d|
+ files = {}
+ IO::read("#{$wd}/a_#{d}.filei_output").each_line do |l|
+ f,t = l.split($;,2)
+ t.chomp!
+ f.gsub!(/:$/,'')
+ files[f] = t
+ end
+ files.each_pair do |f,t|
+ filecmp(d, "#{$wd}/p_a/#{d}", "#{$wd}/p_b/#{d}", f, t)
+ end
+end
+
+system("rm -rf #{$wd}")
+exit(0)
+
+# Compare file lists
+
+#
+#tar tvzf + output filtering
+#interdiff
+#
+## binaries
+#ar x pathto.deb
+#compare content of data.tar.gz
+#extract control.tar.gz and compare content
+#extract data. for each file, the result of "file" must be the same:
+#find . -exec file {} +
+#same with file -i
+#
+#use file -i output and check some files:
+#- same content
+#- same size: equal, or +/- 1,5,10%
+#
+#By filetype:
+# /application\/x-executable .* dynamically linked/
+# nm -D
+# objdump -x # headers
+# objdump -T # dynamic relocation records
+#
Property changes on: tools/debcmp/debcmp.rb
___________________________________________________________________
Name: svn:executable
+ *
More information about the Collab-qa-commits
mailing list