[Collab-qa-commits] r2104 - rc-buggy-leaf-packages
Niels Thykier
nthykier at alioth.debian.org
Thu Dec 15 14:02:32 UTC 2011
Author: nthykier
Date: 2011-12-15 14:02:32 +0000 (Thu, 15 Dec 2011)
New Revision: 2104
Added:
rc-buggy-leaf-packages/rt-find-rc-buggy-leaf-packages
rc-buggy-leaf-packages/rt-get-rcbuggy-packages
rc-buggy-leaf-packages/rt-source-rcbugs.sql
Log:
Added some RT specific tools/queries
Added: rc-buggy-leaf-packages/rt-find-rc-buggy-leaf-packages
===================================================================
--- rc-buggy-leaf-packages/rt-find-rc-buggy-leaf-packages (rev 0)
+++ rc-buggy-leaf-packages/rt-find-rc-buggy-leaf-packages 2011-12-15 14:02:32 UTC (rev 2104)
@@ -0,0 +1,312 @@
+#!/usr/bin/perl
+#
+# Exploits the Debian Release Team infrastructure to find RC-buggy
+# leaf packages. This tool uses the data available to Britney (the
+# testing migration tool) to find its results.
+#
+#
+# Copyright 2011 Niels Thykier <niels at thykier.net>
+# License: GPL-2 (or at your option any later)
+#
+# Parts taken form Lintian, which is also licensed under GPL-2 (or at
+# your option any later). The copyright holder is most likely
+# Christian Schwarz (1998), though the code has been modified since
+# then original commit.
+
+# Usage: $0 <britney.conf> [bugfile] [deadline]
+#
+# - britney.conf is (usually) /srv/release.debian.org/etc/britney.conf
+# - bugfile is the bug file generated from UDD
+# - can also be the "BugsV" file from britney (which is the default)
+# - deadline is the proposed deadline (may be empty to skip it)
+# - format is "YYYY-MM-DD"
+#
+#
+# Most likely you want to re-direct stderr to a file or /dev/null as
+# it complains about all "unknown" packages in dependency relations.
+#
+# - These are usually not errors (e.g. the missing packages are only
+# part of an or-group).
+
+use strict;
+use warnings;
+
+# $needs->{src}->{bin}->{$dep} = 1
+# $needs->{src}->{_BD}->{$dep} = 1
+# $needs->{src}->{_verison}->{$version} = 1
+my $needs;
+# $buggy->{src} = [bug1, bug2, ...]
+my $buggy;
+
+# $rdep->{srcX} > 0 (if something depends on something from srcX)
+my $rdeps;
+
+# $bin2src->{bin}->{_main} = $src
+# - _main is the "main" provider of $src (if any)
+# $bin2src->{bin}->{src} = 1
+# - bin may map to multiple sources (hi Provides)
+my $bin2src;
+
+my $britney_conf = shift @ARGV;
+my $bugfile = shift @ARGV;
+my $deadline = shift @ARGV;
+
+my $conf = _parse_britney_conf ($britney_conf);
+
+($needs, $bin2src) = _parse_package_data ($conf);
+
+$rdeps = _calculate_rdeps ($needs, $bin2src);
+
+unless ($bugfile) {
+ my $path = $conf->{'testing'};
+ $bugfile = "$path/BugsV";
+}
+
+$buggy = _parse_bugs ($bugfile, $bin2src);
+
+foreach my $buggy_src (sort keys %$buggy) {
+ # If it is not in testing, ignore it.
+ next unless $needs->{$buggy_src};
+ next if $rdeps->{$buggy_src};
+ print "Package: $buggy_src\n";
+ print "Version: " . join (' ', keys %{ $needs->{$buggy_src}->{'_version'}}) . "\n";
+ print "Bugs: ". join (' ', @{ $buggy->{$buggy_src} }) ."\n";
+ print "Deadline: $deadline\n" if $deadline;
+ print "\n";
+}
+
+exit 0;
+
+sub _calculate_rdeps {
+ my ($needs, $bin2src) = @_;
+ my $rdeps = {};
+ my %once = ();
+
+ foreach my $src (keys %$needs) {
+ foreach my $el (keys %{$needs->{$src}}) {
+ next if $el eq '_version';
+ foreach my $dep (keys %{$needs->{$src}->{$el}}) {
+ my $providers = $bin2src->{$dep};
+ #print STDERR "N: $src ($el) -> $prov_src ($dep)\n" if $prov_src;
+ unless ($providers) {
+ print STDERR "warning: cannot determine the provider of $dep ($src via $el)\n"
+ unless $once{$dep}++;
+ next;
+ }
+ foreach my $prov (keys %$providers) {
+ next if $prov eq '_main'; # fake entry
+ next if $src eq $prov; # self depends does not count
+ #print STDERR "N: $src deps on $prov (via $el)\n";
+ $rdeps->{$prov}++;
+ }
+ }
+ }
+ }
+
+ return $rdeps;
+}
+
+sub _parse_bugs {
+ my ($bugfile, $bin2src) = @_;
+ my $buggy = {};
+
+ open my $fd, '<', $bugfile or die "opening $bugfile: $!";
+
+ while ( my $line = <$fd> ) {
+ my ($pkg, $bug);
+ chomp $line;
+
+ ($pkg, $bug) = split m/\s++/, $line, 2;
+
+ if ($pkg =~ s/^src://) {
+ push @{ $buggy->{$pkg} }, split m/\s*,\s*/, $bug;
+ } else {
+ my $src = $bin2src->{$pkg}->{'_main'};
+ next unless $src;
+ push @{ $buggy->{$src} }, split m/\s*,\s*/, $bug;
+ }
+ }
+
+ close $fd;
+ return $buggy;
+}
+
+sub _parse_britney_conf {
+ my ($conffile) = @_;
+ my $conf = {};
+
+ open my $fd, '<', $conffile or die "opening $conffile: $!";
+ while ( my $line = <$fd> ) {
+ my ($var, $value);
+ chomp $line;
+ $line =~ s/^\s*+//o;
+ $line =~ s/\#.*+$//o;
+ next unless $line;
+
+ ($var, $value) = split m/\s*+=\s*+/o, $line, 2;
+ die "Unknown line ($.:\"$line\")\n" unless defined $var and defined $value;
+
+ $conf->{lc $var} = $value;
+ }
+ close $fd;
+
+ return $conf;
+}
+
+sub _parse_package_data {
+ my ($conf) = @_;
+ my $path = $conf->{'testing'};
+ my @archs = split m/\s++/o, $conf->{'architectures'};
+ my $needs = {};
+ my $bin2src = {};
+ my $sources = "$path/Sources";
+
+ open my $fd, '<', $sources or die "opening $sources: $!";
+ _parse_file ($fd, sub { _parse_source ($needs, @_) });
+ close $fd;
+
+ foreach my $file (map { "$path/Packages_$_" } @archs) {
+ open $fd, '<', $file or die "opening $file: $!";
+ _parse_file ($fd, sub { _parse_pg ($needs, $bin2src, @_) });
+ close $fd;
+ }
+
+ return ($needs, $bin2src);
+}
+
+sub _parse_source {
+ my ($needs, $pg) = @_;
+ my $src = $pg->{'package'};
+ my $version = $pg->{'version'};
+ my $depstr = '';
+ die "Sources has paragraph without Source and Version field\n"
+ unless defined $src && defined $version;
+ foreach my $f (qw(build-depends build-depends-indep)) {
+ my $val = $pg->{$f};
+ next unless $val;
+ $depstr .= ', ' if $depstr;
+ $depstr .= $val;
+ }
+ foreach my $dep (_split_dep ($depstr)) {
+ $needs->{$src}->{'_BD'}->{$dep} = 1;
+ }
+ $needs->{$src}->{'_version'}->{$version} = 1;
+}
+
+sub _parse_pg {
+ my ($needs, $bin2src, $pg) = @_;
+ my $src = $pg->{'source'};
+ my $src_version;
+ my $pkg = $pg->{'package'};
+ my $depstr = '';
+
+ if ($src) {
+ # strip src version (if any)
+ if ($src =~ s/\s*\((.*)\)$//) {
+ $src_version = $1;
+ }
+ } else {
+ $src = $pkg;
+ }
+ $src_version = $pg->{'version'} unless defined $src_version;
+
+ die "Packages_X has paragraph without Package or Version field\n"
+ unless $pkg && defined $src_version;
+
+ next unless $needs->{$src}->{'_version'}->{$src_version};
+
+ $bin2src->{$pkg}->{'_main'} = $src;
+ $bin2src->{$pkg}->{$src} = 1;
+ if ($pg->{'provides'}) {
+ my $prov = $pg->{'provides'};
+ $prov =~ s/^\s*+//o;
+ $prov =~ s/\s*+$//o;
+ foreach my $p (split m/\s*+,\s*+/o, $prov) {
+ $bin2src->{$p}->{$src} = 1;
+ #print STDERR "N: $src (via $pkg) provides $p\n";
+ }
+ }
+
+ foreach my $f (qw(pre-depends depends)) {
+ my $val = $pg->{$f};
+ next unless $val;
+ $depstr .= ', ' if $depstr;
+ $depstr .= $val;
+ }
+
+ foreach my $dep (_split_dep ($depstr)) {
+ $needs->{$src}->{$pkg}->{$dep} = 1;
+ #print STDERR "N: $src depends on $dep (via $pkg)\n";
+ }
+}
+
+sub _split_dep {
+ my ($dep) = @_;
+ # Remove version and architecture
+ $dep =~ s/\[[^\]]*\]//og;
+ $dep =~ s/\([^\)]*\)//og;
+ $dep =~ s/^\s*+//o;
+ $dep =~ s/\s*+$//o;
+ return split m/\s*+[,|]\s*+/o, $dep;
+}
+
+
+
+## More or less stolen from Lintian, so under GPL-2 and
+# (most likely) Copyright (C) 1998 Christian Schwarz
+sub _parse_file {
+ my ($fd, $code) = @_;
+ my $open_section = 0;
+ my $section;
+ my $last_tag;
+ while ( my $line = <$fd> ) {
+ chomp ($line);
+ if ($line =~ m/^\s*$/o) {
+ if ($open_section) { # end of current section
+ # pass the current section to the handler
+ $code->($section);
+ $section = {};
+ $open_section = 0;
+ }
+ }
+
+ if ($line =~ m/^([^: \t]+):\s*$/o) {
+ $open_section = 1;
+
+ my ($tag) = (lc $1);
+ $section->{$tag} = '';
+
+ $last_tag = $tag;
+ }
+ # new field?
+ elsif ($line =~ m/^([^: \t]+):\s*(.*)$/o) {
+ $open_section = 1;
+
+ # Policy: Horizontal whitespace (spaces and tabs) may occur
+ # immediately before or after the value and is ignored there.
+ my ($tag,$value) = (lc $1,$2);
+ $value =~ s/\s+$//;
+ $section->{$tag} = $value;
+
+ $last_tag = $tag;
+ }
+ # continued field?
+ elsif ($line =~ m/^([ \t].*)$/o) {
+ $open_section or next;
+
+ # Policy: Many fields' values may span several lines; in this case
+ # each continuation line must start with a space or a tab. Any
+ # trailing spaces or tabs at the end of individual lines of a
+ # field value are ignored.
+ my $value = $1;
+ $value =~ s/\s+$//;
+ $section->{$last_tag} .= "\n" . $value;
+ } else {
+ next;
+ }
+ }
+
+ # pass the last section (if not already done).
+ $code->($section) if $open_section;
+}
+
Property changes on: rc-buggy-leaf-packages/rt-find-rc-buggy-leaf-packages
___________________________________________________________________
Added: svn:executable
+ *
Added: rc-buggy-leaf-packages/rt-get-rcbuggy-packages
===================================================================
--- rc-buggy-leaf-packages/rt-get-rcbuggy-packages (rev 0)
+++ rc-buggy-leaf-packages/rt-get-rcbuggy-packages 2011-12-15 14:02:32 UTC (rev 2104)
@@ -0,0 +1,14 @@
+#!/bin/sh
+#
+# Run on quantz.d.o (or any other machine with access to UDD).
+#
+# Produces a (crude) "src" -> rc bug mapping; duplicate src entries
+# may occur and should be merged. Luckily,
+# rt-find-rc-buggy-leaf-packages handles this for us.
+
+SQLFILE=rt-source-rcbugs.sql
+
+/usr/bin/psql -h localhost -p 5441 -U guest -f "$SQLFILE" udd --tuples-only \
+ --no-align --field-separator="||" | \
+ perl -ne '($pkgs,$bugs) = split m,\|\|,; for $pkg (split m/\s*,\s*/, $pkgs) { print "src:$pkg $bugs"; }'
+
Added: rc-buggy-leaf-packages/rt-source-rcbugs.sql
===================================================================
--- rc-buggy-leaf-packages/rt-source-rcbugs.sql (rev 0)
+++ rc-buggy-leaf-packages/rt-source-rcbugs.sql 2011-12-15 14:02:32 UTC (rev 2104)
@@ -0,0 +1,46 @@
+--
+-- Copyright (c) 2011 Alexander Reichle-Schmehl <tolimar at debian.org>
+--
+-- Permission is hereby granted, free of charge, to any person obtaining a
+-- copy of this software and associated documentation files (the
+-- "Software"),
+-- to deal in the Software without restriction, including without limitation
+-- the rights to use, copy, modify, merge, publish, distribute, sublicense,
+-- and/or sell copies of the Software, and to permit persons to whom the
+-- Software is furnished to do so, subject to the following conditions:
+--
+-- The above copyright notice and this permission notice shall be included
+-- in
+-- all copies or substantial portions of the Software.
+--
+-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+-- OR
+-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+-- THE
+-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+-- FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+-- DEALINGS IN THE SOFTWARE.
+--
+
+SELECT b.source,
+ id AS "RC Bugs" --,
+-- (SELECT array_to_string(array(
+-- SELECT DISTINCT(packages.package)
+-- FROM packages JOIN packages_summary
+-- ON packages.package=packages_summary.package
+-- WHERE packages.source=b.source
+-- AND packages_summary.release='sid'), ' ')) AS "Builds"
+ -- id AS "RC Bug",
+ -- b.package,
+ --(SELECT COUNT(DISTINCT packages.package) FROM packages WHERE packages.depends LIKE '%.' || b.package || '%') AS Dependencies
+FROM bugs b
+WHERE b.severity >= 'serious'
+ AND b.status != 'done'
+ AND b.affects_testing = true AND b.affects_unstable = true
+ AND b.last_modified < CURRENT_TIMESTAMP - INTERVAL '14 days'
+GROUP BY b.source, b.id --, b.package
+-- LIMIT 10;
+
+
More information about the Collab-qa-commits
mailing list