[pkg-bioc] svn commit r363 r363 - in /trunk/tools-ng: PBuilder.pm README r_pkg_ordering.pl r_pkg_prepare.sh r_pkg_upload.pl
smoe-guest at users.alioth.debian.org
smoe-guest at users.alioth.debian.org
Tue Nov 27 20:14:34 UTC 2007
Author: smoe-guest
Date: Tue Nov 27 20:14:34 2007
New Revision: 363
URL: http://svn.debian.org/wsvn/pkg-bioc/?sc=1&rev=363
Log:
Optionally using a web server to communicate with build directories ... I am afraid r_pkg_prepare.sh now needs the getopts somehow implemented.
Added:
trunk/tools-ng/r_pkg_upload.pl
Modified:
trunk/tools-ng/PBuilder.pm
trunk/tools-ng/README
trunk/tools-ng/r_pkg_ordering.pl
trunk/tools-ng/r_pkg_prepare.sh
Modified: trunk/tools-ng/PBuilder.pm
URL: http://svn.debian.org/wsvn/pkg-bioc/trunk/tools-ng/PBuilder.pm?rev=363&op=diff
==============================================================================
--- trunk/tools-ng/PBuilder.pm (original)
+++ trunk/tools-ng/PBuilder.pm Tue Nov 27 20:14:34 2007
@@ -32,9 +32,11 @@ sub PBuilder_update () {
unless ( defined($main::PBuilder_Base)
or ( $main::PBuilder_Base !~ /^\s*$/m ) );
if ( ( !-f $main::PBuilder_Base ) || ( !-s $main::PBuilder_Base ) ) {
+ print "Creating new file '$main::PBuilder_Base'.\n" if $main::verbose;
$buildercommand .= "--create ";
}
else {
+ print "UPdating file '$main::PBuilder_Base'.\n" if $main::verbose;
$buildercommand .= "--update ";
}
die "${name}_ExtraPackages is undefined, please read the documentation\n"
Modified: trunk/tools-ng/README
URL: http://svn.debian.org/wsvn/pkg-bioc/trunk/tools-ng/README?rev=363&op=diff
==============================================================================
--- trunk/tools-ng/README (original)
+++ trunk/tools-ng/README Tue Nov 27 20:14:34 2007
@@ -2,7 +2,8 @@
./r_pkg_prepare.sh --create-all --eu
./r_pkg_update.pl --dobuilderupdate
./r_pkg_update.pl --doupdate
-./r_pkg_ordering.pl
+# avoid day of the week in changelog file to appear in your local language
+LANG=C ./r_pkg_ordering.pl
# to generate the file list.d/AlreadyIncludeInDebian.list
Modified: trunk/tools-ng/r_pkg_ordering.pl
URL: http://svn.debian.org/wsvn/pkg-bioc/trunk/tools-ng/r_pkg_ordering.pl?rev=363&op=diff
==============================================================================
--- trunk/tools-ng/r_pkg_ordering.pl (original)
+++ trunk/tools-ng/r_pkg_ordering.pl Tue Nov 27 20:14:34 2007
@@ -106,6 +106,10 @@ This script will be executed under Debia
TODO option not yet implemented
+=item --pattern <regexp>
+
+ Only packages with a particular pattern are prepared.
+
=back
=head1 SEE ALSO
@@ -124,7 +128,8 @@ use vars qw($help $man $debug $verbose $
#the specific variables
use vars
- qw($rebuildlocal $rebuildmirror $rebuilddebian $rebuildall $noQAcheck $builddebian $inspectfailedbuild);
+ qw($rebuildlocal $rebuildmirror $rebuilddebian $rebuildall $noQAcheck
+ $builddebian $inspectfailedbuild $pattern);
my $origpath = cwd;
@@ -185,6 +190,7 @@ my %options = (
"rebuildlocal" => \$rebuildlocal,
"rebuildmirror" => \$rebuildmirror,
"rebuilddebian" => \$rebuilddebian,
+ "pattern:s" => \$pattern,
"noQAcheck" => \$noQAcheck,
"builddebian" => \$builddebian,
"inspectfailedbuild" => \$inspectfailedbuild,
@@ -465,7 +471,7 @@ if ($debug) {
\%repositoryOf, \%globalpackage, \%globalstuff )
if ( $RepositoryListFlat =~ /omegahat/ );
-### all the preparation should be done now....
+### all the preparation should be completed now....
# TODO dump for storage here, so I can extanalize the end of this function outside.
@@ -485,6 +491,10 @@ my $g = Graph::Directed->new;
foreach my $package (@packagenames) {
my @dbds;
+ if ($pattern) {
+ next unless $package =~ /$pattern/i;
+ }
+
$g->add_vertex($package);
if ( defined( $globalpackage{$package}{debianbuilddependencies} ) ) {
@@ -765,7 +775,8 @@ foreach my $package (@nodes) {
# renaming the extracted directory to the Debian model
if ( -d $packagehashref->{BuildDir} ) {
- print "\tnuking BuildDir for $package\n";
+ print "\tnuking BuildDir for $package at "
+ .$packagehashref->{BuildDir}."\n";
system( "rm -rf " . $packagehashref->{BuildDir} );
}
if (
@@ -799,7 +810,8 @@ foreach my $package (@nodes) {
# unlink the debian directory provided by the tar.gz, if one is provided
if ( -d $packagehashref->{BuildDir} . "/debian" ) {
- print "\tnuking debian directory for $package\n";
+ print "\tnuking debian directory for $package at "
+ . $packagehashref->{BuildDir} . "/debian" ."\n";
system( "rm -rf " . $packagehashref->{BuildDir} . "/debian" );
}
@@ -854,7 +866,9 @@ foreach my $package (@nodes) {
doQAcheck( $debfile, $qalogfile );
}
if ( -d $packagehashref->{BuildDir} ) {
- system("rm -rf $packagehashref->{BuildDir}");
+ print "Found build directory still, removing "
+ .$packagehashref->{BuildDir}."\n";
+ system("rm -rf ".$packagehashref->{BuildDir});
}
}
Modified: trunk/tools-ng/r_pkg_prepare.sh
URL: http://svn.debian.org/wsvn/pkg-bioc/trunk/tools-ng/r_pkg_prepare.sh?rev=363&op=diff
==============================================================================
--- trunk/tools-ng/r_pkg_prepare.sh (original)
+++ trunk/tools-ng/r_pkg_prepare.sh Tue Nov 27 20:14:34 2007
@@ -34,6 +34,10 @@
#installMethods="CowBuilder"
installMethods="PBuilder"
+hostname="pc171.inb.uni-luebeck.de"
+
+accessLocalFilesByHttp=1
+
### END TO EDIT, START MODE READ-ONLY!!!
# Name of the file where I store my intermediate variable.
@@ -230,13 +234,47 @@ do
done
echo
-echo "creating the web directory for storing repport"
+echo "creating the web directory for storing report"
echo
if [ ! -d "${webdir}" ]; then
mkdir -p "${webdir}"
fi
+if [ -z "$hostname" ]; then
+hostname=`hostname -l`
+fi
+
+if [ ! -r "${webdir}/index.html" ]; then
+ cat <<EOINDEX > "${webdir}/index.html"
+<html>
+<head>
+<title>Server for Debian packages for BioConductor</title>
+</head>
+<body>
+<h1>Debian Packages for BioConductor</h1>
+
+To access the files on this server, consider to add the following lines to your /etc/sources/apt.conf file:
+<br/>
+<pre>
+deb http://$hostname:8080/bioc/cran/ builds/
+deb http://$hostname:8080/bioc/bioc-2.1/ builds/
+deb http://$hostname:8080/bioc/omegahat/ builds/
+</pre>
+
+<a href="cannotbuild.html">Packages experiencing build failures</a><br />
+<a href="duplicated.html">Packages found in multiple repositories</a><br />
+
+</body>
+</html>
+EOINDEX
+fi
+
+echo "DONE"
+
+# Preparing links from the web server to the prepared packages
+( cd ${webdir} && for i in $RepositoryList; do ln -sf ../$i .; done)
+
echo
echo "creating all directories for using apt 'magic'"
echo " for the standard Debian mirror"
@@ -316,11 +354,21 @@ done
EOPBUILDERHOOKSCRIPT
chmod a+x "${Builder_BuildPlaceHook}/D70apt"
-cat <<EOPBUILDERHOOKSCRIPTTT > "${Builder_AptConfDir}/sources.list"
+if [ -n "$accessLocalFilesByHttp" ]; then
+ echo "# local web server" > "${Builder_AptConfDir}/sources.list"
+ for m in ${RepositoryList}; do
+ echo "deb http://localhost:8080/bioc/$m/ builds/" >> "${Builder_AptConfDir}/sources.list"
+ done
+else
+ cat <<EOPBUILDERHOOKSCRIPTTT >> "${Builder_AptConfDir}/sources.list"
# Locally generated files
deb file://${FILEDIR}/${bioccurrent}/ builds/
deb file://${FILEDIR}/cran/ builds/
deb file://${FILEDIR}/omegahat/ builds/
+EOPBUILDERHOOKSCRIPTTT
+fi
+
+cat <<EOPBUILDERHOOKSCRIPTTT >> "${Builder_AptConfDir}/sources.list"
# Main Debian distribution
deb http://ftp.${Lang}.debian.org/debian stable main non-free contrib
deb http://ftp.${Lang}.debian.org/debian testing main non-free contrib
@@ -337,11 +385,20 @@ fi
Builder_Post=${Builder_Post}" --aptconfdir ${Builder_AptConfDir}"
Builder_Post=${Builder_Post}" --aptcache ${Builder_AptCacheDir}"
Builder_Post=${Builder_Post}" --hookdir ${Builder_BuildPlaceHook}"
+if [ -z "$accessLocalFilesByHttp" ]; then
for i in ${RepositoryList}; do
Builder_Post=${Builder_Post}" --bindmounts ${FILEDIR}/${i}/builds/"
done
+fi
if [ ! -e ${AptSourcesList} ] || [ -f ${AptSourcesList} ] ; then
+ rm -f "${AptSourcesList}"
+ if [ -n "$accessLocalFilesByHttp" ]; then
+ echo "# Local Apache" >> ${AptSourcesList}
+ for i in ${RepositoryList}; do
+ echo "deb http://localhost:8080/bioc/$i/ builds/" >> ${AptSourcesList}
+ done
+ fi
cat <<EOAPTSCRIPT > "${AptSourcesList}"
# Main Debian distribution
deb http://ftp.${Lang}.debian.org/debian stable main non-free contrib
Added: trunk/tools-ng/r_pkg_upload.pl
URL: http://svn.debian.org/wsvn/pkg-bioc/trunk/tools-ng/r_pkg_upload.pl?rev=363&op=file
==============================================================================
--- trunk/tools-ng/r_pkg_upload.pl (added)
+++ trunk/tools-ng/r_pkg_upload.pl Tue Nov 27 20:14:34 2007
@@ -0,0 +1,368 @@
+#!/usr/bin/perl -w
+use strict;
+use warnings;
+
+use Getopt::Long;
+
+=head1 NAME
+
+r_pkg_upload.pl - select files for an upload to the alioth server
+
+=head1 SYNOPSIS
+
+./r_pkg_upload.pl --repos <full path to cran, bioc or omegahat dir>
+
+=head1 DESCRIPTION
+
+This script selects and prints those files that are supposed to be
+uploded to the bioc repository on Alioth.
+
+=head1 OPTIONS
+
+=over 4
+
+=item --verbose
+
+ Give insights about all files read in the local repository.
+
+=item --repos <path to repository, expecting a folder "build" underneath>
+
+ The location to the cran, bioc-1.9 or omegahat project directory.
+
+=item --pattern <regexp>
+
+ Case insensitive selection of packages to be uploaded.
+
+=item --status
+
+ Similar to verbose, only lists packages that are accepted or
+ rejected.
+
+=item --username <alioth account name>
+
+ Specification of the username with which to access the shared
+ repository.
+
+=back
+
+=head1 SEE ALSO
+
+http://pkg-bioc.alioth.debian.org
+
+=cut
+
+use File::Basename;
+
+my ( $repository, $verbose, $status, $username, $pattern, $packages ) =
+ ( "cran", 0, 0, undef, undef, undef );
+my ( $help, $man ) = ( 0, 0 );
+
+if (
+ !GetOptions(
+ "repos:s" => \$repository,
+ "verbose" => \$verbose,
+ "status|dryrun" => \$status,
+ "username:s" => \$username,
+ "packages" => \$packages,
+ "subset|pattern:s" => \$pattern,
+ "man" => \$man,
+ "help" => \$help
+ )
+ or $help
+ )
+{
+ require "Pod/Usage.pm";
+ import Pod::Usage;
+ pod2usage(1);
+}
+
+print "\$pattern\n" if $pattern;
+print "defined(\$pattern)\n" if $pattern;
+
+if ($man) {
+ require "Pod/Usage.pm";
+ import Pod::Usage;
+ pod2usage( -verbose => 2 );
+}
+
+if ($packages) {
+ my $cmd = "wget http://pkg-bioc.alioth.debian.org/debian/Packages";
+ print STDERR "Executing '$cmd'.\n";
+ system($cmd);
+}
+
+die "Could not locate directory '$repository' from which to upload packages.\n"
+ unless ( -d "$repository" );
+
+die
+"Could not locate directory '$repository/builds' from which to upload packages.\n"
+ unless ( -d "$repository/builds" );
+
+if ( ( !defined($username) or "" eq $username ) and !$status ) {
+ print STDERR "Username not defined. Performing dry run.\n";
+ $status = 1;
+}
+
+# now check all packages for their eligibility of an
+# upload ... howevert this will be done in future
+
+die "Cannot access lynx at /usr/bin/lynx\n" unless -x "/usr/bin/lynx";
+
+# globally define the repository basename
+my $reposbn = basename $repository;
+
+# assigns the packages to the source packages that are already uploaded
+my %packagesAlreadyUploaded;
+
+# set to true for all packages that are independent of the architecture
+my %sourcePackageIsPlatformindependent;
+my %sourcePackageVersion;
+
+my $arch = `dpkg --print-architecture|tr -d '\n'`;
+die "Could not determine host architecture.\n" unless defined($arch);
+
+my $tmpSep = $/;
+$/ = "Package: ";
+
+#$/="Package: "
+open( PACKAGES,
+ "lynx --source http://pkg-bioc.alioth.debian.org/debian/Packages |" )
+ or die "Could not access target repository\n";
+
+use AptPkg::Config '$_config';
+use AptPkg::System '$_system';
+
+#use AptPkg::Cache;
+#use AptPkg::Version;
+$_config->init;
+$_system = $_config->system;
+my $vs = $_system->versioning;
+
+# Iterate over all packages that should possibly
+# be listed as already available.
+while (<PACKAGES>) {
+ chomp;
+ my ($package) = /^\s*(\S+)/s;
+ my ($source) = /^Source:\s*(\S+)/m;
+ my ($architecture) = /^Architecture:\s*(\S+)/m;
+ my ($version) = /^Version:\s*(\S+)/m;
+ next unless ( defined $source );
+ unless ( "all" eq $architecture or $arch eq $architecture ) {
+ print STDERR
+"Ignoring package '$package' in repository: $source, $architecture != $arch\n"
+ if $verbose;
+ next;
+ }
+ print STDERR "$package, $source, $architecture\n" if $verbose;
+ if ( exists( $packagesAlreadyUploaded{$source} ) ) {
+ push( @{ $packagesAlreadyUploaded{$source} }, $package );
+ }
+ else {
+ $packagesAlreadyUploaded{$source} = [$package];
+ }
+ $sourcePackageIsPlatformindependent{$package} = "all" eq $architecture;
+ if ( exists( $sourcePackageIsPlatformindependent{$source} ) ) {
+ $sourcePackageIsPlatformindependent{$source} =
+ $sourcePackageIsPlatformindependent{$source} +
+ ( "all" ne $architecture );
+ }
+ else {
+ $sourcePackageIsPlatformindependent{$source} = "all" ne $architecture;
+ }
+
+ if ( exists( $sourcePackageVersion{$source} ) ) {
+ if ( $sourcePackageVersion{$source} eq $version ) {
+ print STDERR "$source:$package:$architecture:$version: "
+ . "Something is wrong, "
+ . "version of sources differ for the same architecture: "
+ . $sourcePackageVersion{$source} . "\n";
+ exit;
+ }
+ else {
+ print STDERR $source . " 2nd:"
+ . $sourcePackageVersion{$source} . " 1st:"
+ . $version . "\n";
+ if ( 0 > $vs->compare( $sourcePackageVersion{$source}, $version ) )
+ {
+ print STDERR
+"$source: Update $sourcePackageVersion{$source} -> $version\n";
+ $sourcePackageVersion{$source} = $version;
+ }
+ else {
+ }
+ }
+ }
+ else {
+ $sourcePackageVersion{$source} = $version;
+ }
+
+}
+close(PACKAGES);
+$/ = $tmpSep;
+
+my @files = glob("$repository/builds/[a-zA-z0-9]*");
+print STDERR "Found " . ( $#files + 1 ) . " files.\n" if $verbose;
+
+my %checkedSourcePackages;
+my %failedPackages;
+my %package2files;
+
+#print join(",", at files)."\n";
+my $packageno = 0;
+foreach my $f (@files) {
+
+ next if $f =~ /Packages\.gz$/;
+ next if $f =~ /Packages$/;
+ next if $f =~ /^r-base-dev/;
+ next if $f =~ /\.log$/;
+ next if $f =~ /\.build$/;
+ next if -d $f;
+
+ my $isdebfile = 0;
+ my ( $packagename, $versionComplete );
+ my $bn = basename($f);
+ if ( $f =~ /\.deb/ ) {
+ $isdebfile = 1;
+ ( $packagename, $versionComplete ) = $bn =~ /^r-[^-]+-([^_]+)_(.+)/;
+ }
+ else {
+ $isdebfile = 0;
+ my $red = $bn;
+ $red =~ s/\.orig\.tar\.gz//;
+ $red =~ s/\.diff\.gz//;
+ $red =~ s/\.dsc//;
+ $red =~ s/_${arch}.changes//;
+ $red =~ s/_source.changes//;
+ ( $packagename, $versionComplete ) = $red =~ /^([^_]+)_(.+)/;
+ ( $packagename, $versionComplete ) = $red =~ /^([^_]+)_(.+)/;
+ }
+ my ($version) = $versionComplete =~ /([^-]+)/;
+
+ if ( !defined($packagename) ) {
+ print STDERR "Could not parse packagename from '$bn'\n";
+ next;
+ }
+ elsif ( !defined($version) ) {
+ print STDERR
+ "Could not parse version from '$bn' (package is '$packagename'.\n";
+ next;
+ }
+
+ next
+ if defined($pattern)
+ and "" ne $pattern
+ and !( $packagename =~ m/$pattern/i );
+
+ my $accepted = 0;
+ print STDERR "Packagename not defined for file '$f'"
+ unless defined $packagename;
+ if ( !exists( $checkedSourcePackages{$packagename} ) ) {
+
+ if ( exists( $packagesAlreadyUploaded{$packagename} ) ) {
+ print STDERR "AlreadyUploaded: $packagename\n" if $verbose;
+ my $reposVersion = $sourcePackageVersion{$packagename};
+ my $compareUpstreamVersionsOnly = 1;
+ if ($compareUpstreamVersionsOnly) {
+ ($reposVersion) =
+ $sourcePackageVersion{$packagename} =~ /^([^-]+)/;
+ }
+ if ( 0 < $vs->compare( $version, $reposVersion ) ) {
+ print STDERR "$packagename: Update $reposVersion -> $version\n";
+ $checkedSourcePackages{$packagename} = 1;
+ }
+ else {
+ $failedPackages{$packagename} = 1;
+ $checkedSourcePackages{$packagename} = 0;
+ }
+ }
+ else {
+ my $bnr = basename $repository;
+ $bnr =~ s/-.*$//;
+
+ if ( grep( /\/r-$bnr-$packagename/, @files ) > 0 ) {
+ $checkedSourcePackages{$packagename} = 1;
+ $package2files{$packagename} = [];
+ }
+ else {
+ $checkedSourcePackages{$packagename} = 0;
+ $failedPackages{$packagename} = 1;
+ }
+ }
+ }
+ $accepted = $checkedSourcePackages{$packagename};
+ if ( $accepted and !$status ) {
+ print STDERR "Accepted: $bn -> $packagename - $version -> $accepted\n"
+ if $verbose;
+ push @{ $package2files{$packagename} }, $bn;
+
+ #print "$bn\n" unless $status;
+ }
+ else {
+ print STDERR "Rejected: $bn -> $packagename - $version -> $accepted\n"
+ if $verbose;
+ }
+}
+
+my @ks = sort keys %checkedSourcePackages;
+
+if ( $verbose or $status ) {
+
+ print STDERR "Number of known packages: " . ( 1 + $#ks ) . "\n";
+
+ print STDERR "Accepted packages:\n";
+ foreach my $k (@ks) {
+ print STDERR " $k" if $checkedSourcePackages{$k};
+ }
+ print STDERR "\n\n";
+ print STDERR "Rejected packages:\n";
+ foreach my $k (@ks) {
+ print STDERR " $k" unless $checkedSourcePackages{$k};
+ }
+ print STDERR "\n\n";
+}
+
+if ( defined($username) and !$status ) {
+ my $command = "cd $repository/builds && scp ";
+ my $added = 0;
+ foreach my $k (@ks) {
+ unless ( $checkedSourcePackages{$k} ) {
+ print STDERR "No positive entry in '\$checkedSourcePackages{$k}'\n"
+ if defined($pattern)
+ and "" ne $pattern
+ and $verbose;
+ next;
+ }
+
+ # double checked
+ next
+ if defined($pattern)
+ and "" ne $pattern
+ and !( $k =~ m/$pattern/i );
+ $added++;
+ die "Programming error: Accepted package '$k' "
+ . "has no files assigned.\n"
+ unless exists( $package2files{$k} );
+ $command .= " " . join( " ", @{ $package2files{$k} } );
+ }
+ print STDERR "Added: $added, Pattern: '$pattern'\n" if $verbose;
+ if ( $added > 0 ) {
+ $command .=
+ " $username\@alioth.debian.org:pkg-bioc/htdocs/debian/$reposbn/";
+ print STDERR "$command\n" if $verbose;
+ my $ret = system "$command";
+ if ($ret) {
+ print STDERR "Problems occurred during execution of scp (return code $ret): $!\n";
+ }
+ $command =
+"ssh $username\@alioth.debian.org make -f pkg-bioc/htdocs/debian/Makefile $reposbn";
+ print STDERR "Now executing $command\n";
+ $ret = system "$command";
+ if ($ret) {
+ print STDERR "Problems occurred during execution of ssp (return code $ret): $!\n";
+ }
+ }
+}
+elsif ($status) {
+ print STDERR "Not performing upload since '--status' was set.\n";
+}
+
More information about the pkg-bioc-devel
mailing list