[libdatetime-timezone-perl] 03/09: Run parse_olson in parallel
gregor herrmann
gregoa at debian.org
Wed Apr 15 19:03:41 UTC 2015
This is an automated email from the git hooks/post-receive script.
gregoa pushed a commit to annotated tag v1.83
in repository libdatetime-timezone-perl.
commit 4825de873d5c8f1ad0ffeaa4c43eb901d4532180
Author: Dave Rolsky <autarch at urth.org>
Date: Wed Dec 24 13:52:01 2014 -0600
Run parse_olson in parallel
---
tools/parse_olson | 88 ++++++++++++++++++++++++++++++++++++++-----------------
1 file changed, 61 insertions(+), 27 deletions(-)
diff --git a/tools/parse_olson b/tools/parse_olson
index af1ff0d..6916951 100755
--- a/tools/parse_olson
+++ b/tools/parse_olson
@@ -18,6 +18,7 @@ use File::Spec;
use Getopt::Long;
use List::AllUtils qw( max );
use Locale::Country 3.11 qw( code2country );
+use Parallel::ForkManager;
$Data::Dumper::Indent = 1;
$Data::Dumper::Sortkeys = 1;
@@ -32,6 +33,7 @@ GetOptions(
'dir:s' => \$opts{dir},
'clean' => \$opts{clean},
'version:s' => \$opts{version},
+ 'jobs:i' => \$opts{jobs},
'old' => \$opts{old},
'file:s' => \$opts{file},
'name:s' => \$opts{name},
@@ -47,15 +49,17 @@ $opts{help} = 1
$opts{version} ||= 'test';
+$opts{jobs} ||= 4;
+
if ( $opts{help} ) {
print <<'EOF';
This script parses the Olson time zone database files and turns them into a
-set of Perl modules. It also generates the DateTime::TimeZone::Catalog
+set of Perl modules. It also generates the DateTime::TimeZone::Catalog
module, which contains a list of all the available time zone names.
By default, it looks for files named africa, antarctica, asia, australasia,
-europe, northamerica, pacificnew, southamerica, and backward. All other files
+europe, northamerica, pacificnew, southamerica, and backward. All other files
are ignored.
It takes the following arguments:
@@ -68,9 +72,11 @@ It takes the following arguments:
--clean Remove old generated modules (which may not be valid with
the latest Olson database)
- --file Parse just the file with the given name. For debugging.
+ --jobs Number of parallel jobs to run. Defaults to 4.
+
+ --file Parse just the file with the given name. For debugging.
- --name Only create the specified time zone. For debugging.
+ --name Only create the specified time zone. For debugging.
--old Also look for files named etcetera and systemv
@@ -103,15 +109,36 @@ else {
if $opts{old};
}
-my ( @zones, %categories, %links );
-
my $autogen_warning = <<"EOF";
# This file is auto-generated by the Perl DateTime Suite time zone
# code generator ($VERSION) This code generator comes with the
# DateTime::TimeZone module distribution in the tools/ directory
EOF
-parse_file($_) for sort @files;
+my $pm = Parallel::ForkManager->new( $opts{jobs} );
+
+my ( @zones, %categories, %links );
+$pm->run_on_finish(
+ sub {
+ my ( $exit_code, $data ) = @_[ 1, 5 ];
+ die "Bad exit: $exit_code" if $exit_code;
+
+ push @zones, @{ $data->{zones} };
+ $links{$_} = $data->{links}{$_} for keys %{ $data->{links} };
+ for my $category ( keys %{ $data->{categories} } ) {
+ push @{ $categories{$category} },
+ @{ $data->{categories}{$category} };
+ }
+ }
+);
+
+for my $file ( sort @files ) {
+ $pm->start() and next;
+
+ $pm->finish( 0, parse_file($file) );
+}
+
+$pm->wait_all_children();
exit if $opts{name};
@@ -143,8 +170,8 @@ sub parse_file {
$odb->parse_file($file);
- %links = ( %links, $odb->links );
-
+ my @zones;
+ my %categories;
foreach my $zone_name ( sort $odb->zone_names ) {
if ( $opts{name} ) {
next unless $zone_name eq $opts{name};
@@ -166,15 +193,15 @@ sub parse_file {
$name = $zone_name;
}
- my $outfile1 = $name;
- $outfile1 =~ s/-(\d)/_Minus$1/;
- $outfile1 =~ s/\+/_Plus/;
- $outfile1 =~ s/-/_/g;
+ my $outfile1 = $name;
+ $outfile1 =~ s/-(\d)/_Minus$1/;
+ $outfile1 =~ s/\+/_Plus/;
+ $outfile1 =~ s/-/_/g;
- ( my $mod_name = $zone_name ) =~ s/\//::/g;
- $mod_name =~ s/-(\d)/_Minus$1/;
- $mod_name =~ s/\+/_Plus/;
- $mod_name =~ s/-/_/g;
+ ( my $mod_name = $zone_name ) =~ s/\//::/g;
+ $mod_name =~ s/-(\d)/_Minus$1/;
+ $mod_name =~ s/\+/_Plus/;
+ $mod_name =~ s/-/_/g;
my $zone = $odb->zone($zone_name);
my $max_year = max(
@@ -203,7 +230,7 @@ sub parse_file {
if defined $opts{version};
# Helps avoid mis indexing on (meta)cpan
- my $pkg = 'package DateTime::TimeZone::' .$mod_name;
+ my $pkg = 'package DateTime::TimeZone::' . $mod_name;
my $body = <<"EOF";
$autogen_warning
@@ -225,14 +252,13 @@ use DateTime::TimeZone::OlsonDB;
my \$spans =
$spans;
-sub olson_version { '$opts{version}' }
+sub olson_version {'$opts{version}'}
-sub has_dst_changes { $has_dst_changes }
+sub has_dst_changes {$has_dst_changes}
-sub _max_year { $max_year }
+sub _max_year {$max_year}
-sub _new_instance
-{
+sub _new_instance {
return shift->_init( \@_, spans => \$spans );
}
@@ -258,6 +284,12 @@ EOF
print $fh $body or die "Cannot write to $outfile2: $!";
close $fh or die "Cannot write to $outfile2: $!";
}
+
+ return {
+ links => { $odb->links() },
+ categories => \%categories,
+ zones => \@zones,
+ };
}
sub zone_as_spans {
@@ -336,7 +368,8 @@ sub serialize_span {
# must correspond to constants in DT::TZ, and short_name is always last
my $string = " [\n";
for my $key (qw( utc_start utc_end local_start local_end )) {
- my $comment = sprintf( '# %12s %s', $key, _num_as_datetime( $span->{$key} ) );
+ my $comment
+ = sprintf( '# %12s %s', $key, _num_as_datetime( $span->{$key} ) );
$comment =~ s/\s+$//;
$string .= "$span->{$key}, $comment\n";
}
@@ -353,6 +386,7 @@ sub serialize_span {
{
{
+
package FakeCal;
sub utc_rd_values {
@@ -372,7 +406,7 @@ sub serialize_span {
my $dt = DateTime->from_object( object => $obj );
- return $dt->strftime( '%Y-%m-%d %H:%M:%S (%a)' );
+ return $dt->strftime('%Y-%m-%d %H:%M:%S (%a)');
}
}
@@ -457,7 +491,7 @@ sub make_catalog_pm {
$links =~ s/{/(/;
$links =~ s/}/)/;
- my $zones = join "\n", map {" $_"} sort @zones, 'UTC';;
+ my $zones = join "\n", map {" $_"} sort @zones, 'UTC';
my $cat_names = join "\n", map {" $_"} sort keys %categories;
my $cat = '';
foreach my $c ( sort keys %categories ) {
@@ -571,7 +605,7 @@ EOF
$zonecatalog .= "=head3 $country->[0] ($country->[1])\n\n";
for my $zone ( @{ $countries{ $country->[1] } } ) {
- my $line = join ' - ', grep { defined } @{$zone};
+ my $line = join ' - ', grep {defined} @{$zone};
$zonecatalog .= " $line\n";
}
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-perl/packages/libdatetime-timezone-perl.git
More information about the Pkg-perl-cvs-commits
mailing list