[libparser-mgc-perl] 05/12: Import of PEVANS/Parser-MGC-0.05 from CPAN.
Jonas Smedegaard
dr at jones.dk
Sat Dec 17 17:37:01 UTC 2016
This is an automated email from the git hooks/post-receive script.
js pushed a commit to tag PEVANS
in repository libparser-mgc-perl.
commit 059907fea15b933ebbb8659e341e82d42afc4b3a
Author: Paul Evans <leonerd at leonerd.org.uk>
Date: Fri Feb 25 16:47:44 2011 +0000
Import of PEVANS/Parser-MGC-0.05 from CPAN.
gitpan-cpan-distribution: Parser-MGC
gitpan-cpan-version: 0.05
gitpan-cpan-path: PEVANS/Parser-MGC-0.05.tar.gz
gitpan-cpan-author: PEVANS
gitpan-cpan-maturity: released
---
Changes | 4 ++
MANIFEST | 6 ++-
META.yml | 4 +-
README | 24 +++++++++++
lib/Parser/MGC.pm | 92 +++++++++++++++++++++++++++++++++++++-----
t/03reader.t | 32 +++++++++++++++
t/{03where.t => 04where.t} | 0
t/{04comment.t => 05comment.t} | 0
t/31scope_level.t | 30 ++++++++++++++
9 files changed, 179 insertions(+), 13 deletions(-)
diff --git a/Changes b/Changes
index 18985a7..5b7312c 100644
--- a/Changes
+++ b/Changes
@@ -1,5 +1,9 @@
Revision history for Parser-MGC
+0.05 CHANGES:
+ * Added ->scope_level
+ * Added ->from_reader as a new potential source of string input
+
0.04 CHANGES:
* Added ->token_float
* Optionally parse 0o... ad octal integers
diff --git a/MANIFEST b/MANIFEST
index af37bd3..d2bcc68 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -11,8 +11,9 @@ README
t/00use.t
t/01base.t
t/02expect.t
-t/03where.t
-t/04comment.t
+t/03reader.t
+t/04where.t
+t/05comment.t
t/10token_int.t
t/11token_float.t
t/12token_string.t
@@ -24,4 +25,5 @@ t/22list_of.t
t/23sequence_of.t
t/24one_of.t
t/30commit.t
+t/31scope_level.t
t/99pod.t
diff --git a/META.yml b/META.yml
index 912d436..4a88122 100644
--- a/META.yml
+++ b/META.yml
@@ -15,9 +15,9 @@ name: Parser-MGC
provides:
Parser::MGC:
file: lib/Parser/MGC.pm
- version: 0.04
+ version: 0.05
requires:
File::Slurp: 0
resources:
license: http://dev.perl.org/licenses/
-version: 0.04
+version: 0.05
diff --git a/README b/README
index 4024ed1..6b211da 100644
--- a/README
+++ b/README
@@ -93,6 +93,24 @@ METHODS
Parse the given file, which may be a pathname in a string, or an opened
IO handle, and return the result from the "parse" method.
+ $result = $parser->from_reader( \&reader )
+ Parse the input which is read by the "reader" function. This function
+ will be called in scalar context to generate portions of string to
+ parse, being passed the $parser object. The function should return
+ "undef" when it has no more string to return.
+
+ $reader->( $parser )
+
+ Note that because it is not generally possible to detect exactly when
+ more input may be required due to failed regexp parsing, the reader
+ function is only invoked during searching for skippable whitespace. This
+ makes it suitable for reading lines of a file in the common case where
+ lines are considered as skippable whitespace, or for reading lines of
+ input interractively from a user. It cannot be used in all cases (for
+ example, reading fixed-size buffers from a file) because two successive
+ invocations may split a single token across the buffer boundaries, and
+ cause parse failures parse failures.
+
( $lineno, $col, $text ) = $parser->where
Returns the current parse position, as a line and column number, and the
entire current line of text. The first line is numbered 1, and the first
@@ -106,6 +124,9 @@ METHODS
$eos = $parser->at_eos
Returns true if the input string is at the end of the string.
+ $level = $parser->scope_level
+ Returns the number of nested "scope_of" calls that have been made.
+
STRUCTURE-FORMING METHODS
The following methods may be used to build a grammatical structure out
of the defined basic token-parsing methods. Each takes at least one code
@@ -259,6 +280,9 @@ TODO
* Easy ability for subclasses to define more token types
+ * Investigate how well "from_reader" can cope with buffer splitting
+ across other tokens than simply skippable whitespace
+
AUTHOR
Paul Evans <leonerd at leonerd.org.uk>
diff --git a/lib/Parser/MGC.pm b/lib/Parser/MGC.pm
index 73d442f..235ec8d 100644
--- a/lib/Parser/MGC.pm
+++ b/lib/Parser/MGC.pm
@@ -1,14 +1,14 @@
# You may distribute under the terms of either the GNU General Public License
# or the Artistic License (the same terms as Perl itself)
#
-# (C) Paul Evans, 2010 -- leonerd at leonerd.org.uk
+# (C) Paul Evans, 2010-2011 -- leonerd at leonerd.org.uk
package Parser::MGC;
use strict;
use warnings;
-our $VERSION = '0.04';
+our $VERSION = '0.05';
use Carp;
@@ -146,6 +146,7 @@ sub new
my $self = bless {
patterns => {},
+ scope_level => 0,
}, $class;
$self->{patterns}{$_} = $args{patterns}{$_} || $self->${\"pattern_$_"} for @patterns;
@@ -201,6 +202,44 @@ sub from_file
$self->from_string( scalar(slurp $filename) );
}
+=head2 $result = $parser->from_reader( \&reader )
+
+Parse the input which is read by the C<reader> function. This function will be
+called in scalar context to generate portions of string to parse, being passed
+the C<$parser> object. The function should return C<undef> when it has no more
+string to return.
+
+ $reader->( $parser )
+
+Note that because it is not generally possible to detect exactly when more
+input may be required due to failed regexp parsing, the reader function is
+only invoked during searching for skippable whitespace. This makes it suitable
+for reading lines of a file in the common case where lines are considered as
+skippable whitespace, or for reading lines of input interractively from a
+user. It cannot be used in all cases (for example, reading fixed-size buffers
+from a file) because two successive invocations may split a single token
+across the buffer boundaries, and cause parse failures parse failures.
+
+=cut
+
+sub from_reader
+{
+ my $self = shift;
+ my ( $reader ) = @_;
+
+ local $self->{reader} = $reader;
+
+ $self->{str} = "";
+ pos $self->{str} = 0;
+
+ my $result = $self->parse;
+
+ $self->at_eos or
+ $self->fail( "Expected end of input" );
+
+ return $result;
+}
+
=head2 ( $lineno, $col, $text ) = $parser->where
Returns the current parse position, as a line and column number, and
@@ -273,6 +312,18 @@ sub at_eos
return $at_eos;
}
+=head2 $level = $parser->scope_level
+
+Returns the number of nested C<scope_of> calls that have been made.
+
+=cut
+
+sub scope_level
+{
+ my $self = shift;
+ return $self->{scope_level};
+}
+
=head1 STRUCTURE-FORMING METHODS
The following methods may be used to build a grammatical structure out of the
@@ -353,6 +404,7 @@ sub scope_of
$self->expect( $start );
local $self->{endofscope} = $stop;
+ local $self->{scope_level} = $self->{scope_level} + 1;
my $ret = $code->( $self );
@@ -519,8 +571,26 @@ sub skip_ws
my $ws = $self->{patterns}{ws};
my $c = $self->{patterns}{comment};
- 1 while $self->{str} =~ m/\G$ws/gc or
- ( $c and $self->{str} =~ m/\G$c/gc );
+ {
+ 1 while $self->{str} =~ m/\G$ws/gc or
+ ( $c and $self->{str} =~ m/\G$c/gc );
+
+ return if pos( $self->{str} ) < length $self->{str};
+
+ return unless $self->{reader};
+
+ my $more = $self->{reader}->( $self );
+ if( defined $more ) {
+ my $pos = pos( $self->{str} );
+ $self->{str} .= $more;
+ pos( $self->{str} ) = $pos;
+
+ redo;
+ }
+
+ undef $self->{reader};
+ return;
+ }
}
=head2 $parser->expect( $string )
@@ -694,11 +764,6 @@ sub STRING
# Provide fallback operators for cmp, eq, etc...
use overload fallback => 1;
-# Keep perl happy; keep Britain tidy
-1;
-
-__END__
-
=head1 TODO
=over 4
@@ -711,8 +776,17 @@ Unescaping of string constants; customisable
Easy ability for subclasses to define more token types
+=item *
+
+Investigate how well C<from_reader> can cope with buffer splitting across
+other tokens than simply skippable whitespace
+
=back
=head1 AUTHOR
Paul Evans <leonerd at leonerd.org.uk>
+
+=cut
+
+0x55AA;
diff --git a/t/03reader.t b/t/03reader.t
new file mode 100644
index 0000000..5118fc8
--- /dev/null
+++ b/t/03reader.t
@@ -0,0 +1,32 @@
+#!/usr/bin/perl -w
+
+use strict;
+
+use Test::More tests => 1;
+
+package TestParser;
+use base qw( Parser::MGC );
+
+sub parse
+{
+ my $self = shift;
+
+ my @tokens;
+ push @tokens, $self->expect( qr/[a-z]+/ ) while !$self->at_eos;
+
+ return \@tokens;
+}
+
+package main;
+
+my $parser = TestParser->new;
+
+my @strings = (
+ "here is a list ",
+ "of some more ",
+ "tokens"
+);
+
+is_deeply( $parser->from_reader( sub { return shift @strings } ),
+ [qw( here is a list of some more tokens )],
+ 'tokens from reader' );
diff --git a/t/03where.t b/t/04where.t
similarity index 100%
rename from t/03where.t
rename to t/04where.t
diff --git a/t/04comment.t b/t/05comment.t
similarity index 100%
rename from t/04comment.t
rename to t/05comment.t
diff --git a/t/31scope_level.t b/t/31scope_level.t
new file mode 100644
index 0000000..06e87ac
--- /dev/null
+++ b/t/31scope_level.t
@@ -0,0 +1,30 @@
+#!/usr/bin/perl -w
+
+use strict;
+
+use Test::More tests => 3;
+
+package TestParser;
+use base qw( Parser::MGC );
+
+sub parse
+{
+ my $self = shift;
+
+ $self->sequence_of(
+ sub {
+ $self->one_of(
+ sub { $self->expect( qr/[a-z]+/ ) . "/" . $self->scope_level },
+ sub { $self->scope_of( "(", \&parse, ")" ) },
+ );
+ },
+ );
+}
+
+package main;
+
+my $parser = TestParser->new;
+
+is_deeply( $parser->from_string( "a" ), [ "a/0" ], 'a' );
+is_deeply( $parser->from_string( "(b)" ), [ [ "b/1" ] ], '(b)' );
+is_deeply( $parser->from_string( "c (d) e" ), [ "c/0", [ "d/1" ], "e/0" ], 'c (d) e' );
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-perl/packages/libparser-mgc-perl.git
More information about the Pkg-perl-cvs-commits
mailing list