r20890 - in /branches/upstream/libtext-csv-xs-perl/current: CSV_XS.pm CSV_XS.xs ChangeLog MANIFEST META.yml Makefile.PL README examples/csv2xls files/ files/utf8.csv t/50_utf8.t t/80_diag.t
vdanjean at users.alioth.debian.org
vdanjean at users.alioth.debian.org
Tue Jun 10 09:16:03 UTC 2008
Author: vdanjean
Date: Tue Jun 10 09:16:02 2008
New Revision: 20890
URL: http://svn.debian.org/wsvn/?sc=1&rev=20890
Log:
[svn-upgrade] Integrating new upstream version, libtext-csv-xs-perl (0.50)
Added:
branches/upstream/libtext-csv-xs-perl/current/files/
branches/upstream/libtext-csv-xs-perl/current/files/utf8.csv
Modified:
branches/upstream/libtext-csv-xs-perl/current/CSV_XS.pm
branches/upstream/libtext-csv-xs-perl/current/CSV_XS.xs
branches/upstream/libtext-csv-xs-perl/current/ChangeLog
branches/upstream/libtext-csv-xs-perl/current/MANIFEST
branches/upstream/libtext-csv-xs-perl/current/META.yml
branches/upstream/libtext-csv-xs-perl/current/Makefile.PL
branches/upstream/libtext-csv-xs-perl/current/README
branches/upstream/libtext-csv-xs-perl/current/examples/csv2xls
branches/upstream/libtext-csv-xs-perl/current/t/50_utf8.t
branches/upstream/libtext-csv-xs-perl/current/t/80_diag.t
Modified: branches/upstream/libtext-csv-xs-perl/current/CSV_XS.pm
URL: http://svn.debian.org/wsvn/branches/upstream/libtext-csv-xs-perl/current/CSV_XS.pm?rev=20890&op=diff
==============================================================================
--- branches/upstream/libtext-csv-xs-perl/current/CSV_XS.pm (original)
+++ branches/upstream/libtext-csv-xs-perl/current/CSV_XS.pm Tue Jun 10 09:16:02 2008
@@ -30,7 +30,7 @@
use Carp;
use vars qw( $VERSION @ISA );
-$VERSION = "0.45";
+$VERSION = "0.50";
@ISA = qw( DynaLoader );
sub PV { 0 }
@@ -547,6 +547,15 @@
while (my $row = $csv->getline ($io)) {
my @fields = @$row;
+=head2 Unicode (UTF8)
+
+On parsing (both for C<getline ()> and C<parse ()>), if the source is
+marked being UTF8, then parsing that source will mark all fields that
+are marked binary will also be marked UTF8.
+
+On combining (C<print ()> and C<combine ()>), if any of the combining
+fields was marked UTF8, the resulting string will be marked UTF8.
+
=head1 SPECIFICATION
While no formal specification for CSV exists, RFC 4180 1) describes a common
@@ -1165,7 +1174,7 @@
my $err = $csv->error_input;
print "combine () failed on argument: ", $err, "\n";
}
- close $csv_fh;
+ close $csv_fh or die "hello.csv: $!";
An example for parsing CSV strings:
@@ -1194,9 +1203,9 @@
$sth->execute;
$csv->print ($fh, $sth->{NAME_lc});
while (my $row = $sth->fetch) {
- $csv->print ($fh, $row);
+ $csv->print ($fh, $row) or ...;
}
- close $fh;
+ close $fh or die "$tbl.csv: $!";
Reading a CSV file line by line:
@@ -1205,7 +1214,7 @@
while (my $row = $csv->getline ($fh)) {
# do something with @$row
}
- close $fh;
+ close $fh or die "file.csv: $!";;
For more extended examples, see the C<examples/> subdirectory in the
original distribution. Included is C<examples/parser-xs.pl>, that could
@@ -1262,11 +1271,10 @@
=item Unicode
-Make C<parse ()> and C<combine ()> do the right thing for Unicode
-(UTF-8) if requested. See t/50_utf8.t. More complicated, but evenly
-important, also for C<getline ()> and C<print ()>.
-
-Probably the best way to do this is to make a subclass
+We probably need many more tests to check if all edge-cases are covered.
+See t/50_utf8.t.
+
+Probably the best way to do this more reliable is to make a subclass
Text::CSV_XS::Encoded that can be passed the required encoding and
then behaves transparently (but slower), something like this:
Modified: branches/upstream/libtext-csv-xs-perl/current/CSV_XS.xs
URL: http://svn.debian.org/wsvn/branches/upstream/libtext-csv-xs-perl/current/CSV_XS.xs?rev=20890&op=diff
==============================================================================
--- branches/upstream/libtext-csv-xs-perl/current/CSV_XS.xs (original)
+++ branches/upstream/libtext-csv-xs-perl/current/CSV_XS.xs Tue Jun 10 09:16:02 2008
@@ -11,6 +11,10 @@
#define NEED_newRV_noinc
#define NEED_vload_module
#include "ppport.h"
+#if (PERL_BCDVERSION <= 0x5005005)
+# define SvUTF8_on(sv) /* no-op */
+# define SvUTF8(sv) 0
+# endif
#define MAINT_DEBUG 0
#define ALLOW_ALLOW 1
@@ -68,43 +72,44 @@
#define byte unsigned char
typedef struct {
- byte quote_char;
- byte escape_char;
- byte sep_char;
- byte binary;
-
- byte keep_meta_info;
- byte always_quote;
- byte useIO; /* Also used to indicate EOF */
- byte eol_is_cr;
-
-#if ALLOW_ALLOW
- byte allow_loose_quotes;
- byte allow_loose_escapes;
- byte allow_double_quoted;
- byte allow_whitespace;
-
- byte blank_is_undef;
- byte verbatim;
- byte is_bound;
- byte reserved1;
-#endif
-
- byte cache[CACHE_SIZE];
-
- HV* self;
- SV* bound;
-
- char *eol;
- STRLEN eol_len;
- char *types;
- STRLEN types_len;
-
- char *bptr;
- SV *tmp;
- STRLEN size;
- STRLEN used;
- char buffer[BUFFER_SIZE];
+ byte quote_char;
+ byte escape_char;
+ byte sep_char;
+ byte binary;
+
+ byte keep_meta_info;
+ byte always_quote;
+ byte useIO; /* Also used to indicate EOF */
+ byte eol_is_cr;
+
+#if ALLOW_ALLOW
+ byte allow_loose_quotes;
+ byte allow_loose_escapes;
+ byte allow_double_quoted;
+ byte allow_whitespace;
+
+ byte blank_is_undef;
+ byte verbatim;
+ byte is_bound;
+ byte reserved1;
+#endif
+
+ byte cache[CACHE_SIZE];
+
+ HV * self;
+ SV * bound;
+
+ char * eol;
+ STRLEN eol_len;
+ char * types;
+ STRLEN types_len;
+
+ char * bptr;
+ SV * tmp;
+ int utf8;
+ STRLEN size;
+ STRLEN used;
+ char buffer[BUFFER_SIZE];
} csv_t;
#define bool_opt(o) \
@@ -327,6 +332,7 @@
else
csv->is_bound = 0;
}
+ csv->utf8 = 0;
csv->used = 0;
} /* SetupCsv */
@@ -354,11 +360,13 @@
sv_catpvn (SvRV (dst), csv->buffer, csv->used);
result = TRUE;
}
+ if (csv->utf8 && SvROK (dst))
+ SvUTF8_on (SvRV (dst));
csv->used = 0;
return result;
} /* Print */
-#define CSV_PUT(csv,dst,c) { \
+#define CSV_PUT(csv,dst,c) { \
if ((csv)->used == sizeof ((csv)->buffer) - 1) \
Print ((csv), (dst)); \
(csv)->buffer[(csv)->used++] = (c); \
@@ -387,9 +395,10 @@
int quoteMe = csv->always_quote;
unless ((SvOK (*svp) || (
- (SvMAGICAL (*svp) && (mg_get (*svp), 1) && SvOK (*svp)))
+ (SvGMAGICAL (*svp) && (mg_get (*svp), 1) && SvOK (*svp)))
)) continue;
ptr = SvPV (*svp, len);
+ if (len && SvUTF8 (*svp)) csv->utf8 = 1;
/* Do we need quoting? We do quote, if the user requested
* (always_quote), if binary or blank characters are found
* and if the string contains quote or escape characters.
@@ -507,6 +516,7 @@
}
}
#endif
+ if (SvUTF8 (csv->tmp)) csv->utf8 = 1;
if (csv->size)
return ((byte)csv->bptr[csv->used++]);
}
@@ -525,7 +535,7 @@
return FALSE; \
}
-#define CSV_PUT_SV(sv,c) { \
+#define CSV_PUT_SV(c) { \
len = SvCUR ((sv)); \
SvGROW ((sv), len + 2); \
*SvEND ((sv)) = c; \
@@ -549,6 +559,8 @@
else { \
if (csv->allow_whitespace && ! (f & CSV_FLAGS_QUO)) \
strip_trail_whitespace (sv); \
+ if (f & CSV_FLAGS_BIN && csv->utf8) \
+ SvUTF8_on (sv); \
unless (csv->is_bound) av_push (fields, sv); \
} \
sv = NULL; \
@@ -666,7 +678,7 @@
}
else
if (f & CSV_FLAGS_QUO)
- CSV_PUT_SV (sv, c)
+ CSV_PUT_SV (c)
else {
AV_PUSH;
}
@@ -698,7 +710,7 @@
unless (csv->binary)
ERROR_INSIDE_QUOTES (2021);
- CSV_PUT_SV (sv, c);
+ CSV_PUT_SV (c);
}
#if ALLOW_ALLOW
else
@@ -707,7 +719,7 @@
unless (csv->binary)
/* uncovered */ ERROR_INSIDE_FIELD (2030);
- CSV_PUT_SV (sv, c);
+ CSV_PUT_SV (c);
}
#endif
else {
@@ -756,7 +768,7 @@
unless (csv->binary)
ERROR_INSIDE_QUOTES (2022);
- CSV_PUT_SV (sv, c);
+ CSV_PUT_SV (c);
}
else {
int c2;
@@ -836,7 +848,7 @@
}
if (csv->allow_loose_quotes) {
- CSV_PUT_SV (sv, c);
+ CSV_PUT_SV (c);
c = c2;
goto restart;
}
@@ -865,10 +877,10 @@
}
else
if (c2 == '0')
- CSV_PUT_SV (sv, 0)
+ CSV_PUT_SV (0)
else
if (c2 == csv->quote_char || c2 == csv->sep_char)
- CSV_PUT_SV (sv, c2)
+ CSV_PUT_SV (c2)
else
if (c2 == CH_NL) {
AV_PUSH;
@@ -893,7 +905,7 @@
}
if (csv->allow_loose_quotes && csv->escape_char != csv->quote_char) {
- CSV_PUT_SV (sv, c);
+ CSV_PUT_SV (c);
c = c2;
goto restart;
}
@@ -918,7 +930,7 @@
#if ALLOW_ALLOW
if (csv->allow_loose_quotes) { /* 1,foo "boo" d'uh,1 */
f |= CSV_FLAGS_EIF;
- CSV_PUT_SV (sv, c);
+ CSV_PUT_SV (c);
}
else
#endif
@@ -943,7 +955,7 @@
}
if (c2 == '0')
- CSV_PUT_SV (sv, 0)
+ CSV_PUT_SV (0)
else
if ( c2 == csv->quote_char || c2 == csv->sep_char ||
c2 == csv->escape_char
@@ -951,7 +963,7 @@
|| csv->allow_loose_escapes
#endif
)
- CSV_PUT_SV (sv, c2)
+ CSV_PUT_SV (c2)
else {
csv->used--;
ERROR_INSIDE_QUOTES (2025);
@@ -966,7 +978,7 @@
ERROR_INSIDE_FIELD (2035);
}
- CSV_PUT_SV (sv, c2);
+ CSV_PUT_SV (c2);
}
else
/* uncovered */ ERROR_INSIDE_FIELD (2036); /* I think there's no way to get here */
@@ -995,8 +1007,7 @@
unless (csv->binary)
ERROR_INSIDE_QUOTES (2026);
}
-
- CSV_PUT_SV (sv, c);
+ CSV_PUT_SV (c);
}
else {
if (is_csv_binary (c)) {
@@ -1004,8 +1015,7 @@
unless (csv->binary)
ERROR_INSIDE_FIELD (2037);
}
-
- CSV_PUT_SV (sv, c);
+ CSV_PUT_SV (c);
}
}
@@ -1060,6 +1070,7 @@
}
else {
csv.tmp = src;
+ csv.utf8 = SvUTF8 (src);
csv.bptr = SvPV (src, csv.size);
}
hv_delete (hv, "_ERROR_INPUT", 12, G_DISCARD);
Modified: branches/upstream/libtext-csv-xs-perl/current/ChangeLog
URL: http://svn.debian.org/wsvn/branches/upstream/libtext-csv-xs-perl/current/ChangeLog?rev=20890&op=diff
==============================================================================
--- branches/upstream/libtext-csv-xs-perl/current/ChangeLog (original)
+++ branches/upstream/libtext-csv-xs-perl/current/ChangeLog Tue Jun 10 09:16:02 2008
@@ -1,3 +1,21 @@
+2008-06-04 0.50 - H.Merijn Brand <h.m.brand at xs4all.nl>
+
+ * Skip a few tests in automated testing, as they confuse
+ reports. This is important for the automated sites that
+ mark modules as fail if it is not an obvious full PASS
+ * 0.46 caused the last still open RT bug to be closed!
+ * Tested on 5.11.0, 5.10.0, 5.8.8, 5.6.2, and 5.005_04,
+ Strawberry and Cygwin
+
+2008-06-04 0.46 - H.Merijn Brand <h.m.brand at xs4all.nl>
+
+ * In examples add die on failed close calls
+ * Use Test::MinimumVersion (not distributed)
+ * Added option -F to examples/csv2xls
+ * More source code cleanup
+ * Nailed the UTF-8 issues for parsing
+ * Nailed the UTF-8 issues for combining
+
2008-04-23 0.45 - H.Merijn Brand <h.m.brand at xs4all.nl>
* Forgot to pack examples/parser-xs.pl
Modified: branches/upstream/libtext-csv-xs-perl/current/MANIFEST
URL: http://svn.debian.org/wsvn/branches/upstream/libtext-csv-xs-perl/current/MANIFEST?rev=20890&op=diff
==============================================================================
--- branches/upstream/libtext-csv-xs-perl/current/MANIFEST (original)
+++ branches/upstream/libtext-csv-xs-perl/current/MANIFEST Tue Jun 10 09:16:02 2008
@@ -30,4 +30,5 @@
examples/csv-check Script to check a CSV file/stream
examples/parser-xs.pl Parse CSV stream, be forgiving on bad lines
examples/speed.pl Small benchmark script
+files/utf8.csv A UTF-8 encode test file
META.yml Module meta-data (added by MakeMaker)
Modified: branches/upstream/libtext-csv-xs-perl/current/META.yml
URL: http://svn.debian.org/wsvn/branches/upstream/libtext-csv-xs-perl/current/META.yml?rev=20890&op=diff
==============================================================================
--- branches/upstream/libtext-csv-xs-perl/current/META.yml (original)
+++ branches/upstream/libtext-csv-xs-perl/current/META.yml Tue Jun 10 09:16:02 2008
@@ -1,6 +1,6 @@
--- #YAML:1.0
name: Text-CSV_XS
-version: 0.45
+version: 0.50
abstract: Comma-Separated Values manipulation routines
license: perl
author:
@@ -10,7 +10,7 @@
provides:
Text::CSV_XS:
file: CSV_XS.pm
- version: 0.45
+ version: 0.50
requires:
perl: 5.005
DynaLoader: 0
Modified: branches/upstream/libtext-csv-xs-perl/current/Makefile.PL
URL: http://svn.debian.org/wsvn/branches/upstream/libtext-csv-xs-perl/current/Makefile.PL?rev=20890&op=diff
==============================================================================
--- branches/upstream/libtext-csv-xs-perl/current/Makefile.PL (original)
+++ branches/upstream/libtext-csv-xs-perl/current/Makefile.PL Tue Jun 10 09:16:02 2008
@@ -64,6 +64,13 @@
"test_harness($(TEST_VERBOSE), '$(INST_LIB)', '$(INST_ARCHLIB)')"
$(TEST_FILES) 3>valgrind.log
);
+
+ my $min_vsn = ($] >= 5.010 && -d "xt" && ($ENV{AUTOMATED_TESTING} || 0) != 1)
+ ? join "\n" =>
+ 'test ::',
+ ' - at env TEST_FILES="xt/*.t" make -e test_dynamic',
+ ''
+ : "";
join "\n" =>
'cover test_cover:',
' cover -delete',
@@ -81,7 +88,7 @@
q{ sandbox/leaktest $(FULLPERLRUN) "test_harness($(TEST_VERBOSE), '$(INST_LIB)', '$(INST_ARCHLIB)')" $(TEST_FILES)},
'',
'fixmeta: distmeta',
- ' perl genMETA.pl',
+ ' perl sandbox/genMETA.pl',
'',
'tgzdist: fixmeta $(DISTVNAME).tar.gz distcheck',
' - at mv -f $(DISTVNAME).tar.gz $(DISTVNAME).tgz',
@@ -89,5 +96,6 @@
'',
'test_speed: pure_all',
' PERL_DL_NONLAZY=1 $(FULLPERLRUN) -I"$(INST_LIB)" -I"$(INST_ARCHLIB)" examples/speed.pl',
- '';
+ '',
+ $min_vsn;
} # postamble
Modified: branches/upstream/libtext-csv-xs-perl/current/README
URL: http://svn.debian.org/wsvn/branches/upstream/libtext-csv-xs-perl/current/README?rev=20890&op=diff
==============================================================================
--- branches/upstream/libtext-csv-xs-perl/current/README (original)
+++ branches/upstream/libtext-csv-xs-perl/current/README Tue Jun 10 09:16:02 2008
@@ -21,7 +21,7 @@
perl 5.005. 5.005 will not be able to build the manual pages.
Build/Installation:
- Standard build/installation supported by ExtUtils::MakeMaker(3)...
+ Standard build/installation:
perl Makefile.PL
make
make test
@@ -37,6 +37,6 @@
A pure-perl version is being maintained by Makamaka Hannyaharamitu
as Text::CSV_PP, which tries to follow Text::CSV_XS very closely.
- Expect Text::CSV to be a wrapper module over Text::CSV_XS and
- Text::CSV_PP that will select Text::CSV_XS if available, and
- otherwise use Text::PP.
+ Text::CSV_PP is bundled in Text::CSV, which now is a wrapper module over
+ Text::CSV_XS and Text::CSV_PP that will select Text::CSV_XS if available,
+ and otherwise use Text::PP.
Modified: branches/upstream/libtext-csv-xs-perl/current/examples/csv2xls
URL: http://svn.debian.org/wsvn/branches/upstream/libtext-csv-xs-perl/current/examples/csv2xls?rev=20890&op=diff
==============================================================================
--- branches/upstream/libtext-csv-xs-perl/current/examples/csv2xls (original)
+++ branches/upstream/libtext-csv-xs-perl/current/examples/csv2xls Tue Jun 10 09:16:02 2008
@@ -1,12 +1,12 @@
-#!/usr/bin/perl
+#!/pro/bin/perl
# csv2xls: Convert csv to xls
-# (m)'07 [21 Sep 2007] Copyright H.M.Brand 2007-2008
+# (m)'08 [22 May 2008] Copyright H.M.Brand 2007-2008
use strict;
use warnings;
-our $VERSION = "1.5"; # 2008-04-10
+our $VERSION = "1.6";
sub usage
{
@@ -18,8 +18,10 @@
-q <quot> use <quot> as quotation char. Default = '"'
-w <width> use <width> as default minimum column width (4)
-o <xls> write output to file named <xls>, defaults
- to input file name with .csv replaced with .xls
- if from standard input, defaults to csv2xls.xls
+ to input file name with .csv replaced with .xls
+ if from standard input, defaults to csv2xls.xls
+ -F allow formula's. Otherwise fields starting with
+ an equal sign are forced to string
-f force usage of <xls> if already exists (unlink before use)
-d <dtfmt> use <dtfmt> as date formats. Default = 'dd-mm-yyyy'
-u CSV is UTF8
@@ -34,6 +36,7 @@
my $xls; # Excel out file name
my $frc = 0; # Force use of file
my $utf = 0; # Data is encoded in Unicode
+my $frm = 0; # Allow formula's
my $dtf = "dd-mm-yyyy"; # Date format
GetOptions (
@@ -45,6 +48,7 @@
"o|x=s" => \$xls,
"d=s" => \$dtf,
"f" => \$frc,
+ "F" => \$frm,
"u" => \$utf,
) or usage (1);
@@ -151,7 +155,12 @@
next;
}
- $wks->write ($h, $c, $val);
+ if (!$frm && $val =~ m/^=/) {
+ $wks->write_string ($h, $c, $val);
+ }
+ else {
+ $wks->write ($h, $c, $val);
+ }
}
++$h % 100 or printf STDERR "%6d x %6d\r", $w, $h;
} continue { $row = undef }
Added: branches/upstream/libtext-csv-xs-perl/current/files/utf8.csv
URL: http://svn.debian.org/wsvn/branches/upstream/libtext-csv-xs-perl/current/files/utf8.csv?rev=20890&op=file
==============================================================================
--- branches/upstream/libtext-csv-xs-perl/current/files/utf8.csv (added)
+++ branches/upstream/libtext-csv-xs-perl/current/files/utf8.csv Tue Jun 10 09:16:02 2008
@@ -1,0 +1,1 @@
+"Øl/Vin",0
Modified: branches/upstream/libtext-csv-xs-perl/current/t/50_utf8.t
URL: http://svn.debian.org/wsvn/branches/upstream/libtext-csv-xs-perl/current/t/50_utf8.t?rev=20890&op=diff
==============================================================================
--- branches/upstream/libtext-csv-xs-perl/current/t/50_utf8.t (original)
+++ branches/upstream/libtext-csv-xs-perl/current/t/50_utf8.t Tue Jun 10 09:16:02 2008
@@ -10,7 +10,7 @@
plan skip_all => "UTF8 tests useless in this ancient perl version";
}
else {
- plan tests => 57;
+ plan tests => 64;
}
}
@@ -20,7 +20,11 @@
require "t/util.pl";
}
-my $csv = Text::CSV_XS->new ({ binary => 1, always_quote => 1 });
+my $csv = Text::CSV_XS->new ({
+ binary => 1,
+ always_quote => 1,
+ keep_meta_info => 1,
+ });
# Special characters to check:
# 0A = \n 2C = , 20 = 22 = "
@@ -57,3 +61,23 @@
is_binary ($in[$_], $out[$_], "field $_ $msg");
}
}
+
+# As all utf tests are skipped for older pers, It's safe to use 3-arg open this way
+my $file = "files/utf8.csv";
+SKIP: {
+ open my $fh, "<:encoding(utf8)", $file or
+ skip "Cannot open UTF-8 test file", 6;
+
+ my $row;
+ ok ($row = $csv->getline ($fh), "read/parse");
+
+ is ($csv->is_quoted (0), 1, "First field is quoted");
+ is ($csv->is_quoted (1), 0, "Second field is not quoted");
+ is ($csv->is_binary (0), 1, "First field is binary");
+ is ($csv->is_binary (1), 0, "Second field is not binary");
+
+ ok (utf8::valid ($row->[0]), "First field is valid utf8");
+
+ $csv->combine (@$row);
+ ok (utf8::valid ($csv->string), "Combined string is valid utf8");
+ }
Modified: branches/upstream/libtext-csv-xs-perl/current/t/80_diag.t
URL: http://svn.debian.org/wsvn/branches/upstream/libtext-csv-xs-perl/current/t/80_diag.t?rev=20890&op=diff
==============================================================================
--- branches/upstream/libtext-csv-xs-perl/current/t/80_diag.t (original)
+++ branches/upstream/libtext-csv-xs-perl/current/t/80_diag.t Tue Jun 10 09:16:02 2008
@@ -57,8 +57,10 @@
parse_err 2034, 4, qq{1, "bar",2};
parse_err 2037, 1, qq{\0 };
-diag ("Next line should be an error message");
-$csv->error_diag ();
+unless (($ENV{AUTOMATED_TESTING} || 0) == "1") {
+ diag ("Next line should be an error message");
+ $csv->error_diag ();
+ }
is (Text::CSV_XS->new ({ ecs_char => ":" }), undef, "Unsupported option");
More information about the Pkg-perl-cvs-commits
mailing list