[python-hdf5storage] 51/84: Writing non-ASCII numpy.bytes_ when using MATLAB formatting now throws an exception.

[Ghislain Vaillant]

This is an automated email from the git hooks/post-receive script.

ghisvail-guest pushed a commit to annotated tag 0.1.10
in repository python-hdf5storage.

commit 24e17a8b40eff6a50e55afe413310f94539145e0
Author: Freja Nordsiek <fnordsie at gmail.com>
Date:   Sun May 17 19:18:20 2015 -0400

    Writing non-ASCII numpy.bytes_ when using MATLAB formatting now throws an exception.
---
 hdf5storage/Marshallers.py   | 13 +++++++++----
 tests/test_write_readback.py | 18 ++++++++++++++++++
 2 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/hdf5storage/Marshallers.py b/hdf5storage/Marshallers.py
index ca081ad..56f0e61 100644
--- a/hdf5storage/Marshallers.py
+++ b/hdf5storage/Marshallers.py
@@ -578,10 +578,12 @@ class NumpyScalarArrayMarshaller(TypeMarshaller):
         if isinstance(data_to_store, np.core.records.recarray):
             data_to_store = data_to_store.view(np.ndarray)
 
-        # Optionally convert ASCII strings to UTF-16. This is done by
-        # simply converting to uint16's. This will require making them
-        # at least 1 dimensinal.
-
+        # Optionally convert bytes_ strings to UTF-16, if possible (all
+        # are in the ASCII character set). This is done by simply
+        # converting to uint16's and checking that each one's value is
+        # less than 128 (in the ASCII character set). This will require
+        # making them at least 1 dimensional. If it fails, throw an
+        # exception.
         if data.dtype.type == np.bytes_ \
                 and options.convert_numpy_bytes_to_utf16:
             if data_to_store.nbytes == 0:
@@ -589,6 +591,9 @@ class NumpyScalarArrayMarshaller(TypeMarshaller):
             else:
                 data_to_store = np.uint16(np.atleast_1d( \
                     data_to_store).view(np.uint8))
+                if np.any(data_to_store >= 128):
+                    raise NotImplementedError( \
+                        'Can''t write non-ASCII numpy.bytes_.')
 
         # As of 2013-12-13, h5py cannot write numpy.str_ (UTF-32
         # encoding) types (its numpy.unicode_ in Python 2, which is an
diff --git a/tests/test_write_readback.py b/tests/test_write_readback.py
index 2751553..dd86440 100644
--- a/tests/test_write_readback.py
+++ b/tests/test_write_readback.py
@@ -444,6 +444,7 @@ class TestPythonMatlabFormat(object):
                                   self.options)
         self.assert_equal(out, data)
 
+    @raises(NotImplementedError)
     def test_str_ascii_encoded_utf8(self):
         data = self.random_str_some_unicode(random.randint(1,
             self.max_string_length)).encode('UTF-8')
@@ -556,6 +557,15 @@ class TestPythonFormat(TestPythonMatlabFormat):
         self.options = hdf5storage.Options(matlab_compatible=False)
         self.filename = 'data.h5'
 
+    # Won't throw an exception unlike the parent.
+    def test_str_ascii_encoded_utf8(self):
+        data = self.random_str_some_unicode(random.randint(1,
+            self.max_string_length)).encode('UTF-8')
+        out = self.write_readback(data, self.random_name(),
+                                  self.options)
+        self.assert_equal(out, data)
+
+
 
 class TestNoneFormat(TestPythonMatlabFormat):
     def __init__(self):
@@ -569,6 +579,14 @@ class TestNoneFormat(TestPythonMatlabFormat):
         # Add in float16 to the set of types tested.
         self.dtypes.append('float16')
 
+    # Won't throw an exception unlike the parent.
+    def test_str_ascii_encoded_utf8(self):
+        data = self.random_str_some_unicode(random.randint(1,
+            self.max_string_length)).encode('UTF-8')
+        out = self.write_readback(data, self.random_name(),
+                                  self.options)
+        self.assert_equal(out, data)
+
     def assert_equal(self, a, b):
         assert_equal_none_format(a, b)
 

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/python-hdf5storage.git