[h5py] 319/455: Merge refs
Ghislain Vaillant
ghisvail-guest at moszumanska.debian.org
Thu Jul 2 18:19:46 UTC 2015
This is an automated email from the git hooks/post-receive script.
ghisvail-guest pushed a commit to annotated tag 1.3.0
in repository h5py.
commit a761a379a8a9f0f0fff27fc10dc9d3a96ec8ebb2
Author: andrewcollette <andrew.collette at gmail.com>
Date: Mon Oct 12 01:26:13 2009 +0000
Merge refs
---
h5py/_conv.pxd | 37 ++++
h5py/_conv.pyx | 470 ++++++++++++++++++++++++++++++++++++++++++
h5py/_proxy.pxd | 19 ++
h5py/_proxy.pyx | 289 ++++++++++++++++++++++++++
h5py/defs.pxd | 12 +-
h5py/h5.pxd | 22 --
h5py/h5.pyx | 27 +--
h5py/h5a.pyx | 7 +-
h5py/h5d.pyx | 7 +-
h5py/h5e.pyx | 6 +-
h5py/h5r.pyx | 96 +++++----
h5py/h5t.pyx | 125 ++++++++++--
h5py/python.pxd | 83 --------
h5py/tests/__init__.py | 3 +-
h5py/tests/test_conv.py | 61 ++++++
h5py/tests/test_h5r.py | 8 +-
h5py/typeconv.c | 529 ------------------------------------------------
h5py/typeconv.h | 28 ---
h5py/typeproxy.c | 450 ----------------------------------------
h5py/typeproxy.h | 55 -----
setup.py | 8 +-
21 files changed, 1062 insertions(+), 1280 deletions(-)
diff --git a/h5py/_conv.pxd b/h5py/_conv.pxd
new file mode 100644
index 0000000..9dca753
--- /dev/null
+++ b/h5py/_conv.pxd
@@ -0,0 +1,37 @@
+#+
+#
+# This file is part of h5py, a low-level Python interface to the HDF5 library.
+#
+# Copyright (C) 2008 Andrew Collette
+# http://h5py.alfven.org
+# License: BSD (See LICENSE.txt for full license)
+#
+# $Date$
+#
+#-
+
+include "defs.pxd"
+
+from h5r cimport Reference
+
+# Minimal interface for Python objects immune to Cython refcounting
+cdef extern from "Python.h":
+
+ # From Cython declarations
+ ctypedef void PyTypeObject
+ ctypedef struct PyObject:
+ Py_ssize_t ob_refcnt
+ PyTypeObject *ob_type
+
+ PyObject* PyString_FromString(char* str) except NULL
+ int PyString_CheckExact(PyObject* str) except *
+ int PyString_Size(PyObject* obj) except *
+ PyObject* PyObject_Str(PyObject* obj) except NULL
+ char* PyString_AsString(PyObject* obj) except NULL
+
+ PyObject* Py_None
+ void Py_INCREF(PyObject* obj)
+ void Py_DECREF(PyObject* obj)
+ void Py_XDECREF(PyObject* obj)
+
+
diff --git a/h5py/_conv.pyx b/h5py/_conv.pyx
new file mode 100644
index 0000000..772b19e
--- /dev/null
+++ b/h5py/_conv.pyx
@@ -0,0 +1,470 @@
+#+
+#
+# This file is part of h5py, a low-level Python interface to the HDF5 library.
+#
+# Copyright (C) 2008 Andrew Collette
+# http://h5py.alfven.org
+# License: BSD (See LICENSE.txt for full license)
+#
+# $Date$
+#
+#-
+
+"""
+ Low-level type-conversion routines.
+"""
+
+
+
+# Create Python object equivalents
+cdef hid_t H5PY_OBJ = 0
+
+cpdef hid_t get_python_obj():
+ global H5PY_OBJ
+ if H5PY_OBJ <= 0:
+ H5PY_OBJ = H5Tcreate(H5T_OPAQUE, sizeof(PyObject*))
+ H5Tset_tag(H5PY_OBJ, "PYTHON:OBJECT")
+ H5Tlock(H5PY_OBJ)
+ return H5PY_OBJ
+
+ctypedef int (*conv_operator_t)(void* ipt, void* opt, void* bkg, void* priv) except -1
+ctypedef herr_t (*init_operator_t)(hid_t src, hid_t dst, void** priv) except -1
+
+# Generic conversion callback
+#
+# The actual conversion routines are one-liners which plug the appropriate
+# operator callback into this function. This prevents us from having to
+# repeat all the conversion boilerplate for every single callback.
+#
+# While this is somewhat slower than a custom function, the added overhead is
+# likely small compared to the cost of the Python-side API calls required to
+# implement the conversions.
+cdef herr_t generic_converter(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata,
+ size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i,
+ void *bkg_i, hid_t dxpl, conv_operator_t op,
+ init_operator_t initop, H5T_bkg_t need_bkg) except -1:
+
+ cdef int command = cdata[0].command
+ cdef conv_size_t *sizes
+ cdef int i
+
+ cdef char* buf = <char*>buf_i
+ cdef char* bkg = <char*>bkg_i
+
+ if command == H5T_CONV_INIT:
+
+ cdata[0].need_bkg = need_bkg
+ return initop(src_id, dst_id, &(cdata[0].priv))
+
+ elif command == H5T_CONV_FREE:
+
+ free(cdata[0].priv)
+ cdata[0].priv = NULL
+
+ elif command == H5T_CONV_CONV:
+
+ sizes = <conv_size_t*>cdata[0].priv
+
+ if bkg_stride==0: bkg_stride = sizes[0].dst_size;
+
+ if buf_stride == 0:
+ # No explicit stride seems to mean that the elements are packed
+ # contiguously in the buffer. In this case we must be careful
+ # not to "stomp on" input elements if the output elements are
+ # of a larger size.
+
+ if sizes[0].src_size >= sizes[0].dst_size:
+ for i from 0<=i<nl:
+ op( buf + (i*sizes[0].src_size), # input pointer
+ buf + (i*sizes[0].dst_size), # output pointer
+ bkg + (i*bkg_stride), # backing buffer
+ cdata[0].priv) # conversion context
+ else:
+ for i from nl>i>=0:
+ op( buf + (i*sizes[0].src_size),
+ buf + (i*sizes[0].dst_size),
+ bkg + (i*bkg_stride),
+ cdata[0].priv)
+ else:
+ # With explicit strides, we assume that the library knows the
+ # alignment better than us. Therefore we use the given stride
+ # offsets exclusively.
+ for i from 0<=i<nl:
+ op( buf + (i*buf_stride),
+ buf + (i*buf_stride), # note this is the same!
+ bkg + (i*bkg_stride),
+ cdata[0].priv)
+ else:
+ return -2 # Unrecognized command. Note this is NOT an exception.
+
+ return 0
+
+# =============================================================================
+# String to VLEN routines
+
+ctypedef struct conv_size_t:
+ size_t src_size
+ size_t dst_size
+
+cdef herr_t init_generic(hid_t src, hid_t dst, void** priv) except -1:
+
+ cdef conv_size_t *sizes
+ priv[0] = sizes = <conv_size_t*>malloc(sizeof(conv_size_t))
+ sizes[0].src_size = H5Tget_size(src)
+ sizes[0].dst_size = H5Tget_size(dst)
+
+ return 0
+
+cdef int conv_vlen2str(void* ipt, void* opt, void* bkg, void* priv) except -1:
+
+ cdef PyObject** buf_obj = <PyObject**>opt
+ cdef PyObject** bkg_obj = <PyObject**>bkg
+ cdef char** buf_cstring = <char**>ipt
+ cdef PyObject* temp_obj = NULL
+
+ if buf_cstring[0] == NULL:
+ temp_obj = PyString_FromString("")
+ else:
+ temp_obj = PyString_FromString(buf_cstring[0])
+
+ # Since all data conversions are by definition in-place, it
+ # is our responsibility to free the memory used by the vlens.
+ free(buf_cstring[0])
+
+ # HDF5 will eventuallly overwrite this target location, so we
+ # make sure to decref the object there.
+ Py_XDECREF(bkg_obj[0])
+
+ # Write the new string object to the buffer in-place
+ buf_obj[0] = temp_obj
+
+ return 0
+
+cdef int conv_str2vlen(void* ipt, void* opt, void* bkg, void* priv) except -1:
+
+ cdef PyObject** buf_obj = <PyObject**>ipt
+ cdef char** buf_cstring = <char**>opt
+
+ cdef PyObject* temp_object = NULL
+ cdef char* temp_string = NULL
+ cdef size_t temp_string_len = 0 # Not including null term
+
+ try:
+ if buf_obj[0] == NULL or buf_obj[0] == Py_None:
+ temp_string = ""
+ temp_string_len = 0
+ else:
+ if PyString_CheckExact(buf_obj[0]):
+ temp_object = buf_obj[0]
+ Py_INCREF(temp_object)
+ temp_string = PyString_AsString(temp_object)
+ temp_string_len = PyString_Size(temp_object)
+ else:
+ temp_object = PyObject_Str(buf_obj[0])
+ temp_string = PyString_AsString(temp_object)
+ temp_string_len = PyString_Size(temp_object)
+
+ if strlen(temp_string) != temp_string_len:
+ raise ValueError("VLEN strings do not support embedded NULLs")
+
+ buf_cstring[0] = <char*>malloc(temp_string_len+1)
+ memcpy(buf_cstring[0], temp_string, temp_string_len+1)
+
+ return 0
+ finally:
+ Py_XDECREF(temp_object)
+
+
+# =============================================================================
+# VLEN to fixed-width strings
+
+cdef herr_t init_vlen2fixed(hid_t src, hid_t dst, void** priv) except -1:
+
+ cdef conv_size_t *sizes
+
+ if not (H5Tis_variable_str(src) and (not H5Tis_variable_str(dst))):
+ return -2
+
+ priv[0] = sizes = <conv_size_t*>malloc(sizeof(conv_size_t))
+ sizes[0].src_size = H5Tget_size(src)
+ sizes[0].dst_size = H5Tget_size(dst)
+
+ return 0
+
+cdef herr_t init_fixed2vlen(hid_t src, hid_t dst, void** priv) except -1:
+
+ cdef conv_size_t *sizes
+
+ if not (H5Tis_variable_str(dst) and (not H5Tis_variable_str(src))):
+ return -2
+
+ priv[0] = sizes = <conv_size_t*>malloc(sizeof(conv_size_t))
+ sizes[0].src_size = H5Tget_size(src)
+ sizes[0].dst_size = H5Tget_size(dst)
+
+ return 0
+
+cdef int conv_vlen2fixed(void* ipt, void* opt, void* bkg, void* priv) except -1:
+
+ cdef char** buf_vlen = <char**>ipt
+ cdef char* buf_fixed = <char*>opt
+ cdef char* temp_string = NULL
+ cdef size_t temp_string_len = 0 # Without null term
+ cdef conv_size_t *sizes = <conv_size_t*>priv
+
+ if buf_vlen[0] != NULL:
+ temp_string = buf_vlen[0]
+ temp_string_len = strlen(temp_string)
+
+ if temp_string_len <= sizes[0].dst_size:
+ # Pad with zeros
+ memcpy(buf_fixed, temp_string, temp_string_len)
+ memset(buf_fixed + temp_string_len, c'\0', sizes[0].dst_size - temp_string_len)
+ else:
+ # Simply truncate the string
+ memcpy(buf_fixed, temp_string, sizes[0].dst_size)
+ else:
+ memset(buf_fixed, c'\0', sizes[0].dst_size)
+
+ return 0
+
+cdef int conv_fixed2vlen(void* ipt, void* opt, void* bkg, void* priv) except -1:
+
+ cdef char** buf_vlen = <char**>opt
+ cdef char* buf_fixed = <char*>ipt
+ cdef char* temp_string = NULL
+ cdef conv_size_t *sizes = <conv_size_t*>priv
+
+ temp_string = <char*>malloc(sizes[0].src_size+1)
+ memcpy(temp_string, buf_fixed, sizes[0].src_size)
+ temp_string[sizes[0].src_size] = c'\0'
+
+ buf_vlen[0] = temp_string
+
+ return 0
+
+# =============================================================================
+# HDF5 references to Python instances of h5r.Reference
+
+cdef int conv_objref2pyref(void* ipt, void* opt, void* bkg, void* priv) except -1:
+
+ cdef PyObject** buf_obj = <PyObject**>opt
+ cdef PyObject** bkg_obj = <PyObject**>bkg
+ cdef hobj_ref_t* buf_ref = <hobj_ref_t*>ipt
+
+ cdef Reference ref = Reference()
+ cdef PyObject* ref_ptr = NULL
+
+ ref.ref.obj_ref = buf_ref[0]
+ ref.typecode = H5R_OBJECT
+
+ ref_ptr = <PyObject*>ref
+ Py_INCREF(ref_ptr) # because Cython discards its reference when the
+ # function exits
+
+ Py_XDECREF(bkg_obj[0])
+ buf_obj[0] = ref_ptr
+
+ return 0
+
+cdef int conv_pyref2objref(void* ipt, void* opt, void* bkg, void* priv) except -1:
+
+ cdef PyObject** buf_obj = <PyObject**>ipt
+ cdef hobj_ref_t* buf_ref = <hobj_ref_t*>opt
+
+ cdef object obj
+ cdef Reference ref
+
+ if buf_obj[0] != NULL and buf_obj[0] != Py_None:
+ obj = <object>(buf_obj[0])
+ if not isinstance(obj, Reference):
+ raise TypeError("Can't convert incompatible object to HDF5 object reference")
+ ref = <Reference>(buf_obj[0])
+ buf_ref[0] = ref.ref.obj_ref
+ else:
+ memset(buf_ref, c'\0', sizeof(hobj_ref_t))
+
+ return 0
+
+# =============================================================================
+# Conversion functions
+
+
+cdef herr_t vlen2str(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata,
+ size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i,
+ void *bkg_i, hid_t dxpl) except -1:
+ return generic_converter(src_id, dst_id, cdata, nl, buf_stride, bkg_stride,
+ buf_i, bkg_i, dxpl, conv_vlen2str, init_generic, H5T_BKG_YES)
+
+cdef herr_t str2vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata,
+ size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i,
+ void *bkg_i, hid_t dxpl) except -1:
+ return generic_converter(src_id, dst_id, cdata, nl, buf_stride, bkg_stride,
+ buf_i, bkg_i, dxpl, conv_str2vlen, init_generic, H5T_BKG_NO)
+
+cdef herr_t vlen2fixed(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata,
+ size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i,
+ void *bkg_i, hid_t dxpl) except -1:
+ return generic_converter(src_id, dst_id, cdata, nl, buf_stride, bkg_stride,
+ buf_i, bkg_i, dxpl, conv_vlen2fixed, init_vlen2fixed, H5T_BKG_NO)
+
+cdef herr_t fixed2vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata,
+ size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i,
+ void *bkg_i, hid_t dxpl) except -1:
+ return generic_converter(src_id, dst_id, cdata, nl, buf_stride, bkg_stride,
+ buf_i, bkg_i, dxpl, conv_fixed2vlen, init_fixed2vlen, H5T_BKG_NO)
+
+cdef herr_t objref2pyref(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata,
+ size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i,
+ void *bkg_i, hid_t dxpl) except -1:
+ return generic_converter(src_id, dst_id, cdata, nl, buf_stride, bkg_stride,
+ buf_i, bkg_i, dxpl, conv_objref2pyref, init_generic, H5T_BKG_YES)
+
+cdef herr_t pyref2objref(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata,
+ size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i,
+ void *bkg_i, hid_t dxpl) except -1:
+ return generic_converter(src_id, dst_id, cdata, nl, buf_stride, bkg_stride,
+ buf_i, bkg_i, dxpl, conv_pyref2objref, init_generic, H5T_BKG_NO)
+
+# =============================================================================
+# Enum to integer converter
+
+cdef struct conv_enum_t:
+ size_t src_size
+ size_t dst_size
+ hid_t supertype
+ int identical
+
+# Direction ("forward"): 1 = enum to int, 0 = int to enum
+cdef herr_t enum_int_converter(hid_t src, hid_t dst, H5T_cdata_t *cdata,
+ size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i,
+ void *bkg_i, hid_t dxpl, int forward) except -1 with gil:
+
+ cdef int command = cdata[0].command
+ cdef conv_enum_t *info
+ cdef size_t nalloc
+ cdef int i
+ cdef char* cbuf = NULL
+ cdef char* buf = <char*>buf_i
+
+ if command == H5T_CONV_INIT:
+ cdata[0].need_bkg = H5T_BKG_NO
+ cdata[0].priv = info = <conv_enum_t*>malloc(sizeof(conv_enum_t))
+ info[0].src_size = H5Tget_size(src)
+ info[0].dst_size = H5Tget_size(dst)
+ if forward:
+ info[0].supertype = H5Tget_super(src)
+ info[0].identical = H5Tequal(info[0].supertype, dst)
+ else:
+ info[0].supertype = H5Tget_super(dst)
+ info[0].identical = H5Tequal(info[0].supertype, src)
+
+ elif command == H5T_CONV_FREE:
+
+ info = <conv_enum_t*>cdata[0].priv
+ #H5Tclose(info[0].supertype)
+ free(info)
+ cdata[0].priv = NULL
+
+ elif command == H5T_CONV_CONV:
+
+ info = <conv_enum_t*>cdata[0].priv
+
+ # Short-circuit success
+ if info[0].identical:
+ return 0
+
+ if buf_stride == 0:
+ # Contiguous case: call H5Tconvert directly
+ if forward:
+ H5Tconvert(info[0].supertype, dst, nl, buf, NULL, dxpl)
+ else:
+ H5Tconvert(src, info[0].supertype, nl, buf, NULL, dxpl)
+ else:
+ # Non-contiguous: gather, convert and then scatter
+ if info[0].src_size > info[0].dst_size:
+ nalloc = info[0].src_size*nl
+ else:
+ nalloc = info[0].dst_size*nl
+
+ cbuf = <char*>malloc(nalloc)
+ if cbuf == NULL:
+ raise MemoryError("Can't allocate conversion buffer")
+ try:
+ for i from 0<=i<nl:
+ memcpy(cbuf + (i*info[0].src_size), buf + (i*buf_stride),
+ info[0].src_size)
+
+ if forward:
+ H5Tconvert(info[0].supertype, dst, nl, cbuf, NULL, dxpl)
+ else:
+ H5Tconvert(src, info[0].supertype, nl, cbuf, NULL, dxpl)
+
+ for i from 0<=i<nl:
+ memcpy(buf + (i*buf_stride), cbuf + (i*info[0].dst_size),
+ info[0].dst_size)
+ finally:
+ free(cbuf)
+ cbuf = NULL
+ else:
+ return -2
+
+ return 0
+
+cdef herr_t enum2int(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata,
+ size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i,
+ void *bkg_i, hid_t dxpl) except -1:
+ return enum_int_converter(src_id, dst_id, cdata, nl, buf_stride, bkg_stride,
+ buf_i, bkg_i, dxpl, 1)
+
+cdef herr_t int2enum(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata,
+ size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i,
+ void *bkg_i, hid_t dxpl) except -1:
+ return enum_int_converter(src_id, dst_id, cdata, nl, buf_stride, bkg_stride,
+ buf_i, bkg_i, dxpl, 0)
+
+
+cpdef int register_converters() except -1:
+
+ cdef hid_t vlstring
+ cdef hid_t pyobj
+ cdef hid_t enum
+
+ vlstring = H5Tcopy(H5T_C_S1)
+ H5Tset_size(vlstring, H5T_VARIABLE)
+
+ enum = H5Tenum_create(H5T_STD_I32LE)
+
+ pyobj = get_python_obj()
+
+ H5Tregister(H5T_PERS_HARD, "vlen2str", vlstring, pyobj, vlen2str)
+ H5Tregister(H5T_PERS_HARD, "str2vlen", pyobj, vlstring, str2vlen)
+ H5Tregister(H5T_PERS_SOFT, "vlen2fixed", vlstring, H5T_C_S1, vlen2fixed)
+ H5Tregister(H5T_PERS_SOFT, "fixed2vlen", H5T_C_S1, vlstring, fixed2vlen)
+ H5Tregister(H5T_PERS_HARD, "objref2pyref", H5T_STD_REF_OBJ, pyobj, objref2pyref)
+ H5Tregister(H5T_PERS_HARD, "pyref2objref", pyobj, H5T_STD_REF_OBJ, pyref2objref)
+ H5Tregister(H5T_PERS_SOFT, "enum2int", enum, H5T_STD_I32LE, enum2int)
+ H5Tregister(H5T_PERS_SOFT, "int2enum", H5T_STD_I32LE, enum, int2enum)
+
+ H5Tclose(vlstring)
+ H5Tclose(enum)
+
+ return 0
+
+cpdef int unregister_converters() except -1:
+
+ H5Tunregister(H5T_PERS_HARD, "vlen2str", -1, -1, vlen2str)
+ H5Tunregister(H5T_PERS_HARD, "str2vlen", -1, -1, str2vlen)
+ H5Tunregister(H5T_PERS_SOFT, "vlen2fixed", -1, -1, vlen2fixed)
+ H5Tunregister(H5T_PERS_SOFT, "fixed2vlen", -1, -1, fixed2vlen)
+ H5Tunregister(H5T_PERS_HARD, "objref2pyref", -1, -1, objref2pyref)
+ H5Tunregister(H5T_PERS_HARD, "pyref2objref", -1, -1, pyref2objref)
+ H5Tunregister(H5T_PERS_SOFT, "enum2int", -1, -1, enum2int)
+ H5Tunregister(H5T_PERS_SOFT, "int2enum", -1, -1, int2enum)
+
+ return 0
+
+
+
+
+
diff --git a/h5py/_proxy.pxd b/h5py/_proxy.pxd
new file mode 100644
index 0000000..ea3e0f1
--- /dev/null
+++ b/h5py/_proxy.pxd
@@ -0,0 +1,19 @@
+#+
+#
+# This file is part of h5py, a low-level Python interface to the HDF5 library.
+#
+# Copyright (C) 2008 Andrew Collette
+# http://h5py.alfven.org
+# License: BSD (See LICENSE.txt for full license)
+#
+# $Date$
+#
+#-
+
+include "defs.pxd"
+
+cdef herr_t attr_rw(hid_t attr, hid_t mtype, void *progbuf, int read) except -1
+
+cdef herr_t dset_rw(hid_t dset, hid_t mtype, hid_t mspace, hid_t fspace,
+ hid_t dxpl, void* progbuf, int read) except -1
+
diff --git a/h5py/_proxy.pyx b/h5py/_proxy.pyx
new file mode 100644
index 0000000..97d2956
--- /dev/null
+++ b/h5py/_proxy.pyx
@@ -0,0 +1,289 @@
+#+
+#
+# This file is part of h5py, a low-level Python interface to the HDF5 library.
+#
+# Copyright (C) 2008 Andrew Collette
+# http://h5py.alfven.org
+# License: BSD (See LICENSE.txt for full license)
+#
+# $Date$
+#
+#-
+
+"""
+ Proxy functions for read/write, to work around the HDF5 bogus type issue.
+"""
+
+cdef enum copy_dir:
+ H5PY_SCATTER = 0,
+ H5PY_GATHER
+
+cdef herr_t attr_rw(hid_t attr, hid_t mtype, void *progbuf, int read) except -1:
+
+ cdef htri_t need_bkg
+ cdef hid_t atype = -1
+ cdef hid_t aspace = -1
+ cdef hsize_t npoints
+
+ cdef size_t msize, asize
+ cdef void* conv_buf = NULL
+ cdef void* back_buf = NULL
+
+ try:
+ atype = H5Aget_type(attr)
+
+ if not (needs_proxy(atype) or needs_proxy(mtype)):
+ if read:
+ H5Aread(attr, mtype, progbuf)
+ else:
+ H5Awrite(attr, mtype, progbuf)
+
+ else:
+
+ asize = H5Tget_size(atype)
+ msize = H5Tget_size(mtype)
+ aspace = H5Aget_space(attr)
+ npoints = H5Sget_select_npoints(aspace)
+
+ conv_buf = create_buffer(asize, msize, npoints)
+
+ if read:
+ need_bkg = needs_bkg_buffer(atype, mtype)
+ else:
+ need_bkg = needs_bkg_buffer(mtype, atype)
+ if need_bkg:
+ back_buf = malloc(msize*npoints)
+ memcpy(back_buf, progbuf, msize*npoints)
+
+ if read:
+ H5Aread(attr, atype, conv_buf)
+ H5Tconvert(atype, mtype, npoints, conv_buf, back_buf, H5P_DEFAULT)
+ memcpy(progbuf, conv_buf, msize*npoints)
+ else:
+ memcpy(conv_buf, progbuf, msize*npoints)
+ H5Tconvert(mtype, atype, npoints, conv_buf, back_buf, H5P_DEFAULT)
+ H5Awrite(attr, atype, conv_buf)
+ finally:
+ free(conv_buf)
+ free(back_buf)
+ if atype > 0:
+ H5Tclose(atype)
+ if aspace > 0:
+ H5Sclose(aspace)
+
+ return 0
+
+# =============================================================================
+# Proxy functions to safely release the GIL around read/write operations
+
+cdef herr_t H5PY_H5Dread(hid_t dset, hid_t mtype, hid_t mspace,
+ hid_t fspace, hid_t dxpl, void* buf) except -1:
+ cdef herr_t retval
+ with nogil:
+ retval = H5Dread(dset, mtype, mspace, fspace, dxpl, buf)
+ if retval < 0:
+ return -1
+ return retval
+
+cdef herr_t H5PY_H5Dwrite(hid_t dset, hid_t mtype, hid_t mspace,
+ hid_t fspace, hid_t dxpl, void* buf) except -1:
+ cdef herr_t retval
+ with nogil:
+ retval = H5Dwrite(dset, mtype, mspace, fspace, dxpl, buf)
+ if retval < 0:
+ return -1
+ return retval
+
+# =============================================================================
+# Proxy for vlen buf workaround
+
+cdef herr_t dset_rw(hid_t dset, hid_t mtype, hid_t mspace, hid_t fspace,
+ hid_t dxpl, void* progbuf, int read) except -1:
+
+ cdef htri_t need_bkg
+ cdef hid_t dstype = -1 # Dataset datatype
+ cdef hid_t dspace = -1 # Dataset dataspace
+ cdef hid_t cspace = -1 # Temporary contiguous dataspaces
+
+ cdef void* back_buf = NULL
+ cdef void* conv_buf = NULL
+ cdef hsize_t npoints
+
+ try:
+ dstype = H5Dget_type(dset)
+
+ if not (needs_proxy(dstype) or needs_proxy(mtype)):
+ if read:
+ H5PY_H5Dread(dset, mtype, mspace, fspace, dxpl, progbuf)
+ else:
+ H5PY_H5Dwrite(dset, mtype, mspace, fspace, dxpl, progbuf)
+ else:
+
+ if mspace == H5S_ALL and fspace != H5S_ALL:
+ mspace = fspace
+ elif mspace != H5S_ALL and fspace == H5S_ALL:
+ fspace = mspace
+ elif mspace == H5S_ALL and fspace == H5S_ALL:
+ fspace = mspace = dspace = H5Dget_space(dset)
+
+ npoints = H5Sget_select_npoints(mspace)
+ cspace = H5Screate_simple(1, &npoints, NULL)
+
+ conv_buf = create_buffer(H5Tget_size(dstype), H5Tget_size(mtype), npoints)
+
+ # Only create a (contiguous) backing buffer if absolutely
+ # necessary. Note this buffer always has memory type.
+ if read:
+ need_bkg = needs_bkg_buffer(dstype, mtype)
+ else:
+ need_bkg = needs_bkg_buffer(mtype, dstype)
+ if need_bkg:
+ back_buf = malloc(H5Tget_size(mtype)*npoints)
+ h5py_copy(mtype, mspace, back_buf, progbuf, H5PY_GATHER)
+
+ if read:
+ H5PY_H5Dread(dset, dstype, cspace, fspace, dxpl, conv_buf)
+ H5Tconvert(dstype, mtype, npoints, conv_buf, back_buf, dxpl)
+ h5py_copy(mtype, mspace, conv_buf, progbuf, H5PY_SCATTER)
+ else:
+ h5py_copy(mtype, mspace, conv_buf, progbuf, H5PY_GATHER)
+ H5Tconvert(mtype, dstype, npoints, conv_buf, back_buf, dxpl)
+ H5PY_H5Dwrite(dset, dstype, cspace, fspace, dxpl, conv_buf)
+
+ finally:
+ free(back_buf)
+ free(conv_buf)
+ if dstype > 0:
+ H5Tclose(dstype)
+ if dspace > 0:
+ H5Sclose(dspace)
+ if cspace > 0:
+ H5Sclose(cspace)
+
+ return 0
+
+cdef void* create_buffer(size_t ipt_size, size_t opt_size, nl) except NULL:
+
+ cdef size_t final_size
+ cdef void* buf
+
+ if ipt_size >= opt_size:
+ final_size = ipt_size*nl
+ else:
+ final_size = opt_size*nl
+
+ buf = malloc(final_size)
+ if buf == NULL:
+ raise MemoryError("Failed to allocate conversion buffer")
+
+ return buf
+
+# =============================================================================
+# Scatter/gather routines
+
+ctypedef struct h5py_scatter_t:
+ size_t i
+ size_t elsize
+ void* buf
+
+cdef herr_t h5py_scatter_cb(void* elem, hid_t type_id, unsigned ndim,
+ hsize_t *point, void *operator_data) except -1:
+
+ cdef h5py_scatter_t* info = <h5py_scatter_t*>operator_data
+
+ memcpy(elem, (<char*>info[0].buf)+((info[0].i)*(info[0].elsize)),
+ info[0].elsize)
+
+ info[0].i += 1
+
+ return 0
+
+cdef herr_t h5py_gather_cb(void* elem, hid_t type_id, unsigned ndim,
+ hsize_t *point, void *operator_data) except -1:
+
+ cdef h5py_scatter_t* info = <h5py_scatter_t*>operator_data
+
+ memcpy((<char*>info[0].buf)+((info[0].i)*(info[0].elsize)), elem,
+ info[0].elsize)
+
+ info[0].i += 1
+
+ return 0
+
+# Copy between a contiguous and non-contiguous buffer, with the layout
+# of the latter specified by a dataspace selection.
+cdef herr_t h5py_copy(hid_t tid, hid_t space, void* contig, void* noncontig,
+ copy_dir op) except -1:
+
+ cdef h5py_scatter_t info
+ cdef hsize_t elsize
+
+ elsize = H5Tget_size(tid)
+
+ info.i = 0
+ info.elsize = elsize
+ info.buf = contig
+
+ if op == H5PY_SCATTER:
+ H5Diterate(noncontig, tid, space, h5py_scatter_cb, &info)
+ elif op == H5PY_GATHER:
+ H5Diterate(noncontig, tid, space, h5py_gather_cb, &info)
+ else:
+ raise RuntimeError("Illegal direction")
+
+ return 0
+
+# =============================================================================
+# VLEN support routines
+
+cdef htri_t needs_bkg_buffer(hid_t src, hid_t dst) except -1:
+
+ cdef H5T_cdata_t *info = NULL
+
+ if H5Tdetect_class(src, H5T_COMPOUND) or H5Tdetect_class(dst, H5T_COMPOUND):
+ return 1
+
+ H5Tfind(src, dst, &info)
+ if info[0].need_bkg == H5T_BKG_YES:
+ return 1
+
+ return 0
+
+# Determine if the given type requires proxy buffering
+cdef htri_t needs_proxy(hid_t tid) except -1:
+
+ cdef H5T_class_t cls
+ cdef hid_t supertype
+ cdef int i, n
+ cdef htri_t result
+
+ cls = H5Tget_class(tid)
+
+ if cls == H5T_VLEN or cls == H5T_REFERENCE:
+ return 1
+
+ elif cls == H5T_STRING:
+ return H5Tis_variable_str(tid)
+
+ elif cls == H5T_ARRAY:
+
+ supertype = H5Tget_super(tid)
+ try:
+ return needs_proxy(supertype)
+ finally:
+ H5Tclose(supertype)
+
+ elif cls == H5T_COMPOUND:
+
+ n = H5Tget_nmembers(tid)
+ for i from 0<=i<n:
+ supertype = H5Tget_member_type(tid, i)
+ try:
+ result = needs_proxy(supertype)
+ if result > 0:
+ return 1
+ finally:
+ H5Tclose(supertype)
+ return 0
+
+ return 0
diff --git a/h5py/defs.pxd b/h5py/defs.pxd
index b45a8b4..b451a7a 100644
--- a/h5py/defs.pxd
+++ b/h5py/defs.pxd
@@ -38,6 +38,7 @@ cdef extern from "string.h":
int strcmp(char *s1, char *s2)
char *strdup(char *s)
void *memcpy(void *dest, void *src, size_t n)
+ void *memset(void *s, int c, size_t n)
cdef extern from "time.h":
ctypedef int time_t
@@ -167,9 +168,8 @@ cdef extern from "hdf5.h":
haddr_t H5Dget_offset(hid_t dset_id) except *
hsize_t H5Dget_storage_size(hid_t dset_id) except? 0
- # These must have their return values checked manually. The functions
- # H5PY_H5Dread and H5PY_HDwrite return -1 specifically, for use when
- # the GIL is released and PyErr_Occurred() is inadvisable.
+ # These must have their return values checked manually, in order to
+ # allow the GIL to be released during reading and writing.
herr_t H5Dread(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id,
hid_t file_space_id, hid_t plist_id, void *buf) nogil
herr_t H5Dwrite(hid_t dset_id, hid_t mem_type, hid_t mem_space, hid_t
@@ -185,7 +185,7 @@ cdef extern from "hdf5.h":
hid_t plist, void *buf) except *
ctypedef herr_t (*H5D_operator_t)(void *elem, hid_t type_id, unsigned ndim,
- hsize_t *point, void *operator_data)
+ hsize_t *point, void *operator_data) except -1
herr_t H5Diterate(void *buf, hid_t type_id, hid_t space_id,
H5D_operator_t operator, void* operator_data) except *
herr_t H5Dset_extent(hid_t dset_id, hsize_t* size)
@@ -1168,9 +1168,9 @@ cdef extern from "hdf5.h":
ctypedef herr_t (*H5T_conv_t)(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata,
size_t nelmts, size_t buf_stride, size_t bkg_stride, void *buf,
- void *bkg, hid_t dset_xfer_plist)
+ void *bkg, hid_t dset_xfer_plist) except -1
- H5T_conv_t H5Tfind(hid_t src_id, hid_t dst_id, H5T_cdata_t **pcdata)
+ H5T_conv_t H5Tfind(hid_t src_id, hid_t dst_id, H5T_cdata_t **pcdata) except *
herr_t H5Tregister(H5T_pers_t pers, char *name, hid_t src_id,
hid_t dst_id, H5T_conv_t func) except *
diff --git a/h5py/h5.pxd b/h5py/h5.pxd
index 028a200..89379d7 100644
--- a/h5py/h5.pxd
+++ b/h5py/h5.pxd
@@ -50,27 +50,5 @@ cdef class SmartStruct:
# Library init. Safe to call more than once.
cdef int init_hdf5() except -1
-cdef extern from "typeconv.h":
-
- hid_t h5py_object_type() except *
- int h5py_register_conv() except -1
-
-cdef extern from "typeproxy.h":
- ctypedef enum h5py_rw_t:
- H5PY_WRITE = 0,
- H5PY_READ
-
- herr_t H5PY_dset_rw(hid_t dset, hid_t mtype, hid_t mspace_in, hid_t fspace_in,
- hid_t xfer_plist, void* buf, h5py_rw_t dir) except *
-
- herr_t H5PY_attr_rw(hid_t attr, hid_t mtype, void* buf, h5py_rw_t dir) except *
-
-cdef hid_t get_object_type() except -1
-
-cdef herr_t attr_rw(hid_t attr_id, hid_t mem_type_id, void *buf, h5py_rw_t dir) except *
-
-cdef herr_t dset_rw(hid_t dataset_id, hid_t mem_type_id, hid_t mem_space_id,
- hid_t file_space_id, hid_t xfer_plist_id, void *outbuf,
- h5py_rw_t dir) except *
diff --git a/h5py/h5.pyx b/h5py/h5.pyx
index 7de639b..cea307a 100644
--- a/h5py/h5.pyx
+++ b/h5py/h5.pyx
@@ -423,29 +423,16 @@ def _exithack():
finally:
free(objs)
-hdf5_inited = 0
-
-# Proxy function which returns the predefined HDF5 type corresponding to
-# a Python object pointer (opaque type with size sizeof(void*)).
-cdef hid_t get_object_type() except -1:
- return h5py_object_type()
-
-# Proxy functions for r/w with workarounds for various HDF5 bugs.
-cdef herr_t dset_rw(hid_t dataset_id, hid_t mem_type_id, hid_t mem_space_id,
- hid_t file_space_id, hid_t xfer_plist_id, void *outbuf,
- h5py_rw_t dir) except *:
+ _conv.unregister_converters()
- return H5PY_dset_rw(dataset_id, mem_type_id, mem_space_id, file_space_id,
- xfer_plist_id, outbuf, dir)
-
-cdef herr_t attr_rw(hid_t attr_id, hid_t mem_type_id, void *buf, h5py_rw_t dir) except *:
- return H5PY_attr_rw(attr_id, mem_type_id, buf, dir)
+hdf5_inited = 0
cdef int init_hdf5() except -1:
# Initialize the library and register Python callbacks for exception
# handling. Safe to call more than once.
global hdf5_inited
+ import _conv
if not hdf5_inited:
IF H5PY_DEBUG:
log_lib.info("* Initializing h5py library")
@@ -455,12 +442,13 @@ cdef int init_hdf5() except -1:
if register_lzf() < 0:
raise RuntimeError("Failed to register LZF filter")
atexit.register(_exithack)
- h5py_register_conv()
+ #h5py_register_conv()
+ _conv.register_converters()
hdf5_inited = 1
return 0
-init_hdf5()
+
# === Module init =============================================================
@@ -473,4 +461,5 @@ _version_string = H5PY_VERSION
# being in this module.
from h5e import H5Error
-
+init_hdf5()
+import _conv
diff --git a/h5py/h5a.pyx b/h5py/h5a.pyx
index e23a630..75dedfe 100644
--- a/h5py/h5a.pyx
+++ b/h5py/h5a.pyx
@@ -17,12 +17,13 @@
include "config.pxi"
# Compile-time imports
-from h5 cimport init_hdf5, SmartStruct, attr_rw, H5PY_READ, H5PY_WRITE
+from h5 cimport init_hdf5, SmartStruct
from h5t cimport TypeID, typewrap, py_create
from h5s cimport SpaceID
from h5p cimport PropID, pdefault
from numpy cimport import_array, ndarray, PyArray_DATA
from utils cimport check_numpy_read, check_numpy_write, emalloc, efree
+from _proxy cimport attr_rw
# Initialization
import_array()
@@ -451,7 +452,7 @@ cdef class AttrID(ObjectID):
mtype = py_create(arr.dtype)
- attr_rw(self.id, mtype.id, PyArray_DATA(arr), H5PY_READ)
+ attr_rw(self.id, mtype.id, PyArray_DATA(arr), 1)
finally:
if space_id:
@@ -477,7 +478,7 @@ cdef class AttrID(ObjectID):
check_numpy_read(arr, space_id)
mtype = py_create(arr.dtype)
- attr_rw(self.id, mtype.id, PyArray_DATA(arr), H5PY_WRITE)
+ attr_rw(self.id, mtype.id, PyArray_DATA(arr), 0)
finally:
if space_id:
diff --git a/h5py/h5d.pyx b/h5py/h5d.pyx
index 767d86d..d0f0aca 100644
--- a/h5py/h5d.pyx
+++ b/h5py/h5d.pyx
@@ -17,13 +17,14 @@
include "config.pxi"
# Compile-time imports
-from h5 cimport init_hdf5, dset_rw, H5PY_READ, H5PY_WRITE
+from h5 cimport init_hdf5
from numpy cimport ndarray, import_array, PyArray_DATA, NPY_WRITEABLE
from utils cimport check_numpy_read, check_numpy_write, \
convert_tuple, emalloc, efree
from h5t cimport TypeID, typewrap, py_create
from h5s cimport SpaceID
from h5p cimport PropID, propwrap, pdefault
+from _proxy cimport dset_rw
# Initialization
import_array()
@@ -173,7 +174,7 @@ cdef class DatasetID(ObjectID):
arr_obj.flags &= (~NPY_WRITEABLE) # Wish-it-was-a-mutex approach
try:
- dset_rw(self_id, mtype_id, mspace_id, fspace_id, plist_id, data, H5PY_READ)
+ dset_rw(self_id, mtype_id, mspace_id, fspace_id, plist_id, data, 1)
finally:
arr_obj.flags |= NPY_WRITEABLE
@@ -207,7 +208,7 @@ cdef class DatasetID(ObjectID):
arr_obj.flags &= (~NPY_WRITEABLE) # Wish-it-was-a-mutex approach
try:
- dset_rw(self_id, mtype_id, mspace_id, fspace_id, plist_id, data, H5PY_WRITE)
+ dset_rw(self_id, mtype_id, mspace_id, fspace_id, plist_id, data, 0)
finally:
arr_obj.flags |= NPY_WRITEABLE
diff --git a/h5py/h5e.pyx b/h5py/h5e.pyx
index 07ee60b..0a9f570 100644
--- a/h5py/h5e.pyx
+++ b/h5py/h5e.pyx
@@ -17,7 +17,7 @@
include "config.pxi"
-from python_exc cimport PyErr_SetString
+from python_exc cimport PyErr_SetString, PyErr_Occurred
from h5 cimport SmartStruct
import _stub
@@ -362,6 +362,10 @@ cdef herr_t err_callback(void* client_data) with gil:
# By definition any function for which this can be called already
# holds the PHIL.
+ if PyErr_Occurred() != NULL:
+ # Native Python exceptions can occur inside HDF5 callbacks
+ return 1
+
stack = ErrorStack()
a = stack.get_exc_msg()
diff --git a/h5py/h5r.pyx b/h5py/h5r.pyx
index e9241be..f94f72e 100644
--- a/h5py/h5r.pyx
+++ b/h5py/h5r.pyx
@@ -65,73 +65,69 @@ def create(ObjectID loc not None, char* name, int ref_type, SpaceID space=None):
return ref
-cdef class Reference:
+ at sync
+def dereference(Reference ref not None, ObjectID id not None):
+ """(Reference ref, ObjectID id) => ObjectID
- """
- Represents an HDF5 reference.
+ Open the object pointed to by the reference and return its
+ identifier. The file identifier (or the identifier for any object
+ in the file) must also be provided.
- Objects of this class are created exclusively by the library and
- cannot be modified. The read-only attribute "typecode" determines
- whether the reference is to an object in an HDF5 file (OBJECT)
- or a dataset region (DATASET_REGION).
+ The reference type may be either OBJECT or DATASET_REGION.
"""
+ return wrap_identifier(H5Rdereference(id.id, <H5R_type_t>ref.typecode, &ref.ref))
+
+ at sync
+def get_region(Reference ref not None, ObjectID id not None):
+ """(Reference ref, ObjectID id) => SpaceID
- @sync
- def dereference(self, ObjectID id not None):
- """(ObjectID id) => ObjectID
+ Retrieve the dataspace selection pointed to by the reference.
+ Returns a copy of the dataset's dataspace, with the appropriate
+ elements selected. The file identifier or the identifier of any
+ object in the file (including the dataset itself) must also be
+ provided.
- Open the object pointed to by this reference and return its
- identifier. The file identifier (or the identifier for any object
- in the file) must also be provided.
+ The reference object must be of type DATASET_REGION.
+ """
+ return SpaceID(H5Rget_region(id.id, <H5R_type_t>ref.typecode, &ref.ref))
- The reference type may be either OBJECT or DATASET_REGION.
- """
- return wrap_identifier(H5Rdereference(id.id, <H5R_type_t>self.typecode, &self.ref))
+ at sync
+def get_obj_type(Reference ref not None, ObjectID id not None):
+ """(Reference ref, ObjectID id) => INT obj_code
- @sync
- def get_region(self, ObjectID id not None):
- """(ObjectID id) => SpaceID
+ Determine what type of object the eference points to. The
+ reference may be either type OBJECT or DATASET_REGION. The file
+ identifier or the identifier of any object in the file must also
+ be provided.
- Retrieve the dataspace selection pointed to by this reference.
- Returns a copy of the dataset's dataspace, with the appropriate
- elements selected. The file identifier or the identifier of any
- object in the file (including the dataset itself) must also be
- provided.
+ The return value is one of:
- The reference object must be of type DATASET_REGION.
- """
- return SpaceID(H5Rget_region(id.id, <H5R_type_t>self.typecode, &self.ref))
+ - h5g.LINK
+ - h5g.GROUP
+ - h5g.DATASET
+ - h5g.TYPE
+ """
+ return <int>H5Rget_obj_type(id.id, <H5R_type_t>ref.typecode, &ref.ref)
- @sync
- def get_obj_type(self, ObjectID id not None):
- """(ObjectID id) => INT obj_code
+cdef class Reference:
- Determine what type of object this eference points to. The
- reference may be either type OBJECT or DATASET_REGION. The file
- identifier or the identifier of any object in the file must also
- be provided.
+ """
+ Opaque representation of an HDF5 reference.
- The return value is one of:
+ Objects of this class are created exclusively by the library and
+ cannot be modified. The read-only attribute "typecode" determines
+ whether the reference is to an object in an HDF5 file (OBJECT)
+ or a dataset region (DATASET_REGION).
+ """
- - h5g.LINK
- - h5g.GROUP
- - h5g.DATASET
- - h5g.TYPE
- """
- return <int>H5Rget_obj_type(id.id, <H5R_type_t>self.typecode, &self.ref)
@nosync
- def __str__(self):
+ def __repr__(self):
if self.typecode == H5R_OBJECT:
- return "HDF5 object reference"
+ return "<HDF5 object reference>"
elif self.typecode == H5R_DATASET_REGION:
- return "HDF5 dataset region reference"
-
- return "Unknown HDF5 reference"
-
- def __repr__(self):
- return self.__str__()
-
+ return "<HDF5 dataset region reference>"
+ return "<Invalid HDF5 reference>"
diff --git a/h5py/h5t.pyx b/h5py/h5t.pyx
index 0d8928c..80a74c2 100644
--- a/h5py/h5t.pyx
+++ b/h5py/h5t.pyx
@@ -21,13 +21,14 @@
include "config.pxi"
# Pyrex compile-time imports
-from h5 cimport init_hdf5, H5PYConfig, get_config, PHIL, get_phil, get_object_type
+from h5 cimport init_hdf5, H5PYConfig, get_config, PHIL, get_phil
from h5p cimport PropID, pdefault
from numpy cimport dtype, ndarray
from python_string cimport PyString_FromStringAndSize
from utils cimport emalloc, efree, \
require_tuple, convert_dims, convert_tuple
+import _conv
# Initialization
init_hdf5()
@@ -134,6 +135,11 @@ if sys.byteorder == "little": # Custom python addition
ORDER_NATIVE = H5T_ORDER_LE
else:
ORDER_NATIVE = H5T_ORDER_BE
+
+# For conversion
+BKG_NO = H5T_BKG_NO
+BKG_TEMP = H5T_BKG_TEMP
+BKG_YES = H5T_BKG_YES
# --- Built-in HDF5 datatypes -------------------------------------------------
@@ -192,8 +198,10 @@ C_S1 = lockid(H5T_C_S1)
FORTRAN_S1 = lockid(H5T_FORTRAN_S1)
VARIABLE = H5T_VARIABLE
-# Custom Python object pointer type
-PYTHON_OBJECT = lockid(get_object_type())
+# Custom Python object pointer type
+PYTHON_OBJECT = lockid(_conv.get_python_obj())
+
+#PYTHON_OBJECT = lockid(get_object_type())
# Translation tables for HDF5 -> NumPy dtype conversion
cdef dict _order_map = { H5T_ORDER_NONE: '|', H5T_ORDER_LE: '<', H5T_ORDER_BE: '>'}
@@ -644,7 +652,14 @@ cdef class TypeReferenceID(TypeID):
"""
HDF5 object or region reference
"""
- pass
+
+ cdef object py_dtype(self):
+ if H5Tequal(self.id, H5T_STD_REF_OBJ):
+ return py_new_ref(H5R_OBJECT)
+ elif H5Tequal(self.id, H5T_STD_REF_DSETREG):
+ return py_new_ref(H5R_DATASET_REGION)
+ else:
+ raise TypeError("Unknown reference type")
# === Numeric classes (integers and floats) ===================================
@@ -1298,17 +1313,19 @@ cdef TypeCompoundID _c_compound(dtype dt, int logical):
return TypeCompoundID(tid)
-cdef TypeOpaqueID _c_object(dtype dt):
- # Object types are represented by a custom opaque type
- # Currently no other logic is required
- return PYTHON_OBJECT
-
cdef TypeStringID _c_vlen_str(object basetype):
# Variable-length strings
cdef hid_t tid
tid = H5Tcopy(H5T_C_S1)
H5Tset_size(tid, H5T_VARIABLE)
return TypeStringID(tid)
+
+cdef TypeReferenceID _c_ref(int typecode):
+ if typecode == H5R_OBJECT:
+ return STD_REF_OBJ
+ elif typecode == H5R_DATASET_REGION:
+ return STD_REF_DSETREG
+ raise TypeError("Unrecognized reference code")
cpdef TypeID py_create(object dtype_in, bint logical=0):
"""(OBJECT dtype_in, BOOL logical=False) => TypeID
@@ -1372,15 +1389,19 @@ cpdef TypeID py_create(object dtype_in, bint logical=0):
# Object types (including those with vlen hints)
elif kind == c'O':
-
+
if logical:
- # Check for vlen hints
vlen = py_get_vlen(dt)
if vlen is not None:
- return _c_vlen_str(vlen)
- raise TypeError("Object dtype has no native HDF5 equivalent")
+ return _c_vlen_str(vlen)
+
+ refcode = py_get_ref(dt)
+ if refcode is not None:
+ return _c_ref(refcode)
- return _c_object(dt)
+ raise TypeError("Object dtype %r has no native HDF5 equivalent" % (dt,))
+
+ return PYTHON_OBJECT
# Unrecognized
else:
@@ -1421,6 +1442,37 @@ cpdef dict py_get_enum(object dt):
return info_dict['vals']
return None
+
+cpdef dtype py_new_ref(int typecode):
+ """ (INT typecode) => DTYPE
+
+ Create a NumPy object type representing an HDF5 reference. The typecode
+ should be one of:
+
+ - h5r.OBJECT
+ - h5r.DATASET_REGION
+ """
+ return dtype(('O', [( ({'type': typecode},'hdf5ref'), 'O' )] ))
+
+cpdef object py_get_ref(object dt_in):
+ """ (DTYPE dt_in) => INT typecode or None
+
+ Determine what kind of reference this dtype represents. Returns None
+ if it's not a reference.
+ """
+ cdef dtype dt = dtype(dt_in)
+
+ if dt.kind != 'O':
+ return None
+
+ if dt.fields is not None and 'hdf5ref' in dt.fields:
+ tpl = dt.fields['hdf5ref']
+ if len(tpl) == 3:
+ hint_dict = tpl[2]
+ if 'type' in hint_dict:
+ return hint_dict['type']
+
+ return None
cpdef dtype py_new_vlen(object kind):
""" (OBJECT kind) => DTYPE
@@ -1454,14 +1506,45 @@ cpdef object py_get_vlen(object dt_in):
return None
-
-def _path_exists(TypeID src not None, TypeID dst not None):
- """ Undocumented and unsupported """
+def convert(TypeID src not None, TypeID dst not None, size_t n,
+ ndarray buf not None, ndarray bkg=None, PropID dxpl=None):
+ """ (TypeID src, TypeID dst, UINT n, NDARRAY buf, NDARRAY bkg=None,
+ PropID dxpl=None)
+
+ Convert n contiguous elements of a buffer in-place. The array dtype
+ is ignored. The backing buffer is optional; for conversion of compound
+ types, a temporary copy of conversion buffer will used for backing if
+ one is not supplied.
+ """
+ cdef void* bkg_ = NULL
+ cdef void* buf_ = buf.data
+
+ if bkg is None and (src.detect_class(H5T_COMPOUND) or
+ dst.detect_class(H5T_COMPOUND)):
+ bkg = buf.copy()
+ if bkg is not None:
+ bkg_ = bkg.data
+
+ H5Tconvert(src.id, dst.id, n, buf_, bkg_, pdefault(dxpl))
+
+def find(TypeID src not None, TypeID dst not None):
+ """ (TypeID src, TypeID dst) => TUPLE or None
+
+ Determine if a conversion path exists from src to dst. Result is None
+ or a tuple describing the conversion path. Currently tuple entries are:
+
+ 1. INT need_bkg: Whether this routine requires a backing buffer.
+ Values are BKG_NO, BKG_TEMP and BKG_YES.
+ """
cdef H5T_cdata_t *data
cdef H5T_conv_t result = NULL
-
- result = H5Tfind(src.id, dst.id, &data)
- return result != NULL
-
+
+ try:
+ result = H5Tfind(src.id, dst.id, &data)
+ if result == NULL:
+ return None
+ return (data[0].need_bkg,)
+ except:
+ return None
diff --git a/h5py/python.pxd b/h5py/python.pxd
deleted file mode 100644
index 767078f..0000000
--- a/h5py/python.pxd
+++ /dev/null
@@ -1,83 +0,0 @@
-#+
-#
-# This file is part of h5py, a low-level Python interface to the HDF5 library.
-#
-# Copyright (C) 2008 Andrew Collette
-# http://h5py.alfven.org
-# License: BSD (See LICENSE.txt for full license)
-#
-# $Date$
-#
-#-
-
-# This file is based on code from the PyTables project. The complete PyTables
-# license is available at licenses/pytables.txt, in the distribution root
-# directory.
-
-# Some helper routines from the Python API
-cdef extern from "Python.h":
-
- # special types
- ctypedef int Py_ssize_t
-
- # references
- void Py_INCREF(object)
- void Py_DECREF(object)
-
- # To release global interpreter lock (GIL) for threading
- void Py_BEGIN_ALLOW_THREADS()
- void Py_END_ALLOW_THREADS()
-
- # Functions for integers
- object PyInt_FromLong(long)
- long PyInt_AsLong(object)
- object PyLong_FromLongLong(long long)
- long long PyLong_AsLongLong(object)
-
- # Functions for floating points
- object PyFloat_FromDouble(double)
-
- # Functions for strings
- object PyString_FromStringAndSize(char *s, int len)
- char *PyString_AsString(object string)
- object PyString_FromString(char *)
-
- # Functions for lists
- int PyList_Append(object list, object item)
- int PyList_Check(object list_)
-
- # Functions for tuples
- object PyTuple_New(int)
- int PyTuple_SetItem(object, int, object)
- object PyTuple_GetItem(object, int)
- int PyTuple_Size(object tuple)
- int PyTuple_Check(object tpl)
-
- # Functions for dicts
- int PyDict_Contains(object p, object key)
- object PyDict_GetItem(object p, object key)
-
- # Functions for objects
- object PyObject_GetItem(object o, object key)
- int PyObject_SetItem(object o, object key, object v)
- int PyObject_DelItem(object o, object key)
- long PyObject_Length(object o)
- int PyObject_Compare(object o1, object o2)
- int PyObject_AsReadBuffer(object obj, void **buffer, Py_ssize_t *buffer_len)
-
- # Exception handling (manual)
- ctypedef extern class __builtin__.BaseException [object PyBaseExceptionObject]:
- cdef object dict
- cdef object args
- cdef object message
-
- void* PyExc_Exception # Not allowed to declare objects "extern C" (why not?)
- void PyErr_SetString(object type_, char* msg)
- void PyErr_SetNone(object type_)
- void PyErr_SetObject(object type_, object args)
- object PyErr_NewException(char* name, object base, object dict_)
-
-
-
-
-
diff --git a/h5py/tests/__init__.py b/h5py/tests/__init__.py
index 1d1b532..1d7a1b5 100644
--- a/h5py/tests/__init__.py
+++ b/h5py/tests/__init__.py
@@ -32,7 +32,8 @@ mnames = [
'test_slicing',
'test_threads',
'test_utils',
-'test_vlen']
+'test_vlen',
+'test_conv']
def runtests():
diff --git a/h5py/tests/test_conv.py b/h5py/tests/test_conv.py
new file mode 100644
index 0000000..cdf9f65
--- /dev/null
+++ b/h5py/tests/test_conv.py
@@ -0,0 +1,61 @@
+
+
+from unittest import TestCase
+import numpy as np
+
+from h5py import h5t
+import ctypes
+
+strings = ["Hi", "Hello", "This is a string", "HDF5 is awesome!"]
+vlen_dtype = h5t.py_new_vlen(str)
+vlen_htype = h5t.py_create(vlen_dtype, logical=1)
+obj_htype = h5t.py_create(vlen_dtype)
+
+class TestVlenObject(TestCase):
+
+ """
+ Test conversion routines between string vlens and object pointers
+ """
+
+ def test_obj2vlen_simple(self):
+ """ Object to vlen (contiguous) """
+
+ objarr = np.array(strings, dtype=vlen_dtype)
+
+ destbuffer = np.ndarray(objarr.shape, dtype=np.uintp, buffer=objarr).copy()
+
+ h5t.convert(obj_htype, vlen_htype, len(strings), destbuffer)
+
+ for idx, val in enumerate(destbuffer):
+ self.assertEqual(ctypes.string_at(int(val)), strings[idx])
+
+ def test_obj2vlen_complex(self):
+ """ Object to vlen (compound) """
+
+ obj_ptr_size = h5t.PYTHON_OBJECT.get_size()
+ vlen_ptr_size = vlen_htype.get_size()
+
+ input_htype = h5t.create(h5t.COMPOUND, obj_ptr_size+4)
+ input_htype.insert('a', 0, obj_htype)
+ input_htype.insert('b', obj_ptr_size, h5t.STD_I32LE)
+
+ output_htype = h5t.create(h5t.COMPOUND, vlen_ptr_size+4)
+ output_htype.insert('a', 0, vlen_htype)
+ output_htype.insert('b', vlen_ptr_size, h5t.STD_I32LE)
+
+ objarr = np.ndarray((len(strings),), dtype=[('a', vlen_dtype), ('b', '<i4')])
+ objarr['a'] = strings
+
+ destbuffer = np.ndarray(objarr.shape, dtype=[('a', np.uintp), ('b', '<i4')], buffer=objarr).copy()
+
+ h5t.convert(input_htype, output_htype, len(strings), destbuffer, destbuffer.copy())
+
+ for idx, val in enumerate(destbuffer):
+ self.assertEqual(ctypes.string_at(int(val[0])), strings[idx])
+
+
+
+
+
+
+
diff --git a/h5py/tests/test_h5r.py b/h5py/tests/test_h5r.py
index 1604162..b914c80 100644
--- a/h5py/tests/test_h5r.py
+++ b/h5py/tests/test_h5r.py
@@ -32,10 +32,10 @@ class TestH5R(unittest.TestCase):
def test_objref(self):
ref = h5r.create(self.f.id, "grp", h5r.OBJECT)
- deref = ref.dereference(self.f.id)
+ deref = h5r.dereference(ref, self.f.id)
self.assertEqual(deref, self.grp.id)
#self.assertEqual(h5i.get_name(deref), h5i.get_name(self.grp.id))
- self.assertEqual(ref.get_obj_type(self.f.id), h5g.GROUP)
+ self.assertEqual(h5r.get_obj_type(ref, self.f.id), h5g.GROUP)
def test_regref(self):
space = self.dset.id.get_space()
@@ -46,11 +46,11 @@ class TestH5R(unittest.TestCase):
#self.assertEqual(deref, self.grp.id)
#self.assertEqual(h5i.get_name(deref), h5i.get_name(self.grp.id))
- deref_space = ref.get_region(self.f.id)
+ deref_space = h5r.get_region(ref, self.f.id)
self.assertEqual(space.shape, deref_space.shape)
self.assert_(space.get_select_bounds(), deref_space.get_select_bounds())
- self.assertEqual(ref.get_obj_type(self.f.id), h5g.DATASET)
+ self.assertEqual(h5r.get_obj_type(ref, self.f.id), h5g.DATASET)
diff --git a/h5py/typeconv.c b/h5py/typeconv.c
deleted file mode 100644
index f9af6e2..0000000
--- a/h5py/typeconv.c
+++ /dev/null
@@ -1,529 +0,0 @@
-/***** Preamble block *********************************************************
-*
-* This file is part of h5py, a low-level Python interface to the HDF5 library.
-*
-* Copyright (C) 2008 Andrew Collette
-* http://h5py.alfven.org
-* License: BSD (See LICENSE.txt for full license)
-*
-* $Date$
-*
-****** End preamble block ****************************************************/
-
-#include <stdio.h>
-#include <string.h>
-#include "Python.h"
-#include "hdf5.h"
-#include "typeconv.h"
-
-hid_t _H5PY_OBJ = 0;
-
-/* Return the (locked) canonical type corresponding to a Python object
- pointer in memory. This is an ephemeral type; it should never be stored
- in a file. */
-hid_t h5py_object_type(void){
- if(_H5PY_OBJ == 0){
- _H5PY_OBJ = H5Tcreate(H5T_OPAQUE, sizeof(PyObject*));
- H5Tset_tag(_H5PY_OBJ, "PYTHON:OBJECT");
- H5Tlock(_H5PY_OBJ);
- }
- return _H5PY_OBJ;
-}
-
-
-/* === Type-conversion callbacks & support === */
-
-/* Check types for Python string/vlen conversion */
-htri_t h5py_match_str_ptr(hid_t str, hid_t pyptr){
-
- htri_t is_var_str = 0;
- htri_t is_pyptr = 0;
- char* tagval;
-
- is_var_str = H5Tis_variable_str(str);
- if(is_var_str<0) goto failed;
-
- tagval = H5Tget_tag(pyptr);
- if(tagval != NULL){
- is_pyptr = !strcmp(tagval, "PYTHON:OBJECT");
- }
- free(tagval);
-
- return is_var_str && is_pyptr;
-
- failed: /* Error target */
-
- return -1;
-}
-
-typedef struct {
- size_t src_size;
- size_t dst_size;
-} conv_size_t;
-
-/* Convert from HDF5 variable-length strings to Python string objects.
-*/
-herr_t vlen_to_str(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata,
- size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i,
- void *bkg_i, hid_t dxpl){
-
- PyGILState_STATE gil;
-
- char** str;
- PyObject** obj;
- PyObject** obj_bkg;
- PyObject* obj_tmp;
-
- char *buf;
- char *bkg;
-
- conv_size_t *sizes = NULL;
-
- herr_t retval = -1;
- int i;
-
- buf = (char*)buf_i;
- bkg = (char*)bkg_i;
-
- switch(cdata->command){
-
- /* Determine if we can convert between src_id and dst_id; return 0 if
- possible, -1 otherwise */
- case H5T_CONV_INIT:
-
- /* Only accept the case of vlen H5T_STRING to Python string */
- if(h5py_match_str_ptr(src_id, dst_id) <= 0) goto init_failed;
-
- cdata->need_bkg = H5T_BKG_YES;
- if((cdata->priv = sizes = (conv_size_t*)malloc(sizeof(conv_size_t))) == NULL) goto init_failed;
-
- if((sizes->src_size = H5Tget_size(src_id)) == 0) goto init_failed;
- if((sizes->dst_size = H5Tget_size(dst_id)) == 0) goto init_failed;
-
- return 0;
-
- init_failed: /* Error target */
- free(sizes);
- return -1;
-
- case H5T_CONV_CONV:
-
- gil = PyGILState_Ensure();
-
- sizes = (conv_size_t*)(cdata->priv);
-
- if(buf_stride==0) buf_stride = sizes->src_size;
- if(bkg_stride==0) bkg_stride = sizes->dst_size;
-
- for(i=0;i<nl;i++){
-
- obj = (PyObject**)(buf+(i*buf_stride));
- str = (char**)(buf+(i*buf_stride));
- obj_bkg = (PyObject**)(bkg+(i*bkg_stride));
-
- if((*str)==NULL){
- obj_tmp = PyString_FromString("");
- } else {
- obj_tmp = PyString_FromString(*str);
- }
- if(obj_tmp==NULL) goto conv_failed;
-
- /* Since all data conversions are by convention in-place, it
- is our responsibility to free the memory used by the vlens. */
- free(*str);
-
- Py_XDECREF(*obj_bkg);
- *obj = obj_tmp;
- }
-
- PyGILState_Release(gil);
- return 0;
-
- conv_failed: /* Error target */
-
- PyGILState_Release(gil);
- return -1;
-
- case H5T_CONV_FREE:
-
- free(cdata->priv);
- return 0;
-
- default:
-
- return -1;
- }
-}
-
-
-/* Convert from Python strings to HDF5 vlens.
-*/
-herr_t str_to_vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata,
- size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i,
- void *bkg_i, hid_t dset_xfer_plist){
-
- PyGILState_STATE gil;
-
- PyObject** obj;
- PyObject* inter_string = NULL;
- char** str;
- char* str_tmp;
- Py_ssize_t len;
-
- conv_size_t *sizes = NULL;
-
- herr_t retval = -1;
- int i;
-
- char* buf = (char*)buf_i;
- char* bkg = (char*)bkg_i;
-
- switch(cdata->command){
-
- case H5T_CONV_INIT:
-
- /* Only accept Python string -> HDF5 vlen */
- if(h5py_match_str_ptr(dst_id, src_id) <= 0) goto init_failed;
-
- cdata->need_bkg = H5T_BKG_NO;
- if((cdata->priv = sizes = (conv_size_t*)malloc(sizeof(conv_size_t))) == NULL) goto init_failed;
-
- if((sizes->src_size = H5Tget_size(src_id)) == 0) goto init_failed;
- if((sizes->dst_size = H5Tget_size(dst_id)) == 0) goto init_failed;
-
- return 0;
-
- init_failed: /* Error target */
- free(sizes);
- return -1;
-
- case H5T_CONV_CONV:
-
-
- gil = PyGILState_Ensure();
- sizes = (conv_size_t*)(cdata->priv);
-
- if(buf_stride==0) buf_stride = sizes->src_size;
-
- for(i=0;i<nl;i++){
-
- obj = (PyObject**)(buf+(i*buf_stride));
- str = (char**)(buf+(i*buf_stride));
-
- if(*obj == NULL || *obj == Py_None){
- len = 1;
- str_tmp = "";
-
- } else { /* If it's not a string, take the result of str(obj) */
-
- if(PyString_CheckExact(*obj)) {
- len = PyString_Size(*obj)+1;
- str_tmp = PyString_AsString(*obj);
- } else {
- inter_string = PyObject_Str(*obj);
- if(inter_string == NULL) goto conv_failed;
- len = PyString_Size(inter_string)+1;
- str_tmp = PyString_AsString(inter_string);
- }
-
- }
-
- *str = (char*)malloc(len); /* len already includes null term */
- memcpy(*str, str_tmp, len);
-
- }
-
- retval = 0;
-
- conv_out:
-
- /* Note we do NOT decref obj, as it is a borrowed reference */
- Py_XDECREF(inter_string);
- PyGILState_Release(gil);
- return retval;
-
- conv_failed: /* Error target */
- retval = -1;
- goto conv_out;
-
- case H5T_CONV_FREE:
-
- free(cdata->priv);
- return 0;
-
- default:
-
- return -1;
- }
-
-}
-
-typedef struct {
- size_t src_size;
- size_t dst_size;
- int vlen_to_fixed;
-} h5py_vlfix_conv_t;
-
-/* Convert back & forth between fixed and vlen strings. When converting from
- vlen to fixed, if the string is shorted, the space will be padded with
- nulls; when longer, it will simply be truncated with no null termination.
- */
-herr_t vlen_fixed(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata,
- size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i,
- void *bkg_i, hid_t dset_xfer_plist){
-
- htri_t svlen, dvlen;
- h5py_vlfix_conv_t *info = NULL;
- char* buf_ptr;
- char* str_tmp;
- size_t str_tmp_len;
-
- int i, start, stop, incr;
-
- char* buf = (char*)buf_i;
- char* bkg = (char*)bkg_i;
-
- switch(cdata->command){
-
- case H5T_CONV_INIT:
-
- if((svlen = H5Tis_variable_str(src_id)) < 0) goto init_failed;
- if((dvlen = H5Tis_variable_str(dst_id)) < 0) goto init_failed;
-
- /* Exactly one must be variable-length */
- if((svlen && dvlen) || !(svlen || dvlen)) goto init_failed;
-
- if((cdata->priv = info = malloc(sizeof(h5py_vlfix_conv_t))) == NULL) goto init_failed;
-
- if((info->src_size = H5Tget_size(src_id)) < 0) goto init_failed;
- if((info->dst_size = H5Tget_size(dst_id)) < 0) goto init_failed;
-
- info->vlen_to_fixed = svlen;
-
- return 0;
-
- init_failed:
- free(info);
- return -1;
-
- case H5T_CONV_CONV:
-
- info = (h5py_vlfix_conv_t*)(cdata->priv);
-
- if(buf_stride==0) buf_stride = info->src_size;
-
- if(info->src_size >= info->dst_size){
- start = 0;
- stop = nl;
- incr = 1;
- } else {
- start = nl-1;
- stop = -1;
- incr = -1;
- }
-
- if(info->vlen_to_fixed){
-
- for(i=start; i!=stop; i+=incr){
- buf_ptr = buf + (i*buf_stride);
- str_tmp = *((char**)buf_ptr);
- str_tmp_len = strlen(str_tmp);
- if(str_tmp_len <= info->dst_size){
- memcpy(buf_ptr, str_tmp, str_tmp_len);
- memset(buf_ptr + str_tmp_len, info->dst_size - str_tmp_len, '\0');
- } else {
- memcpy(buf_ptr, str_tmp, info->dst_size);
- }
- free(str_tmp);
- }
-
- } else {
-
- for(i=start; i!=stop; i+=incr){
- buf_ptr = buf + (i*buf_stride);
- if((str_tmp = (char*)malloc(info->src_size + 1))==NULL) goto conv_failed;
- memcpy(str_tmp, buf_ptr, info->src_size);
- str_tmp[info->src_size] = '\0';
- *((char**)buf_ptr) = str_tmp;
- }
-
- }
-
- return 0;
-
- conv_failed:
- return -1;
-
- case H5T_CONV_FREE:
-
- return 0;
-
- default:
-
- return -1;
- }
-}
-
-/* Convert back & forth between enums and ints */
-
-typedef struct {
- H5T_class_t src_cls;
- size_t src_size;
- size_t dst_size;
- hid_t int_src_id; /* Integer type appropriate for source */
- hid_t int_dst_id; /* Integer type appropriate for destination */
- int identical; /* Tells if the above types are the same */
-} h5py_enum_conv_t;
-
-/* This function is registered on both paths ENUM -> INT and INT -> ENUM */
-herr_t enum_int(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata,
- size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i,
- void *bkg_i, hid_t dset_xfer_plist){
-
- h5py_enum_conv_t* info = NULL;
- hid_t conv_src_id, conv_dst_id;
-
- char* conv_buf = NULL;
- size_t nalloc;
-
- herr_t cresult;
- int i;
-
- char* buf = (char*)buf_i;
- char* bkg = (char*)bkg_i;
-
- switch(cdata->command){
-
- case H5T_CONV_INIT:
-
- cdata->need_bkg = H5T_BKG_NO;
- if((cdata->priv = info = (h5py_enum_conv_t*)malloc(sizeof(h5py_enum_conv_t))) == NULL) goto init_failed;
-
- info->int_src_id = 0;
- info->int_dst_id = 0;
-
- if((info->src_cls = H5Tget_class(src_id)) < 0) goto init_failed;
-
- if((info->src_size = H5Tget_size(src_id)) == 0) goto init_failed;
- if((info->dst_size = H5Tget_size(dst_id)) == 0) goto init_failed;
-
- if(info->src_cls == H5T_ENUM){
- /* We're trying to convert an ENUM to an INT */
- info->int_src_id = H5Tget_super(src_id);
- info->int_dst_id = dst_id;
- if(H5Iinc_ref(dst_id) < 0) goto init_failed;
- } else {
- /* We're trying to convert an INT to an ENUM */
- info->int_src_id = src_id;
- info->int_dst_id = H5Tget_super(dst_id);
- if(H5Iinc_ref(src_id) < 0) goto init_failed;
- }
- if(info->int_src_id<0) goto init_failed;
- if(info->int_dst_id<0) goto init_failed;
-
- if((info->identical = H5Tequal(info->int_src_id, info->int_dst_id)) < 0) goto init_failed;
-
- return 0;
-
- init_failed:
-
- if(info!=NULL){
- if(info->int_src_id>0) H5Idec_ref(info->int_src_id);
- if(info->int_dst_id>0) H5Idec_ref(info->int_dst_id);
- }
- free(info);
-
- return -1;
-
- case H5T_CONV_CONV:
-
- info = (h5py_enum_conv_t*)(cdata->priv);
-
- /* Shortcut */
- if(info->identical) return 0;
-
- if(buf_stride==0){
- /* Contiguous data: H5Tconvert can do this directly */
-
- if(H5Tconvert(info->int_src_id, info->int_dst_id,
- nl, buf, NULL, dset_xfer_plist) < 0) goto conv_failed;
-
- } else {
- /* Can't tell H5Tconvert about strides; use a buffer */
-
- if( (info->src_size) > (info->dst_size)){
- nalloc = (info->src_size)*nl;
- } else {
- nalloc = (info->dst_size)*nl;
- }
- if((conv_buf = malloc(nalloc)) == NULL) goto conv_failed;
-
- /* Copy into temp buffer */
- for(i=0;i<nl;i++){
- memcpy(conv_buf+(i*(info->src_size)), buf+(i*buf_stride),
- info->src_size);
- }
-
- /* Convert in-place */
- if(H5Tconvert(info->int_src_id, info->int_dst_id,
- nl, conv_buf, NULL, dset_xfer_plist) < 0) goto conv_failed;
-
- /* Copy back out to source buffer. Remember these elements
- are now of size info->dst_size. */
- for(i=0;i<nl;i++){
- memcpy(buf+(i*buf_stride), conv_buf+(i*(info->dst_size)),
- info->dst_size);
- }
-
- } /* if ... else */
-
- free(conv_buf);
- return 0;
-
- conv_failed:
- free(conv_buf);
- return -1;
-
- case H5T_CONV_FREE:
-
- /* Segfault on cleanup; something's wrong with cdata->priv */
- return 0;
-
- default:
-
- return -1;
-
- } /* case */
-
-}
-
-int h5py_register_conv(void){
-
- hid_t h5py_enum = H5Tenum_create(H5T_NATIVE_INT);
- hid_t h5py_obj = h5py_object_type();
- hid_t vlen_str = H5Tcopy(H5T_C_S1);
- H5Tset_size(vlen_str, H5T_VARIABLE);
-
- /* "Soft" registration means the conversion is tested for any two types
- which match the given classes (in this case H5T_STRING and H55_OPAQUE) */
- H5Tregister(H5T_PERS_SOFT, "vlen_to_str", vlen_str, h5py_obj, vlen_to_str);
- H5Tregister(H5T_PERS_SOFT, "str_to_vlen", h5py_obj, vlen_str, str_to_vlen);
-
- H5Tregister(H5T_PERS_SOFT, "enum to int", h5py_enum, H5T_NATIVE_INT, enum_int);
- H5Tregister(H5T_PERS_SOFT, "int to enum", H5T_NATIVE_INT, h5py_enum, enum_int);
-
- H5Tregister(H5T_PERS_SOFT, "fix to vlen", H5T_C_S1, vlen_str, vlen_fixed);
- H5Tregister(H5T_PERS_SOFT, "vlen to fix", vlen_str, H5T_C_S1, vlen_fixed);
-
- H5Tclose(vlen_str);
-
- return 0;
-}
-
-
-
-
-
-
-
-
diff --git a/h5py/typeconv.h b/h5py/typeconv.h
deleted file mode 100644
index 90b78c0..0000000
--- a/h5py/typeconv.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/***** Preamble block *********************************************************
-*
-* This file is part of h5py, a low-level Python interface to the HDF5 library.
-*
-* Copyright (C) 2008 Andrew Collette
-* http://h5py.alfven.org
-* License: BSD (See LICENSE.txt for full license)
-*
-* $Date$
-*
-****** End preamble block ****************************************************/
-
-/*
- Implements low-level infrastructure for vlen and enum types
-*/
-
-#include "hdf5.h"
-
-#ifndef H5PY_TYPECONV_H
-#define H5PY_TYPECONV_H
-
-/* Register all new conversion functions */
-int h5py_register_conv(void);
-
-/* Return the canonical Python object pointer type */
-hid_t h5py_object_type(void);
-
-#endif
diff --git a/h5py/typeproxy.c b/h5py/typeproxy.c
deleted file mode 100644
index 305c1a4..0000000
--- a/h5py/typeproxy.c
+++ /dev/null
@@ -1,450 +0,0 @@
-/***** Preamble block *********************************************************
-*
-* This file is part of h5py, a low-level Python interface to the HDF5 library.
-*
-* Copyright (C) 2008 Andrew Collette
-* http://h5py.alfven.org
-* License: BSD (See LICENSE.txt for full license)
-*
-* $Date$
-*
-****** End preamble block ****************************************************/
-
-/*
- Contains proxy functions for reading and writing data from datasets and
- attributes. Importantly, these functions implement the proper workarounds
- required for variable-length type support, as implemented in typeconv.c.
-*/
-
-#include <string.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include "hdf5.h"
-#include "typeproxy.h"
-
-/* ------- Private function prototypes ------ */
-
-herr_t h5py_resolve_spaces(hid_t dset_id, hid_t ifspace, hid_t imspace,
- hid_t *ofspace, hid_t *omspace);
-
-void* h5py_setup_buffer(hid_t itype, hid_t otype, hid_t space_id, hsize_t* nl);
-
-htri_t h5py_detect_vlen(hid_t type_id);
-
-
-/* ------- Attribute read/write support ------- */
-
-herr_t H5PY_attr_rw(hid_t attr, hid_t mtype, void *buf, h5py_rw_t dir){
-
- hid_t atype = 0; /* Attribute data type */
- hid_t aspace = 0; /* Attribute data space */
-
- htri_t vlen_present;
- herr_t status; /* API function result */
- herr_t retval; /* Return value for this function */
-
- hsize_t mtype_size;
- hsize_t nl;
- void* bkg_buf = NULL;
- void* conv_buf = NULL;
-
- atype = H5Aget_type(attr);
- if(atype<0) goto failed;
-
- vlen_present = h5py_detect_vlen(atype);
- if(vlen_present<0) goto failed;
-
- if(!vlen_present){
- /* Direct read/write */
-
- switch(dir){
- case H5PY_READ:
- status = H5Aread(attr, mtype, buf);
- break;
- case H5PY_WRITE:
- status = H5Awrite(attr, mtype, buf);
- break;
- default:
- goto failed;
- }
- if(status<0) goto failed;
-
- } else {
- /* Buffered read/write */
-
- aspace = H5Aget_space(attr);
- if(aspace<0) goto failed;
-
- conv_buf = h5py_setup_buffer(atype, mtype, aspace, &nl);
- if(conv_buf==NULL) goto failed;
-
- mtype_size = H5Tget_size(mtype);
- if(mtype_size==0) goto failed;
-
- bkg_buf = malloc(mtype_size*nl);
- if(bkg_buf==NULL) goto failed;
-
- memcpy(bkg_buf, buf, mtype_size*nl);
-
- switch(dir){
-
- case H5PY_READ:
- status = H5Aread(attr, atype, conv_buf);
- if(status<0) goto failed;
- status = H5Tconvert(atype, mtype, nl, conv_buf, bkg_buf, H5P_DEFAULT);
- if(status<0) goto failed;
- memcpy(buf, conv_buf, mtype_size*nl);
- break;
-
- case H5PY_WRITE:
- memcpy(conv_buf, buf, mtype_size*nl);
- status = H5Tconvert(mtype, atype, nl, conv_buf, bkg_buf, H5P_DEFAULT);
- if(status<0) goto failed;
- status = H5Awrite(attr, atype, conv_buf);
- if(status<0) goto failed;
- break;
-
- default:
- goto failed;
- }
- }
-
- retval = 0;
-
- out: /* Cleanup */
-
- free(bkg_buf);
- free(conv_buf);
- if(atype>0) H5Tclose(atype);
- if(aspace>0) H5Sclose(aspace);
-
- return retval;
-
- failed: /* Error target */
-
- retval = -1;
- goto out;
-
-}
-
-
-
-/* H5PY_dset_rw
-
- Read & write datasets with proxy support for vlen bug. "Direction"
- determines whether to read or write data.
-*/
-herr_t H5PY_dset_rw(hid_t dset, hid_t mtype, hid_t mspace_in, hid_t fspace_in,
- hid_t xfer_plist, void* buf, h5py_rw_t dir){
-
- hid_t dstype = 0;
- hid_t mspace = 0, fspace =0;
- htri_t vlen_present;
- herr_t status; /* Status flag for API calls */
- herr_t retval; /* Return value for this function */
-
- hsize_t nl; /* Number of elements for read/write */
- size_t mtype_size;
- hid_t cspace = 0; /* Dataspace for conversion buffer */
- void* conv_buf = NULL; /* Conversion buffer */
- void* bkg_buf = NULL; /* Backing buffer */
-
-
- dstype = H5Dget_type(dset);
- if(dstype<0) goto failed;
-
- vlen_present = h5py_detect_vlen(dstype);
- if(vlen_present<0) goto failed;
-
- if(!vlen_present){
- /* Standard read/write */
-
- switch(dir){
- case H5PY_READ:
- status = H5Dread(dset, mtype, mspace_in, fspace_in, xfer_plist, buf);
- break;
- case H5PY_WRITE:
- status = H5Dwrite(dset, mtype, mspace_in, fspace_in, xfer_plist, buf);
- break;
- default:
- goto failed;
- }
- if(status<0) goto failed;
-
- } else {
- /* Buffered read/write */
-
- status = h5py_resolve_spaces(dset, fspace_in, mspace_in, &fspace, &mspace);
- if(status<0) goto failed;
-
- conv_buf = h5py_setup_buffer(dstype, mtype, fspace, &nl);
- if(conv_buf==NULL) goto failed;
-
- cspace = H5Screate_simple(1, &nl, NULL);
- if(cspace<0) goto failed;
-
- /* Populate the backing buffer with in-memory data */
- /* TODO: skip unless (1) reading (any type), or (2) writing compound */
- mtype_size = H5Tget_size(mtype);
- if(mtype_size==0) goto failed;
-
- bkg_buf = malloc(mtype_size*nl);
-
- status = h5py_copy(mtype, mspace, bkg_buf, buf, H5PY_GATHER);
- if(status<0) goto failed;
-
- switch(dir){
-
- case H5PY_READ:
- status = H5Dread(dset, dstype, cspace, fspace, xfer_plist, conv_buf);
- if(status<0) goto failed;
- status = H5Tconvert(dstype, mtype, nl, conv_buf, bkg_buf, xfer_plist);
- if(status<0) goto failed;
- status = h5py_copy(mtype, mspace, conv_buf, buf, H5PY_SCATTER);
- if(status<0) goto failed;
- break;
-
- case H5PY_WRITE:
- status = h5py_copy(mtype, mspace, conv_buf, buf, H5PY_GATHER);
- if(status<0) goto failed;
- status = H5Tconvert(mtype, dstype, nl, conv_buf, bkg_buf, xfer_plist);
- if(status<0) goto failed;
- status = H5Dwrite(dset, dstype, cspace, fspace, xfer_plist, conv_buf);
- if(status<0) goto failed;
- break;
-
- default:
- goto failed;
- }
-
- }
-
- retval = 0;
-
- out: /* Cleanup */
-
- free(conv_buf);
- free(bkg_buf);
-
- if(dstype>0) H5Tclose(dstype);
- if(fspace>0) H5Sclose(fspace);
- if(mspace>0) H5Sclose(mspace);
- if(cspace>0) H5Sclose(cspace);
-
- return retval;
-
- failed: /* Error target */
-
- retval = -1;
- goto out;
-
-}
-
-/* ------- Support functions ------- */
-
-
-/* Normalize a pair of file and memory dataspaces to get rid of H5S_ALL's.
- The new dataspaces returned via ofspace and omspace must be closed. */
-herr_t h5py_resolve_spaces(hid_t dset_id, hid_t ifspace, hid_t imspace,
- hid_t *ofspace, hid_t *omspace){
-
- hid_t of_tmp, om_tmp;
-
- if(ifspace==H5S_ALL){
- of_tmp = H5Dget_space(dset_id);
- } else {
- of_tmp = H5Scopy(ifspace);
- }
- if(of_tmp<0) goto failed;
-
- if(imspace==H5S_ALL){
- om_tmp = H5Scopy(of_tmp);
- } else {
- om_tmp = H5Scopy(imspace);
- }
- if(om_tmp<0) goto failed;
-
- *ofspace = of_tmp;
- *omspace = om_tmp;
-
- return 0;
-
- failed:
-
- return -1;
-}
-
-void* h5py_setup_buffer(hid_t itype, hid_t otype, hid_t space_id, hsize_t* nl){
-
- void* buf = NULL;
- size_t isize, osize, buflen;
- hssize_t nelements;
-
- isize = H5Tget_size(itype);
- if(isize==0) goto failed;
-
- osize = H5Tget_size(otype);
- if(osize==0) goto failed;
-
- if(isize>osize){
- buflen = isize;
- } else {
- buflen = osize;
- }
-
- nelements = H5Sget_select_npoints(space_id);
- if(nelements<0) goto failed;
-
- buf = malloc(nelements*buflen);
- if(buf==NULL) goto failed;
-
- *nl = nelements;
- return buf;
-
- failed:
-
- free(buf);
- return NULL;
-
-}
-
-
-/*
- Determine if a type is variable-length (H5T_STRING or H5T_VLEN) or in the
- case of compound or array types, contains one.
-*/
-htri_t h5py_detect_vlen(hid_t type_id){
-
- H5T_class_t typeclass;
- htri_t retval;
-
- htri_t is_vlen;
- hid_t stype=0;
- int nmembers;
- int i;
-
- typeclass = H5Tget_class(type_id);
- if(typeclass<0) return -1;
-
- switch(typeclass){
-
- case H5T_STRING:
- return H5Tis_variable_str(type_id);
-
- case H5T_VLEN:
- return 1;
-
- case H5T_ARRAY:
- stype = H5Tget_super(type_id);
- if(stype<0){
- return -1;
- }
- retval = h5py_detect_vlen(stype);
- H5Tclose(stype);
- return retval;
-
- case H5T_COMPOUND:
- nmembers = H5Tget_nmembers(type_id);
- if(nmembers<0){
- return -1;
- }
- for(i=0;i<nmembers;i++){
- stype = H5Tget_member_type(type_id, i);
- if(stype<0){
- return -1;
- }
- retval = h5py_detect_vlen(stype);
- if(retval!=0){ /* short-circuit success */
- H5Tclose(stype);
- return retval;
- }
- H5Tclose(stype);
- }
- return 0;
-
- default:
- return 0;
-
- } /* switch */
-
-}
-
-
-/* ------ Implements buffer-to-buffer scatter/gather operations ------- */
-
-typedef struct {
- size_t i;
- size_t el_size;
- void* buf;
-} h5py_scatter_t;
-
-herr_t h5py_scatter_cb(void* elem, hid_t type_id, unsigned ndim,
- const hsize_t *point, void *operator_data){
-
- h5py_scatter_t* info = (h5py_scatter_t*)operator_data;
-
- memcpy(elem, ((char*)info->buf)+((info->i)*(info->el_size)), info->el_size);
-
- info->i++;
-
- return 0;
-}
-
-herr_t h5py_gather_cb(void* elem, hid_t type_id, unsigned ndim,
- const hsize_t *point, void *operator_data){
-
- h5py_scatter_t* info = (h5py_scatter_t*)operator_data;
-
- memcpy(((char*)info->buf)+((info->i)*(info->el_size)), elem, info->el_size);
-
- info->i++;
-
- return 0;
-}
-
-herr_t h5py_copy(hid_t type_id, hid_t space_id, void* contig_buf,
- void* scatter_buf, h5py_copy_t op){
-
- size_t el_size;
- hssize_t nl;
- herr_t call_result;
-
- h5py_scatter_t info;
- H5D_operator_t cb;
-
- el_size = H5Tget_size(type_id);
- if(el_size==0) goto failed;
-
- nl = H5Sget_select_npoints(space_id);
- if(nl<0) goto failed;
-
- info.i = 0;
- info.el_size = el_size;
- info.buf = contig_buf;
-
- switch(op){
- case H5PY_SCATTER:
- cb = h5py_scatter_cb;
- break;
- case H5PY_GATHER:
- cb = h5py_gather_cb;
- break;
- default:
- goto failed;
- }
-
- call_result = H5Diterate(scatter_buf, type_id, space_id, cb, &info);
- if(call_result<0) goto failed;
-
- return 0;
-
- failed:
-
- return -1;
-}
-
-
-
-
-
-
diff --git a/h5py/typeproxy.h b/h5py/typeproxy.h
deleted file mode 100644
index f2b7c7b..0000000
--- a/h5py/typeproxy.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/***** Preamble block *********************************************************
-*
-* This file is part of h5py, a low-level Python interface to the HDF5 library.
-*
-* Copyright (C) 2008 Andrew Collette
-* http://h5py.alfven.org
-* License: BSD (See LICENSE.txt for full license)
-*
-* $Date$
-*
-****** End preamble block ****************************************************/
-
-/*
- Conversion routines designed to support the use of variable-length,
- reference and other types which suffer from the HDF5 type-conversion
- bug.
-*/
-
-#include "hdf5.h"
-
-#ifndef H5PY_TYPEPROXY_H
-#define H5PY_TYPEPROXY_H
-
-/* Proxy functions for reading and writing datasets and attributes */
-
-typedef enum {
- H5PY_WRITE = 0,
- H5PY_READ
-} h5py_rw_t;
-
-herr_t H5PY_dset_rw(hid_t dset, hid_t mtype, hid_t mspace_in, hid_t fspace_in,
- hid_t xfer_plist, void* buf, h5py_rw_t dir);
-
-herr_t H5PY_attr_rw(hid_t attr, hid_t mtype, void* buf, h5py_rw_t dir);
-
-
-/* Copy data back & forth between a contiguous buffer and a dataspace
- selection. The dataspace must be a "real" dataspace; the value
- H5S_ALL is not acceptable. */
-
-typedef enum {
- H5PY_SCATTER = 0,
- H5PY_GATHER
-} h5py_copy_t;
-
-herr_t h5py_copy(hid_t type_id, hid_t space_id, void* contig_buf,
- void* scatter_buf, h5py_copy_t op);
-
-
-#endif
-
-
-
-
-
diff --git a/setup.py b/setup.py
index 9013c90..678350b 100644
--- a/setup.py
+++ b/setup.py
@@ -57,13 +57,11 @@ def localpath(*args):
return op.abspath(reduce(op.join, (op.dirname(__file__),)+args))
MODULES = ['h5', 'h5e', 'h5f', 'h5g', 'h5s', 'h5t', 'h5d', 'h5a', 'h5p', 'h5z',
- 'h5i', 'h5r', 'h5fd', 'utils', 'h5o', 'h5l']
+ 'h5i', 'h5r', 'h5fd', 'utils', 'h5o', 'h5l', '_conv', '_proxy']
EXTRA_SRC = {'h5': [ localpath("lzf/lzf_filter.c"),
localpath("lzf/lzf/lzf_c.c"),
- localpath("lzf/lzf/lzf_d.c"),
- localpath("h5py/typeproxy.c"),
- localpath("h5py/typeconv.c") ]}
+ localpath("lzf/lzf/lzf_d.c")]}
# --- Imports -----------------------------------------------------------------
@@ -435,7 +433,7 @@ class hbuild_ext(build_ext):
else:
debug("Copying %s -> %s" % (src, dst))
shutil.copy(src, dst)
- self.force = True # If any files are out of date, we need to
+ #self.force = True # If any files are out of date, we need to
# recompile the whole thing for consistency
build_ext.run(self)
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/h5py.git
More information about the debian-science-commits
mailing list