[h5py] 221/455: LZF updates, IPython completer plugin
Ghislain Vaillant
ghisvail-guest at moszumanska.debian.org
Thu Jul 2 18:19:35 UTC 2015
This is an automated email from the git hooks/post-receive script.
ghisvail-guest pushed a commit to annotated tag 1.3.0
in repository h5py.
commit 30875afd1d7f62069b03c099e3d3ad8e86e30783
Author: andrewcollette <andrew.collette at gmail.com>
Date: Sat Feb 7 02:24:47 2009 +0000
LZF updates, IPython completer plugin
---
h5py/__init__.py | 8 +++
h5py/_ipy_completer.py | 141 +++++++++++++++++++++++++++++++++++++++++++++++++
h5py/filters.py | 2 +
lzf/example.c | 34 +++++++++---
lzf/lzf_filter.c | 118 ++++++++++++++++++++++++-----------------
lzf/lzf_filter.h | 8 ++-
6 files changed, 252 insertions(+), 59 deletions(-)
diff --git a/h5py/__init__.py b/h5py/__init__.py
index 2936154..3990e4e 100644
--- a/h5py/__init__.py
+++ b/h5py/__init__.py
@@ -44,3 +44,11 @@ if version.api_version_tuple >= (1,8):
import h5o, h5l
__all__ += ['h5l', 'h5o']
+try:
+ import IPython as _IP
+ if _IP.ipapi.get() is not None:
+ import _ipy_completer
+ _ipy_completer.activate()
+except Exception:
+ pass
+
diff --git a/h5py/_ipy_completer.py b/h5py/_ipy_completer.py
new file mode 100644
index 0000000..dc90095
--- /dev/null
+++ b/h5py/_ipy_completer.py
@@ -0,0 +1,141 @@
+#+
+#
+# This file is part of h5py, a low-level Python interface to the HDF5 library.
+#
+# Contributed by Darren Dale
+#
+# Copyright (C) 2009 Darren Dale
+#
+# http://h5py.alfven.org
+# License: BSD (See LICENSE.txt for full license)
+#
+#-
+
+"""
+h5py completer extension for ipython. Drop this file in your ~/.ipython direcory
+and add the following lines to ~/.ipython/ipy_user_conf.py main()::
+
+ from ipy_h5py_completer.py import activate
+ activate()
+
+It will let you do things like::
+
+ f=File('foo.h5')
+ f['<tab>
+ # or:
+ f['ite<tab>
+
+which will do tab completion based on the subgroups of `f`. Also::
+
+ f['item1'].at<tab>
+
+will perform tab completion for the attributes in the usual way. This should
+also work::
+
+ a = b = f['item1'].attrs.<tab>
+
+as should::
+
+ f['item1/item2/it<tab>
+
+"""
+
+import posixpath
+import re
+
+from IPython import generics
+from IPython.ipapi import TryNext, get as ipget
+
+import readline
+
+from h5py.highlevel import AttributeManager, HLObject
+
+re_attr_match = re.compile(r"(?:.*\=)?(.+\[.*\].*)\.(\w*)$")
+re_item_match = re.compile(r"""(?:.*\=)?(.*)\[(?P<s>['|"])(?!.*(?P=s))(.*)$""")
+re_object_match = re.compile(r"(?:.*\=)?(.+?)(?:\[)")
+
+
+def h5py_item_completer(context, command):
+ """Compute possible item matches for dict-like objects"""
+
+ base, item = re_item_match.split(command)[1:4:2]
+
+ # we don't want to call any functions, but I couldn't find a robust regex
+ # that filtered them without unintended side effects. So keys containing
+ # "(" will not complete.
+ try:
+ assert '(' not in base
+ except AssertionError:
+ raise ValueError()
+
+ try:
+ obj = eval(base, context.shell.user_ns)
+ except:
+ return []
+
+ path, item = posixpath.split(item)
+ if path:
+ target = obj[path]
+ items = (posixpath.join(path, item) for item in target.iternames())
+ else:
+ items = obj.iternames()
+
+ readline.set_completer_delims(' \t\n`!@#$^&*()=+[{]}\\|;:\'",<>?')
+
+ return [i for i in items if i[:len(item)] == item]
+
+
+def h5py_attr_completer(context, command):
+ """Compute possible attr matches for nested dict-like objects"""
+
+ base, attr = re_attr_match.split(command)[1:3]
+ base = base.strip()
+
+ try:
+ assert '(' not in base
+ except AssertionError:
+ raise ValueError()
+
+ try:
+ obj = eval(base, context.shell.user_ns)
+ except:
+ return []
+
+ attrs = dir(obj)
+ try:
+ attrs = generics.complete_object(obj, attrs)
+ except TryNext:
+ pass
+
+ omit__names = ipget().options.readline_omit__names
+ if omit__names == 1:
+ attrs = [a for a in attrs if not a.startswith('__')]
+ elif omit__names == 2:
+ attrs = [a for a in attrs if not a.startswith('_')]
+
+ readline.set_completer_delims(' =')
+
+ return ["%s.%s" % (base, a) for a in attrs if a[:len(attr)] == attr]
+
+
+def h5py_completer(self, event):
+ base = re_object_match.split(event.line)[1]
+
+ if not isinstance(self._ofind(base)['obj'], (AttributeManager, HLObject)):
+ raise TryNext
+
+ try:
+ return h5py_attr_completer(self, event.line)
+ except ValueError:
+ pass
+
+ try:
+ return h5py_item_completer(self, event.line)
+ except ValueError:
+ pass
+
+ return []
+
+def activate():
+ ip = ipget()
+ ip.set_hook('complete_command', h5py_completer, re_key=r"(?:.*\=)?(.+?)\[")
diff --git a/h5py/filters.py b/h5py/filters.py
index 454193f..33bf171 100644
--- a/h5py/filters.py
+++ b/h5py/filters.py
@@ -182,6 +182,8 @@ def get_filters(plist):
else:
raise TypeError("Unknown SZIP configuration")
vals = (mask, pixels)
+ elif code == h5z.FILTER_LZF:
+ vals = None
else:
if len(vals) == 0:
vals = None
diff --git a/lzf/example.c b/lzf/example.c
index f37abc9..7277c34 100644
--- a/lzf/example.c
+++ b/lzf/example.c
@@ -4,8 +4,6 @@
License: BSD (see LICENSE.txt)
Example program demonstrating use of the LZF filter from C code.
- The filter is completely stateless, and so is safe to statically
- link into the final program.
The LZF filter provides high-speed compression with acceptable compression
performance, resulting in much faster performance than DEFLATE, at the
@@ -17,7 +15,12 @@
free, supported by all current versions of HDF5, and can significantly
improve the compression ratio.
- To compile:
+ The filter is completely stateless, and so is safe to statically
+ link into the final program if desired. Using gcc with option -O1
+ or higher is recommended. If building with HDF5 1.8, you must
+ define H5_USE_16_API when compiling.
+
+ To compile this program:
h5cc -DH5_USE_16_API lzf/*.c lzf_filter.c example.c -o example
@@ -33,24 +36,30 @@
Chunks: {1, 100, 100} 40000 bytes
Storage: 4000000 logical bytes, 529745 allocated bytes, 755.08% utilization
Filter-0: shuffle-2 OPT {4}
- Filter-1: lzf-32000 OPT {}
+ Filter-1: lzf-32000 OPT {1, 261, 0}
Type: native float
*/
+#include <stdio.h>
#include "hdf5.h"
#include "lzf_filter.h"
+#define SIZE 100*100*100
+#define SHAPE {100,100,100}
+#define CHUNKSHAPE {1,100,100}
+
int main(){
- float data[100*100*100];
- const hsize_t shape[] = {100,100,100};
- const hsize_t chunkshape[] = {1,100,100};
+ static float data[SIZE];
+ static float data_out[SIZE];
+ const hsize_t shape[] = SHAPE;
+ const hsize_t chunkshape[] = CHUNKSHAPE;
int r, i;
int return_code = 1;
hid_t fid, sid, dset, plist = 0;
- for(i=0; i<100*100*100; i++){
+ for(i=0; i<SIZE; i++){
data[i] = i;
}
@@ -88,6 +97,15 @@ int main(){
r = H5Dwrite(dset, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data);
if(r<0) goto failed;
+ r = H5Dread(dset, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data_out);
+ if(r<0) goto failed;
+
+ for(i=0;i<SIZE;i++){
+ if(data[i] != data_out[i]) goto failed;
+ }
+
+ fprintf(stdout, "Success!\n");
+
return_code = 0;
failed:
diff --git a/lzf/lzf_filter.c b/lzf/lzf_filter.c
index fc6251d..b2bcb00 100644
--- a/lzf/lzf_filter.c
+++ b/lzf/lzf_filter.c
@@ -29,9 +29,6 @@
#include "lzf/lzf.h"
#include "lzf_filter.h"
-
-#define H5PY_LZF_MAX_BUF (100L*1024L*1024L) /* max decompress buffer */
-
#if H5_VERS_MAJOR == 1 && H5_VERS_MINOR < 7
#define H5PY_LZF_16API 1
@@ -44,40 +41,12 @@
#endif
-/* In HDF5, one filter function handles both compression and decompression */
size_t lzf_filter(unsigned flags, size_t cd_nelmts,
const unsigned cd_values[], size_t nbytes,
size_t *buf_size, void **buf);
-/* TODO: Store chunk size in DCPL? This is not thread safe.
-*/
-static size_t chunksize = 0;
-
-/* Compute the chunk size to help guess a decompression buffer */
-herr_t lzf_set_local(hid_t dcpl, hid_t type, hid_t space){
+herr_t lzf_set_local(hid_t dcpl, hid_t type, hid_t space);
- int ndims;
- int i;
- size_t typesize;
- hsize_t chunkdims[32];
-
- ndims = H5Pget_chunk(dcpl, 32, &chunkdims);
- if(ndims<0) return -1;
- if(ndims>32) return -1;
-
- typesize = H5Tget_size(type);
- if(typesize==0) return -1;
-
- chunksize = typesize;
- for(i=0;i<ndims;i++){
- chunksize *= chunkdims[i];
- }
-
-#ifdef H5PY_LZF_DEBUG
- fprintf(stderr, "Computed chunk size of %d\n", chunksize);
-#endif
- return 1;
-}
/* Try to register the filter, passing on the HDF5 return value */
int register_lzf(void){
@@ -111,6 +80,64 @@ int register_lzf(void){
return retval;
}
+/* Filter setup. Records the following inside the DCPL:
+
+ 1. If version information is not present, set slots 0 and 1 to the filter
+ revision and LZF API version, respectively.
+
+ 2. Compute the chunk size in bytes and store it in slot 2.
+*/
+herr_t lzf_set_local(hid_t dcpl, hid_t type, hid_t space){
+
+ int ndims;
+ int i;
+ herr_t r;
+
+ unsigned int bufsize;
+ hsize_t chunkdims[32];
+
+ unsigned int flags;
+ size_t nelements = 8;
+ unsigned int values[] = {0,0,0,0,0,0,0,0};
+
+ r = H5Pget_filter_by_id(dcpl, H5PY_FILTER_LZF, &flags, &nelements, &values, 0, NULL);
+ if(r<0) return -1;
+
+ if(nelements < 3) nelements = 3; /* First 3 slots reserved. If any higher
+ slots are used, preserve the contents. */
+
+ /* It seems the H5Z_FLAG_REVERSE flag doesn't work here, so we have to be
+ careful not to clobber any existing version info */
+ if(values[0]==0) values[0] = H5PY_FILTER_LZF_VERSION;
+ if(values[1]==0) values[1] = LZF_VERSION;
+
+ ndims = H5Pget_chunk(dcpl, 32, &chunkdims);
+ if(ndims<0) return -1;
+ if(ndims>32){
+ PUSH_ERR("lzf_set_local", H5E_CALLBACK, "Chunk rank exceeds limit");
+ return -1;
+ }
+
+ bufsize = H5Tget_size(type);
+ if(bufsize==0) return -1;
+
+ for(i=0;i<ndims;i++){
+ bufsize *= chunkdims[i];
+ }
+
+ values[2] = bufsize;
+
+#ifdef H5PY_LZF_DEBUG
+ fprintf(stderr, "LZF: Computed buffer size %d\n", bufsize);
+#endif
+
+ r = H5Pmodify_filter(dcpl, H5PY_FILTER_LZF, flags, nelements, values);
+ if(r<0) return -1;
+
+ return 1;
+}
+
+
/* The filter function */
size_t lzf_filter(unsigned flags, size_t cd_nelmts,
const unsigned cd_values[], size_t nbytes,
@@ -143,9 +170,11 @@ size_t lzf_filter(unsigned flags, size_t cd_nelmts,
/* We're decompressing */
} else {
- outbuf_size = chunksize; /* From global precomputed by set_local */
-
- if(outbuf_size==0) outbuf_size = (*buf_size); /* just in case */
+ if((cd_nelmts>=3)&&(cd_values[2]!=0)){
+ outbuf_size = cd_values[2]; /* Precomputed buffer guess */
+ }else{
+ outbuf_size = (*buf_size);
+ }
#ifdef H5PY_LZF_DEBUG
fprintf(stderr, "Decompress %d chunk w/buffer %d\n", nbytes, outbuf_size);
@@ -163,28 +192,18 @@ size_t lzf_filter(unsigned flags, size_t cd_nelmts,
status = lzf_decompress(*buf, nbytes, outbuf, outbuf_size);
+ if(!status){ /* compression failed */
- /* compression failed */
- if(!status){
-
- /* Output buffer too small; make it bigger */
if(errno == E2BIG){
+ outbuf_size += (*buf_size);
#ifdef H5PY_LZF_DEBUG
fprintf(stderr, " Too small: %d\n", outbuf_size);
#endif
- outbuf_size += (*buf_size);
- if(outbuf_size > H5PY_LZF_MAX_BUF){
- PUSH_ERR("lzf_filter", H5E_CALLBACK, "Requested LZF buffer too big");
- goto failed;
- }
-
- /* Horrible internal error (data corruption) */
} else if(errno == EINVAL) {
PUSH_ERR("lzf_filter", H5E_CALLBACK, "Invalid data for LZF decompression");
goto failed;
- /* Unknown error */
} else {
PUSH_ERR("lzf_filter", H5E_CALLBACK, "Unknown LZF decompression error");
goto failed;
@@ -206,8 +225,9 @@ size_t lzf_filter(unsigned flags, size_t cd_nelmts,
}
failed:
- free(outbuf);
- return 0;
+
+ free(outbuf);
+ return 0;
} /* End filter function */
diff --git a/lzf/lzf_filter.h b/lzf/lzf_filter.h
index 0421423..e7ee8d5 100644
--- a/lzf/lzf_filter.h
+++ b/lzf/lzf_filter.h
@@ -14,8 +14,12 @@
#ifndef H5PY_LZF_H
#define H5PY_LZF_H
-#define H5PY_FILTER_LZF 32000 /* Semi-official as of 2/5/09. This value can
- never change. */
+#define H5PY_FILTER_LZF_VERSION 1
+
+/* Filter ID registered with the HDF Group as of 2/6/09. For maintenance
+ requests, contact the filter author directly. */
+#define H5PY_FILTER_LZF 32000
+
/* Register the filter with the library. Returns a negative value on failure,
and a non-negative value on success.
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/h5py.git
More information about the debian-science-commits
mailing list