[h5py] 219/455: Fixes for LZF, new filter code and a 6x decompressor speedup

[Ghislain Vaillant]

This is an automated email from the git hooks/post-receive script.

ghisvail-guest pushed a commit to annotated tag 1.3.0
in repository h5py.

commit 43d897d8b287ee0dac228f97fa4fb7a81c3e5950
Author: andrewcollette <andrew.collette at gmail.com>
Date:   Fri Feb 6 01:22:40 2009 +0000

    Fixes for LZF, new filter code and a 6x decompressor speedup
---
 lzf/example.c    |  2 +-
 lzf/lzf/lzf_d.c  |  2 ++
 lzf/lzf_filter.c | 52 ++++++++++++++++++++++++++++++++++++++++++++--------
 lzf/lzf_filter.h |  7 ++-----
 4 files changed, 49 insertions(+), 14 deletions(-)

diff --git a/lzf/example.c b/lzf/example.c
index 63bb6ed..f37abc9 100644
--- a/lzf/example.c
+++ b/lzf/example.c
@@ -33,7 +33,7 @@
         Chunks:    {1, 100, 100} 40000 bytes
         Storage:   4000000 logical bytes, 529745 allocated bytes, 755.08% utilization
         Filter-0:  shuffle-2 OPT {4}
-        Filter-1:  lzf-315 OPT {}
+        Filter-1:  lzf-32000 OPT {}
         Type:      native float
 */
 
diff --git a/lzf/lzf/lzf_d.c b/lzf/lzf/lzf_d.c
index 5897580..2e2eeda 100644
--- a/lzf/lzf/lzf_d.c
+++ b/lzf/lzf/lzf_d.c
@@ -43,6 +43,7 @@
 # define SET_ERRNO(n) errno = (n)
 #endif
 
+/* ASM is slower than C in HDF5 tests -- A.C. 2/5/09
 #ifndef __STRICT_ANSI__
 #ifndef H5PY_DISABLE_LZF_ASM
 #if (__i386 || __amd64) && __GNUC__ >= 3
@@ -53,6 +54,7 @@
 #endif
 #endif
 #endif
+*/
 
 unsigned int 
 lzf_decompress (const void *const in_data,  unsigned int in_len,
diff --git a/lzf/lzf_filter.c b/lzf/lzf_filter.c
index 96bb30a..fc6251d 100644
--- a/lzf/lzf_filter.c
+++ b/lzf/lzf_filter.c
@@ -29,8 +29,8 @@
 #include "lzf/lzf.h"
 #include "lzf_filter.h"
 
-/* Max size of compress/decompress buffer */
-#define H5PY_LZF_MAX_BUF (100L*1024L*1024L)
+
+#define H5PY_LZF_MAX_BUF (100L*1024L*1024L)  /* max decompress buffer */
 
 #if H5_VERS_MAJOR == 1 && H5_VERS_MINOR < 7
 
@@ -49,6 +49,35 @@ size_t lzf_filter(unsigned flags, size_t cd_nelmts,
 		    const unsigned cd_values[], size_t nbytes,
 		    size_t *buf_size, void **buf);
 
+/* TODO: Store chunk size in DCPL?  This is not thread safe.
+*/
+static size_t chunksize = 0;
+
+/* Compute the chunk size to help guess a decompression buffer */
+herr_t lzf_set_local(hid_t dcpl, hid_t type, hid_t space){
+
+    int ndims;
+    int i;
+    size_t typesize;
+    hsize_t chunkdims[32];
+
+    ndims = H5Pget_chunk(dcpl, 32, &chunkdims);
+    if(ndims<0) return -1;
+    if(ndims>32) return -1;
+
+    typesize = H5Tget_size(type);
+    if(typesize==0) return -1;
+
+    chunksize = typesize;
+    for(i=0;i<ndims;i++){
+        chunksize *= chunkdims[i];
+    }
+
+#ifdef H5PY_LZF_DEBUG
+    fprintf(stderr, "Computed chunk size of %d\n", chunksize);
+#endif
+    return 1;
+}
 
 /* Try to register the filter, passing on the HDF5 return value */
 int register_lzf(void){
@@ -60,7 +89,7 @@ int register_lzf(void){
         (H5Z_filter_t)(H5PY_FILTER_LZF),
         "lzf",
         NULL,
-        NULL,
+        (H5Z_func_t)(lzf_set_local),
         (H5Z_func_t)(lzf_filter)
     };
 #else
@@ -70,7 +99,7 @@ int register_lzf(void){
         1, 1,
         "lzf",
         NULL,
-        NULL,
+        (H5Z_func_t)(lzf_set_local),
         (H5Z_func_t)(lzf_filter)
     };
 #endif
@@ -101,7 +130,7 @@ size_t lzf_filter(unsigned flags, size_t cd_nelmts,
            proceeds.
         */
 
-        outbuf_size = nbytes;
+        outbuf_size = (*buf_size);
         outbuf = malloc(outbuf_size);
 
         if(outbuf == NULL){
@@ -114,10 +143,16 @@ size_t lzf_filter(unsigned flags, size_t cd_nelmts,
     /* We're decompressing */
     } else {
 
-        outbuf_size = (*buf_size);
+        outbuf_size = chunksize;  /* From global precomputed by set_local */
+
+        if(outbuf_size==0) outbuf_size = (*buf_size);  /* just in case */
+
+#ifdef H5PY_LZF_DEBUG
+        fprintf(stderr, "Decompress %d chunk w/buffer %d\n", nbytes, outbuf_size);
+#endif
 
         while(!status){
-        
+            
             free(outbuf);
             outbuf = malloc(outbuf_size);
 
@@ -128,13 +163,14 @@ size_t lzf_filter(unsigned flags, size_t cd_nelmts,
 
             status = lzf_decompress(*buf, nbytes, outbuf, outbuf_size);
 
+
             /* compression failed */
             if(!status){
 
                 /* Output buffer too small; make it bigger */
                 if(errno == E2BIG){
 #ifdef H5PY_LZF_DEBUG
-                    fprintf(stderr, "LZF filter: Buffer guess too small: %d", outbuf_size);
+                    fprintf(stderr, "    Too small: %d\n", outbuf_size);
 #endif
                     outbuf_size += (*buf_size);
                     if(outbuf_size > H5PY_LZF_MAX_BUF){
diff --git a/lzf/lzf_filter.h b/lzf/lzf_filter.h
index 7c3f54c..0421423 100644
--- a/lzf/lzf_filter.h
+++ b/lzf/lzf_filter.h
@@ -10,15 +10,12 @@
 * 
 ****** End preamble block ****************************************************/
 
-/*
-    Filter code is chosen in an ad-hoc manner to avoid conflict
-    with PyTables LZO/BZIP2 implementation.
-*/
 
 #ifndef H5PY_LZF_H
 #define H5PY_LZF_H
 
-#define H5PY_FILTER_LZF 315
+#define H5PY_FILTER_LZF 32000   /* Semi-official as of 2/5/09.  This value can
+                                   never change. */
 
 /* Register the filter with the library. Returns a negative value on failure, 
    and a non-negative value on success.

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/h5py.git