[h5py] 123/455: More fixes, revised logging system
Ghislain Vaillant
ghisvail-guest at moszumanska.debian.org
Thu Jul 2 18:19:24 UTC 2015
This is an automated email from the git hooks/post-receive script.
ghisvail-guest pushed a commit to annotated tag 1.3.0
in repository h5py.
commit 48e9d190d27db357970aba9d84193871481036ae
Author: andrewcollette <andrew.collette at gmail.com>
Date: Mon Sep 22 04:00:31 2008 +0000
More fixes, revised logging system
---
h5py/__init__.py | 1 -
h5py/_extras.py | 21 +++--
h5py/h5.pxd | 8 +-
h5py/h5.pyx | 46 +++++-----
h5py/h5d.pyx | 11 ++-
h5py/h5f.pyx | 18 +++-
h5py/highlevel.py | 5 +-
h5py/tests/test_threads.py | 207 +--------------------------------------------
setup.py | 2 +-
9 files changed, 77 insertions(+), 242 deletions(-)
diff --git a/h5py/__init__.py b/h5py/__init__.py
index 5d5eec0..e9d3d2e 100644
--- a/h5py/__init__.py
+++ b/h5py/__init__.py
@@ -22,7 +22,6 @@ __doc__ = \
HDF5 %s (using %s API)
"""
-from h5 import _config as config
import utils, h5, h5a, h5d, h5f, h5g, h5i, h5p, h5r, h5s, h5t, h5z, highlevel
from highlevel import File, Group, Dataset, Datatype, AttributeManager, CoordsList
diff --git a/h5py/_extras.py b/h5py/_extras.py
index 1cbcc94..b1bb975 100644
--- a/h5py/_extras.py
+++ b/h5py/_extras.py
@@ -32,7 +32,6 @@ def h5sync(logger=None):
def wrap(*args, **kwds):
with phil:
return func(*args, **kwds)
-
uw_apply(wrap, func)
return wrap
@@ -44,10 +43,13 @@ def h5sync(logger=None):
def wrap(*args, **kwds):
logger.debug("$ Threadsafe function entry: %s" % func.__name__)
- with phil:
- logger.debug("> Acquired lock on %s" % func.__name__)
- retval = func(*args, **kwds)
- logger.debug("< Released lock on %s" % func.__name__)
+ try:
+ with phil:
+ retval = func(*args, **kwds)
+ except:
+ logger.debug("! Exception in %s" % func.__name__)
+ raise
+ logger.debug("# Threadsafe function exit: %s" % func.__name__)
return retval
uw_apply(wrap, func)
@@ -61,8 +63,13 @@ def h5sync_dummy(logger):
def wrap(*args, **kwds):
logger.debug("$ Function entry: %s" % func.__name__)
- return func(*args, **kwds)
-
+ try:
+ retval = func(*args, **kwds)
+ except:
+ logger.debug("! Exception in %s" % func.__name__)
+ raise
+ logger.debug("# Function exit: %s" % func.__name__)
+ return retval
uw_apply(wrap, func)
return wrap
diff --git a/h5py/h5.pxd b/h5py/h5.pxd
index 3a0571d..51a2792 100644
--- a/h5py/h5.pxd
+++ b/h5py/h5.pxd
@@ -27,10 +27,10 @@ cdef class PHIL:
cdef object lock
- cpdef int __enter__(self) except -1
- cpdef int __exit__(self, a, b, c) except -1
- cpdef int acquire(self) except -1
- cpdef int release(self) except -1
+ cpdef bint __enter__(self) except -1
+ cpdef bint __exit__(self, a, b, c) except -1
+ cpdef bint acquire(self, int blocking=*) except -1
+ cpdef bint release(self) except -1
cpdef PHIL get_phil()
diff --git a/h5py/h5.pyx b/h5py/h5.pyx
index 8ef1678..f529cdd 100644
--- a/h5py/h5.pyx
+++ b/h5py/h5.pyx
@@ -40,10 +40,13 @@ import threading
# Logging is only enabled when compiled with H5PY_DEBUG nonzero
IF H5PY_DEBUG:
import logging
- logger = logging.getLogger('h5py') # Set up the root logger
- logger.setLevel(H5PY_DEBUG)
- logger.addHandler(logging.StreamHandler())
+ for x in ('h5py.identifiers', 'h5py.functions', 'h5py.threads'):
+ l = logging.getLogger(x)
+ l.setLevel(H5PY_DEBUG)
+ l.addHandler(logging.StreamHandler())
log_ident = logging.getLogger('h5py.identifiers')
+ log_threads = logging.getLogger('h5py.threads')
+
# --- C extensions and classes ------------------------------------------------
@@ -82,34 +85,39 @@ cdef class PHIL:
The Primary HDF5 Interface Lock (PHIL) is a global reentrant lock
which manages access to the library. HDF5 is not guaranteed to
be thread-safe, and certain callbacks in h5py can execute arbitrary
- threaded Python code. Therefore, in threading mode all routines
- acquire this lock first. With threading support disabled, the
- object's methods do nothing.
+ threaded Python code, defeating the normal GIL-based protection for
+ extension modules. Therefore, in threading mode all routines
+ acquire this lock first. When h5py is built without thread awareness,
+ all locking methods are no-ops.
"""
IF H5PY_THREADS:
def __init__(self):
self.lock = threading.RLock()
- cpdef int __enter__(self) except -1:
+ cpdef bint __enter__(self) except -1:
self.lock.acquire()
return 0
- cpdef int __exit__(self,a,b,c) except -1:
+ cpdef bint __exit__(self,a,b,c) except -1:
self.lock.release()
return 0
- cpdef int acquire(self) except -1:
- self.lock.acquire()
- return 0
- cpdef int release(self) except -1:
+ cpdef bint acquire(self, int blocking=1) except -1:
+ cdef bint rval = self.lock.acquire(blocking)
+ IF H5PY_DEBUG:
+ log_threads.debug('> PHIL acquired')
+ return rval
+ cpdef bint release(self) except -1:
self.lock.release()
+ IF H5PY_DEBUG:
+ log_threads.debug('< PHIL released')
return 0
ELSE:
- cpdef int __enter__(self) except -1:
- return 0
- cpdef int __exit__(self,a,b,c) except -1:
+ cpdef bint __enter__(self) except -1:
return 0
- cpdef int acquire(self) except -1:
+ cpdef bint __exit__(self,a,b,c) except -1:
return 0
- cpdef int release(self) except -1:
+ cpdef bint acquire(self, int blocking=1) except -1:
+ return 1
+ cpdef bint release(self) except -1:
return 0
cdef PHIL phil = PHIL()
@@ -206,7 +214,7 @@ cdef class ObjectID:
ref = str(H5Iget_ref(self.id)) if self._valid else "X"
lck = "L" if self._locked else "U"
- return "%9d [%s] (%s) %s" % (self.id, ref, lck, self.__class__.__name__)
+ return "%s [%s] (%s) %d" % (self.__class__.__name__, ref, lck, self.id)
def __repr__(self):
return self.__str__()
@@ -600,7 +608,7 @@ api_version = "%d.%d" % api_version_tuple
version = H5PY_VERSION
version_tuple = tuple([int(x) for x in version.split('.')])
-_config = H5PYConfig()
+config = H5PYConfig()
diff --git a/h5py/h5d.pyx b/h5py/h5d.pyx
index 7c874d3..3d0ef86 100644
--- a/h5py/h5d.pyx
+++ b/h5py/h5d.pyx
@@ -32,7 +32,6 @@ import h5
import h5t
import h5s
import h5g
-from h5 import _config as config
import_array()
@@ -61,6 +60,7 @@ FILL_VALUE_USER_DEFINED = H5D_FILL_VALUE_USER_DEFINED
# === Basic dataset operations ================================================
+ at sync
def create(ObjectID loc not None, char* name, TypeID tid not None,
SpaceID space not None, PropDCID dcpl=None):
""" (ObjectID loc, STRING name, TypeID tid, SpaceID space,
@@ -72,6 +72,7 @@ def create(ObjectID loc not None, char* name, TypeID tid not None,
"""
return DatasetID(H5Dcreate(loc.id, name, tid.id, space.id, pdefault(dcpl)))
+ at sync
def open(ObjectID loc not None, char* name):
""" (ObjectID loc, STRING name) => DatasetID
@@ -147,6 +148,7 @@ cdef class DatasetID(ObjectID):
sid = self.get_space()
return sid.get_simple_extent_ndims()
+ @sync
def _close(self):
""" ()
@@ -243,6 +245,7 @@ cdef class DatasetID(ObjectID):
finally:
arr_obj.flags |= NPY_WRITEABLE
+ @sync
def extend(self, object shape):
""" (TUPLE shape)
@@ -271,6 +274,7 @@ cdef class DatasetID(ObjectID):
if space_id:
H5Sclose(space_id)
+ @sync
def get_space(self):
""" () => SpaceID
@@ -278,6 +282,7 @@ cdef class DatasetID(ObjectID):
"""
return SpaceID(H5Dget_space(self.id))
+ @sync
def get_space_status(self):
""" () => INT space_status_code
@@ -291,6 +296,7 @@ cdef class DatasetID(ObjectID):
H5Dget_space_status(self.id, &status)
return <int>status
+ @sync
def get_type(self):
""" () => TypeID
@@ -298,6 +304,7 @@ cdef class DatasetID(ObjectID):
"""
return typewrap(H5Dget_type(self.id))
+ @sync
def get_create_plist(self):
""" () => PropDCID
@@ -306,6 +313,7 @@ cdef class DatasetID(ObjectID):
"""
return propwrap(H5Dget_create_plist(self.id))
+ @sync
def get_offset(self):
""" () => LONG offset or None
@@ -320,6 +328,7 @@ cdef class DatasetID(ObjectID):
return None
return offset
+ @sync
def get_storage_size(self):
""" () => LONG storage_size
diff --git a/h5py/h5f.pyx b/h5py/h5f.pyx
index 602cac7..2ccf169 100644
--- a/h5py/h5f.pyx
+++ b/h5py/h5f.pyx
@@ -13,6 +13,7 @@
"""
Low-level operations on HDF5 file objects.
"""
+include "std_code.pxi"
# Pyrex compile-time imports
from h5 cimport standard_richcmp
@@ -54,6 +55,7 @@ OBJ_LOCAL = H5F_OBJ_LOCAL
# === File operations =========================================================
+ at sync
def open(char* name, unsigned int flags=H5F_ACC_RDWR, PropFAID fapl=None):
""" (STRING name, UINT flags=ACC_RDWR, PropFAID fapl=None)
=> FileID
@@ -63,6 +65,7 @@ def open(char* name, unsigned int flags=H5F_ACC_RDWR, PropFAID fapl=None):
"""
return FileID(H5Fopen(name, flags, pdefault(fapl)))
+ at sync
def create(char* name, int flags=H5F_ACC_TRUNC, PropFCID fcpl=None,
PropFAID fapl=None):
""" (STRING name, INT flags=ACC_TRUNC, PropFCID fcpl=None,
@@ -78,6 +81,7 @@ def create(char* name, int flags=H5F_ACC_TRUNC, PropFCID fcpl=None,
"""
return FileID(H5Fcreate(name, flags, pdefault(fcpl), pdefault(fapl)))
+ at sync
def flush(ObjectID obj not None, int scope=H5F_SCOPE_LOCAL):
""" (ObjectID obj, INT scope=SCOPE_LOCAL)
@@ -89,6 +93,7 @@ def flush(ObjectID obj not None, int scope=H5F_SCOPE_LOCAL):
"""
H5Fflush(obj.id, <H5F_scope_t>scope)
+ at sync
def is_hdf5(char* name):
""" (STRING name) => BOOL is_hdf5
@@ -97,13 +102,15 @@ def is_hdf5(char* name):
"""
return pybool(H5Fis_hdf5(name))
+ at sync
def mount(ObjectID loc not None, char* name, FileID fid not None):
""" (ObjectID loc, STRING name, FileID fid)
Mount an open file as "name" under group loc_id.
"""
H5Fmount(loc.id, name, fid.id, H5P_DEFAULT)
-
+
+ at sync
def unmount(ObjectID loc not None, char* name):
""" (ObjectID loc, STRING name)
@@ -111,6 +118,7 @@ def unmount(ObjectID loc not None, char* name):
"""
H5Funmount(loc.id, name)
+ at sync
def get_name(ObjectID obj not None):
""" (ObjectID obj) => STRING file_name
@@ -130,6 +138,7 @@ def get_name(ObjectID obj not None):
finally:
efree(name)
+ at sync
def get_obj_count(object where=OBJ_ALL, int types=H5F_OBJ_ALL):
""" (OBJECT where=OBJ_ALL, types=OBJ_ALL) => INT n_objs
@@ -178,6 +187,7 @@ cdef object wrap_identifier(hid_t ident):
H5Iinc_ref(ident)
return obj
+ at sync
def get_obj_ids(object where=OBJ_ALL, int types=H5F_OBJ_ALL):
""" (OBJECT where=OBJ_ALL, types=OBJ_ALL) => LIST open_ids
@@ -240,6 +250,7 @@ cdef class FileID(ObjectID):
def __get__(self):
return get_name(self)
+ @sync
def close(self):
""" ()
@@ -250,6 +261,7 @@ cdef class FileID(ObjectID):
"""
H5Fclose(self.id)
+ @sync
def reopen(self):
""" () => FileID
@@ -259,6 +271,7 @@ cdef class FileID(ObjectID):
"""
return FileID(H5Freopen(self.id))
+ @sync
def get_filesize(self):
""" () => LONG size
@@ -269,6 +282,7 @@ cdef class FileID(ObjectID):
H5Fget_filesize(self.id, &size)
return size
+ @sync
def get_create_plist(self):
""" () => PropFCID
@@ -276,6 +290,7 @@ cdef class FileID(ObjectID):
"""
return propwrap(H5Fget_create_plist(self.id))
+ @sync
def get_access_plist(self):
""" () => PropFAID
@@ -284,6 +299,7 @@ cdef class FileID(ObjectID):
"""
return propwrap(H5Fget_access_plist(self.id))
+ @sync
def get_freespace(self):
""" () => LONG freespace
diff --git a/h5py/highlevel.py b/h5py/highlevel.py
index 52b7a6f..abc18b7 100644
--- a/h5py/highlevel.py
+++ b/h5py/highlevel.py
@@ -52,7 +52,7 @@ import inspect
import threading
import sys
-from h5py import h5, h5f, h5g, h5s, h5t, h5d, h5a, h5p, h5z, h5i, config
+from h5py import h5, h5f, h5g, h5s, h5t, h5d, h5a, h5p, h5z, h5i
from h5py.h5 import H5Error
from utils_hl import slice_select, hbasename, strhdr, strlist, guess_chunk
from utils_hl import CoordsList
@@ -76,8 +76,7 @@ class LockableObject(object):
Base class which provides rudimentary locking support.
"""
- _lock = property(lambda self: config.lock,
- doc = "A reentrant lock for internal thread safety")
+ _lock = h5.get_phil()
class HLObject(LockableObject):
diff --git a/h5py/tests/test_threads.py b/h5py/tests/test_threads.py
index 6d23795..74d8160 100644
--- a/h5py/tests/test_threads.py
+++ b/h5py/tests/test_threads.py
@@ -9,211 +9,8 @@
# $Date$
#
#-
-from __future__ import with_statement
import unittest
-import threading
-import dummy_threading
-import tempfile
-from threading import Thread
-import os
-import numpy
-import time
-
-from h5py import *
-from h5py.h5 import H5Error
-import h5py
-
-LOCKTYPE = threading.RLock
-SHAPE = (10,10)
-BIGSHAPE = (5300,5000) # About 200MB worth of doubles
-
-class WriterThread(Thread):
-
- def __init__(self, name, value, dset, reclist):
- Thread.__init__(self)
- self.arr = numpy.ones(SHAPE)*value # Value to be written
- self.dset = dset # Dataset to write to
- self.reclist = reclist # We'll append our name to this when done
- self.name = name
- self.next_thread = None # Thread we'll try to get to break the lock
- self.sleeptime = 0 # How long do we give that thread to try
-
- def run(self):
- # Try to fill the dataset with our values
-
- with self.dset._lock:
- if self.next_thread is not None:
- self.next_thread.start() # Try to make the next thread steal the dataset
- time.sleep(self.sleeptime) # Make sure it has a chance to misbehave
- self.dset[...] = self.arr
- self.reclist.append(self.name) # Add our name to the record, before releasing the lock.
-
-class TimedWriter(Thread):
-
- def __init__(self, dset, arr):
- Thread.__init__(self)
- self.dset = dset
- self.arr = arr
- self.timestart = 0
- self.timestop = 0
-
- def run(self):
- self.timestart = time.time()
- self.dset.id.write(h5s.ALL, h5s.ALL, self.arr)
- self.timestop = time.time()
-
-class NullWriter(Thread):
-
- def __init__(self):
- Thread.__init__(self)
- self.time = 0
-
- def run(self):
- self.time = time.time()
-
-class TestThreads(unittest.TestCase):
-
- def setUp(self):
-
- self.fname = tempfile.mktemp('.hdf5')
- self.f = File(self.fname, 'w')
- self.old_lock = h5py.config.lock
- h5py.config.lock = LOCKTYPE()
-
- def tearDown(self):
- self.f.close()
- os.unlink(self.fname)
- h5py.config.lock = self.old_lock
-
- def test_hl_pos(self):
-
- reclist = []
-
- dset = self.f.create_dataset('ds',(10,10), dtype='=f8')
- thread_a = WriterThread('A', 1.0, dset, reclist)
- thread_b = WriterThread('B', 2.0, dset, reclist)
- thread_c = WriterThread('C', 3.0, dset, reclist)
- thread_d = WriterThread('D', 4.0, dset, reclist)
-
- thread_a.next_thread = thread_b
- thread_b.next_thread = thread_c
- thread_c.next_thread = thread_d
-
- thread_a.sleeptime = 3 # Must be larger than b, so that b attempts to steal the lock.
- thread_b.sleeptime = 2 # Must be larger than c, for the same reason
- thread_c.sleeptime = 1
-
- thread_a.start()
-
- # Ensure they all finish
- # TODO: Handle a possible deadlock more gracefully.
- thread_a.join()
- thread_b.join()
- thread_c.join()
- thread_d.join()
-
- self.assertEqual(reclist, ['A','B','C','D'])
- self.assert_(numpy.all(dset.value == numpy.ones(SHAPE)*4.0))
-
- def test_hl_neg(self):
-
- oldlock = h5py.config.lock
- try:
- # Force the threads to operate in reverse order, by defeating locks
- h5py.config.lock = dummy_threading.RLock()
-
- reclist = []
-
- dset = self.f.create_dataset('ds',(10,10), dtype='=f8')
- thread_a = WriterThread('A', 1.0, dset, reclist)
- thread_b = WriterThread('B', 2.0, dset, reclist)
- thread_c = WriterThread('C', 3.0, dset, reclist)
- thread_d = WriterThread('D', 4.0, dset, reclist)
-
- thread_a.next_thread = thread_b
- thread_b.next_thread = thread_c
- thread_c.next_thread = thread_d
-
- thread_a.sleeptime = 3 # Must be larger than b, so that b attempts to steal the lock.
- thread_b.sleeptime = 2 # Must be larger than c, for the same reason
- thread_c.sleeptime = 1
-
- thread_a.start()
-
- # Ensure they all finish
- # TODO: Handle a possible deadlock more gracefully.
- thread_a.join()
- thread_b.join()
- thread_c.join()
- thread_d.join()
-
- self.assertEqual(reclist, ['D','C','B','A'])
- self.assert_(numpy.all(dset.value == numpy.ones(SHAPE)*1.0))
- finally:
- h5py.config.lock = oldlock
-
- def test_nonblock(self):
- # Ensure low-level I/O blocking behavior
- # This test is very badly designed and will have to reworked
- print "\nNonblock test skipped"
- return
-
- dset = self.f.create_dataset('ds', BIGSHAPE, '=f8')
- arr = numpy.ones(BIGSHAPE, '=f8')
- DELAY = 0.1
-
- thread_a = TimedWriter(dset, arr)
- thread_b = NullWriter()
-
- thread_a.start()
- time.sleep(DELAY)
- thread_b.start()
-
- thread_a.join()
- thread_b.join()
-
- write_time = thread_a.timestop - thread_a.timestart
- if write_time < DELAY:
- raise Exception("Write was too fast to test blocking (%f sec; need %f)" % (write_time, DELAY))
-
- if h5py.config.THREADS:
- self.assert_(thread_b.time < thread_a.timestop, "%f !< %f" % (thread_b.time, thread_a.timestop))
- else:
- self.assert_(thread_b.time > thread_a.timestop, "%f !> %f" % (thread_b.time, thread_a.timestop))
-
- def test_lock_behavior(self):
- # Check to make sure the user-provided lock class behaves correctly
- # when called from C code
-
- dset = self.f.create_dataset('ds', SHAPE, '=f8')
- arr = numpy.ones(SHAPE, '=f8')
-
- writethread = TimedWriter(dset, arr)
-
- with h5py.config.lock:
- writethread.start()
- time.sleep(2) # give it more than enough time to finish, if it ignores the lock
- exit_lock_time = time.time()
- time.sleep(2)
- writethread.join()
-
- if h5py.config.THREADS:
- # With non-blocking I/O, the library will double-check that the
- # global lock isn't held, to prevent more than one thread from
- # calling into the HDF5 API.
- self.assert_(writethread.timestop > exit_lock_time)
- else:
- # In blocking mode, the GIL ensures that only one thread at a time
- # can access the HDF5 library. Therefore the library ignores the
- # state of the soft lock. If this were a real program, the author
- # of "writethread" should acquire the lock first.
- self.assert_(writethread.timestop < exit_lock_time)
-
-
-
-
-
-
-
+class TestThreads(unittest.TestCase):
+ pass
diff --git a/setup.py b/setup.py
index 2b53405..b3fbce1 100644
--- a/setup.py
+++ b/setup.py
@@ -306,7 +306,7 @@ class test(cybuild):
cybuild.finalize_options(self)
def run(self):
- self._explicit_only = True
+ self._explicit_only = True # Ignore config.pxi disagreement unless --cython
cybuild.run(self)
oldpath = sys.path
try:
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/h5py.git
More information about the debian-science-commits
mailing list