[h5py] 91/455: Don't expose locking interface
Ghislain Vaillant
ghisvail-guest at moszumanska.debian.org
Thu Jul 2 18:19:21 UTC 2015
This is an automated email from the git hooks/post-receive script.
ghisvail-guest pushed a commit to annotated tag 1.3.0
in repository h5py.
commit 3c13b5ad1ab7aa3754c9302b5168076c593d90d9
Author: andrewcollette <andrew.collette at gmail.com>
Date: Wed Jul 30 22:03:08 2008 +0000
Don't expose locking interface
---
docs/info.txt | 92 +++++++++++++---------------------------------
h5py/extras.py | 2 +-
h5py/highlevel.py | 46 +++++++++++------------
h5py/tests/test_threads.py | 2 +-
4 files changed, 50 insertions(+), 92 deletions(-)
diff --git a/docs/info.txt b/docs/info.txt
index 0b87148..9854a62 100644
--- a/docs/info.txt
+++ b/docs/info.txt
@@ -3,52 +3,30 @@
Threading
=========
-All h5py routines are intended to be thread-safe. In this context "thread-safe"
-means that if a method is called on an object by one thread, no other thread
-can execute the same method on the same object until the first one finishes.
+Threading is an issue in h5py because HDF5 doesn't support thread-level
+concurrency. Some versions of HDF5 are not even thread-safe. The package
+tries to hide as much of these problems as possible using a combination of
+the GIL and Python-side reentrant locks.
-Most routines are written in C, and accomplish this by holding the global
-interpreter lock (GIL) until they finish, automatically guaranteeing that no
-other thread can execute. Native-Python routines like those in h5py.highlevel
-enforce thread safety through the use of reentrant locks.
+High-level
+----------
-Since HDF5 does not (yet) provide concurrency support for threads, the same
-global lock is used for all objects. It is available on the global config
-object as h5.config.lock, and comes attached to all high-level objects
-(Dataset, etc.) as "<obj>.lock".
+The objects in h5py.highlevel (File, Dataset, etc) are always thread-safe. You
+don't need to do any explicit locking, regardless of how the library is
+configured.
-You are encouraged to use this locking mechanism for blocks of Python
-statements that need to be executed in a thread-safe manner (i.e. atomically).
+Low-level
+---------
-A few examples are:
+The low-level API (h5py.h5*) is also thread-safe, unless you use the
+experimental non-blocking option to compile h5py. Then, and only then, you
+must acquire a global lock before calling into the low-level API. This lock
+is available on the global configuration object at "h5py.config.lock". The
+decorator "h5sync" in h5py.extras can wrap functions to do this automatically.
- # Example 1 (h5py.highlevel API)
- def fiddle_with_data(ds): # ds is a highlevel.Dataset object
- with ds.lock:
- ds[0,0] = 4.0
- ds[1,2] = 8.0
- ... more stuff ...
- # lock released at end of block
- # Example 2 (h5py.h5* API)
- def write_some_data(shared_file, data_array):
-
- with h5.config.lock:
- dset = h5d.open(shared_file, "dsname")
- dset.write(h5s.ALL, h5s.ALL, data_array)
-
- # Example 3 (using a decorator)
-
- from h5py.extras import h5sync
-
- @h5sync # Does exactly the same thing as "with" in example 2
- def write_some_data(shared_file, data_array):
-
- dset = h5d.open(shared_file, "dsname")
- dset.write(h5s.ALL, h5s.ALL, data_array)
-
-Non-Blocking Routines
----------------------
+More about Non-Blocking Routines
+--------------------------------
By default, all low-level HDF5 routines will lock the entire interpreter
until they complete, even in the case of lengthy I/O operations. This is
@@ -77,13 +55,14 @@ The following operations will release the GIL during I/O:
* DatasetID.write
-Customizing Locks
------------------
+Customizing the lock type
+-------------------------
-Because applications that use h5py may have their own threading systems, the
-lock used is settable at runtime. The lock is stored as settable property
-"h5py.config.lock" and should be a lock instance (not a constructor) which
-provides the following methods:
+Applications that use h5py may have their own threading systems. Since the
+h5py locks are acquired and released alongside application code, you can
+set the type of lock used internally by h5py. The lock is stored as settable
+property "h5py.config.lock" and should be a lock instance (not a constructor)
+which provides the following methods:
__enter__(), __exit__() For the Python context manager protocol
acquire(), release() For manual lock management
@@ -92,27 +71,6 @@ The default lock type is the native Python threading.RLock, but h5py makes no
assumptions about the behavior or implementation of locks beyond reentrance and
the existence of the four required methods above.
-todo: make this a new section
-
-ObjectID Hashing
-----------------
-
-H5py uses a global weak-reference dictionary to keep track of which lock goes
-with which ObjectID instance. For this to work, there must be a way to
-identify which ObjectID instances point to the same HDF5 structure. Two rules
-make this possible:
-
- A. ObjectID instances which point to the same HDF5 structure must both
- have the same hash() value.
- B. ObjectID instances which point to the same HDF5 structure must evauluate
- as equal.
-
-For named objects like datasets, groups, and files, the hash is derived from
-properties like fileno and objno, which are guaranteed by the library to be
-unique among open files. For all other objects, the HDF5 identifier determines
-uniqueness.
-
-
diff --git a/h5py/extras.py b/h5py/extras.py
index 8a2e9a3..5135358 100644
--- a/h5py/extras.py
+++ b/h5py/extras.py
@@ -13,7 +13,7 @@ from __future__ import with_statement
from h5py import config
-# Decorator utility for threads
+# Decorator utility for low-level thread safety
from functools import update_wrapper
def h5sync(func):
diff --git a/h5py/highlevel.py b/h5py/highlevel.py
index 198c3f6..0b09db6 100644
--- a/h5py/highlevel.py
+++ b/h5py/highlevel.py
@@ -70,8 +70,8 @@ class LockableObject(object):
Base class which provides rudimentary locking support.
"""
- lock = property(lambda self: config.lock,
- doc = "A reentrant lock for thread-safe use of this object")
+ _lock = property(lambda self: config.lock,
+ doc = "A reentrant lock for internal thread safety")
class HLObject(LockableObject):
@@ -129,7 +129,7 @@ class Group(HLObject):
raising an exception if it doesn't exist. If "create" is True,
create a new HDF5 group and link it into the parent group.
"""
- with parent_object.lock:
+ with parent_object._lock:
if create:
self.id = h5g.create(parent_object.id, name)
else:
@@ -166,7 +166,7 @@ class Group(HLObject):
This limitation is intentional, and may be lifted in the future.
"""
- with self.lock:
+ with self._lock:
if isinstance(obj, Group) or isinstance(obj, Dataset) or isinstance(obj, Datatype):
self.id.link(h5i.get_name(obj.id), name, link_type=h5g.LINK_HARD)
@@ -184,7 +184,7 @@ class Group(HLObject):
Currently can open groups, datasets, and named types.
"""
- with self.lock:
+ with self._lock:
info = h5g.get_objinfo(self.id, name)
if info.type == h5g.DATASET:
@@ -216,7 +216,7 @@ class Group(HLObject):
def iteritems(self):
""" Iterate over the group members as (name, value) pairs """
- with self.lock:
+ with self._lock:
for name in self:
yield (name, self[name])
@@ -250,7 +250,7 @@ class Group(HLObject):
def desc(self):
""" Extended (multi-line) description of this group, as a string.
"""
- with self.lock:
+ with self._lock:
outstr = 'Group "%s" in file "%s":' % \
(hbasename(h5i.get_name(self.id)), os.path.basename(h5f.get_name(self.id)))
outstr = strhdr(outstr)
@@ -265,7 +265,7 @@ class Group(HLObject):
return outstr
def __str__(self):
- with self.lock:
+ with self._lock:
try:
return 'Group "%s" (%d members)' % (hbasename(self.name), len(self))
except:
@@ -346,7 +346,7 @@ class File(Group):
def close(self):
""" Close this HDF5 file. All open objects will be invalidated.
"""
- with self.lock:
+ with self._lock:
self.id._close()
self.fid.close()
@@ -359,12 +359,12 @@ class File(Group):
return self
def __exit__(self,*args):
- with self.lock:
+ with self._lock:
if self.id._valid:
self.close()
def __str__(self):
- with self.lock:
+ with self._lock:
try:
return 'File "%s", root members: %s' % (self.name, ', '.join(['"%s"' % name for name in self]))
except:
@@ -429,7 +429,7 @@ class Dataset(HLObject):
doc = "Numpy dtype representing the datatype")
def _getval(self):
- with self.lock:
+ with self._lock:
arr = self[...]
if arr.shape == ():
return numpy.asscalar(arr)
@@ -468,7 +468,7 @@ class Dataset(HLObject):
shuffle: Use the shuffle filter? (requires compression) T/F*
fletcher32: Enable Fletcher32 error detection? T/F*
"""
- with group.lock:
+ with group._lock:
if data is None and shape is None:
if any((data,dtype,shape,chunks,compression,shuffle,fletcher32)):
raise ValueError('You cannot specify keywords when opening a dataset.')
@@ -524,7 +524,7 @@ class Dataset(HLObject):
ds[1,2,3,"a"]
ds[0:5:2, ..., 0:2, "a", "b"]
"""
- with self.lock:
+ with self._lock:
start, count, stride, names = slicer(self.shape, args)
if not (len(start) == len(count) == len(stride) == self.id.rank):
@@ -577,7 +577,7 @@ class Dataset(HLObject):
Numpy array must match the shape of the selection, and the Numpy
array's datatype must be convertible to the HDF5 datatype.
"""
- with self.lock:
+ with self._lock:
start, count, stride, names = slicer(self.shape, args)
if len(names) != 0:
raise ValueError("Field name selections are not allowed for write.")
@@ -598,7 +598,7 @@ class Dataset(HLObject):
self.id.write(mspace, fspace, numpy.array(val))
def __str__(self):
- with self.lock:
+ with self._lock:
try:
return 'Dataset "%s": %s %s' % (hbasename(self.name),
str(self.shape), repr(self.dtype))
@@ -643,7 +643,7 @@ class AttributeManager(LockableObject):
will be returned as a Numpy scalar. Otherwise, it will be returned
as a Numpy ndarray.
"""
- with self.lock:
+ with self._lock:
attr = h5a.open_name(self.id, name)
arr = numpy.ndarray(attr.shape, dtype=attr.dtype)
@@ -661,7 +661,7 @@ class AttributeManager(LockableObject):
Any existing value is destroyed just before the call to h5a.create.
If the creation fails, the data is not recoverable.
"""
- with self.lock:
+ with self._lock:
if not isinstance(value, numpy.ndarray):
value = numpy.array(value)
@@ -686,13 +686,13 @@ class AttributeManager(LockableObject):
def __iter__(self):
""" Iterate over the names of attributes. """
- with self.lock:
+ with self._lock:
for name in h5a.py_listattrs(self.id):
yield name
def iteritems(self):
""" Iterate over (name, value) tuples. """
- with self.lock:
+ with self._lock:
for name in self:
yield (name, self[name])
@@ -701,7 +701,7 @@ class AttributeManager(LockableObject):
return h5a.py_exists(self.id, name)
def __str__(self):
- with self.lock:
+ with self._lock:
try:
rstr = 'Attributes of "%s": ' % hbasename(h5i.get_name(self.id))
if len(self) == 0:
@@ -738,12 +738,12 @@ class Datatype(HLObject):
def __init__(self, grp, name):
""" Private constructor; you should not create these.
"""
- with grp.lock:
+ with grp._lock:
self.id = h5t.open(grp.id, name)
self._attrs = AttributeManager(self)
def __str__(self):
- with self.lock:
+ with self._lock:
try:
return "Named datatype object (%s)" % str(self.dtype)
except:
diff --git a/h5py/tests/test_threads.py b/h5py/tests/test_threads.py
index d70bf49..e29dfda 100644
--- a/h5py/tests/test_threads.py
+++ b/h5py/tests/test_threads.py
@@ -43,7 +43,7 @@ class WriterThread(Thread):
def run(self):
# Try to fill the dataset with our values
- with self.dset.lock:
+ with self.dset._lock:
if self.next_thread is not None:
self.next_thread.start() # Try to make the next thread steal the dataset
time.sleep(self.sleeptime) # Make sure it has a chance to misbehave
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/h5py.git
More information about the debian-science-commits
mailing list