r736 - / zope-managableindex zope-managableindex/branches zope-managableindex/branches/upstream zope-managableindex/branches/upstream/current zope-managableindex/branches/upstream/current/doc zope-managableindex/branches/upstream/current/tests zope-managableindex/branches/upstream/current/zpt

Bernd Zeimetz bzed-guest at alioth.debian.org
Wed Apr 4 00:01:35 UTC 2007


Author: bzed-guest
Date: 2007-04-04 00:01:33 +0000 (Wed, 04 Apr 2007)
New Revision: 736

Added:
   zope-managableindex/
   zope-managableindex/branches/
   zope-managableindex/branches/upstream/
   zope-managableindex/branches/upstream/current/
   zope-managableindex/branches/upstream/current/Evaluation.py
   zope-managableindex/branches/upstream/current/FieldIndex.py
   zope-managableindex/branches/upstream/current/KeywordIndex.py
   zope-managableindex/branches/upstream/current/LICENSE.txt
   zope-managableindex/branches/upstream/current/ManagableIndex.py
   zope-managableindex/branches/upstream/current/PathIndex.py
   zope-managableindex/branches/upstream/current/RangeIndex.py
   zope-managableindex/branches/upstream/current/Utils.py
   zope-managableindex/branches/upstream/current/VERSION.txt
   zope-managableindex/branches/upstream/current/ValueProvider.py
   zope-managableindex/branches/upstream/current/WordIndex.py
   zope-managableindex/branches/upstream/current/__init__.py
   zope-managableindex/branches/upstream/current/doc/
   zope-managableindex/branches/upstream/current/doc/ManagableIndex.html
   zope-managableindex/branches/upstream/current/fixPluginIndexes.py
   zope-managableindex/branches/upstream/current/tests/
   zope-managableindex/branches/upstream/current/tests/TestBase.py
   zope-managableindex/branches/upstream/current/tests/__init__.py
   zope-managableindex/branches/upstream/current/tests/test_ManagableIndex.py
   zope-managableindex/branches/upstream/current/zpt/
   zope-managableindex/branches/upstream/current/zpt/addForm.zpt
   zope-managableindex/tags/
Log:
[svn-inject] Installing original source of zope-managableindex

Added: zope-managableindex/branches/upstream/current/Evaluation.py
===================================================================
--- zope-managableindex/branches/upstream/current/Evaluation.py	2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/Evaluation.py	2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,135 @@
+# Copyright (C) 2003 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+#       $Id: Evaluation.py,v 1.5 2006/04/09 16:52:03 dieter Exp $
+'''TALES evaluation for different purposes.'''
+
+from AccessControl import getSecurityManager
+from Acquisition import Explicit, Implicit
+from Persistence import Persistent
+
+from OFS.PropertyManager import PropertyManager
+from DocumentTemplate.DT_Util import safe_callable
+
+from Products.PageTemplates.Expressions import getEngine, SecureModuleImporter
+
+import ManagableIndex
+
+
+class Eval(PropertyManager,Explicit,Persistent):
+  '''evaluate a TALES expression and return the result.'''
+  # to be overridden by derived classes
+  ExpressionProperty= 'Expression'
+
+  _properties= (
+    { 'id' : ExpressionProperty, 'type' : 'string', 'mode' : 'w',},
+    )
+
+  Expression= ''
+
+  def __init__(self,exprProperty= None):
+    if exprProperty: self.ExpressionProperty= exprProperty
+
+  def _evaluate(self,value,object):
+    '''evaluate our expression property in a context containing
+    *value*, *object*, 'index', 'catalog' and standard names.'''
+    index= self._findIndex()
+    catalog= index.aq_inner.aq_parent # ATT: do we need to go up an additional level?
+    # work around bug in "aq_acquire" (has 'default' argument but ignores it)
+    try: request= object.aq_acquire('REQUEST',default=None)
+    except AttributeError: request= None
+    try: container= object.aq_inner.aq_parent
+    except AttributeError: container= None
+    data= {
+      'value' : value,
+      'index' : index,
+      'catalog' : catalog,
+      'object' : object,
+      'here' : object, # compatibility
+      'container' : container,
+      'nothing' : None,
+      'root' : index.getPhysicalRoot(),
+      #'request': object.aq_acquire('REQUEST',default=None),
+      'request': request,
+      'modules' : SecureModuleImporter,
+      'user' : getSecurityManager().getUser(),
+      }
+    context= getEngine().getContext(data)
+    expr= self._getExpression()
+    return expr(context)
+
+  def _findIndex(self):
+    '''return the nearest 'ManagableIndex' above us.'''
+    obj= self
+    while not isinstance(obj,ManagableIndex.ManagableIndex):
+      obj= obj.aq_inner.aq_parent
+    return obj
+
+  _v_expression= 0
+  _v_expr_string= None
+  def _getExpression(self):
+    '''return the TALES expression to be evaluated.'''
+    expr= self._v_expression
+    expr_string= self.aq_acquire(self.ExpressionProperty)
+    if expr != 0 and self._v_expr_string == expr_string: return expr
+    expr= self._v_expression= getEngine().compile(expr_string)
+    self._v_expr_string= expr_string
+    return expr
+
+  def _getExpressionString(self):
+    # this uses 'aq_aquire', because we want to support independent multiple inheritance
+    return self.aq_acquire(self.ExpressionProperty)
+
+
+class EvalAndCall(Eval):
+  '''evaluate and then call with *value*, if possible.'''
+  def _evaluate(self,value,object):
+    v= EvalAndCall.inheritedAttribute('_evaluate')(self,value,object)
+    if safe_callable(v): v= v(value)
+    return v
+
+
+class Ignore(PropertyManager,Implicit):
+  '''ignore values for which 'IgnorePredicate' gives true.'''
+  IgnoreProperty= 'IgnorePredicate'
+
+  _properties= (
+    {'id' : IgnoreProperty, 'type' : 'string', 'mode' : 'w',},
+    )
+  IgnorePredicate= ''
+
+  _v_IgnoreEvaluator= None
+  def _ignore(self,value,object):
+    if value is None: return
+    if not self._hasIgnorer(): return value
+    evaluator= self._v_IgnoreEvaluator
+    if evaluator is None:
+      evaluator= self._v_IgnoreEvaluator= EvalAndCall(self.IgnoreProperty)
+      evaluator= evaluator.__of__(self)
+    if evaluator._evaluate(value,object): return
+    return value
+
+  def _hasIgnorer(self):
+    return getattr(self,self.IgnoreProperty,None)
+
+class Normalize(PropertyManager,Implicit):
+  '''normalize value by the 'Normalizer' expression.'''
+  NormalizerProperty= 'Normalizer'
+
+  _properties= (
+    {'id' : NormalizerProperty, 'type' : 'string', 'mode' : 'w',},
+    )
+  Normalizer= ''
+
+  _v_NormalizeEvaluator= None
+  def _normalize(self,value,object):
+    if value is None: return
+    if not self._hasNormalizer(): return value
+    evaluator= self._v_NormalizeEvaluator
+    if evaluator is None:
+      evaluator= self._v_NormalizeEvaluator= EvalAndCall(self.NormalizerProperty)
+      evaluator= evaluator.__of__(self)
+    if not evaluator._getExpressionString(): return value
+    return evaluator._evaluate(value,object)
+
+  def _hasNormalizer(self):
+    return getattr(self,self.NormalizerProperty,None)

Added: zope-managableindex/branches/upstream/current/FieldIndex.py
===================================================================
--- zope-managableindex/branches/upstream/current/FieldIndex.py	2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/FieldIndex.py	2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,43 @@
+# Copyright (C) 2003 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+#       $Id: FieldIndex.py,v 1.3 2006/04/09 16:52:03 dieter Exp $
+'''Managable FieldIndex.'''
+
+from ManagableIndex import ManagableIndex, addForm
+
+class FieldIndex(ManagableIndex):
+  '''a managable 'FieldIndex'.'''
+  meta_type= 'Managable FieldIndex'
+
+  _properties = (
+    ManagableIndex._properties
+    + (
+    {'id':'ReverseOrder',
+     'label':'Maintain reverse order (used by AdvancedQuery to efficiently support descending order). Remember to clear the index when you change this value!',
+     'type':'boolean', 'mode':'rw',},
+    )
+    )
+
+  def _indexValue(self,documentId,val,threshold):
+    self._insert(val,documentId)
+    return 1
+
+  def _unindexValue(self,documentId,val):
+    self._remove(val,documentId)
+
+  # newly required for Zope 2.7
+  def documentToKeyMap(self):
+    '''must return a map from document ids to object value.'''
+    return self._unindex
+
+  # filtering support
+  supportFiltering = True
+
+
+def addFieldIndexForm(self):
+  '''add FieldIndex form.'''
+  return addForm.__of__(self)(
+    type= FieldIndex.meta_type,
+    description= '''A FieldIndex indexes an object under a single (atomic) value.''',
+    action= 'addIndex',
+    )

Added: zope-managableindex/branches/upstream/current/KeywordIndex.py
===================================================================
--- zope-managableindex/branches/upstream/current/KeywordIndex.py	2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/KeywordIndex.py	2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,115 @@
+# Copyright (C) 2003 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+#       $Id: KeywordIndex.py,v 1.8 2006/05/17 19:53:07 dieter Exp $
+'''Managable KeywordIndex.'''
+
+from sys import maxint
+
+from BTrees.OOBTree import difference, OOSet, union, OOTreeSet
+
+from ManagableIndex import ManagableIndex, addForm
+
+
+class KeywordIndex(ManagableIndex):
+  '''a managable 'KeywordIndex'.'''
+  meta_type= 'Managable KeywordIndex'
+
+  Combiners= ManagableIndex.Combiners + ('union',)
+  CombineType= Combiners[-1]
+
+  def _createDefaultValueProvider(self):
+    ManagableIndex._createDefaultValueProvider(self)
+    # add (value) normalizer to let it behave identical to
+    # a standard Zope KeywordIndex
+    vp= self.objectValues()[0]
+    setattr(vp, vp.NormalizerProperty, 'python: hasattr(value,"capitalize") and (value,) or value')
+
+  def _indexValue(self,documentId,val,threshold):
+    if not threshold: threshold= maxint
+    n= i= 0; T= get_transaction()
+    for v in val.keys():
+      self._insert(v,documentId)
+      n+=1; i+=1
+      if i == threshold: T.commit(1); i= 0
+    return n
+
+  def _unindexValue(self,documentId,val):
+    for v in val.keys():
+      self._remove(v,documentId)
+
+  def _update(self,documentId,val,oldval,threshold):
+    add= difference(val,oldval)
+    rem= difference(oldval,val)
+    if add: self._indexValue(documentId,add,threshold)
+    if rem: self._unindexValue(documentId,rem)
+    self._updateOldval(oldval, val, add, rem)
+    return len(add),
+
+  def _updateOldval(self, oldval, newval, add, rem):
+    # optimize transaction size by not writing _unindex bucket
+    oldval.clear(); oldval.update(newval)
+
+  def _equalValues(self,val1,val2):
+    if val1 == val2: return 1
+    if val1 is None or val2 is None: return 0
+    return tuple(val1.keys()) == tuple(val2.keys())
+
+  def _combine_union(self,values,object):
+    if not values: return
+    set= None
+    for v in values:
+      sv= self._standardizeValue(v,object)
+      if not sv: continue
+      set= union(set,sv)
+    return set
+
+  _SETTYPE = OOSet
+
+  def _standardizeValue(self,value,object):
+    '''convert to a set of standardized terms.'''
+    if not value: return
+    set= self._SETTYPE([st for st in [self._standardizeTerm(t,object) for t in value] if st is not None])
+    return set or None
+
+  # filtering support
+  supportFiltering = True
+
+  def _makeFilter(self, pred):
+    '''a document filter 'did -> True/False' checking term predicate *pred*.'''
+    def check(did):
+      dv = self._unindex.get(did)
+      if dv is None: return False
+      for t in dv.keys():
+        if pred(t): return True
+      return False
+    return check
+
+def addKeywordIndexForm(self):
+  '''add KeywordIndex form.'''
+  return addForm.__of__(self)(
+    type= KeywordIndex.meta_type,
+    description= '''A KeywordIndex indexes an object under a set of terms.''',
+    action= 'addIndex',
+    )
+
+class KeywordIndex_scalable(KeywordIndex):
+  '''a Keyword index that can efficiently handle huge keyword sets per object.'''
+  _SETTYPE = OOTreeSet
+  meta_type = 'Managable KeywordIndex (scalable)'
+
+  def _updateOldval(self, oldval, newval, add, rem):
+    for t in rem: oldval.remove(t)
+    oldval.update(add)
+
+def addKeywordIndex_scalableForm(self):
+  '''add KeywordIndex form.'''
+  return addForm.__of__(self)(
+    type= KeywordIndex_scalable.meta_type,
+    description= '''A KeywordIndex (scalable) indexes an object under a (potentially huge) set of terms.''',
+    action= 'addIndex',
+    )
+
+try:
+  import transaction # ZODB 3.4 (Zope 2.8)
+  def get_transaction(): return transaction
+except ImportError: pass # pre ZODB 3.4

Added: zope-managableindex/branches/upstream/current/LICENSE.txt
===================================================================
--- zope-managableindex/branches/upstream/current/LICENSE.txt	2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/LICENSE.txt	2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,32 @@
+Copyright (C) 2003-2006 by Dr. Dieter Maurer <dieter at handshake.de>
+D-66386 St. Ingbert, Eichendorffstr. 23, Germany
+
+		All Rights Reserved
+
+Permission to use, copy, modify, and distribute this software and its
+documentation for any purpose and without fee is hereby granted,
+provided that the above copyright notice and this permission
+notice appear in all copies, modified copies and in
+supporting documentation.
+
+Dieter Maurer DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL Dieter Maurer
+BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL
+DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
+PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
+
+
+IMPORTANT NOTE: I consider to forbid institutions of the
+European Union (Commission, Council of Ministers, Parliament, ...)
+to use software developped by me.
+This is in protest against plans of the Council of Ministers to make
+logic (general ideas outside of a concrete technical system)
+and especially software patentable in Europe.
+Note that with such a restriction, this software will cease to be
+compatible with the GPL (GNU Public License).
+This version does not yet have the restriction but a future version
+may.
+

Added: zope-managableindex/branches/upstream/current/ManagableIndex.py
===================================================================
--- zope-managableindex/branches/upstream/current/ManagableIndex.py	2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/ManagableIndex.py	2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,963 @@
+# Copyright (C) 2003-2006 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+#       $Id: ManagableIndex.py,v 1.16 2006/11/23 19:43:04 dieter Exp $
+'''Managable Index abstract base class.'''
+
+import copy
+from types import IntType, LongType, FloatType, \
+     StringType, UnicodeType, \
+     TupleType, InstanceType
+from sys import modules, getdefaultencoding
+from re import escape, compile
+
+from Globals import InitializeClass
+from Acquisition import aq_base
+from AccessControl import ClassSecurityInfo
+from BTrees.IOBTree import IOBTree
+from BTrees.IIBTree import IISet, IITreeSet, union, intersection, multiunion
+from BTrees.OOBTree import OOBTree, OOTreeSet
+from BTrees.Length import Length
+from DateTime.DateTime import DateTime
+
+from Products.OFolder.OFolder import OFolder
+from Products.PluginIndexes.common.PluggableIndex import PluggableIndexInterface
+from Products.PluginIndexes.common.util import parseIndexRequest
+from Products.PageTemplates.PageTemplateFile import PageTemplateFile
+
+from fixPluginIndexes import parseIndexRequest
+from Evaluation import Normalize, Ignore, EvalAndCall
+from ValueProvider import AttributeLookup, ExpressionEvaluator
+from Utils import reverseOrder, _LazyMap, \
+     convertToDateTime, convertToDateTimeInteger, convertToDateInteger
+
+_mdict= globals()
+
+ManageManagableIndexes= "ManagableIndex: manage"
+
+
+_TermTypeList= (
+      'not checked',
+      'string',
+      'ustring',
+      'integer',
+      'numeric',
+      'DateTime',
+      'DateTimeInteger',
+      'DateInteger',
+      'tuple',
+      'instance',
+      'expression checked',
+      )
+
+_IntegerTypes = 'integer DateTimeInteger DateInteger'.split()
+
+_TermCopyList= (
+      'none',
+      'shallow',
+      'deep',
+      )
+
+
+class ManagableIndex(OFolder,Normalize, Ignore):
+  '''Abstract base class for 'ManagableIndex'.'''
+
+  security= ClassSecurityInfo()
+  security.declareProtected(
+    ManageManagableIndexes,
+    'addAttributeLookupForm',
+    'addExpressionEvaluatorForm',
+    'addValueProvider',
+    'manage_propertiesForm', 'manage_changeProperties', 'manage_editProperties',
+    'indexSize',
+    )
+  security.declarePrivate(
+    'getReverseOrder',
+    )
+
+
+  icon= 'misc_/PluginIndexes/index.gif'
+
+  manage_options= (
+    OFolder.manage_options[:1]
+    + OFolder.manage_options[2:]
+    )
+
+  operators= ('or', 'and',)
+  useOperator= 'or'
+  query_options= ('operator', 'range', 'usage', 'match', 'isearch', 'isearch_filter')
+  Combiners= ('useFirst',)
+
+  NormalizerProperty= 'NormalizeTerm'
+  IgnoreProperty= 'StopTermPredicate'
+
+  _properties= (
+    (
+      { 'id' : 'CombineType', 'type' : 'selection', 'mode' : 'w', 'select_variable' : 'listCombiners',
+        'label':'Combine type: determines how values from value providers are combined'},
+      {'id':'PrenormalizeTerm', 'type':'string', 'mode':'w',
+       'label':'Term prenormalizer: applied to terms before term expansion in queries and (always) before stop term elimination; used e.g. for case normalization, stemming, phonetic normalization, ...',},
+      {'id' : IgnoreProperty, 'type' : 'string', 'mode' : 'w',
+       'label':'Stop term predicate: used to recognized and eliminate stop terms; used always (except in range queries) after prenormalization',},
+      {'id' : NormalizerProperty, 'type' : 'string', 'mode' : 'w',
+       'label':'Term normalizer: applied to terms before type checking; used e.g. for encoding the term into a efficient form',},
+      { 'id' : 'TermType', 'type' : 'selection', 'mode' : 'w', 'select_variable' : 'listTermTypes',
+        'label':'Term type: used to convert and check the terms type; may allows to choose specially optimized index structures (e.g. for integer types) or provide additional features (e.g. term expansions for string types) -- clear+reindex after change!',
+        },
+      { 'id' : 'TermTypeExtra', 'type' : 'string', 'mode' : 'w',
+        'label':'Term type argument: required by some term types (see the documentation)',},
+      { 'id' : 'TermCopy', 'type' : 'selection', 'mode' : 'w', 'select_variable' : 'listCopyTypes',
+        'label':'Control term copying: may be necessary for mutable terms to prevent index corruption',},
+      )
+    )
+  TermType= _TermTypeList[0]
+  TermTypeExtra= ''
+  TermCopy= _TermCopyList[0]
+  CombineType= Combiners[0]
+  NormalizeTerm= ''
+  PrenormalizeTerm= ''
+  StopTermPredicate= ''
+
+  __implements__= PluggableIndexInterface
+
+  def __init__(self, name, extra=None, caller=None):
+    self.id= name
+    def setProperties(obj, values, special):
+      __traceback_info__ = obj
+      allowed = dict.fromkeys(special)
+      allowed.update(dict.fromkeys([p['id'] for p in obj._properties]))
+      for k in values.keys():
+        if k not in allowed: raise ValueError('not a known property: %s' % k)
+      obj.manage_changeProperties(**values)
+    if extra: setProperties(self, extra, ('ValueProviders',))
+    providers = extra and extra.get('ValueProviders')
+    if providers is None: self._createDefaultValueProvider()
+    else:
+      for p in providers:
+        vp = self.addValueProvider(p['id'], p['type'])
+        setProperties(vp, p, ('id', 'type',))
+    self.clear()
+
+  def clear(self):
+    '''clear the index.'''
+    l = self.__len__
+    if isinstance(l, Length): l.set(0)
+    else: self.__len__ = Length()
+    try: self.numObjects.set(0)
+    except AttributeError: self.numObjects= Length()
+    if self.ReverseOrder: self._reverseOrder = OOTreeSet()
+    self._setup()
+
+  def __len__(self):
+    '''Python 2.4 requires this to be defined inside the class.'''
+    l = self.__len__
+    if not isinstance(l, Length): l = self.__len__ = Length()
+    return l()
+
+  def indexSize(self):
+    return self.__len__()
+
+  def _setup(self):
+    self._unindex= IOBTree()
+    treeType = self.TermType in _IntegerTypes and IOBTree or OOBTree
+    self._index= treeType()
+
+  def _createDefaultValueProvider(self):
+    self.addValueProvider(self.id,'AttributeLookup')
+
+  ## term expansion -- provided for indexes with declared "string" and "ustring"
+  ## term types
+  def matchGlob(self, t):
+    '''match '*' (match any sequence) and '?' (match any character) in *t* returning a list of terms.'''
+    # leads to wrong result -- would need to check against index
+    # if not ('*' in t or '?' in t): return [t]
+    regexp = glob2regexp(t)
+    return self.matchRegexp(regexp+'$')
+
+  _matchType = None
+  def matchRegexp(self, regexp):
+    '''match *regexp* into a list of matching terms.
+
+    Note that for efficiency reasons, the regular expression
+    should have an easily recognizable plain text prefix -- at
+    least for large indexes.
+    '''
+    prefix, regexp = _splitPrefixRegexp(regexp)
+    termType = self._matchType or self.TermType
+    if termType == 'string': prefix = str(prefix); regexp = str(regexp)
+    elif termType == 'ustring': prefix = unicode(prefix); regexp = unicode(regexp)
+    elif termType == 'asis': pass
+    else: raise ValueError(
+      "Index %s has 'TermType/MatchType' %s not supporting glob/regexp expansion"
+      % (self.id, termType)
+      )
+    index = self._getMatchIndex(); pn = len(prefix)
+    l = []; match = compile(regexp).match
+    for t in index.keys(prefix): # terms starting prefix
+      if not t.startswith(prefix): break
+      if match(t[pn:]): l.append(t)
+    return l
+
+  def _getMatchIndex(self):
+    '''the index used for expansion'''
+    return self._index
+
+  ## match filtering
+  def matchFilterGlob(self, t):
+    '''see 'matchGlob' but for filtering.'''
+    regexp = glob2regexp(t)
+    return self.matchFilterRegexp(regexp+'$')
+
+  def matchFilterRegexp(self, regexp):
+    '''see 'matchRegexp' but for filtering.'''
+    termType = self._matchType or self.TermType
+    if termType == 'string': regexp = str(regexp)
+    elif termType == 'ustring': regexp = unicode(regexp)
+    elif termType == 'asis': pass
+    else: raise ValueError(
+      "Index %s has 'TermType/MatchType' %s not supporting glob/regexp expansion"
+      % (self.id, termType)
+      )
+    return compile(regexp).match
+
+
+  ## Responsibilities from 'PluggableIndexInterface'
+  # 'getId' -- inherited from 'SimpleItem'
+
+  def getEntryForObject(self,documentId, default= None):
+    '''Information for *documentId*.'''
+    info= self._unindex.get(documentId)
+    if info is None: return default
+    return repr(info)
+
+
+  def index_object(self,documentId,obj,threshold=None):
+    '''index *obj* as *documentId*.
+
+    Commit subtransaction when *threshold* index terms have been indexed.
+    Return the number of index terms indexed.
+    '''
+    # Note: objPath should be provided by the catalog -- but it is not
+    try: objPath = obj.getPhysicalPath()
+    except: objPath = None
+    __traceback_info__ = self.id, objPath
+
+    val= self._evaluate(obj)
+
+    # see whether something changed - do nothing, if it did not
+    oldval= self._unindex.get(documentId)
+    if val == oldval: return 0
+    # some data types, e.g. "OOSet"s do not define a senseful "==".
+    #  provide a hook to handle this case
+    customEqual= self._equalValues
+    if customEqual is not None and customEqual(val,oldval): return 0
+
+    # remove state info
+    update= self._update
+    if update is None or oldval is None or val is None:
+      # no optimization
+      if oldval is not None: self._unindex_object(documentId,oldval,val is None)
+      if val is None: return 0
+      rv= self._indexValue(documentId,val,threshold)
+      if oldval is None: self.numObjects.change(1)
+    else:
+      # optimization
+      rv= update(documentId,val,oldval,threshold)
+      if isinstance(rv, tuple): return rv[0]
+
+    # remember indexed value
+    self._unindex[documentId]= val
+    return rv
+
+
+  def unindex_object(self,documentId):
+    '''unindex *documentId*.
+
+    ATT: why do we not have a *threshold*????
+    '''
+    # Note: objPath/documentId should be provided by the catalog -- but it is not
+    __traceback_info__ = self.id, documentId
+
+    val= self._unindex.get(documentId)
+    if val is None: return # not indexed
+    self._unindex_object(documentId,val,1)
+
+  def _unindex_object(self,documentId,val,remove):
+    self._unindexValue(documentId,val)
+    if remove:
+      del self._unindex[documentId]
+      self.numObjects.change(-1)
+
+
+  def uniqueValues(self, name=None, withLengths=0):
+    '''unique values for *name* (???).
+
+    If *withLength*, returns sequence of tuples *(value,length)*.
+    '''
+    if name is None: name= self.id
+    if name != self.id: return ()
+    values= self._index.keys()
+    if not withLengths: return tuple(values)
+    return tuple([(value,self._len(value)) for value in values])
+
+
+  def _apply_index(self,request, cid= ''):
+    '''see 'PluggableIndex'.
+
+    What is *cid* for???
+    '''
+    __traceback_info__ = self.id
+
+    record= parseIndexRequest(request, self.id, self.query_options)
+    terms= record.keys
+    if terms is None: return
+
+    __traceback_info__ = self.id, record.keys
+
+    op= record.get('operator', self.useOperator)
+    if op not in self.operators:
+      raise ValueError("operator not permitted: %s" % op)
+    combine= op == 'or' and union or intersection
+
+    filteredSearch = None
+    if record.get('isearch') and record.get('isearch_filter') \
+       and self.supportFiltering and IFilter is not None:
+      filteredSearch = self._getFilteredISearch(record)
+
+    if filteredSearch is None:
+      match = record.get('match')
+      if match is not None:
+        l = []; match = getattr(self, 'match' + match.capitalize())
+        prenorm = self._prenormalizeTerm
+        for t in terms:
+          t = prenorm(t, None)
+          if t is not None: l.extend(match(t))
+        terms = l
+
+      range= record.get('range')
+      if range is not None:
+        terms= [self._standardizeTerm(t,elimStopTerm=0, prenormalize=not match) for t in terms]
+        range= range.split(':'); lo= hi= None
+        if 'min' in range: lo= min(terms)
+        if 'max' in range: hi= max(terms)
+        terms= self._enumerateRange(lo,hi)
+      else:
+        terms= [self._standardizeTerm(t, prenormalize=not match) for t in terms]
+
+    if filteredSearch is None: r = self._search(terms,combine,record)
+    else: r = filteredSearch
+    if r is None: return
+    return r, self.id
+
+
+  #################################################################
+  # search
+  def _search(self,terms,combine,record):
+    return setOperation(
+      combine is union and 'or' or 'and',
+      [self._searchTerm(t,record) for t in terms],
+      record.get('isearch'),
+      )
+
+  def _searchTerm(self,term,record):
+    return self._load(term)
+
+  def _enumerateRange(self,min,max):
+    '''enumerate terms between *min* and *max*.'''
+    return self._index.keys(min,max)
+
+
+  #################################################################
+  # filtering
+  supportFiltering = False
+
+  def _getFilteredISearch(self, query):
+    '''return a filtered search for *query*, if this seems promissing, or 'None'.
+    '''
+    preds = []
+    enumerator = self._getFilterEnumerator(); makeFilter = self._makeFilter
+
+    terms = query.keys
+    match = query.get('match'); range = query.get('range')
+    op = query.get('operator', self.useOperator)
+
+    if match is not None:
+      # we do not want to filter combined 'match' and 'range' queries
+      if range is not None: return
+      # can only filter 'or' matches
+      if op != 'or': return
+      # we can filter 'match' queries only if there is no 'normalizer'
+      #  maybe, we should not filter, if there is an 'ignorer'?
+      if self._hasNormalizer(): return
+      l = []; match = getattr(self, 'matchFilter' + match.capitalize())
+      prenorm = self._prenormalizeTerm
+      for t in terms:
+        t = prenorm(t, None)
+        if t is not None: preds.append(match(t))
+    else:
+      range= query.get('range')
+      if range is not None:
+        # can only filter 'or' ranges
+        if op != 'or': return
+        terms= [self._standardizeTerm(t,elimStopTerm=0, prenormalize=not match) for t in terms]
+        range= range.split(':'); lo= hi= None
+        if 'min' in range: lo= min(terms)
+        if 'max' in range: hi= max(terms)
+        preds.append(_rangeChecker(lo,hi))
+      else:
+        makePred = self._makeTermPredicate; standardize = self._standardizeTerm
+        preds = [
+          makePred(standardize(t, prenormalize=not match))
+          for t in terms
+          ]
+    subsearches = [IFilter(makeFilter(pred), enumerator) for pred in preds]
+
+    return self._combineSubsearches(subsearches, op)
+
+  def _combineSubsearches(self, subsearches, op):
+    if len(subsearches) == 1: return subsearches[0]
+    combine = op == 'or' and IOr or IAnd
+    search = combine(*subsearches); search.complete()
+    return search
+
+  def _getFilterEnumerator(self):
+    return Enumerator(self._unindex)
+
+  def _makeTermPredicate(self, term):
+    '''this is adequate for field and keyword indexes.'''
+    return lambda x, t=term: x == t
+
+  def _makeFilter(self, pred):
+    '''a document filter 'did -> True/False' checking term predicate *pred*.
+
+    This definition is adequate, when the predicate can be directly
+    applied to the 'unindex' value.
+    '''
+    def check(did):
+      dv = self._unindex.get(did)
+      return dv is not None and pred(dv)
+    return check
+
+
+  #################################################################
+  # required for sorting
+  # no longer needed for Zope 2.7 -- keep for compatibility
+  def keyForDocument(self, docId): return self._unindex[docId]
+  def items(self):
+    return [(k,self._load(k)) for k in self._index.keys()]
+
+
+  #################################################################
+  # Reverse ordering support
+  def getReverseOrder(self):
+    '''return the keys in reverse order.'''
+    if self.ReverseOrder:
+      return _LazyMap(lambda x: x.getValue(), self._reverseOrder.keys())
+
+
+  #################################################################
+  # Storage API
+  # we implement a small optimization
+  # a single document is stored as integer; more are stored as an IITreeSet
+  ReverseOrder = 0
+
+  def _insert(self,term,docId, _isInstance= isinstance, _IntType= IntType):
+    '''index *docId* under *term*.'''
+    index= self._index
+    dl= index.get(term)
+    if dl is None:
+      index[term]= docId; self.__len__.change(1)
+      if self.ReverseOrder: self._reverseOrder.insert(reverseOrder(term))
+      return
+    if _isInstance(dl,_IntType): dl= index[term]= IITreeSet((dl,))
+    dl.insert(docId)
+
+  def _remove(self,term,docId, _isInstance= isinstance, _IntType= IntType):
+    '''unindex *docId* under *term*.'''
+    index= self._index
+    dl= index.get(term); isInt= _isInstance(dl,_IntType)
+    if dl is None or isInt and dl != docId:
+      raise ValueError('Attempt to remove nonexisting document %s from %s'
+                       % (docId, self.id)
+                       )
+    if isInt: dl = None
+    else: dl.remove(docId)
+    if not dl:
+      del index[term]; self.__len__.change(-1)
+      if self.ReverseOrder: self._reverseOrder.remove(reverseOrder(term))
+
+  def _load(self,term, _isInstance= isinstance, _IntType= IntType):
+    '''the docId list for *term*.'''
+    index= self._index
+    dl= index.get(term)
+    if dl is None: return IISet()
+    if _isInstance(dl,_IntType): return IISet((dl,))
+    return dl
+
+  def _len(self,term):
+    '''the number of documents indexed under *term*.'''
+    return len(self._load(term))
+
+
+  ###########################################################################
+  ## methods to maintain auxiliary indexes
+  ## we implement the same optimization as for the main index
+  def _insertAux(self, index, term, docId):
+    '''index *docId* under *term*.'''
+    dl= index.get(term)
+    if dl is None: index[term]= docId; return
+    if isinstance(dl,int): dl= index[term]= IITreeSet((dl,))
+    dl.insert(docId)
+
+  def _removeAux(self, index, term, docId):
+    '''unindex *docId* under *term*.'''
+    dl= index.get(term); isInt= isinstance(dl,int)
+    if dl is None or isInt and dl != docId:
+      raise ValueError('Attempt to remove nonexisting document %s from %s'
+                       % (docId, self.id)
+                       )
+    if isInt: dl = None
+    else: dl.remove(docId)
+    if not dl: del index[term]
+
+  def _loadAux(self,index, term):
+    '''the docId list for *term*.'''
+    dl= index.get(term)
+    if dl is None: return IISet()
+    if isinstance(dl,int): return IISet((dl,))
+    return dl
+    
+
+
+  #################################################################
+  # Term standardization and checking
+  def listTermTypes(self):
+    '''the sequence of supported term types.'''
+    return _TermTypeList
+
+  def listCopyTypes(self):
+    '''the sequence of term copy types.'''
+    return _TermCopyList
+
+  def listCombiners(self):
+    '''the sequence of combine types.'''
+    return self.Combiners
+
+  def _standardizeTerm(self, value, object=None, copy=False, elimStopTerm=True, prenormalize=True):
+    if prenormalize:
+      value = self._prenormalizeTerm(value, object)
+      if value is None: return
+    if elimStopTerm:
+      value= self._ignore(value,object)
+      if value is None: return
+    value= self._normalize(value,object)
+    if value is None: return
+    tt= self.TermType
+    checker= _TermTypeMap[tt]
+    if checker: value= checker(self,value,object)
+    if copy and tt in ('not checked', 'instance', 'expression checked',):
+      copy= _CopyTypeMap[self.TermCopy]
+      if copy: value= copy(value)
+    return value
+
+  _prenormalizer = None
+  def _prenormalizeTerm(self, value, object):
+    PT = self.PrenormalizeTerm
+    if not PT: return value
+    normalizer = self._prenormalizer
+    if normalizer is None:
+      normalizer = self._prenormalizer = Normalize()
+      normalizer.NormalizerProperty = 'PrenormalizeTerm'
+    return normalizer._normalize(value, object)
+    
+
+
+  #################################################################
+  # Evaluation
+  def _evaluate(self,object):
+    '''evaluate *object* with respect to this index.'''
+    l= []; v= None
+    combiner= self.CombineType; useFirst= combiner == 'useFirst'
+    for vp in self.objectValues():
+      v= vp.evaluate(object)
+      if v is not None:
+        if useFirst: break
+        l.append(v)
+    if useFirst:
+      if v is None: return
+      return self._standardizeValue(v,object)
+    return getattr(self,'_combine_' + combiner)(l,object)
+
+  def _standardizeValue(self,value,object):
+    return self._standardizeTerm(value,object,1)
+    
+
+
+  #################################################################
+  # to be defined by concrete deriving classes
+  # _indexValue(self,documentId,val,threshold)
+  # _unindexValue(self,documentId,val)
+
+
+  #################################################################
+  # optionally defined by concrete deriving classes
+  # _update(self,documentId,val,oldval,threshold)
+  #   returns number of entries added; if tuple, _unindex already updated
+  # _equalValues(self,val1,val2) -- true, if standardized values are equal
+  _update= None
+  _equalValues= None
+
+
+  #################################################################
+  # Value provider management
+  def all_meta_types(self):
+    return (
+      { 'name' : AttributeLookup.meta_type,
+        'action' : 'addAttributeLookupForm',
+        'permission' : ManageManagableIndexes,
+        },
+      { 'name' : ExpressionEvaluator.meta_type,
+        'action' : 'addExpressionEvaluatorForm',
+        'permission' : ManageManagableIndexes,
+        },
+      )
+
+  def addAttributeLookupForm(self):
+    '''addForm for 'AttributeLookup' value provider.'''
+    return addForm.__of__(self)(
+      type= 'AttributeLookup',
+      description= '''An AttributeLookup is a value provider which evaluates an object by looking up an attribute of the object.''',
+      action= 'addValueProvider',
+      )
+
+  def addExpressionEvaluatorForm(self):
+    '''addForm for 'ExpressionEvaluator' value provider.'''
+    return addForm.__of__(self)(
+      type= 'ExpressionEvaluator',
+      description= '''An ExpressionEvaluator is a value provider which evaluates an expression.''',
+      action= 'addValueProvider',
+      )
+
+  def addValueProvider(self,id,type, RESPONSE=None):
+    '''add a value provider with *id* of *type*.'''
+    if type not in ('AttributeLookup', 'ExpressionEvaluator'):
+      raise ValueError('unknown type')
+    cl= _mdict[type]
+    # try to avaid a name conflict
+    eid= id
+    if not id.endswith('_') and hasattr(aq_base(self),id): eid= id + '_'
+    vp= cl(); vp.id= eid
+    if id != eid and type == 'AttributeLookup': vp.Name= id
+    self._setObject(eid, vp)
+    vp= self._getOb(eid)
+    if RESPONSE is None: return vp
+    RESPONSE.redirect('%s/manage_workspace' % vp.absolute_url())
+
+    
+InitializeClass(ManagableIndex)
+
+
+#################################################################
+# Term checking and copying
+
+_CopyTypeMap= {
+  'none' : None,
+  'shallow' : copy.copy,
+  'deep' : copy.deepcopy,
+  }
+
+def _isNumeric(value, _NumericType= (IntType, FloatType, LongType,)):
+  try: return isinstance(value,_NumericType)
+  except TypeError: # pre 2.3
+    for t in _NumericType:
+      if isinstance(value,t): return 1
+  return 0
+
+def _isString(value, _StringType= (StringType, UnicodeType,)):
+  try: return isinstance(value,_StringType)
+  except TypeError: # pre 2.3
+    for t in _StringType:
+      if isinstance(value,t): return 1
+  return 0
+
+
+def _checkNumeric(index,value,object):
+  '''return *value*, maybe converted, if it is numeric.'''
+  # see whether is has already the correct type
+  if _isNumeric(value): return value
+  try:
+    if _isString(value):
+      if '.' in value or 'E' in value or 'e' in value: value= float(value)
+      else:
+        try: value= int(value)
+        except ValueError: value= long(value)
+  except:
+    raise TypeError("cannot convert %s to numeric" % str(value))
+  return value
+
+
+def _checkInteger(index,value,object):
+  '''return *value*, maybe converted, if it is integer.'''
+  if hasattr(index.aq_base, 'convertToInteger'):
+    return index.convertToInteger(value, object)
+  return int(value)
+
+
+def _checkString(index,value,object):
+  '''return *value*, maybe converted, if it is a string.'''
+  if isinstance(value,StringType): return value
+  try:
+    nv= str(value)
+  except: nv= None
+  if value is None or nv.startswith('<'):
+    raise TypeError("cannot convert %s to string" % str(value))
+  return nv
+
+
+def _checkUnicode(index,value,object, encode=None):
+  '''return *value*, maybe converted, if it is a unicode string.'''
+  if isinstance(value,UnicodeType): return value
+  try:
+    nv= unicode(value, encode or getdefaultencoding())
+  except:
+    raise TypeError("cannot convert %s to string" % str(value))
+  return nv
+
+def _checkUnicode_encode(index, value, object):
+  # use 'TermTypeExtra' as encoding
+  return _checkUnicode(index, value, object, index.TermTypeExtra)
+
+def _checkUnicode_encode2(index, value, object):
+  # use the value after ';' in 'TermTypeExtra' as encoding
+  encoding = index.TermTypeExtra
+  encoding = ';' in encoding and encoding.split(';',1)[1]
+  return _checkUnicode(index, value, object, encoding)
+
+
+def _checkDateTime(index,value,object):
+  '''return *value* (in sec since epoch), if it is a 'DateTime' instance.'''
+  if isinstance(value, float): return value
+  return convertToDateTime(value)._t # float
+
+def _checkDateTimeInteger(index, value, object):
+  return convertToDateTimeInteger(value)
+
+def _checkDateInteger(index, value, object):
+  return convertToDateInteger(value)
+
+def _checkInstance(index,value,object):
+  '''return *value*, if it is an instance of class 'index.TermTypeExtra'.'''
+  fullClassName= index.TermTypeExtra
+  cl= _findClass(fullClassName)
+  if isinstance(value,cl):
+    if hasattr(cl,'__cmp__'): return value
+    raise TypeError("class %s does not define '__cmp__'" % fullClassName)
+  raise TypeError("cannot convert %s to %s" % (str(value),fullClassName))
+
+
+def _findClass(cl):
+  '''return the class identified by full class name *cl*.'''
+  cs= cl.split('.')
+  mod,cn= '.'.join(cs[:-1]), cs[-1]
+  return getattr(modules[mod],cn)
+
+def _checkTuple(index,value,object):
+  '''return *value*, if it matches the tuple spec in 'index.TermTypeExtra'.'''
+  spec= index.TermTypeExtra.split(';',1)[0]
+  value,pos= _checkTuple_(index,value,spec,0)
+  if pos != len(spec):
+    raise TypeError("%s does not conform to %s" % (str(value),spec))
+  return value
+
+
+def _checkTuple_(index,value,spec,pos):
+  '''return *value*, if it conforms to *spec*.'''
+  if _isString(value):
+    raise TypeError("%s does not conform to %s" % (str(value),spec))
+  try:
+    value= tuple(value)
+  except TypeError:
+      raise TypeError("%s does not conform to %s" % (str(value),spec))
+  i= 0; n= len(value)
+  while i < n:
+    v= value[i]
+    if pos >= len(spec):
+      raise TypeError("%s does not conform to %s" % (str(value),spec))
+    si= spec[pos]; pos+= 1
+    if si == '(':
+      nv,pos= _checkTuple_(index,v,spec,pos)
+      if spec[pos] != ')':
+        raise TypeError("%s does not conform to %s" % (str(value),spec))
+      pos+= 1
+    else:
+      checker= _TupleCheck[si]
+      nv= checker(index,v,None)
+    if v != nv:
+      if isinstance(value,TupleType): value= list(value)
+      value[i]= nv
+    i+= 1
+  return value, pos
+
+def _checkWithExpression(index,value,object):
+  '''return 'index.TermTypeExtra' applied to *value*, if not None.'''
+  evaluator= getattr(index,'_v_checkEvaluator',None)
+  if evaluator is None:
+    evaluator= index._v_checkEvaluator= EvalAndCall('TermTypeExtra')
+    evaluator= evaluator.__of__(index)
+  nv= evaluator._evaluate(value,object)
+  if nv is None:
+    raise TypeError('%s is not accepted by %s' % (str(value),index.TermTypeExtra))
+  return nv
+    
+
+_TermTypeMap= {
+  'not checked' : None,
+  'numeric' : _checkNumeric,
+  'string' : _checkString,
+  'integer' : _checkInteger,
+  'ustring' : _checkUnicode_encode,
+  'DateTime' : _checkDateTime,
+  'DateTimeInteger' : _checkDateTimeInteger,
+  'DateInteger' : _checkDateInteger,
+  'tuple' : _checkTuple,
+  'instance' : _checkInstance,
+  'expression checked' : _checkWithExpression,
+  }
+
+_TupleCheck= {
+  'n' : _checkNumeric,
+  's' : _checkString,
+  'u' : _checkUnicode_encode2,
+  'd' : _checkDateTime,
+  }
+
+
+#################################################################
+# constructor support
+addForm= PageTemplateFile('zpt/addForm',_mdict)
+
+def addIndex(self,id,type, REQUEST= None, RESPONSE= None, URL3= None):
+  '''add index of *type* with *id*.'''
+  return self.manage_addIndex(id, type, extra=None,
+             REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)
+
+
+
+#################################################################
+# auxiliaries
+_escs = dict('a\a f\f n\n r\r t\t v\v'.split(' '))
+_escs.update(dict([(c,0) for c in 'AbBdDsSwWZ']))
+
+def _splitPrefixRegexp(regexp,
+                       special_=dict(map(None, '.^$*+?{}|[]()', ())),
+                       leftOps_=dict(map(None, '*+?{|', ())),
+                       escs_=_escs.get,
+                       ):
+  '''return pair of plain text prefix and remaining regexp.'''
+  if not regexp or regexp[0] in special_: return '', regexp
+  prefix = ''; i = 0; n = len(regexp)
+  while i < n:
+    c = regexp[i]
+    if c in special_:
+      if c in leftOps_: return prefix[:-1], escape(prefix[-1]) + regexp[i:]
+      return prefix, regexp[i:]
+    elif c == '\\':
+      c = regexp[i+1]
+      # could be optimized but who cares
+      if c.isdigit() or c == 'x': return prefix, regexp[i:]
+      ec = escs_(c, c)
+      if ec == 0: return prefix, regexp[i:]
+      prefix += ec # ATT: quadratic -- but we do not expect it to become huge
+      i += 1
+    else: prefix += c # ATT: quadratic -- see above
+    i += 1
+  return prefix, ''
+
+def glob2regexp(glob):
+    return escape(glob).replace(r'\*','.*', ).replace(r'\?','.')
+
+def _rangeChecker(lo, hi):
+  if lo is None and hi is None: return lambda x: True
+  if lo is None: return lambda x: x <= hi
+  if hi is None: return lambda x: lo <= x
+  return lambda x: lo <= x <= hi
+
+
+
+#################################################################
+# monkey patches
+
+# give ZCatalogIndexes an id such that urls are correct
+from Products.ZCatalog.ZCatalogIndexes import ZCatalogIndexes
+ZCatalogIndexes.id= "Indexes"
+
+
+
+#################################################################
+# setOperation -- using 'IncrementalSearch', if available
+def setOperation(op, sets, isearch):
+  '''perform *op* on *sets*. if *isearch*, return an incremental search.
+
+  *op* may be '"and"' or '"or"'.
+
+  Uses 'IncrementalSearch', if available.
+  '''
+  if not sets:
+    if op == 'and': return # None means all results
+    if isearch: search = IOr(); search.complete(); return search
+    return IISet()
+  # Note: "multiunion" is *much* faster than "IOr"!
+  #if IAnd is not None and (isearch or len(sets) > 1):
+  if IAnd is not None and (isearch or (op == 'and' and len(sets) > 1)):
+    isets = []
+    for set in sets:
+      if set is None:
+        # all results
+        if op == 'and': continue
+        else: return
+      if not isinstance(set, ISearch): set = IBTree(set)
+      isets.append(set)
+    if op == 'and' and not isets: return # empty 'and'
+    cl = op == 'and' and IAnd or IOr
+    if len(isets) == 1:
+      # do not wrap a one element search
+      search = isets[0]
+    else: search = cl(*isets); search.complete()
+    if isearch: return search
+    if hasattr(search, 'asSet'): r = search.asSet()
+    else: r = IISet(); r.__setstate__((tuple(search),))
+    return r
+  if op == 'or' and len(sets) > 5:
+    r = multiunion(sets)
+  else:
+    combine = op == 'and' and intersection or union
+    r= None
+    for set in sets: r= combine(r,set)
+    if r is None:
+      if combine is union: r = IISet()
+      else: return
+    if isearch: r = IBTree(r)
+  return r
+  
+
+
+IAnd = IOr = IBTree = IFilter = None
+
+# try IncrementalSearch2 (the C implementation of IncrementalSearch)
+try:
+  from IncrementalSearch2 import \
+       IAnd_int as IAnd, IOr_int as IOr, IBTree, ISearch
+  try: from IncrementalSearch2 import IFilter_int as IFilter, \
+       Enumerator
+  except ImportError: pass
+except ImportError: IAnd = None
+
+# try IncrementalSearch
+if IAnd is None:
+  try:
+    from IncrementalSearch import IAnd, IOr, IBTree, ISearch, \
+         EBTree as Enumerator
+    try:
+      from IncrementalSearch import IFilter
+    except ImportError: pass
+  except ImportError: pass
+

Added: zope-managableindex/branches/upstream/current/PathIndex.py
===================================================================
--- zope-managableindex/branches/upstream/current/PathIndex.py	2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/PathIndex.py	2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,203 @@
+# Copyright (C) 2004 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+#       $Id: PathIndex.py,v 1.4 2006/11/14 15:05:59 dieter Exp $
+'''Path Index.'''
+
+from BTrees.IOBTree import IOBTree
+from BTrees.IIBTree import IISet
+
+from ManagableIndex import ManagableIndex, addForm, setOperation, IFilter
+from FieldIndex import FieldIndex
+
+class PathIndex(ManagableIndex):
+  '''a managable 'PathIndex'.'''
+  meta_type= 'Managable PathIndex'
+
+  _properties = ManagableIndex._properties[:4]
+
+  query_options= ('operator', 'level', 'depth', 'isearch', 'isearch_filter')
+
+  def _setup(self):
+    PathIndex.inheritedAttribute('_setup')(self)
+    # create auxiliary index
+    self._lengthIndex = IOBTree()
+    self._depth = 0
+
+  # do we need this?
+  def uniqueValues(self, name=None, withLength=0): raise NotImplementedError
+
+  # we do not support range searches
+  def _enumerateRange(self, lo, hi): raise NotImplementedError
+
+  # we do not support sorting
+  def keyForDocument(self, docId): raise NotImplementedError
+  def items(self): raise NotImplementedError
+
+  # we do not support expanding
+  def _getExpansionIndex(self): raise NotImplementedError
+
+  # term normalization -- we currently do not handle unicode appropriately
+  def _normalize(self, value, object):
+    '''convert into a tuple.'''
+    value = PathIndex.inheritedAttribute('_normalize')(self, value, object)
+    if value is None: return
+    if hasattr(value, 'upper'):
+      value = value.split('/')
+      if not value[-1]: del value[-1]
+    return tuple(value)
+
+  # basic methods
+  def _indexValue(self, docId, val, threshold):
+    for t in enumerate(val): self._insert(t, docId)
+    vn = len(val)
+    self._insertAux(self._lengthIndex, vn, docId)
+    if vn > self._depth: self._depth = vn
+    return 1
+
+  def _unindexValue(self, docId, val):
+    for t in enumerate(val): self._remove(t, docId)
+    self._removeAux(self._lengthIndex, len(val), docId)
+
+  # modified storage -- we implement Zope's PathIndex scheme
+  #  At first, it appeared to be superior than our standard scheme.
+  #  However, the implementation proved this wrong: it is slightly
+  #  inferior (with respect to number of loads, load size and load time).
+  #  Nevertheless, we keep it as it will make the matching implementation
+  #  easier (should someone wants it).
+  # Our length reflects the number of different segs, not the number
+  #  of total index entries
+  def _insert(self, (pos, seg), docId):
+    index = self._index
+    si = index.get(seg)
+    if si is None: index[seg] = si = IOBTree(); self.__len__.change(1)
+    self._insertAux(si, pos, docId)
+
+  def _remove(self, (pos, seg), docId):
+    index = self._index
+    si = index[seg]
+    self._removeAux(si, pos, docId)
+    if not si: del index[seg]; self.__len__.change(1)
+
+  def _load(self, (pos, seg)):
+    index = self._index
+    si = index.get(seg)
+    if si is None: return IISet()
+    return self._loadAux(si, pos)
+
+
+  # search implementation
+  def _searchTerm(self, path, record):
+    level = record.get('level', 0)
+    depth = record.get('depth')
+    isearch = record.get('isearch')
+    if not path: return self._searchLength(level, depth, isearch)
+    if level is not None and level >= 0:
+      return self._searchAt(path, level, depth, isearch)
+    try: limit = self._depth + 1
+    except ValueError: limit = 0
+    limit -= len(path)
+    if level is not None: limit = min(limit, -level+1)
+    return setOperation(
+      'or',
+      [self._searchAt(path, l, depth, isearch) for l in range(limit)],
+      isearch,
+      )
+
+  def _searchAt(self, path, pos, depth, isearch):
+    '''search for *path* at *pos* restricted by *depth*.'''
+    load = self._load
+    sets = [load((i+pos, seg)) for (i,seg) in enumerate(path)]
+    if depth is not None:
+      sets.append(self._searchDepth(depth, pos + len(path), isearch))
+    return setOperation('and', sets, isearch)
+
+  def _searchDepth(self, depth, len, isearch):
+    li = self._lengthIndex; load = self._loadAux
+    if depth >= 0: return load(li, len+depth)
+    return setOperation('or',
+                        [load(li, d) for d in range(len, len-depth+1)],
+                        isearch,
+                        )
+
+  def _searchLength(self, level, depth, isearch):
+    try: limit = self._depth + 1
+    except: limit = 0
+    if level is None: level = -limit
+    if depth is None: depth = -limit
+    lo = hi = 0
+    if level >= 0: lo += level; hi += level
+    else: hi += -level
+    if depth >= 0: lo += depth; hi += depth
+    else: hi += -depth
+    li = self._lengthIndex; load = self._loadAux
+    if lo == hi: return load(li, lo)
+    return setOperation('or',
+                        [load(li,l) for l in range(lo, min(hi+1,limit))],
+                        isearch,
+                        )
+
+  # filtering support
+  supportFiltering = True
+
+  def _getFilteredISearch(self, query):
+    terms = query.keys
+    level = query.get('level', 0)
+    depth = query.get('depth')
+    try: limit = self._depth + 1
+    except: limit = 0
+    if level is None: level = -limit
+    if depth is None: depth = -limit
+    op = query.get('operator', self.useOperator)
+    if depth >= 0:
+      if level >= 0:
+        def predFactory(t):
+          tn = len(t)
+          rn = level + depth + tn
+          return lambda v: len(v) == rn and v[level:level+tn] == t
+      else: # level < 0
+        def predFactory(t):
+          tn = len(t)
+          rn = depth + tn
+          def pred(v):
+            vn = len(v)
+            return vn >= rn and rn - vn <= -level and v[-rn:-rn+tn] == t
+          return pred
+    else: # depth < 0
+      if level >= 0:
+        def predFactory(t):
+          tn = len(t)
+          rn = level + tn; mn = rn - depth
+          def pred(v):
+            vn = len(v)
+            return rn <= vn <= mn and v[level:level+tn] == t
+          return pred
+      else: # level < 0
+        def predFactory(t):
+          tn = len(t)
+          def pred(v):
+            vn = len(v)
+            for l in range(max(0, vn-tn+depth), min(-level,vn-tn)+1):
+              if v[l:l+tn] == t: return True
+            return False
+          return pred
+    subsearches = []
+    makeFilter = self._makeFilter; enumerator = self._getFilterEnumerator()
+    for t in terms:
+      t = self._standardizeTerm(t)
+      subsearches.append(IFilter(makeFilter(predFactory(t)), enumerator))
+    return self._combineSubsearches(subsearches, op)
+
+
+    
+
+
+def addPathIndexForm(self):
+  '''add PathIndex form.'''
+  return addForm.__of__(self)(
+    type= PathIndex.meta_type,
+    description= '''A PathIndex indexes an object under a path (a tuple or '/' separated string).
+    It can be queried for objects the path of which contains a given path with
+    various possibilities to contrain where the given path must
+    lie within the objects path.''',
+    action= 'addIndex',
+    )

Added: zope-managableindex/branches/upstream/current/RangeIndex.py
===================================================================
--- zope-managableindex/branches/upstream/current/RangeIndex.py	2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/RangeIndex.py	2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,247 @@
+# Copyright (C) 2004 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+#       $Id: RangeIndex.py,v 1.7 2006/04/09 16:52:03 dieter Exp $
+'''An efficient index for ranges.'''
+
+from BTrees.IIBTree import IISet, multiunion, IITreeSet
+
+from ManagableIndex import ManagableIndex, addForm, setOperation
+from Evaluation import Eval
+from fixPluginIndexes import parseIndexRequest
+
+
+class RangeIndex(ManagableIndex):
+  '''A 'RangeIndex' has exactly 2 'ValueProviders': for the lower
+  and upper bound, respectively.
+  
+  For a term *t*, it can efficiently return the documents *d*
+  with: *lowerbound(d) <= t <= upperbound(d)*.
+  '''
+
+  meta_type = 'Managable RangeIndex'
+
+  query_options= ('operator', 'isearch', 'isearch_filter')
+
+  _properties = (
+    ManagableIndex._properties
+    + (
+    {'id':'IgnoreIncompleteRange',
+     'label':'Ignore incomplete ranges (otherwise, we raise a ValueError)',
+     'type':'boolean', 'mode':'rw',},
+    {'id':'BoundaryNames',
+     'label':'Boundary names: "low" "high" pair specifying that this index should handle "low <= value <= high" queries"',
+     'type':'tokens', 'mode':'rw',},
+    {'id':'MinimalValue',
+     'label':'Minimal value: values at or below this value are identified -- clear+reindex after change!',
+     'type':'string', 'mode':'rw',},
+    {'id':'MaximalValue',
+     'label':'Maximal value: values at or above this value are identified -- clear+reindex after change!',
+     'type':'string', 'mode':'rw',},
+    {'id':'OrganisationHighThenLow',
+     'label':"Organisation 'high-then-low': check when 'x <= high' is less likely than 'low <= x' -- clear+reindex after change",
+     'type':'boolean', 'mode':'rw', },
+    )
+    )
+  IgnoreIncompleteRange = 1
+  BoundaryNames = ()
+  MinimalValue = MaximalValue = ''
+  OrganisationHighThenLow = False
+
+  Combiners = ManagableIndex.Combiners + ('aggregate',)
+
+  def __init__(self, name):
+    self._minEvaluator = Eval('MinimalValue')
+    self._maxEvaluator = Eval('MaximalValue')
+    RangeIndex.inheritedAttribute('__init__')(self, name)
+
+  _minValue = _maxValue = None # backward compatibility
+  def _setup(self):
+    RangeIndex.inheritedAttribute('_setup')(self)
+    self._minValue = self._maxValue = None
+    self._unrestriced = self._upperOnly = self._lowerOnly = None
+    if hasattr(self, 'aq_base'): # wrapped
+      eval = self._minEvaluator
+      if eval._getExpressionString(): min = eval._evaluate(None, None)
+      else: min = None
+      if min is not None: min = self._standardizeTerm(min, None, 1, 0)
+      self._minValue = min
+      eval = self._maxEvaluator
+      if eval._getExpressionString(): max = eval._evaluate(None, None)
+      else: max = None
+      if max is not None: max = self._standardizeTerm(max, None, 1, 0)
+      self._maxValue = max
+      it = self._index.__class__
+      if min is not None and max is not None:
+        if min > max:
+          raise ValueError('Rangeindex %s: minimal exceeds maximal value: %s %s'
+                           % (self.id, str(min), str(max))
+                           )
+        self._unrestricted = it()
+      if min is not None: self._upperOnly = it()
+      if max is not None: self._lowerOnly = it()
+
+  def _standardizeValue(self, value, object):
+    if not value: return
+    value = [self._standardizeTerm(v, object, 1) for v in value]
+    value = [v for v in value if v is not None]
+    if not value: return
+    if len(value) != 2:
+      if self.IgnoreIncompleteRange: return
+      raise ValueError('RangeIndex %s: wrong value %s for object %s'
+                       % (self.id, str(value), object.absolute_url(1),),
+                       )
+    if value[0] > value[1]: return # empty range
+    return value
+
+  _combine_aggregate = _standardizeValue
+
+  # do we need this?
+  def uniqueValues(self, name=None, withLength=0): raise NotImplementedError
+
+  # we do not support range searches
+  def _enumerateRange(self, lo, hi): raise NotImplementedError
+
+  # we do not support sorting
+  def keyForDocument(self, docId): raise NotImplementedError
+  def items(self): raise NotImplementedError
+
+  # we do not support expanding
+  def _getExpansionIndex(self): raise NotImplementedError
+
+
+  # basic specialized methods
+  def _indexValue(self,documentId,val,threshold):
+    self._insert(val,documentId)
+    return 1
+
+  def _unindexValue(self,documentId,val):
+    self._remove(val,documentId)
+
+
+  # apply -- for (partial) plugin replacement for effective/expires
+  def _apply_index(self,request, cid= ''):
+    r = RangeIndex.inheritedAttribute('_apply_index')(self,request, cid)
+    if r is not None or not self.BoundaryNames: return r
+    bn = self.BoundaryNames
+    if len(bn) != 2:
+      raise ValueError('RangeIndex %s: "BoundaryNames" is not a pair'
+                       % self.id
+                       )
+    ln, hn = bn
+    lq = parseIndexRequest(request, ln, ManagableIndex.query_options)
+    hq = parseIndexRequest(request, hn, ManagableIndex.query_options)
+    if lq.keys != hq.keys \
+       or lq.get('range') != 'min' \
+       or hq.get('range') != 'max' \
+       or lq.get('operator') \
+       or hq.get('operator') \
+       : return
+    return RangeIndex.inheritedAttribute('_apply_index')(
+      self,
+      {self.id:{'query':lq.keys}},
+      cid
+      )
+    
+
+  # overridden _searchTerm to pass "isearch" into "_load"
+  # ATT: the base class should do that but it would break compatibility
+  def _searchTerm(self,term,record):
+    return self._load(term, record.get('isearch'))
+
+  
+  # adapted storage
+  def _findDocList(self, term, create):
+    '''return for *term* a docList access path consisting
+    of (index, key) pairs or 'None'.
+
+    If *create*, create missing intermediates.
+    '''
+    lv, hv = term
+    min = self._minValue; max = self._maxValue
+    if min is not None and lv <= min:
+      # no lower restriction
+      if max is not None and hv >= max:
+        # no upper restriction, i.e. unrestricted
+        return ((self._unrestricted,min),)
+      else: return ((self._upperOnly, hv),)
+    elif max is not None and hv >= max:
+      # no upper restriction
+      return ((self._lowerOnly, lv),)
+    elif self.OrganisationHighThenLow:
+      idx = self._index; hi = idx.get(hv)
+      if hi is None:
+        if not create: return
+        hi = idx[hv] = idx.__class__()
+      return ((idx, hv), (hi, lv),)
+    else:
+      idx = self._index; li = idx.get(lv)
+      if li is None:
+        if not create: return
+        li = idx[lv] = idx.__class__()
+      return ((idx, lv), (li, hv),)
+
+  def _insert(self, term, docId, _isInstance= isinstance, _IntType= int):
+    '''index *docId* under *term*.'''
+    i,k = self._findDocList(term, 1)[-1]
+    dl = i.get(k)
+    if dl is None: i[k] = docId; self.__len__.change(1); return
+    if _isInstance(dl,_IntType): dl = i[k]= IITreeSet((dl,))
+    dl.insert(docId)
+    
+
+  def _remove(self, term, docId, _isInstance= isinstance, _IntType= int):
+    '''unindex *docId* under *term*.'''
+    access = self._findDocList(term, 0)
+    if access is not None: i,k = access[-1]; dl = i.get(k)
+    else: dl = None
+    isInt = _isInstance(dl,_IntType)
+    if dl is None or isInt and dl != docId:
+      raise ValueError('Attempt to remove nonexisting document %s from %s'
+                       % (docId, self.id)
+                       )
+    if isInt: dl = None
+    else: dl.remove(docId)
+    if not dl:
+      del i[k]; self.__len__.change(-1)
+      if not i and len(access)==2: del access[0][0][access[0][1]]
+    
+  def _mkSet(self, dl,  _isInstance= isinstance, _IntType= int):
+    if dl is None: return IISet()
+    if _isInstance(dl, _IntType): dl = IISet((dl,))
+    return dl
+
+  def _load(self, term, isearch=False):
+    '''return IISet for documents matching *term*.'''
+    sets = []; mkSet = self._mkSet
+    if self.OrganisationHighThenLow:
+      for hi in self._index.values(term):
+        for dl in hi.values(None, term): sets.append(mkSet(dl))
+    else:
+      for li in self._index.values(None, term):
+        for dl in li.values(term): sets.append(mkSet(dl))
+    min = self._minValue; max = self._maxValue
+    if min is not None:
+      for dl in self._upperOnly.values(term): sets.append(mkSet(dl))
+    if max is not None:
+      for dl in self._lowerOnly.values(None, term): sets.append(mkSet(dl))
+      if min is not None: sets.append(mkSet(self._unrestricted.get(min)))
+    return setOperation('or', sets, isearch)
+
+  # filtering support
+  supportFiltering = True
+
+  def _makeTermPredicate(self, term):
+    min = self._minValue; max = self._maxValue
+    if min is not None and term < min: term = min
+    if max is not None and term > max: term = max
+    return lambda (low,high), t=term: low <= t <= high
+  
+
+
+def addRangeIndexForm(self):
+  '''add RangeIndex form.'''
+  return addForm.__of__(self)(
+    type= RangeIndex.meta_type,
+    description= '''A RangeIndex indexes an object under a range of terms.''',
+    action= 'addIndex',
+    )

Added: zope-managableindex/branches/upstream/current/Utils.py
===================================================================
--- zope-managableindex/branches/upstream/current/Utils.py	2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/Utils.py	2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,94 @@
+# Copyright (C) 2003 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+#       $Id: Utils.py,v 1.1 2004/05/22 17:40:18 dieter Exp $
+'''Utilities.'''
+
+from sys import maxint
+from DateTime import DateTime
+
+
+##############################################################################
+## order support
+class _OrderReverse:
+  '''auxiliary class to provide order reversal.'''
+
+  def __init__(self, value):
+    self._OrderReverse_value = value
+
+  getValue__roles__ = None # public
+  def getValue(self): return self._OrderReverse_value
+
+  def __cmp__(self,other):
+    return - cmp(self._OrderReverse_value, other._OrderReverse_value)
+
+##  def __getattr__(self, attr):
+##    return getattr(_OrderReverse_value, attr)
+
+_mdict = globals(); _cldict = _OrderReverse.__dict__
+
+for _f,_op in [spec.split(':') for spec in
+               'lt:> le:>= eq:== ne:!= gt:< ge:<='.split()]:
+    exec('def %s(self, other): return self._OrderReverse_value %s other._OrderReverse_value\n\n' %
+         (_f, _op),
+         _mdict,
+         _cldict,
+         )
+
+def reverseOrder(value):
+  if isinstance(value, _OrderReverse): return value._OrderReverse_value
+  return _OrderReverse(value)
+
+
+##############################################################################
+## Lazy
+## this is private for the moment as otherwise, we would need to
+## respect security aspects
+class _LazyMap:
+  '''an object applying a function lazyly.'''
+  def __init__(self, f, seq):
+    self._f = f
+    self._seq = seq
+
+  def __getitem__(self, i): return self._f(self._seq[i])
+
+
+
+##############################################################################
+## DateTime conversions
+
+def convertToDateTime(value):
+  '''convert *value* to a 'DateTime' object.'''
+  if isinstance(value, DateTime): return value
+  if isinstance(value, tuple): return DateTime(*value)
+  return DateTime(value)
+
+def convertToDateTimeInteger(value, exc=0):
+  '''convert *value* into a DateTime integer (representing secs since
+  epoch).
+
+  *exc* controls whether an exception should be raised when the
+  value cannot be represented in the integer range. If *exc* is
+  false, values are truncated, if necessary.
+  '''
+  if isinstance(value, int): return value
+  value = round(convertToDateTime(value)._t) # seconds since epoch
+  ma = maxint; mi = -ma - 1
+  if exc and value < mi or value > ma:
+    raise TypeError('not in integer range: %s' % value)
+  if value < mi: value = mi
+  elif value > ma: value = ma
+  return int(value)
+
+def convertToDateInteger(value, round_dir=-1):
+  '''convert *value* into a Date integer (400*y + 31*(m-1) + d-1).
+
+  *round_dir* controls rounding: '0' means 'round', '1' means 'ceil'
+  and '-1' 'floor')
+  '''
+  if isinstance(value, int): return value
+  adjust = (0, 0.5, (0.999999))[round_dir]
+  dt = convertToDateTime(value)
+  if adjust: dt += adjust
+  y,m,d = dt._year, dt._month, dt._day
+  return 400*y +(m-1)*31 +d-1
+

Added: zope-managableindex/branches/upstream/current/VERSION.txt
===================================================================
--- zope-managableindex/branches/upstream/current/VERSION.txt	2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/VERSION.txt	2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,2 @@
+1.6.1 for Zope 2.7+, Python 2.3+/Python 2.4 // Zope 2.8
+requires "OFolder", see <http://www.dieter.handskake.de/pyprojects/zope>

Added: zope-managableindex/branches/upstream/current/ValueProvider.py
===================================================================
--- zope-managableindex/branches/upstream/current/ValueProvider.py	2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/ValueProvider.py	2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,170 @@
+# Copyright (C) 2003 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+#       $Id: ValueProvider.py,v 1.5 2004/07/30 13:38:02 dieter Exp $
+'''ValueProvider.
+
+A 'ValueProvider' evaluates an object and returns a value.
+A 'None' value is interpreted as if the object does not have
+a value.
+
+A 'ValueProvider' can ignore more values than 'None'.
+It can specify a normalization for values which have not been ignored.
+
+A 'ValueProvider' can specify how exceptions during evaluation
+and postprocessing should be handled. They can be ignored
+(this means, the object does not have a value) or propagated.
+
+'ValueProviders' are 'SimpleItems' and provide comfiguration
+via Properties, i.e. they are 'PropertyManagers'.
+'''
+
+from Globals import InitializeClass
+from AccessControl import ClassSecurityInfo
+from ComputedAttribute import ComputedAttribute
+from Acquisition import Acquired
+
+from OFS.SimpleItem import SimpleItem
+from OFS.PropertyManager import PropertyManager
+from DocumentTemplate.DT_Util import safe_callable
+from ZODB.POSException import ConflictError
+
+from Products.PageTemplates.TALES import CompilerError
+
+from Evaluation import EvalAndCall, Ignore, Normalize
+
+
+class ValueProvider(SimpleItem,PropertyManager,Ignore,Normalize):
+  '''configuration for object evaluation.'''
+
+  manage_options= (
+    PropertyManager.manage_options
+    + SimpleItem.manage_options
+    )
+
+  _properties= (
+  (
+    { 'id' : 'IgnoreExceptions', 'type' : 'boolean', 'mode' : 'w',},
+    )
+    + Ignore._properties
+    + Normalize._properties
+    )
+  IgnoreExceptions= 1
+
+  security= ClassSecurityInfo()
+  security.declarePrivate('evaluate')
+
+
+  def evaluate(self,object):
+    '''evaluate *object* to a value.
+
+    'None' means, the object does not have a value.
+    '''
+    __traceback_info__ = self.id
+    try:
+      value= self._evaluate(object)
+      value= self._ignore(value,object)
+      value= self._normalize(value,object)
+    except (CompilerError, ConflictError): raise
+    except:
+      if self.IgnoreExceptions: return
+      raise
+    return value
+
+
+  def __str__(self):
+    return '; '.join(['%s : %s' % pi for pi in self.propertyItems()])
+
+  title= ComputedAttribute(__str__)
+
+
+  # abstract methods
+  def _evaluate(self,object):
+    raise NotImplementedError("'_ValueProvider._evaluate' is abstract")
+
+InitializeClass(ValueProvider)
+
+
+class AttributeLookup(ValueProvider):
+  '''Configures attribute/method lookup.
+
+  Configuration properties:
+
+   'Name' -- the attribute/method which determines the value
+
+   'AcquisitionType' -- controls how acquisition is used
+     
+     * 'implicit' -- use implicit acquisition
+
+     * 'explicit' -- equivalent to 'implicit' unless the
+       looked up value is a method and is called.
+       In this case, the passed in object is an explicit
+       acquisition wrapper.
+
+     * 'none' -- do not use acquisition
+
+   'CallType' -- controls how a callable result should be handled
+
+     * 'call' -- call it and return the result
+
+     * 'ignore' -- ignore value
+
+     * 'return' -- return the value
+   '''
+
+  meta_type= 'Attribute Lookup'
+
+  _properties= (
+   (
+     { 'id' : 'Name', 'type' : 'string', 'mode' : 'w',},
+     { 'id' : 'AcquisitionType', 'type' : 'selection', 'select_variable' : 'listAcquisitionTypes', 'mode' : 'w',},
+     { 'id' : 'CallType', 'type' : 'selection', 'select_variable' : 'listCallTypes', 'mode' : 'w',},
+     )
+     + ValueProvider._properties
+     )
+
+  Name= ''
+  AcquisitionType= 'implicit'
+  CallType= 'call'
+   
+  def _evaluate(self,object):
+     aqType= self.AcquisitionType; orgObj = object
+     name= self.Name or self.id
+     # determine value
+     if not hasattr(object,'aq_acquire'): aqType= 'implicit' # not wrapped
+     elif hasattr(object.aq_base, name+'__index_aqType__'):
+       aqType = getattr(object, name+'__index_aqType__')
+     if aqType == 'explicit':
+       # work around bug in "aq_acquire" (has 'default' argument but ignores it)
+       try: value= object.aq_explicit.aq_acquire(name,default=None)
+       except AttributeError: value= None
+     else:
+       if aqType == 'none': object= object.aq_base
+       value= getattr(object,name,None)
+       # allow acquisition for objets that explicitly called for it
+       if value is Acquired: value = getattr(orgObj, name, None)
+     # handle calls
+     if safe_callable(value): 
+       callType= self.CallType
+       if callType == 'call': value= value()
+       elif callType == 'ignore': value= None
+     return value
+
+  def listAcquisitionTypes(self):
+    return ('implicit', 'none', 'explicit',)
+
+  def listCallTypes(self):
+    return ('call', 'return', 'ignore',)
+
+
+class ExpressionEvaluator(ValueProvider,EvalAndCall):
+  '''configures TALES evaluation.'''
+
+  meta_type= 'Expression Evaluator'
+
+  _properties= (
+    EvalAndCall._properties
+    + ValueProvider._properties
+    )
+
+  def _evaluate(self,object):
+    return EvalAndCall._evaluate(self,object,object)

Added: zope-managableindex/branches/upstream/current/WordIndex.py
===================================================================
--- zope-managableindex/branches/upstream/current/WordIndex.py	2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/WordIndex.py	2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,161 @@
+# Copyright (C) 2004 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+#       $Id: KeywordIndex.py,v 1.4 2004/07/30 11:12:09 dieter Exp $
+'''Managable WordIndex.
+
+A word index lies between a 'KeywordIndex' and a 'TextIndex'.
+
+Like a 'TextIndex', it uses a 'Lexicon' to split values into
+a sequence of words, stores integer word ids in its index
+and does not support range queries.
+Like a 'KeywordIndex', it indexes an object under a sequence of words --
+no near or phrase queries and no relevancy ranking.
+Due to these restrictions, a 'WordIndex' is very efficient with respect
+to transaction and load size.
+
+The motivation to implement a
+'WordIndex' came from my observation, that almost all our ZEO loads
+above 100ms were caused by loading large word frequency
+IOBuckets used by 'TextIndex'es for relevancy ranking -- a feature
+we do not need and use. Many of these loads transfered buckets
+of several 10 kB and a considerable portion of them took more
+than a second. As word frequency information is necessary for
+each document in a hit, you can imagine how fast our queries
+were.
+
+On the other hand, a 'WordIndex' is only useful when you
+use a flexible query framework, such as e.g. 'AdvancedQuery'
+or 'CatalogQuery'.
+The standard 'ZCatalog' framework is too weak as it does not
+allow to have several subqueries against the same index
+in a query. Thats the reason why 'TextIndex'es come with
+their own query parser. I live in Germany and therefore
+the standard query parser is useless for us (we use 'und', 'oder',
+'nicht' instead of 'and', 'or' and 'not') and I have 'AdvancedQuery' --
+thus I did not care to give the new 'WordIndex' a query parser.
+You could easily provide one -- should you feel a need for it.
+'''
+
+from Acquisition import aq_base
+
+from BTrees.IIBTree import IITreeSet, difference
+
+from ManagableIndex import addForm, ManagableIndex
+from KeywordIndex import KeywordIndex
+
+class WordIndex(KeywordIndex):
+  '''a managable 'WordIndex'.'''
+  meta_type = 'Managable WordIndex'
+
+  query_options = ('operator', 'match', 'isearch')
+  TermType = 'integer'
+
+  _properties = ManagableIndex._properties[:1] + (
+    {'id':'PrenormalizeTerm', 'type':'string', 'mode':'w',
+     'label':'Match normalizer: applied to match patterns -- this should match the lexicons normalization',},
+    {'id':'Lexicon', 'type':'string', 'mode':'w',
+     'label':'Lexicon id of a ZCTextIndex like lexicon (resolved with respect to the catalog) -- clear+reindex after change',},
+    )
+  Lexicon = ''
+
+  _createDefaultValueProvider = ManagableIndex._createDefaultValueProvider
+
+  # override for better readability
+  def getEntryForObject(self,documentId, default= None):
+    '''Information for *documentId*.'''
+    info= self._unindex.get(documentId)
+    if info is None: return default
+    lexicon = self._getLexicon()
+    l = [lexicon.get_word(wid) for wid in info.keys()]; l.sort()
+    return repr(l)
+  
+  # overrides: we could use the corresponding 'KeywordIndex' definitions
+  # but as the word sets tend to be much larger,
+  # we use definitions which are more efficient for large sets
+  def _update(self,documentId,val,oldval,threshold):
+    add= difference(val,oldval)
+    rem= difference(oldval,val)
+    if add: self._indexValue(documentId,add,threshold)
+    if rem: self._unindexValue(documentId,rem)
+    # optimize transaction size by not writing _unindex bucket
+    if len(rem) < 100:
+      for x in rem: oldval.remove(x) # sad that we do not have a mass remove
+      oldval.update(add)
+    else: oldval.clear(); oldval.update(val)
+    return len(add),
+
+## the KeywordIndex implementation is more efficient
+##  def _equalValues(self, val1, val2):
+##    if val1 == val2: return True
+##    if val1 is None or val2 is None: return False
+##    it1 = val1.keys(); it2 = val2.keys(); i = 0
+##    while True:
+##      k1 = k2 = self
+##      try: k1 = it1[i]
+##      except IndexError: pass
+##      try: k2 = it2[i]
+##      except IndexError: pass
+##      if k1 != k2: return False
+##      if k2 is self: return True
+##      i+= 1
+
+  def _combine_union(self, values, object):
+    if not values: return
+    set= None
+    for v in values:
+      sv= self._standardizeValue_(v, object)
+      if not sv: continue
+      if set is None: set = IITreeSet(sv)
+      else: set.update(sv)
+    return set
+
+  def _standardizeValue(self, value, object):
+    '''convert to an IITreeSet of standardized terms.'''
+    terms = self._standardizeValue_(self, value, object)
+    if terms: return IITreeSet(terms)
+
+  ## essential work delegated to lexicon
+  def _standardizeValue_(self, value, object):
+    '''convert to a sequence of standardized terms.'''
+    if not value: return
+    lexicon = self._getLexicon()
+    # we assume some type of "Products.ZCTextIndex.Lexicon.Lexicon" instance
+    return lexicon.sourceToWordIds(value)
+
+  def _normalize(self, value, object):
+    '''convert term *value* to word id.'''
+    if isinstance(value, int): return value # already normalized
+    # ATT: returns 0, when the word is unknown
+    # This implies that searches for unknown words cannot get matches
+    wids = self._getLexicon().termToWordIds(value)
+    if len(wids) != 1:
+      raise ValueError('Word index %s can only handle word queries: %s'
+                       % (self.id, value)
+                       )
+    return wids[0]
+    
+  def _getLexicon(self):
+    # resolve with respect to catalog -- this is nasty but necessary
+    # to avoid acquisition from the internal "_catalog".
+    obj = self
+    while not hasattr(aq_base(obj), 'Indexes'): obj = obj.aq_inner.aq_parent
+    lexicon = getattr(obj, self.Lexicon, None)
+    if lexicon is None:
+      raise ValueError('Lexicon not found: ' + self.Lexicon)
+    return lexicon
+
+  _matchType = 'asis'
+  def _getMatchIndex(self):
+    # ATT: internal knowledge about ZCTextIndex.Lexicon.Lexicon!
+    return self._getLexicon()._wids
+
+
+
+def addWordIndexForm(self):
+  '''add KeywordIndex form.'''
+  return addForm.__of__(self)(
+    type= WordIndex.meta_type,
+    description= '''A WordIndex indexes an object under a set of word ids determined via a 'ZCTextIndex' like lexicon.''',
+    action= 'addIndex',
+    )
+    

Added: zope-managableindex/branches/upstream/current/__init__.py
===================================================================
--- zope-managableindex/branches/upstream/current/__init__.py	2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/__init__.py	2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,38 @@
+# Copyright (C) 2003 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+#       $Id: __init__.py,v 1.5 2006/05/17 19:53:07 dieter Exp $
+'''Managable Indexes
+
+see 'ManagableIndexes.html' for documentation.
+'''
+
+from FieldIndex import FieldIndex, addFieldIndexForm
+from KeywordIndex import KeywordIndex, addKeywordIndexForm, \
+     KeywordIndex_scalable, addKeywordIndex_scalableForm
+from RangeIndex import RangeIndex, addRangeIndexForm
+from WordIndex import WordIndex, addWordIndexForm
+from PathIndex import PathIndex, addPathIndexForm
+from ManagableIndex import addIndex, ManageManagableIndexes
+from AccessControl import allow_module
+
+_indexes= (
+  'FieldIndex',
+  'KeywordIndex',
+  'KeywordIndex_scalable',
+  'RangeIndex',
+  'WordIndex',
+  'PathIndex',
+  )
+
+_mdict= globals()
+
+def initialize(context):
+  for idx in _indexes:
+    context.registerClass(
+      _mdict[idx],
+      permission= ManageManagableIndexes,
+      constructors= (_mdict['add%sForm' % idx], addIndex,),
+      visibility= None,
+      )
+
+allow_module('Products.ManagableIndex.Utils')

Added: zope-managableindex/branches/upstream/current/doc/ManagableIndex.html
===================================================================
--- zope-managableindex/branches/upstream/current/doc/ManagableIndex.html	2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/doc/ManagableIndex.html	2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,448 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2//EN">
+<html>
+  <head>
+    <title>ManagableIndex</title>
+    <style type="text/css">
+      .warning {
+      	color: red;
+        font-weight: bold;
+      }
+    </style>
+  </head>
+
+  <body>
+    <h1>ManagableIndex</h1>
+
+<h2>Introduction</h2>
+<p><code>ManagableIndex</code> can mean different things for different
+people. For a content manager, it brings flexible field, keyword
+and efficient range indexes managable via the ZMI to</p>
+<ul>
+<li>combine values from different sources, ignoring irrelevant values
+  and transforming values in an inadequate form,<br>
+  Example: index an object via title if it is non-empty but use
+  id otherwise<br>
+  Example: add <code>portal_type</code> to the <code>Subject</code> index
+  transforming it into a sequence (as required by a keyword index).
+<li>control acquisition<br>
+  and prevent acquired values to be indexed
+<li>ignore stop-terms, normalize terms and check types<br>
+  Example: when you do not want that an object is indexed under
+    a meaningless default value, you can define this value
+    as a stop-term.<br>
+  Example: when you want case insensitive searches, you
+    can perform case folding during normalization.<br>
+  Example: the BTrees underlying Zope indexes require its
+    keys to have a persistently consistent ordering.
+    When this is not the case, they become corrupt.
+    While modern Python versions (at least Python 2.3)
+    compare values from different elementary types
+    persistently consistent, the result can be quite
+    unintuitive. Therefore, it is best when all terms
+    in an index belong to the same types. Normalizing
+    and type checking helps to ensure this.
+</ul>
+
+<p>For a developer, <code>ManagableIndex</code> provides
+a framework for index definition, improving on <code>PluginIndexes</code>.
+It provides for managability,
+automatically and intelligently handles unindexing when an object
+is reindexed and implements <em>and</em>, <em>or</em> and
+<em>range</em> queries (for not too complex indexes).</p>
+
+<h2>Concepts</h2>
+<p>The main tasks of an index are to index objects under
+a set of index terms derived from the object and efficiently locate
+all objects indexed under a given term.</p>
+
+<p>Indexing consists of 3 stages: evaluation of the object
+to obtain the object's value (with respect to the index),
+deriving index terms from the value and storing the association
+term/object in a way such that
+the objects associated with a term can quickly be retrieved.</p>
+
+
+<h3>ValueProvider</h3>
+<p>Evaluation is specified by a sequence of <em>ValueProvider</em>s
+associated with the index. 
+A ValueProvider is a device that returns a value for an object.
+If the return value is not <code>None</code>,
+then it is interpreted as the
+object's value with respect to this ValueProvider.</p>
+
+<p>A <code>ValueProvider</code> can have an associated
+<em>IgnorePredicate</em>, a TALES expression.
+When the <code>IgnorePredicate</code> yields true for
+a value, then it is ignored. You may e.g. specify that
+empty strings should be ignored.</p>
+
+<p>A <code>ValueProvider</code> can have an associated
+<em>Normalizer</em>, a TALES expression.
+The <code>Normalizer</code> is applied to not ignored values
+to transform them in a standard form, e.g. the normalizer
+for a keyword index can transform a string into a one element
+tuple containing the string as a keyword index requires a sequence
+value.</p>
+
+
+<p>The most essential ValueProviders are <em>AttributeLookup</em>s.
+An AttributeLookup determines the object's value through
+the lookup of an attribute. The AttributeLookup's configuration
+determines whether acquisition can be used, whether a callable value
+should be called and whether exceptions should be ignored.
+</p>
+
+<p><em>ExpressionEvaluator</em>s are another type of ValueProviders.
+These are TALES expressions defining an objects value.
+<code>ExpressionEvaluator</code>s often avoid to define
+simple scripts just for indexing purposes.<br>
+<span class="warning">Warning: </span>Until Zope 2.7, TALES
+expressions have been trusted when used outside of Zope; from
+Zope 2.8 on, TALES expression evaluation is always subject
+to Zope's security restrictions even when used outside of Zope.
+This may have strange consequences when you perform index management
+(e.g. mass reindexing) in an external script (run outside of Zope).
+In such cases, you should probably let the script login as a Zope user
+with sufficient priviledges.
+</p>
+
+<h3>Value Combination</h3>
+<p>When an index has several ValueProviders, their values
+must be combined to define the object's value.</p>
+
+<p><code>None</code> and ignored values are always ignored
+and are not combined.</p>
+
+<p>Currently, there are three combiners:</p>
+<dl>
+<dt><code>useFirst</code><dd>use the first non-ignored value<br>
+  This is available for all kinds of indexes.</dd>
+<dt><code>union</code><dd>combine all non-ignored values in a union like way.<br>
+  This is only available for indexes that split the object's value
+  into a set of index terms (such as keyword or text indexes).</dd>
+<dt><code>aggregate</code><dd>combine all values into a list.<br>
+  This is only available for index types that expect aggregate values
+  such as e.g. a <code>RangeIndex</code>.</dd>
+</dl>
+
+<h3>Term Standardization</h3>
+<p>Once the object's value (with respect to the index) has been
+determined, the value is split into a set of index terms
+in an index specific way (e.g. a <code>FieldIndex</code>
+uses the value directly, a <code>KeyWordIndex</code>
+requires the value to be a sequence and uses its elements as
+index terms and a <code>TextIndex</code> splits the value into
+words and uses the words).
+The terms are then standardized.</p>
+
+<p>Standardization consists of serveral steps:
+prenormalization such as case normalization, stemming,
+phonetic normalization, ...
+elimination of
+stop terms, normalization such as type conversion,
+type checking and copying.</p>
+
+
+<h4>Prenormalization</h4>
+<p>A index can define a term prenormalizer, a TALES expression.
+The prenormalizer is applied to terms before term expansion
+in queries and always as the first step of normalization during
+indexing. It can be used e.g. for case normalization, stemming,
+phonetic normalization, synonym normalization, ...</p>
+
+<h4>Stop Terms</h4>
+<p>An index can define a <em>StopTermPredicate</em>, a TALES
+expression. When the predicate yields true for a term, the term
+is ignored.</p>
+
+<h4>Term Normalization</h4>
+<p>An index can define a <em>NormalizeTerm</em> TALES expression.
+The expression can be used to transform the term into some standard
+form, e.g.
+convert a descriptive string or complex object into a code
+used for efficient indexing.</p>
+
+<h3>Type Restriction</h3>
+<p>The BTrees used in index implementation require that any index term must be
+persistently comparable to any other index term in this index
+(otherwise, the index gets corrupted). To help observing this
+essential property, the index term's type can be restricted.</p>
+
+<p>The following type restrictions are defined:</p>
+<dl>
+<dt><code>not checked</code><dd>the type is not restricted at all,</dd>
+<dt><code>numeric</code><dd>the type must be a numeric type (<code>int</code>,
+<code>float</code> or <code>long</code>),</dd>
+<dt><code>string</code><dd>the type must be a string,</dd>
+<dt><code>ustring</code><dd>the type must be a unicode string; <code>TermTypeExtra</code> can specify an encoding used for conversion.</dd>
+<dt><code>integer</code><dd>the type must be an integer,</dd>
+<dt><code>DateTime</code><dd>the type must be a <code>DateTime</code> object or float --
+the index stores the date as a float (seconds since epoch),</dd>
+<dt><code>DateTimeInteger</code><dd>the type must be a <code>DateTime</code> object or integer -- the index stores the date as an integer (seconds since epoch; truncated, if necessary),</dd>
+<dt><code>DateInteger</code><dd>the type must be a <code>DateTime</code> object or integer -- the index stores an integer representing the date
+(<code>400 * <var>year</var> + 31 * (<var>month</var>-1) + (<var>day</var>-1)</code>,</dd>
+<dt><code>tuple</code> with <var>tuplespec</var> in <code>TermTypeExtra</code><dd>The value must be a tuple
+  of the structure given by <var>tuplespec</var>. Each
+  tuple component is specified by a letter or by a parenthesized
+  <var>tuplespec</var> (for "subtuples"). Recognized
+  letters are <code>n</code> (numeric), <code>s</code> (string),
+  <code>u</code> (unicode string),
+  <code>d</code> (datetime).
+  An encoding for unicode conversion can be specified after <var>tuplespec</var>,
+  separated by <code>;</code>.
+  </dd>
+<dt><code>instance</code> with <var>fullClassName</var> in <code>TermTypeExtra</code><dd>the type must be an instance of the given class and the class must define an <code>__cmp__</code> method.
+It is assumed without check, that this <code>__cmp__</code>
+implements a persistent comparison.
+</dd>
+<dt><code>expression checked</code> with <code>TermTypeExtra</code> specifying checking TALES expression<dd>the expression may try to transform the
+value into something acceptable. If it yields <code>None</code> or
+raises an exception, the type is considered inacceptable, otherwise
+the value is used instead of the original term.
+ </dd>
+</dl>
+
+<p>All checkers try to convert a term into an acceptable type.
+Only when this fails, an exception is raised.</p>
+
+<p>If you choose one of the integer types, i.e. <code>integer</code>,
+<code>DateTimeInteger</code> or <code>DateInteger</code>,
+an especially efficient index type is build.
+You must clear and reindex the index when the index type is changed.
+You must clear the index even when it is already empty (because the data
+structures may need to be changed).</p>
+
+
+<h4>Copying</h4>
+<p>It is dangerous to index mutable types. If a indexed mutable object
+is later changed, its ordering with respect to other indexed
+objects may change, corrupting the index. Corruption of this
+kind can be avoided when the mutable object is copied before it
+is stored in the index.</p>
+
+<p><code>TermCopy</code>
+  specifies whether the value should be directly used,
+  shallow copied or deep copied.</p>
+
+
+<h2>Index Types</h2>
+<p>Currently, <code>ManagableIndexes</code> supports 3 types of
+indexes: <code>FieldIndex</code>, <code>KeywordIndex</code> and
+<code>RangeIndex</code>. Further types can easily be implemented.</p>
+
+<h3><code>FieldIndex</code></h3>
+<p>A <code>FieldIndex</code> indexes an object under a single term,
+the object's value with respect to the index.</p>
+
+<p>You get efficient DateTime and Date indexes by selecting
+<code>DateTime</code> (stored as float),
+<code>DateTimeInteger</code> (stored as integer) or
+<code>DateInteger</code> (stored as integer) as
+<code>TermType</code>. You must clear and reindex when you
+change the type.</p>
+
+<p>You can very efficiently sort search results via a <code>FieldIndex</code>.
+The <code>ZCatalog</code>'s method <code>searchResults</code>
+(aka <code>__call__</code>) supports this via its <code>sort_on</code>
+argument. My
+<a href="http://www.dieter.handshake.de/pyprojects/zope/AdvancedQuery.html"
+> AdvancedQuery</a> product supports arbitrary levels of sorting
+via <code>FieldIndex</code>es.</p>
+
+<p><code>AdvancedQuery</code>'s <code>FieldIndex</code> based ascending sorting
+is much more efficient than descending sorting. To make descending sorting
+more efficient, you can tell your <code>FieldIndex</code> to
+maintain the sort keys (also) in reverse order. This will slow indexing down
+a bit but make descending sorts much faster. When you change this
+attribute, you must clear and reindex the index (otherwise, the
+reverse order is not consistent).</p>
+
+<h3><code>KeywordIndex</code></h3>
+<p>A <code>KeywordIndex</code> expects the value of an object to
+be a sequence. It indexes the object under each element of this sequence.</p>
+
+<h3><code>RangeIndex</code></h3>
+<p>A <code>RangeIndex</code> expects an object's value
+to be a pair specifying an interval <var>low</var> to <var>high</var>.
+The index can efficiently locate documents for which a given term
+<var>t</var> lies within the document's <var>low</var> and <var>high</var>
+bounds.</p>
+
+<p>The object's value can either by constructed by a single
+attribute or with an <code>aggregate</code> combiner.</p>
+
+<p>To provide for a partial plugin replacement for CMF's
+<code>effective</code> and <code>expires</code> indexes,
+<code>RangeIndex</code> supports a <code>Boundary names</code>
+property. If set, it should be a pair of two names
+<var>low</var> and <var>high</var>. The index will then
+execute queries of the form <code><var>low</var> <= <var>val</var> <= <var>high</var></code>.
+To be compatible with <a href="AdvancedQuery"><code>AdvancedQuery</code></a>,
+the index replacing <code>effective</code> and <code>expires</code>
+should have the name <code>ValidityRange</code>.</p>
+
+<p><code>RangeIndex</code> efficiently supports improper ranges,
+i.e. those where at least one boundary is unlimited. You
+use its <code>Mininal value</code> and <code>Maximal value</code>
+properties to define which values should be considered as
+unlimited. These properties are TALES valued and are evaluated
+when the index is cleared. All values at or below (the value of)
+<code>Minimal value</code> are identified and interpreted
+as no lower limit; similarly, all values at or above (the value of)
+<code>Maximal value</code> are identified and interpreted as no
+upper limit.</p>
+
+<p>The boolean property <code>Organisation 'high-then-low'</code>
+controls the index organisation. With <code>high-then-low</code>
+organisation, the <code>high</code> index is primary and
+the <code>low</code> index is subordinate; <code>low-then-high</code>
+indicates the opposite organisation. You should use <code>high-then-low</code>
+when <code><var>val</var> <= <var>high</var></code>
+is less likely than <code><var>low</var> <= <var>val</var></code>.
+This is to be expected for date ranges and typical queries against them.</p>
+
+<h3><code>WordIndex</code></h3>
+<p>A <code>WordIndex</code> indexes an object under a set of words.
+It uses a <code>ZCTextIndex.PLexicon</code> for splitting a
+text into a sequence of words.</p>
+
+<p>A <code>WordIndex</code> lies between a <code>KeywordIndex</code> and a <code>TextIndex</code>.
+Like a <code>TextIndex</code>, it uses a <code>Lexicon</code> to split values into
+a sequence of words, stores integer word ids in its index
+and does not support range queries.
+Like a <code>KeywordIndex</code>, it indexes an object under a set of words --
+no near or phrase queries and no relevancy ranking.
+Due to these restrictions, a <code>WordIndex</code> is very efficient with respect
+to transaction and load size.</p>
+
+<p>The motivation to implement a
+<code>WordIndex</code> came from my observation, that almost all our ZEO loads
+above 100ms were caused by loading large word frequency
+IOBuckets used by <code>TextIndex</code>es for relevancy ranking -- a feature
+we do not need and use. Many of these loads transfered buckets
+of several 10 kB and a considerable portion of them took more
+than a second. As word frequency information is necessary for
+each document in a hit, you can imagine how fast our queries
+were.</p>
+
+<p>On the other hand, a <code>WordIndex</code> is only useful when you
+use a flexible query framework, such as e.g. <code>AdvancedQuery</code>
+or <code>CatalogQuery</code>.
+The standard <code>ZCatalog</code> framework is too weak as it does not
+allow to have several subqueries against the same index
+in a query. That's the reason why <code>TextIndex</code>es come with
+their own query parser. I live in Germany and therefore
+the standard query parser is useless for us (we use <code>und</code>, <code>oder</code>,
+<code>nicht</code> instead of <code>and</code>, <code>or</code> and <code>not</code>) and I have <code>AdvancedQuery</code> --
+thus I did not care to give the new <code>WordIndex</code> a query parser.
+You could easily provide one -- should you feel a need for it.</p>
+
+<h3><code>PathIndex</code></h3>
+<p>A <code>PathIndex</code> indexes an object under a path. A path is
+is either a tuple or a '/' separated string. The index supports
+path queries. For a given path, it locates objects that contain
+this path as subpath in its path value. Two search parameters
+<var>level</var> and <var>depth</var> control where in the
+object's path the given path may occur.</p>
+
+<p><var>level</var> and <var>depth</var> can be either <code>None</code>
+or an integer. When <var>level</var> and <var>depth</var> are
+both greater or equal 0, then an object with path <var>op</var>
+matches path <var>p</var> with respect to <var>level</var>
+and <var>depth</var>, iff <code><var>op</var> = <var>p1</var> <var>p</var> <var>p2</var></code> and
+<code>len(<var>p1</var>) = <var>level</var></code> and
+<code>len(<var>p2</var>) = <var>depth</var></code>.
+This means that <var>level</var> controls <var>p</var>'s distance
+from the beginning
+and <var>depth</var> that from the end of <var>op</var>.
+A <code>None</code> value means
+that there is no restriction for the respective side.
+A negative value means that the distance from the respective side
+may be up to (including) the negated value. E.g. a "-2" allows
+up to 2 segments.</p>
+
+<p>The default value for <var>level</var> is <code>0</code>,
+that for <var>depth</var> is <code>None</code>.</p>
+
+<p>Note, that <code>getPhysicalPath</code> requires acquisition
+to work properly. You must not set the acquisition type for
+such a value provider to <code>none</code>.</p>
+
+
+<h2>Matching</h2>
+<p>Most string and unicode based indexes (exception <code>RangeIndex</code>)
+support regular expression and glob matching. You use
+the query option <code>match</code> with a value of <code>'glob'</code>
+or <code>'regexp'</code> to call for this feature.</p>
+
+<p>Note that for large indexes, the match term should begin with
+an (easily recognizable) plain text string. Otherwise, this
+query type can be very inefficient.</p>
+
+
+
+<h2>TALES Expressions</h2>
+<p><code>ManagableIndex</code> uses TALES expressions at many places.
+They always can use the predefined variables:</p>
+<dl>
+<dt><code>index</code><dd>the index</dd>
+<dt><code>catalog</code><dd>the catalog</dd>
+<dt><code>root</code><dd>the zope root object</dd>
+<dt><code>modules</code><dd>the modules importer</dd>
+<dt><code>nothing</code><dd><code>None</code></dd>
+<dt><code>value</code><dd>the value to be checked</dd>
+<dt><code>object</code><dd>the object currently indexed</dd>
+</dl>
+
+<p>If the TALES expression evaluates to a callable object, then
+this is called on the value and the result used; otherwise, the
+evaluation result is used directly.</p>
+
+<h2>Utilities</h2>
+<p>The module <code>Utils</code> contains some useful functions:
+<code>convertToDateTime(value)</code>,
+<code>convertToDateTimeInteger(value, exc=0)</code>
+and 
+<code>convertToDateInteger(value, round_dir=-1)</code>.
+Please see the source documentation, for details.</p>
+
+<h2>Programmatic Creation</h2>
+<p>A primary goal for <code>ManagableIndex</code> is the easy and flexible
+configuration via the Zope Management Interface (ZMI). Occasionally, however,
+you want to create your ManagableIndexes programmatically and want to
+configure them in the same step. The <code>ZCatalog</code>'s
+<code>[manage_]addIndex</code> allow you to pass configuration information
+down to the index construction in the <code>extra</code> argument.</p>
+
+<p>For the creation of a ManagableIndex, <code>extra</code> must have
+a dict value.
+Its key <code>'ValueProviders'</code> determines the indexes'
+value providers. All other keys can provide values for the indexes
+properties.</p>
+
+<p>If the key <code>'ValueProviders'</code> is present, its value
+must be a sequence of value provider specifications which define
+the value providers for the index. If the key is not present, a
+default value provider is created.</p>
+
+<p>A value provider specification is a dict with the mandatory keys
+<code>'id'</code> and <code>'type'</code>. <code>type</code>
+specifies the value provider's type (<code>'AttributeLookup'</code> or
+<code>'ExpressionEvaluator'</code>). Additional keys can defined values
+for the value providers properties.</p>
+
+<p>If the configuration dicts contain keys different from the above mentioned
+special ones and not corresponding to a property,
+a <code>ValueError</code> exception is raised.</p>
+
+<p>Look for the <code>_properties</code> definitions in the source, to
+determine the available properties for the various types of objects.</p>
+
+    <hr>
+    <address><a href="mailto:dieter at handshake.de">Dieter Maurer</a></address>
+<!-- Created: Tue Sep  2 21:02:52 CEST 2003 -->
+<!-- hhmts start -->
+Last modified: Thu Nov 23 20:42:04 CET 2006
+<!-- hhmts end -->
+  </body>
+</html>

Added: zope-managableindex/branches/upstream/current/fixPluginIndexes.py
===================================================================
--- zope-managableindex/branches/upstream/current/fixPluginIndexes.py	2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/fixPluginIndexes.py	2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,99 @@
+#       $Id: fixPluginIndexes.py,v 1.3 2004/08/22 08:41:57 dieter Exp $
+'''Fixes for broken 'PluginIndexes'.
+
+I am sad that this is necessary.
+'''
+
+from Products.PluginIndexes.common.util import parseIndexRequest, \
+     InstanceType, DateTime, StringType, DictType, SequenceTypes
+
+
+class parseIndexRequest(parseIndexRequest):
+    def __init__(self, request, iid, options=[]):
+        '''this is essentially a copy of the inherited '__init__'.
+        Unfortunately, this turns empty query sequences into 'None'
+        which effectively means "no restriction".
+        However, an empty sequence of terms is the opposite
+        on "no restriction" (for "or" searches).
+
+        I hate that this extensive copying is necessary.
+        '''
+        self.id = iid
+
+        if not request.has_key(iid):
+            self.keys = None
+            return
+
+        # We keep this for backward compatility
+        usage_param = iid + '_usage'
+        if request.has_key(usage_param):
+            # we only understand 'range' -- thus convert it here
+            #self.usage = request[usage_param]
+            usage = request[usage_param]
+            if usage.startswith('range:'): range= usage[6:]
+            else: ValueError('unrecognized usage: %s' % usage)
+            self.range = range
+
+        param = request[iid]
+        keys = None
+        t = type(param)
+
+        if t is InstanceType and not isinstance(param, DateTime):
+            """ query is of type record """
+
+            record = param
+
+            if not hasattr(record, 'query'):
+                raise self.ParserException, (
+                    "record for '%s' *must* contain a "
+                    "'query' attribute" % self.id)
+            keys = record.query
+
+            if type(keys) is StringType:
+                keys = [keys.strip()]
+
+            for op in options:
+                if op == "query": continue
+
+                if hasattr(record, op):
+                    setattr(self, op, getattr(record, op))
+
+        elif t is DictType:
+            """ query is a dictionary containing all parameters """
+
+            query = param.get("query", ())
+            if type(query) in SequenceTypes:
+                keys = query
+            else:
+                keys = [ query ]
+
+            for op in options:
+                if op == "query": continue
+
+                if param.has_key(op):
+                    setattr(self, op, param[op])
+
+        else:
+            """ query is tuple, list, string, number, or something else """
+
+            if t in SequenceTypes:
+                keys = param
+            else:
+                keys = [param]
+
+            for op in options:
+                field = iid + "_" + op
+                if request.has_key(field):
+                    setattr(self, op, request[field])
+
+##        DM: turns empty sequences into 'None', the opposite of an empty sequence of search terms (for "or" searches).
+##        if not keys:
+##            keys = None
+
+        self.keys = keys
+
+    # fix broken inherited get
+    def get(self, key, default=None):
+        v = getattr(self, key, self)
+        if v is self: return default
+        return v

Added: zope-managableindex/branches/upstream/current/tests/TestBase.py
===================================================================
--- zope-managableindex/branches/upstream/current/tests/TestBase.py	2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/tests/TestBase.py	2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,118 @@
+# Copyright (C) 2003 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+#       $Id: TestBase.py,v 1.5 2005/09/17 08:42:03 dieter Exp $
+'''Test infrastructure.'''
+
+
+
+#######################################################################
+# Hack to find our "INSTANCE_HOME" infrastructure.
+# Note, that "testrunner.py" overrides "INSTANCE_HOME". Therefore,
+# we look also for "TEST_INSTANCE_HOME"
+from os import environ, path
+import sys
+
+def _updatePath(path,dir):
+  if dir not in path: path.insert(0,dir)
+
+_ih= environ.get('TEST_INSTANCE_HOME') or environ.get('INSTANCE_HOME')
+if _ih:
+  _updatePath(sys.path, path.join(_ih,'lib','python'))
+  import Products; _updatePath(Products.__path__,path.join(_ih,'Products'))
+
+
+#######################################################################
+# Standard imports
+from unittest import TestCase, TestSuite, makeSuite, TextTestRunner
+
+from Acquisition import Explicit, Implicit
+from OFS.Application import Application
+from OFS.SimpleItem import SimpleItem
+from Products.ZCatalog.ZCatalog import ZCatalog
+from Products.ZCTextIndex.Lexicon import Lexicon, Splitter
+
+from Products.ManagableIndex.FieldIndex import FieldIndex
+from Products.ManagableIndex.KeywordIndex import KeywordIndex
+from Products.ManagableIndex.RangeIndex import RangeIndex
+from Products.ManagableIndex.WordIndex import WordIndex
+from Products.ManagableIndex.PathIndex import PathIndex
+
+def genSuite(*testClasses,**kw):
+  prefix= kw.get('prefix','test')
+  return TestSuite([makeSuite(cl,prefix) for cl in testClasses])
+
+def runSuite(suite):
+  tester= TextTestRunner()
+  tester.run(suite)
+
+def runTests(*testClasses,**kw):
+  runSuite(genSuite(*testClasses,**kw))
+
+
+class Lexicon(Lexicon, SimpleItem): pass
+
+#######################################################################
+# Test base class
+class TestBase(TestCase):
+  '''An application with a catalog with field index 'id',
+  keyword index 'ki', range index 'ri', word index 'wi', path index 'pi'
+  and two objects 'obj1' and 'obj2'.
+  '''
+  def setUp(self):
+    app= Application()
+    catalog= ZCatalog('Catalog')
+    app._setObject('Catalog',catalog)
+    self.catalog= catalog= app._getOb('Catalog')
+    # create indexes -- avoid the official API because it requires
+    # product setup and this takes ages
+    cat= catalog._catalog
+    # field
+    fi= FieldIndex('id'); cat.addIndex('id',fi)
+    self.fi= cat.getIndex('id')
+    # keyword
+    ki= KeywordIndex('kw'); cat.addIndex('kw',ki)
+    self.ki= cat.getIndex('kw')
+    # range
+    ri= RangeIndex('ri'); cat.addIndex('ri',ri)
+    self.ri = ri = cat.getIndex('ri')
+    ri._delObject('ri'); ri.CombineType = 'aggregate'
+    ri.addValueProvider('rlow','AttributeLookup')
+    ri.addValueProvider('rhigh','AttributeLookup')
+    # word
+    lexicon = Lexicon(Splitter())
+    app._setObject('lexicon', lexicon)
+    wi = WordIndex('wi'); cat.addIndex('wi',wi)
+    wi.Lexicon = 'lexicon'
+    self.wi = cat.getIndex('wi')
+    # path
+    pi= PathIndex('pi'); cat.addIndex('pi',pi)
+    self.pi= cat.getIndex('pi')
+    # create objects
+    self.obj1= obj1= _Object()
+    obj1.kw= (1,2)
+    obj1.fkw= _Caller(lambda obj: obj.kw)
+    obj1.fid= _Caller(lambda obj: obj.id)
+    self.obj2= obj2= _Object().__of__(obj1)
+    obj2.id= 'id'
+
+  def _check(self, index, query, should):
+    rs, _ = index._apply_index({index.id:query})
+    self.assertEqual(''.join(map(repr, rs.keys())), should)
+
+
+#######################################################################
+# Auxiliaries
+
+class _Caller(Explicit):
+  def __init__(self,f):
+    self._f= f
+
+  def __call__(self):
+    return self._f(self.aq_parent)
+
+class _Object(Implicit):
+  __roles__ = None
+  __allow_access_to_unprotected_subobjects__ = True
+  def __cmp__(self,other):
+    if not isinstance(other,_Object): raise TypeError('type mismatch in comparison')
+    return cmp(self.__dict__,other.__dict__)

Added: zope-managableindex/branches/upstream/current/tests/__init__.py
===================================================================

Added: zope-managableindex/branches/upstream/current/tests/test_ManagableIndex.py
===================================================================
--- zope-managableindex/branches/upstream/current/tests/test_ManagableIndex.py	2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/tests/test_ManagableIndex.py	2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,611 @@
+# Copyright (C) 2003 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+#       $Id: test_ManagableIndex.py,v 1.13 2006/11/23 19:43:05 dieter Exp $
+
+from TestBase import TestBase, genSuite, runSuite, TestCase
+
+from re import escape
+
+from BTrees.OOBTree import OOSet
+from BTrees.IIBTree import IISet
+from BTrees.IOBTree import IOBTree
+from DateTime.DateTime import DateTime
+
+from Products.ManagableIndex.ValueProvider import ExpressionEvaluator
+from Products.ManagableIndex.ManagableIndex import _splitPrefixRegexp
+from Products.ManagableIndex.ManagableIndex import IFilter, ManagableIndex
+from Products.ManagableIndex.KeywordIndex import KeywordIndex_scalable
+
+
+class TestManagableIndex(TestBase):
+  def test_AttributeLookup(self):
+    fi= self.fi; ki= self.ki # field and keyword index
+    obj1= self.obj1; obj2= self.obj2
+    kw= OOSet((1,2,))
+    ## Acquisition type
+    # implicit
+    self.assertEqual(fi._evaluate(obj2),'id')
+    self.assertEqual(fi._evaluate(obj1),None)
+    # "OOSet" does not support intelligent equality test
+    self.assertEqual(ki._evaluate(obj2).keys(),kw.keys())
+    self.assertEqual(ki._evaluate(obj1).keys(),kw.keys())
+    # none
+    ki.kw.AcquisitionType= 'none'
+    self.assertEqual(ki._evaluate(obj2),None)
+    self.assertEqual(ki._evaluate(obj1).keys(),[1,2])
+    # explicit
+    #  same as implicit for non-methods
+    ki.kw.AcquisitionType= 'explicit'
+    self.assertEqual(fi._evaluate(obj2),'id')
+    self.assertEqual(fi._evaluate(obj1),None)
+    self.assertEqual(ki._evaluate(obj2).keys(),kw.keys())
+    self.assertEqual(ki._evaluate(obj1).keys(),kw.keys())
+    #  now check methods
+    fi.id_.Name= 'fid'; ki.kw.Name= 'fkw'
+    self.assertEqual(fi._evaluate(obj2),'id')
+    self.assertEqual(fi._evaluate(obj1),None)
+    self.assertEqual(ki._evaluate(obj2),None)
+    self.assertEqual(ki._evaluate(obj1).keys(),kw.keys())
+
+    ## call types
+    # call -- already checked
+    # return
+    fi.id_.CallType= 'return'
+    self.assertEqual(fi._evaluate(obj2), obj1.fid)
+    # ignore
+    fi.id_.CallType= 'ignore'
+    self.assertEqual(fi._evaluate(obj2), None)
+    fi.id_.CallType= 'call'
+
+    ## IgnoreExceptions
+    fi.id_.IgnoreExceptions= 0
+    self.assertRaises(AttributeError,fi._evaluate,obj1)
+    self.assertEqual(fi._evaluate(obj2),'id')
+
+  def test_ExpressionEvaluator(self):
+    ki= self.ki; obj2= self.obj2
+    ee= ExpressionEvaluator(); ee.id= 'ee'
+    ki._setObject(ee.id,ee); ee= ki._getOb(ee.id)
+    ee.manage_changeProperties(Expression= 'python: (3,4,)')
+    self.assertEqual(ki._evaluate(obj2).keys(),OOSet((1,2,3,4)).keys())
+    # ignore
+    ee.manage_changeProperties(IgnorePredicate= 'python: 3 in value')
+    self.assertEqual(ki._evaluate(obj2).keys(),OOSet((1,2,)).keys())
+    # ignore - call it
+    ee.manage_changeProperties(IgnorePredicate= 'python: lambda v: 3 in v')
+    # normalize
+    ee.manage_changeProperties(Expression= 'python: (4,)')
+    ee.manage_changeProperties(Normalizer= 'python: (0,) + value')
+    self.assertEqual(ki._evaluate(obj2).keys(),OOSet((0,1,2,4,)).keys())
+    # normalize - call it
+    ee.manage_changeProperties(Normalizer= 'python: lambda v: (0,) + v')
+    self.assertEqual(ki._evaluate(obj2).keys(),OOSet((0,1,2,4,)).keys())
+    # method
+    ee.manage_changeProperties(Expression= "python: lambda object: object.kw")
+    self.assertEqual(ki._evaluate(obj2).keys(),OOSet((0,1,2,)).keys())
+    ## combine
+    # 'union' - already tested
+    # 'useFirst'
+    ki.CombineType= 'useFirst'
+    self.assertEqual(ki._evaluate(obj2).keys(),OOSet((1,2,)).keys())
+
+  def test_TypeChecking(self):
+    fi= self.fi; obj= self.obj2
+    # numeric
+    fi.TermType= 'numeric'
+    obj.id= 1; self.assertEqual(fi._evaluate(obj),1)
+    obj.id= '1'; self.assertEqual(fi._evaluate(obj),1)
+    obj.id= '1.0'; self.assertEqual(fi._evaluate(obj),1.0)
+    obj.id= '1.0+'; self.assertRaises(Exception,fi._evaluate,obj)
+    # string
+    fi.TermType= 'string'
+    obj.id= 1; self.assertEqual(fi._evaluate(obj),'1')
+    obj.id= '1'; self.assertEqual(fi._evaluate(obj),'1')
+    obj.id= obj; self.assertRaises(Exception,fi._evaluate,obj)
+    # unicode
+    fi.TermType= 'ustring'
+    obj.id= u'1'; self.assertEqual(fi._evaluate(obj),u'1')
+    obj.id= '1'; self.assertEqual(fi._evaluate(obj),u'1')
+    # integer
+    fi.TermType= 'integer'
+    obj.id= 1; self.assertEqual(fi._evaluate(obj),1)
+    obj.id= '1'; self.assertEqual(fi._evaluate(obj),1)
+    obj.id= 1.1; self.assertEqual(fi._evaluate(obj),1)
+    # DateTime
+    fi.TermType= 'DateTime'; now= DateTime()
+    obj.id= now; self.assertEqual(fi._evaluate(obj),now)
+    obj.id= '1'; self.assertRaises(Exception,fi._evaluate,obj)
+    # DateTimeInteger
+    fi.TermType= 'DateTimeInteger'
+    obj.id= now; v = fi._evaluate(obj)
+    self.assert_(isinstance(v, int))
+    self.assert_(abs(v-now._t) <= 1)
+    # DateInteger
+    fi.TermType= 'DateInteger'
+    obj.id = DateTime('1000-01-01')
+    v = fi._evaluate(obj)
+    self.assert_(isinstance(v, int))
+    self.assertEqual(v, 400000)
+    # tuple
+    fi.TermType= 'tuple'
+    fi.TermTypeExtra= 'n(su)d'
+    obj.id= (1,('1',u'1'),now); self.assertEqual(fi._evaluate(obj),obj.id)
+    fi.TermTypeExtra+= 'n'
+    self.assertRaises(Exception,fi._evaluate,obj)
+    fi.TermTypeExtra= fi.TermTypeExtra[:-2]
+    self.assertRaises(Exception,fi._evaluate,obj)
+    # instance
+    fi.TermType= 'instance'
+    b= obj.aq_base; cl= b.__class__
+    fi.TermTypeExtra= '%s.%s' % (cl.__module__,cl.__name__)
+    obj.id= b; self.assertEqual(fi._evaluate(obj),b)
+    obj.id= '1'; self.assertRaises(Exception,fi._evaluate,obj)
+    # expression
+    fi.TermType= 'expression checked'
+    fi.TermTypeExtra= 'python: 1'
+    self.assertEqual(fi._evaluate(obj),1)
+    fi.TermTypeExtra= 'python: lambda v: 1'
+    self.assertEqual(fi._evaluate(obj),1)
+
+    ## term copy
+    fi.TermType= 'instance'
+    fi.TermTypeExtra= '%s._Object' % __name__
+    b= _Object()
+    obj.id= b; self.assert_(fi._evaluate(obj) is b)
+    fi.TermCopy= 'shallow'
+    v= fi._evaluate(obj)
+    self.assertEqual(v,b)
+    self.assert_(v is not b)
+    fi.TermCopy= 'deep'
+    b.l= []
+    v= fi._evaluate(obj)
+    self.assertEqual(v,b)
+    self.assert_(v.l is not b.l)
+
+  def test_Terms(self):
+    # normalize term
+    ki= self.ki; obj= self.obj1
+    ki.NormalizeTerm= 'python: lambda value: value-1'
+    self.assertEqual(ki._evaluate(obj).keys(),OOSet((0,1)).keys())
+    # stop term
+    ki.StopTermPredicate= 'python: value == 1'
+    self.assertEqual(ki._evaluate(obj).keys(),OOSet((1,)).keys())
+
+  def test_IntegerOptimization(self):
+    fi= self.fi
+    for t in 'integer DateTimeInteger DateInteger'.split():
+      fi.TermType = t
+      fi.clear()
+      self.assert_(isinstance(fi._index, IOBTree))
+
+
+  def test_len(self):
+    fi = self.fi; obj = self.obj2
+    self.assertEqual(len(fi), 0)
+    fi.index_object(1, obj)
+    self.assertEqual(len(fi), 1)
+    fi.index_object(2, obj)
+    self.assertEqual(len(fi), 1)
+    obj.id = 'id2'
+    fi.index_object(2, obj)
+    self.assertEqual(len(fi), 2)
+
+  def test_FieldIndex(self):
+    fi= self.fi; obj= self.obj2
+    # index_object
+    fi.index_object(1,obj)
+    self.assertEqual(fi.numObjects(),1)
+    self.assertEqual(len(fi),1)
+    # simple succeeding search
+    r,i= fi._apply_index({'id' : 'id'})
+    self.assertEqual(i,fi.id)
+    self.assertEqual(r.keys(),[1])
+    self._check(fi, 'id', '1')
+    # simple failing search
+    r,i= fi._apply_index({'id' : ''})
+    self.assertEqual(i,fi.id)
+    self.assertEqual(r.keys(), [])
+    # or search
+    q = ('','id',)
+    r,i= fi._apply_index({'id':q})
+    self.assertEqual(i,fi.id)
+    self.assertEqual(r.keys(), [1])
+    self._check(fi, q, '1')
+    # empty or search
+    r,i= fi._apply_index({'id' : ()})
+    self.assertEqual(i,fi.id)
+    self.assertEqual(r.keys(), [])
+    # range searches
+    q = {'query' : ('a','z'), 'range' : 'min:max'}
+    r,i= fi._apply_index({'id':q})
+    self.assertEqual(i,fi.id)
+    self.assertEqual(r.keys(), [1])
+    self._check(fi, q, '1')
+    q = {'query' : ('a',), 'range' : 'min'}
+    r,i= fi._apply_index({'id':q})
+    self.assertEqual(i,fi.id)
+    self.assertEqual(r.keys(), [1])
+    self._check(fi, q, '1')
+    q = {'query' : ('z',), 'range' : 'max'}
+    r,i= fi._apply_index({'id':q})
+    self.assertEqual(i,fi.id)
+    self.assertEqual(r.keys(), [1])
+    self._check(fi, q, '1')
+    r,i= fi._apply_index({'id' : {'query' : ('a','i'), 'range' : 'min:max'}})
+    self.assertEqual(i,fi.id)
+    self.assertEqual(r.keys(), [])
+    r,i= fi._apply_index({'id' : {'query' : ('j','z'), 'range' : 'min:max'}})
+    self.assertEqual(i,fi.id)
+    self.assertEqual(r.keys(), [])
+    r,i= fi._apply_index({'id' : {'query' : ('j',), 'range' : 'min'}})
+    self.assertEqual(i,fi.id)
+    self.assertEqual(r.keys(), [])
+    r,i= fi._apply_index({'id' : {'query' : ('i',), 'range' : 'max'}})
+    self.assertEqual(i,fi.id)
+    self.assertEqual(r.keys(), [])
+    # simple and search
+    r,i= fi._apply_index({'id' : {'query' : ('id',), 'operator' : 'and'}})
+    self.assertEqual(i,fi.id)
+    self.assertEqual(r.keys(), [1])
+    # multi and search
+    r,i= fi._apply_index({'id' : {'query' : ('id','id',), 'operator' : 'and'}})
+    self.assertEqual(i,fi.id)
+    self.assertEqual(r.keys(), [1])
+    # empty and search
+    r= fi._apply_index({'id' : {'query' : (), 'operator' : 'and'}})
+    self.assertEqual(r,None)
+    # failing and search
+    r,i= fi._apply_index({'id' : {'query' : ('id','',), 'operator' : 'and'}})
+    self.assertEqual(i,fi.id)
+    self.assertEqual(r.keys(), [])
+    # reindex
+    obj.id= 'id1'
+    fi.index_object(1,obj)
+    self.assertEqual(fi.numObjects(),1)
+    self.assertEqual(len(fi),1)
+    # unindex
+    fi.unindex_object(1)
+    self.assertEqual(fi.numObjects(),0)
+    self.assertEqual(len(fi),0)
+
+  def test_RangeIndex(self):
+    ri= self.ri; obj= self.obj2
+    # index_object
+    ri.index_object(1,obj)
+    self.assertEqual(ri.numObjects(),0)
+    obj.rlow = 10; ri.index_object(1,obj)
+    self.assertEqual(ri.numObjects(),0)
+    obj.rhigh = 20; ri.index_object(1,obj)
+    self.assertEqual(ri.numObjects(),1)
+    obj.rlow = obj.rhigh = 15; ri.index_object(2,obj)
+    self.assertEqual(ri.numObjects(),2)
+    # search
+    r,i = ri._apply_index({'ri': 9})
+    self.assertEqual(i,ri.id)
+    self.assertEqual(r.keys(), [])
+    r,i = ri._apply_index({'ri': 15})
+    self.assertEqual(i,ri.id)
+    self.assertEqual(r.keys(), [1,2])
+    self._check(ri, 15, '12')
+    r,i = ri._apply_index({'ri': 20})
+    self.assertEqual(i,ri.id)
+    self.assertEqual(r.keys(), [1])
+    self._check(ri, 20, '1')
+    self.assertEqual(len(ri), 2)
+    # boundary emulation
+    ri.BoundaryNames = ('low','high')
+    r = ri._apply_index({'low': {'query':15, 'range':'min'},})
+    self.assertEqual(r, None)
+    r = ri._apply_index({'low':{'query':15, 'range':'min'},
+                         'high':{'query':16, 'range':'max'},
+                         })
+    self.assertEqual(r, None)
+    r,i = ri._apply_index({'low':{'query':15, 'range':'min'},
+                         'high':{'query':15, 'range':'max'},
+                         })
+    self.assertEqual(i,ri.id)
+    self.assertEqual(r.keys(), [1,2])
+    # unindex object
+    ri.unindex_object(1)
+    self.assertEqual(len(ri), 1)
+    self.assertEqual(ri.numObjects(),1)
+    ri.index_object(1, obj)
+    self.assertEqual(len(ri), 1)
+    self.assertEqual(ri.numObjects(),2)
+    ri.unindex_object(1); ri.unindex_object(2)
+    self.assertEqual(len(ri), 0)
+    self.assertEqual(ri.numObjects(),0)
+
+  def test_ImproperRanges(self):
+    ri= self.ri; obj= self.obj2
+    ri.MinimalValue = 'python:10'; ri.MaximalValue = 'python:20'
+    ri.clear()
+    self._indexForRange(obj, ri)
+    r,i = ri._apply_index({'ri':0}); self.assertEqual(r.keys(),[1,2])
+    r,i = ri._apply_index({'ri':20}); self.assertEqual(r.keys(),[1,3])
+    r,i = ri._apply_index({'ri':15}); self.assertEqual(r.keys(),[1,2,3,4])
+    # check empty min
+    ri.MinimalValue = ''; ri.MaximalValue = 'python:20'
+    ri.clear()
+    self._indexForRange(obj, ri)
+    r,i = ri._apply_index({'ri':0}); self.assertEqual(r.keys(),[])
+    r,i = ri._apply_index({'ri':20}); self.assertEqual(r.keys(),[1,3])
+    r,i = ri._apply_index({'ri':15}); self.assertEqual(r.keys(),[1,2,3,4])
+    ri.MinimalValue = 'python:10'; ri.MaximalValue = ''
+    ri.clear()
+    self._indexForRange(obj, ri)
+    r,i = ri._apply_index({'ri':0}); self.assertEqual(r.keys(),[1,2])
+    r,i = ri._apply_index({'ri':21}); self.assertEqual(r.keys(),[])
+    r,i = ri._apply_index({'ri':15}); self.assertEqual(r.keys(),[1,2,3,4])
+    ri.MinimalValue = ''; ri.MaximalValue = ''
+    ri.clear()
+    self._indexForRange(obj, ri)
+    r,i = ri._apply_index({'ri':0}); self.assertEqual(r.keys(),[])
+    r,i = ri._apply_index({'ri':21}); self.assertEqual(r.keys(),[])
+    r,i = ri._apply_index({'ri':15}); self.assertEqual(r.keys(),[1,2,3,4])
+
+  def test_Organisation(self):
+    ri = self.ri; ri.OrganisationHighThenLow = True
+    ri.clear()
+    self.test_RangeIndex()
+    self.test_ImproperRanges()
+
+  def _indexForRange(self, obj, ri):
+      obj.rlow = 10; obj.rhigh = 20; ri.index_object(1, obj) # unrestricted
+      obj.rlow = 10; obj.rhigh = 15; ri.index_object(2, obj) # low unrestricted
+      obj.rlow = 15; obj.rhigh = 20; ri.index_object(3, obj) # high unrestricted
+      obj.rlow = 15; obj.rhigh = 15; ri.index_object(4, obj) # proper range
+
+  def test_PathIndex(self):
+    pi= self.pi; obj= self.obj2
+    # index_object
+    pi.index_object(1,obj)
+    self.assertEqual(pi.numObjects(),0)
+    obj.pi = 'a'; pi.index_object(1,obj)
+    self.assertEqual(pi.numObjects(),1)
+    obj.pi = 'a/b'; pi.index_object(2,obj)
+    obj.pi = 'a/b/c'.split('/'); pi.index_object(3,obj)
+    # check queries
+    c = self._check
+    c(pi, [()], '123')
+    c(pi, {'query':[()], 'level':2}, '23')
+    c(pi, {'query':[()], 'level':1, 'depth':1}, '2')
+    c(pi, {'query':[()], 'level':None}, '123')
+    c(pi, {'query':[()], 'level':-1}, '123')
+    c(pi, 'a', '123')
+    c(pi, 'b', '')
+    c(pi, 'x', '')
+    c(pi, {'query':'b', 'level':1}, '23')
+    c(pi, {'query':'b', 'level':None}, '23')
+    c(pi, {'query':'b/c', 'level':-1}, '3')
+    c(pi, {'query':['b/c'.split('/')], 'level':-1}, '3')
+    c(pi, {'query':'c', 'level':-1}, '')
+    c(pi, {'query':'c', 'level':-2}, '3')
+    c(pi, {'query':'a', 'depth':0}, '1')
+    c(pi, {'query':'a', 'depth':1}, '2')
+    c(pi, {'query':'a', 'depth':-1}, '12')
+    c(pi, {'query':'a', 'depth':None}, '123')
+    c(pi, {'query':[()], 'depth':-2}, '12')
+    c(pi, {'query':'b', 'level':None, 'depth':-1}, '23')
+    obj.pi = '/a'; pi.index_object(4,obj)
+    c(pi, '/', '4')
+
+
+  def test_ReverseOrder(self):
+    fi= self.fi; obj= self.obj2
+    self.assertEqual(fi.getReverseOrder(), None)
+    fi.ReverseOrder = 1; fi.clear()
+    # index_object
+    obj.id = '1'; fi.index_object(1,obj)
+    obj.id = '2'; fi.index_object(2,obj)
+    self.assertEqual(tuple(fi.getReverseOrder()), ('2','1'))
+    fi.unindex_object(2)
+    self.assertEqual(tuple(fi.getReverseOrder()), ('1',))
+
+  def test_Sorting(self):
+    cat= self.catalog; obj= self.obj2
+    cat.catalog_object(obj,'1')
+    self.assertEqual(len(self.catalog(id='id', sort_on='id')),1)
+    cat.catalog_object(obj,'1')
+
+  def test_KeywordIndex(self):
+    ki= self.ki; obj= self.obj2
+    # index_object
+    ki.index_object(1,obj)
+    self.assertEqual(ki.numObjects(),1)
+    self.assertEqual(len(ki),2)
+    self._check(ki, 1, '1')
+    # reindex_object
+    self.assertEqual(ki.index_object(1,obj),0)
+    obj.kw= (2,3,)
+    ki.index_object(1,obj)
+    self.assertEqual(ki.numObjects(),1)
+    self.assertEqual(len(ki),2)
+    self.assertEqual(ki.uniqueValues(),(2,3,))
+    self.assertEqual(ki.uniqueValues(withLengths=1),((2,1),(3,1),))
+    # index 2
+    ki.index_object(2,self.obj1)
+    self.assertEqual(ki.numObjects(),2)
+    self.assertEqual(len(ki),3)
+    self.assertEqual(ki.uniqueValues(withLengths=1),((1,1),(2,2),(3,1),))
+    # unindex
+    ki.unindex_object(1)
+    self.assertEqual(ki.numObjects(),1)
+    self.assertEqual(len(ki),2)
+    self.assertEqual(ki.uniqueValues(withLengths=1),((1,1),(2,1),))
+    ki.unindex_object(2)
+    self.assertEqual(ki.numObjects(),0)
+    self.assertEqual(len(ki),0)
+    self.assertEqual(ki.uniqueValues(withLengths=1),())
+
+  def test_KeywordIndex_scalable(self):
+    cat = self.catalog._catalog
+    cat.delIndex('kw')
+    ki = KeywordIndex_scalable('kw'); cat.addIndex('kw', ki)
+    self.ki = cat.getIndex('kw')
+    self.test_KeywordIndex()
+
+class TestFiltering(TestBase):
+  test_FieldIndex = TestManagableIndex.test_FieldIndex.im_func
+  test_KeywordIndex = TestManagableIndex.test_KeywordIndex.im_func
+  test_KeywordIndex_scalable = TestManagableIndex.test_KeywordIndex_scalable.im_func
+  test_RangeIndex = TestManagableIndex.test_RangeIndex.im_func
+  test_PathIndex = TestManagableIndex.test_PathIndex.im_func
+
+  def _check(self, index, query, should):
+    if not isinstance(query, dict): query = {'query':query}
+    query['isearch'] = query['isearch_filter'] = True
+    rs, _ = index._apply_index({index.id:query})
+    if hasattr(rs, 'asSet'): rs = rs.asSet().keys()
+    self.assertEqual(''.join(map(repr, rs)), should)
+
+class TestMatching(TestBase):
+  '''test glob and regexp expansion.'''
+  TermType = 'string'
+
+  def setUp(self):
+    '''
+    1 -> a
+    2 -> b
+    3 -> ba
+    4 -> bab
+    5 -> c
+    6 -> cb
+    7 -> d\e
+    '''
+    TestBase.setUp(self)
+    self.index = index = self.fi; obj = self.obj1
+    index.TermType = self.TermType
+    for i, val in enumerate(r'a b ba bab c cb d\e'.split()):
+      obj.id = val
+      index.index_object(i+1, obj)
+
+  def test_glob(self):
+    self._check('glob', '*', '1234567')
+    self._check('glob', 'b*', '234')
+    self._check('glob', '*a', '13')
+    self._check('glob', 'b?', '3')
+    self._check('glob', r'd\e', '7')
+
+  def test_regexp(self):
+    self._check('regexp', 'a*', '1234567')
+    self._check('regexp', 'ba+', '34')
+    self._check('regexp', 'b[a]+', '34')
+
+  def test_splitPrefix(self):
+    self._checkSplit('abc', 'abc')
+    self._checkSplit('+', '')
+    self._checkSplit('a+', '')
+    self._checkSplit(r'\+', '+')
+    self._checkSplit(r'\a', '\a', '')
+    self._checkSplit(r'\d', '')
+    self._checkSplit(r'\c', 'c', '')
+    self._checkSplit(r'\\+', '')
+    self._checkSplit(r'a\\+', 'a')
+
+  def _check(self, match, pattern, result):
+    i = self.index
+    r = i._apply_index({i.id : {'query' : pattern, 'match':match},})[0]
+    self.assertEqual(''.join(map(str, r)), result)
+
+  def _checkSplit(self, regexp, prefix, rep=None):
+    pr,rp = _splitPrefixRegexp(regexp)
+    if rep is None: self.assertEqual(escape(pr) + rp, regexp)
+    else: self.assertEqual(rp, rep)
+    self.assertEqual(pr, prefix)
+
+class TestMatching_Unicode(TestMatching):
+  TermType = 'ustring'
+
+
+class TestMatching_Filtering(TestMatching):
+
+  def _check(self, match, pattern, result):
+    i = self.index
+    r = i._apply_index({i.id : {'query' : pattern, 'match':match, 'isearch':True, 'isearch_filter':True},})[0]
+    if hasattr(r, 'asSet'): r = r.asSet().keys()
+    self.assertEqual(''.join(map(str, r)), result)
+  
+
+
+class TestWordIndex(TestBase):
+  def setUp(self):
+    '''
+    1 -> a ab abc
+    2 -> b ab bca
+    '''
+    TestBase.setUp(self)
+    obj = self.obj1; index = self.wi
+    obj.wi = 'a ab abc'; index.index_object(1, obj)
+    obj.wi = 'b ab bca'; index.index_object(2, obj)
+
+  def test_Lookup(self):
+    self._check('a', '1')
+    self._check('ab', '12')
+    self._check('A', '')
+    self._check('bca', '2')
+    self.assertRaises(ValueError, self._check, 'a b', '')
+
+  def test_Matching(self):
+    self._check({'query':'a*', 'match':'glob'}, '12')
+    self._check({'query':'a b', 'match':'glob'}, '')
+##    self.assertRaises(ValueError,
+##                      self._check,
+##                      {'query':'a b', 'match':'glob'},
+##                      ''
+##                      )
+
+  def _check(self, query, result):
+    i = self.wi
+    r = i._apply_index({i.id : query,})[0]
+    self.assertEqual(''.join(map(str, r.keys())), result)
+
+
+class TestProgrammaticSetup(TestCase):
+  def test_PropertySetup(self):
+    self.assertRaises(ValueError, ManagableIndex, 'mi', {'TermTypeExtra':'tte', 'X':None})
+    mi = ManagableIndex('mi', {'TermTypeExtra':'tte'})
+    self.assertEqual(mi.TermTypeExtra, 'tte')
+
+  def test_NoProvider(self):
+    mi = ManagableIndex('mi', {})
+    self.assertEqual(mi.objectIds(), [])
+
+  def test_ExpliciteProvidersWithProperties(self):
+    mi = ManagableIndex('mi', {
+      'ValueProviders': (
+      {'type':'AttributeLookup', 'id':'al', 'Name':'name',},
+      {'type':'ExpressionEvaluator', 'id':'el', 'Expression':'e',},
+      ),
+      }
+                        )
+    self.assertEqual(len(mi.objectIds()), 2)
+    self.assertEqual(mi.al.Name, 'name')
+    self.assertEqual(mi.el.Expression, 'e')
+
+
+
+class _Object:
+  def __eq__(self,other):
+    return isinstance(other,_Object) and self.__dict__ == other.__dict__
+  def __ne__(self,other): return not (self == other)
+  def __cmp__(self,other):
+    if not isinstance(other,_Object): raise TypeError('type mismatch in comparison')
+    return cmp(self.__dict__,other.__dict__)
+
+
+
+def test_suite():
+  tests = [
+    TestManagableIndex,
+    TestMatching, TestMatching_Unicode,
+    TestWordIndex,
+    TestProgrammaticSetup,
+    ]
+  if IFilter is not None:
+    tests.extend([TestFiltering, TestMatching_Filtering])
+  return genSuite(*tests)
+
+
+if __name__ == '__main__':
+  runSuite(test_suite())

Added: zope-managableindex/branches/upstream/current/zpt/addForm.zpt
===================================================================
--- zope-managableindex/branches/upstream/current/zpt/addForm.zpt	2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/zpt/addForm.zpt	2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,33 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2//EN">
+<html tal:define="
+  type options/type;
+  action options/action;
+  description options/description;
+  title string: Add $type;
+  "
+>
+  <head>
+    <title tal:content="title">Title</title>
+  </head>
+
+  <body>
+    <h2 tal:content="title">Title</h2>
+
+    <p tal:content="description">Description</p>
+
+    <form action="action" method="post"
+      tal:attributes="action action">
+      <table cellpadding=5>
+      <tr>
+        <th>Id</th><td><input name="id"></td>
+      </tr>
+      <tr>
+        <td colspan=2>
+	  <input type="hidden" name="type" tal:attributes="value type">
+	  <input type="submit" name="add">
+	</td>
+      </tr>
+      </table>
+    </form>
+  </body>
+</html>




More information about the pkg-zope-commits mailing list