r736 - / zope-managableindex zope-managableindex/branches
zope-managableindex/branches/upstream
zope-managableindex/branches/upstream/current
zope-managableindex/branches/upstream/current/doc
zope-managableindex/branches/upstream/current/tests
zope-managableindex/branches/upstream/current/zpt
Bernd Zeimetz
bzed-guest at alioth.debian.org
Wed Apr 4 00:01:35 UTC 2007
Author: bzed-guest
Date: 2007-04-04 00:01:33 +0000 (Wed, 04 Apr 2007)
New Revision: 736
Added:
zope-managableindex/
zope-managableindex/branches/
zope-managableindex/branches/upstream/
zope-managableindex/branches/upstream/current/
zope-managableindex/branches/upstream/current/Evaluation.py
zope-managableindex/branches/upstream/current/FieldIndex.py
zope-managableindex/branches/upstream/current/KeywordIndex.py
zope-managableindex/branches/upstream/current/LICENSE.txt
zope-managableindex/branches/upstream/current/ManagableIndex.py
zope-managableindex/branches/upstream/current/PathIndex.py
zope-managableindex/branches/upstream/current/RangeIndex.py
zope-managableindex/branches/upstream/current/Utils.py
zope-managableindex/branches/upstream/current/VERSION.txt
zope-managableindex/branches/upstream/current/ValueProvider.py
zope-managableindex/branches/upstream/current/WordIndex.py
zope-managableindex/branches/upstream/current/__init__.py
zope-managableindex/branches/upstream/current/doc/
zope-managableindex/branches/upstream/current/doc/ManagableIndex.html
zope-managableindex/branches/upstream/current/fixPluginIndexes.py
zope-managableindex/branches/upstream/current/tests/
zope-managableindex/branches/upstream/current/tests/TestBase.py
zope-managableindex/branches/upstream/current/tests/__init__.py
zope-managableindex/branches/upstream/current/tests/test_ManagableIndex.py
zope-managableindex/branches/upstream/current/zpt/
zope-managableindex/branches/upstream/current/zpt/addForm.zpt
zope-managableindex/tags/
Log:
[svn-inject] Installing original source of zope-managableindex
Added: zope-managableindex/branches/upstream/current/Evaluation.py
===================================================================
--- zope-managableindex/branches/upstream/current/Evaluation.py 2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/Evaluation.py 2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,135 @@
+# Copyright (C) 2003 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+# $Id: Evaluation.py,v 1.5 2006/04/09 16:52:03 dieter Exp $
+'''TALES evaluation for different purposes.'''
+
+from AccessControl import getSecurityManager
+from Acquisition import Explicit, Implicit
+from Persistence import Persistent
+
+from OFS.PropertyManager import PropertyManager
+from DocumentTemplate.DT_Util import safe_callable
+
+from Products.PageTemplates.Expressions import getEngine, SecureModuleImporter
+
+import ManagableIndex
+
+
+class Eval(PropertyManager,Explicit,Persistent):
+ '''evaluate a TALES expression and return the result.'''
+ # to be overridden by derived classes
+ ExpressionProperty= 'Expression'
+
+ _properties= (
+ { 'id' : ExpressionProperty, 'type' : 'string', 'mode' : 'w',},
+ )
+
+ Expression= ''
+
+ def __init__(self,exprProperty= None):
+ if exprProperty: self.ExpressionProperty= exprProperty
+
+ def _evaluate(self,value,object):
+ '''evaluate our expression property in a context containing
+ *value*, *object*, 'index', 'catalog' and standard names.'''
+ index= self._findIndex()
+ catalog= index.aq_inner.aq_parent # ATT: do we need to go up an additional level?
+ # work around bug in "aq_acquire" (has 'default' argument but ignores it)
+ try: request= object.aq_acquire('REQUEST',default=None)
+ except AttributeError: request= None
+ try: container= object.aq_inner.aq_parent
+ except AttributeError: container= None
+ data= {
+ 'value' : value,
+ 'index' : index,
+ 'catalog' : catalog,
+ 'object' : object,
+ 'here' : object, # compatibility
+ 'container' : container,
+ 'nothing' : None,
+ 'root' : index.getPhysicalRoot(),
+ #'request': object.aq_acquire('REQUEST',default=None),
+ 'request': request,
+ 'modules' : SecureModuleImporter,
+ 'user' : getSecurityManager().getUser(),
+ }
+ context= getEngine().getContext(data)
+ expr= self._getExpression()
+ return expr(context)
+
+ def _findIndex(self):
+ '''return the nearest 'ManagableIndex' above us.'''
+ obj= self
+ while not isinstance(obj,ManagableIndex.ManagableIndex):
+ obj= obj.aq_inner.aq_parent
+ return obj
+
+ _v_expression= 0
+ _v_expr_string= None
+ def _getExpression(self):
+ '''return the TALES expression to be evaluated.'''
+ expr= self._v_expression
+ expr_string= self.aq_acquire(self.ExpressionProperty)
+ if expr != 0 and self._v_expr_string == expr_string: return expr
+ expr= self._v_expression= getEngine().compile(expr_string)
+ self._v_expr_string= expr_string
+ return expr
+
+ def _getExpressionString(self):
+ # this uses 'aq_aquire', because we want to support independent multiple inheritance
+ return self.aq_acquire(self.ExpressionProperty)
+
+
+class EvalAndCall(Eval):
+ '''evaluate and then call with *value*, if possible.'''
+ def _evaluate(self,value,object):
+ v= EvalAndCall.inheritedAttribute('_evaluate')(self,value,object)
+ if safe_callable(v): v= v(value)
+ return v
+
+
+class Ignore(PropertyManager,Implicit):
+ '''ignore values for which 'IgnorePredicate' gives true.'''
+ IgnoreProperty= 'IgnorePredicate'
+
+ _properties= (
+ {'id' : IgnoreProperty, 'type' : 'string', 'mode' : 'w',},
+ )
+ IgnorePredicate= ''
+
+ _v_IgnoreEvaluator= None
+ def _ignore(self,value,object):
+ if value is None: return
+ if not self._hasIgnorer(): return value
+ evaluator= self._v_IgnoreEvaluator
+ if evaluator is None:
+ evaluator= self._v_IgnoreEvaluator= EvalAndCall(self.IgnoreProperty)
+ evaluator= evaluator.__of__(self)
+ if evaluator._evaluate(value,object): return
+ return value
+
+ def _hasIgnorer(self):
+ return getattr(self,self.IgnoreProperty,None)
+
+class Normalize(PropertyManager,Implicit):
+ '''normalize value by the 'Normalizer' expression.'''
+ NormalizerProperty= 'Normalizer'
+
+ _properties= (
+ {'id' : NormalizerProperty, 'type' : 'string', 'mode' : 'w',},
+ )
+ Normalizer= ''
+
+ _v_NormalizeEvaluator= None
+ def _normalize(self,value,object):
+ if value is None: return
+ if not self._hasNormalizer(): return value
+ evaluator= self._v_NormalizeEvaluator
+ if evaluator is None:
+ evaluator= self._v_NormalizeEvaluator= EvalAndCall(self.NormalizerProperty)
+ evaluator= evaluator.__of__(self)
+ if not evaluator._getExpressionString(): return value
+ return evaluator._evaluate(value,object)
+
+ def _hasNormalizer(self):
+ return getattr(self,self.NormalizerProperty,None)
Added: zope-managableindex/branches/upstream/current/FieldIndex.py
===================================================================
--- zope-managableindex/branches/upstream/current/FieldIndex.py 2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/FieldIndex.py 2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,43 @@
+# Copyright (C) 2003 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+# $Id: FieldIndex.py,v 1.3 2006/04/09 16:52:03 dieter Exp $
+'''Managable FieldIndex.'''
+
+from ManagableIndex import ManagableIndex, addForm
+
+class FieldIndex(ManagableIndex):
+ '''a managable 'FieldIndex'.'''
+ meta_type= 'Managable FieldIndex'
+
+ _properties = (
+ ManagableIndex._properties
+ + (
+ {'id':'ReverseOrder',
+ 'label':'Maintain reverse order (used by AdvancedQuery to efficiently support descending order). Remember to clear the index when you change this value!',
+ 'type':'boolean', 'mode':'rw',},
+ )
+ )
+
+ def _indexValue(self,documentId,val,threshold):
+ self._insert(val,documentId)
+ return 1
+
+ def _unindexValue(self,documentId,val):
+ self._remove(val,documentId)
+
+ # newly required for Zope 2.7
+ def documentToKeyMap(self):
+ '''must return a map from document ids to object value.'''
+ return self._unindex
+
+ # filtering support
+ supportFiltering = True
+
+
+def addFieldIndexForm(self):
+ '''add FieldIndex form.'''
+ return addForm.__of__(self)(
+ type= FieldIndex.meta_type,
+ description= '''A FieldIndex indexes an object under a single (atomic) value.''',
+ action= 'addIndex',
+ )
Added: zope-managableindex/branches/upstream/current/KeywordIndex.py
===================================================================
--- zope-managableindex/branches/upstream/current/KeywordIndex.py 2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/KeywordIndex.py 2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,115 @@
+# Copyright (C) 2003 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+# $Id: KeywordIndex.py,v 1.8 2006/05/17 19:53:07 dieter Exp $
+'''Managable KeywordIndex.'''
+
+from sys import maxint
+
+from BTrees.OOBTree import difference, OOSet, union, OOTreeSet
+
+from ManagableIndex import ManagableIndex, addForm
+
+
+class KeywordIndex(ManagableIndex):
+ '''a managable 'KeywordIndex'.'''
+ meta_type= 'Managable KeywordIndex'
+
+ Combiners= ManagableIndex.Combiners + ('union',)
+ CombineType= Combiners[-1]
+
+ def _createDefaultValueProvider(self):
+ ManagableIndex._createDefaultValueProvider(self)
+ # add (value) normalizer to let it behave identical to
+ # a standard Zope KeywordIndex
+ vp= self.objectValues()[0]
+ setattr(vp, vp.NormalizerProperty, 'python: hasattr(value,"capitalize") and (value,) or value')
+
+ def _indexValue(self,documentId,val,threshold):
+ if not threshold: threshold= maxint
+ n= i= 0; T= get_transaction()
+ for v in val.keys():
+ self._insert(v,documentId)
+ n+=1; i+=1
+ if i == threshold: T.commit(1); i= 0
+ return n
+
+ def _unindexValue(self,documentId,val):
+ for v in val.keys():
+ self._remove(v,documentId)
+
+ def _update(self,documentId,val,oldval,threshold):
+ add= difference(val,oldval)
+ rem= difference(oldval,val)
+ if add: self._indexValue(documentId,add,threshold)
+ if rem: self._unindexValue(documentId,rem)
+ self._updateOldval(oldval, val, add, rem)
+ return len(add),
+
+ def _updateOldval(self, oldval, newval, add, rem):
+ # optimize transaction size by not writing _unindex bucket
+ oldval.clear(); oldval.update(newval)
+
+ def _equalValues(self,val1,val2):
+ if val1 == val2: return 1
+ if val1 is None or val2 is None: return 0
+ return tuple(val1.keys()) == tuple(val2.keys())
+
+ def _combine_union(self,values,object):
+ if not values: return
+ set= None
+ for v in values:
+ sv= self._standardizeValue(v,object)
+ if not sv: continue
+ set= union(set,sv)
+ return set
+
+ _SETTYPE = OOSet
+
+ def _standardizeValue(self,value,object):
+ '''convert to a set of standardized terms.'''
+ if not value: return
+ set= self._SETTYPE([st for st in [self._standardizeTerm(t,object) for t in value] if st is not None])
+ return set or None
+
+ # filtering support
+ supportFiltering = True
+
+ def _makeFilter(self, pred):
+ '''a document filter 'did -> True/False' checking term predicate *pred*.'''
+ def check(did):
+ dv = self._unindex.get(did)
+ if dv is None: return False
+ for t in dv.keys():
+ if pred(t): return True
+ return False
+ return check
+
+def addKeywordIndexForm(self):
+ '''add KeywordIndex form.'''
+ return addForm.__of__(self)(
+ type= KeywordIndex.meta_type,
+ description= '''A KeywordIndex indexes an object under a set of terms.''',
+ action= 'addIndex',
+ )
+
+class KeywordIndex_scalable(KeywordIndex):
+ '''a Keyword index that can efficiently handle huge keyword sets per object.'''
+ _SETTYPE = OOTreeSet
+ meta_type = 'Managable KeywordIndex (scalable)'
+
+ def _updateOldval(self, oldval, newval, add, rem):
+ for t in rem: oldval.remove(t)
+ oldval.update(add)
+
+def addKeywordIndex_scalableForm(self):
+ '''add KeywordIndex form.'''
+ return addForm.__of__(self)(
+ type= KeywordIndex_scalable.meta_type,
+ description= '''A KeywordIndex (scalable) indexes an object under a (potentially huge) set of terms.''',
+ action= 'addIndex',
+ )
+
+try:
+ import transaction # ZODB 3.4 (Zope 2.8)
+ def get_transaction(): return transaction
+except ImportError: pass # pre ZODB 3.4
Added: zope-managableindex/branches/upstream/current/LICENSE.txt
===================================================================
--- zope-managableindex/branches/upstream/current/LICENSE.txt 2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/LICENSE.txt 2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,32 @@
+Copyright (C) 2003-2006 by Dr. Dieter Maurer <dieter at handshake.de>
+D-66386 St. Ingbert, Eichendorffstr. 23, Germany
+
+ All Rights Reserved
+
+Permission to use, copy, modify, and distribute this software and its
+documentation for any purpose and without fee is hereby granted,
+provided that the above copyright notice and this permission
+notice appear in all copies, modified copies and in
+supporting documentation.
+
+Dieter Maurer DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL Dieter Maurer
+BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL
+DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
+PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
+
+
+IMPORTANT NOTE: I consider to forbid institutions of the
+European Union (Commission, Council of Ministers, Parliament, ...)
+to use software developped by me.
+This is in protest against plans of the Council of Ministers to make
+logic (general ideas outside of a concrete technical system)
+and especially software patentable in Europe.
+Note that with such a restriction, this software will cease to be
+compatible with the GPL (GNU Public License).
+This version does not yet have the restriction but a future version
+may.
+
Added: zope-managableindex/branches/upstream/current/ManagableIndex.py
===================================================================
--- zope-managableindex/branches/upstream/current/ManagableIndex.py 2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/ManagableIndex.py 2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,963 @@
+# Copyright (C) 2003-2006 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+# $Id: ManagableIndex.py,v 1.16 2006/11/23 19:43:04 dieter Exp $
+'''Managable Index abstract base class.'''
+
+import copy
+from types import IntType, LongType, FloatType, \
+ StringType, UnicodeType, \
+ TupleType, InstanceType
+from sys import modules, getdefaultencoding
+from re import escape, compile
+
+from Globals import InitializeClass
+from Acquisition import aq_base
+from AccessControl import ClassSecurityInfo
+from BTrees.IOBTree import IOBTree
+from BTrees.IIBTree import IISet, IITreeSet, union, intersection, multiunion
+from BTrees.OOBTree import OOBTree, OOTreeSet
+from BTrees.Length import Length
+from DateTime.DateTime import DateTime
+
+from Products.OFolder.OFolder import OFolder
+from Products.PluginIndexes.common.PluggableIndex import PluggableIndexInterface
+from Products.PluginIndexes.common.util import parseIndexRequest
+from Products.PageTemplates.PageTemplateFile import PageTemplateFile
+
+from fixPluginIndexes import parseIndexRequest
+from Evaluation import Normalize, Ignore, EvalAndCall
+from ValueProvider import AttributeLookup, ExpressionEvaluator
+from Utils import reverseOrder, _LazyMap, \
+ convertToDateTime, convertToDateTimeInteger, convertToDateInteger
+
+_mdict= globals()
+
+ManageManagableIndexes= "ManagableIndex: manage"
+
+
+_TermTypeList= (
+ 'not checked',
+ 'string',
+ 'ustring',
+ 'integer',
+ 'numeric',
+ 'DateTime',
+ 'DateTimeInteger',
+ 'DateInteger',
+ 'tuple',
+ 'instance',
+ 'expression checked',
+ )
+
+_IntegerTypes = 'integer DateTimeInteger DateInteger'.split()
+
+_TermCopyList= (
+ 'none',
+ 'shallow',
+ 'deep',
+ )
+
+
+class ManagableIndex(OFolder,Normalize, Ignore):
+ '''Abstract base class for 'ManagableIndex'.'''
+
+ security= ClassSecurityInfo()
+ security.declareProtected(
+ ManageManagableIndexes,
+ 'addAttributeLookupForm',
+ 'addExpressionEvaluatorForm',
+ 'addValueProvider',
+ 'manage_propertiesForm', 'manage_changeProperties', 'manage_editProperties',
+ 'indexSize',
+ )
+ security.declarePrivate(
+ 'getReverseOrder',
+ )
+
+
+ icon= 'misc_/PluginIndexes/index.gif'
+
+ manage_options= (
+ OFolder.manage_options[:1]
+ + OFolder.manage_options[2:]
+ )
+
+ operators= ('or', 'and',)
+ useOperator= 'or'
+ query_options= ('operator', 'range', 'usage', 'match', 'isearch', 'isearch_filter')
+ Combiners= ('useFirst',)
+
+ NormalizerProperty= 'NormalizeTerm'
+ IgnoreProperty= 'StopTermPredicate'
+
+ _properties= (
+ (
+ { 'id' : 'CombineType', 'type' : 'selection', 'mode' : 'w', 'select_variable' : 'listCombiners',
+ 'label':'Combine type: determines how values from value providers are combined'},
+ {'id':'PrenormalizeTerm', 'type':'string', 'mode':'w',
+ 'label':'Term prenormalizer: applied to terms before term expansion in queries and (always) before stop term elimination; used e.g. for case normalization, stemming, phonetic normalization, ...',},
+ {'id' : IgnoreProperty, 'type' : 'string', 'mode' : 'w',
+ 'label':'Stop term predicate: used to recognized and eliminate stop terms; used always (except in range queries) after prenormalization',},
+ {'id' : NormalizerProperty, 'type' : 'string', 'mode' : 'w',
+ 'label':'Term normalizer: applied to terms before type checking; used e.g. for encoding the term into a efficient form',},
+ { 'id' : 'TermType', 'type' : 'selection', 'mode' : 'w', 'select_variable' : 'listTermTypes',
+ 'label':'Term type: used to convert and check the terms type; may allows to choose specially optimized index structures (e.g. for integer types) or provide additional features (e.g. term expansions for string types) -- clear+reindex after change!',
+ },
+ { 'id' : 'TermTypeExtra', 'type' : 'string', 'mode' : 'w',
+ 'label':'Term type argument: required by some term types (see the documentation)',},
+ { 'id' : 'TermCopy', 'type' : 'selection', 'mode' : 'w', 'select_variable' : 'listCopyTypes',
+ 'label':'Control term copying: may be necessary for mutable terms to prevent index corruption',},
+ )
+ )
+ TermType= _TermTypeList[0]
+ TermTypeExtra= ''
+ TermCopy= _TermCopyList[0]
+ CombineType= Combiners[0]
+ NormalizeTerm= ''
+ PrenormalizeTerm= ''
+ StopTermPredicate= ''
+
+ __implements__= PluggableIndexInterface
+
+ def __init__(self, name, extra=None, caller=None):
+ self.id= name
+ def setProperties(obj, values, special):
+ __traceback_info__ = obj
+ allowed = dict.fromkeys(special)
+ allowed.update(dict.fromkeys([p['id'] for p in obj._properties]))
+ for k in values.keys():
+ if k not in allowed: raise ValueError('not a known property: %s' % k)
+ obj.manage_changeProperties(**values)
+ if extra: setProperties(self, extra, ('ValueProviders',))
+ providers = extra and extra.get('ValueProviders')
+ if providers is None: self._createDefaultValueProvider()
+ else:
+ for p in providers:
+ vp = self.addValueProvider(p['id'], p['type'])
+ setProperties(vp, p, ('id', 'type',))
+ self.clear()
+
+ def clear(self):
+ '''clear the index.'''
+ l = self.__len__
+ if isinstance(l, Length): l.set(0)
+ else: self.__len__ = Length()
+ try: self.numObjects.set(0)
+ except AttributeError: self.numObjects= Length()
+ if self.ReverseOrder: self._reverseOrder = OOTreeSet()
+ self._setup()
+
+ def __len__(self):
+ '''Python 2.4 requires this to be defined inside the class.'''
+ l = self.__len__
+ if not isinstance(l, Length): l = self.__len__ = Length()
+ return l()
+
+ def indexSize(self):
+ return self.__len__()
+
+ def _setup(self):
+ self._unindex= IOBTree()
+ treeType = self.TermType in _IntegerTypes and IOBTree or OOBTree
+ self._index= treeType()
+
+ def _createDefaultValueProvider(self):
+ self.addValueProvider(self.id,'AttributeLookup')
+
+ ## term expansion -- provided for indexes with declared "string" and "ustring"
+ ## term types
+ def matchGlob(self, t):
+ '''match '*' (match any sequence) and '?' (match any character) in *t* returning a list of terms.'''
+ # leads to wrong result -- would need to check against index
+ # if not ('*' in t or '?' in t): return [t]
+ regexp = glob2regexp(t)
+ return self.matchRegexp(regexp+'$')
+
+ _matchType = None
+ def matchRegexp(self, regexp):
+ '''match *regexp* into a list of matching terms.
+
+ Note that for efficiency reasons, the regular expression
+ should have an easily recognizable plain text prefix -- at
+ least for large indexes.
+ '''
+ prefix, regexp = _splitPrefixRegexp(regexp)
+ termType = self._matchType or self.TermType
+ if termType == 'string': prefix = str(prefix); regexp = str(regexp)
+ elif termType == 'ustring': prefix = unicode(prefix); regexp = unicode(regexp)
+ elif termType == 'asis': pass
+ else: raise ValueError(
+ "Index %s has 'TermType/MatchType' %s not supporting glob/regexp expansion"
+ % (self.id, termType)
+ )
+ index = self._getMatchIndex(); pn = len(prefix)
+ l = []; match = compile(regexp).match
+ for t in index.keys(prefix): # terms starting prefix
+ if not t.startswith(prefix): break
+ if match(t[pn:]): l.append(t)
+ return l
+
+ def _getMatchIndex(self):
+ '''the index used for expansion'''
+ return self._index
+
+ ## match filtering
+ def matchFilterGlob(self, t):
+ '''see 'matchGlob' but for filtering.'''
+ regexp = glob2regexp(t)
+ return self.matchFilterRegexp(regexp+'$')
+
+ def matchFilterRegexp(self, regexp):
+ '''see 'matchRegexp' but for filtering.'''
+ termType = self._matchType or self.TermType
+ if termType == 'string': regexp = str(regexp)
+ elif termType == 'ustring': regexp = unicode(regexp)
+ elif termType == 'asis': pass
+ else: raise ValueError(
+ "Index %s has 'TermType/MatchType' %s not supporting glob/regexp expansion"
+ % (self.id, termType)
+ )
+ return compile(regexp).match
+
+
+ ## Responsibilities from 'PluggableIndexInterface'
+ # 'getId' -- inherited from 'SimpleItem'
+
+ def getEntryForObject(self,documentId, default= None):
+ '''Information for *documentId*.'''
+ info= self._unindex.get(documentId)
+ if info is None: return default
+ return repr(info)
+
+
+ def index_object(self,documentId,obj,threshold=None):
+ '''index *obj* as *documentId*.
+
+ Commit subtransaction when *threshold* index terms have been indexed.
+ Return the number of index terms indexed.
+ '''
+ # Note: objPath should be provided by the catalog -- but it is not
+ try: objPath = obj.getPhysicalPath()
+ except: objPath = None
+ __traceback_info__ = self.id, objPath
+
+ val= self._evaluate(obj)
+
+ # see whether something changed - do nothing, if it did not
+ oldval= self._unindex.get(documentId)
+ if val == oldval: return 0
+ # some data types, e.g. "OOSet"s do not define a senseful "==".
+ # provide a hook to handle this case
+ customEqual= self._equalValues
+ if customEqual is not None and customEqual(val,oldval): return 0
+
+ # remove state info
+ update= self._update
+ if update is None or oldval is None or val is None:
+ # no optimization
+ if oldval is not None: self._unindex_object(documentId,oldval,val is None)
+ if val is None: return 0
+ rv= self._indexValue(documentId,val,threshold)
+ if oldval is None: self.numObjects.change(1)
+ else:
+ # optimization
+ rv= update(documentId,val,oldval,threshold)
+ if isinstance(rv, tuple): return rv[0]
+
+ # remember indexed value
+ self._unindex[documentId]= val
+ return rv
+
+
+ def unindex_object(self,documentId):
+ '''unindex *documentId*.
+
+ ATT: why do we not have a *threshold*????
+ '''
+ # Note: objPath/documentId should be provided by the catalog -- but it is not
+ __traceback_info__ = self.id, documentId
+
+ val= self._unindex.get(documentId)
+ if val is None: return # not indexed
+ self._unindex_object(documentId,val,1)
+
+ def _unindex_object(self,documentId,val,remove):
+ self._unindexValue(documentId,val)
+ if remove:
+ del self._unindex[documentId]
+ self.numObjects.change(-1)
+
+
+ def uniqueValues(self, name=None, withLengths=0):
+ '''unique values for *name* (???).
+
+ If *withLength*, returns sequence of tuples *(value,length)*.
+ '''
+ if name is None: name= self.id
+ if name != self.id: return ()
+ values= self._index.keys()
+ if not withLengths: return tuple(values)
+ return tuple([(value,self._len(value)) for value in values])
+
+
+ def _apply_index(self,request, cid= ''):
+ '''see 'PluggableIndex'.
+
+ What is *cid* for???
+ '''
+ __traceback_info__ = self.id
+
+ record= parseIndexRequest(request, self.id, self.query_options)
+ terms= record.keys
+ if terms is None: return
+
+ __traceback_info__ = self.id, record.keys
+
+ op= record.get('operator', self.useOperator)
+ if op not in self.operators:
+ raise ValueError("operator not permitted: %s" % op)
+ combine= op == 'or' and union or intersection
+
+ filteredSearch = None
+ if record.get('isearch') and record.get('isearch_filter') \
+ and self.supportFiltering and IFilter is not None:
+ filteredSearch = self._getFilteredISearch(record)
+
+ if filteredSearch is None:
+ match = record.get('match')
+ if match is not None:
+ l = []; match = getattr(self, 'match' + match.capitalize())
+ prenorm = self._prenormalizeTerm
+ for t in terms:
+ t = prenorm(t, None)
+ if t is not None: l.extend(match(t))
+ terms = l
+
+ range= record.get('range')
+ if range is not None:
+ terms= [self._standardizeTerm(t,elimStopTerm=0, prenormalize=not match) for t in terms]
+ range= range.split(':'); lo= hi= None
+ if 'min' in range: lo= min(terms)
+ if 'max' in range: hi= max(terms)
+ terms= self._enumerateRange(lo,hi)
+ else:
+ terms= [self._standardizeTerm(t, prenormalize=not match) for t in terms]
+
+ if filteredSearch is None: r = self._search(terms,combine,record)
+ else: r = filteredSearch
+ if r is None: return
+ return r, self.id
+
+
+ #################################################################
+ # search
+ def _search(self,terms,combine,record):
+ return setOperation(
+ combine is union and 'or' or 'and',
+ [self._searchTerm(t,record) for t in terms],
+ record.get('isearch'),
+ )
+
+ def _searchTerm(self,term,record):
+ return self._load(term)
+
+ def _enumerateRange(self,min,max):
+ '''enumerate terms between *min* and *max*.'''
+ return self._index.keys(min,max)
+
+
+ #################################################################
+ # filtering
+ supportFiltering = False
+
+ def _getFilteredISearch(self, query):
+ '''return a filtered search for *query*, if this seems promissing, or 'None'.
+ '''
+ preds = []
+ enumerator = self._getFilterEnumerator(); makeFilter = self._makeFilter
+
+ terms = query.keys
+ match = query.get('match'); range = query.get('range')
+ op = query.get('operator', self.useOperator)
+
+ if match is not None:
+ # we do not want to filter combined 'match' and 'range' queries
+ if range is not None: return
+ # can only filter 'or' matches
+ if op != 'or': return
+ # we can filter 'match' queries only if there is no 'normalizer'
+ # maybe, we should not filter, if there is an 'ignorer'?
+ if self._hasNormalizer(): return
+ l = []; match = getattr(self, 'matchFilter' + match.capitalize())
+ prenorm = self._prenormalizeTerm
+ for t in terms:
+ t = prenorm(t, None)
+ if t is not None: preds.append(match(t))
+ else:
+ range= query.get('range')
+ if range is not None:
+ # can only filter 'or' ranges
+ if op != 'or': return
+ terms= [self._standardizeTerm(t,elimStopTerm=0, prenormalize=not match) for t in terms]
+ range= range.split(':'); lo= hi= None
+ if 'min' in range: lo= min(terms)
+ if 'max' in range: hi= max(terms)
+ preds.append(_rangeChecker(lo,hi))
+ else:
+ makePred = self._makeTermPredicate; standardize = self._standardizeTerm
+ preds = [
+ makePred(standardize(t, prenormalize=not match))
+ for t in terms
+ ]
+ subsearches = [IFilter(makeFilter(pred), enumerator) for pred in preds]
+
+ return self._combineSubsearches(subsearches, op)
+
+ def _combineSubsearches(self, subsearches, op):
+ if len(subsearches) == 1: return subsearches[0]
+ combine = op == 'or' and IOr or IAnd
+ search = combine(*subsearches); search.complete()
+ return search
+
+ def _getFilterEnumerator(self):
+ return Enumerator(self._unindex)
+
+ def _makeTermPredicate(self, term):
+ '''this is adequate for field and keyword indexes.'''
+ return lambda x, t=term: x == t
+
+ def _makeFilter(self, pred):
+ '''a document filter 'did -> True/False' checking term predicate *pred*.
+
+ This definition is adequate, when the predicate can be directly
+ applied to the 'unindex' value.
+ '''
+ def check(did):
+ dv = self._unindex.get(did)
+ return dv is not None and pred(dv)
+ return check
+
+
+ #################################################################
+ # required for sorting
+ # no longer needed for Zope 2.7 -- keep for compatibility
+ def keyForDocument(self, docId): return self._unindex[docId]
+ def items(self):
+ return [(k,self._load(k)) for k in self._index.keys()]
+
+
+ #################################################################
+ # Reverse ordering support
+ def getReverseOrder(self):
+ '''return the keys in reverse order.'''
+ if self.ReverseOrder:
+ return _LazyMap(lambda x: x.getValue(), self._reverseOrder.keys())
+
+
+ #################################################################
+ # Storage API
+ # we implement a small optimization
+ # a single document is stored as integer; more are stored as an IITreeSet
+ ReverseOrder = 0
+
+ def _insert(self,term,docId, _isInstance= isinstance, _IntType= IntType):
+ '''index *docId* under *term*.'''
+ index= self._index
+ dl= index.get(term)
+ if dl is None:
+ index[term]= docId; self.__len__.change(1)
+ if self.ReverseOrder: self._reverseOrder.insert(reverseOrder(term))
+ return
+ if _isInstance(dl,_IntType): dl= index[term]= IITreeSet((dl,))
+ dl.insert(docId)
+
+ def _remove(self,term,docId, _isInstance= isinstance, _IntType= IntType):
+ '''unindex *docId* under *term*.'''
+ index= self._index
+ dl= index.get(term); isInt= _isInstance(dl,_IntType)
+ if dl is None or isInt and dl != docId:
+ raise ValueError('Attempt to remove nonexisting document %s from %s'
+ % (docId, self.id)
+ )
+ if isInt: dl = None
+ else: dl.remove(docId)
+ if not dl:
+ del index[term]; self.__len__.change(-1)
+ if self.ReverseOrder: self._reverseOrder.remove(reverseOrder(term))
+
+ def _load(self,term, _isInstance= isinstance, _IntType= IntType):
+ '''the docId list for *term*.'''
+ index= self._index
+ dl= index.get(term)
+ if dl is None: return IISet()
+ if _isInstance(dl,_IntType): return IISet((dl,))
+ return dl
+
+ def _len(self,term):
+ '''the number of documents indexed under *term*.'''
+ return len(self._load(term))
+
+
+ ###########################################################################
+ ## methods to maintain auxiliary indexes
+ ## we implement the same optimization as for the main index
+ def _insertAux(self, index, term, docId):
+ '''index *docId* under *term*.'''
+ dl= index.get(term)
+ if dl is None: index[term]= docId; return
+ if isinstance(dl,int): dl= index[term]= IITreeSet((dl,))
+ dl.insert(docId)
+
+ def _removeAux(self, index, term, docId):
+ '''unindex *docId* under *term*.'''
+ dl= index.get(term); isInt= isinstance(dl,int)
+ if dl is None or isInt and dl != docId:
+ raise ValueError('Attempt to remove nonexisting document %s from %s'
+ % (docId, self.id)
+ )
+ if isInt: dl = None
+ else: dl.remove(docId)
+ if not dl: del index[term]
+
+ def _loadAux(self,index, term):
+ '''the docId list for *term*.'''
+ dl= index.get(term)
+ if dl is None: return IISet()
+ if isinstance(dl,int): return IISet((dl,))
+ return dl
+
+
+
+ #################################################################
+ # Term standardization and checking
+ def listTermTypes(self):
+ '''the sequence of supported term types.'''
+ return _TermTypeList
+
+ def listCopyTypes(self):
+ '''the sequence of term copy types.'''
+ return _TermCopyList
+
+ def listCombiners(self):
+ '''the sequence of combine types.'''
+ return self.Combiners
+
+ def _standardizeTerm(self, value, object=None, copy=False, elimStopTerm=True, prenormalize=True):
+ if prenormalize:
+ value = self._prenormalizeTerm(value, object)
+ if value is None: return
+ if elimStopTerm:
+ value= self._ignore(value,object)
+ if value is None: return
+ value= self._normalize(value,object)
+ if value is None: return
+ tt= self.TermType
+ checker= _TermTypeMap[tt]
+ if checker: value= checker(self,value,object)
+ if copy and tt in ('not checked', 'instance', 'expression checked',):
+ copy= _CopyTypeMap[self.TermCopy]
+ if copy: value= copy(value)
+ return value
+
+ _prenormalizer = None
+ def _prenormalizeTerm(self, value, object):
+ PT = self.PrenormalizeTerm
+ if not PT: return value
+ normalizer = self._prenormalizer
+ if normalizer is None:
+ normalizer = self._prenormalizer = Normalize()
+ normalizer.NormalizerProperty = 'PrenormalizeTerm'
+ return normalizer._normalize(value, object)
+
+
+
+ #################################################################
+ # Evaluation
+ def _evaluate(self,object):
+ '''evaluate *object* with respect to this index.'''
+ l= []; v= None
+ combiner= self.CombineType; useFirst= combiner == 'useFirst'
+ for vp in self.objectValues():
+ v= vp.evaluate(object)
+ if v is not None:
+ if useFirst: break
+ l.append(v)
+ if useFirst:
+ if v is None: return
+ return self._standardizeValue(v,object)
+ return getattr(self,'_combine_' + combiner)(l,object)
+
+ def _standardizeValue(self,value,object):
+ return self._standardizeTerm(value,object,1)
+
+
+
+ #################################################################
+ # to be defined by concrete deriving classes
+ # _indexValue(self,documentId,val,threshold)
+ # _unindexValue(self,documentId,val)
+
+
+ #################################################################
+ # optionally defined by concrete deriving classes
+ # _update(self,documentId,val,oldval,threshold)
+ # returns number of entries added; if tuple, _unindex already updated
+ # _equalValues(self,val1,val2) -- true, if standardized values are equal
+ _update= None
+ _equalValues= None
+
+
+ #################################################################
+ # Value provider management
+ def all_meta_types(self):
+ return (
+ { 'name' : AttributeLookup.meta_type,
+ 'action' : 'addAttributeLookupForm',
+ 'permission' : ManageManagableIndexes,
+ },
+ { 'name' : ExpressionEvaluator.meta_type,
+ 'action' : 'addExpressionEvaluatorForm',
+ 'permission' : ManageManagableIndexes,
+ },
+ )
+
+ def addAttributeLookupForm(self):
+ '''addForm for 'AttributeLookup' value provider.'''
+ return addForm.__of__(self)(
+ type= 'AttributeLookup',
+ description= '''An AttributeLookup is a value provider which evaluates an object by looking up an attribute of the object.''',
+ action= 'addValueProvider',
+ )
+
+ def addExpressionEvaluatorForm(self):
+ '''addForm for 'ExpressionEvaluator' value provider.'''
+ return addForm.__of__(self)(
+ type= 'ExpressionEvaluator',
+ description= '''An ExpressionEvaluator is a value provider which evaluates an expression.''',
+ action= 'addValueProvider',
+ )
+
+ def addValueProvider(self,id,type, RESPONSE=None):
+ '''add a value provider with *id* of *type*.'''
+ if type not in ('AttributeLookup', 'ExpressionEvaluator'):
+ raise ValueError('unknown type')
+ cl= _mdict[type]
+ # try to avaid a name conflict
+ eid= id
+ if not id.endswith('_') and hasattr(aq_base(self),id): eid= id + '_'
+ vp= cl(); vp.id= eid
+ if id != eid and type == 'AttributeLookup': vp.Name= id
+ self._setObject(eid, vp)
+ vp= self._getOb(eid)
+ if RESPONSE is None: return vp
+ RESPONSE.redirect('%s/manage_workspace' % vp.absolute_url())
+
+
+InitializeClass(ManagableIndex)
+
+
+#################################################################
+# Term checking and copying
+
+_CopyTypeMap= {
+ 'none' : None,
+ 'shallow' : copy.copy,
+ 'deep' : copy.deepcopy,
+ }
+
+def _isNumeric(value, _NumericType= (IntType, FloatType, LongType,)):
+ try: return isinstance(value,_NumericType)
+ except TypeError: # pre 2.3
+ for t in _NumericType:
+ if isinstance(value,t): return 1
+ return 0
+
+def _isString(value, _StringType= (StringType, UnicodeType,)):
+ try: return isinstance(value,_StringType)
+ except TypeError: # pre 2.3
+ for t in _StringType:
+ if isinstance(value,t): return 1
+ return 0
+
+
+def _checkNumeric(index,value,object):
+ '''return *value*, maybe converted, if it is numeric.'''
+ # see whether is has already the correct type
+ if _isNumeric(value): return value
+ try:
+ if _isString(value):
+ if '.' in value or 'E' in value or 'e' in value: value= float(value)
+ else:
+ try: value= int(value)
+ except ValueError: value= long(value)
+ except:
+ raise TypeError("cannot convert %s to numeric" % str(value))
+ return value
+
+
+def _checkInteger(index,value,object):
+ '''return *value*, maybe converted, if it is integer.'''
+ if hasattr(index.aq_base, 'convertToInteger'):
+ return index.convertToInteger(value, object)
+ return int(value)
+
+
+def _checkString(index,value,object):
+ '''return *value*, maybe converted, if it is a string.'''
+ if isinstance(value,StringType): return value
+ try:
+ nv= str(value)
+ except: nv= None
+ if value is None or nv.startswith('<'):
+ raise TypeError("cannot convert %s to string" % str(value))
+ return nv
+
+
+def _checkUnicode(index,value,object, encode=None):
+ '''return *value*, maybe converted, if it is a unicode string.'''
+ if isinstance(value,UnicodeType): return value
+ try:
+ nv= unicode(value, encode or getdefaultencoding())
+ except:
+ raise TypeError("cannot convert %s to string" % str(value))
+ return nv
+
+def _checkUnicode_encode(index, value, object):
+ # use 'TermTypeExtra' as encoding
+ return _checkUnicode(index, value, object, index.TermTypeExtra)
+
+def _checkUnicode_encode2(index, value, object):
+ # use the value after ';' in 'TermTypeExtra' as encoding
+ encoding = index.TermTypeExtra
+ encoding = ';' in encoding and encoding.split(';',1)[1]
+ return _checkUnicode(index, value, object, encoding)
+
+
+def _checkDateTime(index,value,object):
+ '''return *value* (in sec since epoch), if it is a 'DateTime' instance.'''
+ if isinstance(value, float): return value
+ return convertToDateTime(value)._t # float
+
+def _checkDateTimeInteger(index, value, object):
+ return convertToDateTimeInteger(value)
+
+def _checkDateInteger(index, value, object):
+ return convertToDateInteger(value)
+
+def _checkInstance(index,value,object):
+ '''return *value*, if it is an instance of class 'index.TermTypeExtra'.'''
+ fullClassName= index.TermTypeExtra
+ cl= _findClass(fullClassName)
+ if isinstance(value,cl):
+ if hasattr(cl,'__cmp__'): return value
+ raise TypeError("class %s does not define '__cmp__'" % fullClassName)
+ raise TypeError("cannot convert %s to %s" % (str(value),fullClassName))
+
+
+def _findClass(cl):
+ '''return the class identified by full class name *cl*.'''
+ cs= cl.split('.')
+ mod,cn= '.'.join(cs[:-1]), cs[-1]
+ return getattr(modules[mod],cn)
+
+def _checkTuple(index,value,object):
+ '''return *value*, if it matches the tuple spec in 'index.TermTypeExtra'.'''
+ spec= index.TermTypeExtra.split(';',1)[0]
+ value,pos= _checkTuple_(index,value,spec,0)
+ if pos != len(spec):
+ raise TypeError("%s does not conform to %s" % (str(value),spec))
+ return value
+
+
+def _checkTuple_(index,value,spec,pos):
+ '''return *value*, if it conforms to *spec*.'''
+ if _isString(value):
+ raise TypeError("%s does not conform to %s" % (str(value),spec))
+ try:
+ value= tuple(value)
+ except TypeError:
+ raise TypeError("%s does not conform to %s" % (str(value),spec))
+ i= 0; n= len(value)
+ while i < n:
+ v= value[i]
+ if pos >= len(spec):
+ raise TypeError("%s does not conform to %s" % (str(value),spec))
+ si= spec[pos]; pos+= 1
+ if si == '(':
+ nv,pos= _checkTuple_(index,v,spec,pos)
+ if spec[pos] != ')':
+ raise TypeError("%s does not conform to %s" % (str(value),spec))
+ pos+= 1
+ else:
+ checker= _TupleCheck[si]
+ nv= checker(index,v,None)
+ if v != nv:
+ if isinstance(value,TupleType): value= list(value)
+ value[i]= nv
+ i+= 1
+ return value, pos
+
+def _checkWithExpression(index,value,object):
+ '''return 'index.TermTypeExtra' applied to *value*, if not None.'''
+ evaluator= getattr(index,'_v_checkEvaluator',None)
+ if evaluator is None:
+ evaluator= index._v_checkEvaluator= EvalAndCall('TermTypeExtra')
+ evaluator= evaluator.__of__(index)
+ nv= evaluator._evaluate(value,object)
+ if nv is None:
+ raise TypeError('%s is not accepted by %s' % (str(value),index.TermTypeExtra))
+ return nv
+
+
+_TermTypeMap= {
+ 'not checked' : None,
+ 'numeric' : _checkNumeric,
+ 'string' : _checkString,
+ 'integer' : _checkInteger,
+ 'ustring' : _checkUnicode_encode,
+ 'DateTime' : _checkDateTime,
+ 'DateTimeInteger' : _checkDateTimeInteger,
+ 'DateInteger' : _checkDateInteger,
+ 'tuple' : _checkTuple,
+ 'instance' : _checkInstance,
+ 'expression checked' : _checkWithExpression,
+ }
+
+_TupleCheck= {
+ 'n' : _checkNumeric,
+ 's' : _checkString,
+ 'u' : _checkUnicode_encode2,
+ 'd' : _checkDateTime,
+ }
+
+
+#################################################################
+# constructor support
+addForm= PageTemplateFile('zpt/addForm',_mdict)
+
+def addIndex(self,id,type, REQUEST= None, RESPONSE= None, URL3= None):
+ '''add index of *type* with *id*.'''
+ return self.manage_addIndex(id, type, extra=None,
+ REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)
+
+
+
+#################################################################
+# auxiliaries
+_escs = dict('a\a f\f n\n r\r t\t v\v'.split(' '))
+_escs.update(dict([(c,0) for c in 'AbBdDsSwWZ']))
+
+def _splitPrefixRegexp(regexp,
+ special_=dict(map(None, '.^$*+?{}|[]()', ())),
+ leftOps_=dict(map(None, '*+?{|', ())),
+ escs_=_escs.get,
+ ):
+ '''return pair of plain text prefix and remaining regexp.'''
+ if not regexp or regexp[0] in special_: return '', regexp
+ prefix = ''; i = 0; n = len(regexp)
+ while i < n:
+ c = regexp[i]
+ if c in special_:
+ if c in leftOps_: return prefix[:-1], escape(prefix[-1]) + regexp[i:]
+ return prefix, regexp[i:]
+ elif c == '\\':
+ c = regexp[i+1]
+ # could be optimized but who cares
+ if c.isdigit() or c == 'x': return prefix, regexp[i:]
+ ec = escs_(c, c)
+ if ec == 0: return prefix, regexp[i:]
+ prefix += ec # ATT: quadratic -- but we do not expect it to become huge
+ i += 1
+ else: prefix += c # ATT: quadratic -- see above
+ i += 1
+ return prefix, ''
+
+def glob2regexp(glob):
+ return escape(glob).replace(r'\*','.*', ).replace(r'\?','.')
+
+def _rangeChecker(lo, hi):
+ if lo is None and hi is None: return lambda x: True
+ if lo is None: return lambda x: x <= hi
+ if hi is None: return lambda x: lo <= x
+ return lambda x: lo <= x <= hi
+
+
+
+#################################################################
+# monkey patches
+
+# give ZCatalogIndexes an id such that urls are correct
+from Products.ZCatalog.ZCatalogIndexes import ZCatalogIndexes
+ZCatalogIndexes.id= "Indexes"
+
+
+
+#################################################################
+# setOperation -- using 'IncrementalSearch', if available
+def setOperation(op, sets, isearch):
+ '''perform *op* on *sets*. if *isearch*, return an incremental search.
+
+ *op* may be '"and"' or '"or"'.
+
+ Uses 'IncrementalSearch', if available.
+ '''
+ if not sets:
+ if op == 'and': return # None means all results
+ if isearch: search = IOr(); search.complete(); return search
+ return IISet()
+ # Note: "multiunion" is *much* faster than "IOr"!
+ #if IAnd is not None and (isearch or len(sets) > 1):
+ if IAnd is not None and (isearch or (op == 'and' and len(sets) > 1)):
+ isets = []
+ for set in sets:
+ if set is None:
+ # all results
+ if op == 'and': continue
+ else: return
+ if not isinstance(set, ISearch): set = IBTree(set)
+ isets.append(set)
+ if op == 'and' and not isets: return # empty 'and'
+ cl = op == 'and' and IAnd or IOr
+ if len(isets) == 1:
+ # do not wrap a one element search
+ search = isets[0]
+ else: search = cl(*isets); search.complete()
+ if isearch: return search
+ if hasattr(search, 'asSet'): r = search.asSet()
+ else: r = IISet(); r.__setstate__((tuple(search),))
+ return r
+ if op == 'or' and len(sets) > 5:
+ r = multiunion(sets)
+ else:
+ combine = op == 'and' and intersection or union
+ r= None
+ for set in sets: r= combine(r,set)
+ if r is None:
+ if combine is union: r = IISet()
+ else: return
+ if isearch: r = IBTree(r)
+ return r
+
+
+
+IAnd = IOr = IBTree = IFilter = None
+
+# try IncrementalSearch2 (the C implementation of IncrementalSearch)
+try:
+ from IncrementalSearch2 import \
+ IAnd_int as IAnd, IOr_int as IOr, IBTree, ISearch
+ try: from IncrementalSearch2 import IFilter_int as IFilter, \
+ Enumerator
+ except ImportError: pass
+except ImportError: IAnd = None
+
+# try IncrementalSearch
+if IAnd is None:
+ try:
+ from IncrementalSearch import IAnd, IOr, IBTree, ISearch, \
+ EBTree as Enumerator
+ try:
+ from IncrementalSearch import IFilter
+ except ImportError: pass
+ except ImportError: pass
+
Added: zope-managableindex/branches/upstream/current/PathIndex.py
===================================================================
--- zope-managableindex/branches/upstream/current/PathIndex.py 2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/PathIndex.py 2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,203 @@
+# Copyright (C) 2004 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+# $Id: PathIndex.py,v 1.4 2006/11/14 15:05:59 dieter Exp $
+'''Path Index.'''
+
+from BTrees.IOBTree import IOBTree
+from BTrees.IIBTree import IISet
+
+from ManagableIndex import ManagableIndex, addForm, setOperation, IFilter
+from FieldIndex import FieldIndex
+
+class PathIndex(ManagableIndex):
+ '''a managable 'PathIndex'.'''
+ meta_type= 'Managable PathIndex'
+
+ _properties = ManagableIndex._properties[:4]
+
+ query_options= ('operator', 'level', 'depth', 'isearch', 'isearch_filter')
+
+ def _setup(self):
+ PathIndex.inheritedAttribute('_setup')(self)
+ # create auxiliary index
+ self._lengthIndex = IOBTree()
+ self._depth = 0
+
+ # do we need this?
+ def uniqueValues(self, name=None, withLength=0): raise NotImplementedError
+
+ # we do not support range searches
+ def _enumerateRange(self, lo, hi): raise NotImplementedError
+
+ # we do not support sorting
+ def keyForDocument(self, docId): raise NotImplementedError
+ def items(self): raise NotImplementedError
+
+ # we do not support expanding
+ def _getExpansionIndex(self): raise NotImplementedError
+
+ # term normalization -- we currently do not handle unicode appropriately
+ def _normalize(self, value, object):
+ '''convert into a tuple.'''
+ value = PathIndex.inheritedAttribute('_normalize')(self, value, object)
+ if value is None: return
+ if hasattr(value, 'upper'):
+ value = value.split('/')
+ if not value[-1]: del value[-1]
+ return tuple(value)
+
+ # basic methods
+ def _indexValue(self, docId, val, threshold):
+ for t in enumerate(val): self._insert(t, docId)
+ vn = len(val)
+ self._insertAux(self._lengthIndex, vn, docId)
+ if vn > self._depth: self._depth = vn
+ return 1
+
+ def _unindexValue(self, docId, val):
+ for t in enumerate(val): self._remove(t, docId)
+ self._removeAux(self._lengthIndex, len(val), docId)
+
+ # modified storage -- we implement Zope's PathIndex scheme
+ # At first, it appeared to be superior than our standard scheme.
+ # However, the implementation proved this wrong: it is slightly
+ # inferior (with respect to number of loads, load size and load time).
+ # Nevertheless, we keep it as it will make the matching implementation
+ # easier (should someone wants it).
+ # Our length reflects the number of different segs, not the number
+ # of total index entries
+ def _insert(self, (pos, seg), docId):
+ index = self._index
+ si = index.get(seg)
+ if si is None: index[seg] = si = IOBTree(); self.__len__.change(1)
+ self._insertAux(si, pos, docId)
+
+ def _remove(self, (pos, seg), docId):
+ index = self._index
+ si = index[seg]
+ self._removeAux(si, pos, docId)
+ if not si: del index[seg]; self.__len__.change(1)
+
+ def _load(self, (pos, seg)):
+ index = self._index
+ si = index.get(seg)
+ if si is None: return IISet()
+ return self._loadAux(si, pos)
+
+
+ # search implementation
+ def _searchTerm(self, path, record):
+ level = record.get('level', 0)
+ depth = record.get('depth')
+ isearch = record.get('isearch')
+ if not path: return self._searchLength(level, depth, isearch)
+ if level is not None and level >= 0:
+ return self._searchAt(path, level, depth, isearch)
+ try: limit = self._depth + 1
+ except ValueError: limit = 0
+ limit -= len(path)
+ if level is not None: limit = min(limit, -level+1)
+ return setOperation(
+ 'or',
+ [self._searchAt(path, l, depth, isearch) for l in range(limit)],
+ isearch,
+ )
+
+ def _searchAt(self, path, pos, depth, isearch):
+ '''search for *path* at *pos* restricted by *depth*.'''
+ load = self._load
+ sets = [load((i+pos, seg)) for (i,seg) in enumerate(path)]
+ if depth is not None:
+ sets.append(self._searchDepth(depth, pos + len(path), isearch))
+ return setOperation('and', sets, isearch)
+
+ def _searchDepth(self, depth, len, isearch):
+ li = self._lengthIndex; load = self._loadAux
+ if depth >= 0: return load(li, len+depth)
+ return setOperation('or',
+ [load(li, d) for d in range(len, len-depth+1)],
+ isearch,
+ )
+
+ def _searchLength(self, level, depth, isearch):
+ try: limit = self._depth + 1
+ except: limit = 0
+ if level is None: level = -limit
+ if depth is None: depth = -limit
+ lo = hi = 0
+ if level >= 0: lo += level; hi += level
+ else: hi += -level
+ if depth >= 0: lo += depth; hi += depth
+ else: hi += -depth
+ li = self._lengthIndex; load = self._loadAux
+ if lo == hi: return load(li, lo)
+ return setOperation('or',
+ [load(li,l) for l in range(lo, min(hi+1,limit))],
+ isearch,
+ )
+
+ # filtering support
+ supportFiltering = True
+
+ def _getFilteredISearch(self, query):
+ terms = query.keys
+ level = query.get('level', 0)
+ depth = query.get('depth')
+ try: limit = self._depth + 1
+ except: limit = 0
+ if level is None: level = -limit
+ if depth is None: depth = -limit
+ op = query.get('operator', self.useOperator)
+ if depth >= 0:
+ if level >= 0:
+ def predFactory(t):
+ tn = len(t)
+ rn = level + depth + tn
+ return lambda v: len(v) == rn and v[level:level+tn] == t
+ else: # level < 0
+ def predFactory(t):
+ tn = len(t)
+ rn = depth + tn
+ def pred(v):
+ vn = len(v)
+ return vn >= rn and rn - vn <= -level and v[-rn:-rn+tn] == t
+ return pred
+ else: # depth < 0
+ if level >= 0:
+ def predFactory(t):
+ tn = len(t)
+ rn = level + tn; mn = rn - depth
+ def pred(v):
+ vn = len(v)
+ return rn <= vn <= mn and v[level:level+tn] == t
+ return pred
+ else: # level < 0
+ def predFactory(t):
+ tn = len(t)
+ def pred(v):
+ vn = len(v)
+ for l in range(max(0, vn-tn+depth), min(-level,vn-tn)+1):
+ if v[l:l+tn] == t: return True
+ return False
+ return pred
+ subsearches = []
+ makeFilter = self._makeFilter; enumerator = self._getFilterEnumerator()
+ for t in terms:
+ t = self._standardizeTerm(t)
+ subsearches.append(IFilter(makeFilter(predFactory(t)), enumerator))
+ return self._combineSubsearches(subsearches, op)
+
+
+
+
+
+def addPathIndexForm(self):
+ '''add PathIndex form.'''
+ return addForm.__of__(self)(
+ type= PathIndex.meta_type,
+ description= '''A PathIndex indexes an object under a path (a tuple or '/' separated string).
+ It can be queried for objects the path of which contains a given path with
+ various possibilities to contrain where the given path must
+ lie within the objects path.''',
+ action= 'addIndex',
+ )
Added: zope-managableindex/branches/upstream/current/RangeIndex.py
===================================================================
--- zope-managableindex/branches/upstream/current/RangeIndex.py 2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/RangeIndex.py 2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,247 @@
+# Copyright (C) 2004 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+# $Id: RangeIndex.py,v 1.7 2006/04/09 16:52:03 dieter Exp $
+'''An efficient index for ranges.'''
+
+from BTrees.IIBTree import IISet, multiunion, IITreeSet
+
+from ManagableIndex import ManagableIndex, addForm, setOperation
+from Evaluation import Eval
+from fixPluginIndexes import parseIndexRequest
+
+
+class RangeIndex(ManagableIndex):
+ '''A 'RangeIndex' has exactly 2 'ValueProviders': for the lower
+ and upper bound, respectively.
+
+ For a term *t*, it can efficiently return the documents *d*
+ with: *lowerbound(d) <= t <= upperbound(d)*.
+ '''
+
+ meta_type = 'Managable RangeIndex'
+
+ query_options= ('operator', 'isearch', 'isearch_filter')
+
+ _properties = (
+ ManagableIndex._properties
+ + (
+ {'id':'IgnoreIncompleteRange',
+ 'label':'Ignore incomplete ranges (otherwise, we raise a ValueError)',
+ 'type':'boolean', 'mode':'rw',},
+ {'id':'BoundaryNames',
+ 'label':'Boundary names: "low" "high" pair specifying that this index should handle "low <= value <= high" queries"',
+ 'type':'tokens', 'mode':'rw',},
+ {'id':'MinimalValue',
+ 'label':'Minimal value: values at or below this value are identified -- clear+reindex after change!',
+ 'type':'string', 'mode':'rw',},
+ {'id':'MaximalValue',
+ 'label':'Maximal value: values at or above this value are identified -- clear+reindex after change!',
+ 'type':'string', 'mode':'rw',},
+ {'id':'OrganisationHighThenLow',
+ 'label':"Organisation 'high-then-low': check when 'x <= high' is less likely than 'low <= x' -- clear+reindex after change",
+ 'type':'boolean', 'mode':'rw', },
+ )
+ )
+ IgnoreIncompleteRange = 1
+ BoundaryNames = ()
+ MinimalValue = MaximalValue = ''
+ OrganisationHighThenLow = False
+
+ Combiners = ManagableIndex.Combiners + ('aggregate',)
+
+ def __init__(self, name):
+ self._minEvaluator = Eval('MinimalValue')
+ self._maxEvaluator = Eval('MaximalValue')
+ RangeIndex.inheritedAttribute('__init__')(self, name)
+
+ _minValue = _maxValue = None # backward compatibility
+ def _setup(self):
+ RangeIndex.inheritedAttribute('_setup')(self)
+ self._minValue = self._maxValue = None
+ self._unrestriced = self._upperOnly = self._lowerOnly = None
+ if hasattr(self, 'aq_base'): # wrapped
+ eval = self._minEvaluator
+ if eval._getExpressionString(): min = eval._evaluate(None, None)
+ else: min = None
+ if min is not None: min = self._standardizeTerm(min, None, 1, 0)
+ self._minValue = min
+ eval = self._maxEvaluator
+ if eval._getExpressionString(): max = eval._evaluate(None, None)
+ else: max = None
+ if max is not None: max = self._standardizeTerm(max, None, 1, 0)
+ self._maxValue = max
+ it = self._index.__class__
+ if min is not None and max is not None:
+ if min > max:
+ raise ValueError('Rangeindex %s: minimal exceeds maximal value: %s %s'
+ % (self.id, str(min), str(max))
+ )
+ self._unrestricted = it()
+ if min is not None: self._upperOnly = it()
+ if max is not None: self._lowerOnly = it()
+
+ def _standardizeValue(self, value, object):
+ if not value: return
+ value = [self._standardizeTerm(v, object, 1) for v in value]
+ value = [v for v in value if v is not None]
+ if not value: return
+ if len(value) != 2:
+ if self.IgnoreIncompleteRange: return
+ raise ValueError('RangeIndex %s: wrong value %s for object %s'
+ % (self.id, str(value), object.absolute_url(1),),
+ )
+ if value[0] > value[1]: return # empty range
+ return value
+
+ _combine_aggregate = _standardizeValue
+
+ # do we need this?
+ def uniqueValues(self, name=None, withLength=0): raise NotImplementedError
+
+ # we do not support range searches
+ def _enumerateRange(self, lo, hi): raise NotImplementedError
+
+ # we do not support sorting
+ def keyForDocument(self, docId): raise NotImplementedError
+ def items(self): raise NotImplementedError
+
+ # we do not support expanding
+ def _getExpansionIndex(self): raise NotImplementedError
+
+
+ # basic specialized methods
+ def _indexValue(self,documentId,val,threshold):
+ self._insert(val,documentId)
+ return 1
+
+ def _unindexValue(self,documentId,val):
+ self._remove(val,documentId)
+
+
+ # apply -- for (partial) plugin replacement for effective/expires
+ def _apply_index(self,request, cid= ''):
+ r = RangeIndex.inheritedAttribute('_apply_index')(self,request, cid)
+ if r is not None or not self.BoundaryNames: return r
+ bn = self.BoundaryNames
+ if len(bn) != 2:
+ raise ValueError('RangeIndex %s: "BoundaryNames" is not a pair'
+ % self.id
+ )
+ ln, hn = bn
+ lq = parseIndexRequest(request, ln, ManagableIndex.query_options)
+ hq = parseIndexRequest(request, hn, ManagableIndex.query_options)
+ if lq.keys != hq.keys \
+ or lq.get('range') != 'min' \
+ or hq.get('range') != 'max' \
+ or lq.get('operator') \
+ or hq.get('operator') \
+ : return
+ return RangeIndex.inheritedAttribute('_apply_index')(
+ self,
+ {self.id:{'query':lq.keys}},
+ cid
+ )
+
+
+ # overridden _searchTerm to pass "isearch" into "_load"
+ # ATT: the base class should do that but it would break compatibility
+ def _searchTerm(self,term,record):
+ return self._load(term, record.get('isearch'))
+
+
+ # adapted storage
+ def _findDocList(self, term, create):
+ '''return for *term* a docList access path consisting
+ of (index, key) pairs or 'None'.
+
+ If *create*, create missing intermediates.
+ '''
+ lv, hv = term
+ min = self._minValue; max = self._maxValue
+ if min is not None and lv <= min:
+ # no lower restriction
+ if max is not None and hv >= max:
+ # no upper restriction, i.e. unrestricted
+ return ((self._unrestricted,min),)
+ else: return ((self._upperOnly, hv),)
+ elif max is not None and hv >= max:
+ # no upper restriction
+ return ((self._lowerOnly, lv),)
+ elif self.OrganisationHighThenLow:
+ idx = self._index; hi = idx.get(hv)
+ if hi is None:
+ if not create: return
+ hi = idx[hv] = idx.__class__()
+ return ((idx, hv), (hi, lv),)
+ else:
+ idx = self._index; li = idx.get(lv)
+ if li is None:
+ if not create: return
+ li = idx[lv] = idx.__class__()
+ return ((idx, lv), (li, hv),)
+
+ def _insert(self, term, docId, _isInstance= isinstance, _IntType= int):
+ '''index *docId* under *term*.'''
+ i,k = self._findDocList(term, 1)[-1]
+ dl = i.get(k)
+ if dl is None: i[k] = docId; self.__len__.change(1); return
+ if _isInstance(dl,_IntType): dl = i[k]= IITreeSet((dl,))
+ dl.insert(docId)
+
+
+ def _remove(self, term, docId, _isInstance= isinstance, _IntType= int):
+ '''unindex *docId* under *term*.'''
+ access = self._findDocList(term, 0)
+ if access is not None: i,k = access[-1]; dl = i.get(k)
+ else: dl = None
+ isInt = _isInstance(dl,_IntType)
+ if dl is None or isInt and dl != docId:
+ raise ValueError('Attempt to remove nonexisting document %s from %s'
+ % (docId, self.id)
+ )
+ if isInt: dl = None
+ else: dl.remove(docId)
+ if not dl:
+ del i[k]; self.__len__.change(-1)
+ if not i and len(access)==2: del access[0][0][access[0][1]]
+
+ def _mkSet(self, dl, _isInstance= isinstance, _IntType= int):
+ if dl is None: return IISet()
+ if _isInstance(dl, _IntType): dl = IISet((dl,))
+ return dl
+
+ def _load(self, term, isearch=False):
+ '''return IISet for documents matching *term*.'''
+ sets = []; mkSet = self._mkSet
+ if self.OrganisationHighThenLow:
+ for hi in self._index.values(term):
+ for dl in hi.values(None, term): sets.append(mkSet(dl))
+ else:
+ for li in self._index.values(None, term):
+ for dl in li.values(term): sets.append(mkSet(dl))
+ min = self._minValue; max = self._maxValue
+ if min is not None:
+ for dl in self._upperOnly.values(term): sets.append(mkSet(dl))
+ if max is not None:
+ for dl in self._lowerOnly.values(None, term): sets.append(mkSet(dl))
+ if min is not None: sets.append(mkSet(self._unrestricted.get(min)))
+ return setOperation('or', sets, isearch)
+
+ # filtering support
+ supportFiltering = True
+
+ def _makeTermPredicate(self, term):
+ min = self._minValue; max = self._maxValue
+ if min is not None and term < min: term = min
+ if max is not None and term > max: term = max
+ return lambda (low,high), t=term: low <= t <= high
+
+
+
+def addRangeIndexForm(self):
+ '''add RangeIndex form.'''
+ return addForm.__of__(self)(
+ type= RangeIndex.meta_type,
+ description= '''A RangeIndex indexes an object under a range of terms.''',
+ action= 'addIndex',
+ )
Added: zope-managableindex/branches/upstream/current/Utils.py
===================================================================
--- zope-managableindex/branches/upstream/current/Utils.py 2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/Utils.py 2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,94 @@
+# Copyright (C) 2003 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+# $Id: Utils.py,v 1.1 2004/05/22 17:40:18 dieter Exp $
+'''Utilities.'''
+
+from sys import maxint
+from DateTime import DateTime
+
+
+##############################################################################
+## order support
+class _OrderReverse:
+ '''auxiliary class to provide order reversal.'''
+
+ def __init__(self, value):
+ self._OrderReverse_value = value
+
+ getValue__roles__ = None # public
+ def getValue(self): return self._OrderReverse_value
+
+ def __cmp__(self,other):
+ return - cmp(self._OrderReverse_value, other._OrderReverse_value)
+
+## def __getattr__(self, attr):
+## return getattr(_OrderReverse_value, attr)
+
+_mdict = globals(); _cldict = _OrderReverse.__dict__
+
+for _f,_op in [spec.split(':') for spec in
+ 'lt:> le:>= eq:== ne:!= gt:< ge:<='.split()]:
+ exec('def %s(self, other): return self._OrderReverse_value %s other._OrderReverse_value\n\n' %
+ (_f, _op),
+ _mdict,
+ _cldict,
+ )
+
+def reverseOrder(value):
+ if isinstance(value, _OrderReverse): return value._OrderReverse_value
+ return _OrderReverse(value)
+
+
+##############################################################################
+## Lazy
+## this is private for the moment as otherwise, we would need to
+## respect security aspects
+class _LazyMap:
+ '''an object applying a function lazyly.'''
+ def __init__(self, f, seq):
+ self._f = f
+ self._seq = seq
+
+ def __getitem__(self, i): return self._f(self._seq[i])
+
+
+
+##############################################################################
+## DateTime conversions
+
+def convertToDateTime(value):
+ '''convert *value* to a 'DateTime' object.'''
+ if isinstance(value, DateTime): return value
+ if isinstance(value, tuple): return DateTime(*value)
+ return DateTime(value)
+
+def convertToDateTimeInteger(value, exc=0):
+ '''convert *value* into a DateTime integer (representing secs since
+ epoch).
+
+ *exc* controls whether an exception should be raised when the
+ value cannot be represented in the integer range. If *exc* is
+ false, values are truncated, if necessary.
+ '''
+ if isinstance(value, int): return value
+ value = round(convertToDateTime(value)._t) # seconds since epoch
+ ma = maxint; mi = -ma - 1
+ if exc and value < mi or value > ma:
+ raise TypeError('not in integer range: %s' % value)
+ if value < mi: value = mi
+ elif value > ma: value = ma
+ return int(value)
+
+def convertToDateInteger(value, round_dir=-1):
+ '''convert *value* into a Date integer (400*y + 31*(m-1) + d-1).
+
+ *round_dir* controls rounding: '0' means 'round', '1' means 'ceil'
+ and '-1' 'floor')
+ '''
+ if isinstance(value, int): return value
+ adjust = (0, 0.5, (0.999999))[round_dir]
+ dt = convertToDateTime(value)
+ if adjust: dt += adjust
+ y,m,d = dt._year, dt._month, dt._day
+ return 400*y +(m-1)*31 +d-1
+
Added: zope-managableindex/branches/upstream/current/VERSION.txt
===================================================================
--- zope-managableindex/branches/upstream/current/VERSION.txt 2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/VERSION.txt 2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,2 @@
+1.6.1 for Zope 2.7+, Python 2.3+/Python 2.4 // Zope 2.8
+requires "OFolder", see <http://www.dieter.handskake.de/pyprojects/zope>
Added: zope-managableindex/branches/upstream/current/ValueProvider.py
===================================================================
--- zope-managableindex/branches/upstream/current/ValueProvider.py 2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/ValueProvider.py 2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,170 @@
+# Copyright (C) 2003 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+# $Id: ValueProvider.py,v 1.5 2004/07/30 13:38:02 dieter Exp $
+'''ValueProvider.
+
+A 'ValueProvider' evaluates an object and returns a value.
+A 'None' value is interpreted as if the object does not have
+a value.
+
+A 'ValueProvider' can ignore more values than 'None'.
+It can specify a normalization for values which have not been ignored.
+
+A 'ValueProvider' can specify how exceptions during evaluation
+and postprocessing should be handled. They can be ignored
+(this means, the object does not have a value) or propagated.
+
+'ValueProviders' are 'SimpleItems' and provide comfiguration
+via Properties, i.e. they are 'PropertyManagers'.
+'''
+
+from Globals import InitializeClass
+from AccessControl import ClassSecurityInfo
+from ComputedAttribute import ComputedAttribute
+from Acquisition import Acquired
+
+from OFS.SimpleItem import SimpleItem
+from OFS.PropertyManager import PropertyManager
+from DocumentTemplate.DT_Util import safe_callable
+from ZODB.POSException import ConflictError
+
+from Products.PageTemplates.TALES import CompilerError
+
+from Evaluation import EvalAndCall, Ignore, Normalize
+
+
+class ValueProvider(SimpleItem,PropertyManager,Ignore,Normalize):
+ '''configuration for object evaluation.'''
+
+ manage_options= (
+ PropertyManager.manage_options
+ + SimpleItem.manage_options
+ )
+
+ _properties= (
+ (
+ { 'id' : 'IgnoreExceptions', 'type' : 'boolean', 'mode' : 'w',},
+ )
+ + Ignore._properties
+ + Normalize._properties
+ )
+ IgnoreExceptions= 1
+
+ security= ClassSecurityInfo()
+ security.declarePrivate('evaluate')
+
+
+ def evaluate(self,object):
+ '''evaluate *object* to a value.
+
+ 'None' means, the object does not have a value.
+ '''
+ __traceback_info__ = self.id
+ try:
+ value= self._evaluate(object)
+ value= self._ignore(value,object)
+ value= self._normalize(value,object)
+ except (CompilerError, ConflictError): raise
+ except:
+ if self.IgnoreExceptions: return
+ raise
+ return value
+
+
+ def __str__(self):
+ return '; '.join(['%s : %s' % pi for pi in self.propertyItems()])
+
+ title= ComputedAttribute(__str__)
+
+
+ # abstract methods
+ def _evaluate(self,object):
+ raise NotImplementedError("'_ValueProvider._evaluate' is abstract")
+
+InitializeClass(ValueProvider)
+
+
+class AttributeLookup(ValueProvider):
+ '''Configures attribute/method lookup.
+
+ Configuration properties:
+
+ 'Name' -- the attribute/method which determines the value
+
+ 'AcquisitionType' -- controls how acquisition is used
+
+ * 'implicit' -- use implicit acquisition
+
+ * 'explicit' -- equivalent to 'implicit' unless the
+ looked up value is a method and is called.
+ In this case, the passed in object is an explicit
+ acquisition wrapper.
+
+ * 'none' -- do not use acquisition
+
+ 'CallType' -- controls how a callable result should be handled
+
+ * 'call' -- call it and return the result
+
+ * 'ignore' -- ignore value
+
+ * 'return' -- return the value
+ '''
+
+ meta_type= 'Attribute Lookup'
+
+ _properties= (
+ (
+ { 'id' : 'Name', 'type' : 'string', 'mode' : 'w',},
+ { 'id' : 'AcquisitionType', 'type' : 'selection', 'select_variable' : 'listAcquisitionTypes', 'mode' : 'w',},
+ { 'id' : 'CallType', 'type' : 'selection', 'select_variable' : 'listCallTypes', 'mode' : 'w',},
+ )
+ + ValueProvider._properties
+ )
+
+ Name= ''
+ AcquisitionType= 'implicit'
+ CallType= 'call'
+
+ def _evaluate(self,object):
+ aqType= self.AcquisitionType; orgObj = object
+ name= self.Name or self.id
+ # determine value
+ if not hasattr(object,'aq_acquire'): aqType= 'implicit' # not wrapped
+ elif hasattr(object.aq_base, name+'__index_aqType__'):
+ aqType = getattr(object, name+'__index_aqType__')
+ if aqType == 'explicit':
+ # work around bug in "aq_acquire" (has 'default' argument but ignores it)
+ try: value= object.aq_explicit.aq_acquire(name,default=None)
+ except AttributeError: value= None
+ else:
+ if aqType == 'none': object= object.aq_base
+ value= getattr(object,name,None)
+ # allow acquisition for objets that explicitly called for it
+ if value is Acquired: value = getattr(orgObj, name, None)
+ # handle calls
+ if safe_callable(value):
+ callType= self.CallType
+ if callType == 'call': value= value()
+ elif callType == 'ignore': value= None
+ return value
+
+ def listAcquisitionTypes(self):
+ return ('implicit', 'none', 'explicit',)
+
+ def listCallTypes(self):
+ return ('call', 'return', 'ignore',)
+
+
+class ExpressionEvaluator(ValueProvider,EvalAndCall):
+ '''configures TALES evaluation.'''
+
+ meta_type= 'Expression Evaluator'
+
+ _properties= (
+ EvalAndCall._properties
+ + ValueProvider._properties
+ )
+
+ def _evaluate(self,object):
+ return EvalAndCall._evaluate(self,object,object)
Added: zope-managableindex/branches/upstream/current/WordIndex.py
===================================================================
--- zope-managableindex/branches/upstream/current/WordIndex.py 2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/WordIndex.py 2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,161 @@
+# Copyright (C) 2004 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+# $Id: KeywordIndex.py,v 1.4 2004/07/30 11:12:09 dieter Exp $
+'''Managable WordIndex.
+
+A word index lies between a 'KeywordIndex' and a 'TextIndex'.
+
+Like a 'TextIndex', it uses a 'Lexicon' to split values into
+a sequence of words, stores integer word ids in its index
+and does not support range queries.
+Like a 'KeywordIndex', it indexes an object under a sequence of words --
+no near or phrase queries and no relevancy ranking.
+Due to these restrictions, a 'WordIndex' is very efficient with respect
+to transaction and load size.
+
+The motivation to implement a
+'WordIndex' came from my observation, that almost all our ZEO loads
+above 100ms were caused by loading large word frequency
+IOBuckets used by 'TextIndex'es for relevancy ranking -- a feature
+we do not need and use. Many of these loads transfered buckets
+of several 10 kB and a considerable portion of them took more
+than a second. As word frequency information is necessary for
+each document in a hit, you can imagine how fast our queries
+were.
+
+On the other hand, a 'WordIndex' is only useful when you
+use a flexible query framework, such as e.g. 'AdvancedQuery'
+or 'CatalogQuery'.
+The standard 'ZCatalog' framework is too weak as it does not
+allow to have several subqueries against the same index
+in a query. Thats the reason why 'TextIndex'es come with
+their own query parser. I live in Germany and therefore
+the standard query parser is useless for us (we use 'und', 'oder',
+'nicht' instead of 'and', 'or' and 'not') and I have 'AdvancedQuery' --
+thus I did not care to give the new 'WordIndex' a query parser.
+You could easily provide one -- should you feel a need for it.
+'''
+
+from Acquisition import aq_base
+
+from BTrees.IIBTree import IITreeSet, difference
+
+from ManagableIndex import addForm, ManagableIndex
+from KeywordIndex import KeywordIndex
+
+class WordIndex(KeywordIndex):
+ '''a managable 'WordIndex'.'''
+ meta_type = 'Managable WordIndex'
+
+ query_options = ('operator', 'match', 'isearch')
+ TermType = 'integer'
+
+ _properties = ManagableIndex._properties[:1] + (
+ {'id':'PrenormalizeTerm', 'type':'string', 'mode':'w',
+ 'label':'Match normalizer: applied to match patterns -- this should match the lexicons normalization',},
+ {'id':'Lexicon', 'type':'string', 'mode':'w',
+ 'label':'Lexicon id of a ZCTextIndex like lexicon (resolved with respect to the catalog) -- clear+reindex after change',},
+ )
+ Lexicon = ''
+
+ _createDefaultValueProvider = ManagableIndex._createDefaultValueProvider
+
+ # override for better readability
+ def getEntryForObject(self,documentId, default= None):
+ '''Information for *documentId*.'''
+ info= self._unindex.get(documentId)
+ if info is None: return default
+ lexicon = self._getLexicon()
+ l = [lexicon.get_word(wid) for wid in info.keys()]; l.sort()
+ return repr(l)
+
+ # overrides: we could use the corresponding 'KeywordIndex' definitions
+ # but as the word sets tend to be much larger,
+ # we use definitions which are more efficient for large sets
+ def _update(self,documentId,val,oldval,threshold):
+ add= difference(val,oldval)
+ rem= difference(oldval,val)
+ if add: self._indexValue(documentId,add,threshold)
+ if rem: self._unindexValue(documentId,rem)
+ # optimize transaction size by not writing _unindex bucket
+ if len(rem) < 100:
+ for x in rem: oldval.remove(x) # sad that we do not have a mass remove
+ oldval.update(add)
+ else: oldval.clear(); oldval.update(val)
+ return len(add),
+
+## the KeywordIndex implementation is more efficient
+## def _equalValues(self, val1, val2):
+## if val1 == val2: return True
+## if val1 is None or val2 is None: return False
+## it1 = val1.keys(); it2 = val2.keys(); i = 0
+## while True:
+## k1 = k2 = self
+## try: k1 = it1[i]
+## except IndexError: pass
+## try: k2 = it2[i]
+## except IndexError: pass
+## if k1 != k2: return False
+## if k2 is self: return True
+## i+= 1
+
+ def _combine_union(self, values, object):
+ if not values: return
+ set= None
+ for v in values:
+ sv= self._standardizeValue_(v, object)
+ if not sv: continue
+ if set is None: set = IITreeSet(sv)
+ else: set.update(sv)
+ return set
+
+ def _standardizeValue(self, value, object):
+ '''convert to an IITreeSet of standardized terms.'''
+ terms = self._standardizeValue_(self, value, object)
+ if terms: return IITreeSet(terms)
+
+ ## essential work delegated to lexicon
+ def _standardizeValue_(self, value, object):
+ '''convert to a sequence of standardized terms.'''
+ if not value: return
+ lexicon = self._getLexicon()
+ # we assume some type of "Products.ZCTextIndex.Lexicon.Lexicon" instance
+ return lexicon.sourceToWordIds(value)
+
+ def _normalize(self, value, object):
+ '''convert term *value* to word id.'''
+ if isinstance(value, int): return value # already normalized
+ # ATT: returns 0, when the word is unknown
+ # This implies that searches for unknown words cannot get matches
+ wids = self._getLexicon().termToWordIds(value)
+ if len(wids) != 1:
+ raise ValueError('Word index %s can only handle word queries: %s'
+ % (self.id, value)
+ )
+ return wids[0]
+
+ def _getLexicon(self):
+ # resolve with respect to catalog -- this is nasty but necessary
+ # to avoid acquisition from the internal "_catalog".
+ obj = self
+ while not hasattr(aq_base(obj), 'Indexes'): obj = obj.aq_inner.aq_parent
+ lexicon = getattr(obj, self.Lexicon, None)
+ if lexicon is None:
+ raise ValueError('Lexicon not found: ' + self.Lexicon)
+ return lexicon
+
+ _matchType = 'asis'
+ def _getMatchIndex(self):
+ # ATT: internal knowledge about ZCTextIndex.Lexicon.Lexicon!
+ return self._getLexicon()._wids
+
+
+
+def addWordIndexForm(self):
+ '''add KeywordIndex form.'''
+ return addForm.__of__(self)(
+ type= WordIndex.meta_type,
+ description= '''A WordIndex indexes an object under a set of word ids determined via a 'ZCTextIndex' like lexicon.''',
+ action= 'addIndex',
+ )
+
Added: zope-managableindex/branches/upstream/current/__init__.py
===================================================================
--- zope-managableindex/branches/upstream/current/__init__.py 2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/__init__.py 2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,38 @@
+# Copyright (C) 2003 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+# $Id: __init__.py,v 1.5 2006/05/17 19:53:07 dieter Exp $
+'''Managable Indexes
+
+see 'ManagableIndexes.html' for documentation.
+'''
+
+from FieldIndex import FieldIndex, addFieldIndexForm
+from KeywordIndex import KeywordIndex, addKeywordIndexForm, \
+ KeywordIndex_scalable, addKeywordIndex_scalableForm
+from RangeIndex import RangeIndex, addRangeIndexForm
+from WordIndex import WordIndex, addWordIndexForm
+from PathIndex import PathIndex, addPathIndexForm
+from ManagableIndex import addIndex, ManageManagableIndexes
+from AccessControl import allow_module
+
+_indexes= (
+ 'FieldIndex',
+ 'KeywordIndex',
+ 'KeywordIndex_scalable',
+ 'RangeIndex',
+ 'WordIndex',
+ 'PathIndex',
+ )
+
+_mdict= globals()
+
+def initialize(context):
+ for idx in _indexes:
+ context.registerClass(
+ _mdict[idx],
+ permission= ManageManagableIndexes,
+ constructors= (_mdict['add%sForm' % idx], addIndex,),
+ visibility= None,
+ )
+
+allow_module('Products.ManagableIndex.Utils')
Added: zope-managableindex/branches/upstream/current/doc/ManagableIndex.html
===================================================================
--- zope-managableindex/branches/upstream/current/doc/ManagableIndex.html 2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/doc/ManagableIndex.html 2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,448 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2//EN">
+<html>
+ <head>
+ <title>ManagableIndex</title>
+ <style type="text/css">
+ .warning {
+ color: red;
+ font-weight: bold;
+ }
+ </style>
+ </head>
+
+ <body>
+ <h1>ManagableIndex</h1>
+
+<h2>Introduction</h2>
+<p><code>ManagableIndex</code> can mean different things for different
+people. For a content manager, it brings flexible field, keyword
+and efficient range indexes managable via the ZMI to</p>
+<ul>
+<li>combine values from different sources, ignoring irrelevant values
+ and transforming values in an inadequate form,<br>
+ Example: index an object via title if it is non-empty but use
+ id otherwise<br>
+ Example: add <code>portal_type</code> to the <code>Subject</code> index
+ transforming it into a sequence (as required by a keyword index).
+<li>control acquisition<br>
+ and prevent acquired values to be indexed
+<li>ignore stop-terms, normalize terms and check types<br>
+ Example: when you do not want that an object is indexed under
+ a meaningless default value, you can define this value
+ as a stop-term.<br>
+ Example: when you want case insensitive searches, you
+ can perform case folding during normalization.<br>
+ Example: the BTrees underlying Zope indexes require its
+ keys to have a persistently consistent ordering.
+ When this is not the case, they become corrupt.
+ While modern Python versions (at least Python 2.3)
+ compare values from different elementary types
+ persistently consistent, the result can be quite
+ unintuitive. Therefore, it is best when all terms
+ in an index belong to the same types. Normalizing
+ and type checking helps to ensure this.
+</ul>
+
+<p>For a developer, <code>ManagableIndex</code> provides
+a framework for index definition, improving on <code>PluginIndexes</code>.
+It provides for managability,
+automatically and intelligently handles unindexing when an object
+is reindexed and implements <em>and</em>, <em>or</em> and
+<em>range</em> queries (for not too complex indexes).</p>
+
+<h2>Concepts</h2>
+<p>The main tasks of an index are to index objects under
+a set of index terms derived from the object and efficiently locate
+all objects indexed under a given term.</p>
+
+<p>Indexing consists of 3 stages: evaluation of the object
+to obtain the object's value (with respect to the index),
+deriving index terms from the value and storing the association
+term/object in a way such that
+the objects associated with a term can quickly be retrieved.</p>
+
+
+<h3>ValueProvider</h3>
+<p>Evaluation is specified by a sequence of <em>ValueProvider</em>s
+associated with the index.
+A ValueProvider is a device that returns a value for an object.
+If the return value is not <code>None</code>,
+then it is interpreted as the
+object's value with respect to this ValueProvider.</p>
+
+<p>A <code>ValueProvider</code> can have an associated
+<em>IgnorePredicate</em>, a TALES expression.
+When the <code>IgnorePredicate</code> yields true for
+a value, then it is ignored. You may e.g. specify that
+empty strings should be ignored.</p>
+
+<p>A <code>ValueProvider</code> can have an associated
+<em>Normalizer</em>, a TALES expression.
+The <code>Normalizer</code> is applied to not ignored values
+to transform them in a standard form, e.g. the normalizer
+for a keyword index can transform a string into a one element
+tuple containing the string as a keyword index requires a sequence
+value.</p>
+
+
+<p>The most essential ValueProviders are <em>AttributeLookup</em>s.
+An AttributeLookup determines the object's value through
+the lookup of an attribute. The AttributeLookup's configuration
+determines whether acquisition can be used, whether a callable value
+should be called and whether exceptions should be ignored.
+</p>
+
+<p><em>ExpressionEvaluator</em>s are another type of ValueProviders.
+These are TALES expressions defining an objects value.
+<code>ExpressionEvaluator</code>s often avoid to define
+simple scripts just for indexing purposes.<br>
+<span class="warning">Warning: </span>Until Zope 2.7, TALES
+expressions have been trusted when used outside of Zope; from
+Zope 2.8 on, TALES expression evaluation is always subject
+to Zope's security restrictions even when used outside of Zope.
+This may have strange consequences when you perform index management
+(e.g. mass reindexing) in an external script (run outside of Zope).
+In such cases, you should probably let the script login as a Zope user
+with sufficient priviledges.
+</p>
+
+<h3>Value Combination</h3>
+<p>When an index has several ValueProviders, their values
+must be combined to define the object's value.</p>
+
+<p><code>None</code> and ignored values are always ignored
+and are not combined.</p>
+
+<p>Currently, there are three combiners:</p>
+<dl>
+<dt><code>useFirst</code><dd>use the first non-ignored value<br>
+ This is available for all kinds of indexes.</dd>
+<dt><code>union</code><dd>combine all non-ignored values in a union like way.<br>
+ This is only available for indexes that split the object's value
+ into a set of index terms (such as keyword or text indexes).</dd>
+<dt><code>aggregate</code><dd>combine all values into a list.<br>
+ This is only available for index types that expect aggregate values
+ such as e.g. a <code>RangeIndex</code>.</dd>
+</dl>
+
+<h3>Term Standardization</h3>
+<p>Once the object's value (with respect to the index) has been
+determined, the value is split into a set of index terms
+in an index specific way (e.g. a <code>FieldIndex</code>
+uses the value directly, a <code>KeyWordIndex</code>
+requires the value to be a sequence and uses its elements as
+index terms and a <code>TextIndex</code> splits the value into
+words and uses the words).
+The terms are then standardized.</p>
+
+<p>Standardization consists of serveral steps:
+prenormalization such as case normalization, stemming,
+phonetic normalization, ...
+elimination of
+stop terms, normalization such as type conversion,
+type checking and copying.</p>
+
+
+<h4>Prenormalization</h4>
+<p>A index can define a term prenormalizer, a TALES expression.
+The prenormalizer is applied to terms before term expansion
+in queries and always as the first step of normalization during
+indexing. It can be used e.g. for case normalization, stemming,
+phonetic normalization, synonym normalization, ...</p>
+
+<h4>Stop Terms</h4>
+<p>An index can define a <em>StopTermPredicate</em>, a TALES
+expression. When the predicate yields true for a term, the term
+is ignored.</p>
+
+<h4>Term Normalization</h4>
+<p>An index can define a <em>NormalizeTerm</em> TALES expression.
+The expression can be used to transform the term into some standard
+form, e.g.
+convert a descriptive string or complex object into a code
+used for efficient indexing.</p>
+
+<h3>Type Restriction</h3>
+<p>The BTrees used in index implementation require that any index term must be
+persistently comparable to any other index term in this index
+(otherwise, the index gets corrupted). To help observing this
+essential property, the index term's type can be restricted.</p>
+
+<p>The following type restrictions are defined:</p>
+<dl>
+<dt><code>not checked</code><dd>the type is not restricted at all,</dd>
+<dt><code>numeric</code><dd>the type must be a numeric type (<code>int</code>,
+<code>float</code> or <code>long</code>),</dd>
+<dt><code>string</code><dd>the type must be a string,</dd>
+<dt><code>ustring</code><dd>the type must be a unicode string; <code>TermTypeExtra</code> can specify an encoding used for conversion.</dd>
+<dt><code>integer</code><dd>the type must be an integer,</dd>
+<dt><code>DateTime</code><dd>the type must be a <code>DateTime</code> object or float --
+the index stores the date as a float (seconds since epoch),</dd>
+<dt><code>DateTimeInteger</code><dd>the type must be a <code>DateTime</code> object or integer -- the index stores the date as an integer (seconds since epoch; truncated, if necessary),</dd>
+<dt><code>DateInteger</code><dd>the type must be a <code>DateTime</code> object or integer -- the index stores an integer representing the date
+(<code>400 * <var>year</var> + 31 * (<var>month</var>-1) + (<var>day</var>-1)</code>,</dd>
+<dt><code>tuple</code> with <var>tuplespec</var> in <code>TermTypeExtra</code><dd>The value must be a tuple
+ of the structure given by <var>tuplespec</var>. Each
+ tuple component is specified by a letter or by a parenthesized
+ <var>tuplespec</var> (for "subtuples"). Recognized
+ letters are <code>n</code> (numeric), <code>s</code> (string),
+ <code>u</code> (unicode string),
+ <code>d</code> (datetime).
+ An encoding for unicode conversion can be specified after <var>tuplespec</var>,
+ separated by <code>;</code>.
+ </dd>
+<dt><code>instance</code> with <var>fullClassName</var> in <code>TermTypeExtra</code><dd>the type must be an instance of the given class and the class must define an <code>__cmp__</code> method.
+It is assumed without check, that this <code>__cmp__</code>
+implements a persistent comparison.
+</dd>
+<dt><code>expression checked</code> with <code>TermTypeExtra</code> specifying checking TALES expression<dd>the expression may try to transform the
+value into something acceptable. If it yields <code>None</code> or
+raises an exception, the type is considered inacceptable, otherwise
+the value is used instead of the original term.
+ </dd>
+</dl>
+
+<p>All checkers try to convert a term into an acceptable type.
+Only when this fails, an exception is raised.</p>
+
+<p>If you choose one of the integer types, i.e. <code>integer</code>,
+<code>DateTimeInteger</code> or <code>DateInteger</code>,
+an especially efficient index type is build.
+You must clear and reindex the index when the index type is changed.
+You must clear the index even when it is already empty (because the data
+structures may need to be changed).</p>
+
+
+<h4>Copying</h4>
+<p>It is dangerous to index mutable types. If a indexed mutable object
+is later changed, its ordering with respect to other indexed
+objects may change, corrupting the index. Corruption of this
+kind can be avoided when the mutable object is copied before it
+is stored in the index.</p>
+
+<p><code>TermCopy</code>
+ specifies whether the value should be directly used,
+ shallow copied or deep copied.</p>
+
+
+<h2>Index Types</h2>
+<p>Currently, <code>ManagableIndexes</code> supports 3 types of
+indexes: <code>FieldIndex</code>, <code>KeywordIndex</code> and
+<code>RangeIndex</code>. Further types can easily be implemented.</p>
+
+<h3><code>FieldIndex</code></h3>
+<p>A <code>FieldIndex</code> indexes an object under a single term,
+the object's value with respect to the index.</p>
+
+<p>You get efficient DateTime and Date indexes by selecting
+<code>DateTime</code> (stored as float),
+<code>DateTimeInteger</code> (stored as integer) or
+<code>DateInteger</code> (stored as integer) as
+<code>TermType</code>. You must clear and reindex when you
+change the type.</p>
+
+<p>You can very efficiently sort search results via a <code>FieldIndex</code>.
+The <code>ZCatalog</code>'s method <code>searchResults</code>
+(aka <code>__call__</code>) supports this via its <code>sort_on</code>
+argument. My
+<a href="http://www.dieter.handshake.de/pyprojects/zope/AdvancedQuery.html"
+> AdvancedQuery</a> product supports arbitrary levels of sorting
+via <code>FieldIndex</code>es.</p>
+
+<p><code>AdvancedQuery</code>'s <code>FieldIndex</code> based ascending sorting
+is much more efficient than descending sorting. To make descending sorting
+more efficient, you can tell your <code>FieldIndex</code> to
+maintain the sort keys (also) in reverse order. This will slow indexing down
+a bit but make descending sorts much faster. When you change this
+attribute, you must clear and reindex the index (otherwise, the
+reverse order is not consistent).</p>
+
+<h3><code>KeywordIndex</code></h3>
+<p>A <code>KeywordIndex</code> expects the value of an object to
+be a sequence. It indexes the object under each element of this sequence.</p>
+
+<h3><code>RangeIndex</code></h3>
+<p>A <code>RangeIndex</code> expects an object's value
+to be a pair specifying an interval <var>low</var> to <var>high</var>.
+The index can efficiently locate documents for which a given term
+<var>t</var> lies within the document's <var>low</var> and <var>high</var>
+bounds.</p>
+
+<p>The object's value can either by constructed by a single
+attribute or with an <code>aggregate</code> combiner.</p>
+
+<p>To provide for a partial plugin replacement for CMF's
+<code>effective</code> and <code>expires</code> indexes,
+<code>RangeIndex</code> supports a <code>Boundary names</code>
+property. If set, it should be a pair of two names
+<var>low</var> and <var>high</var>. The index will then
+execute queries of the form <code><var>low</var> <= <var>val</var> <= <var>high</var></code>.
+To be compatible with <a href="AdvancedQuery"><code>AdvancedQuery</code></a>,
+the index replacing <code>effective</code> and <code>expires</code>
+should have the name <code>ValidityRange</code>.</p>
+
+<p><code>RangeIndex</code> efficiently supports improper ranges,
+i.e. those where at least one boundary is unlimited. You
+use its <code>Mininal value</code> and <code>Maximal value</code>
+properties to define which values should be considered as
+unlimited. These properties are TALES valued and are evaluated
+when the index is cleared. All values at or below (the value of)
+<code>Minimal value</code> are identified and interpreted
+as no lower limit; similarly, all values at or above (the value of)
+<code>Maximal value</code> are identified and interpreted as no
+upper limit.</p>
+
+<p>The boolean property <code>Organisation 'high-then-low'</code>
+controls the index organisation. With <code>high-then-low</code>
+organisation, the <code>high</code> index is primary and
+the <code>low</code> index is subordinate; <code>low-then-high</code>
+indicates the opposite organisation. You should use <code>high-then-low</code>
+when <code><var>val</var> <= <var>high</var></code>
+is less likely than <code><var>low</var> <= <var>val</var></code>.
+This is to be expected for date ranges and typical queries against them.</p>
+
+<h3><code>WordIndex</code></h3>
+<p>A <code>WordIndex</code> indexes an object under a set of words.
+It uses a <code>ZCTextIndex.PLexicon</code> for splitting a
+text into a sequence of words.</p>
+
+<p>A <code>WordIndex</code> lies between a <code>KeywordIndex</code> and a <code>TextIndex</code>.
+Like a <code>TextIndex</code>, it uses a <code>Lexicon</code> to split values into
+a sequence of words, stores integer word ids in its index
+and does not support range queries.
+Like a <code>KeywordIndex</code>, it indexes an object under a set of words --
+no near or phrase queries and no relevancy ranking.
+Due to these restrictions, a <code>WordIndex</code> is very efficient with respect
+to transaction and load size.</p>
+
+<p>The motivation to implement a
+<code>WordIndex</code> came from my observation, that almost all our ZEO loads
+above 100ms were caused by loading large word frequency
+IOBuckets used by <code>TextIndex</code>es for relevancy ranking -- a feature
+we do not need and use. Many of these loads transfered buckets
+of several 10 kB and a considerable portion of them took more
+than a second. As word frequency information is necessary for
+each document in a hit, you can imagine how fast our queries
+were.</p>
+
+<p>On the other hand, a <code>WordIndex</code> is only useful when you
+use a flexible query framework, such as e.g. <code>AdvancedQuery</code>
+or <code>CatalogQuery</code>.
+The standard <code>ZCatalog</code> framework is too weak as it does not
+allow to have several subqueries against the same index
+in a query. That's the reason why <code>TextIndex</code>es come with
+their own query parser. I live in Germany and therefore
+the standard query parser is useless for us (we use <code>und</code>, <code>oder</code>,
+<code>nicht</code> instead of <code>and</code>, <code>or</code> and <code>not</code>) and I have <code>AdvancedQuery</code> --
+thus I did not care to give the new <code>WordIndex</code> a query parser.
+You could easily provide one -- should you feel a need for it.</p>
+
+<h3><code>PathIndex</code></h3>
+<p>A <code>PathIndex</code> indexes an object under a path. A path is
+is either a tuple or a '/' separated string. The index supports
+path queries. For a given path, it locates objects that contain
+this path as subpath in its path value. Two search parameters
+<var>level</var> and <var>depth</var> control where in the
+object's path the given path may occur.</p>
+
+<p><var>level</var> and <var>depth</var> can be either <code>None</code>
+or an integer. When <var>level</var> and <var>depth</var> are
+both greater or equal 0, then an object with path <var>op</var>
+matches path <var>p</var> with respect to <var>level</var>
+and <var>depth</var>, iff <code><var>op</var> = <var>p1</var> <var>p</var> <var>p2</var></code> and
+<code>len(<var>p1</var>) = <var>level</var></code> and
+<code>len(<var>p2</var>) = <var>depth</var></code>.
+This means that <var>level</var> controls <var>p</var>'s distance
+from the beginning
+and <var>depth</var> that from the end of <var>op</var>.
+A <code>None</code> value means
+that there is no restriction for the respective side.
+A negative value means that the distance from the respective side
+may be up to (including) the negated value. E.g. a "-2" allows
+up to 2 segments.</p>
+
+<p>The default value for <var>level</var> is <code>0</code>,
+that for <var>depth</var> is <code>None</code>.</p>
+
+<p>Note, that <code>getPhysicalPath</code> requires acquisition
+to work properly. You must not set the acquisition type for
+such a value provider to <code>none</code>.</p>
+
+
+<h2>Matching</h2>
+<p>Most string and unicode based indexes (exception <code>RangeIndex</code>)
+support regular expression and glob matching. You use
+the query option <code>match</code> with a value of <code>'glob'</code>
+or <code>'regexp'</code> to call for this feature.</p>
+
+<p>Note that for large indexes, the match term should begin with
+an (easily recognizable) plain text string. Otherwise, this
+query type can be very inefficient.</p>
+
+
+
+<h2>TALES Expressions</h2>
+<p><code>ManagableIndex</code> uses TALES expressions at many places.
+They always can use the predefined variables:</p>
+<dl>
+<dt><code>index</code><dd>the index</dd>
+<dt><code>catalog</code><dd>the catalog</dd>
+<dt><code>root</code><dd>the zope root object</dd>
+<dt><code>modules</code><dd>the modules importer</dd>
+<dt><code>nothing</code><dd><code>None</code></dd>
+<dt><code>value</code><dd>the value to be checked</dd>
+<dt><code>object</code><dd>the object currently indexed</dd>
+</dl>
+
+<p>If the TALES expression evaluates to a callable object, then
+this is called on the value and the result used; otherwise, the
+evaluation result is used directly.</p>
+
+<h2>Utilities</h2>
+<p>The module <code>Utils</code> contains some useful functions:
+<code>convertToDateTime(value)</code>,
+<code>convertToDateTimeInteger(value, exc=0)</code>
+and
+<code>convertToDateInteger(value, round_dir=-1)</code>.
+Please see the source documentation, for details.</p>
+
+<h2>Programmatic Creation</h2>
+<p>A primary goal for <code>ManagableIndex</code> is the easy and flexible
+configuration via the Zope Management Interface (ZMI). Occasionally, however,
+you want to create your ManagableIndexes programmatically and want to
+configure them in the same step. The <code>ZCatalog</code>'s
+<code>[manage_]addIndex</code> allow you to pass configuration information
+down to the index construction in the <code>extra</code> argument.</p>
+
+<p>For the creation of a ManagableIndex, <code>extra</code> must have
+a dict value.
+Its key <code>'ValueProviders'</code> determines the indexes'
+value providers. All other keys can provide values for the indexes
+properties.</p>
+
+<p>If the key <code>'ValueProviders'</code> is present, its value
+must be a sequence of value provider specifications which define
+the value providers for the index. If the key is not present, a
+default value provider is created.</p>
+
+<p>A value provider specification is a dict with the mandatory keys
+<code>'id'</code> and <code>'type'</code>. <code>type</code>
+specifies the value provider's type (<code>'AttributeLookup'</code> or
+<code>'ExpressionEvaluator'</code>). Additional keys can defined values
+for the value providers properties.</p>
+
+<p>If the configuration dicts contain keys different from the above mentioned
+special ones and not corresponding to a property,
+a <code>ValueError</code> exception is raised.</p>
+
+<p>Look for the <code>_properties</code> definitions in the source, to
+determine the available properties for the various types of objects.</p>
+
+ <hr>
+ <address><a href="mailto:dieter at handshake.de">Dieter Maurer</a></address>
+<!-- Created: Tue Sep 2 21:02:52 CEST 2003 -->
+<!-- hhmts start -->
+Last modified: Thu Nov 23 20:42:04 CET 2006
+<!-- hhmts end -->
+ </body>
+</html>
Added: zope-managableindex/branches/upstream/current/fixPluginIndexes.py
===================================================================
--- zope-managableindex/branches/upstream/current/fixPluginIndexes.py 2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/fixPluginIndexes.py 2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,99 @@
+# $Id: fixPluginIndexes.py,v 1.3 2004/08/22 08:41:57 dieter Exp $
+'''Fixes for broken 'PluginIndexes'.
+
+I am sad that this is necessary.
+'''
+
+from Products.PluginIndexes.common.util import parseIndexRequest, \
+ InstanceType, DateTime, StringType, DictType, SequenceTypes
+
+
+class parseIndexRequest(parseIndexRequest):
+ def __init__(self, request, iid, options=[]):
+ '''this is essentially a copy of the inherited '__init__'.
+ Unfortunately, this turns empty query sequences into 'None'
+ which effectively means "no restriction".
+ However, an empty sequence of terms is the opposite
+ on "no restriction" (for "or" searches).
+
+ I hate that this extensive copying is necessary.
+ '''
+ self.id = iid
+
+ if not request.has_key(iid):
+ self.keys = None
+ return
+
+ # We keep this for backward compatility
+ usage_param = iid + '_usage'
+ if request.has_key(usage_param):
+ # we only understand 'range' -- thus convert it here
+ #self.usage = request[usage_param]
+ usage = request[usage_param]
+ if usage.startswith('range:'): range= usage[6:]
+ else: ValueError('unrecognized usage: %s' % usage)
+ self.range = range
+
+ param = request[iid]
+ keys = None
+ t = type(param)
+
+ if t is InstanceType and not isinstance(param, DateTime):
+ """ query is of type record """
+
+ record = param
+
+ if not hasattr(record, 'query'):
+ raise self.ParserException, (
+ "record for '%s' *must* contain a "
+ "'query' attribute" % self.id)
+ keys = record.query
+
+ if type(keys) is StringType:
+ keys = [keys.strip()]
+
+ for op in options:
+ if op == "query": continue
+
+ if hasattr(record, op):
+ setattr(self, op, getattr(record, op))
+
+ elif t is DictType:
+ """ query is a dictionary containing all parameters """
+
+ query = param.get("query", ())
+ if type(query) in SequenceTypes:
+ keys = query
+ else:
+ keys = [ query ]
+
+ for op in options:
+ if op == "query": continue
+
+ if param.has_key(op):
+ setattr(self, op, param[op])
+
+ else:
+ """ query is tuple, list, string, number, or something else """
+
+ if t in SequenceTypes:
+ keys = param
+ else:
+ keys = [param]
+
+ for op in options:
+ field = iid + "_" + op
+ if request.has_key(field):
+ setattr(self, op, request[field])
+
+## DM: turns empty sequences into 'None', the opposite of an empty sequence of search terms (for "or" searches).
+## if not keys:
+## keys = None
+
+ self.keys = keys
+
+ # fix broken inherited get
+ def get(self, key, default=None):
+ v = getattr(self, key, self)
+ if v is self: return default
+ return v
Added: zope-managableindex/branches/upstream/current/tests/TestBase.py
===================================================================
--- zope-managableindex/branches/upstream/current/tests/TestBase.py 2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/tests/TestBase.py 2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,118 @@
+# Copyright (C) 2003 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+# $Id: TestBase.py,v 1.5 2005/09/17 08:42:03 dieter Exp $
+'''Test infrastructure.'''
+
+
+
+#######################################################################
+# Hack to find our "INSTANCE_HOME" infrastructure.
+# Note, that "testrunner.py" overrides "INSTANCE_HOME". Therefore,
+# we look also for "TEST_INSTANCE_HOME"
+from os import environ, path
+import sys
+
+def _updatePath(path,dir):
+ if dir not in path: path.insert(0,dir)
+
+_ih= environ.get('TEST_INSTANCE_HOME') or environ.get('INSTANCE_HOME')
+if _ih:
+ _updatePath(sys.path, path.join(_ih,'lib','python'))
+ import Products; _updatePath(Products.__path__,path.join(_ih,'Products'))
+
+
+#######################################################################
+# Standard imports
+from unittest import TestCase, TestSuite, makeSuite, TextTestRunner
+
+from Acquisition import Explicit, Implicit
+from OFS.Application import Application
+from OFS.SimpleItem import SimpleItem
+from Products.ZCatalog.ZCatalog import ZCatalog
+from Products.ZCTextIndex.Lexicon import Lexicon, Splitter
+
+from Products.ManagableIndex.FieldIndex import FieldIndex
+from Products.ManagableIndex.KeywordIndex import KeywordIndex
+from Products.ManagableIndex.RangeIndex import RangeIndex
+from Products.ManagableIndex.WordIndex import WordIndex
+from Products.ManagableIndex.PathIndex import PathIndex
+
+def genSuite(*testClasses,**kw):
+ prefix= kw.get('prefix','test')
+ return TestSuite([makeSuite(cl,prefix) for cl in testClasses])
+
+def runSuite(suite):
+ tester= TextTestRunner()
+ tester.run(suite)
+
+def runTests(*testClasses,**kw):
+ runSuite(genSuite(*testClasses,**kw))
+
+
+class Lexicon(Lexicon, SimpleItem): pass
+
+#######################################################################
+# Test base class
+class TestBase(TestCase):
+ '''An application with a catalog with field index 'id',
+ keyword index 'ki', range index 'ri', word index 'wi', path index 'pi'
+ and two objects 'obj1' and 'obj2'.
+ '''
+ def setUp(self):
+ app= Application()
+ catalog= ZCatalog('Catalog')
+ app._setObject('Catalog',catalog)
+ self.catalog= catalog= app._getOb('Catalog')
+ # create indexes -- avoid the official API because it requires
+ # product setup and this takes ages
+ cat= catalog._catalog
+ # field
+ fi= FieldIndex('id'); cat.addIndex('id',fi)
+ self.fi= cat.getIndex('id')
+ # keyword
+ ki= KeywordIndex('kw'); cat.addIndex('kw',ki)
+ self.ki= cat.getIndex('kw')
+ # range
+ ri= RangeIndex('ri'); cat.addIndex('ri',ri)
+ self.ri = ri = cat.getIndex('ri')
+ ri._delObject('ri'); ri.CombineType = 'aggregate'
+ ri.addValueProvider('rlow','AttributeLookup')
+ ri.addValueProvider('rhigh','AttributeLookup')
+ # word
+ lexicon = Lexicon(Splitter())
+ app._setObject('lexicon', lexicon)
+ wi = WordIndex('wi'); cat.addIndex('wi',wi)
+ wi.Lexicon = 'lexicon'
+ self.wi = cat.getIndex('wi')
+ # path
+ pi= PathIndex('pi'); cat.addIndex('pi',pi)
+ self.pi= cat.getIndex('pi')
+ # create objects
+ self.obj1= obj1= _Object()
+ obj1.kw= (1,2)
+ obj1.fkw= _Caller(lambda obj: obj.kw)
+ obj1.fid= _Caller(lambda obj: obj.id)
+ self.obj2= obj2= _Object().__of__(obj1)
+ obj2.id= 'id'
+
+ def _check(self, index, query, should):
+ rs, _ = index._apply_index({index.id:query})
+ self.assertEqual(''.join(map(repr, rs.keys())), should)
+
+
+#######################################################################
+# Auxiliaries
+
+class _Caller(Explicit):
+ def __init__(self,f):
+ self._f= f
+
+ def __call__(self):
+ return self._f(self.aq_parent)
+
+class _Object(Implicit):
+ __roles__ = None
+ __allow_access_to_unprotected_subobjects__ = True
+ def __cmp__(self,other):
+ if not isinstance(other,_Object): raise TypeError('type mismatch in comparison')
+ return cmp(self.__dict__,other.__dict__)
Added: zope-managableindex/branches/upstream/current/tests/__init__.py
===================================================================
Added: zope-managableindex/branches/upstream/current/tests/test_ManagableIndex.py
===================================================================
--- zope-managableindex/branches/upstream/current/tests/test_ManagableIndex.py 2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/tests/test_ManagableIndex.py 2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,611 @@
+# Copyright (C) 2003 by Dr. Dieter Maurer, Eichendorffstr. 23, D-66386 St. Ingbert, Germany
+# see "LICENSE.txt" for details
+# $Id: test_ManagableIndex.py,v 1.13 2006/11/23 19:43:05 dieter Exp $
+
+from TestBase import TestBase, genSuite, runSuite, TestCase
+
+from re import escape
+
+from BTrees.OOBTree import OOSet
+from BTrees.IIBTree import IISet
+from BTrees.IOBTree import IOBTree
+from DateTime.DateTime import DateTime
+
+from Products.ManagableIndex.ValueProvider import ExpressionEvaluator
+from Products.ManagableIndex.ManagableIndex import _splitPrefixRegexp
+from Products.ManagableIndex.ManagableIndex import IFilter, ManagableIndex
+from Products.ManagableIndex.KeywordIndex import KeywordIndex_scalable
+
+
+class TestManagableIndex(TestBase):
+ def test_AttributeLookup(self):
+ fi= self.fi; ki= self.ki # field and keyword index
+ obj1= self.obj1; obj2= self.obj2
+ kw= OOSet((1,2,))
+ ## Acquisition type
+ # implicit
+ self.assertEqual(fi._evaluate(obj2),'id')
+ self.assertEqual(fi._evaluate(obj1),None)
+ # "OOSet" does not support intelligent equality test
+ self.assertEqual(ki._evaluate(obj2).keys(),kw.keys())
+ self.assertEqual(ki._evaluate(obj1).keys(),kw.keys())
+ # none
+ ki.kw.AcquisitionType= 'none'
+ self.assertEqual(ki._evaluate(obj2),None)
+ self.assertEqual(ki._evaluate(obj1).keys(),[1,2])
+ # explicit
+ # same as implicit for non-methods
+ ki.kw.AcquisitionType= 'explicit'
+ self.assertEqual(fi._evaluate(obj2),'id')
+ self.assertEqual(fi._evaluate(obj1),None)
+ self.assertEqual(ki._evaluate(obj2).keys(),kw.keys())
+ self.assertEqual(ki._evaluate(obj1).keys(),kw.keys())
+ # now check methods
+ fi.id_.Name= 'fid'; ki.kw.Name= 'fkw'
+ self.assertEqual(fi._evaluate(obj2),'id')
+ self.assertEqual(fi._evaluate(obj1),None)
+ self.assertEqual(ki._evaluate(obj2),None)
+ self.assertEqual(ki._evaluate(obj1).keys(),kw.keys())
+
+ ## call types
+ # call -- already checked
+ # return
+ fi.id_.CallType= 'return'
+ self.assertEqual(fi._evaluate(obj2), obj1.fid)
+ # ignore
+ fi.id_.CallType= 'ignore'
+ self.assertEqual(fi._evaluate(obj2), None)
+ fi.id_.CallType= 'call'
+
+ ## IgnoreExceptions
+ fi.id_.IgnoreExceptions= 0
+ self.assertRaises(AttributeError,fi._evaluate,obj1)
+ self.assertEqual(fi._evaluate(obj2),'id')
+
+ def test_ExpressionEvaluator(self):
+ ki= self.ki; obj2= self.obj2
+ ee= ExpressionEvaluator(); ee.id= 'ee'
+ ki._setObject(ee.id,ee); ee= ki._getOb(ee.id)
+ ee.manage_changeProperties(Expression= 'python: (3,4,)')
+ self.assertEqual(ki._evaluate(obj2).keys(),OOSet((1,2,3,4)).keys())
+ # ignore
+ ee.manage_changeProperties(IgnorePredicate= 'python: 3 in value')
+ self.assertEqual(ki._evaluate(obj2).keys(),OOSet((1,2,)).keys())
+ # ignore - call it
+ ee.manage_changeProperties(IgnorePredicate= 'python: lambda v: 3 in v')
+ # normalize
+ ee.manage_changeProperties(Expression= 'python: (4,)')
+ ee.manage_changeProperties(Normalizer= 'python: (0,) + value')
+ self.assertEqual(ki._evaluate(obj2).keys(),OOSet((0,1,2,4,)).keys())
+ # normalize - call it
+ ee.manage_changeProperties(Normalizer= 'python: lambda v: (0,) + v')
+ self.assertEqual(ki._evaluate(obj2).keys(),OOSet((0,1,2,4,)).keys())
+ # method
+ ee.manage_changeProperties(Expression= "python: lambda object: object.kw")
+ self.assertEqual(ki._evaluate(obj2).keys(),OOSet((0,1,2,)).keys())
+ ## combine
+ # 'union' - already tested
+ # 'useFirst'
+ ki.CombineType= 'useFirst'
+ self.assertEqual(ki._evaluate(obj2).keys(),OOSet((1,2,)).keys())
+
+ def test_TypeChecking(self):
+ fi= self.fi; obj= self.obj2
+ # numeric
+ fi.TermType= 'numeric'
+ obj.id= 1; self.assertEqual(fi._evaluate(obj),1)
+ obj.id= '1'; self.assertEqual(fi._evaluate(obj),1)
+ obj.id= '1.0'; self.assertEqual(fi._evaluate(obj),1.0)
+ obj.id= '1.0+'; self.assertRaises(Exception,fi._evaluate,obj)
+ # string
+ fi.TermType= 'string'
+ obj.id= 1; self.assertEqual(fi._evaluate(obj),'1')
+ obj.id= '1'; self.assertEqual(fi._evaluate(obj),'1')
+ obj.id= obj; self.assertRaises(Exception,fi._evaluate,obj)
+ # unicode
+ fi.TermType= 'ustring'
+ obj.id= u'1'; self.assertEqual(fi._evaluate(obj),u'1')
+ obj.id= '1'; self.assertEqual(fi._evaluate(obj),u'1')
+ # integer
+ fi.TermType= 'integer'
+ obj.id= 1; self.assertEqual(fi._evaluate(obj),1)
+ obj.id= '1'; self.assertEqual(fi._evaluate(obj),1)
+ obj.id= 1.1; self.assertEqual(fi._evaluate(obj),1)
+ # DateTime
+ fi.TermType= 'DateTime'; now= DateTime()
+ obj.id= now; self.assertEqual(fi._evaluate(obj),now)
+ obj.id= '1'; self.assertRaises(Exception,fi._evaluate,obj)
+ # DateTimeInteger
+ fi.TermType= 'DateTimeInteger'
+ obj.id= now; v = fi._evaluate(obj)
+ self.assert_(isinstance(v, int))
+ self.assert_(abs(v-now._t) <= 1)
+ # DateInteger
+ fi.TermType= 'DateInteger'
+ obj.id = DateTime('1000-01-01')
+ v = fi._evaluate(obj)
+ self.assert_(isinstance(v, int))
+ self.assertEqual(v, 400000)
+ # tuple
+ fi.TermType= 'tuple'
+ fi.TermTypeExtra= 'n(su)d'
+ obj.id= (1,('1',u'1'),now); self.assertEqual(fi._evaluate(obj),obj.id)
+ fi.TermTypeExtra+= 'n'
+ self.assertRaises(Exception,fi._evaluate,obj)
+ fi.TermTypeExtra= fi.TermTypeExtra[:-2]
+ self.assertRaises(Exception,fi._evaluate,obj)
+ # instance
+ fi.TermType= 'instance'
+ b= obj.aq_base; cl= b.__class__
+ fi.TermTypeExtra= '%s.%s' % (cl.__module__,cl.__name__)
+ obj.id= b; self.assertEqual(fi._evaluate(obj),b)
+ obj.id= '1'; self.assertRaises(Exception,fi._evaluate,obj)
+ # expression
+ fi.TermType= 'expression checked'
+ fi.TermTypeExtra= 'python: 1'
+ self.assertEqual(fi._evaluate(obj),1)
+ fi.TermTypeExtra= 'python: lambda v: 1'
+ self.assertEqual(fi._evaluate(obj),1)
+
+ ## term copy
+ fi.TermType= 'instance'
+ fi.TermTypeExtra= '%s._Object' % __name__
+ b= _Object()
+ obj.id= b; self.assert_(fi._evaluate(obj) is b)
+ fi.TermCopy= 'shallow'
+ v= fi._evaluate(obj)
+ self.assertEqual(v,b)
+ self.assert_(v is not b)
+ fi.TermCopy= 'deep'
+ b.l= []
+ v= fi._evaluate(obj)
+ self.assertEqual(v,b)
+ self.assert_(v.l is not b.l)
+
+ def test_Terms(self):
+ # normalize term
+ ki= self.ki; obj= self.obj1
+ ki.NormalizeTerm= 'python: lambda value: value-1'
+ self.assertEqual(ki._evaluate(obj).keys(),OOSet((0,1)).keys())
+ # stop term
+ ki.StopTermPredicate= 'python: value == 1'
+ self.assertEqual(ki._evaluate(obj).keys(),OOSet((1,)).keys())
+
+ def test_IntegerOptimization(self):
+ fi= self.fi
+ for t in 'integer DateTimeInteger DateInteger'.split():
+ fi.TermType = t
+ fi.clear()
+ self.assert_(isinstance(fi._index, IOBTree))
+
+
+ def test_len(self):
+ fi = self.fi; obj = self.obj2
+ self.assertEqual(len(fi), 0)
+ fi.index_object(1, obj)
+ self.assertEqual(len(fi), 1)
+ fi.index_object(2, obj)
+ self.assertEqual(len(fi), 1)
+ obj.id = 'id2'
+ fi.index_object(2, obj)
+ self.assertEqual(len(fi), 2)
+
+ def test_FieldIndex(self):
+ fi= self.fi; obj= self.obj2
+ # index_object
+ fi.index_object(1,obj)
+ self.assertEqual(fi.numObjects(),1)
+ self.assertEqual(len(fi),1)
+ # simple succeeding search
+ r,i= fi._apply_index({'id' : 'id'})
+ self.assertEqual(i,fi.id)
+ self.assertEqual(r.keys(),[1])
+ self._check(fi, 'id', '1')
+ # simple failing search
+ r,i= fi._apply_index({'id' : ''})
+ self.assertEqual(i,fi.id)
+ self.assertEqual(r.keys(), [])
+ # or search
+ q = ('','id',)
+ r,i= fi._apply_index({'id':q})
+ self.assertEqual(i,fi.id)
+ self.assertEqual(r.keys(), [1])
+ self._check(fi, q, '1')
+ # empty or search
+ r,i= fi._apply_index({'id' : ()})
+ self.assertEqual(i,fi.id)
+ self.assertEqual(r.keys(), [])
+ # range searches
+ q = {'query' : ('a','z'), 'range' : 'min:max'}
+ r,i= fi._apply_index({'id':q})
+ self.assertEqual(i,fi.id)
+ self.assertEqual(r.keys(), [1])
+ self._check(fi, q, '1')
+ q = {'query' : ('a',), 'range' : 'min'}
+ r,i= fi._apply_index({'id':q})
+ self.assertEqual(i,fi.id)
+ self.assertEqual(r.keys(), [1])
+ self._check(fi, q, '1')
+ q = {'query' : ('z',), 'range' : 'max'}
+ r,i= fi._apply_index({'id':q})
+ self.assertEqual(i,fi.id)
+ self.assertEqual(r.keys(), [1])
+ self._check(fi, q, '1')
+ r,i= fi._apply_index({'id' : {'query' : ('a','i'), 'range' : 'min:max'}})
+ self.assertEqual(i,fi.id)
+ self.assertEqual(r.keys(), [])
+ r,i= fi._apply_index({'id' : {'query' : ('j','z'), 'range' : 'min:max'}})
+ self.assertEqual(i,fi.id)
+ self.assertEqual(r.keys(), [])
+ r,i= fi._apply_index({'id' : {'query' : ('j',), 'range' : 'min'}})
+ self.assertEqual(i,fi.id)
+ self.assertEqual(r.keys(), [])
+ r,i= fi._apply_index({'id' : {'query' : ('i',), 'range' : 'max'}})
+ self.assertEqual(i,fi.id)
+ self.assertEqual(r.keys(), [])
+ # simple and search
+ r,i= fi._apply_index({'id' : {'query' : ('id',), 'operator' : 'and'}})
+ self.assertEqual(i,fi.id)
+ self.assertEqual(r.keys(), [1])
+ # multi and search
+ r,i= fi._apply_index({'id' : {'query' : ('id','id',), 'operator' : 'and'}})
+ self.assertEqual(i,fi.id)
+ self.assertEqual(r.keys(), [1])
+ # empty and search
+ r= fi._apply_index({'id' : {'query' : (), 'operator' : 'and'}})
+ self.assertEqual(r,None)
+ # failing and search
+ r,i= fi._apply_index({'id' : {'query' : ('id','',), 'operator' : 'and'}})
+ self.assertEqual(i,fi.id)
+ self.assertEqual(r.keys(), [])
+ # reindex
+ obj.id= 'id1'
+ fi.index_object(1,obj)
+ self.assertEqual(fi.numObjects(),1)
+ self.assertEqual(len(fi),1)
+ # unindex
+ fi.unindex_object(1)
+ self.assertEqual(fi.numObjects(),0)
+ self.assertEqual(len(fi),0)
+
+ def test_RangeIndex(self):
+ ri= self.ri; obj= self.obj2
+ # index_object
+ ri.index_object(1,obj)
+ self.assertEqual(ri.numObjects(),0)
+ obj.rlow = 10; ri.index_object(1,obj)
+ self.assertEqual(ri.numObjects(),0)
+ obj.rhigh = 20; ri.index_object(1,obj)
+ self.assertEqual(ri.numObjects(),1)
+ obj.rlow = obj.rhigh = 15; ri.index_object(2,obj)
+ self.assertEqual(ri.numObjects(),2)
+ # search
+ r,i = ri._apply_index({'ri': 9})
+ self.assertEqual(i,ri.id)
+ self.assertEqual(r.keys(), [])
+ r,i = ri._apply_index({'ri': 15})
+ self.assertEqual(i,ri.id)
+ self.assertEqual(r.keys(), [1,2])
+ self._check(ri, 15, '12')
+ r,i = ri._apply_index({'ri': 20})
+ self.assertEqual(i,ri.id)
+ self.assertEqual(r.keys(), [1])
+ self._check(ri, 20, '1')
+ self.assertEqual(len(ri), 2)
+ # boundary emulation
+ ri.BoundaryNames = ('low','high')
+ r = ri._apply_index({'low': {'query':15, 'range':'min'},})
+ self.assertEqual(r, None)
+ r = ri._apply_index({'low':{'query':15, 'range':'min'},
+ 'high':{'query':16, 'range':'max'},
+ })
+ self.assertEqual(r, None)
+ r,i = ri._apply_index({'low':{'query':15, 'range':'min'},
+ 'high':{'query':15, 'range':'max'},
+ })
+ self.assertEqual(i,ri.id)
+ self.assertEqual(r.keys(), [1,2])
+ # unindex object
+ ri.unindex_object(1)
+ self.assertEqual(len(ri), 1)
+ self.assertEqual(ri.numObjects(),1)
+ ri.index_object(1, obj)
+ self.assertEqual(len(ri), 1)
+ self.assertEqual(ri.numObjects(),2)
+ ri.unindex_object(1); ri.unindex_object(2)
+ self.assertEqual(len(ri), 0)
+ self.assertEqual(ri.numObjects(),0)
+
+ def test_ImproperRanges(self):
+ ri= self.ri; obj= self.obj2
+ ri.MinimalValue = 'python:10'; ri.MaximalValue = 'python:20'
+ ri.clear()
+ self._indexForRange(obj, ri)
+ r,i = ri._apply_index({'ri':0}); self.assertEqual(r.keys(),[1,2])
+ r,i = ri._apply_index({'ri':20}); self.assertEqual(r.keys(),[1,3])
+ r,i = ri._apply_index({'ri':15}); self.assertEqual(r.keys(),[1,2,3,4])
+ # check empty min
+ ri.MinimalValue = ''; ri.MaximalValue = 'python:20'
+ ri.clear()
+ self._indexForRange(obj, ri)
+ r,i = ri._apply_index({'ri':0}); self.assertEqual(r.keys(),[])
+ r,i = ri._apply_index({'ri':20}); self.assertEqual(r.keys(),[1,3])
+ r,i = ri._apply_index({'ri':15}); self.assertEqual(r.keys(),[1,2,3,4])
+ ri.MinimalValue = 'python:10'; ri.MaximalValue = ''
+ ri.clear()
+ self._indexForRange(obj, ri)
+ r,i = ri._apply_index({'ri':0}); self.assertEqual(r.keys(),[1,2])
+ r,i = ri._apply_index({'ri':21}); self.assertEqual(r.keys(),[])
+ r,i = ri._apply_index({'ri':15}); self.assertEqual(r.keys(),[1,2,3,4])
+ ri.MinimalValue = ''; ri.MaximalValue = ''
+ ri.clear()
+ self._indexForRange(obj, ri)
+ r,i = ri._apply_index({'ri':0}); self.assertEqual(r.keys(),[])
+ r,i = ri._apply_index({'ri':21}); self.assertEqual(r.keys(),[])
+ r,i = ri._apply_index({'ri':15}); self.assertEqual(r.keys(),[1,2,3,4])
+
+ def test_Organisation(self):
+ ri = self.ri; ri.OrganisationHighThenLow = True
+ ri.clear()
+ self.test_RangeIndex()
+ self.test_ImproperRanges()
+
+ def _indexForRange(self, obj, ri):
+ obj.rlow = 10; obj.rhigh = 20; ri.index_object(1, obj) # unrestricted
+ obj.rlow = 10; obj.rhigh = 15; ri.index_object(2, obj) # low unrestricted
+ obj.rlow = 15; obj.rhigh = 20; ri.index_object(3, obj) # high unrestricted
+ obj.rlow = 15; obj.rhigh = 15; ri.index_object(4, obj) # proper range
+
+ def test_PathIndex(self):
+ pi= self.pi; obj= self.obj2
+ # index_object
+ pi.index_object(1,obj)
+ self.assertEqual(pi.numObjects(),0)
+ obj.pi = 'a'; pi.index_object(1,obj)
+ self.assertEqual(pi.numObjects(),1)
+ obj.pi = 'a/b'; pi.index_object(2,obj)
+ obj.pi = 'a/b/c'.split('/'); pi.index_object(3,obj)
+ # check queries
+ c = self._check
+ c(pi, [()], '123')
+ c(pi, {'query':[()], 'level':2}, '23')
+ c(pi, {'query':[()], 'level':1, 'depth':1}, '2')
+ c(pi, {'query':[()], 'level':None}, '123')
+ c(pi, {'query':[()], 'level':-1}, '123')
+ c(pi, 'a', '123')
+ c(pi, 'b', '')
+ c(pi, 'x', '')
+ c(pi, {'query':'b', 'level':1}, '23')
+ c(pi, {'query':'b', 'level':None}, '23')
+ c(pi, {'query':'b/c', 'level':-1}, '3')
+ c(pi, {'query':['b/c'.split('/')], 'level':-1}, '3')
+ c(pi, {'query':'c', 'level':-1}, '')
+ c(pi, {'query':'c', 'level':-2}, '3')
+ c(pi, {'query':'a', 'depth':0}, '1')
+ c(pi, {'query':'a', 'depth':1}, '2')
+ c(pi, {'query':'a', 'depth':-1}, '12')
+ c(pi, {'query':'a', 'depth':None}, '123')
+ c(pi, {'query':[()], 'depth':-2}, '12')
+ c(pi, {'query':'b', 'level':None, 'depth':-1}, '23')
+ obj.pi = '/a'; pi.index_object(4,obj)
+ c(pi, '/', '4')
+
+
+ def test_ReverseOrder(self):
+ fi= self.fi; obj= self.obj2
+ self.assertEqual(fi.getReverseOrder(), None)
+ fi.ReverseOrder = 1; fi.clear()
+ # index_object
+ obj.id = '1'; fi.index_object(1,obj)
+ obj.id = '2'; fi.index_object(2,obj)
+ self.assertEqual(tuple(fi.getReverseOrder()), ('2','1'))
+ fi.unindex_object(2)
+ self.assertEqual(tuple(fi.getReverseOrder()), ('1',))
+
+ def test_Sorting(self):
+ cat= self.catalog; obj= self.obj2
+ cat.catalog_object(obj,'1')
+ self.assertEqual(len(self.catalog(id='id', sort_on='id')),1)
+ cat.catalog_object(obj,'1')
+
+ def test_KeywordIndex(self):
+ ki= self.ki; obj= self.obj2
+ # index_object
+ ki.index_object(1,obj)
+ self.assertEqual(ki.numObjects(),1)
+ self.assertEqual(len(ki),2)
+ self._check(ki, 1, '1')
+ # reindex_object
+ self.assertEqual(ki.index_object(1,obj),0)
+ obj.kw= (2,3,)
+ ki.index_object(1,obj)
+ self.assertEqual(ki.numObjects(),1)
+ self.assertEqual(len(ki),2)
+ self.assertEqual(ki.uniqueValues(),(2,3,))
+ self.assertEqual(ki.uniqueValues(withLengths=1),((2,1),(3,1),))
+ # index 2
+ ki.index_object(2,self.obj1)
+ self.assertEqual(ki.numObjects(),2)
+ self.assertEqual(len(ki),3)
+ self.assertEqual(ki.uniqueValues(withLengths=1),((1,1),(2,2),(3,1),))
+ # unindex
+ ki.unindex_object(1)
+ self.assertEqual(ki.numObjects(),1)
+ self.assertEqual(len(ki),2)
+ self.assertEqual(ki.uniqueValues(withLengths=1),((1,1),(2,1),))
+ ki.unindex_object(2)
+ self.assertEqual(ki.numObjects(),0)
+ self.assertEqual(len(ki),0)
+ self.assertEqual(ki.uniqueValues(withLengths=1),())
+
+ def test_KeywordIndex_scalable(self):
+ cat = self.catalog._catalog
+ cat.delIndex('kw')
+ ki = KeywordIndex_scalable('kw'); cat.addIndex('kw', ki)
+ self.ki = cat.getIndex('kw')
+ self.test_KeywordIndex()
+
+class TestFiltering(TestBase):
+ test_FieldIndex = TestManagableIndex.test_FieldIndex.im_func
+ test_KeywordIndex = TestManagableIndex.test_KeywordIndex.im_func
+ test_KeywordIndex_scalable = TestManagableIndex.test_KeywordIndex_scalable.im_func
+ test_RangeIndex = TestManagableIndex.test_RangeIndex.im_func
+ test_PathIndex = TestManagableIndex.test_PathIndex.im_func
+
+ def _check(self, index, query, should):
+ if not isinstance(query, dict): query = {'query':query}
+ query['isearch'] = query['isearch_filter'] = True
+ rs, _ = index._apply_index({index.id:query})
+ if hasattr(rs, 'asSet'): rs = rs.asSet().keys()
+ self.assertEqual(''.join(map(repr, rs)), should)
+
+class TestMatching(TestBase):
+ '''test glob and regexp expansion.'''
+ TermType = 'string'
+
+ def setUp(self):
+ '''
+ 1 -> a
+ 2 -> b
+ 3 -> ba
+ 4 -> bab
+ 5 -> c
+ 6 -> cb
+ 7 -> d\e
+ '''
+ TestBase.setUp(self)
+ self.index = index = self.fi; obj = self.obj1
+ index.TermType = self.TermType
+ for i, val in enumerate(r'a b ba bab c cb d\e'.split()):
+ obj.id = val
+ index.index_object(i+1, obj)
+
+ def test_glob(self):
+ self._check('glob', '*', '1234567')
+ self._check('glob', 'b*', '234')
+ self._check('glob', '*a', '13')
+ self._check('glob', 'b?', '3')
+ self._check('glob', r'd\e', '7')
+
+ def test_regexp(self):
+ self._check('regexp', 'a*', '1234567')
+ self._check('regexp', 'ba+', '34')
+ self._check('regexp', 'b[a]+', '34')
+
+ def test_splitPrefix(self):
+ self._checkSplit('abc', 'abc')
+ self._checkSplit('+', '')
+ self._checkSplit('a+', '')
+ self._checkSplit(r'\+', '+')
+ self._checkSplit(r'\a', '\a', '')
+ self._checkSplit(r'\d', '')
+ self._checkSplit(r'\c', 'c', '')
+ self._checkSplit(r'\\+', '')
+ self._checkSplit(r'a\\+', 'a')
+
+ def _check(self, match, pattern, result):
+ i = self.index
+ r = i._apply_index({i.id : {'query' : pattern, 'match':match},})[0]
+ self.assertEqual(''.join(map(str, r)), result)
+
+ def _checkSplit(self, regexp, prefix, rep=None):
+ pr,rp = _splitPrefixRegexp(regexp)
+ if rep is None: self.assertEqual(escape(pr) + rp, regexp)
+ else: self.assertEqual(rp, rep)
+ self.assertEqual(pr, prefix)
+
+class TestMatching_Unicode(TestMatching):
+ TermType = 'ustring'
+
+
+class TestMatching_Filtering(TestMatching):
+
+ def _check(self, match, pattern, result):
+ i = self.index
+ r = i._apply_index({i.id : {'query' : pattern, 'match':match, 'isearch':True, 'isearch_filter':True},})[0]
+ if hasattr(r, 'asSet'): r = r.asSet().keys()
+ self.assertEqual(''.join(map(str, r)), result)
+
+
+
+class TestWordIndex(TestBase):
+ def setUp(self):
+ '''
+ 1 -> a ab abc
+ 2 -> b ab bca
+ '''
+ TestBase.setUp(self)
+ obj = self.obj1; index = self.wi
+ obj.wi = 'a ab abc'; index.index_object(1, obj)
+ obj.wi = 'b ab bca'; index.index_object(2, obj)
+
+ def test_Lookup(self):
+ self._check('a', '1')
+ self._check('ab', '12')
+ self._check('A', '')
+ self._check('bca', '2')
+ self.assertRaises(ValueError, self._check, 'a b', '')
+
+ def test_Matching(self):
+ self._check({'query':'a*', 'match':'glob'}, '12')
+ self._check({'query':'a b', 'match':'glob'}, '')
+## self.assertRaises(ValueError,
+## self._check,
+## {'query':'a b', 'match':'glob'},
+## ''
+## )
+
+ def _check(self, query, result):
+ i = self.wi
+ r = i._apply_index({i.id : query,})[0]
+ self.assertEqual(''.join(map(str, r.keys())), result)
+
+
+class TestProgrammaticSetup(TestCase):
+ def test_PropertySetup(self):
+ self.assertRaises(ValueError, ManagableIndex, 'mi', {'TermTypeExtra':'tte', 'X':None})
+ mi = ManagableIndex('mi', {'TermTypeExtra':'tte'})
+ self.assertEqual(mi.TermTypeExtra, 'tte')
+
+ def test_NoProvider(self):
+ mi = ManagableIndex('mi', {})
+ self.assertEqual(mi.objectIds(), [])
+
+ def test_ExpliciteProvidersWithProperties(self):
+ mi = ManagableIndex('mi', {
+ 'ValueProviders': (
+ {'type':'AttributeLookup', 'id':'al', 'Name':'name',},
+ {'type':'ExpressionEvaluator', 'id':'el', 'Expression':'e',},
+ ),
+ }
+ )
+ self.assertEqual(len(mi.objectIds()), 2)
+ self.assertEqual(mi.al.Name, 'name')
+ self.assertEqual(mi.el.Expression, 'e')
+
+
+
+class _Object:
+ def __eq__(self,other):
+ return isinstance(other,_Object) and self.__dict__ == other.__dict__
+ def __ne__(self,other): return not (self == other)
+ def __cmp__(self,other):
+ if not isinstance(other,_Object): raise TypeError('type mismatch in comparison')
+ return cmp(self.__dict__,other.__dict__)
+
+
+
+def test_suite():
+ tests = [
+ TestManagableIndex,
+ TestMatching, TestMatching_Unicode,
+ TestWordIndex,
+ TestProgrammaticSetup,
+ ]
+ if IFilter is not None:
+ tests.extend([TestFiltering, TestMatching_Filtering])
+ return genSuite(*tests)
+
+
+if __name__ == '__main__':
+ runSuite(test_suite())
Added: zope-managableindex/branches/upstream/current/zpt/addForm.zpt
===================================================================
--- zope-managableindex/branches/upstream/current/zpt/addForm.zpt 2007-04-03 23:03:41 UTC (rev 735)
+++ zope-managableindex/branches/upstream/current/zpt/addForm.zpt 2007-04-04 00:01:33 UTC (rev 736)
@@ -0,0 +1,33 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2//EN">
+<html tal:define="
+ type options/type;
+ action options/action;
+ description options/description;
+ title string: Add $type;
+ "
+>
+ <head>
+ <title tal:content="title">Title</title>
+ </head>
+
+ <body>
+ <h2 tal:content="title">Title</h2>
+
+ <p tal:content="description">Description</p>
+
+ <form action="action" method="post"
+ tal:attributes="action action">
+ <table cellpadding=5>
+ <tr>
+ <th>Id</th><td><input name="id"></td>
+ </tr>
+ <tr>
+ <td colspan=2>
+ <input type="hidden" name="type" tal:attributes="value type">
+ <input type="submit" name="add">
+ </td>
+ </tr>
+ </table>
+ </form>
+ </body>
+</html>
More information about the pkg-zope-commits
mailing list