support for dict keys that can't be repr/eval'd

direct/src/showbase/ContainerLeakDetector.py

@@ -19,20 +19,25 @@ class CheckContainers(Job):
     
     def run(self):
         self._leakDetector._index2containerName2len[self._index] = {}
-        self._leakDetector.notify.debug(repr(self._leakDetector._id2pathStr))
+        self._leakDetector.notify.debug(repr(self._leakDetector._id2ref))
-        ids = self._leakDetector._id2pathStr.keys()
+        ids = self._leakDetector.getContainerIds()
         # record the current len of each container
         for id in ids:
             yield None
-            name = self._leakDetector._id2pathStr[id]
             try:
-                container = eval(name)
+                container = self._leakDetector.getContainerById(id)
             except Exception, e:
                 # this container no longer exists
-                self.notify.debug('container %s no longer exists; caught exception in eval (%s)' % (name, e))
+                self.notify.debug('container %s no longer exists; caught exception in getContainerById (%s)' % (name, e))
-                del self._leakDetector._id2pathStr[id]
+                self._leakDetector.removeContainerById(id)
+                continue
+            if container is None:
+                # this container no longer exists
+                self.notify.debug('container %s no longer exists; getContainerById returned None (%s)' % (name, e))
+                self._leakDetector.removeContainerById(id)
                 continue
             cLen = len(container)
+            name = self._leakDetector.getContainerNameById(id)
             self._leakDetector._index2containerName2len[self._index][name] = cLen
         # compare the current len of each container to past lens
         if self._index > 0:
@@ -64,6 +69,143 @@ class CheckContainers(Job):
                                 messenger.send(self._leakDetector.getLeakEvent(), [msg])
         yield Job.Done
 
+class NoDictKey:
+    pass
+
+class Indirection:
+    # represents the indirection that brings you from a container to an element of the container
+    # stored as a string to be used as part of an eval
+    # each dictionary dereference is individually eval'd since the dict key might have been garbage-collected
+    class GarbageCollectedDictKey(Exception):
+        pass
+
+    def __init__(self, evalStr=None, dictKey=NoDictKey):
+        # if this is a dictionary lookup, pass dictKey instead of evalStr
+        self.evalStr = evalStr
+        self.dictKey = NoDictKey
+        if dictKey is not NoDictKey:
+            # if we can repr/eval the key, store it as an evalStr
+            keyRepr = repr(dictKey)
+            useEval = False
+            try:
+                keyEval = eval(keyRepr)
+                useEval = True
+            except:
+                pass
+            if useEval:
+                # check to make sure the eval succeeded
+                if hash(keyEval) != hash(dictKey):
+                    useEval = False
+            if useEval:
+                # eval/repr succeeded, store as an evalStr
+                self.evalStr = '[%s]' % keyRepr
+            else:
+                # store a weakref to the key
+                self.dictKey = weakref.ref(dictKey)
+
+    def isDictKey(self):
+        return self.dictKey is not NoDictKey
+
+    def dereferenceDictKey(self, parentDict):
+        key = self.dictKey()
+        if key is None:
+            raise Indirection.GarbageCollectedDictKey()
+        return parentDict[key]
+
+    def getString(self, nextIndirection=None):
+        # return our contribution to the name of the object
+        if self.evalStr is not None:
+            # if we're an instance dict and the next indirection is not a dict key,
+            # skip over this one (obj.__dict__[keyName] == obj.keyName)
+            if nextIndirection is not None and self.evalStr == '.__dict__':
+                return ''
+            return self.evalStr
+
+        # we're stored as a dict key
+        # this might not eval, but that's OK, we're not using this string to find
+        # the object, we dereference the parent dict
+        key = self.dictKey()
+        if key is None:
+            return '<garbage-collected dict key>'
+        return safeRepr(key)
+
+    def __repr__(self):
+        return self.getString()
+
+class ContainerRef:
+    """
+    stores a reference to a container in a way that does not prevent garbage
+    collection of the container
+    stored as a series of 'indirections' (obj.foo -> '.foo', dict[key] -> '[key]', etc.)
+    """
+    class FailedEval(Exception):
+        pass
+    # whatever this is set to will be the default ContainerRef
+    BaseRef = None
+
+    def __init__(self, other=None, indirection=None):
+        self._indirections = []
+        # if no other passed in, try ContainerRef.BaseRef
+        if other is None:
+            other = ContainerRef.BaseRef
+        if other is not None:
+            for ind in other._indirections:
+                self.addIndirection(ind)
+        if indirection:
+            self.addIndirection(indirection)
+
+    def addIndirection(self, indirection):
+        self._indirections.append(indirection)
+
+    def _getContainerByEval(self, evalStr):
+        try:
+            container = eval(evalStr)
+        except NameError, ne:
+            return None
+        return container
+
+    def _evalWithObj(self, evalStr, curObj=None):
+        # eval an evalStr, optionally based off of an existing object
+        if curObj is not None:
+            # eval('curObj.foo.bar.someDict')
+            evalStr = 'curObj%s' % evalStr
+        return self._getContainerByEval(evalStr)
+
+    def getContainer(self):
+        # try to get a handle on the container by eval'ing and looking things
+        # up in dictionaries, depending on the type of each indirection
+        #import pdb;pdb.set_trace()
+        evalStr = ''
+        curObj = None
+        curIndirection = None
+        nextIndirection = None
+        for indirection in self._indirections:
+            if not indirection.isDictKey():
+                # build up a string to be eval'd
+                evalStr += indirection.getString()
+            else:
+                curObj = self._evalWithObj(evalStr, curObj)
+                if curObj is None:
+                    raise FailedEval(evalStr)
+                # try to look up this key in the curObj dictionary
+                curObj = indirection.dereferenceDictKey(curObj)
+                evalStr = ''
+
+        return self._evalWithObj(evalStr, curObj)
+        
+    def __repr__(self):
+        str = ''
+        curIndirection = None
+        nextIndirection = None
+        for i in xrange(len(self._indirections)):
+            curIndirection = self._indirections[i]
+            if i < len(self._indirections)-1:
+                nextIndirection = self._indirections[i+1]
+            else:
+                nextIndirection = None
+            str += curIndirection.getString(nextIndirection=nextIndirection)
+        return str
+
 class ContainerLeakDetector(Job):
     """
     Low-priority Python object-graph walker that looks for leaking containers.
@@ -87,19 +229,37 @@ class ContainerLeakDetector(Job):
         self._index2containerName2len = {}
         self._index2delay = {}
         # set up our data structures
-        self._id2pathStr = {}
+        self._id2ref = {}
-        self._curObjPathStr = '__builtin__.__dict__'
+
+        # set up the base/starting object
+        self._nameContainer(__builtin__.__dict__, ContainerRef(indirection=Indirection(evalStr='__builtin__.__dict__')))
+        try:
+            base
+        except:
+            pass
+        else:
+            ContainerRef.BaseRef = ContainerRef(indirection=Indirection(evalStr='base.__dict__'))
+            self._nameContainer(base.__dict__, ContainerRef.BaseRef)
+        try:
+            simbase
+        except:
+            pass
+        else:
+            ContainerRef.BaseRef = ContainerRef(indirection=Indirection(evalStr='simbase.__dict__'))
+            self._nameContainer(simbase.__dict__, ContainerRef.BaseRef)
+
+        self._curObjRef = ContainerRef()
         jobMgr.add(self)
         ContainerLeakDetector.PrivateIds.update(set([
             id(ContainerLeakDetector.PrivateIds),
-            id(self._id2pathStr),
+            id(self._id2ref),
             ]))
 
     def destroy(self):
         if self._checkContainersJob is not None:
             jobMgr.remove(self._checkContainersJob)
             self._checkContainersJob = None
-        del self._id2pathStr
+        del self._id2ref
         del self._index2containerName2len
         del self._index2delay
 
@@ -113,30 +273,17 @@ class ContainerLeakDetector(Job):
         # passes description string as argument
         return 'containerLeakDetected-%s' % self._serialNum
 
-    def _getContainerByEval(self, evalStr):
+    def getContainerIds(self):
-        try:
+        return self._id2ref.keys()
-            container = eval(evalStr)
+
-        except NameError, ne:
+    def getContainerById(self, id):
-            return None
+        return self._id2ref[id].getContainer()
-        return container
+    def getContainerNameById(self, id):
+        return repr(self._id2ref[id])
+    def removeContainerById(self, id):
+        del self._id2ref[id]
 
     def run(self):
-        # push on a few things that we want to give priority
-        # for the sake of the variable-name printouts
-        self._nameContainer(__builtin__.__dict__, '__builtin__.__dict__')
-        try:
-            base
-        except:
-            pass
-        else:
-            self._nameContainer(base.__dict__, 'base.__dict__')
-        try:
-            simbase
-        except:
-            pass
-        else:
-            self._nameContainer(simbase.__dict__, 'simbase.__dict__')
-
         taskMgr.doMethodLater(self._nextCheckDelay, self._checkForLeaks,
                               self.getCheckTaskName())
 
@@ -146,24 +293,27 @@ class ContainerLeakDetector(Job):
             yield None
             #import pdb;pdb.set_trace()
             curObj = None
-            curObj = self._getContainerByEval(self._curObjPathStr)
+            if self._curObjRef is None:
-            if curObj is None:
+                self._curObjRef = random.choice(self._id2ref.values())
-                self.notify.debug('lost current container: %s' % self._curObjPathStr)
+            try:
-                while len(self._id2pathStr):
+                curObj = self._curObjRef.getContainer()
-                    _id = random.choice(self._id2pathStr.keys())
+            except:
-                    curObj = self._getContainerByEval(self._id2pathStr[_id])
+                self.notify.debug('lost current container: %s' % self._curObjRef)
+                while len(self._id2ref):
+                    _id = random.choice(self._id2ref.keys())
+                    curObj = self.getContainerById(_id)
                     if curObj is not None:
                         break
                     # container is no longer valid
-                    del self._id2pathStr[_id]
+                    del self._id2ref[_id]
-                self._curObjPathStr = self._id2pathStr[_id]
+                self._curObjRef = self._id2ref[_id]
-            #print '%s: %s, %s' % (id(curObj), type(curObj), self._id2pathStr[id(curObj)])
+            #print '%s: %s, %s' % (id(curObj), type(curObj), self._id2ref[id(curObj)])
-            self.notify.debug('--> %s' % self._curObjPathStr)
+            self.notify.debug('--> %s' % self._curObjRef)
 
-            # keep a copy of this obj's eval str, it might not be in _id2pathStr
+            # keep a copy of this obj's eval str, it might not be in _id2ref
-            curObjPathStr = self._curObjPathStr
+            curObjRef = self._curObjRef
-            # if we hit a dead end, go back to __builtin__
+            # if we hit a dead end, start over at a container we know about
-            self._curObjPathStr = '__builtin__'
+            self._curObjRef = None
 
             try:
                 if curObj.__class__.__name__ == 'method-wrapper':
@@ -177,9 +327,9 @@ class ContainerLeakDetector(Job):
             if type(curObj) in (types.ModuleType, types.InstanceType):
                 child = curObj.__dict__
                 if not self._isDeadEnd(child):
-                    self._curObjPathStr = curObjPathStr + '.__dict__'
+                    self._curObjRef = ContainerRef(curObjRef, indirection=Indirection(evalStr='.__dict__'))
                     if self._isContainer(child):
-                        self._nameContainer(child, self._curObjPathStr)
+                        self._nameContainer(child, self._curObjRef)
                 continue
 
             if type(curObj) is types.DictType:
@@ -193,26 +343,16 @@ class ContainerLeakDetector(Job):
                     try:
                         attr = curObj[key]
                     except KeyError, e:
-                        self.notify.warning('could not index into %s with key %s' % (curObjPathStr,
+                        self.notify.warning('could not index into %s with key %s' % (curObjRef, key))
-                                                                                     key))
                         continue
                     if not self._isDeadEnd(attr):
-                        if curObj is __builtin__:
+                        if curObj is __builtin__.__dict__:
-                            self._curObjPathStr = PathStr(key)
+                            indirection=Indirection(evalStr=key)
-                            if key == '__doc__':
-                                import pdb;pdb.set_trace()
-                            if self._isContainer(attr):
-                                self._nameContainer(attr, PathStr(key))
                         else:
-                            # if the parent dictionary is an instance dictionary, remove the __dict__
+                            indirection=Indirection(dictKey=key)
-                            # and use the . operator
+                        self._curObjRef = ContainerRef(curObjRef, indirection=indirection)
-                            dLen = len('__dict__')
-                            if len(self._curObjPathStr) >= dLen and self._curObjPathStr[-dLen:] == '__dict__':
-                                self._curObjPathStr = curObjPathStr[:-dLen] + '.%s' % safeRepr(key)
-                            else:
-                                self._curObjPathStr = curObjPathStr + '[%s]' % safeRepr(key)
                         if self._isContainer(attr):
-                                self._nameContainer(attr, self._curObjPathStr)
+                            self._nameContainer(attr, self._curObjRef)
                 del key
                 del attr
                 continue
@@ -239,9 +379,9 @@ class ContainerLeakDetector(Job):
                         random.shuffle(attrs)
                         for attr in attrs:
                             if not self._isDeadEnd(attr):
-                                self._curObjPathStr = curObjPathStr + '[%s]' % index
+                                self._curObjRef = ContainerRef(curObjRef, indirection=Indirection(evalStr='[%s]' % index))
                                 if self._isContainer(attr):
-                                    self._nameContainer(attr, self._curObjPathStr)
+                                    self._nameContainer(attr, self._curObjRef)
                             index += 1
                         del attr
                     except StopIteration, e:
@@ -262,9 +402,9 @@ class ContainerLeakDetector(Job):
                 for childName in childNames:
                     child = getattr(curObj, childName)
                     if not self._isDeadEnd(child):
-                        self._curObjPathStr = curObjPathStr + '.%s' % childName
+                        self._curObjRef = ContainerRef(curObjRef, indirection=Indirection(evalStr='.%s' % childName))
                         if self._isContainer(child):
-                            self._nameContainer(child, self._curObjPathStr)
+                            self._nameContainer(child, self._curObjRef)
                 del childName
                 del child
                 continue
@@ -281,6 +421,8 @@ class ContainerLeakDetector(Job):
         # if it's an internal object, ignore it
         if id(obj) in ContainerLeakDetector.PrivateIds:
             return True
+        if id(obj) in self._id2ref:
+            return True
         return False
 
     def _isContainer(self, obj):
@@ -290,15 +432,15 @@ class ContainerLeakDetector(Job):
             return False
         return True
 
-    def _nameContainer(self, cont, pathStr):
+    def _nameContainer(self, cont, objRef):
         if self.notify.getDebug():
-            self.notify.debug('_nameContainer: %s' % pathStr)
+            self.notify.debug('_nameContainer: %s' % objRef)
-            printStack()
+            #printStack()
         contId = id(cont)
-        # if this container is new, or the pathStr is shorter than what we already have,
+        # if this container is new, or the objRef repr is shorter than what we already have,
         # put it in the table
-        if contId not in self._id2pathStr or len(pathStr) < len(self._id2pathStr[contId]):
+        if contId not in self._id2ref or len(repr(objRef)) < len(repr(self._id2ref[contId])):
-            self._id2pathStr[contId] = pathStr
+            self._id2ref[contId] = objRef
 
     def _checkForLeaks(self, task=None):
         self._index2delay[len(self._index2containerName2len)] = self._nextCheckDelay