TheCloud/panda3d
1
0
Fork 0
mirror of https://github.com/panda3d/panda3d.git synced 2025-10-02 18:03:56 -04:00
Code Issues Packages Projects Releases Wiki Activity

many bug fixes, almost completed

Browse Source
This commit is contained in:
Darren Ranalli 2007-04-06 11:14:07 +00:00
parent 5e7eeef6e2
commit 88950518ab
1 changed files with 223 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

307
direct/src/showbase/ContainerLeakDetector.py
View File

@ -2,7 +2,15 @@ from direct.directnotify.DirectNotifyGlobal import directNotify
from direct.showbase.PythonUtil import Queue, invertDictLossless
from direct.showbase.PythonUtil import itype, serialNum, safeRepr
from direct.showbase.Job import Job
import types, weakref, random, __builtin__
import types, weakref, gc, random, __builtin__
def _createContainerLeak():
def leakContainer(task):
if not hasattr(simbase, 'leakContainer'):
simbase.leakContainer = []
simbase.leakContainer.append(1)
return task.cont
taskMgr.add(leakContainer, 'leakContainer-%s' % serialNum())
class CheckContainers(Job):
"""
@ -13,13 +21,18 @@ class CheckContainers(Job):
self._leakDetector = leakDetector
self.notify = self._leakDetector.notify
self._index = index
ContainerLeakDetector.addPrivateId(self.__dict__)
def destroy(self):
ContainerLeakDetector.removePrivateId(self.__dict__)
Job.destroy(self)
def getPriority(self):
return Job.Priorities.Normal
def run(self):
self._leakDetector._index2containerName2len[self._index] = {}
self._leakDetector.notify.debug(repr(self._leakDetector._id2ref))
self._leakDetector._index2containerId2len[self._index] = {}
#self._leakDetector.notify.debug(repr(self._leakDetector._id2ref))
ids = self._leakDetector.getContainerIds()
# record the current len of each container
for id in ids:
@ -28,47 +41,89 @@ class CheckContainers(Job):
container = self._leakDetector.getContainerById(id)
except Exception, e:
# this container no longer exists
self.notify.debug('container %s no longer exists; caught exception in getContainerById (%s)' % (name, e))
self.notify.debug('container %s no longer exists; caught exception in getContainerById (%s)' % (
self._leakDetector.getContainerNameById(id), e))
self._leakDetector.removeContainerById(id)
continue
if container is None:
# this container no longer exists
self.notify.debug('container %s no longer exists; getContainerById returned None (%s)' % (name, e))
self.notify.debug('container %s no longer exists; getContainerById returned None' %
self._leakDetector.getContainerNameById(id))
self._leakDetector.removeContainerById(id)
continue
cLen = len(container)
name = self._leakDetector.getContainerNameById(id)
self._leakDetector._index2containerName2len[self._index][name] = cLen
try:
cLen = len(container)
except Exception, e:
# this container no longer exists
self.notify.debug('%s is no longer a container, it is now %s (%s)' %
(self._leakDetector.getContainerNameById(id), safeRepr(container), e))
self._leakDetector.removeContainerById(id)
continue
self._leakDetector._index2containerId2len[self._index][id] = cLen
# compare the current len of each container to past lens
if self._index > 0:
idx2name2len = self._leakDetector._index2containerName2len
for name in idx2name2len[self._index]:
idx2id2len = self._leakDetector._index2containerId2len
for id in idx2id2len[self._index]:
yield None
if name in idx2name2len[self._index-1]:
diff = idx2name2len[self._index][name] - idx2name2len[self._index-1][name]
if id in idx2id2len[self._index-1]:
diff = idx2id2len[self._index][id] - idx2id2len[self._index-1][id]
if diff > 0:
if diff > idx2name2len[self._index-1][name]:
if diff > idx2id2len[self._index-1][id]:
minutes = (self._leakDetector._index2delay[self._index] -
self._leakDetector._index2delay[self._index-1]) / 60.
name = self._leakDetector.getContainerNameById(id)
self.notify.warning('container %s grew > 200%% in %s minutes' % (name, minutes))
if self._index > 3:
diff2 = idx2name2len[self._index-1][name] - idx2name2len[self._index-2][name]
diff3 = idx2name2len[self._index-2][name] - idx2name2len[self._index-3][name]
if (self._index > 3 and
id in idx2id2len[self._index-2] and
id in idx2id2len[self._index-3]):
diff2 = idx2id2len[self._index-1][id] - idx2id2len[self._index-2][id]
diff3 = idx2id2len[self._index-2][id] - idx2id2len[self._index-3][id]
if self._index <= 5:
msg = ('%s consistently increased in length over the last 3 periods (currently %s items)' %
(name, idx2name2len[self._index][name]))
self.notify.warning(msg)
else:
if diff > 0 and diff2 > 0 and diff3 > 0:
name = self._leakDetector.getContainerNameById(id)
msg = ('%s consistently increased in length over the last 3 periods (currently %s items)' %
(name, idx2id2len[self._index][id]))
self.notify.warning(msg)
elif (id in idx2id2len[self._index-4] and
id in idx2id2len[self._index-5]):
# if size has consistently increased over the last 5 checks, send out a warning
diff4 = idx2name2len[self._index-3][name] - idx2name2len[self._index-4][name]
diff5 = idx2name2len[self._index-4][name] - idx2name2len[self._index-5][name]
diff4 = idx2id2len[self._index-3][id] - idx2id2len[self._index-4][id]
diff5 = idx2id2len[self._index-4][id] - idx2id2len[self._index-5][id]
if diff > 0 and diff2 > 0 and diff3 > 0 and diff4 > 0 and diff5 > 0:
name = self._leakDetector.getContainerNameById(id)
msg = ('%s consistently increased in length over the last 5 periods (currently %s items), notifying system' %
(name, idx2name2len[self._index][name]))
(name, idx2id2len[self._index][id]))
self.notify.warning(msg)
messenger.send(self._leakDetector.getLeakEvent(), [msg])
yield Job.Done
class PruneContainerRefs(Job):
"""
Job to destroy any container refs that have Indirections that are holding references
to objects that should be garbage-collected
"""
def __init__(self, name, leakDetector):
Job.__init__(self, name)
self._leakDetector = leakDetector
self.notify = self._leakDetector.notify
self._index = index
ContainerLeakDetector.addPrivateId(self.__dict__)
def destroy(self):
ContainerLeakDetector.removePrivateId(self.__dict__)
Job.destroy(self)
def getPriority(self):
return Job.Priorities.Normal
def run(self):
ids = self._leakDetector._id2ref.keys()
for id in ids:
yield None
ref = self._leakDetector._id2ref[id]
ref.destroyIfGarbageDictKey()
yield Job.Done
class NoDictKey:
pass
@ -76,13 +131,14 @@ class Indirection:
# represents the indirection that brings you from a container to an element of the container
# stored as a string to be used as part of an eval
# each dictionary dereference is individually eval'd since the dict key might have been garbage-collected
class GarbageCollectedDictKey(Exception):
pass
def __init__(self, evalStr=None, dictKey=NoDictKey):
# if this is a dictionary lookup, pass dictKey instead of evalStr
self.evalStr = evalStr
self.dictKey = NoDictKey
# is the dictKey a weak reference?
self._isWeakRef = False
self._refCount = 0
if dictKey is not NoDictKey:
# if we can repr/eval the key, store it as an evalStr
keyRepr = repr(dictKey)
@ -100,34 +156,74 @@ class Indirection:
# eval/repr succeeded, store as an evalStr
self.evalStr = '[%s]' % keyRepr
else:
# store a weakref to the key
self.dictKey = weakref.ref(dictKey)
try:
# store a weakref to the key
self.dictKey = weakref.ref(dictKey)
self._isWeakRef = True
except TypeError, e:
self.dictKey = dictKey
self._isWeakRef = False
def destroy(self):
del self.dictKey
def acquire(self):
self._refCount += 1
def release(self):
self._refCount -= 1
if self._refCount == 0:
self.destroy()
def isDictKey(self):
# is this an indirection through a dictionary?
return self.dictKey is not NoDictKey
def isGarbageDictKey(self):
# are we holding a non-weak reference to an object that should be
# garbage-collected?
if self.isDictKey() and not self._isWeakRef:
referrers = gc.get_referrers(self.dictKey)
print referrers
import pdb;pdb.set_trace()
def _getNonWeakDictKey(self):
if not self._isWeakRef:
return self.dictKey
else:
key = self.dictKey()
if key is None:
return '<garbage-collected dict key>'
return key
def dereferenceDictKey(self, parentDict):
key = self.dictKey()
if key is None:
raise Indirection.GarbageCollectedDictKey()
# look ourselves up in parentDict
key = self._getNonWeakDictKey()
# objects in __builtin__ will have parentDict==None
if parentDict is None:
return key
return parentDict[key]
def getString(self, nextIndirection=None):
# return our contribution to the name of the object
def getString(self, prevIndirection=None, nextIndirection=None):
# return our contribution to the full name of an object
instanceDictStr = '.__dict__'
if self.evalStr is not None:
# if we're an instance dict and the next indirection is not a dict key,
# skip over this one (obj.__dict__[keyName] == obj.keyName)
if nextIndirection is not None and self.evalStr == '.__dict__':
return ''
# if we're an instance dict, skip over this one (obj.__dict__[keyName] == obj.keyName)
if nextIndirection is not None and self.evalStr[-len(instanceDictStr):] == instanceDictStr:
return self.evalStr[:-len(instanceDictStr)]
# if the previous indirection was an instance dict, change our syntax from ['key'] to .key
if prevIndirection is not None and prevIndirection.evalStr is not None:
if prevIndirection.evalStr[-len(instanceDictStr):] == instanceDictStr:
return '.%s' % self.evalStr[2:-2]
return self.evalStr
# we're stored as a dict key
# this might not eval, but that's OK, we're not using this string to find
# the object, we dereference the parent dict
key = self.dictKey()
if key is None:
return '<garbage-collected dict key>'
return safeRepr(key)
keyRepr = safeRepr(self._getNonWeakDictKey())
# if the previous indirection was an instance dict, change our syntax from ['key'] to .key
if prevIndirection is not None and prevIndirection.evalStr is not None:
if prevIndirection.evalStr[-len(instanceDictStr):] == instanceDictStr:
return '.%s' % keyRepr
return '[%s]' % keyRepr
def __repr__(self):
return self.getString()
@ -140,21 +236,30 @@ class ContainerRef:
"""
class FailedEval(Exception):
pass
# whatever this is set to will be the default ContainerRef
BaseRef = None
def __init__(self, other=None, indirection=None):
def __init__(self, indirection, other=None):
self._indirections = []
# if no other passed in, try ContainerRef.BaseRef
if other is None:
other = ContainerRef.BaseRef
if other is not None:
for ind in other._indirections:
self.addIndirection(ind)
if indirection:
self.addIndirection(indirection)
self.addIndirection(indirection)
def destroy(self):
for indirection in self._indirections:
indirection.release()
del self._indirections
def destroyIfGarbageDictKey(self):
# if any of our indirections are holding onto objects that
# should be garbage-collected, destroy
for indirection in self._indirections:
if indirection.isGarbageDictKey():
self.destroy()
return
def addIndirection(self, indirection):
indirection.acquire()
self._indirections.append(indirection)
def _getContainerByEval(self, evalStr):
@ -177,8 +282,6 @@ class ContainerRef:
#import pdb;pdb.set_trace()
evalStr = ''
curObj = None
curIndirection = None
nextIndirection = None
for indirection in self._indirections:
if not indirection.isDictKey():
# build up a string to be eval'd
@ -195,15 +298,21 @@ class ContainerRef:
def __repr__(self):
str = ''
prevIndirection = None
curIndirection = None
nextIndirection = None
for i in xrange(len(self._indirections)):
if i > 0:
prevIndirection = self._indirections[i-1]
else:
prevIndirection = None
curIndirection = self._indirections[i]
if i < len(self._indirections)-1:
nextIndirection = self._indirections[i+1]
else:
nextIndirection = None
str += curIndirection.getString(nextIndirection=nextIndirection)
str += curIndirection.getString(prevIndirection=prevIndirection,
nextIndirection=nextIndirection)
return str
class ContainerLeakDetector(Job):
@ -226,33 +335,39 @@ class ContainerLeakDetector(Job):
if firstCheckDelay is None:
firstCheckDelay = 60. * 15.
self._nextCheckDelay = firstCheckDelay
self._index2containerName2len = {}
self._pruneTaskPeriod = config.GetFloat('leak-detector-prune-period', 60. * 30.)
self._index2containerId2len = {}
self._index2delay = {}
# set up our data structures
self._id2ref = {}
# set up the base/starting object
self._nameContainer(__builtin__.__dict__, ContainerRef(indirection=Indirection(evalStr='__builtin__.__dict__')))
self._baseObjRef = ContainerRef(Indirection(evalStr='__builtin__.__dict__'))
self._nameContainer(__builtin__.__dict__, self._baseObjRef)
try:
base
except:
pass
else:
ContainerRef.BaseRef = ContainerRef(indirection=Indirection(evalStr='base.__dict__'))
self._nameContainer(base.__dict__, ContainerRef.BaseRef)
self._baseObjRef = ContainerRef(Indirection(evalStr='base.__dict__'))
self._nameContainer(base.__dict__, self._baseObjRef)
try:
simbase
except:
pass
else:
ContainerRef.BaseRef = ContainerRef(indirection=Indirection(evalStr='simbase.__dict__'))
self._nameContainer(simbase.__dict__, ContainerRef.BaseRef)
self._baseObjRef = ContainerRef(Indirection(evalStr='simbase.__dict__'))
self._nameContainer(simbase.__dict__, self._baseObjRef)
if config.GetBool('leak-container', 0):
_createContainerLeak()
self._curObjRef = self._baseObjRef
self._curObjRef = ContainerRef()
jobMgr.add(self)
ContainerLeakDetector.PrivateIds.update(set([
id(ContainerLeakDetector.PrivateIds),
id(self._id2ref),
id(self.__dict__),
]))
def destroy(self):
@ -260,14 +375,23 @@ class ContainerLeakDetector(Job):
jobMgr.remove(self._checkContainersJob)
self._checkContainersJob = None
del self._id2ref
del self._index2containerName2len
del self._index2containerId2len
del self._index2delay
def getPriority(self):
return self._priority
def getCheckTaskName(self):
@classmethod
def addPrivateId(cls, obj):
cls.PrivateIds.add(id(obj))
@classmethod
def removePrivateId(cls, obj):
cls.PrivateIds.remove(id(obj))
def _getCheckTaskName(self):
return 'checkForLeakingContainers-%s' % self._serialNum
def _getPruneTaskName(self):
return 'pruneLeakingContainerRefs-%s' % self._serialNum
def getLeakEvent(self):
# passes description string as argument
@ -285,7 +409,9 @@ class ContainerLeakDetector(Job):
def run(self):
taskMgr.doMethodLater(self._nextCheckDelay, self._checkForLeaks,
self.getCheckTaskName())
self._getCheckTaskName())
taskMgr.doMethodLater(self._pruneTaskPeriod, self._pruneContainerRefs,
self._getPruneTaskName())
while True:
# yield up here instead of at the end, since we skip back to the
@ -294,7 +420,7 @@ class ContainerLeakDetector(Job):
#import pdb;pdb.set_trace()
curObj = None
if self._curObjRef is None:
self._curObjRef = random.choice(self._id2ref.values())
self._curObjRef = self._baseObjRef
try:
curObj = self._curObjRef.getContainer()
except:
@ -308,26 +434,17 @@ class ContainerLeakDetector(Job):
del self._id2ref[_id]
self._curObjRef = self._id2ref[_id]
#print '%s: %s, %s' % (id(curObj), type(curObj), self._id2ref[id(curObj)])
self.notify.debug('--> %s' % self._curObjRef)
#self.notify.debug('--> %s' % self._curObjRef)
# keep a copy of this obj's eval str, it might not be in _id2ref
curObjRef = self._curObjRef
# if we hit a dead end, start over at a container we know about
self._curObjRef = None
try:
if curObj.__class__.__name__ == 'method-wrapper':
continue
except:
pass
if type(curObj) in (types.StringType, types.UnicodeType):
continue
if type(curObj) in (types.ModuleType, types.InstanceType):
child = curObj.__dict__
if not self._isDeadEnd(child):
self._curObjRef = ContainerRef(curObjRef, indirection=Indirection(evalStr='.__dict__'))
self._curObjRef = ContainerRef(Indirection(evalStr='.__dict__'), curObjRef)
if self._isContainer(child):
self._nameContainer(child, self._curObjRef)
continue
@ -345,12 +462,13 @@ class ContainerLeakDetector(Job):
except KeyError, e:
self.notify.warning('could not index into %s with key %s' % (curObjRef, key))
continue
if not self._isDeadEnd(attr):
if not self._isDeadEnd(attr, key):
if curObj is __builtin__.__dict__:
indirection=Indirection(evalStr=key)
self._curObjRef = ContainerRef(indirection)
else:
indirection=Indirection(dictKey=key)
self._curObjRef = ContainerRef(curObjRef, indirection=indirection)
self._curObjRef = ContainerRef(indirection, curObjRef)
if self._isContainer(attr):
self._nameContainer(attr, self._curObjRef)
del key
@ -379,7 +497,7 @@ class ContainerLeakDetector(Job):
random.shuffle(attrs)
for attr in attrs:
if not self._isDeadEnd(attr):
self._curObjRef = ContainerRef(curObjRef, indirection=Indirection(evalStr='[%s]' % index))
self._curObjRef = ContainerRef(Indirection(evalStr='[%s]' % index), curObjRef)
if self._isContainer(attr):
self._nameContainer(attr, self._curObjRef)
index += 1
@ -401,8 +519,8 @@ class ContainerLeakDetector(Job):
random.shuffle(childNames)
for childName in childNames:
child = getattr(curObj, childName)
if not self._isDeadEnd(child):
self._curObjRef = ContainerRef(curObjRef, indirection=Indirection(evalStr='.%s' % childName))
if not self._isDeadEnd(child, childName):
self._curObjRef = ContainerRef(Indirection(evalStr='.%s' % childName), curObjRef)
if self._isContainer(child):
self._nameContainer(child, self._curObjRef)
del childName
@ -411,19 +529,31 @@ class ContainerLeakDetector(Job):
yield Job.Done
def _isDeadEnd(self, obj):
def _isDeadEnd(self, obj, objName=None):
if type(obj) in (types.BooleanType, types.BuiltinFunctionType,
types.BuiltinMethodType, types.ComplexType,
types.FloatType, types.IntType, types.LongType,
types.NoneType, types.NotImplementedType,
types.TypeType, types.CodeType, types.FunctionType):
types.TypeType, types.CodeType, types.FunctionType,
types.StringType, types.UnicodeType):
return True
if id(obj) in self._id2ref:
return True
# if it's an internal object, ignore it
if id(obj) in ContainerLeakDetector.PrivateIds:
return True
if id(obj) in self._id2ref:
if objName in ('im_self', 'im_class'):
return True
try:
className = obj.__class__.__name__
except:
pass
else:
# prevent infinite recursion in built-in containers related to methods
if className == 'method-wrapper':
return True
return False
def _isContainer(self, obj):
try:
@ -433,9 +563,11 @@ class ContainerLeakDetector(Job):
return True
def _nameContainer(self, cont, objRef):
"""
if self.notify.getDebug():
self.notify.debug('_nameContainer: %s' % objRef)
#printStack()
"""
contId = id(cont)
# if this container is new, or the objRef repr is shorter than what we already have,
# put it in the table
@ -443,11 +575,18 @@ class ContainerLeakDetector(Job):
self._id2ref[contId] = objRef
def _checkForLeaks(self, task=None):
self._index2delay[len(self._index2containerName2len)] = self._nextCheckDelay
self._index2delay[len(self._index2containerId2len)] = self._nextCheckDelay
self._checkContainersJob = CheckContainers(
'%s-checkForLeaks' % self.getJobName(), self, len(self._index2containerName2len))
'%s-checkForLeaks' % self.getJobName(), self, len(self._index2containerId2len))
jobMgr.add(self._checkContainersJob)
self._nextCheckDelay *= 2
taskMgr.doMethodLater(self._nextCheckDelay, self._checkForLeaks,
self.getCheckTaskName())
self._getCheckTaskName())
return task.done
def _pruneContainerRefs(self, task=None):
taskMgr.doMethodLater(self._pruneTaskPeriod, self._pruneContainerRefs,
self._getPruneTaskName())
return task.done