TheCloud/panda3d
1
0
Fork 0
mirror of https://github.com/panda3d/panda3d.git synced 2025-10-03 10:22:45 -04:00
Code Issues Packages Projects Releases Wiki Activity

reduce end-of-phase chugs

Browse Source
This commit is contained in:
David Rose 2010-02-09 19:51:20 +00:00
parent 381f89d8ea
commit 7c3bd0bd8d
3 changed files with 67 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
direct/src/p3d/PackageInfo.py
View File

@ -1,4 +1,4 @@
from pandac.PandaModules import Filename, URLSpec, DocumentSpec, Ramfile, Multifile, Decompressor, EUOk, EUSuccess, VirtualFileSystem, Thread, getModelPath, ExecutionEnvironment from pandac.PandaModules import Filename, URLSpec, DocumentSpec, Ramfile, Multifile, Decompressor, EUOk, EUSuccess, VirtualFileSystem, Thread, getModelPath, ExecutionEnvironment, PStatCollector
from pandac import PandaModules from pandac import PandaModules
from direct.p3d.FileSpec import FileSpec from direct.p3d.FileSpec import FileSpec
from direct.showbase import VFSImporter from direct.showbase import VFSImporter
@ -37,11 +37,29 @@ class PackageInfo:
represents a single atomic piece of the installation step, and represents a single atomic piece of the installation step, and
the relative effort of that piece. When the plan is executed, the relative effort of that piece. When the plan is executed,
it will call the saved function pointer here. """ it will call the saved function pointer here. """
def __init__(self, func, bytes, factor): def __init__(self, func, bytes, factor, stepType):
self.func = func self.__funcPtr = func
self.bytesNeeded = bytes self.bytesNeeded = bytes
self.bytesDone = 0 self.bytesDone = 0
self.bytesFactor = factor self.bytesFactor = factor
self.stepType = stepType
self.pStatCol = PStatCollector(':App:PackageInstaller:%s' % (stepType))
def func(self):
""" self.__funcPtr(self) will return a generator of
tokens. This function defines a new generator that yields
each of those tokens, but wraps each call into the nested
generator within a pair of start/stop collector calls. """
self.pStatCol.start()
for token in self.__funcPtr(self):
self.pStatCol.stop()
yield token
self.pStatCol.start()
# Shouldn't ever get here.
self.pStatCol.stop()
raise StopIteration
def getEffort(self): def getEffort(self):
""" Returns the relative amount of effort of this step. """ """ Returns the relative amount of effort of this step. """
@ -215,10 +233,13 @@ class PackageInfo:
self.http = http self.http = http
for token in self.__downloadFile( func = lambda step, self = self: self.__downloadFile(
None, self.descFile, None, self.descFile,
urlbase = self.descFile.filename, urlbase = self.descFile.filename,
filename = self.descFileBasename): filename = self.descFileBasename)
step = self.InstallStep(func, self.descFile.size, self.downloadFactor, 'downloadDesc')
for token in step.func():
if token == self.stepContinue: if token == self.stepContinue:
yield token yield token
else: else:
@ -226,10 +247,12 @@ class PackageInfo:
while token == self.restartDownload: while token == self.restartDownload:
# Try again. # Try again.
for token in self.__downloadFile( func = lambda step, self = self: self.__downloadFile(
None, self.descFile, None, self.descFile,
urlbase = self.descFile.filename, urlbase = self.descFile.filename,
filename = self.descFileBasename): filename = self.descFileBasename)
step = self.InstallStep(func, self.descFile.size, self.downloadFactor, 'downloadDesc')
for token in step.func():
if token == self.stepContinue: if token == self.stepContinue:
yield token yield token
else: else:
@ -350,6 +373,9 @@ class PackageInfo:
""" Sets up self.installPlans, a list of one or more "plans" """ Sets up self.installPlans, a list of one or more "plans"
to download and install the package. """ to download and install the package. """
pc = PStatCollector(':App:PackageInstaller:buildInstallPlans')
pc.start()
self.hasPackage = False self.hasPackage = False
if self.asMirror: if self.asMirror:
@ -361,9 +387,10 @@ class PackageInfo:
downloadSize = self.compressedArchive.size downloadSize = self.compressedArchive.size
func = lambda step, fileSpec = self.compressedArchive: self.__downloadFile(step, fileSpec, allowPartial = True) func = lambda step, fileSpec = self.compressedArchive: self.__downloadFile(step, fileSpec, allowPartial = True)
step = self.InstallStep(func, downloadSize, self.downloadFactor) step = self.InstallStep(func, downloadSize, self.downloadFactor, 'download')
installPlan = [step] installPlan = [step]
self.installPlans = [installPlan] self.installPlans = [installPlan]
pc.stop()
return return
# The normal download process. Determine what we will need to # The normal download process. Determine what we will need to
@ -375,7 +402,7 @@ class PackageInfo:
unpackSize = 0 unpackSize = 0
for file in self.extracts: for file in self.extracts:
unpackSize += file.size unpackSize += file.size
step = self.InstallStep(self.__unpackArchive, unpackSize, self.unpackFactor) step = self.InstallStep(self.__unpackArchive, unpackSize, self.unpackFactor, 'unpack')
planA = [step] planA = [step]
# If the uncompressed archive file is good, that's all we'll # If the uncompressed archive file is good, that's all we'll
@ -383,14 +410,16 @@ class PackageInfo:
self.uncompressedArchive.actualFile = None self.uncompressedArchive.actualFile = None
if self.uncompressedArchive.quickVerify(self.getPackageDir(), notify = self.notify): if self.uncompressedArchive.quickVerify(self.getPackageDir(), notify = self.notify):
self.installPlans = [planA] self.installPlans = [planA]
pc.stop()
return return
# Maybe the compressed archive file is good. # Maybe the compressed archive file is good.
if self.compressedArchive.quickVerify(self.getPackageDir(), notify = self.notify): if self.compressedArchive.quickVerify(self.getPackageDir(), notify = self.notify):
uncompressSize = self.uncompressedArchive.size uncompressSize = self.uncompressedArchive.size
step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor) step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor, 'uncompress')
planA = [step] + planA planA = [step] + planA
self.installPlans = [planA] self.installPlans = [planA]
pc.stop()
return return
# Maybe we can download one or more patches. We'll come back # Maybe we can download one or more patches. We'll come back
@ -399,13 +428,13 @@ class PackageInfo:
planB = planA[:] planB = planA[:]
uncompressSize = self.uncompressedArchive.size uncompressSize = self.uncompressedArchive.size
step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor) step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor, 'uncompress')
planB = [step] + planB planB = [step] + planB
downloadSize = self.compressedArchive.size downloadSize = self.compressedArchive.size
func = lambda step, fileSpec = self.compressedArchive: self.__downloadFile(step, fileSpec, allowPartial = True) func = lambda step, fileSpec = self.compressedArchive: self.__downloadFile(step, fileSpec, allowPartial = True)
step = self.InstallStep(func, downloadSize, self.downloadFactor) step = self.InstallStep(func, downloadSize, self.downloadFactor, 'download')
planB = [step] + planB planB = [step] + planB
# Now look for patches. Start with the md5 hash from the # Now look for patches. Start with the md5 hash from the
@ -430,6 +459,8 @@ class PackageInfo:
# plan B as the only plan. # plan B as the only plan.
self.installPlans = [planB] self.installPlans = [planB]
pc.stop()
def __scanDirectoryRecursively(self, dirname): def __scanDirectoryRecursively(self, dirname):
""" Generates a list of Filename objects: all of the files """ Generates a list of Filename objects: all of the files
(not directories) within and below the indicated dirname. """ (not directories) within and below the indicated dirname. """
@ -589,7 +620,7 @@ class PackageInfo:
for step in plan: for step in plan:
self.currentStepEffort = step.getEffort() self.currentStepEffort = step.getEffort()
for token in step.func(step): for token in step.func():
if token == self.stepContinue: if token == self.stepContinue:
yield token yield token
else: else:
@ -633,12 +664,12 @@ class PackageInfo:
for patchfile in patchChain: for patchfile in patchChain:
downloadSize = patchfile.file.size downloadSize = patchfile.file.size
func = lambda step, fileSpec = patchfile.file: self.__downloadFile(step, fileSpec, allowPartial = True) func = lambda step, fileSpec = patchfile.file: self.__downloadFile(step, fileSpec, allowPartial = True)
step = self.InstallStep(func, downloadSize, self.downloadFactor) step = self.InstallStep(func, downloadSize, self.downloadFactor, 'download')
plan.append(step) plan.append(step)
patchSize = patchfile.targetFile.size patchSize = patchfile.targetFile.size
func = lambda step, patchfile = patchfile: self.__applyPatch(step, patchfile) func = lambda step, patchfile = patchfile: self.__applyPatch(step, patchfile)
step = self.InstallStep(func, patchSize, self.patchFactor) step = self.InstallStep(func, patchSize, self.patchFactor, 'patch')
plan.append(step) plan.append(step)
patchMaker.cleanup() patchMaker.cleanup()

26
direct/src/p3d/PackageInstaller.py
View File

@ -3,7 +3,7 @@ from direct.stdpy.threading import Lock, RLock
from direct.showbase.MessengerGlobal import messenger from direct.showbase.MessengerGlobal import messenger
from direct.task.TaskManagerGlobal import taskMgr from direct.task.TaskManagerGlobal import taskMgr
from direct.p3d.PackageInfo import PackageInfo from direct.p3d.PackageInfo import PackageInfo
from pandac.PandaModules import TPLow from pandac.PandaModules import TPLow, PStatCollector
from direct.directnotify.DirectNotifyGlobal import directNotify from direct.directnotify.DirectNotifyGlobal import directNotify
class PackageInstaller(DirectObject): class PackageInstaller(DirectObject):
@ -495,8 +495,10 @@ class PackageInstaller(DirectObject):
# files, stop the task. # files, stop the task.
if not self.needsDescFile: if not self.needsDescFile:
self.descFileTask = None self.descFileTask = None
messenger.send('PackageInstaller-%s-allHaveDesc' % self.uniqueId,
taskChain = 'default') eventName = 'PackageInstaller-%s-allHaveDesc' % self.uniqueId
messenger.send(eventName, taskChain = 'default')
return task.done return task.done
pp = self.needsDescFile[0] pp = self.needsDescFile[0]
del self.needsDescFile[0] del self.needsDescFile[0]
@ -545,11 +547,12 @@ class PackageInstaller(DirectObject):
del self.needsDownload[0] del self.needsDownload[0]
except: except:
self.packageLock.release() self.packageLock.release()
raise
self.packageLock.release() self.packageLock.release()
# Now serve this one package. # Now serve this one package.
messenger.send('PackageInstaller-%s-packageStarted' % self.uniqueId, eventName = 'PackageInstaller-%s-packageStarted' % self.uniqueId
[pp], taskChain = 'default') messenger.send(eventName, [pp], taskChain = 'default')
if not pp.package.hasPackage: if not pp.package.hasPackage:
for token in pp.package.downloadPackageGenerator(self.appRunner.http): for token in pp.package.downloadPackageGenerator(self.appRunner.http):
@ -559,12 +562,18 @@ class PackageInstaller(DirectObject):
break break
if token != pp.package.stepComplete: if token != pp.package.stepComplete:
pc = PStatCollector(':App:PackageInstaller:donePackage:%s' % (pp.package.packageName))
pc.start()
self.__donePackage(pp, False) self.__donePackage(pp, False)
pc.stop()
yield task.cont yield task.cont
continue continue
# Successfully downloaded and installed. # Successfully downloaded and installed.
pc = PStatCollector(':App:PackageInstaller:donePackage:%s' % (pp.package.packageName))
pc.start()
self.__donePackage(pp, True) self.__donePackage(pp, True)
pc.stop()
# Continue the loop without yielding, so we pick up the # Continue the loop without yielding, so we pick up the
# next package within this same frame. # next package within this same frame.
@ -575,7 +584,10 @@ class PackageInstaller(DirectObject):
assert not pp.done assert not pp.done
if success: if success:
pc = PStatCollector(':App:PackageInstaller:install:%s' % (pp.package.packageName))
pc.start()
pp.package.installPackage(self.appRunner) pp.package.installPackage(self.appRunner)
pc.stop()
self.packageLock.acquire() self.packageLock.acquire()
try: try:
@ -588,8 +600,8 @@ class PackageInstaller(DirectObject):
finally: finally:
self.packageLock.release() self.packageLock.release()
messenger.send('PackageInstaller-%s-packageDone' % self.uniqueId, eventName = 'PackageInstaller-%s-packageDone' % self.uniqueId
[pp], taskChain = 'default') messenger.send(eventName, [pp], taskChain = 'default')
def __progressTask(self, task): def __progressTask(self, task):
self.callbackLock.acquire() self.callbackLock.acquire()

4
panda/src/downloader/config_downloader.cxx
View File

@ -49,12 +49,12 @@ ConfigVariableInt downloader_timeout_retries
("downloader-timeout-retries", 5); ("downloader-timeout-retries", 5);
ConfigVariableDouble decompressor_step_time ConfigVariableDouble decompressor_step_time
("decompressor-step-time", 0.1, ("decompressor-step-time", 0.005,
PRC_DESC("Specifies the maximum amount of time that should be consumed by " PRC_DESC("Specifies the maximum amount of time that should be consumed by "
"a single call to Decompressor::run().")); "a single call to Decompressor::run()."));
ConfigVariableDouble extractor_step_time ConfigVariableDouble extractor_step_time
("extractor-step-time", 0.1, ("extractor-step-time", 0.005,
PRC_DESC("Specifies the maximum amount of time that should be consumed by " PRC_DESC("Specifies the maximum amount of time that should be consumed by "
"a single call to Extractor::step().")); "a single call to Extractor::step()."));