From 50b9e51aa61bde41457201927b38ca7799cd98b0 Mon Sep 17 00:00:00 2001 From: David Rose Date: Mon, 28 Dec 2009 21:16:56 +0000 Subject: [PATCH] fix shutdown while downloading --- direct/src/p3d/AppRunner.py | 4 +- direct/src/p3d/FileSpec.py | 37 +++++++++----- direct/src/p3d/HostInfo.py | 19 ++++--- direct/src/p3d/PackageInfo.py | 96 +++++++++++++++++++++++------------ direct/src/task/Task.py | 4 +- 5 files changed, 105 insertions(+), 55 deletions(-) diff --git a/direct/src/p3d/AppRunner.py b/direct/src/p3d/AppRunner.py index 20af581f40..06f3c90fc9 100644 --- a/direct/src/p3d/AppRunner.py +++ b/direct/src/p3d/AppRunner.py @@ -335,7 +335,7 @@ class AppRunner(DirectObject): host = self.hosts.get(hostUrl, None) if not host: - host = HostInfo(hostUrl, self) + host = HostInfo(hostUrl, appRunner = self) self.hosts[hostUrl] = host return host @@ -541,7 +541,7 @@ class AppRunner(DirectObject): host = self.getHost(hostUrl) if hostDir and not host.hostDir: - host.hostDir = hostDir + host.hostDir = Filename.fromOsSpecific(hostDir) if not host.readContentsFile(): if not host.downloadContentsFile(self.http): diff --git a/direct/src/p3d/FileSpec.py b/direct/src/p3d/FileSpec.py index 4aa19eeba2..4c12e77fb2 100644 --- a/direct/src/p3d/FileSpec.py +++ b/direct/src/p3d/FileSpec.py @@ -77,7 +77,8 @@ class FileSpec: if self.hash: xelement.SetAttribute('hash', self.hash) - def quickVerify(self, packageDir = None, pathname = None): + def quickVerify(self, packageDir = None, pathname = None, + notify = None): """ Performs a quick test to ensure the file has not been modified. This test is vulnerable to people maliciously attempting to fool the program (by setting datestamps etc.). @@ -91,30 +92,37 @@ class FileSpec: st = os.stat(pathname.toOsSpecific()) except OSError: # If the file is missing, the file fails. - #print "file not found: %s" % (pathname) + if notify: + notify.debug("file not found: %s" % (pathname)) return False if st.st_size != self.size: # If the size is wrong, the file fails. - #print "size wrong: %s" % (pathname) + if notify: + notify.debug("size wrong: %s" % (pathname)) return False if st.st_mtime == self.timestamp: # If the size is right and the timestamp is right, the # file passes. - #print "file ok: %s" % (pathname) + if notify: + notify.debug("file ok: %s" % (pathname)) return True - #print "modification time wrong: %s" % (pathname) + if notify: + notify.debug("modification time wrong: %s" % (pathname)) # If the size is right but the timestamp is wrong, the file # soft-fails. We follow this up with a hash check. if not self.checkHash(packageDir, pathname, st): # Hard fail, the hash is wrong. - #print "hash check wrong: %s" % (pathname) + if notify: + notify.debug("hash check wrong: %s" % (pathname)) + notify.debug(" found %s, expected %s" % (self.actualFile.hash, self.hash)) return False - #print "hash check ok: %s" % (pathname) + if notify: + notify.debug("hash check ok: %s" % (pathname)) # The hash is OK after all. Change the file's timestamp back # to what we expect it to be, so we can quick-verify it @@ -124,7 +132,7 @@ class FileSpec: return True - def fullVerify(self, packageDir = None, pathname = None): + def fullVerify(self, packageDir = None, pathname = None, notify = None): """ Performs a more thorough test to ensure the file has not been modified. This test is less vulnerable to malicious attacks, since it reads and verifies the entire file. @@ -138,20 +146,25 @@ class FileSpec: st = os.stat(pathname.toOsSpecific()) except OSError: # If the file is missing, the file fails. - #print "file not found: %s" % (pathname) + if notify: + notify.debug("file not found: %s" % (pathname)) return False if st.st_size != self.size: # If the size is wrong, the file fails; - #print "size wrong: %s" % (pathname) + if notify: + notify.debug("size wrong: %s" % (pathname)) return False if not self.checkHash(packageDir, pathname, st): # Hard fail, the hash is wrong. - #print "hash check wrong: %s" % (pathname) + if notify: + notify.debug("hash check wrong: %s" % (pathname)) + notify.debug(" found %s, expected %s" % (self.actualFile.hash, self.hash)) return False - #print "hash check ok: %s" % (pathname) + if notify: + notify.debug("hash check ok: %s" % (pathname)) # The hash is OK. If the timestamp is wrong, change it back # to what we expect it to be, so we can quick-verify it diff --git a/direct/src/p3d/HostInfo.py b/direct/src/p3d/HostInfo.py index 6c41b85473..a4fa0358f8 100644 --- a/direct/src/p3d/HostInfo.py +++ b/direct/src/p3d/HostInfo.py @@ -2,6 +2,7 @@ from pandac.PandaModules import HashVal, Filename, PandaSystem, DocumentSpec, Ra from pandac import PandaModules from direct.p3d.PackageInfo import PackageInfo from direct.p3d.FileSpec import FileSpec +from direct.directnotify.DirectNotifyGlobal import directNotify import time class HostInfo: @@ -9,6 +10,8 @@ class HostInfo: Panda3D packages. It is the Python equivalent of the P3DHost class in the core API. """ + notify = directNotify.newCategory("HostInfo") + def __init__(self, hostUrl, appRunner = None, hostDir = None, rootDir = None, asMirror = False, perPlatform = None): @@ -94,13 +97,13 @@ class HostInfo: # We start with the "super mirror", if it's defined. url = self.appRunner.superMirrorUrl + 'contents.xml' request = DocumentSpec(url) - print "Downloading contents file %s" % (request) + self.notify.info("Downloading contents file %s" % (request)) rf = Ramfile() channel = http.makeChannel(False) channel.getDocument(request) if not channel.downloadToRam(rf): - print "Unable to download %s" % (url) + self.notify.warning("Unable to download %s" % (url)) rf = None if not rf: @@ -118,13 +121,13 @@ class HostInfo: request = DocumentSpec(url) request.setCacheControl(DocumentSpec.CCNoCache) - print "Downloading contents file %s" % (request) + self.notify.info("Downloading contents file %s" % (request)) rf = Ramfile() channel = http.makeChannel(False) channel.getDocument(request) if not channel.downloadToRam(rf): - print "Unable to download %s" % (url) + self.notify.warning("Unable to download %s" % (url)) rf = None tempFilename = Filename.temporary('', 'p3d_', '.xml') @@ -134,7 +137,7 @@ class HostInfo: f.close() if not self.readContentsFile(tempFilename): - print "Failure reading %s" % (url) + self.notify.warning("Failure reading %s" % (url)) tempFilename.unlink() return False @@ -151,7 +154,7 @@ class HostInfo: assert self.hasContentsFile url = self.hostUrlPrefix + 'contents.xml' - print "Redownloading %s" % (url) + self.notify.info("Redownloading %s" % (url)) # Get the hash of the original file. assert self.hostDir @@ -168,10 +171,10 @@ class HostInfo: hv2.hashFile(filename) if hv1 != hv2: - print "%s has changed." % (url) + self.notify.info("%s has changed." % (url)) return True else: - print "%s has not changed." % (url) + self.notify.info("%s has not changed." % (url)) return False diff --git a/direct/src/p3d/PackageInfo.py b/direct/src/p3d/PackageInfo.py index d26c043951..85ac4d3f36 100644 --- a/direct/src/p3d/PackageInfo.py +++ b/direct/src/p3d/PackageInfo.py @@ -2,6 +2,8 @@ from pandac.PandaModules import Filename, URLSpec, DocumentSpec, Ramfile, Multif from pandac import PandaModules from direct.p3d.FileSpec import FileSpec from direct.showbase import VFSImporter +from direct.directnotify.DirectNotifyGlobal import directNotify +from direct.task.TaskManagerGlobal import taskMgr import os import sys import random @@ -13,6 +15,8 @@ class PackageInfo: can be (or has been) installed into the current runtime. It is the Python equivalent of the P3DPackage class in the core API. """ + notify = directNotify.newCategory("PackageInfo") + # Weight factors for computing download progress. This # attempts to reflect the relative time-per-byte of each of # these operations. @@ -171,7 +175,7 @@ class PackageInfo: if not self.hasDescFile: filename = Filename(self.getPackageDir(), self.descFileBasename) - if self.descFile.quickVerify(self.getPackageDir(), pathname = filename): + if self.descFile.quickVerify(self.getPackageDir(), pathname = filename, notify = self.notify): if self.__readDescFile(): # Successfully read. We don't need to call # checkArchiveStatus again, since readDescFile() @@ -223,7 +227,7 @@ class PackageInfo: if not self.__readDescFile(): # Weird, it passed the hash check, but we still can't read # it. - print "Failure reading %s" % (filename) + self.notify.warning("Failure reading %s" % (filename)) return False return True @@ -356,12 +360,12 @@ class PackageInfo: # If the uncompressed archive file is good, that's all we'll # need to do. self.uncompressedArchive.actualFile = None - if self.uncompressedArchive.quickVerify(self.getPackageDir()): + if self.uncompressedArchive.quickVerify(self.getPackageDir(), notify = self.notify): self.installPlans = [planA] return # Maybe the compressed archive file is good. - if self.compressedArchive.quickVerify(self.getPackageDir()): + if self.compressedArchive.quickVerify(self.getPackageDir(), notify = self.notify): uncompressSize = self.uncompressedArchive.size step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor) planA = [step] + planA @@ -448,16 +452,16 @@ class PackageInfo: # case there is a problem with ambiguous filenames or # something (e.g. case insensitivity). for filename in contents: - print "Removing %s" % (filename) + self.notify.info("Removing %s" % (filename)) pathname = Filename(self.getPackageDir(), filename) pathname.unlink() if self.asMirror: - return self.compressedArchive.quickVerify(self.getPackageDir()) + return self.compressedArchive.quickVerify(self.getPackageDir(), notify = self.notify) allExtractsOk = True - if not self.uncompressedArchive.quickVerify(self.getPackageDir()): - #print "File is incorrect: %s" % (self.uncompressedArchive.filename) + if not self.uncompressedArchive.quickVerify(self.getPackageDir(), notify = self.notify): + self.notify.debug("File is incorrect: %s" % (self.uncompressedArchive.filename)) allExtractsOk = False if allExtractsOk: @@ -467,14 +471,14 @@ class PackageInfo: pathname.unlink() for file in self.extracts: - if not file.quickVerify(self.getPackageDir()): - #print "File is incorrect: %s" % (file.filename) + if not file.quickVerify(self.getPackageDir(), notify = self.notify): + self.notify.debug("File is incorrect: %s" % (file.filename)) allExtractsOk = False break -## if allExtractsOk: -## print "All %s extracts of %s seem good." % ( -## len(self.extracts), self.packageName) + if allExtractsOk: + self.notify.debug("All %s extracts of %s seem good." % ( + len(self.extracts), self.packageName)) return allExtractsOk @@ -553,6 +557,9 @@ class PackageInfo: # Successfully downloaded! return self.stepComplete + if taskMgr.destroyed: + return self.stepFailed + # All plans failed. return self.stepFailed @@ -636,7 +643,7 @@ class PackageInfo: request = DocumentSpec(url) request.setCacheControl(DocumentSpec.CCNoCache) - print "%s downloading %s" % (self.packageName, url) + self.notify.info("%s downloading %s" % (self.packageName, url)) if not filename: filename = fileSpec.filename @@ -659,7 +666,7 @@ class PackageInfo: bytesStarted = 0 if bytesStarted: - print "Resuming %s after %s bytes already downloaded" % (url, bytesStarted) + self.notify.info("Resuming %s after %s bytes already downloaded" % (url, bytesStarted)) # Make sure the file is writable. os.chmod(targetPathname.toOsSpecific(), 0644) channel.beginGetSubdocument(request, bytesStarted, 0) @@ -679,6 +686,13 @@ class PackageInfo: break self.__updateStepProgress(step) + + if taskMgr.destroyed: + # If the task manager has been destroyed, we must + # be shutting down. Get out of here. + self.notify.warning("Task Manager destroyed, aborting %s" % (url)) + return self.stepFailed + Thread.considerYield() if step: @@ -686,10 +700,10 @@ class PackageInfo: self.__updateStepProgress(step) if not channel.isValid(): - print "Failed to download %s" % (url) + self.notify.warning("Failed to download %s" % (url)) - elif not fileSpec.fullVerify(self.getPackageDir(), pathname = targetPathname): - print "After downloading, %s incorrect" % (Filename(fileSpec.filename).getBasename()) + elif not fileSpec.fullVerify(self.getPackageDir(), pathname = targetPathname, notify = self.notify): + self.notify.warning("After downloading, %s incorrect" % (Filename(fileSpec.filename).getBasename())) # This attempt failed. Maybe the original contents.xml # file is stale. Try re-downloading it now, just to be @@ -724,7 +738,7 @@ class PackageInfo: origPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename) patchPathname = Filename(self.getPackageDir(), patchfile.file.filename) result = Filename.temporary('', 'patch_') - print "Patching %s with %s" % (origPathname, patchPathname) + self.notify.info("Patching %s with %s" % (origPathname, patchPathname)) p = PandaModules.Patchfile() # The C++ class @@ -734,18 +748,24 @@ class PackageInfo: while ret == EUOk: step.bytesDone = step.bytesNeeded * p.getProgress() self.__updateStepProgress(step) + if taskMgr.destroyed: + # If the task manager has been destroyed, we must + # be shutting down. Get out of here. + self.notify.warning("Task Manager destroyed, aborting patch %s" % (origPathname)) + return self.stepFailed + Thread.considerYield() ret = p.run() del p patchPathname.unlink() if ret < 0: - print "Patching failed." + self.notify.warning("Patching of %s failed." % (origPathname)) result.unlink() return self.stepFailed if not result.renameTo(origPathname): - print "Couldn't rename %s to %s" % (result, origPathname) + self.notify.warning("Couldn't rename %s to %s" % (result, origPathname)) return self.stepFailed return self.stepComplete @@ -758,7 +778,7 @@ class PackageInfo: sourcePathname = Filename(self.getPackageDir(), self.compressedArchive.filename) targetPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename) targetPathname.unlink() - print "Uncompressing %s to %s" % (sourcePathname, targetPathname) + self.notify.info("Uncompressing %s to %s" % (sourcePathname, targetPathname)) decompressor = Decompressor() decompressor.initiate(sourcePathname, targetPathname) totalBytes = self.uncompressedArchive.size @@ -767,6 +787,12 @@ class PackageInfo: step.bytesDone = int(totalBytes * decompressor.getProgress()) self.__updateStepProgress(step) result = decompressor.run() + if taskMgr.destroyed: + # If the task manager has been destroyed, we must + # be shutting down. Get out of here. + self.notify.warning("Task Manager destroyed, aborting decompresss %s" % (sourcePathname)) + return self.stepFailed + Thread.considerYield() if result != EUSuccess: @@ -775,9 +801,9 @@ class PackageInfo: step.bytesDone = totalBytes self.__updateStepProgress(step) - if not self.uncompressedArchive.quickVerify(self.getPackageDir()): - print "after uncompressing, %s still incorrect" % ( - self.uncompressedArchive.filename) + if not self.uncompressedArchive.quickVerify(self.getPackageDir(), notify= self.notify): + self.notify.warning("after uncompressing, %s still incorrect" % ( + self.uncompressedArchive.filename)) return self.stepFailed # Now that we've verified the archive, make it read-only. @@ -798,10 +824,10 @@ class PackageInfo: return self.stepComplete mfPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename) - print "Unpacking %s" % (mfPathname) + self.notify.info("Unpacking %s" % (mfPathname)) mf = Multifile() if not mf.openRead(mfPathname): - print "Couldn't open %s" % (mfPathname) + self.notify.warning("Couldn't open %s" % (mfPathname)) return self.stepFailed allExtractsOk = True @@ -809,19 +835,19 @@ class PackageInfo: for file in self.extracts: i = mf.findSubfile(file.filename) if i == -1: - print "Not in Multifile: %s" % (file.filename) + self.notify.warning("Not in Multifile: %s" % (file.filename)) allExtractsOk = False continue targetPathname = Filename(self.getPackageDir(), file.filename) targetPathname.unlink() if not mf.extractSubfile(i, targetPathname): - print "Couldn't extract: %s" % (file.filename) + self.notify.warning("Couldn't extract: %s" % (file.filename)) allExtractsOk = False continue - if not file.quickVerify(self.getPackageDir()): - print "After extracting, still incorrect: %s" % (file.filename) + if not file.quickVerify(self.getPackageDir(), notify = self.notify): + self.notify.warning("After extracting, still incorrect: %s" % (file.filename)) allExtractsOk = False continue @@ -830,6 +856,12 @@ class PackageInfo: step.bytesDone += file.size self.__updateStepProgress(step) + if taskMgr.destroyed: + # If the task manager has been destroyed, we must + # be shutting down. Get out of here. + self.notify.warning("Task Manager destroyed, aborting unpacking %s" % (mfPathname)) + return self.stepFailed + Thread.considerYield() if not allExtractsOk: @@ -851,7 +883,7 @@ class PackageInfo: mfPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename) mf = Multifile() if not mf.openRead(mfPathname): - print "Couldn't open %s" % (mfPathname) + self.notify.warning("Couldn't open %s" % (mfPathname)) return False # We mount it under its actual location on disk. diff --git a/direct/src/task/Task.py b/direct/src/task/Task.py index a80f224f3d..49d3a6dd31 100644 --- a/direct/src/task/Task.py +++ b/direct/src/task/Task.py @@ -108,6 +108,7 @@ class TaskManager: self.globalClock = self.mgr.getClock() self.stepping = False self.running = False + self.destroyed = False self.fKeyboardInterrupt = False self.interruptCount = 0 @@ -135,7 +136,8 @@ class TaskManager: def destroy(self): # This should be safe to call multiple times. - + self.notify.info("TaskManager.destroy()") + self.destroyed = True self._frameProfileQueue.clear() self.mgr.cleanup()