TheCloud/panda3d
1
0
Fork 0
mirror of https://github.com/panda3d/panda3d.git synced 2025-10-03 10:22:45 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix shutdown while downloading

Browse Source
This commit is contained in:
David Rose 2009-12-28 21:16:56 +00:00
parent a5303df87a
commit 50b9e51aa6
5 changed files with 105 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
direct/src/p3d/AppRunner.py
View File

@ -335,7 +335,7 @@ class AppRunner(DirectObject):
host = self.hosts.get(hostUrl, None)
if not host:
host = HostInfo(hostUrl, self)
host = HostInfo(hostUrl, appRunner = self)
self.hosts[hostUrl] = host
return host
@ -541,7 +541,7 @@ class AppRunner(DirectObject):
host = self.getHost(hostUrl)
if hostDir and not host.hostDir:
host.hostDir = hostDir
host.hostDir = Filename.fromOsSpecific(hostDir)
if not host.readContentsFile():
if not host.downloadContentsFile(self.http):

37
direct/src/p3d/FileSpec.py
View File

@ -77,7 +77,8 @@ class FileSpec:
if self.hash:
xelement.SetAttribute('hash', self.hash)
def quickVerify(self, packageDir = None, pathname = None):
def quickVerify(self, packageDir = None, pathname = None,
notify = None):
""" Performs a quick test to ensure the file has not been
modified. This test is vulnerable to people maliciously
attempting to fool the program (by setting datestamps etc.).
@ -91,30 +92,37 @@ class FileSpec:
st = os.stat(pathname.toOsSpecific())
except OSError:
# If the file is missing, the file fails.
#print "file not found: %s" % (pathname)
if notify:
notify.debug("file not found: %s" % (pathname))
return False
if st.st_size != self.size:
# If the size is wrong, the file fails.
#print "size wrong: %s" % (pathname)
if notify:
notify.debug("size wrong: %s" % (pathname))
return False
if st.st_mtime == self.timestamp:
# If the size is right and the timestamp is right, the
# file passes.
#print "file ok: %s" % (pathname)
if notify:
notify.debug("file ok: %s" % (pathname))
return True
#print "modification time wrong: %s" % (pathname)
if notify:
notify.debug("modification time wrong: %s" % (pathname))
# If the size is right but the timestamp is wrong, the file
# soft-fails. We follow this up with a hash check.
if not self.checkHash(packageDir, pathname, st):
# Hard fail, the hash is wrong.
#print "hash check wrong: %s" % (pathname)
if notify:
notify.debug("hash check wrong: %s" % (pathname))
notify.debug(" found %s, expected %s" % (self.actualFile.hash, self.hash))
return False
#print "hash check ok: %s" % (pathname)
if notify:
notify.debug("hash check ok: %s" % (pathname))
# The hash is OK after all. Change the file's timestamp back
# to what we expect it to be, so we can quick-verify it
@ -124,7 +132,7 @@ class FileSpec:
return True
def fullVerify(self, packageDir = None, pathname = None):
def fullVerify(self, packageDir = None, pathname = None, notify = None):
""" Performs a more thorough test to ensure the file has not
been modified. This test is less vulnerable to malicious
attacks, since it reads and verifies the entire file.
@ -138,20 +146,25 @@ class FileSpec:
st = os.stat(pathname.toOsSpecific())
except OSError:
# If the file is missing, the file fails.
#print "file not found: %s" % (pathname)
if notify:
notify.debug("file not found: %s" % (pathname))
return False
if st.st_size != self.size:
# If the size is wrong, the file fails;
#print "size wrong: %s" % (pathname)
if notify:
notify.debug("size wrong: %s" % (pathname))
return False
if not self.checkHash(packageDir, pathname, st):
# Hard fail, the hash is wrong.
#print "hash check wrong: %s" % (pathname)
if notify:
notify.debug("hash check wrong: %s" % (pathname))
notify.debug(" found %s, expected %s" % (self.actualFile.hash, self.hash))
return False
#print "hash check ok: %s" % (pathname)
if notify:
notify.debug("hash check ok: %s" % (pathname))
# The hash is OK. If the timestamp is wrong, change it back
# to what we expect it to be, so we can quick-verify it

19
direct/src/p3d/HostInfo.py
View File

@ -2,6 +2,7 @@ from pandac.PandaModules import HashVal, Filename, PandaSystem, DocumentSpec, Ra
from pandac import PandaModules
from direct.p3d.PackageInfo import PackageInfo
from direct.p3d.FileSpec import FileSpec
from direct.directnotify.DirectNotifyGlobal import directNotify
import time
class HostInfo:
@ -9,6 +10,8 @@ class HostInfo:
Panda3D packages. It is the Python equivalent of the P3DHost
class in the core API. """
notify = directNotify.newCategory("HostInfo")
def __init__(self, hostUrl, appRunner = None, hostDir = None,
rootDir = None, asMirror = False, perPlatform = None):
@ -94,13 +97,13 @@ class HostInfo:
# We start with the "super mirror", if it's defined.
url = self.appRunner.superMirrorUrl + 'contents.xml'
request = DocumentSpec(url)
print "Downloading contents file %s" % (request)
self.notify.info("Downloading contents file %s" % (request))
rf = Ramfile()
channel = http.makeChannel(False)
channel.getDocument(request)
if not channel.downloadToRam(rf):
print "Unable to download %s" % (url)
self.notify.warning("Unable to download %s" % (url))
rf = None
if not rf:
@ -118,13 +121,13 @@ class HostInfo:
request = DocumentSpec(url)
request.setCacheControl(DocumentSpec.CCNoCache)
print "Downloading contents file %s" % (request)
self.notify.info("Downloading contents file %s" % (request))
rf = Ramfile()
channel = http.makeChannel(False)
channel.getDocument(request)
if not channel.downloadToRam(rf):
print "Unable to download %s" % (url)
self.notify.warning("Unable to download %s" % (url))
rf = None
tempFilename = Filename.temporary('', 'p3d_', '.xml')
@ -134,7 +137,7 @@ class HostInfo:
f.close()
if not self.readContentsFile(tempFilename):
print "Failure reading %s" % (url)
self.notify.warning("Failure reading %s" % (url))
tempFilename.unlink()
return False
@ -151,7 +154,7 @@ class HostInfo:
assert self.hasContentsFile
url = self.hostUrlPrefix + 'contents.xml'
print "Redownloading %s" % (url)
self.notify.info("Redownloading %s" % (url))
# Get the hash of the original file.
assert self.hostDir
@ -168,10 +171,10 @@ class HostInfo:
hv2.hashFile(filename)
if hv1 != hv2:
print "%s has changed." % (url)
self.notify.info("%s has changed." % (url))
return True
else:
print "%s has not changed." % (url)
self.notify.info("%s has not changed." % (url))
return False

96
direct/src/p3d/PackageInfo.py
View File

@ -2,6 +2,8 @@ from pandac.PandaModules import Filename, URLSpec, DocumentSpec, Ramfile, Multif
from pandac import PandaModules
from direct.p3d.FileSpec import FileSpec
from direct.showbase import VFSImporter
from direct.directnotify.DirectNotifyGlobal import directNotify
from direct.task.TaskManagerGlobal import taskMgr
import os
import sys
import random
@ -13,6 +15,8 @@ class PackageInfo:
can be (or has been) installed into the current runtime. It is
the Python equivalent of the P3DPackage class in the core API. """
notify = directNotify.newCategory("PackageInfo")
# Weight factors for computing download progress. This
# attempts to reflect the relative time-per-byte of each of
# these operations.
@ -171,7 +175,7 @@ class PackageInfo:
if not self.hasDescFile:
filename = Filename(self.getPackageDir(), self.descFileBasename)
if self.descFile.quickVerify(self.getPackageDir(), pathname = filename):
if self.descFile.quickVerify(self.getPackageDir(), pathname = filename, notify = self.notify):
if self.__readDescFile():
# Successfully read. We don't need to call
# checkArchiveStatus again, since readDescFile()
@ -223,7 +227,7 @@ class PackageInfo:
if not self.__readDescFile():
# Weird, it passed the hash check, but we still can't read
# it.
print "Failure reading %s" % (filename)
self.notify.warning("Failure reading %s" % (filename))
return False
return True
@ -356,12 +360,12 @@ class PackageInfo:
# If the uncompressed archive file is good, that's all we'll
# need to do.
self.uncompressedArchive.actualFile = None
if self.uncompressedArchive.quickVerify(self.getPackageDir()):
if self.uncompressedArchive.quickVerify(self.getPackageDir(), notify = self.notify):
self.installPlans = [planA]
return
# Maybe the compressed archive file is good.
if self.compressedArchive.quickVerify(self.getPackageDir()):
if self.compressedArchive.quickVerify(self.getPackageDir(), notify = self.notify):
uncompressSize = self.uncompressedArchive.size
step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor)
planA = [step] + planA
@ -448,16 +452,16 @@ class PackageInfo:
# case there is a problem with ambiguous filenames or
# something (e.g. case insensitivity).
for filename in contents:
print "Removing %s" % (filename)
self.notify.info("Removing %s" % (filename))
pathname = Filename(self.getPackageDir(), filename)
pathname.unlink()
if self.asMirror:
return self.compressedArchive.quickVerify(self.getPackageDir())
return self.compressedArchive.quickVerify(self.getPackageDir(), notify = self.notify)
allExtractsOk = True
if not self.uncompressedArchive.quickVerify(self.getPackageDir()):
#print "File is incorrect: %s" % (self.uncompressedArchive.filename)
if not self.uncompressedArchive.quickVerify(self.getPackageDir(), notify = self.notify):
self.notify.debug("File is incorrect: %s" % (self.uncompressedArchive.filename))
allExtractsOk = False
if allExtractsOk:
@ -467,14 +471,14 @@ class PackageInfo:
pathname.unlink()
for file in self.extracts:
if not file.quickVerify(self.getPackageDir()):
#print "File is incorrect: %s" % (file.filename)
if not file.quickVerify(self.getPackageDir(), notify = self.notify):
self.notify.debug("File is incorrect: %s" % (file.filename))
allExtractsOk = False
break
## if allExtractsOk:
## print "All %s extracts of %s seem good." % (
## len(self.extracts), self.packageName)
if allExtractsOk:
self.notify.debug("All %s extracts of %s seem good." % (
len(self.extracts), self.packageName))
return allExtractsOk
@ -553,6 +557,9 @@ class PackageInfo:
# Successfully downloaded!
return self.stepComplete
if taskMgr.destroyed:
return self.stepFailed
# All plans failed.
return self.stepFailed
@ -636,7 +643,7 @@ class PackageInfo:
request = DocumentSpec(url)
request.setCacheControl(DocumentSpec.CCNoCache)
print "%s downloading %s" % (self.packageName, url)
self.notify.info("%s downloading %s" % (self.packageName, url))
if not filename:
filename = fileSpec.filename
@ -659,7 +666,7 @@ class PackageInfo:
bytesStarted = 0
if bytesStarted:
print "Resuming %s after %s bytes already downloaded" % (url, bytesStarted)
self.notify.info("Resuming %s after %s bytes already downloaded" % (url, bytesStarted))
# Make sure the file is writable.
os.chmod(targetPathname.toOsSpecific(), 0644)
channel.beginGetSubdocument(request, bytesStarted, 0)
@ -679,6 +686,13 @@ class PackageInfo:
break
self.__updateStepProgress(step)
if taskMgr.destroyed:
# If the task manager has been destroyed, we must
# be shutting down. Get out of here.
self.notify.warning("Task Manager destroyed, aborting %s" % (url))
return self.stepFailed
Thread.considerYield()
if step:
@ -686,10 +700,10 @@ class PackageInfo:
self.__updateStepProgress(step)
if not channel.isValid():
print "Failed to download %s" % (url)
self.notify.warning("Failed to download %s" % (url))
elif not fileSpec.fullVerify(self.getPackageDir(), pathname = targetPathname):
print "After downloading, %s incorrect" % (Filename(fileSpec.filename).getBasename())
elif not fileSpec.fullVerify(self.getPackageDir(), pathname = targetPathname, notify = self.notify):
self.notify.warning("After downloading, %s incorrect" % (Filename(fileSpec.filename).getBasename()))
# This attempt failed. Maybe the original contents.xml
# file is stale. Try re-downloading it now, just to be
@ -724,7 +738,7 @@ class PackageInfo:
origPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename)
patchPathname = Filename(self.getPackageDir(), patchfile.file.filename)
result = Filename.temporary('', 'patch_')
print "Patching %s with %s" % (origPathname, patchPathname)
self.notify.info("Patching %s with %s" % (origPathname, patchPathname))
p = PandaModules.Patchfile() # The C++ class
@ -734,18 +748,24 @@ class PackageInfo:
while ret == EUOk:
step.bytesDone = step.bytesNeeded * p.getProgress()
self.__updateStepProgress(step)
if taskMgr.destroyed:
# If the task manager has been destroyed, we must
# be shutting down. Get out of here.
self.notify.warning("Task Manager destroyed, aborting patch %s" % (origPathname))
return self.stepFailed
Thread.considerYield()
ret = p.run()
del p
patchPathname.unlink()
if ret < 0:
print "Patching failed."
self.notify.warning("Patching of %s failed." % (origPathname))
result.unlink()
return self.stepFailed
if not result.renameTo(origPathname):
print "Couldn't rename %s to %s" % (result, origPathname)
self.notify.warning("Couldn't rename %s to %s" % (result, origPathname))
return self.stepFailed
return self.stepComplete
@ -758,7 +778,7 @@ class PackageInfo:
sourcePathname = Filename(self.getPackageDir(), self.compressedArchive.filename)
targetPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename)
targetPathname.unlink()
print "Uncompressing %s to %s" % (sourcePathname, targetPathname)
self.notify.info("Uncompressing %s to %s" % (sourcePathname, targetPathname))
decompressor = Decompressor()
decompressor.initiate(sourcePathname, targetPathname)
totalBytes = self.uncompressedArchive.size
@ -767,6 +787,12 @@ class PackageInfo:
step.bytesDone = int(totalBytes * decompressor.getProgress())
self.__updateStepProgress(step)
result = decompressor.run()
if taskMgr.destroyed:
# If the task manager has been destroyed, we must
# be shutting down. Get out of here.
self.notify.warning("Task Manager destroyed, aborting decompresss %s" % (sourcePathname))
return self.stepFailed
Thread.considerYield()
if result != EUSuccess:
@ -775,9 +801,9 @@ class PackageInfo:
step.bytesDone = totalBytes
self.__updateStepProgress(step)
if not self.uncompressedArchive.quickVerify(self.getPackageDir()):
print "after uncompressing, %s still incorrect" % (
self.uncompressedArchive.filename)
if not self.uncompressedArchive.quickVerify(self.getPackageDir(), notify= self.notify):
self.notify.warning("after uncompressing, %s still incorrect" % (
self.uncompressedArchive.filename))
return self.stepFailed
# Now that we've verified the archive, make it read-only.
@ -798,10 +824,10 @@ class PackageInfo:
return self.stepComplete
mfPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename)
print "Unpacking %s" % (mfPathname)
self.notify.info("Unpacking %s" % (mfPathname))
mf = Multifile()
if not mf.openRead(mfPathname):
print "Couldn't open %s" % (mfPathname)
self.notify.warning("Couldn't open %s" % (mfPathname))
return self.stepFailed
allExtractsOk = True
@ -809,19 +835,19 @@ class PackageInfo:
for file in self.extracts:
i = mf.findSubfile(file.filename)
if i == -1:
print "Not in Multifile: %s" % (file.filename)
self.notify.warning("Not in Multifile: %s" % (file.filename))
allExtractsOk = False
continue
targetPathname = Filename(self.getPackageDir(), file.filename)
targetPathname.unlink()
if not mf.extractSubfile(i, targetPathname):
print "Couldn't extract: %s" % (file.filename)
self.notify.warning("Couldn't extract: %s" % (file.filename))
allExtractsOk = False
continue
if not file.quickVerify(self.getPackageDir()):
print "After extracting, still incorrect: %s" % (file.filename)
if not file.quickVerify(self.getPackageDir(), notify = self.notify):
self.notify.warning("After extracting, still incorrect: %s" % (file.filename))
allExtractsOk = False
continue
@ -830,6 +856,12 @@ class PackageInfo:
step.bytesDone += file.size
self.__updateStepProgress(step)
if taskMgr.destroyed:
# If the task manager has been destroyed, we must
# be shutting down. Get out of here.
self.notify.warning("Task Manager destroyed, aborting unpacking %s" % (mfPathname))
return self.stepFailed
Thread.considerYield()
if not allExtractsOk:
@ -851,7 +883,7 @@ class PackageInfo:
mfPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename)
mf = Multifile()
if not mf.openRead(mfPathname):
print "Couldn't open %s" % (mfPathname)
self.notify.warning("Couldn't open %s" % (mfPathname))
return False
# We mount it under its actual location on disk.

4
direct/src/task/Task.py
View File

@ -108,6 +108,7 @@ class TaskManager:
self.globalClock = self.mgr.getClock()
self.stepping = False
self.running = False
self.destroyed = False
self.fKeyboardInterrupt = False
self.interruptCount = 0
@ -135,7 +136,8 @@ class TaskManager:
def destroy(self):
# This should be safe to call multiple times.
self.notify.info("TaskManager.destroy()")
self.destroyed = True
self._frameProfileQueue.clear()
self.mgr.cleanup()