TheCloud/panda3d
1
0
Fork 0
mirror of https://github.com/panda3d/panda3d.git synced 2025-10-03 10:22:45 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix shutdown while downloading

Browse Source
This commit is contained in:
David Rose 2009-12-28 21:16:56 +00:00
parent a5303df87a
commit 50b9e51aa6
5 changed files with 105 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
direct/src/p3d/AppRunner.py
View File

@ -335,7 +335,7 @@ class AppRunner(DirectObject):
host = self.hosts.get(hostUrl, None) host = self.hosts.get(hostUrl, None)
if not host: if not host:
host = HostInfo(hostUrl, self) host = HostInfo(hostUrl, appRunner = self)
self.hosts[hostUrl] = host self.hosts[hostUrl] = host
return host return host
@ -541,7 +541,7 @@ class AppRunner(DirectObject):
host = self.getHost(hostUrl) host = self.getHost(hostUrl)
if hostDir and not host.hostDir: if hostDir and not host.hostDir:
host.hostDir = hostDir host.hostDir = Filename.fromOsSpecific(hostDir)
if not host.readContentsFile(): if not host.readContentsFile():
if not host.downloadContentsFile(self.http): if not host.downloadContentsFile(self.http):

37
direct/src/p3d/FileSpec.py
View File

@ -77,7 +77,8 @@ class FileSpec:
if self.hash: if self.hash:
xelement.SetAttribute('hash', self.hash) xelement.SetAttribute('hash', self.hash)
def quickVerify(self, packageDir = None, pathname = None): def quickVerify(self, packageDir = None, pathname = None,
notify = None):
""" Performs a quick test to ensure the file has not been """ Performs a quick test to ensure the file has not been
modified. This test is vulnerable to people maliciously modified. This test is vulnerable to people maliciously
attempting to fool the program (by setting datestamps etc.). attempting to fool the program (by setting datestamps etc.).
@ -91,30 +92,37 @@ class FileSpec:
st = os.stat(pathname.toOsSpecific()) st = os.stat(pathname.toOsSpecific())
except OSError: except OSError:
# If the file is missing, the file fails. # If the file is missing, the file fails.
#print "file not found: %s" % (pathname) if notify:
notify.debug("file not found: %s" % (pathname))
return False return False
if st.st_size != self.size: if st.st_size != self.size:
# If the size is wrong, the file fails. # If the size is wrong, the file fails.
#print "size wrong: %s" % (pathname) if notify:
notify.debug("size wrong: %s" % (pathname))
return False return False
if st.st_mtime == self.timestamp: if st.st_mtime == self.timestamp:
# If the size is right and the timestamp is right, the # If the size is right and the timestamp is right, the
# file passes. # file passes.
#print "file ok: %s" % (pathname) if notify:
notify.debug("file ok: %s" % (pathname))
return True return True
#print "modification time wrong: %s" % (pathname) if notify:
notify.debug("modification time wrong: %s" % (pathname))
# If the size is right but the timestamp is wrong, the file # If the size is right but the timestamp is wrong, the file
# soft-fails. We follow this up with a hash check. # soft-fails. We follow this up with a hash check.
if not self.checkHash(packageDir, pathname, st): if not self.checkHash(packageDir, pathname, st):
# Hard fail, the hash is wrong. # Hard fail, the hash is wrong.
#print "hash check wrong: %s" % (pathname) if notify:
notify.debug("hash check wrong: %s" % (pathname))
notify.debug(" found %s, expected %s" % (self.actualFile.hash, self.hash))
return False return False
#print "hash check ok: %s" % (pathname) if notify:
notify.debug("hash check ok: %s" % (pathname))
# The hash is OK after all. Change the file's timestamp back # The hash is OK after all. Change the file's timestamp back
# to what we expect it to be, so we can quick-verify it # to what we expect it to be, so we can quick-verify it
@ -124,7 +132,7 @@ class FileSpec:
return True return True
def fullVerify(self, packageDir = None, pathname = None): def fullVerify(self, packageDir = None, pathname = None, notify = None):
""" Performs a more thorough test to ensure the file has not """ Performs a more thorough test to ensure the file has not
been modified. This test is less vulnerable to malicious been modified. This test is less vulnerable to malicious
attacks, since it reads and verifies the entire file. attacks, since it reads and verifies the entire file.
@ -138,20 +146,25 @@ class FileSpec:
st = os.stat(pathname.toOsSpecific()) st = os.stat(pathname.toOsSpecific())
except OSError: except OSError:
# If the file is missing, the file fails. # If the file is missing, the file fails.
#print "file not found: %s" % (pathname) if notify:
notify.debug("file not found: %s" % (pathname))
return False return False
if st.st_size != self.size: if st.st_size != self.size:
# If the size is wrong, the file fails; # If the size is wrong, the file fails;
#print "size wrong: %s" % (pathname) if notify:
notify.debug("size wrong: %s" % (pathname))
return False return False
if not self.checkHash(packageDir, pathname, st): if not self.checkHash(packageDir, pathname, st):
# Hard fail, the hash is wrong. # Hard fail, the hash is wrong.
#print "hash check wrong: %s" % (pathname) if notify:
notify.debug("hash check wrong: %s" % (pathname))
notify.debug(" found %s, expected %s" % (self.actualFile.hash, self.hash))
return False return False
#print "hash check ok: %s" % (pathname) if notify:
notify.debug("hash check ok: %s" % (pathname))
# The hash is OK. If the timestamp is wrong, change it back # The hash is OK. If the timestamp is wrong, change it back
# to what we expect it to be, so we can quick-verify it # to what we expect it to be, so we can quick-verify it

19
direct/src/p3d/HostInfo.py
View File

@ -2,6 +2,7 @@ from pandac.PandaModules import HashVal, Filename, PandaSystem, DocumentSpec, Ra
from pandac import PandaModules from pandac import PandaModules
from direct.p3d.PackageInfo import PackageInfo from direct.p3d.PackageInfo import PackageInfo
from direct.p3d.FileSpec import FileSpec from direct.p3d.FileSpec import FileSpec
from direct.directnotify.DirectNotifyGlobal import directNotify
import time import time
class HostInfo: class HostInfo:
@ -9,6 +10,8 @@ class HostInfo:
Panda3D packages. It is the Python equivalent of the P3DHost Panda3D packages. It is the Python equivalent of the P3DHost
class in the core API. """ class in the core API. """
notify = directNotify.newCategory("HostInfo")
def __init__(self, hostUrl, appRunner = None, hostDir = None, def __init__(self, hostUrl, appRunner = None, hostDir = None,
rootDir = None, asMirror = False, perPlatform = None): rootDir = None, asMirror = False, perPlatform = None):
@ -94,13 +97,13 @@ class HostInfo:
# We start with the "super mirror", if it's defined. # We start with the "super mirror", if it's defined.
url = self.appRunner.superMirrorUrl + 'contents.xml' url = self.appRunner.superMirrorUrl + 'contents.xml'
request = DocumentSpec(url) request = DocumentSpec(url)
print "Downloading contents file %s" % (request) self.notify.info("Downloading contents file %s" % (request))
rf = Ramfile() rf = Ramfile()
channel = http.makeChannel(False) channel = http.makeChannel(False)
channel.getDocument(request) channel.getDocument(request)
if not channel.downloadToRam(rf): if not channel.downloadToRam(rf):
print "Unable to download %s" % (url) self.notify.warning("Unable to download %s" % (url))
rf = None rf = None
if not rf: if not rf:
@ -118,13 +121,13 @@ class HostInfo:
request = DocumentSpec(url) request = DocumentSpec(url)
request.setCacheControl(DocumentSpec.CCNoCache) request.setCacheControl(DocumentSpec.CCNoCache)
print "Downloading contents file %s" % (request) self.notify.info("Downloading contents file %s" % (request))
rf = Ramfile() rf = Ramfile()
channel = http.makeChannel(False) channel = http.makeChannel(False)
channel.getDocument(request) channel.getDocument(request)
if not channel.downloadToRam(rf): if not channel.downloadToRam(rf):
print "Unable to download %s" % (url) self.notify.warning("Unable to download %s" % (url))
rf = None rf = None
tempFilename = Filename.temporary('', 'p3d_', '.xml') tempFilename = Filename.temporary('', 'p3d_', '.xml')
@ -134,7 +137,7 @@ class HostInfo:
f.close() f.close()
if not self.readContentsFile(tempFilename): if not self.readContentsFile(tempFilename):
print "Failure reading %s" % (url) self.notify.warning("Failure reading %s" % (url))
tempFilename.unlink() tempFilename.unlink()
return False return False
@ -151,7 +154,7 @@ class HostInfo:
assert self.hasContentsFile assert self.hasContentsFile
url = self.hostUrlPrefix + 'contents.xml' url = self.hostUrlPrefix + 'contents.xml'
print "Redownloading %s" % (url) self.notify.info("Redownloading %s" % (url))
# Get the hash of the original file. # Get the hash of the original file.
assert self.hostDir assert self.hostDir
@ -168,10 +171,10 @@ class HostInfo:
hv2.hashFile(filename) hv2.hashFile(filename)
if hv1 != hv2: if hv1 != hv2:
print "%s has changed." % (url) self.notify.info("%s has changed." % (url))
return True return True
else: else:
print "%s has not changed." % (url) self.notify.info("%s has not changed." % (url))
return False return False

96
direct/src/p3d/PackageInfo.py
View File

@ -2,6 +2,8 @@ from pandac.PandaModules import Filename, URLSpec, DocumentSpec, Ramfile, Multif
from pandac import PandaModules from pandac import PandaModules
from direct.p3d.FileSpec import FileSpec from direct.p3d.FileSpec import FileSpec
from direct.showbase import VFSImporter from direct.showbase import VFSImporter
from direct.directnotify.DirectNotifyGlobal import directNotify
from direct.task.TaskManagerGlobal import taskMgr
import os import os
import sys import sys
import random import random
@ -13,6 +15,8 @@ class PackageInfo:
can be (or has been) installed into the current runtime. It is can be (or has been) installed into the current runtime. It is
the Python equivalent of the P3DPackage class in the core API. """ the Python equivalent of the P3DPackage class in the core API. """
notify = directNotify.newCategory("PackageInfo")
# Weight factors for computing download progress. This # Weight factors for computing download progress. This
# attempts to reflect the relative time-per-byte of each of # attempts to reflect the relative time-per-byte of each of
# these operations. # these operations.
@ -171,7 +175,7 @@ class PackageInfo:
if not self.hasDescFile: if not self.hasDescFile:
filename = Filename(self.getPackageDir(), self.descFileBasename) filename = Filename(self.getPackageDir(), self.descFileBasename)
if self.descFile.quickVerify(self.getPackageDir(), pathname = filename): if self.descFile.quickVerify(self.getPackageDir(), pathname = filename, notify = self.notify):
if self.__readDescFile(): if self.__readDescFile():
# Successfully read. We don't need to call # Successfully read. We don't need to call
# checkArchiveStatus again, since readDescFile() # checkArchiveStatus again, since readDescFile()
@ -223,7 +227,7 @@ class PackageInfo:
if not self.__readDescFile(): if not self.__readDescFile():
# Weird, it passed the hash check, but we still can't read # Weird, it passed the hash check, but we still can't read
# it. # it.
print "Failure reading %s" % (filename) self.notify.warning("Failure reading %s" % (filename))
return False return False
return True return True
@ -356,12 +360,12 @@ class PackageInfo:
# If the uncompressed archive file is good, that's all we'll # If the uncompressed archive file is good, that's all we'll
# need to do. # need to do.
self.uncompressedArchive.actualFile = None self.uncompressedArchive.actualFile = None
if self.uncompressedArchive.quickVerify(self.getPackageDir()): if self.uncompressedArchive.quickVerify(self.getPackageDir(), notify = self.notify):
self.installPlans = [planA] self.installPlans = [planA]
return return
# Maybe the compressed archive file is good. # Maybe the compressed archive file is good.
if self.compressedArchive.quickVerify(self.getPackageDir()): if self.compressedArchive.quickVerify(self.getPackageDir(), notify = self.notify):
uncompressSize = self.uncompressedArchive.size uncompressSize = self.uncompressedArchive.size
step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor) step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor)
planA = [step] + planA planA = [step] + planA
@ -448,16 +452,16 @@ class PackageInfo:
# case there is a problem with ambiguous filenames or # case there is a problem with ambiguous filenames or
# something (e.g. case insensitivity). # something (e.g. case insensitivity).
for filename in contents: for filename in contents:
print "Removing %s" % (filename) self.notify.info("Removing %s" % (filename))
pathname = Filename(self.getPackageDir(), filename) pathname = Filename(self.getPackageDir(), filename)
pathname.unlink() pathname.unlink()
if self.asMirror: if self.asMirror:
return self.compressedArchive.quickVerify(self.getPackageDir()) return self.compressedArchive.quickVerify(self.getPackageDir(), notify = self.notify)
allExtractsOk = True allExtractsOk = True
if not self.uncompressedArchive.quickVerify(self.getPackageDir()): if not self.uncompressedArchive.quickVerify(self.getPackageDir(), notify = self.notify):
#print "File is incorrect: %s" % (self.uncompressedArchive.filename) self.notify.debug("File is incorrect: %s" % (self.uncompressedArchive.filename))
allExtractsOk = False allExtractsOk = False
if allExtractsOk: if allExtractsOk:
@ -467,14 +471,14 @@ class PackageInfo:
pathname.unlink() pathname.unlink()
for file in self.extracts: for file in self.extracts:
if not file.quickVerify(self.getPackageDir()): if not file.quickVerify(self.getPackageDir(), notify = self.notify):
#print "File is incorrect: %s" % (file.filename) self.notify.debug("File is incorrect: %s" % (file.filename))
allExtractsOk = False allExtractsOk = False
break break
## if allExtractsOk: if allExtractsOk:
## print "All %s extracts of %s seem good." % ( self.notify.debug("All %s extracts of %s seem good." % (
## len(self.extracts), self.packageName) len(self.extracts), self.packageName))
return allExtractsOk return allExtractsOk
@ -553,6 +557,9 @@ class PackageInfo:
# Successfully downloaded! # Successfully downloaded!
return self.stepComplete return self.stepComplete
if taskMgr.destroyed:
return self.stepFailed
# All plans failed. # All plans failed.
return self.stepFailed return self.stepFailed
@ -636,7 +643,7 @@ class PackageInfo:
request = DocumentSpec(url) request = DocumentSpec(url)
request.setCacheControl(DocumentSpec.CCNoCache) request.setCacheControl(DocumentSpec.CCNoCache)
print "%s downloading %s" % (self.packageName, url) self.notify.info("%s downloading %s" % (self.packageName, url))
if not filename: if not filename:
filename = fileSpec.filename filename = fileSpec.filename
@ -659,7 +666,7 @@ class PackageInfo:
bytesStarted = 0 bytesStarted = 0
if bytesStarted: if bytesStarted:
print "Resuming %s after %s bytes already downloaded" % (url, bytesStarted) self.notify.info("Resuming %s after %s bytes already downloaded" % (url, bytesStarted))
# Make sure the file is writable. # Make sure the file is writable.
os.chmod(targetPathname.toOsSpecific(), 0644) os.chmod(targetPathname.toOsSpecific(), 0644)
channel.beginGetSubdocument(request, bytesStarted, 0) channel.beginGetSubdocument(request, bytesStarted, 0)
@ -679,6 +686,13 @@ class PackageInfo:
break break
self.__updateStepProgress(step) self.__updateStepProgress(step)
if taskMgr.destroyed:
# If the task manager has been destroyed, we must
# be shutting down. Get out of here.
self.notify.warning("Task Manager destroyed, aborting %s" % (url))
return self.stepFailed
Thread.considerYield() Thread.considerYield()
if step: if step:
@ -686,10 +700,10 @@ class PackageInfo:
self.__updateStepProgress(step) self.__updateStepProgress(step)
if not channel.isValid(): if not channel.isValid():
print "Failed to download %s" % (url) self.notify.warning("Failed to download %s" % (url))
elif not fileSpec.fullVerify(self.getPackageDir(), pathname = targetPathname): elif not fileSpec.fullVerify(self.getPackageDir(), pathname = targetPathname, notify = self.notify):
print "After downloading, %s incorrect" % (Filename(fileSpec.filename).getBasename()) self.notify.warning("After downloading, %s incorrect" % (Filename(fileSpec.filename).getBasename()))
# This attempt failed. Maybe the original contents.xml # This attempt failed. Maybe the original contents.xml
# file is stale. Try re-downloading it now, just to be # file is stale. Try re-downloading it now, just to be
@ -724,7 +738,7 @@ class PackageInfo:
origPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename) origPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename)
patchPathname = Filename(self.getPackageDir(), patchfile.file.filename) patchPathname = Filename(self.getPackageDir(), patchfile.file.filename)
result = Filename.temporary('', 'patch_') result = Filename.temporary('', 'patch_')
print "Patching %s with %s" % (origPathname, patchPathname) self.notify.info("Patching %s with %s" % (origPathname, patchPathname))
p = PandaModules.Patchfile() # The C++ class p = PandaModules.Patchfile() # The C++ class
@ -734,18 +748,24 @@ class PackageInfo:
while ret == EUOk: while ret == EUOk:
step.bytesDone = step.bytesNeeded * p.getProgress() step.bytesDone = step.bytesNeeded * p.getProgress()
self.__updateStepProgress(step) self.__updateStepProgress(step)
if taskMgr.destroyed:
# If the task manager has been destroyed, we must
# be shutting down. Get out of here.
self.notify.warning("Task Manager destroyed, aborting patch %s" % (origPathname))
return self.stepFailed
Thread.considerYield() Thread.considerYield()
ret = p.run() ret = p.run()
del p del p
patchPathname.unlink() patchPathname.unlink()
if ret < 0: if ret < 0:
print "Patching failed." self.notify.warning("Patching of %s failed." % (origPathname))
result.unlink() result.unlink()
return self.stepFailed return self.stepFailed
if not result.renameTo(origPathname): if not result.renameTo(origPathname):
print "Couldn't rename %s to %s" % (result, origPathname) self.notify.warning("Couldn't rename %s to %s" % (result, origPathname))
return self.stepFailed return self.stepFailed
return self.stepComplete return self.stepComplete
@ -758,7 +778,7 @@ class PackageInfo:
sourcePathname = Filename(self.getPackageDir(), self.compressedArchive.filename) sourcePathname = Filename(self.getPackageDir(), self.compressedArchive.filename)
targetPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename) targetPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename)
targetPathname.unlink() targetPathname.unlink()
print "Uncompressing %s to %s" % (sourcePathname, targetPathname) self.notify.info("Uncompressing %s to %s" % (sourcePathname, targetPathname))
decompressor = Decompressor() decompressor = Decompressor()
decompressor.initiate(sourcePathname, targetPathname) decompressor.initiate(sourcePathname, targetPathname)
totalBytes = self.uncompressedArchive.size totalBytes = self.uncompressedArchive.size
@ -767,6 +787,12 @@ class PackageInfo:
step.bytesDone = int(totalBytes * decompressor.getProgress()) step.bytesDone = int(totalBytes * decompressor.getProgress())
self.__updateStepProgress(step) self.__updateStepProgress(step)
result = decompressor.run() result = decompressor.run()
if taskMgr.destroyed:
# If the task manager has been destroyed, we must
# be shutting down. Get out of here.
self.notify.warning("Task Manager destroyed, aborting decompresss %s" % (sourcePathname))
return self.stepFailed
Thread.considerYield() Thread.considerYield()
if result != EUSuccess: if result != EUSuccess:
@ -775,9 +801,9 @@ class PackageInfo:
step.bytesDone = totalBytes step.bytesDone = totalBytes
self.__updateStepProgress(step) self.__updateStepProgress(step)
if not self.uncompressedArchive.quickVerify(self.getPackageDir()): if not self.uncompressedArchive.quickVerify(self.getPackageDir(), notify= self.notify):
print "after uncompressing, %s still incorrect" % ( self.notify.warning("after uncompressing, %s still incorrect" % (
self.uncompressedArchive.filename) self.uncompressedArchive.filename))
return self.stepFailed return self.stepFailed
# Now that we've verified the archive, make it read-only. # Now that we've verified the archive, make it read-only.
@ -798,10 +824,10 @@ class PackageInfo:
return self.stepComplete return self.stepComplete
mfPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename) mfPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename)
print "Unpacking %s" % (mfPathname) self.notify.info("Unpacking %s" % (mfPathname))
mf = Multifile() mf = Multifile()
if not mf.openRead(mfPathname): if not mf.openRead(mfPathname):
print "Couldn't open %s" % (mfPathname) self.notify.warning("Couldn't open %s" % (mfPathname))
return self.stepFailed return self.stepFailed
allExtractsOk = True allExtractsOk = True
@ -809,19 +835,19 @@ class PackageInfo:
for file in self.extracts: for file in self.extracts:
i = mf.findSubfile(file.filename) i = mf.findSubfile(file.filename)
if i == -1: if i == -1:
print "Not in Multifile: %s" % (file.filename) self.notify.warning("Not in Multifile: %s" % (file.filename))
allExtractsOk = False allExtractsOk = False
continue continue
targetPathname = Filename(self.getPackageDir(), file.filename) targetPathname = Filename(self.getPackageDir(), file.filename)
targetPathname.unlink() targetPathname.unlink()
if not mf.extractSubfile(i, targetPathname): if not mf.extractSubfile(i, targetPathname):
print "Couldn't extract: %s" % (file.filename) self.notify.warning("Couldn't extract: %s" % (file.filename))
allExtractsOk = False allExtractsOk = False
continue continue
if not file.quickVerify(self.getPackageDir()): if not file.quickVerify(self.getPackageDir(), notify = self.notify):
print "After extracting, still incorrect: %s" % (file.filename) self.notify.warning("After extracting, still incorrect: %s" % (file.filename))
allExtractsOk = False allExtractsOk = False
continue continue
@ -830,6 +856,12 @@ class PackageInfo:
step.bytesDone += file.size step.bytesDone += file.size
self.__updateStepProgress(step) self.__updateStepProgress(step)
if taskMgr.destroyed:
# If the task manager has been destroyed, we must
# be shutting down. Get out of here.
self.notify.warning("Task Manager destroyed, aborting unpacking %s" % (mfPathname))
return self.stepFailed
Thread.considerYield() Thread.considerYield()
if not allExtractsOk: if not allExtractsOk:
@ -851,7 +883,7 @@ class PackageInfo:
mfPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename) mfPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename)
mf = Multifile() mf = Multifile()
if not mf.openRead(mfPathname): if not mf.openRead(mfPathname):
print "Couldn't open %s" % (mfPathname) self.notify.warning("Couldn't open %s" % (mfPathname))
return False return False
# We mount it under its actual location on disk. # We mount it under its actual location on disk.

4
direct/src/task/Task.py
View File

@ -108,6 +108,7 @@ class TaskManager:
self.globalClock = self.mgr.getClock() self.globalClock = self.mgr.getClock()
self.stepping = False self.stepping = False
self.running = False self.running = False
self.destroyed = False
self.fKeyboardInterrupt = False self.fKeyboardInterrupt = False
self.interruptCount = 0 self.interruptCount = 0
@ -135,7 +136,8 @@ class TaskManager:
def destroy(self): def destroy(self):
# This should be safe to call multiple times. # This should be safe to call multiple times.
self.notify.info("TaskManager.destroy()")
self.destroyed = True
self._frameProfileQueue.clear() self._frameProfileQueue.clear()
self.mgr.cleanup() self.mgr.cleanup()