diff --git a/direct/src/p3d/AppRunner.py b/direct/src/p3d/AppRunner.py index 5ede54c820..8962a204c1 100644 --- a/direct/src/p3d/AppRunner.py +++ b/direct/src/p3d/AppRunner.py @@ -113,6 +113,12 @@ class AppRunner(DirectObject): # hosts we have imported packages from. self.hosts = {} + # The altHost string that is in effect from the HTML tokens, + # if any, and the dictionary of URL remapping: orig host url + # -> alt host url. + self.altHost = None + self.altHostMap = {} + # Application code can assign a callable object here; if so, # it will be invoked when an uncaught exception propagates to # the top of the TaskMgr.run() loop. @@ -213,7 +219,7 @@ class AppRunner(DirectObject): finished; see the PackageInstaller class if you want this to happen asynchronously instead. """ - host = self.getHost(hostUrl) + host = self.getHostWithAlt(hostUrl) if not host.downloadContentsFile(self.http): return False @@ -235,10 +241,47 @@ class AppRunner(DirectObject): print "Package %s %s installed." % (packageName, version) + def getHostWithAlt(self, hostUrl): + """ Returns a suitable HostInfo object for downloading + contents from the indicated URL. This is almost always the + same thing as getHost(), except in the rare case when we have + an alt_host specified in the HTML tokens; in this case, we may + actually want to download the contents from a different URL + than the one given, for instance to download a version in + testing. """ + + altUrl = self.altHostMap.get(hostUrl, None) + if altUrl: + # We got an alternate host. Use it. + return self.getHost(altUrl) + + # We didn't get an aternate host, use the original. + host = self.getHost(hostUrl) + + # But we might need to consult the host itself to see if *it* + # recommends an altHost. + if self.altHost: + # This means forcing the host to download its contents + # file on the spot, a blocking operation. This is a + # little unfortunate, but since alt_host is so rarely + # used, probably not really a problem. + host.downloadContentsFile(self.http) + altUrl = host.altHosts.get(self.altHost, None) + if altUrl: + return self.getHost(altUrl) + + # No shenanigans, just return the requested host. + return host + def getHost(self, hostUrl): """ Returns a new HostInfo object corresponding to the indicated host URL. If we have already seen this URL - previously, returns the same object. """ + previously, returns the same object. + + This returns the literal referenced host. To return the + mapped host, which is the one we should actually download + from, see getHostWithAlt(). """ + if hostUrl is None: hostUrl = PandaSystem.getPackageHostUrl() @@ -261,7 +304,7 @@ class AppRunner(DirectObject): # It's stale, get a new one. url = URLSpec(host.hostUrlPrefix + fileSpec.filename) - print "Downloading %s" % (url) + print "Freshening %s" % (url) doc = self.http.getDocument(url) if not doc.isValid(): return False @@ -343,7 +386,6 @@ class AppRunner(DirectObject): # Now set up Python to import this stuff. VFSImporter.register() sys.path.append(self.multifileRoot) - print "sys.path is: %s" % (sys.path) # Put our root directory on the model-path, too. getModelPath().appendDirectory(self.multifileRoot) @@ -490,6 +532,9 @@ class AppRunner(DirectObject): # aren't instance-ready. sys.argv = argv + # That means we now know the altHost in effect. + self.altHost = self.tokenDict.get('alt_host', None) + # Tell the browser that Python is up and running, and ready to # respond to queries. self.notifyRequest('onpythonload') @@ -527,6 +572,11 @@ class AppRunner(DirectObject): if allowPythonDev: self.allowPythonDev = int(allowPythonDev) + xhost = self.p3dConfig.FirstChildElement('host') + while xhost: + self.__readHostXml(xhost) + xhost = xhost.NextSiblingElement('host') + # The interactiveConsole flag can only be set true if the # application has allow_python_dev set. if not self.allowPythonDev and interactiveConsole: @@ -550,6 +600,27 @@ class AppRunner(DirectObject): # Send this call to the main thread; don't call it directly. messenger.send('AppRunner_startIfReady', taskChain = 'default') + def __readHostXml(self, xhost): + """ Reads the data in the indicated entry. """ + + url = xhost.Attribute('url') + host = self.getHost(url) + host.readHostXml(xhost) + + # Scan for a matching . If found, it means we + # should use the alternate URL instead of the original URL. + if self.altHost: + xalthost = xhost.FirstChildElement('alt_host') + while xalthost: + keyword = xalthost.Attribute('keyword') + if keyword == self.altHost: + origUrl = xhost.Attribute('url') + newUrl = xalthost.Attribute('url') + self.altHostMap[origUrl] = newUrl + break + + xalthost = xalthost.NextSiblingElement('alt_host') + def loadMultifilePrcFiles(self, mf, root): """ Loads any prc files in the root of the indicated Multifile, which is presumbed to have been mounted already @@ -765,6 +836,7 @@ def dummyAppRunner(tokens = [], argv = None): if argv is None: argv = sys.argv appRunner.argv = argv + appRunner.altHost = appRunner.tokenDict.get('alt_host', None) appRunner.p3dInfo = None appRunner.p3dPackage = None diff --git a/direct/src/p3d/FileSpec.py b/direct/src/p3d/FileSpec.py index 729af1a992..fe926912ae 100644 --- a/direct/src/p3d/FileSpec.py +++ b/direct/src/p3d/FileSpec.py @@ -119,12 +119,7 @@ class FileSpec: # The hash is OK after all. Change the file's timestamp back # to what we expect it to be, so we can quick-verify it # successfully next time. - - # On Windows, we have to change the file to read-write before - # we can successfully update its timestamp. - os.chmod(pathname.toOsSpecific(), 0755) - os.utime(pathname.toOsSpecific(), (st.st_atime, self.timestamp)) - os.chmod(pathname.toOsSpecific(), 0555) + self.__updateTimestamp(pathname, st) return True @@ -162,10 +157,17 @@ class FileSpec: # to what we expect it to be, so we can quick-verify it # successfully next time. if st.st_mtime != self.timestamp: - os.utime(pathname.toOsSpecific(), (st.st_atime, self.timestamp)) + self.__updateTimestamp(pathname, st) return True + def __updateTimestamp(self, pathname, st): + # On Windows, we have to change the file to read-write before + # we can successfully update its timestamp. + os.chmod(pathname.toOsSpecific(), 0755) + os.utime(pathname.toOsSpecific(), (st.st_atime, self.timestamp)) + os.chmod(pathname.toOsSpecific(), 0555) + def checkHash(self, packageDir, pathname, st): """ Returns true if the file has the expected md5 hash, false otherwise. As a side effect, stores a FileSpec corresponding diff --git a/direct/src/p3d/HostInfo.py b/direct/src/p3d/HostInfo.py index 1e2dc9d696..bbf7b61901 100644 --- a/direct/src/p3d/HostInfo.py +++ b/direct/src/p3d/HostInfo.py @@ -23,7 +23,16 @@ class HostInfo: # descriptiveName will be filled in later, when the # contents file is read. - self.descriptiveName = '' + self.descriptiveName = None + + # A list of known mirrors for this host. + self.mirrors = [] + + # A map of keyword -> altHost URL's. An altHost is different + # than a mirror; an altHost is an alternate URL to download a + # different (e.g. testing) version of this host's contents. + # It is rarely used. + self.altHosts = {} # This is a dictionary of packages by (name, version). It # will be filled in when the contents file is read. @@ -52,7 +61,7 @@ class HostInfo: request = DocumentSpec(url) request.setCacheControl(DocumentSpec.CCNoCache) - print "Downloading %s" % (request) + print "Downloading contents file %s" % (request) rf = Ramfile() channel = http.makeChannel(False) @@ -76,7 +85,8 @@ class HostInfo: def readContentsFile(self): """ Reads the contents.xml file for this particular host. - Presumably this has already been downloaded and installed. """ + Raises ValueError if the contents file is not already on disk + or is unreadable. """ if self.hasContentsFile: # No need to read it again. @@ -92,7 +102,8 @@ class HostInfo: if not xcontents: raise ValueError - self.descriptiveName = xcontents.Attribute('descriptive_name') + # Look for our own entry in the hosts table. + self.__findHostXml(xcontents) # Get the list of packages available for download and/or import. xpackage = xcontents.FirstChildElement('package') @@ -115,6 +126,51 @@ class HostInfo: self.hasContentsFile = True + def __findHostXml(self, xcontents): + """ Looks for the or entry in the + contents.xml that corresponds to the URL that we actually + downloaded from. """ + + xhost = xcontents.FirstChildElement('host') + while xhost: + url = xhost.Attribute('url') + if url == self.hostUrl: + self.readHostXml(xhost) + return + + xalthost = xhost.FirstChildElement('alt_host') + while xalthost: + url = xalthost.Attribute('url') + if url == self.hostUrl: + self.readHostXml(xalthost) + return + xalthost = xalthost.NextSiblingElement('alt_host') + + xhost = xhost.NextSiblingElement('host') + + def readHostXml(self, xhost): + """ Reads a or entry and applies the data to + this object. """ + + descriptiveName = xhost.Attribute('descriptive_name') + if descriptiveName and not self.descriptiveName: + self.descriptiveName = descriptiveName + + xmirror = xhost.FirstChildElement('mirror') + while xmirror: + url = xmirror.Attribute('url') + if url and url not in self.mirrors: + self.mirrors.append(url) + xmirror = xmirror.NextSiblingElement('mirror') + + xalthost = xhost.FirstChildElement('alt_host') + while xalthost: + keyword = xalthost.Attribute('keyword') + url = xalthost.Attribute('url') + if url and keyword: + self.altHosts[keyword] = url + xalthost = xalthost.NextSiblingElement('alt_host') + def __makePackage(self, name, platform, version): """ Creates a new PackageInfo entry for the given name, version, and platform. If there is already a matching diff --git a/direct/src/p3d/PackageInfo.py b/direct/src/p3d/PackageInfo.py index a1d035b691..b39bac2c09 100644 --- a/direct/src/p3d/PackageInfo.py +++ b/direct/src/p3d/PackageInfo.py @@ -111,6 +111,12 @@ class PackageInfo: filename = Filename(self.packageDir, self.descFileBasename) if self.descFile.quickVerify(self.packageDir, pathname = filename): self.readDescFile() + if self.hasDescFile: + # Successfully read. We don't need to call + # checkArchiveStatus again, since readDescFile() + # has just done it. + self.hasPackage = True + return True if self.hasDescFile: if self.__checkArchiveStatus(): @@ -131,7 +137,7 @@ class PackageInfo: return True url = URLSpec(self.descFileUrl) - print "Downloading %s" % (url) + print "Downloading desc file %s" % (url) rf = Ramfile() channel = http.getDocument(url) @@ -336,9 +342,9 @@ class PackageInfo: allExtractsOk = False break - if allExtractsOk: - print "All %s extracts of %s seem good." % ( - len(self.extracts), self.packageName) +## if allExtractsOk: +## print "All %s extracts of %s seem good." % ( +## len(self.extracts), self.packageName) return allExtractsOk @@ -432,7 +438,7 @@ class PackageInfo: url = self.descFileUrl.rsplit('/', 1)[0] url += '/' + fileSpec.filename url = DocumentSpec(url) - print "Downloading %s" % (url) + print "Downloading package file %s" % (url) targetPathname = Filename(self.packageDir, fileSpec.filename) targetPathname.setBinary() diff --git a/direct/src/p3d/PackageInstaller.py b/direct/src/p3d/PackageInstaller.py index 1f8f1b68f4..026488caaa 100644 --- a/direct/src/p3d/PackageInstaller.py +++ b/direct/src/p3d/PackageInstaller.py @@ -223,7 +223,7 @@ class PackageInstaller(DirectObject): if self.state != self.S_initial: raise ValueError, 'addPackage called after donePackages' - host = self.appRunner.getHost(hostUrl) + host = self.appRunner.getHostWithAlt(hostUrl) pp = self.PendingPackage(packageName, version, host) self.packageLock.acquire() @@ -374,7 +374,7 @@ class PackageInstaller(DirectObject): def __packageStarted(self, pp): """ This method is called when a single package is beginning to download. """ - print "Downloading %s" % (pp.packageName) + print "Downloading package %s" % (pp.packageName) self.__callDownloadStarted() self.__callPackageStarted(pp) diff --git a/direct/src/p3d/Packager.py b/direct/src/p3d/Packager.py index 9009aca74f..7d057d8274 100644 --- a/direct/src/p3d/Packager.py +++ b/direct/src/p3d/Packager.py @@ -207,6 +207,56 @@ class Packager: return xpackage + class HostEntry: + def __init__(self, url = None, descriptiveName = None, mirrors = None): + self.url = url + self.descriptiveName = descriptiveName + self.mirrors = mirrors or [] + self.altHosts = {} + + def loadXml(self, xhost, packager): + self.url = xhost.Attribute('url') + self.descriptiveName = xhost.Attribute('descriptive_name') + self.mirrors = [] + xmirror = xhost.FirstChildElement('mirror') + while xmirror: + url = xmirror.Attribute('url') + self.mirrors.append(url) + xmirror = xmirror.NextSiblingElement('mirror') + + xalthost = xhost.FirstChildElement('alt_host') + while xalthost: + url = xalthost.Attribute('url') + he = packager.addHost(url) + he.loadXml(xalthost, packager) + xalthost = xalthost.NextSiblingElement('alt_host') + + def makeXml(self, packager = None): + """ Returns a new TiXmlElement. """ + xhost = TiXmlElement('host') + xhost.SetAttribute('url', self.url) + if self.descriptiveName: + xhost.SetAttribute('descriptive_name', self.descriptiveName) + + for mirror in self.mirrors: + xmirror = TiXmlElement('mirror') + xmirror.SetAttribute('url', mirror) + xhost.InsertEndChild(xmirror) + + if packager: + altHosts = self.altHosts.items() + altHosts.sort() + for keyword, alt in altHosts: + he = packager.hosts.get(alt, None) + if he: + xalthost = he.makeXml() + xalthost.SetValue('alt_host') + xalthost.SetAttribute('keyword', keyword) + xhost.InsertEndChild(xalthost) + + return xhost + + class Package: """ This is the full information on a particular package we are constructing. Don't confuse it with PackageEntry, above, @@ -921,20 +971,21 @@ class Packager: self.__addConfigs(xpackage) - requireThisHost = False + requireHosts = {} for package in self.requires: xrequires = TiXmlElement('requires') xrequires.SetAttribute('name', package.packageName) if package.version: xrequires.SetAttribute('version', package.version) xrequires.SetAttribute('host', package.host) - if package.host == self.packager.host: - requireThisHost = True + requireHosts[package.host] = True xpackage.InsertEndChild(xrequires) - if requireThisHost: - xhost = self.packager.makeHostXml() - xpackage.InsertEndChild(xhost) + for host in requireHosts.keys(): + he = self.packager.hosts.get(host, None) + if he: + xhost = he.makeXml(packager = self.packager) + xpackage.InsertEndChild(xhost) doc.InsertEndChild(xpackage) @@ -1485,10 +1536,9 @@ class Packager: # The download URL at which these packages will eventually be # hosted. + self.hosts = {} self.host = PandaSystem.getPackageHostUrl() - self.hostDescriptiveName = None - self.hostMirrors = [] - self.altHosts = {} + self.addHost(self.host) # A search list for previously-built local packages. self.installSearch = ConfigVariableSearchPath('pdef-path') @@ -1650,20 +1700,49 @@ class Packager: # file. self.contents = {} - def setHost(self, host, descriptiveName = None, mirrors = []): + def setHost(self, host, descriptiveName = None, mirrors = None): """ Specifies the URL that will ultimately host these contents. """ - + self.host = host - self.hostDescriptiveName = descriptiveName - self.hostMirrors = mirrors + self.addHost(host, descriptiveName, mirrors) - def addAltHost(self, keyword, host, descriptiveName = None, mirrors = []): - """ Adds an alternate host from which an alternate version of - these contents may be downloaded, if specified on the HTML - page. """ + def addHost(self, host, descriptiveName = None, mirrors = None): + """ Adds a host to the list of known download hosts. This + information will be written into any p3d files that reference + this host; this can be used to pre-define the possible mirrors + for a given host, for instance. Returns the newly-created + HostEntry object.""" - self.altHosts[keyword] = (host, descriptiveName, mirrors) + he = self.hosts.get(host, None) + if he is None: + # Define a new host entry + he = self.HostEntry(host, descriptiveName, mirrors) + self.hosts[host] = he + else: + # Update an existing host entry + if descriptiveName: + he.descriptiveName = descriptiveName + if mirrors: + he.mirrors = mirrors + + return he + + def addAltHost(self, keyword, altHost, origHost = None, + descriptiveName = None, mirrors = None): + """ Adds an alternate host to any already-known host. This + defines an alternate server that may be contacted, if + specified on the HTML page, which hosts a different version of + the server's contents. (This is different from a mirror, + which hosts an identical version of the server's contents.) + """ + + if not origHost: + origHost = self.host + + self.addHost(altHost, descriptiveName, mirrors) + he = self.addHost(origHost) + he.altHosts[keyword] = altHost def addWindowsSearchPath(self, searchPath, varname): """ Expands $varname, interpreting as a Windows-style search @@ -2576,6 +2655,7 @@ class Packager: """ Reads the contents.xml file at the beginning of processing. """ + self.hosts = {} self.contents = {} self.contentsChanged = False @@ -2587,8 +2667,16 @@ class Packager: xcontents = doc.FirstChildElement('contents') if xcontents: - if self.hostDescriptiveName is None: - self.hostDescriptiveName = xcontents.Attribute('descriptive_name') + xhost = xcontents.FirstChildElement('host') + while xhost: + he = self.HostEntry() + he.loadXml(xhost, self) + self.hosts[he.url] = he + xhost = xhost.NextSiblingElement('host') + + host = xcontents.Attribute('host') + if host: + self.host = host xpackage = xcontents.FirstChildElement('package') while xpackage: @@ -2597,6 +2685,10 @@ class Packager: self.contents[pe.getKey()] = pe xpackage = xpackage.NextSiblingElement('package') + # Since we've blown away the self.hosts map, we have to make + # sure that our own host at least is added to the map. + self.addHost(self.host) + def writeContentsFile(self): """ Rewrites the contents.xml file at the end of processing. """ @@ -2611,9 +2703,12 @@ class Packager: doc.InsertEndChild(decl) xcontents = TiXmlElement('contents') - - xhost = self.makeHostXml() - xcontents.InsertEndChild(xhost) + if self.host: + xcontents.SetAttribute('host', self.host) + he = self.hosts.get(self.host, None) + if he: + xhost = he.makeXml(packager = self) + xcontents.InsertEndChild(xhost) contents = self.contents.items() contents.sort() @@ -2623,32 +2718,6 @@ class Packager: doc.InsertEndChild(xcontents) doc.SaveFile() - - def makeHostXml(self): - """ Constructs the entry for this host. """ - xhost = self.makeHostXmlLine('host', self.host, self.hostDescriptiveName, self.hostMirrors) - - for keyword, (host, descriptiveName, mirrors) in self.altHosts.items(): - xalthost = self.makeHostXmlLine('alt_host', host, descriptiveName, mirrors) - xalthost.SetAttribute('keyword', keyword) - xhost.InsertEndChild(xalthost) - return xhost - - def makeHostXmlLine(self, element, host, descriptiveName, mirrors): - """ Constructs the or entry for the - indicated host and its mirrors. """ - - xhost = TiXmlElement(element) - xhost.SetAttribute('url', host) - if descriptiveName: - xhost.SetAttribute('descriptive_name', descriptiveName) - - for mirror in mirrors: - xmirror = TiXmlElement('mirror') - xmirror.SetAttribute('url', mirror) - xhost.InsertEndChild(xmirror) - - return xhost # The following class and function definitions represent a few sneaky diff --git a/direct/src/plugin/p3dHost.cxx b/direct/src/plugin/p3dHost.cxx index 9a113b790a..415a02ba1c 100644 --- a/direct/src/plugin/p3dHost.cxx +++ b/direct/src/plugin/p3dHost.cxx @@ -149,7 +149,6 @@ read_contents_file(const string &contents_filename) { const char *keyword = xalthost->Attribute("keyword"); const char *url = xalthost->Attribute("url"); if (keyword != NULL && url != NULL) { - cerr << "got alt host " << keyword << ": " << url << "\n"; _alt_hosts[keyword] = url; } xalthost = xalthost->NextSiblingElement("alt_host"); diff --git a/direct/src/plugin/p3dInstance.cxx b/direct/src/plugin/p3dInstance.cxx index 0306803908..daf12325ec 100644 --- a/direct/src/plugin/p3dInstance.cxx +++ b/direct/src/plugin/p3dInstance.cxx @@ -1255,7 +1255,19 @@ scan_app_desc_file(TiXmlDocument *doc) { version = ""; } P3DHost *host = inst_mgr->get_host(host_url); - P3DPackage *package = host->get_package(name, version, alt_host); + string this_alt_host = alt_host; + + // Look up in the p3d_info.xml file to see if this p3d file has + // a specific alt_host indication for this host_url. + string alt_host_url = find_alt_host_url(xpackage, host_url, alt_host); + if (!alt_host_url.empty()) { + // If it does, we go ahead and switch to that host now, + // instead of bothering to contact the original host. + host = inst_mgr->get_host(alt_host_url); + this_alt_host.clear(); + } + + P3DPackage *package = host->get_package(name, version, this_alt_host); add_package(package); } @@ -1263,6 +1275,40 @@ scan_app_desc_file(TiXmlDocument *doc) { } } +//////////////////////////////////////////////////////////////////// +// Function: P3DInstance::find_alt_host_url +// Access: Private +// Description: Looks in the p3d_info.xml file for the alt_host +// associated with the indicated host_url, if any. +// Returns empty string if there is no match. +//////////////////////////////////////////////////////////////////// +string P3DInstance:: +find_alt_host_url(TiXmlElement *xpackage, + const string &host_url, const string &alt_host) { + TiXmlElement *xhost = xpackage->FirstChildElement("host"); + while (xhost != NULL) { + const char *url = xhost->Attribute("url"); + if (url != NULL && host_url == url) { + // This matches the host. Now do we have a matching alt_host + // keyword for this host? + TiXmlElement *xalt_host = xhost->FirstChildElement("alt_host"); + while (xalt_host != NULL) { + const char *keyword = xalt_host->Attribute("keyword"); + if (keyword != NULL && alt_host == keyword) { + const char *alt_host_url = xalt_host->Attribute("url"); + if (alt_host_url != NULL) { + return alt_host_url; + } + } + xalt_host = xalt_host->NextSiblingElement("alt_host"); + } + } + xhost = xhost->NextSiblingElement("host"); + } + + return string(); +} + //////////////////////////////////////////////////////////////////// // Function: P3DInstance::send_browser_script_object // Access: Private diff --git a/direct/src/plugin/p3dInstance.h b/direct/src/plugin/p3dInstance.h index ddec17a44f..e233334790 100644 --- a/direct/src/plugin/p3dInstance.h +++ b/direct/src/plugin/p3dInstance.h @@ -153,6 +153,8 @@ private: void mark_p3d_untrusted(); void mark_p3d_trusted(); void scan_app_desc_file(TiXmlDocument *doc); + string find_alt_host_url(TiXmlElement *xpackage, + const string &host_url, const string &alt_host); void send_browser_script_object(); P3D_request *make_p3d_request(TiXmlElement *xrequest);