alt_host

direct/src/p3d/AppRunner.py

@@ -113,6 +113,12 @@ class AppRunner(DirectObject):
         # hosts we have imported packages from.
         self.hosts = {}
 
+        # The altHost string that is in effect from the HTML tokens,
+        # if any, and the dictionary of URL remapping: orig host url
+        # -> alt host url.
+        self.altHost = None
+        self.altHostMap = {}
+
         # Application code can assign a callable object here; if so,
         # it will be invoked when an uncaught exception propagates to
         # the top of the TaskMgr.run() loop.
@@ -213,7 +219,7 @@ class AppRunner(DirectObject):
         finished; see the PackageInstaller class if you want this to
         happen asynchronously instead. """
 
-        host = self.getHost(hostUrl)
+        host = self.getHostWithAlt(hostUrl)
         if not host.downloadContentsFile(self.http):
             return False
 
@@ -235,10 +241,47 @@ class AppRunner(DirectObject):
 
         print "Package %s %s installed." % (packageName, version)
 
+    def getHostWithAlt(self, hostUrl):
+        """ Returns a suitable HostInfo object for downloading
+        contents from the indicated URL.  This is almost always the
+        same thing as getHost(), except in the rare case when we have
+        an alt_host specified in the HTML tokens; in this case, we may
+        actually want to download the contents from a different URL
+        than the one given, for instance to download a version in
+        testing. """
+
+        altUrl = self.altHostMap.get(hostUrl, None)
+        if altUrl:
+            # We got an alternate host.  Use it.
+            return self.getHost(altUrl)
+
+        # We didn't get an aternate host, use the original.
+        host = self.getHost(hostUrl)
+
+        # But we might need to consult the host itself to see if *it*
+        # recommends an altHost.
+        if self.altHost:
+            # This means forcing the host to download its contents
+            # file on the spot, a blocking operation.  This is a
+            # little unfortunate, but since alt_host is so rarely
+            # used, probably not really a problem.
+            host.downloadContentsFile(self.http)
+            altUrl = host.altHosts.get(self.altHost, None)
+            if altUrl:
+                return self.getHost(altUrl)
+
+        # No shenanigans, just return the requested host.
+        return host
+        
     def getHost(self, hostUrl):
         """ Returns a new HostInfo object corresponding to the
         indicated host URL.  If we have already seen this URL
-        previously, returns the same object. """
+        previously, returns the same object.
+
+        This returns the literal referenced host.  To return the
+        mapped host, which is the one we should actually download
+        from, see getHostWithAlt().  """
+
 
         if hostUrl is None:
             hostUrl = PandaSystem.getPackageHostUrl()
@@ -261,7 +304,7 @@ class AppRunner(DirectObject):
 
         # It's stale, get a new one.
         url = URLSpec(host.hostUrlPrefix + fileSpec.filename)
-        print "Downloading %s" % (url)
+        print "Freshening %s" % (url)
         doc = self.http.getDocument(url)
         if not doc.isValid():
             return False
@@ -343,7 +386,6 @@ class AppRunner(DirectObject):
         # Now set up Python to import this stuff.
         VFSImporter.register()
         sys.path.append(self.multifileRoot)
-        print "sys.path is: %s" % (sys.path)
 
         # Put our root directory on the model-path, too.
         getModelPath().appendDirectory(self.multifileRoot)
@@ -490,6 +532,9 @@ class AppRunner(DirectObject):
         # aren't instance-ready.
         sys.argv = argv
 
+        # That means we now know the altHost in effect.
+        self.altHost = self.tokenDict.get('alt_host', None)
+
         # Tell the browser that Python is up and running, and ready to
         # respond to queries.
         self.notifyRequest('onpythonload')
@@ -527,6 +572,11 @@ class AppRunner(DirectObject):
             if allowPythonDev:
                 self.allowPythonDev = int(allowPythonDev)
 
+            xhost = self.p3dConfig.FirstChildElement('host')
+            while xhost:
+                self.__readHostXml(xhost)
+                xhost = xhost.NextSiblingElement('host')
+
         # The interactiveConsole flag can only be set true if the
         # application has allow_python_dev set.
         if not self.allowPythonDev and interactiveConsole:
@@ -550,6 +600,27 @@ class AppRunner(DirectObject):
         # Send this call to the main thread; don't call it directly.
         messenger.send('AppRunner_startIfReady', taskChain = 'default')
 
+    def __readHostXml(self, xhost):
+        """ Reads the data in the indicated <host> entry. """
+
+        url = xhost.Attribute('url')
+        host = self.getHost(url)
+        host.readHostXml(xhost)
+
+        # Scan for a matching <alt_host>.  If found, it means we
+        # should use the alternate URL instead of the original URL.
+        if self.altHost:
+            xalthost = xhost.FirstChildElement('alt_host')
+            while xalthost:
+                keyword = xalthost.Attribute('keyword')
+                if keyword == self.altHost:
+                    origUrl = xhost.Attribute('url')
+                    newUrl = xalthost.Attribute('url')
+                    self.altHostMap[origUrl] = newUrl
+                    break
+                
+                xalthost = xalthost.NextSiblingElement('alt_host')
+    
     def loadMultifilePrcFiles(self, mf, root):
         """ Loads any prc files in the root of the indicated
         Multifile, which is presumbed to have been mounted already
@@ -765,6 +836,7 @@ def dummyAppRunner(tokens = [], argv = None):
     if argv is None:
         argv = sys.argv
     appRunner.argv = argv
+    appRunner.altHost = appRunner.tokenDict.get('alt_host', None)
 
     appRunner.p3dInfo = None
     appRunner.p3dPackage = None

direct/src/p3d/FileSpec.py

@@ -119,12 +119,7 @@ class FileSpec:
         # The hash is OK after all.  Change the file's timestamp back
         # to what we expect it to be, so we can quick-verify it
         # successfully next time.
-
+        self.__updateTimestamp(pathname, st)
-        # On Windows, we have to change the file to read-write before
-        # we can successfully update its timestamp.
-        os.chmod(pathname.toOsSpecific(), 0755)
-        os.utime(pathname.toOsSpecific(), (st.st_atime, self.timestamp))
-        os.chmod(pathname.toOsSpecific(), 0555)
 
         return True
         
@@ -162,10 +157,17 @@ class FileSpec:
         # to what we expect it to be, so we can quick-verify it
         # successfully next time.
         if st.st_mtime != self.timestamp:
-            os.utime(pathname.toOsSpecific(), (st.st_atime, self.timestamp))
+            self.__updateTimestamp(pathname, st)
 
         return True
 
+    def __updateTimestamp(self, pathname, st):
+        # On Windows, we have to change the file to read-write before
+        # we can successfully update its timestamp.
+        os.chmod(pathname.toOsSpecific(), 0755)
+        os.utime(pathname.toOsSpecific(), (st.st_atime, self.timestamp))
+        os.chmod(pathname.toOsSpecific(), 0555)
+
     def checkHash(self, packageDir, pathname, st):
         """ Returns true if the file has the expected md5 hash, false
         otherwise.  As a side effect, stores a FileSpec corresponding

direct/src/p3d/HostInfo.py

@@ -23,7 +23,16 @@ class HostInfo:
 
         # descriptiveName will be filled in later, when the
         # contents file is read.
-        self.descriptiveName = ''
+        self.descriptiveName = None
+
+        # A list of known mirrors for this host.
+        self.mirrors = []
+
+        # A map of keyword -> altHost URL's.  An altHost is different
+        # than a mirror; an altHost is an alternate URL to download a
+        # different (e.g. testing) version of this host's contents.
+        # It is rarely used.
+        self.altHosts = {}
 
         # This is a dictionary of packages by (name, version).  It
         # will be filled in when the contents file is read.
@@ -52,7 +61,7 @@ class HostInfo:
         request = DocumentSpec(url)
         request.setCacheControl(DocumentSpec.CCNoCache)
 
-        print "Downloading %s" % (request)
+        print "Downloading contents file %s" % (request)
 
         rf = Ramfile()
         channel = http.makeChannel(False)
@@ -76,7 +85,8 @@ class HostInfo:
 
     def readContentsFile(self):
         """ Reads the contents.xml file for this particular host.
-        Presumably this has already been downloaded and installed. """
+        Raises ValueError if the contents file is not already on disk
+        or is unreadable. """
 
         if self.hasContentsFile:
             # No need to read it again.
@@ -92,7 +102,8 @@ class HostInfo:
         if not xcontents:
             raise ValueError
 
-        self.descriptiveName = xcontents.Attribute('descriptive_name')
+        # Look for our own entry in the hosts table.
+        self.__findHostXml(xcontents)
 
         # Get the list of packages available for download and/or import.
         xpackage = xcontents.FirstChildElement('package')
@@ -115,6 +126,51 @@ class HostInfo:
 
         self.hasContentsFile = True
 
+    def __findHostXml(self, xcontents):
+        """ Looks for the <host> or <alt_host> entry in the
+        contents.xml that corresponds to the URL that we actually
+        downloaded from. """
+        
+        xhost = xcontents.FirstChildElement('host')
+        while xhost:
+            url = xhost.Attribute('url')
+            if url == self.hostUrl:
+                self.readHostXml(xhost)
+                return
+
+            xalthost = xhost.FirstChildElement('alt_host')
+            while xalthost:
+                url = xalthost.Attribute('url')
+                if url == self.hostUrl:
+                    self.readHostXml(xalthost)
+                    return
+                xalthost = xalthost.NextSiblingElement('alt_host')
+            
+            xhost = xhost.NextSiblingElement('host')
+
+    def readHostXml(self, xhost):
+        """ Reads a <host> or <alt_host> entry and applies the data to
+        this object. """
+
+        descriptiveName = xhost.Attribute('descriptive_name')
+        if descriptiveName and not self.descriptiveName:
+            self.descriptiveName = descriptiveName
+            
+        xmirror = xhost.FirstChildElement('mirror')
+        while xmirror:
+            url = xmirror.Attribute('url')
+            if url and url not in self.mirrors:
+                self.mirrors.append(url)
+            xmirror = xmirror.NextSiblingElement('mirror')
+
+        xalthost = xhost.FirstChildElement('alt_host')
+        while xalthost:
+            keyword = xalthost.Attribute('keyword')
+            url = xalthost.Attribute('url')
+            if url and keyword:
+                self.altHosts[keyword] = url
+            xalthost = xalthost.NextSiblingElement('alt_host')
+
     def __makePackage(self, name, platform, version):
         """ Creates a new PackageInfo entry for the given name,
         version, and platform.  If there is already a matching

direct/src/p3d/PackageInfo.py

@@ -111,6 +111,12 @@ class PackageInfo:
             filename = Filename(self.packageDir, self.descFileBasename)
             if self.descFile.quickVerify(self.packageDir, pathname = filename):
                 self.readDescFile()
+                if self.hasDescFile:
+                    # Successfully read.  We don't need to call
+                    # checkArchiveStatus again, since readDescFile()
+                    # has just done it.
+                    self.hasPackage = True
+                    return True
 
         if self.hasDescFile:
             if self.__checkArchiveStatus():
@@ -131,7 +137,7 @@ class PackageInfo:
             return True
 
         url = URLSpec(self.descFileUrl)
-        print "Downloading %s" % (url)
+        print "Downloading desc file %s" % (url)
 
         rf = Ramfile()
         channel = http.getDocument(url)
@@ -336,9 +342,9 @@ class PackageInfo:
                     allExtractsOk = False
                     break
 
-        if allExtractsOk:
+##         if allExtractsOk:
-            print "All %s extracts of %s seem good." % (
+##             print "All %s extracts of %s seem good." % (
-                len(self.extracts), self.packageName)
+##                 len(self.extracts), self.packageName)
 
         return allExtractsOk
 
@@ -432,7 +438,7 @@ class PackageInfo:
         url = self.descFileUrl.rsplit('/', 1)[0]
         url += '/' + fileSpec.filename
         url = DocumentSpec(url)
-        print "Downloading %s" % (url)
+        print "Downloading package file %s" % (url)
 
         targetPathname = Filename(self.packageDir, fileSpec.filename)
         targetPathname.setBinary()

direct/src/p3d/PackageInstaller.py

@@ -223,7 +223,7 @@ class PackageInstaller(DirectObject):
         if self.state != self.S_initial:
             raise ValueError, 'addPackage called after donePackages'
 
-        host = self.appRunner.getHost(hostUrl)
+        host = self.appRunner.getHostWithAlt(hostUrl)
         pp = self.PendingPackage(packageName, version, host)
 
         self.packageLock.acquire()
@@ -374,7 +374,7 @@ class PackageInstaller(DirectObject):
     def __packageStarted(self, pp):
         """ This method is called when a single package is beginning
         to download. """
-        print "Downloading %s" % (pp.packageName)
+        print "Downloading package %s" % (pp.packageName)
         self.__callDownloadStarted()
         self.__callPackageStarted(pp)
 

direct/src/p3d/Packager.py

@@ -207,6 +207,56 @@ class Packager:
             
             return xpackage
 
+    class HostEntry:
+        def __init__(self, url = None, descriptiveName = None, mirrors = None):
+            self.url = url
+            self.descriptiveName = descriptiveName
+            self.mirrors = mirrors or []
+            self.altHosts = {}
+
+        def loadXml(self, xhost, packager):
+            self.url = xhost.Attribute('url')
+            self.descriptiveName = xhost.Attribute('descriptive_name')
+            self.mirrors = []
+            xmirror = xhost.FirstChildElement('mirror')
+            while xmirror:
+                url = xmirror.Attribute('url')
+                self.mirrors.append(url)
+                xmirror = xmirror.NextSiblingElement('mirror')
+
+            xalthost = xhost.FirstChildElement('alt_host')
+            while xalthost:
+                url = xalthost.Attribute('url')
+                he = packager.addHost(url)
+                he.loadXml(xalthost, packager)
+                xalthost = xalthost.NextSiblingElement('alt_host')
+
+        def makeXml(self, packager = None):
+            """ Returns a new TiXmlElement. """
+            xhost = TiXmlElement('host')
+            xhost.SetAttribute('url', self.url)
+            if self.descriptiveName:
+                xhost.SetAttribute('descriptive_name', self.descriptiveName)
+
+            for mirror in self.mirrors:
+                xmirror = TiXmlElement('mirror')
+                xmirror.SetAttribute('url', mirror)
+                xhost.InsertEndChild(xmirror)
+
+            if packager:
+                altHosts = self.altHosts.items()
+                altHosts.sort()
+                for keyword, alt in altHosts:
+                    he = packager.hosts.get(alt, None)
+                    if he:
+                        xalthost = he.makeXml()
+                        xalthost.SetValue('alt_host')
+                        xalthost.SetAttribute('keyword', keyword)
+                        xhost.InsertEndChild(xalthost)
+
+            return xhost
+
+
     class Package:
         """ This is the full information on a particular package we
         are constructing.  Don't confuse it with PackageEntry, above,
@@ -921,20 +971,21 @@ class Packager:
 
             self.__addConfigs(xpackage)
 
-            requireThisHost = False
+            requireHosts = {}
             for package in self.requires:
                 xrequires = TiXmlElement('requires')
                 xrequires.SetAttribute('name', package.packageName)
                 if package.version:
                     xrequires.SetAttribute('version', package.version)
                 xrequires.SetAttribute('host', package.host)
-                if package.host == self.packager.host:
+                requireHosts[package.host] = True
-                    requireThisHost = True
                 xpackage.InsertEndChild(xrequires)
 
-            if requireThisHost:
+            for host in requireHosts.keys():
-                xhost = self.packager.makeHostXml()
+                he = self.packager.hosts.get(host, None)
-                xpackage.InsertEndChild(xhost)
+                if he:
+                    xhost = he.makeXml(packager = self.packager)
+                    xpackage.InsertEndChild(xhost)
 
             doc.InsertEndChild(xpackage)
 
@@ -1485,10 +1536,9 @@ class Packager:
 
         # The download URL at which these packages will eventually be
         # hosted.
+        self.hosts = {}
         self.host = PandaSystem.getPackageHostUrl()
-        self.hostDescriptiveName = None
+        self.addHost(self.host)
-        self.hostMirrors = []
-        self.altHosts = {}
 
         # A search list for previously-built local packages.
         self.installSearch = ConfigVariableSearchPath('pdef-path')
@@ -1650,20 +1700,49 @@ class Packager:
         # file.
         self.contents = {}
 
-    def setHost(self, host, descriptiveName = None, mirrors = []):
+    def setHost(self, host, descriptiveName = None, mirrors = None):
         """ Specifies the URL that will ultimately host these
         contents. """
-        
+
         self.host = host
-        self.hostDescriptiveName = descriptiveName
+        self.addHost(host, descriptiveName, mirrors)
-        self.hostMirrors = mirrors
 
-    def addAltHost(self, keyword, host, descriptiveName = None, mirrors = []):
+    def addHost(self, host, descriptiveName = None, mirrors = None):
-        """ Adds an alternate host from which an alternate version of
+        """ Adds a host to the list of known download hosts.  This
-        these contents may be downloaded, if specified on the HTML
+        information will be written into any p3d files that reference
-        page. """
+        this host; this can be used to pre-define the possible mirrors
+        for a given host, for instance.  Returns the newly-created
+        HostEntry object."""
 
-        self.altHosts[keyword] = (host, descriptiveName, mirrors)
+        he = self.hosts.get(host, None)
+        if he is None:
+            # Define a new host entry
+            he = self.HostEntry(host, descriptiveName, mirrors)
+            self.hosts[host] = he
+        else:
+            # Update an existing host entry
+            if descriptiveName:
+                he.descriptiveName = descriptiveName
+            if mirrors:
+                he.mirrors = mirrors
+
+        return he
+        
+    def addAltHost(self, keyword, altHost, origHost = None,
+                   descriptiveName = None, mirrors = None):
+        """ Adds an alternate host to any already-known host.  This
+        defines an alternate server that may be contacted, if
+        specified on the HTML page, which hosts a different version of
+        the server's contents.  (This is different from a mirror,
+        which hosts an identical version of the server's contents.)
+        """
+
+        if not origHost:
+            origHost = self.host
+
+        self.addHost(altHost, descriptiveName, mirrors)
+        he = self.addHost(origHost)
+        he.altHosts[keyword] = altHost
 
     def addWindowsSearchPath(self, searchPath, varname):
         """ Expands $varname, interpreting as a Windows-style search
@@ -2576,6 +2655,7 @@ class Packager:
         """ Reads the contents.xml file at the beginning of
         processing. """
 
+        self.hosts = {}
         self.contents = {}
         self.contentsChanged = False
 
@@ -2587,8 +2667,16 @@ class Packager:
 
         xcontents = doc.FirstChildElement('contents')
         if xcontents:
-            if self.hostDescriptiveName is None:
+            xhost = xcontents.FirstChildElement('host')
-                self.hostDescriptiveName = xcontents.Attribute('descriptive_name')
+            while xhost:
+                he = self.HostEntry()
+                he.loadXml(xhost, self)
+                self.hosts[he.url] = he
+                xhost = xhost.NextSiblingElement('host')
+
+            host = xcontents.Attribute('host')
+            if host:
+                self.host = host
             
             xpackage = xcontents.FirstChildElement('package')
             while xpackage:
@@ -2597,6 +2685,10 @@ class Packager:
                 self.contents[pe.getKey()] = pe
                 xpackage = xpackage.NextSiblingElement('package')
 
+        # Since we've blown away the self.hosts map, we have to make
+        # sure that our own host at least is added to the map.
+        self.addHost(self.host)
+
     def writeContentsFile(self):
         """ Rewrites the contents.xml file at the end of
         processing. """
@@ -2611,9 +2703,12 @@ class Packager:
         doc.InsertEndChild(decl)
 
         xcontents = TiXmlElement('contents')
-
+        if self.host:
-        xhost = self.makeHostXml()
+            xcontents.SetAttribute('host', self.host)
-        xcontents.InsertEndChild(xhost)
+            he = self.hosts.get(self.host, None)
+            if he:
+                xhost = he.makeXml(packager = self)
+                xcontents.InsertEndChild(xhost)
 
         contents = self.contents.items()
         contents.sort()
@@ -2623,32 +2718,6 @@ class Packager:
 
         doc.InsertEndChild(xcontents)
         doc.SaveFile()
-
-    def makeHostXml(self):
-        """ Constructs the <host> entry for this host. """
-        xhost = self.makeHostXmlLine('host', self.host, self.hostDescriptiveName, self.hostMirrors)
-
-        for keyword, (host, descriptiveName, mirrors) in self.altHosts.items():
-            xalthost = self.makeHostXmlLine('alt_host', host, descriptiveName, mirrors)
-            xalthost.SetAttribute('keyword', keyword)
-            xhost.InsertEndChild(xalthost)
-        return xhost
-
-    def makeHostXmlLine(self, element, host, descriptiveName, mirrors):
-        """ Constructs the <host> or <alt_host> entry for the
-        indicated host and its mirrors. """
-
-        xhost = TiXmlElement(element)
-        xhost.SetAttribute('url', host)
-        if descriptiveName:
-            xhost.SetAttribute('descriptive_name', descriptiveName)
-
-        for mirror in mirrors:
-            xmirror = TiXmlElement('mirror')
-            xmirror.SetAttribute('url', mirror)
-            xhost.InsertEndChild(xmirror)
-
-        return xhost
         
 
 # The following class and function definitions represent a few sneaky

direct/src/plugin/p3dHost.cxx

@@ -149,7 +149,6 @@ read_contents_file(const string &contents_filename) {
         const char *keyword = xalthost->Attribute("keyword");
         const char *url = xalthost->Attribute("url");
         if (keyword != NULL && url != NULL) {
-          cerr << "got alt host " << keyword << ": " << url << "\n";
           _alt_hosts[keyword] = url;
         }
         xalthost = xalthost->NextSiblingElement("alt_host");

direct/src/plugin/p3dInstance.cxx

@@ -1255,7 +1255,19 @@ scan_app_desc_file(TiXmlDocument *doc) {
         version = "";
       }
       P3DHost *host = inst_mgr->get_host(host_url);
-      P3DPackage *package = host->get_package(name, version, alt_host);
+      string this_alt_host = alt_host;
+
+      // Look up in the p3d_info.xml file to see if this p3d file has
+      // a specific alt_host indication for this host_url.
+      string alt_host_url = find_alt_host_url(xpackage, host_url, alt_host);
+      if (!alt_host_url.empty()) {
+        // If it does, we go ahead and switch to that host now,
+        // instead of bothering to contact the original host.
+        host = inst_mgr->get_host(alt_host_url);
+        this_alt_host.clear();
+      }
+        
+      P3DPackage *package = host->get_package(name, version, this_alt_host);
       add_package(package);
     }
 
@@ -1263,6 +1275,40 @@ scan_app_desc_file(TiXmlDocument *doc) {
   }
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: P3DInstance::find_alt_host_url
+//       Access: Private
+//  Description: Looks in the p3d_info.xml file for the alt_host
+//               associated with the indicated host_url, if any.
+//               Returns empty string if there is no match.
+////////////////////////////////////////////////////////////////////
+string P3DInstance::
+find_alt_host_url(TiXmlElement *xpackage, 
+                  const string &host_url, const string &alt_host) {
+  TiXmlElement *xhost = xpackage->FirstChildElement("host");
+  while (xhost != NULL) {
+    const char *url = xhost->Attribute("url");
+    if (url != NULL && host_url == url) {
+      // This matches the host.  Now do we have a matching alt_host
+      // keyword for this host?
+      TiXmlElement *xalt_host = xhost->FirstChildElement("alt_host");
+      while (xalt_host != NULL) {
+        const char *keyword = xalt_host->Attribute("keyword");
+        if (keyword != NULL && alt_host == keyword) {
+          const char *alt_host_url = xalt_host->Attribute("url");
+          if (alt_host_url != NULL) {
+            return alt_host_url;
+          }
+        }
+        xalt_host = xalt_host->NextSiblingElement("alt_host");
+      }
+    }
+    xhost = xhost->NextSiblingElement("host");
+  }
+
+  return string();
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: P3DInstance::send_browser_script_object
 //       Access: Private

direct/src/plugin/p3dInstance.h

@@ -153,6 +153,8 @@ private:
   void mark_p3d_untrusted();
   void mark_p3d_trusted();
   void scan_app_desc_file(TiXmlDocument *doc);
+  string find_alt_host_url(TiXmlElement *xpackage, 
+                           const string &host_url, const string &alt_host);
 
   void send_browser_script_object();
   P3D_request *make_p3d_request(TiXmlElement *xrequest);