fix caching issues, mirror issues

direct/src/p3d/Packager.py

@@ -208,14 +208,17 @@ class Packager:
             return xpackage
 
     class HostEntry:
-        def __init__(self, url = None, descriptiveName = None, mirrors = None):
+        def __init__(self, url = None, downloadUrl = None,
+                     descriptiveName = None, mirrors = None):
             self.url = url
+            self.downloadUrl = downloadUrl
             self.descriptiveName = descriptiveName
             self.mirrors = mirrors or []
             self.altHosts = {}
 
         def loadXml(self, xhost, packager):
             self.url = xhost.Attribute('url')
+            self.downloadUrl = xhost.Attribute('download_url')
             self.descriptiveName = xhost.Attribute('descriptive_name')
             self.mirrors = []
             xmirror = xhost.FirstChildElement('mirror')
@@ -235,6 +238,8 @@ class Packager:
             """ Returns a new TiXmlElement. """
             xhost = TiXmlElement('host')
             xhost.SetAttribute('url', self.url)
+            if self.downloadUrl and self.downloadUrl != self.url:
+                xhost.SetAttribute('download_url', self.downloadUrl)
             if self.descriptiveName:
                 xhost.SetAttribute('descriptive_name', self.descriptiveName)
 
@@ -1704,14 +1709,17 @@ class Packager:
         # file.
         self.contents = {}
 
-    def setHost(self, host, descriptiveName = None, mirrors = None):
+    def setHost(self, host, downloadUrl = None,
+                descriptiveName = None, mirrors = None):
         """ Specifies the URL that will ultimately host these
         contents. """
 
         self.host = host
-        self.addHost(host, descriptiveName, mirrors)
+        self.addHost(host, downloadUrl = downloadUrl,
+                     descriptiveName = descriptiveName, mirrors = mirrors)
 
-    def addHost(self, host, descriptiveName = None, mirrors = None):
+    def addHost(self, host, downloadUrl = None, descriptiveName = None,
+                mirrors = None):
         """ Adds a host to the list of known download hosts.  This
         information will be written into any p3d files that reference
         this host; this can be used to pre-define the possible mirrors
@@ -1721,11 +1729,15 @@ class Packager:
         he = self.hosts.get(host, None)
         if he is None:
             # Define a new host entry
-            he = self.HostEntry(host, descriptiveName, mirrors)
+            he = self.HostEntry(host, downloadUrl = downloadUrl,
+                                descriptiveName = descriptiveName,
+                                mirrors = mirrors)
             self.hosts[host] = he
         else:
             # Update an existing host entry
-            if descriptiveName:
+            if downloadUrl is not None:
+                he.downloadUrl = downloadUrl
+            if descriptiveName is not None:
                 he.descriptiveName = descriptiveName
             if mirrors is not None:
                 he.mirrors = mirrors

direct/src/p3d/coreapi.pdef

@@ -77,4 +77,3 @@ class p3dcert(package):
     filename = Filename(cvar.getValue())
     if not filename.empty():
         file(filename, newName = 'ca-bundle.crt', extract = True)
-        

direct/src/plugin/fileSpec.cxx

@@ -136,12 +136,23 @@ load_xml(TiXmlElement *xelement) {
 ////////////////////////////////////////////////////////////////////
 bool FileSpec::
 quick_verify(const string &package_dir) {
+  return quick_verify_pathname(get_pathname(package_dir));
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: FileSpec::quick_verify_pathname
+//       Access: Public
+//  Description: Works like quick_verify(), above, with an explicit
+//               pathname.  Useful for verifying the copy of a file in
+//               a temporary location.
+////////////////////////////////////////////////////////////////////
+bool FileSpec::
+quick_verify_pathname(const string &pathname) {
   if (_actual_file != NULL) {
     delete _actual_file;
     _actual_file = NULL;
   }
 
-  string pathname = get_pathname(package_dir);
   struct stat st;
   if (stat(pathname.c_str(), &st) != 0) {
     //cerr << "file not found: " << _filename << "\n";

direct/src/plugin/fileSpec.h

@@ -41,6 +41,7 @@ public:
   inline size_t get_size() const;
   
   bool quick_verify(const string &package_dir);
+  bool quick_verify_pathname(const string &pathname);
   bool full_verify(const string &package_dir);
   inline const FileSpec *get_actual_file() const;
   

direct/src/plugin/p3dDownload.cxx

@@ -84,6 +84,23 @@ cancel() {
   }    
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: P3DDownload::clear
+//       Access: Public
+//  Description: Resets the download to its initial state, for
+//               re-trying the same download.
+////////////////////////////////////////////////////////////////////
+void P3DDownload::
+clear() {
+  _status = P3D_RC_in_progress;
+  _http_status_code = 0;
+  _total_data = 0;
+  _last_reported_time = 0;
+  
+  _canceled = false;
+  _download_id = 0;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: P3DDownload::feed_url_stream
 //       Access: Public

direct/src/plugin/p3dDownload.h

@@ -43,6 +43,7 @@ public:
   inline size_t get_total_data() const;
 
   void cancel();
+  void clear();
 
 public:
   // These are intended to be called only by P3DInstance.

direct/src/plugin/p3dPackage.cxx

@@ -1172,14 +1172,13 @@ resume_download_finished(bool success) {
     string url = _try_urls.back();
     _try_urls.pop_back();
 
-    Download *new_download = new Download(*this);
+    clear();
-    new_download->set_filename(get_filename());
+    set_url(url);
-    new_download->set_url(url);
+    set_filename(get_filename());
-
+    _package->_active_download = this;
-    _package->_active_download = new_download;
 
     assert(!_package->_instances.empty());
-    _package->_instances[0]->start_download(new_download);
+    _package->_instances[0]->start_download(this);
     return;
   }
 

direct/src/plugin_npapi/ppInstance.cxx

@@ -26,6 +26,7 @@
 #include "dtool_platform.h"
 
 #include <fstream>
+#include <algorithm>
 #include <string.h>  // strcmp()
 #include <time.h>
 
@@ -113,6 +114,7 @@ begin() {
     if (!url.empty() && url[url.length() - 1] != '/') {
       url += '/';
     }
+    _download_url_prefix = url;
     ostringstream strm;
     strm << url << "contents.xml";
 
@@ -362,6 +364,20 @@ url_notify(const char *url, NPReason reason, void *notifyData) {
     }
     break;
     
+  case PPDownloadRequest::RT_core_dll:
+    if (reason != NPRES_DONE) {
+      nout << "Failure downloading " << url << "\n";
+      // Couldn't download from this mirror.  Try the next one.
+      if (!_core_urls.empty()) {
+        string url = _core_urls.back();
+        _core_urls.pop_back();
+        
+        PPDownloadRequest *req2 = new PPDownloadRequest(PPDownloadRequest::RT_core_dll);
+        start_download(url, req2);
+      }
+    }
+    break;
+
   default:
     nout << "Unexpected url_notify on stream type " << req->_rtype << "\n";
     break;
@@ -670,6 +686,84 @@ variant_to_p3dobj(const NPVariant *variant) {
   return P3D_new_none_object();
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: PPInstance::read_xhost
+//       Access: Private
+//  Description: Reads the host data from the <host> (or <alt_host>)
+//               entry in the contents.xml file.
+////////////////////////////////////////////////////////////////////
+void PPInstance::
+read_xhost(TiXmlElement *xhost) {
+  // Get the "download" URL, which is the source from which we
+  // download everything other than the contents.xml file.
+  const char *download_url = xhost->Attribute("download_url");
+  if (download_url != NULL) {
+    _download_url_prefix = download_url;
+  } else {
+    _download_url_prefix = PANDA_PACKAGE_HOST_URL;
+  }
+  if (!_download_url_prefix.empty()) {
+    if (_download_url_prefix[_download_url_prefix.size() - 1] != '/') {
+      _download_url_prefix += "/";
+    }
+  }
+        
+  TiXmlElement *xmirror = xhost->FirstChildElement("mirror");
+  while (xmirror != NULL) {
+    const char *url = xmirror->Attribute("url");
+    if (url != NULL) {
+      add_mirror(url);
+    }
+    xmirror = xmirror->NextSiblingElement("mirror");
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: PPInstance::add_mirror
+//       Access: Private
+//  Description: Adds a new URL to serve as a mirror for this host.
+//               The mirrors will be consulted first, before
+//               consulting the host directly.
+////////////////////////////////////////////////////////////////////
+void PPInstance::
+add_mirror(string mirror_url) {
+  // Ensure the URL ends in a slash.
+  if (!mirror_url.empty() && mirror_url[mirror_url.size() - 1] != '/') {
+    mirror_url += '/';
+  }
+  
+  // Add it to the _mirrors list, but only if it's not already
+  // there.
+  if (find(_mirrors.begin(), _mirrors.end(), mirror_url) == _mirrors.end()) {
+    _mirrors.push_back(mirror_url);
+  }
+}
+    
+////////////////////////////////////////////////////////////////////
+//     Function: PPInstance::choose_random_mirrors
+//       Access: Public
+//  Description: Selects num_mirrors elements, chosen at random, from
+//               the _mirrors list.  Adds the selected mirrors to
+//               result.  If there are fewer than num_mirrors elements
+//               in the list, adds only as many mirrors as we can get.
+////////////////////////////////////////////////////////////////////
+void PPInstance::
+choose_random_mirrors(vector<string> &result, int num_mirrors) {
+  vector<size_t> selected;
+
+  size_t num_to_select = min(_mirrors.size(), (size_t)num_mirrors);
+  while (num_to_select > 0) {
+    size_t i = (size_t)(((double)rand() / (double)RAND_MAX) * _mirrors.size());
+    while (find(selected.begin(), selected.end(), i) != selected.end()) {
+      // Already found this i, find a new one.
+      i = (size_t)(((double)rand() / (double)RAND_MAX) * _mirrors.size());
+    }
+    selected.push_back(i);
+    result.push_back(_mirrors[i]);
+    --num_to_select;
+  }
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: PPInstance::request_ready
 //       Access: Private, Static
@@ -719,6 +813,7 @@ request_ready(P3D_instance *instance) {
 ////////////////////////////////////////////////////////////////////
 void PPInstance::
 start_download(const string &url, PPDownloadRequest *req) {
+  nout << "start_download: " << url << "\n";
   if (url.substr(0, 7) == "file://") {
     // If we're "downloading" a file URL, just go read the file directly.
     downloaded_file(req, get_filename_from_url(url));
@@ -744,6 +839,32 @@ read_contents_file(const string &contents_filename) {
 
   TiXmlElement *xcontents = doc.FirstChildElement("contents");
   if (xcontents != NULL) {
+    // Look for the <host> entry; it might point us at a different
+    // download URL, and it might mention some mirrors.
+    string host_url = PANDA_PACKAGE_HOST_URL;
+    TiXmlElement *xhost = xcontents->FirstChildElement("host");
+    if (xhost != NULL) {
+      const char *url = xhost->Attribute("url");
+      if (url != NULL && host_url == string(url)) {
+        // We're the primary host.  This is the normal case.
+        read_xhost(xhost);
+
+      } else {
+        // We're not the primary host; perhaps we're an alternate host.
+        TiXmlElement *xalthost = xhost->FirstChildElement("alt_host");
+        while (xalthost != NULL) {
+          const char *url = xalthost->Attribute("url");
+          if (url != NULL && host_url == string(url)) {
+            // Yep, we're this alternate host.
+            read_xhost(xhost);
+            break;
+          }
+          xalthost = xalthost->NextSiblingElement("alt_host");
+        }
+      }
+    }
+
+    // Now look for the core API package.
     TiXmlElement *xpackage = xcontents->FirstChildElement("package");
     while (xpackage != NULL) {
       const char *name = xpackage->Attribute("name");
@@ -874,13 +995,39 @@ get_core_api(TiXmlElement *xpackage) {
     do_load_plugin();
 
   } else {
-    // The DLL file needs to be downloaded.  Go get it.
+    // The DLL file needs to be downloaded.  Build up our list of
-    string url = PANDA_PACKAGE_HOST_URL;
+    // URL's to attempt to download it from, in reverse order.
-    if (!url.empty() && url[url.length() - 1] != '/') {
+    string url;
-      url += '/';
+
-    }
+    // Our last act of desperation: hit the original host, with a
+    // query uniquifier, to break through any caches.
+    ostringstream strm;
+    strm << _download_url_prefix << _core_api_dll.get_filename()
+         << "?" << time(NULL);
+    url = strm.str();
+    _core_urls.push_back(url);
+
+    // Before we try that, we'll hit the original host, without a
+    // uniquifier.
+    url = _download_url_prefix;
     url += _core_api_dll.get_filename();
-    
+    _core_urls.push_back(url);
+
+    // And before we try that, we'll try two mirrors, at random.
+    vector<string> mirrors;
+    choose_random_mirrors(mirrors, 2);
+    for (vector<string>::iterator si = mirrors.begin();
+         si != mirrors.end(); 
+         ++si) {
+      url = (*si) + _core_api_dll.get_filename();
+      _core_urls.push_back(url);
+    }
+
+    // Now pick the first URL off the list, and try it.
+    assert(!_core_urls.empty());
+    url = _core_urls.back();
+    _core_urls.pop_back();
+
     PPDownloadRequest *req = new PPDownloadRequest(PPDownloadRequest::RT_core_dll);
     start_download(url, req);
   }
@@ -906,40 +1053,40 @@ downloaded_plugin(const string &filename) {
     return;
   }
 
-  // Copy the file onto the target.
+  // Make sure the DLL was correctly downloaded before continuing.
-  string pathname = _core_api_dll.get_pathname(_root_dir);
+  if (!_core_api_dll.quick_verify_pathname(filename)) {
-  mkfile_complete(pathname, nout);
+    nout << "After download, " << _core_api_dll.get_filename() << " is no good.\n";
 
-  ifstream in(filename.c_str(), ios::in | ios::binary);
+    // That DLL came out wrong.  Try the next URL.
-  ofstream out(pathname.c_str(), ios::out | ios::binary);
+    if (!_core_urls.empty()) {
+      string url = _core_urls.back();
+      _core_urls.pop_back();
+      
+      PPDownloadRequest *req = new PPDownloadRequest(PPDownloadRequest::RT_core_dll);
+      start_download(url, req);
+      return;
+    }
 
-  static const size_t buffer_size = 4096;
-  static char buffer[buffer_size];
-
-  in.read(buffer, buffer_size);
-  size_t count = in.gcount();
-  while (count != 0) {
-    out.write(buffer, count);
-    in.read(buffer, buffer_size);
-    count = in.gcount();
-  }
-
-  if (!out) {
-    nout << "Could not write " << pathname << "\n";
     // TODO: fail
     return;
   }
-  in.close();
-  out.close();
 
-  if (_core_api_dll.quick_verify(_root_dir)) {
+  // Copy the file onto the target.
-    // We downloaded and installed it successfully.  Now load it.
+  string pathname = _core_api_dll.get_pathname(_root_dir);
-    do_load_plugin();
+  if (!copy_file(filename, pathname)) {
+    nout << "Couldn't copy " << pathname << "\n";
     return;
   }
 
-  nout << "After download, " << pathname << " is no good.\n";
+  if (!_core_api_dll.quick_verify(_root_dir)) {
-  // TODO: fail
+    nout << "After copying, " << pathname << " is no good.\n";
+    // TODO: fail
+    return;
+  }
+
+  // We downloaded and installed it successfully.  Now load it.
+  do_load_plugin();
+  return;
 }
 
 
@@ -1141,6 +1288,7 @@ cleanup_window() {
 ////////////////////////////////////////////////////////////////////
 bool PPInstance::
 copy_file(const string &from_filename, const string &to_filename) {
+  mkfile_complete(to_filename, nout);
   ifstream in(from_filename.c_str(), ios::in | ios::binary);
   ofstream out(to_filename.c_str(), ios::out | ios::binary);
         

direct/src/plugin_npapi/ppInstance.h

@@ -64,6 +64,10 @@ public:
   static void output_np_variant(ostream &out, const NPVariant &result);
 
 private:
+  void read_xhost(TiXmlElement *xhost);
+  void add_mirror(string mirror_url);
+  void choose_random_mirrors(vector<string> &result, int num_mirrors);
+
   static void request_ready(P3D_instance *instance);
 
   void start_download(const string &url, PPDownloadRequest *req);
@@ -96,6 +100,15 @@ private:
   Tokens _tokens;
 
   string _root_dir;
+  string _download_url_prefix;
+  typedef vector<string> Mirrors;
+  Mirrors _mirrors;
+
+  // A list of URL's that we will attempt to download the core API
+  // from.
+  typedef vector<string> CoreUrls;
+  CoreUrls _core_urls;
+
   FileSpec _core_api_dll;
 
   bool _got_instance_url;