robustify initial download

direct/src/plugin/p3dFileDownload.cxx

@@ -59,11 +59,14 @@ set_filename(const string &filename) {
 bool P3DFileDownload::
 open_file() {
   if (!mkfile_complete(_filename, nout)) {
+    nout << "Failed to create " << _filename << "\n";
     return false;
   }
-  
+
-  _file.open(_filename.c_str(), ios::out | ios::ate | ios::binary);
+  _file.clear();
+  _file.open(_filename.c_str(), ios::out | ios::trunc | ios::binary);
   if (!_file) {
+    nout << "Failed to open " << _filename << " in write mode\n";
     return false;
   }
   

direct/src/plugin/p3dPackage.cxx

@@ -91,13 +91,11 @@ P3DPackage::
   // Cancel any pending download.
   if (_active_download != NULL) {
     _active_download->cancel();
-    delete _active_download;
+    set_active_download(NULL);
-    _active_download = NULL;
   }
   if (_saved_download != NULL) {
     _saved_download->cancel();
-    delete _saved_download;
+    set_saved_download(NULL);
-    _saved_download = NULL;
   }
 
   if (_temp_contents_file != NULL) {
@@ -196,8 +194,7 @@ remove_instance(P3DInstance *inst) {
     // move to the next instance.
     if (_active_download != NULL) {
       _active_download->cancel();
-      delete _active_download;
+      set_active_download(NULL);
-      _active_download = NULL;
     }
   }
 
@@ -355,9 +352,8 @@ redownload_contents_file(P3DPackage::Download *download) {
     host_got_contents_file();
     return;
   }
-
+  
-  _saved_download = download;
+  set_saved_download(download);
-  _saved_download->ref();
 
   // Download contents.xml to a temporary filename first.
   if (_temp_contents_file != NULL) {
@@ -410,8 +406,7 @@ contents_file_redownload_finished(bool success) {
   if (contents_changed) {
     // OK, the contents.xml has changed; this means we have to restart
     // the whole download process from the beginning.
-    unref_delete(_saved_download);
+    set_saved_download(NULL);
-    _saved_download = NULL;
     host_got_contents_file();
 
   } else {
@@ -1045,13 +1040,51 @@ start_download(P3DPackage::DownloadType dtype, const string &urlbase,
   download->set_url(url);
   download->set_filename(pathname);
 
-  _active_download = download;
+  set_active_download(download);
   assert(!_instances.empty());
 
   _instances[0]->start_download(download);
   return download;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: P3DPackage::set_active_download
+//       Access: Private
+//  Description: Changes _active_download to point to the indicated
+//               object, respecting reference counts.
+////////////////////////////////////////////////////////////////////
+void P3DPackage::
+set_active_download(Download *download) {
+  if (_active_download != download) {
+    if (_active_download != NULL) {
+      unref_delete(_active_download);
+    }
+    _active_download = download;
+    if (_active_download != NULL) {
+      _active_download->ref();
+    }
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: P3DPackage::set_saved_download
+//       Access: Private
+//  Description: Changes _saved_download to point to the indicated
+//               object, respecting reference counts.
+////////////////////////////////////////////////////////////////////
+void P3DPackage::
+set_saved_download(Download *download) {
+  if (_saved_download != download) {
+    if (_saved_download != NULL) {
+      unref_delete(_saved_download);
+    }
+    _saved_download = download;
+    if (_saved_download != NULL) {
+      _saved_download->ref();
+    }
+  }
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: P3DPackage::is_extractable
 //       Access: Private
@@ -1129,7 +1162,15 @@ void P3DPackage::Download::
 download_finished(bool success) {
   P3DFileDownload::download_finished(success);
   assert(_package->_active_download == this);
-  _package->_active_download = NULL;
+  if (get_ref_count() == 1) {
+    // No one cares anymore.
+    nout << "No one cares about " << get_url() << "\n";
+    _package->set_active_download(NULL);
+    return;
+  }
+
+  _package->set_active_download(NULL);
+  assert(get_ref_count() > 0);
 
   if (success && !_file_spec.get_filename().empty()) {
     // We think we downloaded it correctly.  Check the hash to be
@@ -1175,7 +1216,7 @@ resume_download_finished(bool success) {
     clear();
     set_url(url);
     set_filename(get_filename());
-    _package->_active_download = this;
+    _package->set_active_download(this);
 
     assert(!_package->_instances.empty());
     _package->_instances[0]->start_download(this);

direct/src/plugin/p3dPackage.h

@@ -206,6 +206,8 @@ private:
   void report_done(bool success);
   Download *start_download(DownloadType dtype, const string &urlbase, 
                            const string &pathname, const FileSpec &file_spec);
+  void set_active_download(Download *download);
+  void set_saved_download(Download *download);
 
   bool is_extractable(FileSpec &file, const string &filename) const;
 

direct/src/plugin/p3dReferenceCount.I

@@ -45,7 +45,7 @@ ref() const {
 
 ////////////////////////////////////////////////////////////////////
 //     Function: P3DReferenceCount::unref
-//       Access: Published, Virtual
+//       Access: Public
 //  Description: Explicitly decrements the reference count.  Usually,
 //               you should call unref_delete() instead.
 //
@@ -57,6 +57,16 @@ unref() const {
   return --(((P3DReferenceCount *)this)->_ref_count) != 0;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: P3DReferenceCount::get_ref_count
+//       Access: Public
+//  Description: Returns the current reference count.
+////////////////////////////////////////////////////////////////////
+inline int P3DReferenceCount::
+get_ref_count() const {
+  return _ref_count;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: unref_delete
 //  Description: This global helper function will unref the given

direct/src/plugin/p3dReferenceCount.h

@@ -31,6 +31,7 @@ public:
 
   inline void ref() const;
   inline bool unref() const;
+  inline int get_ref_count() const;
 
 private:
   int _ref_count;

direct/src/plugin_activex/P3DActiveX.cpp

@@ -167,6 +167,10 @@ BOOL CP3DActiveXApp::InitInstance()
     if (bInit)
     {
         // TODO: Add your own module initialization code here.
+
+      // Seed the lame random number generator in rand(); we use it to
+      // select a mirror for downloading.
+      srand((unsigned int)time(NULL));
     }
 
     return bInit;

direct/src/plugin_activex/PPInstance.cpp

@@ -18,6 +18,7 @@
 #include <sstream>
 #include <iostream>
 #include <fstream>
+#include <algorithm>
 
 #include <stdlib.h>
 #include <stdio.h>
@@ -39,6 +40,10 @@
 #include "find_root_dir.h"
 #include "mkdir_complete.h"
 
+// We can include this header file to get the DTOOL_PLATFORM
+// definition, even though we don't link with dtool.
+#include "dtool_platform.h"
+
 #define P3D_CONTENTS_FILENAME "contents.xml"
 #define P3D_DEFAULT_PLUGIN_FILENAME "p3d_plugin.dll"
 
@@ -101,11 +106,11 @@ PPInstance::~PPInstance(  )
 int PPInstance::DownloadFile( const std::string& from, const std::string& to )
 {
     int error( 0 );
-    PPDownloadRequest p3dContentsDownloadRequest( *this, to ); 
+    PPDownloadRequest p3dFileDownloadRequest( *this, to ); 
-    PPDownloadCallback dcForContents( p3dContentsDownloadRequest );
+    PPDownloadCallback dcForFile( p3dFileDownloadRequest );
 
     nout << "Downloading " << from << " into " << to << "\n";
-    HRESULT hr = ::URLOpenStream( m_parentCtrl.GetControllingUnknown(), from.c_str(), 0, &dcForContents );
+    HRESULT hr = ::URLOpenStream( m_parentCtrl.GetControllingUnknown(), from.c_str(), 0, &dcForFile );
     if ( FAILED( hr ) )
     {   
         error = 1;
@@ -140,35 +145,144 @@ int PPInstance::CopyFile( const std::string& from, const std::string& to )
   return 0;
 }
 
-int PPInstance::ReadContents( const std::string& contentsFilename, FileSpec& p3dDllFile )
+////////////////////////////////////////////////////////////////////
-{
+//     Function: PPInstance::read_contents_file
-    int error(1);
+//       Access: Private
+//  Description: Reads the contents.xml file and starts the core API
+//               DLL downloading, if necessary.
+////////////////////////////////////////////////////////////////////
+bool PPInstance::
+read_contents_file(const string &contents_filename) {
+  TiXmlDocument doc(contents_filename.c_str());
+  if (!doc.LoadFile()) {
+    return false;
+  }
 
-    TiXmlDocument doc( contentsFilename.c_str( ) );
+  TiXmlElement *xcontents = doc.FirstChildElement("contents");
-    if ( doc.LoadFile( ) ) 
+  if (xcontents != NULL) {
-    {
+    // Look for the <host> entry; it might point us at a different
-        TiXmlElement *xcontents = doc.FirstChildElement( "contents" );
+    // download URL, and it might mention some mirrors.
-        if ( xcontents != NULL ) 
+    string host_url = PANDA_PACKAGE_HOST_URL;
-        {
+    TiXmlElement *xhost = xcontents->FirstChildElement("host");
-            TiXmlElement *xpackage = xcontents->FirstChildElement( "package" );
+    if (xhost != NULL) {
-            while ( xpackage != NULL ) 
+      const char *url = xhost->Attribute("url");
-            {
+      if (url != NULL && host_url == string(url)) {
-                const char *name = xpackage->Attribute( "name" );
+        // We're the primary host.  This is the normal case.
-                if ( name != NULL && strcmp( name, "coreapi" ) == 0 ) 
+        read_xhost(xhost);
-                {
+
-                    const char *platform = xpackage->Attribute( "platform" );
+      } else {
-                    if ( platform != NULL && !strcmp(platform, "win32") ) 
+        // We're not the primary host; perhaps we're an alternate host.
-                    {
+        TiXmlElement *xalthost = xhost->FirstChildElement("alt_host");
-                        p3dDllFile.load_xml(xpackage);
+        while (xalthost != NULL) {
-                        error = 0;
+          const char *url = xalthost->Attribute("url");
-                        break;
+          if (url != NULL && host_url == string(url)) {
-                    }
+            // Yep, we're this alternate host.
-                }
+            read_xhost(xhost);
-                xpackage = xpackage->NextSiblingElement( "package" );
+            break;
-            }
+          }
+          xalthost = xalthost->NextSiblingElement("alt_host");
         }
+      }
     }
-    return error;
+
+    // Now look for the core API package.
+    TiXmlElement *xpackage = xcontents->FirstChildElement("package");
+    while (xpackage != NULL) {
+      const char *name = xpackage->Attribute("name");
+      if (name != NULL && strcmp(name, "coreapi") == 0) {
+        const char *platform = xpackage->Attribute("platform");
+        if (platform != NULL && strcmp(platform, DTOOL_PLATFORM) == 0) {
+          _core_api_dll.load_xml(xpackage);
+          return true;
+        }
+      }
+    
+      xpackage = xpackage->NextSiblingElement("package");
+    }
+  }
+
+  // Couldn't find the coreapi package description.
+  nout << "No coreapi package defined in contents file for "
+       << DTOOL_PLATFORM << "\n";
+  return false;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: PPInstance::read_xhost
+//       Access: Private
+//  Description: Reads the host data from the <host> (or <alt_host>)
+//               entry in the contents.xml file.
+////////////////////////////////////////////////////////////////////
+void PPInstance::
+read_xhost(TiXmlElement *xhost) {
+  // Get the "download" URL, which is the source from which we
+  // download everything other than the contents.xml file.
+  const char *download_url = xhost->Attribute("download_url");
+  if (download_url != NULL) {
+    _download_url_prefix = download_url;
+  } else {
+    _download_url_prefix = PANDA_PACKAGE_HOST_URL;
+  }
+  if (!_download_url_prefix.empty()) {
+    if (_download_url_prefix[_download_url_prefix.size() - 1] != '/') {
+      _download_url_prefix += "/";
+    }
+  }
+        
+  TiXmlElement *xmirror = xhost->FirstChildElement("mirror");
+  while (xmirror != NULL) {
+    const char *url = xmirror->Attribute("url");
+    if (url != NULL) {
+      add_mirror(url);
+    }
+    xmirror = xmirror->NextSiblingElement("mirror");
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: PPInstance::add_mirror
+//       Access: Private
+//  Description: Adds a new URL to serve as a mirror for this host.
+//               The mirrors will be consulted first, before
+//               consulting the host directly.
+////////////////////////////////////////////////////////////////////
+void PPInstance::
+add_mirror(std::string mirror_url) {
+  // Ensure the URL ends in a slash.
+  if (!mirror_url.empty() && mirror_url[mirror_url.size() - 1] != '/') {
+    mirror_url += '/';
+  }
+  
+  // Add it to the _mirrors list, but only if it's not already
+  // there.
+  if (std::find(_mirrors.begin(), _mirrors.end(), mirror_url) == _mirrors.end()) {
+    _mirrors.push_back(mirror_url);
+  }
+}
+    
+////////////////////////////////////////////////////////////////////
+//     Function: PPInstance::choose_random_mirrors
+//       Access: Public
+//  Description: Selects num_mirrors elements, chosen at random, from
+//               the _mirrors list.  Adds the selected mirrors to
+//               result.  If there are fewer than num_mirrors elements
+//               in the list, adds only as many mirrors as we can get.
+////////////////////////////////////////////////////////////////////
+void PPInstance::
+choose_random_mirrors(std::vector<std::string> &result, int num_mirrors) {
+  std::vector<size_t> selected;
+
+  size_t num_to_select = min(_mirrors.size(), (size_t)num_mirrors);
+  while (num_to_select > 0) {
+    size_t i = (size_t)(((double)rand() / (double)RAND_MAX) * _mirrors.size());
+    while (std::find(selected.begin(), selected.end(), i) != selected.end()) {
+      // Already found this i, find a new one.
+      i = (size_t)(((double)rand() / (double)RAND_MAX) * _mirrors.size());
+    }
+    selected.push_back(i);
+    result.push_back(_mirrors[i]);
+    --num_to_select;
+  }
 }
 
 int PPInstance::DownloadP3DComponents( std::string& p3dDllFilename )
@@ -204,17 +318,18 @@ int PPInstance::DownloadP3DComponents( std::string& p3dDllFilename )
     strm << hostUrl << P3D_CONTENTS_FILENAME << "?" << time(NULL);
     std::string remoteContentsUrl( strm.str() );
 
-    FileSpec p3dDllFile;
     error = DownloadFile( remoteContentsUrl, localContentsFileName );
     if ( !error )
     {
-        error = ReadContents( localContentsFileName, p3dDllFile );
+      if ( !read_contents_file( localContentsFileName ) )
+        error = 1;
     }
 
     if ( error ) {
       // If we couldn't download or read the contents.xml file, check
       // to see if there's a good one on disk already, as a fallback.
-      error = ReadContents( finalContentsFileName, p3dDllFile );
+      if ( !read_contents_file( finalContentsFileName ) )
+        error = 1;
 
     } else {
       // If we have successfully read the downloaded version,
@@ -226,34 +341,63 @@ int PPInstance::DownloadP3DComponents( std::string& p3dDllFilename )
     // We don't need the temporary file any more.
     ::DeleteFile( localContentsFileName.c_str() );
 
-    if ( !error )
+    if (!error) {
-    {
+      // OK, at this point we have successfully read contents.xml,
-        // OK, at this point we have successfully read contents.xml,
+      // and we have a good file spec in _core_api_dll.
-        // and we have a good file spec in p3dDllFile.
+      if (_core_api_dll.quick_verify(m_rootDir)) {
-        if ( p3dDllFile.quick_verify( m_rootDir ) )
+        // The DLL is already on-disk, and is good.
-        {
+        p3dDllFilename = _core_api_dll.get_pathname(m_rootDir);
-            // The DLL is already on-disk, and is good.
+      } else {
-            p3dDllFilename = p3dDllFile.get_pathname( m_rootDir );
+        // The DLL is not already on-disk, or it's stale.  Go get it.
+        std::string p3dLocalModuleFileName(_core_api_dll.get_pathname(m_rootDir));
+        mkfile_complete(p3dLocalModuleFileName, nout);
+
+        // Try one of the mirrors first.
+        std::vector<std::string> mirrors;
+        choose_random_mirrors(mirrors, 2);
+
+        error = 1;
+        for (std::vector<std::string>::iterator si = mirrors.begin();
+             si != mirrors.end() && error; 
+             ++si) {
+          std::string url = (*si) + _core_api_dll.get_filename();
+          error = DownloadFile(url, p3dLocalModuleFileName);
+          if (!error && !_core_api_dll.full_verify(m_rootDir)) {
+            // If it's not right after downloading, it's an error.
+            error = 1;
+          }
         }
-        else
+
-        {
+        // If that failed, go get it from the authoritative host.
-            // The DLL is not already on-disk, or it's stale.
+        if (error) {
-            std::string p3dLocalModuleFileName( p3dDllFile.get_pathname( m_rootDir ) );
+          std::string url = _download_url_prefix + _core_api_dll.get_filename();
-            mkfile_complete( p3dLocalModuleFileName, nout );
+          error = DownloadFile(url, p3dLocalModuleFileName);
-            std::string p3dRemoteModuleUrl( hostUrl );
+          if (!error && !_core_api_dll.full_verify(m_rootDir)) {
-            p3dRemoteModuleUrl += p3dDllFile.get_filename();
+            error = 1;
-            error = DownloadFile( p3dRemoteModuleUrl, p3dLocalModuleFileName );
+          }
-            if ( !error )
-            {
-                error = 1;
-                if ( p3dDllFile.full_verify( m_rootDir ) )
-                {
-                    // Downloaded successfully.
-                    p3dDllFilename = p3dDllFile.get_pathname( m_rootDir );
-                    error = 0;
-                }
-            }
         }
+
+        // If *that* failed, go get it again from the same URL, this
+        // time with a query prefix to bust through any caches.
+        if (error) {
+          std::ostringstream strm;
+          strm << _download_url_prefix << _core_api_dll.get_filename();
+          strm << "?" << time(NULL);
+
+          std::string url = strm.str();
+          error = DownloadFile(url, p3dLocalModuleFileName);
+          if (!error && !_core_api_dll.full_verify(m_rootDir)) {
+            nout << "After download, " << _core_api_dll.get_filename()
+                 << " is no good.\n";
+            error = 1;
+          }
+        }
+
+        if (!error) {
+          // Downloaded successfully.
+          p3dDllFilename = _core_api_dll.get_pathname(m_rootDir);
+        }
+      }
     }
     
     return error;
@@ -438,22 +582,18 @@ void PPInstance::HandleRequestGetUrl( void* data )
     PPDownloadRequest p3dObjectDownloadRequest( parent->m_instance, request ); 
     PPDownloadCallback bsc( p3dObjectDownloadRequest );
     HRESULT hr = ::URLOpenStream( parent->GetControllingUnknown(), url.c_str(), 0, &bsc );
+
+    P3D_result_code result_code = P3D_RC_done;
     if ( FAILED( hr ) )
     {
         nout << "Error handling P3D_RT_get_url request" << " :" << hr << "\n"; 
-        return;
+        result_code = P3D_RC_generic_error;
     }
-    //inet_InternetSession inet(parent->m_pythonEmbed.m_threadData.m_parent);
-    //std::string outdata;
-    //if ( !inet.getURLMemory( url, outdata, request ) )
-    //{
-    //    handled = false;
-    //}
 
     P3D_instance_feed_url_stream( 
         request->_instance, 
         request->_request._get_url._unique_id, 
-        P3D_RC_done, 
+        result_code, 
         0, 
         0, 
         (const void*)NULL, 

direct/src/plugin_activex/PPInstance.h

@@ -15,6 +15,7 @@
 #pragma once
 
 #include <string>
+#include <vector>
 #include <math.h>
 #include "afxmt.h"
 
@@ -69,7 +70,11 @@ protected:
 
     int DownloadFile( const std::string& from, const std::string& to );
     int CopyFile( const std::string& from, const std::string& to );
-    int ReadContents( const std::string& contentsFilename, FileSpec& p3dDllFile );
+
+    bool read_contents_file(const std::string &contents_filename);
+    void read_xhost(TiXmlElement *xhost);
+    void add_mirror(std::string mirror_url);
+    void choose_random_mirrors(std::vector<std::string> &result, int num_mirrors);
 
     void HandleRequest( P3D_request *request );
     static void HandleRequestGetUrl( void *data );
@@ -82,5 +87,10 @@ protected:
     bool m_isInit;
     bool m_pluginLoaded;
 
+    std::string _download_url_prefix;
+    typedef std::vector<std::string> Mirrors;
+    Mirrors _mirrors;
+    FileSpec _core_api_dll;
+
     std::string m_rootDir;
 };