mirrors at the C++ level too

direct/src/plugin/fileSpec.cxx

@@ -46,6 +46,35 @@ FileSpec() {
   _got_hash = false;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: FileSpec::Copy Constructor
+//       Access: Public
+//  Description: 
+////////////////////////////////////////////////////////////////////
+FileSpec::
+FileSpec(const FileSpec &copy) :
+  _filename(copy._filename),
+  _size(copy._size),
+  _timestamp(copy._timestamp),
+  _got_hash(copy._got_hash)
+{
+  memcpy(_hash, copy._hash, sizeof(_hash));
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: FileSpec::Copy Assignment Operator
+//       Access: Public
+//  Description: 
+////////////////////////////////////////////////////////////////////
+void FileSpec::
+operator = (const FileSpec &copy) {
+  _filename = copy._filename;
+  _size = copy._size;
+  _timestamp = copy._size;
+  memcpy(_hash, copy._hash, sizeof(_hash));
+  _got_hash = copy._hash;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: FileSpec::load_xml
 //       Access: Public

direct/src/plugin/fileSpec.h

@@ -29,6 +29,8 @@ using namespace std;
 class FileSpec {
 public:
   FileSpec();
+  FileSpec(const FileSpec &copy);
+  void operator = (const FileSpec &copy);
   void load_xml(TiXmlElement *xelement);
 
   inline const string &get_filename() const;

direct/src/plugin/p3dDownload.I

@@ -27,13 +27,13 @@ get_url() const {
 //     Function: P3DDownload::get_download_progress
 //       Access: Public
 //  Description: Returns an indication of the progress through the
-//               download file, 0.0 to 1.0.  Returns -1 if the size of
+//               download file, 0.0 to 1.0.  Returns 0.0 if the size
-//               the file is not known.
+//               of the file is not known.
 ////////////////////////////////////////////////////////////////////
 inline double P3DDownload::
 get_download_progress() const {
   if (_total_expected_data == 0) {
-    return -1.0;
+    return 0.0;
   }
 
   return (double)_total_data / (double)_total_expected_data;

direct/src/plugin/p3dDownload.cxx

@@ -31,6 +31,25 @@ P3DDownload() {
   _download_id = 0;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: P3DDownload::Copy Constructor
+//       Access: Public
+//  Description: 
+////////////////////////////////////////////////////////////////////
+P3DDownload::
+P3DDownload(const P3DDownload &copy) :
+  _url(copy._url),
+  _total_expected_data(copy._total_expected_data)
+{
+  _status = P3D_RC_in_progress;
+  _http_status_code = 0;
+  _total_data = 0;
+  _last_reported_time = 0;
+  
+  _canceled = false;
+  _download_id = 0;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: P3DDownload::Destructor
 //       Access: Public, Virtual
@@ -69,7 +88,8 @@ cancel() {
 //     Function: P3DDownload::feed_url_stream
 //       Access: Public
 //  Description: Called by P3DInstance as more data arrives from the
-//               host.
+//               host.  Returns true on success, false if the download
+//               should be aborted.
 ////////////////////////////////////////////////////////////////////
 bool P3DDownload::
 feed_url_stream(P3D_result_code result_code,

direct/src/plugin/p3dDownload.h

@@ -16,6 +16,7 @@
 #define P3DDOWNLOAD_H
 
 #include "p3d_plugin_common.h"
+#include "p3dReferenceCount.h"
 
 #include <time.h>
 
@@ -27,9 +28,10 @@
 //               it, subclass it and redefine the appropriate callback
 //               methods.
 ////////////////////////////////////////////////////////////////////
-class P3DDownload {
+class P3DDownload : public P3DReferenceCount {
 public:
   P3DDownload();
+  P3DDownload(const P3DDownload &copy);
   virtual ~P3DDownload();
 
   void set_url(const string &url);

direct/src/plugin/p3dFileDownload.cxx

@@ -25,6 +25,18 @@ P3DFileDownload::
 P3DFileDownload() {
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: P3DFileDownload::Copy Constructor
+//       Access: Public
+//  Description: 
+////////////////////////////////////////////////////////////////////
+P3DFileDownload::
+P3DFileDownload(const P3DFileDownload &copy) : 
+  P3DDownload(copy)
+{
+  // We don't copy the filename.  You have to copy it yourself.
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: P3DFileDownload::set_filename
 //       Access: Public
@@ -58,6 +70,16 @@ open_file() {
   return true;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: P3DFileDownload::close_file
+//       Access: Protected
+//  Description: Closes the local file.
+////////////////////////////////////////////////////////////////////
+void P3DFileDownload::
+close_file() {
+  _file.close();
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: P3DFileDownload::receive_data
 //       Access: Protected, Virtual

direct/src/plugin/p3dFileDownload.h

@@ -28,12 +28,14 @@
 class P3DFileDownload : public P3DDownload {
 public:
   P3DFileDownload();
+  P3DFileDownload(const P3DFileDownload &copy);
 
   bool set_filename(const string &filename);
   inline const string &get_filename() const;
 
 protected:
   virtual bool open_file();
+  void close_file();
   virtual bool receive_data(const unsigned char *this_data,
                             size_t this_data_size);
   virtual void download_finished(bool success);

direct/src/plugin/p3dHost.cxx

@@ -17,6 +17,8 @@
 #include "p3dPackage.h"
 #include "openssl/md5.h"
 
+#include <algorithm>
+
 ////////////////////////////////////////////////////////////////////
 //     Function: P3DHost::Constructor
 //       Access: Private
@@ -326,6 +328,52 @@ migrate_package(P3DPackage *package, const string &alt_host, P3DHost *new_host)
   assert(inserted);
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: P3DHost::choose_random_mirrors
+//       Access: Public
+//  Description: Selects num_mirrors elements, chosen at random, from
+//               the _mirrors list.  Adds the selected mirrors to
+//               result.  If there are fewer than num_mirrors elements
+//               in the list, adds only as many mirrors as we can get.
+////////////////////////////////////////////////////////////////////
+void P3DHost::
+choose_random_mirrors(vector<string> &result, int num_mirrors) {
+  vector<size_t> selected;
+
+  size_t num_to_select = min(_mirrors.size(), (size_t)num_mirrors);
+  while (num_to_select > 0) {
+    size_t i = (size_t)(((double)rand() / (double)RAND_MAX) * _mirrors.size());
+    while (find(selected.begin(), selected.end(), i) != selected.end()) {
+      // Already found this i, find a new one.
+      i = (size_t)(((double)rand() / (double)RAND_MAX) * _mirrors.size());
+    }
+    selected.push_back(i);
+    result.push_back(_mirrors[i]);
+    --num_to_select;
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: P3DHost::add_mirror
+//       Access: Public
+//  Description: Adds a new URL to serve as a mirror for this host.
+//               The mirrors will be consulted first, before
+//               consulting the host directly.
+////////////////////////////////////////////////////////////////////
+void P3DHost::
+add_mirror(string mirror_url) {
+  // Ensure the URL ends in a slash.
+  if (!mirror_url.empty() && mirror_url[mirror_url.size() - 1] != '/') {
+    mirror_url += '/';
+  }
+  
+  // Add it to the _mirrors list, but only if it's not already
+  // there.
+  if (find(_mirrors.begin(), _mirrors.end(), mirror_url) == _mirrors.end()) {
+    _mirrors.push_back(mirror_url);
+  }
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: P3DHost::determine_host_dir
 //       Access: Private
@@ -423,7 +471,7 @@ read_xhost(TiXmlElement *xhost) {
   while (xmirror != NULL) {
     const char *url = xmirror->Attribute("url");
     if (url != NULL) {
-      _mirrors.push_back(url);
+      add_mirror(url);
     }
     xmirror = xmirror->NextSiblingElement("mirror");
   }

direct/src/plugin/p3dHost.h

@@ -56,6 +56,9 @@ public:
 
   void migrate_package(P3DPackage *package, const string &alt_host, P3DHost *new_host);
 
+  void choose_random_mirrors(vector<string> &result, int num_mirrors);
+  void add_mirror(string mirror_url);
+
 private:
   void determine_host_dir();
   void read_xhost(TiXmlElement *xhost);

direct/src/plugin/p3dInstance.cxx

@@ -635,7 +635,7 @@ feed_url_stream(int unique_id,
   if (!download_ok || download->get_download_finished()) {
     // All done.
     _downloads.erase(di);
-    delete download;
+    unref_delete(download);
   }
 
   return download_ok;
@@ -835,9 +835,11 @@ get_packages_failed() const {
 //  Description: Adds a newly-allocated P3DDownload object to the
 //               download queue, and issues the request to start it
 //               downloading.  As the download data comes in, it will
-//               be fed to the download object.  After
+//               be fed to the download object.
-//               download_finished() has been called, the P3DDownload
+//
-//               object will be deleted.
+//               This increments the P3DDownload object's reference
+//               count, and will decrement it (and possibly delete the
+//               object) after download_finished() has been called.
 ////////////////////////////////////////////////////////////////////
 void P3DInstance::
 start_download(P3DDownload *download) {
@@ -852,6 +854,7 @@ start_download(P3DDownload *download) {
   int download_id = inst_mgr->get_unique_id();
   download->set_download_id(download_id);
 
+  download->ref();
   bool inserted = _downloads.insert(Downloads::value_type(download_id, download)).second;
   assert(inserted);
 

direct/src/plugin/p3dInstanceManager.cxx

@@ -70,6 +70,10 @@ P3DInstanceManager() {
 
   _auth_session = NULL;
 
+  // Seed the lame random number generator in rand(); we use it to
+  // select a mirror for downloading.
+  srand((unsigned int)time(NULL));
+
 #ifdef _WIN32
   // Ensure the appropriate Windows common controls are available to
   // this application.

direct/src/plugin/p3dPackage.cxx

@@ -66,7 +66,6 @@ P3DPackage(P3DHost *host, const string &package_name,
   _ready = false;
   _failed = false;
   _active_download = NULL;
-  _partial_download = false;
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -276,12 +275,12 @@ download_contents_file() {
 
   // Get the URL for contents.xml.
   ostringstream strm;
-  strm << _host->get_host_url_prefix() << "contents.xml";
+  strm << "contents.xml";
   // Append a uniquifying query string to the URL to force the
   // download to go all the way through any caches.  We use the time
   // in seconds; that's unique enough.
   strm << "?" << time(NULL);
-  string url = strm.str();
+  string urlbase = strm.str();
 
   // Download contents.xml to a temporary filename first, in case
   // multiple packages are downloading it simultaneously.
@@ -291,7 +290,8 @@ download_contents_file() {
   }
   _temp_contents_file = new P3DTemporaryFile(".xml");
 
-  start_download(DT_contents_file, url, _temp_contents_file->get_filename(), false);
+  start_download(DT_contents_file, urlbase, _temp_contents_file->get_filename(), 
+                 FileSpec());
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -380,8 +380,7 @@ download_desc_file() {
   // Attempt to check the desc file for freshness.  If it already
   // exists, and is consistent with the server contents file, we don't
   // need to re-download it.
-  FileSpec desc_file;
+  if (!_host->get_package_desc_file(_desc_file, _package_platform, 
-  if (!_host->get_package_desc_file(desc_file, _package_platform, 
                                     _package_solo,
                                     _package_name, _package_version)) {
     nout << "Couldn't find package " << _package_fullname
@@ -389,23 +388,24 @@ download_desc_file() {
     return;
   }
 
-  // The desc file might have a different path on the host server than
+  string url_filename = _desc_file.get_filename();
-  // it has locally, because we strip out the platform directory
-  // locally.  Adjust desc_file to point to the local file.
-  string url_filename = desc_file.get_filename();
-
-  _desc_file_url = _host->get_host_url_prefix();
-  _desc_file_url += url_filename;
 
+  _desc_file_dirname = "";
   _desc_file_basename = url_filename;
   size_t slash = _desc_file_basename.rfind('/');
   if (slash != string::npos) {
+    _desc_file_dirname = _desc_file_basename.substr(0, slash);
     _desc_file_basename = _desc_file_basename.substr(slash + 1);
   }
-  desc_file.set_filename(_desc_file_basename);
-  _desc_file_pathname = desc_file.get_pathname(_package_dir);
 
-  if (!desc_file.full_verify(_package_dir)) {
+  // The desc file might have a different path on the host server than
+  // it has locally, because we strip out the platform directory
+  // locally.
+  FileSpec local_desc_file = _desc_file;
+  local_desc_file.set_filename(_desc_file_basename);
+  _desc_file_pathname = local_desc_file.get_pathname(_package_dir);
+
+  if (!local_desc_file.full_verify(_package_dir)) {
     nout << _desc_file_pathname << " is stale.\n";
 
   } else {
@@ -424,7 +424,8 @@ download_desc_file() {
   }
 
   // The desc file is not current.  Go download it.
-  start_download(DT_desc_file, _desc_file_url, _desc_file_pathname, false);
+  start_download(DT_desc_file, _desc_file.get_filename(), 
+                 _desc_file_pathname, local_desc_file);
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -615,7 +616,7 @@ begin_data_download() {
 
   } else {
     // Shoot, we need to download the archive.
-    download_compressed_archive(true);
+    download_compressed_archive();
   }
 }
 
@@ -625,17 +626,15 @@ begin_data_download() {
 //  Description: Starts downloading the archive file for the package.
 ////////////////////////////////////////////////////////////////////
 void P3DPackage::
-download_compressed_archive(bool allow_partial) {
+download_compressed_archive() {
-  string url = _desc_file_url;
+  string urlbase = _desc_file_dirname;
-  size_t slash = url.rfind('/');
+  urlbase += "/";
-  if (slash != string::npos) {
+  urlbase += _compressed_archive.get_filename();
-    url = url.substr(0, slash + 1);
-  }
-  url += _compressed_archive.get_filename();
 
   string target_pathname = _package_dir + "/" + _compressed_archive.get_filename();
 
-  start_download(DT_compressed_archive, url, target_pathname, allow_partial);
+  start_download(DT_compressed_archive, urlbase, target_pathname, 
+                 _compressed_archive);
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -660,21 +659,8 @@ compressed_archive_download_finished(bool success) {
     return;
   }
 
-  if (_compressed_archive.full_verify(_package_dir)) {
+  // Go on to uncompress the archive.
-    // Go on to uncompress the archive.
+  uncompress_archive();
-    uncompress_archive();
-    return;
-  }
-
-  // Oof, didn't download it correctly.
-  if (_partial_download) {
-    // Go back and get the whole file this time.
-    download_compressed_archive(false);
-  }
-
-  nout << _compressed_archive.get_filename()
-       << " failed hash check after download\n";
-  report_done(false);
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -918,11 +904,13 @@ report_done(bool success) {
 //  Description: Initiates a download of the indicated file.
 ////////////////////////////////////////////////////////////////////
 void P3DPackage::
-start_download(P3DPackage::DownloadType dtype, const string &url, 
+start_download(P3DPackage::DownloadType dtype, const string &urlbase, 
-               const string &pathname, bool allow_partial) {
+               const string &pathname, const FileSpec &file_spec) {
   // Only one download should be active at a time
   assert(_active_download == NULL);
-  
+
+  // TODO: support partial downloads.
+  static const bool allow_partial = false;
   if (!allow_partial) {
 #ifdef _WIN32
     // Windows can't delete a file if it's read-only.
@@ -934,16 +922,37 @@ start_download(P3DPackage::DownloadType dtype, const string &url,
     chmod(pathname.c_str(), 0644);
   }
     
-  Download *download = new Download(this, dtype);
+  Download *download = new Download(this, dtype, file_spec);
+
+  // Fill up the _try_urls vector for URL's to try getting this file
+  // from, in reverse order.
+
+  // The last thing we try is the actual authoritative host.
+  string url = _host->get_host_url_prefix() + urlbase;
+  download->_try_urls.push_back(url);
+
+  // The first thing we try is a couple of mirrors, chosen at random
+  // (except for the contents.xml file, which always goes straight to
+  // the host).
+  if (dtype != DT_contents_file) {
+    vector<string> mirrors;
+    _host->choose_random_mirrors(mirrors, 2);
+    for (vector<string>::iterator si = mirrors.begin();
+         si != mirrors.end(); 
+         ++si) {
+      url = (*si) + urlbase;
+      download->_try_urls.push_back(url);
+    }
+  }
+
+  // OK, start the download.
+  assert(!download->_try_urls.empty());
+  url = download->_try_urls.back();
+  download->_try_urls.pop_back();
   download->set_url(url);
   download->set_filename(pathname);
 
-  // TODO: implement partial file re-download.
-  allow_partial = false;
-
   _active_download = download;
-  _partial_download = false;
-
   assert(!_instances.empty());
 
   _instances[0]->start_download(download);
@@ -973,9 +982,25 @@ is_extractable(const string &filename) const {
 //  Description: 
 ////////////////////////////////////////////////////////////////////
 P3DPackage::Download::
-Download(P3DPackage *package, DownloadType dtype) :
+Download(P3DPackage *package, DownloadType dtype, const FileSpec &file_spec) :
   _package(package),
-  _dtype(dtype)
+  _dtype(dtype),
+  _file_spec(file_spec)
+{
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: P3DPackage::Download::Copy Constructor
+//       Access: Public
+//  Description: 
+////////////////////////////////////////////////////////////////////
+P3DPackage::Download::
+Download(const P3DPackage::Download &copy) :
+  P3DFileDownload(copy),
+  _try_urls(copy._try_urls),
+  _package(copy._package),
+  _dtype(copy._dtype),
+  _file_spec(copy._file_spec)
 {
 }
 
@@ -1010,6 +1035,33 @@ download_finished(bool success) {
   assert(_package->_active_download == this);
   _package->_active_download = NULL;
 
+  if (success && !_file_spec.get_filename().empty()) {
+    // We think we downloaded it correctly.  Check the hash to be
+    // sure.
+    if (!_file_spec.full_verify(_package->_package_dir)) {
+      nout << "After downloading " << get_url()
+           << ", failed hash check\n";
+      success = false;
+    }
+  }
+
+  if (!success && !_try_urls.empty()) {
+    // Well, that URL failed, but we can try another mirror.
+    close_file();
+    string url = _try_urls.back();
+    _try_urls.pop_back();
+
+    Download *new_download = new Download(*this);
+    new_download->set_filename(get_filename());
+    new_download->set_url(url);
+
+    _package->_active_download = new_download;
+
+    assert(!_package->_instances.empty());
+    _package->_instances[0]->start_download(new_download);
+    return;
+  }
+
   switch (_dtype) {
   case DT_contents_file:
     _package->contents_file_download_finished(success);

direct/src/plugin/p3dPackage.h

@@ -75,17 +75,28 @@ private:
     DT_compressed_archive
   };
 
+  typedef vector<string> TryUrls;
+
   class Download : public P3DFileDownload {
   public:
-    Download(P3DPackage *package, DownloadType dtype);
+    Download(P3DPackage *package, DownloadType dtype,
+             const FileSpec &file_spec);
+    Download(const Download &copy);
 
   protected:
     virtual void download_progress();
     virtual void download_finished(bool success);
 
+  public:
+    // URL's to try downloading from, in reverse order.
+    TryUrls _try_urls;
+
   private:
     P3DPackage *_package;
     DownloadType _dtype;
+
+    // FileSpec to validate the download against.
+    FileSpec _file_spec;
   };
 
   void begin_info_download();
@@ -98,7 +109,7 @@ private:
   void got_desc_file(TiXmlDocument *doc, bool freshly_downloaded);
 
   void begin_data_download();
-  void download_compressed_archive(bool allow_partial);
+  void download_compressed_archive();
   void compressed_archive_download_progress(double progress);
   void compressed_archive_download_finished(bool success);
 
@@ -108,8 +119,8 @@ private:
   void report_progress(double progress);
   void report_info_ready();
   void report_done(bool success);
-  void start_download(DownloadType dtype, const string &url, 
+  void start_download(DownloadType dtype, const string &urlbase, 
-                      const string &pathname, bool allow_partial);
+                      const string &pathname, const FileSpec &file_spec);
 
   bool is_extractable(const string &filename) const;
 
@@ -129,7 +140,8 @@ private:
 
   P3DTemporaryFile *_temp_contents_file;
 
-  string _desc_file_url;
+  FileSpec _desc_file;
+  string _desc_file_dirname;
   string _desc_file_basename;
   string _desc_file_pathname;
 
@@ -139,7 +151,6 @@ private:
   bool _ready;
   bool _failed;
   Download *_active_download;
-  bool _partial_download;
 
   typedef vector<P3DInstance *> Instances;
   Instances _instances;