From 17133ebc8cbc905cbc05873544204df5cd87f791 Mon Sep 17 00:00:00 2001 From: David Rose Date: Wed, 23 Sep 2009 19:45:21 +0000 Subject: [PATCH] mirrors at the C++ level too --- direct/src/plugin/fileSpec.cxx | 29 +++++ direct/src/plugin/fileSpec.h | 2 + direct/src/plugin/p3dDownload.I | 6 +- direct/src/plugin/p3dDownload.cxx | 22 +++- direct/src/plugin/p3dDownload.h | 4 +- direct/src/plugin/p3dFileDownload.cxx | 22 ++++ direct/src/plugin/p3dFileDownload.h | 2 + direct/src/plugin/p3dHost.cxx | 50 +++++++- direct/src/plugin/p3dHost.h | 3 + direct/src/plugin/p3dInstance.cxx | 11 +- direct/src/plugin/p3dInstanceManager.cxx | 4 + direct/src/plugin/p3dPackage.cxx | 156 +++++++++++++++-------- direct/src/plugin/p3dPackage.h | 23 +++- 13 files changed, 266 insertions(+), 68 deletions(-) diff --git a/direct/src/plugin/fileSpec.cxx b/direct/src/plugin/fileSpec.cxx index 3e3d95ece9..2bf1b69c29 100755 --- a/direct/src/plugin/fileSpec.cxx +++ b/direct/src/plugin/fileSpec.cxx @@ -46,6 +46,35 @@ FileSpec() { _got_hash = false; } +//////////////////////////////////////////////////////////////////// +// Function: FileSpec::Copy Constructor +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +FileSpec:: +FileSpec(const FileSpec ©) : + _filename(copy._filename), + _size(copy._size), + _timestamp(copy._timestamp), + _got_hash(copy._got_hash) +{ + memcpy(_hash, copy._hash, sizeof(_hash)); +} + +//////////////////////////////////////////////////////////////////// +// Function: FileSpec::Copy Assignment Operator +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +void FileSpec:: +operator = (const FileSpec ©) { + _filename = copy._filename; + _size = copy._size; + _timestamp = copy._size; + memcpy(_hash, copy._hash, sizeof(_hash)); + _got_hash = copy._hash; +} + //////////////////////////////////////////////////////////////////// // Function: FileSpec::load_xml // Access: Public diff --git a/direct/src/plugin/fileSpec.h b/direct/src/plugin/fileSpec.h index f1652caa14..8656c8e736 100755 --- a/direct/src/plugin/fileSpec.h +++ b/direct/src/plugin/fileSpec.h @@ -29,6 +29,8 @@ using namespace std; class FileSpec { public: FileSpec(); + FileSpec(const FileSpec ©); + void operator = (const FileSpec ©); void load_xml(TiXmlElement *xelement); inline const string &get_filename() const; diff --git a/direct/src/plugin/p3dDownload.I b/direct/src/plugin/p3dDownload.I index 994e7788e0..05dcb55751 100755 --- a/direct/src/plugin/p3dDownload.I +++ b/direct/src/plugin/p3dDownload.I @@ -27,13 +27,13 @@ get_url() const { // Function: P3DDownload::get_download_progress // Access: Public // Description: Returns an indication of the progress through the -// download file, 0.0 to 1.0. Returns -1 if the size of -// the file is not known. +// download file, 0.0 to 1.0. Returns 0.0 if the size +// of the file is not known. //////////////////////////////////////////////////////////////////// inline double P3DDownload:: get_download_progress() const { if (_total_expected_data == 0) { - return -1.0; + return 0.0; } return (double)_total_data / (double)_total_expected_data; diff --git a/direct/src/plugin/p3dDownload.cxx b/direct/src/plugin/p3dDownload.cxx index bc19c3e3b4..fbef0187c8 100755 --- a/direct/src/plugin/p3dDownload.cxx +++ b/direct/src/plugin/p3dDownload.cxx @@ -31,6 +31,25 @@ P3DDownload() { _download_id = 0; } +//////////////////////////////////////////////////////////////////// +// Function: P3DDownload::Copy Constructor +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +P3DDownload:: +P3DDownload(const P3DDownload ©) : + _url(copy._url), + _total_expected_data(copy._total_expected_data) +{ + _status = P3D_RC_in_progress; + _http_status_code = 0; + _total_data = 0; + _last_reported_time = 0; + + _canceled = false; + _download_id = 0; +} + //////////////////////////////////////////////////////////////////// // Function: P3DDownload::Destructor // Access: Public, Virtual @@ -69,7 +88,8 @@ cancel() { // Function: P3DDownload::feed_url_stream // Access: Public // Description: Called by P3DInstance as more data arrives from the -// host. +// host. Returns true on success, false if the download +// should be aborted. //////////////////////////////////////////////////////////////////// bool P3DDownload:: feed_url_stream(P3D_result_code result_code, diff --git a/direct/src/plugin/p3dDownload.h b/direct/src/plugin/p3dDownload.h index b79e34743f..8393197eab 100755 --- a/direct/src/plugin/p3dDownload.h +++ b/direct/src/plugin/p3dDownload.h @@ -16,6 +16,7 @@ #define P3DDOWNLOAD_H #include "p3d_plugin_common.h" +#include "p3dReferenceCount.h" #include @@ -27,9 +28,10 @@ // it, subclass it and redefine the appropriate callback // methods. //////////////////////////////////////////////////////////////////// -class P3DDownload { +class P3DDownload : public P3DReferenceCount { public: P3DDownload(); + P3DDownload(const P3DDownload ©); virtual ~P3DDownload(); void set_url(const string &url); diff --git a/direct/src/plugin/p3dFileDownload.cxx b/direct/src/plugin/p3dFileDownload.cxx index 81a3521eba..3444a46f37 100755 --- a/direct/src/plugin/p3dFileDownload.cxx +++ b/direct/src/plugin/p3dFileDownload.cxx @@ -25,6 +25,18 @@ P3DFileDownload:: P3DFileDownload() { } +//////////////////////////////////////////////////////////////////// +// Function: P3DFileDownload::Copy Constructor +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +P3DFileDownload:: +P3DFileDownload(const P3DFileDownload ©) : + P3DDownload(copy) +{ + // We don't copy the filename. You have to copy it yourself. +} + //////////////////////////////////////////////////////////////////// // Function: P3DFileDownload::set_filename // Access: Public @@ -58,6 +70,16 @@ open_file() { return true; } +//////////////////////////////////////////////////////////////////// +// Function: P3DFileDownload::close_file +// Access: Protected +// Description: Closes the local file. +//////////////////////////////////////////////////////////////////// +void P3DFileDownload:: +close_file() { + _file.close(); +} + //////////////////////////////////////////////////////////////////// // Function: P3DFileDownload::receive_data // Access: Protected, Virtual diff --git a/direct/src/plugin/p3dFileDownload.h b/direct/src/plugin/p3dFileDownload.h index 4a7762a095..1caaeffff0 100755 --- a/direct/src/plugin/p3dFileDownload.h +++ b/direct/src/plugin/p3dFileDownload.h @@ -28,12 +28,14 @@ class P3DFileDownload : public P3DDownload { public: P3DFileDownload(); + P3DFileDownload(const P3DFileDownload ©); bool set_filename(const string &filename); inline const string &get_filename() const; protected: virtual bool open_file(); + void close_file(); virtual bool receive_data(const unsigned char *this_data, size_t this_data_size); virtual void download_finished(bool success); diff --git a/direct/src/plugin/p3dHost.cxx b/direct/src/plugin/p3dHost.cxx index 415a02ba1c..8774a1b678 100644 --- a/direct/src/plugin/p3dHost.cxx +++ b/direct/src/plugin/p3dHost.cxx @@ -17,6 +17,8 @@ #include "p3dPackage.h" #include "openssl/md5.h" +#include + //////////////////////////////////////////////////////////////////// // Function: P3DHost::Constructor // Access: Private @@ -326,6 +328,52 @@ migrate_package(P3DPackage *package, const string &alt_host, P3DHost *new_host) assert(inserted); } +//////////////////////////////////////////////////////////////////// +// Function: P3DHost::choose_random_mirrors +// Access: Public +// Description: Selects num_mirrors elements, chosen at random, from +// the _mirrors list. Adds the selected mirrors to +// result. If there are fewer than num_mirrors elements +// in the list, adds only as many mirrors as we can get. +//////////////////////////////////////////////////////////////////// +void P3DHost:: +choose_random_mirrors(vector &result, int num_mirrors) { + vector selected; + + size_t num_to_select = min(_mirrors.size(), (size_t)num_mirrors); + while (num_to_select > 0) { + size_t i = (size_t)(((double)rand() / (double)RAND_MAX) * _mirrors.size()); + while (find(selected.begin(), selected.end(), i) != selected.end()) { + // Already found this i, find a new one. + i = (size_t)(((double)rand() / (double)RAND_MAX) * _mirrors.size()); + } + selected.push_back(i); + result.push_back(_mirrors[i]); + --num_to_select; + } +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DHost::add_mirror +// Access: Public +// Description: Adds a new URL to serve as a mirror for this host. +// The mirrors will be consulted first, before +// consulting the host directly. +//////////////////////////////////////////////////////////////////// +void P3DHost:: +add_mirror(string mirror_url) { + // Ensure the URL ends in a slash. + if (!mirror_url.empty() && mirror_url[mirror_url.size() - 1] != '/') { + mirror_url += '/'; + } + + // Add it to the _mirrors list, but only if it's not already + // there. + if (find(_mirrors.begin(), _mirrors.end(), mirror_url) == _mirrors.end()) { + _mirrors.push_back(mirror_url); + } +} + //////////////////////////////////////////////////////////////////// // Function: P3DHost::determine_host_dir // Access: Private @@ -423,7 +471,7 @@ read_xhost(TiXmlElement *xhost) { while (xmirror != NULL) { const char *url = xmirror->Attribute("url"); if (url != NULL) { - _mirrors.push_back(url); + add_mirror(url); } xmirror = xmirror->NextSiblingElement("mirror"); } diff --git a/direct/src/plugin/p3dHost.h b/direct/src/plugin/p3dHost.h index 157335a817..90e9aa3cb5 100644 --- a/direct/src/plugin/p3dHost.h +++ b/direct/src/plugin/p3dHost.h @@ -56,6 +56,9 @@ public: void migrate_package(P3DPackage *package, const string &alt_host, P3DHost *new_host); + void choose_random_mirrors(vector &result, int num_mirrors); + void add_mirror(string mirror_url); + private: void determine_host_dir(); void read_xhost(TiXmlElement *xhost); diff --git a/direct/src/plugin/p3dInstance.cxx b/direct/src/plugin/p3dInstance.cxx index 783977fa53..af800e5322 100644 --- a/direct/src/plugin/p3dInstance.cxx +++ b/direct/src/plugin/p3dInstance.cxx @@ -635,7 +635,7 @@ feed_url_stream(int unique_id, if (!download_ok || download->get_download_finished()) { // All done. _downloads.erase(di); - delete download; + unref_delete(download); } return download_ok; @@ -835,9 +835,11 @@ get_packages_failed() const { // Description: Adds a newly-allocated P3DDownload object to the // download queue, and issues the request to start it // downloading. As the download data comes in, it will -// be fed to the download object. After -// download_finished() has been called, the P3DDownload -// object will be deleted. +// be fed to the download object. +// +// This increments the P3DDownload object's reference +// count, and will decrement it (and possibly delete the +// object) after download_finished() has been called. //////////////////////////////////////////////////////////////////// void P3DInstance:: start_download(P3DDownload *download) { @@ -852,6 +854,7 @@ start_download(P3DDownload *download) { int download_id = inst_mgr->get_unique_id(); download->set_download_id(download_id); + download->ref(); bool inserted = _downloads.insert(Downloads::value_type(download_id, download)).second; assert(inserted); diff --git a/direct/src/plugin/p3dInstanceManager.cxx b/direct/src/plugin/p3dInstanceManager.cxx index 23acccba3d..517780cc20 100644 --- a/direct/src/plugin/p3dInstanceManager.cxx +++ b/direct/src/plugin/p3dInstanceManager.cxx @@ -70,6 +70,10 @@ P3DInstanceManager() { _auth_session = NULL; + // Seed the lame random number generator in rand(); we use it to + // select a mirror for downloading. + srand((unsigned int)time(NULL)); + #ifdef _WIN32 // Ensure the appropriate Windows common controls are available to // this application. diff --git a/direct/src/plugin/p3dPackage.cxx b/direct/src/plugin/p3dPackage.cxx index dc80e6c25c..9250ecd22a 100755 --- a/direct/src/plugin/p3dPackage.cxx +++ b/direct/src/plugin/p3dPackage.cxx @@ -66,7 +66,6 @@ P3DPackage(P3DHost *host, const string &package_name, _ready = false; _failed = false; _active_download = NULL; - _partial_download = false; } //////////////////////////////////////////////////////////////////// @@ -276,12 +275,12 @@ download_contents_file() { // Get the URL for contents.xml. ostringstream strm; - strm << _host->get_host_url_prefix() << "contents.xml"; + strm << "contents.xml"; // Append a uniquifying query string to the URL to force the // download to go all the way through any caches. We use the time // in seconds; that's unique enough. strm << "?" << time(NULL); - string url = strm.str(); + string urlbase = strm.str(); // Download contents.xml to a temporary filename first, in case // multiple packages are downloading it simultaneously. @@ -291,7 +290,8 @@ download_contents_file() { } _temp_contents_file = new P3DTemporaryFile(".xml"); - start_download(DT_contents_file, url, _temp_contents_file->get_filename(), false); + start_download(DT_contents_file, urlbase, _temp_contents_file->get_filename(), + FileSpec()); } //////////////////////////////////////////////////////////////////// @@ -380,8 +380,7 @@ download_desc_file() { // Attempt to check the desc file for freshness. If it already // exists, and is consistent with the server contents file, we don't // need to re-download it. - FileSpec desc_file; - if (!_host->get_package_desc_file(desc_file, _package_platform, + if (!_host->get_package_desc_file(_desc_file, _package_platform, _package_solo, _package_name, _package_version)) { nout << "Couldn't find package " << _package_fullname @@ -389,23 +388,24 @@ download_desc_file() { return; } - // The desc file might have a different path on the host server than - // it has locally, because we strip out the platform directory - // locally. Adjust desc_file to point to the local file. - string url_filename = desc_file.get_filename(); - - _desc_file_url = _host->get_host_url_prefix(); - _desc_file_url += url_filename; + string url_filename = _desc_file.get_filename(); + _desc_file_dirname = ""; _desc_file_basename = url_filename; size_t slash = _desc_file_basename.rfind('/'); if (slash != string::npos) { + _desc_file_dirname = _desc_file_basename.substr(0, slash); _desc_file_basename = _desc_file_basename.substr(slash + 1); } - desc_file.set_filename(_desc_file_basename); - _desc_file_pathname = desc_file.get_pathname(_package_dir); - if (!desc_file.full_verify(_package_dir)) { + // The desc file might have a different path on the host server than + // it has locally, because we strip out the platform directory + // locally. + FileSpec local_desc_file = _desc_file; + local_desc_file.set_filename(_desc_file_basename); + _desc_file_pathname = local_desc_file.get_pathname(_package_dir); + + if (!local_desc_file.full_verify(_package_dir)) { nout << _desc_file_pathname << " is stale.\n"; } else { @@ -424,7 +424,8 @@ download_desc_file() { } // The desc file is not current. Go download it. - start_download(DT_desc_file, _desc_file_url, _desc_file_pathname, false); + start_download(DT_desc_file, _desc_file.get_filename(), + _desc_file_pathname, local_desc_file); } //////////////////////////////////////////////////////////////////// @@ -615,7 +616,7 @@ begin_data_download() { } else { // Shoot, we need to download the archive. - download_compressed_archive(true); + download_compressed_archive(); } } @@ -625,17 +626,15 @@ begin_data_download() { // Description: Starts downloading the archive file for the package. //////////////////////////////////////////////////////////////////// void P3DPackage:: -download_compressed_archive(bool allow_partial) { - string url = _desc_file_url; - size_t slash = url.rfind('/'); - if (slash != string::npos) { - url = url.substr(0, slash + 1); - } - url += _compressed_archive.get_filename(); +download_compressed_archive() { + string urlbase = _desc_file_dirname; + urlbase += "/"; + urlbase += _compressed_archive.get_filename(); string target_pathname = _package_dir + "/" + _compressed_archive.get_filename(); - start_download(DT_compressed_archive, url, target_pathname, allow_partial); + start_download(DT_compressed_archive, urlbase, target_pathname, + _compressed_archive); } //////////////////////////////////////////////////////////////////// @@ -660,21 +659,8 @@ compressed_archive_download_finished(bool success) { return; } - if (_compressed_archive.full_verify(_package_dir)) { - // Go on to uncompress the archive. - uncompress_archive(); - return; - } - - // Oof, didn't download it correctly. - if (_partial_download) { - // Go back and get the whole file this time. - download_compressed_archive(false); - } - - nout << _compressed_archive.get_filename() - << " failed hash check after download\n"; - report_done(false); + // Go on to uncompress the archive. + uncompress_archive(); } //////////////////////////////////////////////////////////////////// @@ -918,11 +904,13 @@ report_done(bool success) { // Description: Initiates a download of the indicated file. //////////////////////////////////////////////////////////////////// void P3DPackage:: -start_download(P3DPackage::DownloadType dtype, const string &url, - const string &pathname, bool allow_partial) { +start_download(P3DPackage::DownloadType dtype, const string &urlbase, + const string &pathname, const FileSpec &file_spec) { // Only one download should be active at a time assert(_active_download == NULL); - + + // TODO: support partial downloads. + static const bool allow_partial = false; if (!allow_partial) { #ifdef _WIN32 // Windows can't delete a file if it's read-only. @@ -934,16 +922,37 @@ start_download(P3DPackage::DownloadType dtype, const string &url, chmod(pathname.c_str(), 0644); } - Download *download = new Download(this, dtype); + Download *download = new Download(this, dtype, file_spec); + + // Fill up the _try_urls vector for URL's to try getting this file + // from, in reverse order. + + // The last thing we try is the actual authoritative host. + string url = _host->get_host_url_prefix() + urlbase; + download->_try_urls.push_back(url); + + // The first thing we try is a couple of mirrors, chosen at random + // (except for the contents.xml file, which always goes straight to + // the host). + if (dtype != DT_contents_file) { + vector mirrors; + _host->choose_random_mirrors(mirrors, 2); + for (vector::iterator si = mirrors.begin(); + si != mirrors.end(); + ++si) { + url = (*si) + urlbase; + download->_try_urls.push_back(url); + } + } + + // OK, start the download. + assert(!download->_try_urls.empty()); + url = download->_try_urls.back(); + download->_try_urls.pop_back(); download->set_url(url); download->set_filename(pathname); - // TODO: implement partial file re-download. - allow_partial = false; - _active_download = download; - _partial_download = false; - assert(!_instances.empty()); _instances[0]->start_download(download); @@ -973,9 +982,25 @@ is_extractable(const string &filename) const { // Description: //////////////////////////////////////////////////////////////////// P3DPackage::Download:: -Download(P3DPackage *package, DownloadType dtype) : +Download(P3DPackage *package, DownloadType dtype, const FileSpec &file_spec) : _package(package), - _dtype(dtype) + _dtype(dtype), + _file_spec(file_spec) +{ +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::Download::Copy Constructor +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +P3DPackage::Download:: +Download(const P3DPackage::Download ©) : + P3DFileDownload(copy), + _try_urls(copy._try_urls), + _package(copy._package), + _dtype(copy._dtype), + _file_spec(copy._file_spec) { } @@ -1010,6 +1035,33 @@ download_finished(bool success) { assert(_package->_active_download == this); _package->_active_download = NULL; + if (success && !_file_spec.get_filename().empty()) { + // We think we downloaded it correctly. Check the hash to be + // sure. + if (!_file_spec.full_verify(_package->_package_dir)) { + nout << "After downloading " << get_url() + << ", failed hash check\n"; + success = false; + } + } + + if (!success && !_try_urls.empty()) { + // Well, that URL failed, but we can try another mirror. + close_file(); + string url = _try_urls.back(); + _try_urls.pop_back(); + + Download *new_download = new Download(*this); + new_download->set_filename(get_filename()); + new_download->set_url(url); + + _package->_active_download = new_download; + + assert(!_package->_instances.empty()); + _package->_instances[0]->start_download(new_download); + return; + } + switch (_dtype) { case DT_contents_file: _package->contents_file_download_finished(success); diff --git a/direct/src/plugin/p3dPackage.h b/direct/src/plugin/p3dPackage.h index 8b1e744d91..1334e53f41 100755 --- a/direct/src/plugin/p3dPackage.h +++ b/direct/src/plugin/p3dPackage.h @@ -75,17 +75,28 @@ private: DT_compressed_archive }; + typedef vector TryUrls; + class Download : public P3DFileDownload { public: - Download(P3DPackage *package, DownloadType dtype); + Download(P3DPackage *package, DownloadType dtype, + const FileSpec &file_spec); + Download(const Download ©); protected: virtual void download_progress(); virtual void download_finished(bool success); + public: + // URL's to try downloading from, in reverse order. + TryUrls _try_urls; + private: P3DPackage *_package; DownloadType _dtype; + + // FileSpec to validate the download against. + FileSpec _file_spec; }; void begin_info_download(); @@ -98,7 +109,7 @@ private: void got_desc_file(TiXmlDocument *doc, bool freshly_downloaded); void begin_data_download(); - void download_compressed_archive(bool allow_partial); + void download_compressed_archive(); void compressed_archive_download_progress(double progress); void compressed_archive_download_finished(bool success); @@ -108,8 +119,8 @@ private: void report_progress(double progress); void report_info_ready(); void report_done(bool success); - void start_download(DownloadType dtype, const string &url, - const string &pathname, bool allow_partial); + void start_download(DownloadType dtype, const string &urlbase, + const string &pathname, const FileSpec &file_spec); bool is_extractable(const string &filename) const; @@ -129,7 +140,8 @@ private: P3DTemporaryFile *_temp_contents_file; - string _desc_file_url; + FileSpec _desc_file; + string _desc_file_dirname; string _desc_file_basename; string _desc_file_pathname; @@ -139,7 +151,6 @@ private: bool _ready; bool _failed; Download *_active_download; - bool _partial_download; typedef vector Instances; Instances _instances;