From 4545421f6ed37777a844072032efc4e2a1b25448 Mon Sep 17 00:00:00 2001 From: David Rose Date: Sat, 3 Oct 2009 15:54:46 +0000 Subject: [PATCH] fix caching issues, mirror issues --- direct/src/p3d/Packager.py | 24 ++- direct/src/p3d/coreapi.pdef | 1 - direct/src/plugin/fileSpec.cxx | 13 +- direct/src/plugin/fileSpec.h | 1 + direct/src/plugin/p3dDownload.cxx | 17 ++ direct/src/plugin/p3dDownload.h | 1 + direct/src/plugin/p3dPackage.cxx | 11 +- direct/src/plugin_npapi/ppInstance.cxx | 210 +++++++++++++++++++++---- direct/src/plugin_npapi/ppInstance.h | 13 ++ 9 files changed, 246 insertions(+), 45 deletions(-) diff --git a/direct/src/p3d/Packager.py b/direct/src/p3d/Packager.py index e20509348e..bdc7e1d2df 100644 --- a/direct/src/p3d/Packager.py +++ b/direct/src/p3d/Packager.py @@ -208,14 +208,17 @@ class Packager: return xpackage class HostEntry: - def __init__(self, url = None, descriptiveName = None, mirrors = None): + def __init__(self, url = None, downloadUrl = None, + descriptiveName = None, mirrors = None): self.url = url + self.downloadUrl = downloadUrl self.descriptiveName = descriptiveName self.mirrors = mirrors or [] self.altHosts = {} def loadXml(self, xhost, packager): self.url = xhost.Attribute('url') + self.downloadUrl = xhost.Attribute('download_url') self.descriptiveName = xhost.Attribute('descriptive_name') self.mirrors = [] xmirror = xhost.FirstChildElement('mirror') @@ -235,6 +238,8 @@ class Packager: """ Returns a new TiXmlElement. """ xhost = TiXmlElement('host') xhost.SetAttribute('url', self.url) + if self.downloadUrl and self.downloadUrl != self.url: + xhost.SetAttribute('download_url', self.downloadUrl) if self.descriptiveName: xhost.SetAttribute('descriptive_name', self.descriptiveName) @@ -1704,14 +1709,17 @@ class Packager: # file. self.contents = {} - def setHost(self, host, descriptiveName = None, mirrors = None): + def setHost(self, host, downloadUrl = None, + descriptiveName = None, mirrors = None): """ Specifies the URL that will ultimately host these contents. """ self.host = host - self.addHost(host, descriptiveName, mirrors) + self.addHost(host, downloadUrl = downloadUrl, + descriptiveName = descriptiveName, mirrors = mirrors) - def addHost(self, host, descriptiveName = None, mirrors = None): + def addHost(self, host, downloadUrl = None, descriptiveName = None, + mirrors = None): """ Adds a host to the list of known download hosts. This information will be written into any p3d files that reference this host; this can be used to pre-define the possible mirrors @@ -1721,11 +1729,15 @@ class Packager: he = self.hosts.get(host, None) if he is None: # Define a new host entry - he = self.HostEntry(host, descriptiveName, mirrors) + he = self.HostEntry(host, downloadUrl = downloadUrl, + descriptiveName = descriptiveName, + mirrors = mirrors) self.hosts[host] = he else: # Update an existing host entry - if descriptiveName: + if downloadUrl is not None: + he.downloadUrl = downloadUrl + if descriptiveName is not None: he.descriptiveName = descriptiveName if mirrors is not None: he.mirrors = mirrors diff --git a/direct/src/p3d/coreapi.pdef b/direct/src/p3d/coreapi.pdef index 4268043660..91763f9a87 100644 --- a/direct/src/p3d/coreapi.pdef +++ b/direct/src/p3d/coreapi.pdef @@ -77,4 +77,3 @@ class p3dcert(package): filename = Filename(cvar.getValue()) if not filename.empty(): file(filename, newName = 'ca-bundle.crt', extract = True) - \ No newline at end of file diff --git a/direct/src/plugin/fileSpec.cxx b/direct/src/plugin/fileSpec.cxx index 3d28850989..c100784841 100755 --- a/direct/src/plugin/fileSpec.cxx +++ b/direct/src/plugin/fileSpec.cxx @@ -136,12 +136,23 @@ load_xml(TiXmlElement *xelement) { //////////////////////////////////////////////////////////////////// bool FileSpec:: quick_verify(const string &package_dir) { + return quick_verify_pathname(get_pathname(package_dir)); +} + +//////////////////////////////////////////////////////////////////// +// Function: FileSpec::quick_verify_pathname +// Access: Public +// Description: Works like quick_verify(), above, with an explicit +// pathname. Useful for verifying the copy of a file in +// a temporary location. +//////////////////////////////////////////////////////////////////// +bool FileSpec:: +quick_verify_pathname(const string &pathname) { if (_actual_file != NULL) { delete _actual_file; _actual_file = NULL; } - string pathname = get_pathname(package_dir); struct stat st; if (stat(pathname.c_str(), &st) != 0) { //cerr << "file not found: " << _filename << "\n"; diff --git a/direct/src/plugin/fileSpec.h b/direct/src/plugin/fileSpec.h index 4c8297679b..27a409fc76 100755 --- a/direct/src/plugin/fileSpec.h +++ b/direct/src/plugin/fileSpec.h @@ -41,6 +41,7 @@ public: inline size_t get_size() const; bool quick_verify(const string &package_dir); + bool quick_verify_pathname(const string &pathname); bool full_verify(const string &package_dir); inline const FileSpec *get_actual_file() const; diff --git a/direct/src/plugin/p3dDownload.cxx b/direct/src/plugin/p3dDownload.cxx index fbef0187c8..944ffae480 100755 --- a/direct/src/plugin/p3dDownload.cxx +++ b/direct/src/plugin/p3dDownload.cxx @@ -84,6 +84,23 @@ cancel() { } } +//////////////////////////////////////////////////////////////////// +// Function: P3DDownload::clear +// Access: Public +// Description: Resets the download to its initial state, for +// re-trying the same download. +//////////////////////////////////////////////////////////////////// +void P3DDownload:: +clear() { + _status = P3D_RC_in_progress; + _http_status_code = 0; + _total_data = 0; + _last_reported_time = 0; + + _canceled = false; + _download_id = 0; +} + //////////////////////////////////////////////////////////////////// // Function: P3DDownload::feed_url_stream // Access: Public diff --git a/direct/src/plugin/p3dDownload.h b/direct/src/plugin/p3dDownload.h index 5062ea09a0..054cba2e5a 100755 --- a/direct/src/plugin/p3dDownload.h +++ b/direct/src/plugin/p3dDownload.h @@ -43,6 +43,7 @@ public: inline size_t get_total_data() const; void cancel(); + void clear(); public: // These are intended to be called only by P3DInstance. diff --git a/direct/src/plugin/p3dPackage.cxx b/direct/src/plugin/p3dPackage.cxx index a580a36b18..8eb4aa27fe 100755 --- a/direct/src/plugin/p3dPackage.cxx +++ b/direct/src/plugin/p3dPackage.cxx @@ -1172,14 +1172,13 @@ resume_download_finished(bool success) { string url = _try_urls.back(); _try_urls.pop_back(); - Download *new_download = new Download(*this); - new_download->set_filename(get_filename()); - new_download->set_url(url); - - _package->_active_download = new_download; + clear(); + set_url(url); + set_filename(get_filename()); + _package->_active_download = this; assert(!_package->_instances.empty()); - _package->_instances[0]->start_download(new_download); + _package->_instances[0]->start_download(this); return; } diff --git a/direct/src/plugin_npapi/ppInstance.cxx b/direct/src/plugin_npapi/ppInstance.cxx index c82efd1837..a3d4c50987 100644 --- a/direct/src/plugin_npapi/ppInstance.cxx +++ b/direct/src/plugin_npapi/ppInstance.cxx @@ -26,6 +26,7 @@ #include "dtool_platform.h" #include +#include #include // strcmp() #include @@ -113,6 +114,7 @@ begin() { if (!url.empty() && url[url.length() - 1] != '/') { url += '/'; } + _download_url_prefix = url; ostringstream strm; strm << url << "contents.xml"; @@ -362,6 +364,20 @@ url_notify(const char *url, NPReason reason, void *notifyData) { } break; + case PPDownloadRequest::RT_core_dll: + if (reason != NPRES_DONE) { + nout << "Failure downloading " << url << "\n"; + // Couldn't download from this mirror. Try the next one. + if (!_core_urls.empty()) { + string url = _core_urls.back(); + _core_urls.pop_back(); + + PPDownloadRequest *req2 = new PPDownloadRequest(PPDownloadRequest::RT_core_dll); + start_download(url, req2); + } + } + break; + default: nout << "Unexpected url_notify on stream type " << req->_rtype << "\n"; break; @@ -670,6 +686,84 @@ variant_to_p3dobj(const NPVariant *variant) { return P3D_new_none_object(); } +//////////////////////////////////////////////////////////////////// +// Function: PPInstance::read_xhost +// Access: Private +// Description: Reads the host data from the (or ) +// entry in the contents.xml file. +//////////////////////////////////////////////////////////////////// +void PPInstance:: +read_xhost(TiXmlElement *xhost) { + // Get the "download" URL, which is the source from which we + // download everything other than the contents.xml file. + const char *download_url = xhost->Attribute("download_url"); + if (download_url != NULL) { + _download_url_prefix = download_url; + } else { + _download_url_prefix = PANDA_PACKAGE_HOST_URL; + } + if (!_download_url_prefix.empty()) { + if (_download_url_prefix[_download_url_prefix.size() - 1] != '/') { + _download_url_prefix += "/"; + } + } + + TiXmlElement *xmirror = xhost->FirstChildElement("mirror"); + while (xmirror != NULL) { + const char *url = xmirror->Attribute("url"); + if (url != NULL) { + add_mirror(url); + } + xmirror = xmirror->NextSiblingElement("mirror"); + } +} + +//////////////////////////////////////////////////////////////////// +// Function: PPInstance::add_mirror +// Access: Private +// Description: Adds a new URL to serve as a mirror for this host. +// The mirrors will be consulted first, before +// consulting the host directly. +//////////////////////////////////////////////////////////////////// +void PPInstance:: +add_mirror(string mirror_url) { + // Ensure the URL ends in a slash. + if (!mirror_url.empty() && mirror_url[mirror_url.size() - 1] != '/') { + mirror_url += '/'; + } + + // Add it to the _mirrors list, but only if it's not already + // there. + if (find(_mirrors.begin(), _mirrors.end(), mirror_url) == _mirrors.end()) { + _mirrors.push_back(mirror_url); + } +} + +//////////////////////////////////////////////////////////////////// +// Function: PPInstance::choose_random_mirrors +// Access: Public +// Description: Selects num_mirrors elements, chosen at random, from +// the _mirrors list. Adds the selected mirrors to +// result. If there are fewer than num_mirrors elements +// in the list, adds only as many mirrors as we can get. +//////////////////////////////////////////////////////////////////// +void PPInstance:: +choose_random_mirrors(vector &result, int num_mirrors) { + vector selected; + + size_t num_to_select = min(_mirrors.size(), (size_t)num_mirrors); + while (num_to_select > 0) { + size_t i = (size_t)(((double)rand() / (double)RAND_MAX) * _mirrors.size()); + while (find(selected.begin(), selected.end(), i) != selected.end()) { + // Already found this i, find a new one. + i = (size_t)(((double)rand() / (double)RAND_MAX) * _mirrors.size()); + } + selected.push_back(i); + result.push_back(_mirrors[i]); + --num_to_select; + } +} + //////////////////////////////////////////////////////////////////// // Function: PPInstance::request_ready // Access: Private, Static @@ -719,6 +813,7 @@ request_ready(P3D_instance *instance) { //////////////////////////////////////////////////////////////////// void PPInstance:: start_download(const string &url, PPDownloadRequest *req) { + nout << "start_download: " << url << "\n"; if (url.substr(0, 7) == "file://") { // If we're "downloading" a file URL, just go read the file directly. downloaded_file(req, get_filename_from_url(url)); @@ -744,6 +839,32 @@ read_contents_file(const string &contents_filename) { TiXmlElement *xcontents = doc.FirstChildElement("contents"); if (xcontents != NULL) { + // Look for the entry; it might point us at a different + // download URL, and it might mention some mirrors. + string host_url = PANDA_PACKAGE_HOST_URL; + TiXmlElement *xhost = xcontents->FirstChildElement("host"); + if (xhost != NULL) { + const char *url = xhost->Attribute("url"); + if (url != NULL && host_url == string(url)) { + // We're the primary host. This is the normal case. + read_xhost(xhost); + + } else { + // We're not the primary host; perhaps we're an alternate host. + TiXmlElement *xalthost = xhost->FirstChildElement("alt_host"); + while (xalthost != NULL) { + const char *url = xalthost->Attribute("url"); + if (url != NULL && host_url == string(url)) { + // Yep, we're this alternate host. + read_xhost(xhost); + break; + } + xalthost = xalthost->NextSiblingElement("alt_host"); + } + } + } + + // Now look for the core API package. TiXmlElement *xpackage = xcontents->FirstChildElement("package"); while (xpackage != NULL) { const char *name = xpackage->Attribute("name"); @@ -874,13 +995,39 @@ get_core_api(TiXmlElement *xpackage) { do_load_plugin(); } else { - // The DLL file needs to be downloaded. Go get it. - string url = PANDA_PACKAGE_HOST_URL; - if (!url.empty() && url[url.length() - 1] != '/') { - url += '/'; - } + // The DLL file needs to be downloaded. Build up our list of + // URL's to attempt to download it from, in reverse order. + string url; + + // Our last act of desperation: hit the original host, with a + // query uniquifier, to break through any caches. + ostringstream strm; + strm << _download_url_prefix << _core_api_dll.get_filename() + << "?" << time(NULL); + url = strm.str(); + _core_urls.push_back(url); + + // Before we try that, we'll hit the original host, without a + // uniquifier. + url = _download_url_prefix; url += _core_api_dll.get_filename(); - + _core_urls.push_back(url); + + // And before we try that, we'll try two mirrors, at random. + vector mirrors; + choose_random_mirrors(mirrors, 2); + for (vector::iterator si = mirrors.begin(); + si != mirrors.end(); + ++si) { + url = (*si) + _core_api_dll.get_filename(); + _core_urls.push_back(url); + } + + // Now pick the first URL off the list, and try it. + assert(!_core_urls.empty()); + url = _core_urls.back(); + _core_urls.pop_back(); + PPDownloadRequest *req = new PPDownloadRequest(PPDownloadRequest::RT_core_dll); start_download(url, req); } @@ -906,40 +1053,40 @@ downloaded_plugin(const string &filename) { return; } - // Copy the file onto the target. - string pathname = _core_api_dll.get_pathname(_root_dir); - mkfile_complete(pathname, nout); + // Make sure the DLL was correctly downloaded before continuing. + if (!_core_api_dll.quick_verify_pathname(filename)) { + nout << "After download, " << _core_api_dll.get_filename() << " is no good.\n"; - ifstream in(filename.c_str(), ios::in | ios::binary); - ofstream out(pathname.c_str(), ios::out | ios::binary); + // That DLL came out wrong. Try the next URL. + if (!_core_urls.empty()) { + string url = _core_urls.back(); + _core_urls.pop_back(); + + PPDownloadRequest *req = new PPDownloadRequest(PPDownloadRequest::RT_core_dll); + start_download(url, req); + return; + } - static const size_t buffer_size = 4096; - static char buffer[buffer_size]; - - in.read(buffer, buffer_size); - size_t count = in.gcount(); - while (count != 0) { - out.write(buffer, count); - in.read(buffer, buffer_size); - count = in.gcount(); - } - - if (!out) { - nout << "Could not write " << pathname << "\n"; // TODO: fail return; } - in.close(); - out.close(); - if (_core_api_dll.quick_verify(_root_dir)) { - // We downloaded and installed it successfully. Now load it. - do_load_plugin(); + // Copy the file onto the target. + string pathname = _core_api_dll.get_pathname(_root_dir); + if (!copy_file(filename, pathname)) { + nout << "Couldn't copy " << pathname << "\n"; return; } - nout << "After download, " << pathname << " is no good.\n"; - // TODO: fail + if (!_core_api_dll.quick_verify(_root_dir)) { + nout << "After copying, " << pathname << " is no good.\n"; + // TODO: fail + return; + } + + // We downloaded and installed it successfully. Now load it. + do_load_plugin(); + return; } @@ -1141,6 +1288,7 @@ cleanup_window() { //////////////////////////////////////////////////////////////////// bool PPInstance:: copy_file(const string &from_filename, const string &to_filename) { + mkfile_complete(to_filename, nout); ifstream in(from_filename.c_str(), ios::in | ios::binary); ofstream out(to_filename.c_str(), ios::out | ios::binary); diff --git a/direct/src/plugin_npapi/ppInstance.h b/direct/src/plugin_npapi/ppInstance.h index afe4752361..13e224f087 100644 --- a/direct/src/plugin_npapi/ppInstance.h +++ b/direct/src/plugin_npapi/ppInstance.h @@ -64,6 +64,10 @@ public: static void output_np_variant(ostream &out, const NPVariant &result); private: + void read_xhost(TiXmlElement *xhost); + void add_mirror(string mirror_url); + void choose_random_mirrors(vector &result, int num_mirrors); + static void request_ready(P3D_instance *instance); void start_download(const string &url, PPDownloadRequest *req); @@ -96,6 +100,15 @@ private: Tokens _tokens; string _root_dir; + string _download_url_prefix; + typedef vector Mirrors; + Mirrors _mirrors; + + // A list of URL's that we will attempt to download the core API + // from. + typedef vector CoreUrls; + CoreUrls _core_urls; + FileSpec _core_api_dll; bool _got_instance_url;