serialize package download

direct/src/plugin/p3dInstance.cxx

@@ -85,6 +85,8 @@ P3DInstance(P3D_request_ready_func *func,
 
   // Set some initial properties.
   _panda_script_object->set_float_property("downloadProgress", 0.0);
+  _panda_script_object->set_string_property("downloadPackageName", "");
+  _panda_script_object->set_string_property("downloadPackageDisplayName", "");
   _panda_script_object->set_bool_property("downloadComplete", false);
   _panda_script_object->set_string_property("status", "initial");
 }
@@ -108,7 +110,7 @@ P3DInstance::
   // them.
   Packages::iterator pi;
   for (pi = _packages.begin(); pi != _packages.end(); ++pi) {
-    (*pi)->cancel_instance(this);
+    (*pi)->remove_instance(this);
   }
   _packages.clear();
 
@@ -628,13 +630,33 @@ add_package(P3DPackage *package) {
   }
 
   _packages.push_back(package);
-  package->set_instance(this);
+  package->add_instance(this);
 
   if (package->get_package_name() == "panda3d") {
     _panda3d = package;
   }
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: P3DInstance::get_packages_info_ready
+//       Access: Public
+//  Description: Returns true if all of the packages required by the
+//               instance have their information available and are
+//               ready to be downloaded, false if one or more of them
+//               is still waiting for information (or has failed).
+////////////////////////////////////////////////////////////////////
+bool P3DInstance::
+get_packages_info_ready() const {
+  Packages::const_iterator pi;
+  for (pi = _packages.begin(); pi != _packages.end(); ++pi) {
+    if (!(*pi)->get_info_ready()) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: P3DInstance::get_packages_ready
 //       Access: Public
@@ -1089,14 +1111,97 @@ make_splash_window() {
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: P3DInstance::start_package_download
+//     Function: P3DInstance::report_package_info_ready
 //       Access: Private
-//  Description: Notified when the package download begins.
+//  Description: Notified when a package information has been
+//               successfully downloaded and the package is idle,
+//               waiting for activate_download() to be called.
 ////////////////////////////////////////////////////////////////////
 void P3DInstance::
-start_package_download(P3DPackage *package) {
+report_package_info_ready(P3DPackage *package) {
-  _panda_script_object->set_string_property("status", "downloading");
+  if (get_packages_info_ready()) {
-  send_notify("ondownloadbegin");
+    // All packages are ready to go.  Let's start some download
+    // action.
+    _downloading_packages.clear();
+    _total_download_size = 0;
+    Packages::const_iterator pi;
+    for (pi = _packages.begin(); pi != _packages.end(); ++pi) {
+      P3DPackage *package = (*pi);
+      if (package->get_info_ready() && !package->get_ready()) {
+        _downloading_packages.push_back(package);
+        _total_download_size += package->get_download_size();
+      }
+    }
+    _download_package_index = 0;
+    _total_downloaded = 0;
+
+    nout << "Beginning download of " << _downloading_packages.size()
+         << " packages, total " << _total_download_size
+         << " bytes required.\n";
+
+    _panda_script_object->set_string_property("status", "downloading");
+    send_notify("ondownloadbegin");
+
+    start_next_download();
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: P3DInstance::start_next_download
+//       Access: Private
+//  Description: Checks whether all packages are ready and waiting to
+//               be downloaded; if so, starts the next package in
+//               sequence downloading.
+////////////////////////////////////////////////////////////////////
+void P3DInstance::
+start_next_download() {
+  while (_download_package_index < _downloading_packages.size()) {
+    P3DPackage *package = _downloading_packages[_download_package_index];
+    if (package->get_failed()) {
+      // Too bad.  TODO: fail.
+      return;
+    }
+
+    if (!package->get_ready()) {
+      // This package is ready to download.  Begin.
+      string name = package->get_package_display_name();
+      if (name.empty()) {
+        name = package->get_package_name();
+      }
+      _panda_script_object->set_string_property("downloadPackageName", package->get_package_name());
+      _panda_script_object->set_string_property("downloadPackageDisplayName", name);
+      if (_splash_window != NULL) {
+        _splash_window->set_install_label("Installing " + name);
+      }
+
+      nout << "Downloading " << package->get_package_name()
+           << ", package " << _download_package_index + 1
+           << " of " << _downloading_packages.size()
+           << ", " << package->get_download_size()
+           << " bytes.\n";
+
+      package->activate_download();
+      return;
+    }
+    
+    // This package has been downloaded.  Move to the next.
+    _total_downloaded += package->get_download_size();
+    ++_download_package_index;
+  }
+
+  // Looks like we're all done downloading.  Launch!
+  _downloading_packages.clear();
+
+  if (get_packages_ready()) {
+    _panda_script_object->set_bool_property("downloadComplete", true);
+    _panda_script_object->set_string_property("status", "starting");
+    send_notify("ondownloadcomplete");
+    
+    // Notify the session also.
+    if (_session != NULL) {
+      _session->report_packages_done(this, true);
+    }
+  }
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -1106,10 +1211,20 @@ start_package_download(P3DPackage *package) {
 ////////////////////////////////////////////////////////////////////
 void P3DInstance::
 report_package_progress(P3DPackage *package, double progress) {
+  if (_download_package_index >= _downloading_packages.size() ||
+      package != _downloading_packages[_download_package_index]) {
+    // Got a report from an unexpected package.
+    nout << "Got download progress report from " << package->get_package_name()
+         << ", not at download head (head is " << _download_package_index
+         << ")\n";
+    return;
+  }
+
+  // Scale the progress into the range appropriate to this package.
+  progress = (progress * package->get_download_size() + _total_downloaded) / _total_download_size;
+  progress = min(progress, 1.0);
+
   if (_splash_window != NULL) {
-    if (!package->get_package_display_name().empty()) {
-      _splash_window->set_install_label("Installing " + package->get_package_display_name());
-    }
     _splash_window->set_install_progress(progress);
   }
   _panda_script_object->set_float_property("downloadProgress", progress);
@@ -1125,17 +1240,9 @@ void P3DInstance::
 report_package_done(P3DPackage *package, bool success) {
   if (success) {
     report_package_progress(package, 1.0);
-
+    start_next_download();
-    if (get_packages_ready()) {
+  } else {
-      _panda_script_object->set_bool_property("downloadComplete", true);
+    // TODO: fail.
-      _panda_script_object->set_string_property("status", "starting");
-      send_notify("ondownloadcomplete");
-
-      // Notify the session also.
-      if (_session != NULL) {
-        _session->report_packages_done(this, success);
-      }
-    }
   }
 }
 

direct/src/plugin/p3dInstance.h

@@ -80,6 +80,7 @@ public:
   inline P3D_request_ready_func *get_request_ready_func() const;
 
   void add_package(P3DPackage *package);
+  bool get_packages_info_ready() const;
   bool get_packages_ready() const;
   bool get_packages_failed() const;
   
@@ -111,7 +112,8 @@ private:
                              const string &property_name, P3D_object *value,
                              bool needs_response, int unique_id);
   void make_splash_window();
-  void start_package_download(P3DPackage *package);
+  void report_package_info_ready(P3DPackage *package);
+  void start_next_download();
   void report_package_progress(P3DPackage *package, double progress);
   void report_package_done(P3DPackage *package, bool progress);
 
@@ -160,6 +162,10 @@ private:
 
   typedef vector<P3DPackage *> Packages;
   Packages _packages;
+  Packages _downloading_packages;
+  int _download_package_index;
+  size_t _total_download_size;
+  size_t _total_downloaded;
 
   // We keep the _panda3d pointer separately because it's so
   // important, but it's in the above vector also.

direct/src/plugin/p3dOsxSplashWindow.cxx

@@ -240,27 +240,30 @@ paint_window() {
              &src_rect, &dest_rect, srcCopy, 0);
   }
 
-  int bar_width = min((int)(win_width * 0.6), 400);
-  int bar_height = min((int)(win_height * 0.1), 24);
-  int bar_x = (win_width - bar_width) / 2;
-  int bar_y = (win_height - bar_height * 2);
-
-  int progress = bar_x + 1 + (int)((bar_width - 2) * _install_progress);
-
-  Rect rbar = { bar_y, bar_x, bar_y + bar_height, bar_x + bar_width };
-  Rect rneed = { bar_y + 1, progress, bar_y + bar_height - 1, bar_x + bar_width - 1 };
-  Rect rdone = { bar_y + 1, bar_x + 1, bar_y + bar_height - 1, progress };
-  FrameRect(&rbar);
-
-  RGBColor blue = { 27756, 42405, 57568 };
-  RGBForeColor(&blue);
-  PaintRect(&rdone);
-  EraseRect(&rneed);
-
-  RGBColor black = { 0, 0, 0 };
-  RGBForeColor(&black);
-
   if (!_install_label.empty()) {
+    // Draw the progress bar.  We don't draw this bar unless the
+    // install_label has been set nonempty.
+    int bar_width = min((int)(win_width * 0.6), 400);
+    int bar_height = min((int)(win_height * 0.1), 24);
+    int bar_x = (win_width - bar_width) / 2;
+    int bar_y = (win_height - bar_height * 2);
+    
+    int progress = bar_x + 1 + (int)((bar_width - 2) * _install_progress);
+    
+    Rect rbar = { bar_y, bar_x, bar_y + bar_height, bar_x + bar_width };
+    Rect rneed = { bar_y + 1, progress, bar_y + bar_height - 1, bar_x + bar_width - 1 };
+    Rect rdone = { bar_y + 1, bar_x + 1, bar_y + bar_height - 1, progress };
+    FrameRect(&rbar);
+    
+    RGBColor blue = { 27756, 42405, 57568 };
+    RGBForeColor(&blue);
+    PaintRect(&rdone);
+    EraseRect(&rneed);
+    
+    RGBColor black = { 0, 0, 0 };
+    RGBForeColor(&black);
+
+    // Now draw the install_label right above it.
     TextFont(0);
     TextFace(bold);
     TextMode(srcOr);

direct/src/plugin/p3dPackage.I

@@ -13,6 +13,51 @@
 ////////////////////////////////////////////////////////////////////
 
 
+////////////////////////////////////////////////////////////////////
+//     Function: P3DPackage::get_info_ready
+//       Access: Public
+//  Description: Returns true if the package file information has been
+//               downloaded and verified and is ready to be consulted,
+//               false if it is not yet available.
+////////////////////////////////////////////////////////////////////
+inline bool P3DPackage::
+get_info_ready() const {
+  return _info_ready;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: P3DPackage::get_download_size
+//       Access: Public
+//  Description: If get_info_ready() is true but get_ready() is false,
+//               it means the package is ready to be downloaded.  In
+//               this case, this method returns the number of bytes
+//               that need to be downloaded for this package.  This is
+//               intended to be used to estimate the download time for
+//               this package relative to other packages, for instance
+//               to update a progress bar sensibly.
+////////////////////////////////////////////////////////////////////
+inline size_t P3DPackage::
+get_download_size() const {
+  return _download_size;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: P3DPackage::activate_download
+//       Access: Public
+//  Description: Authorizes the package to begin downloading and
+//               unpacking the meat of its data.  Until this is
+//               called, the package will download its file
+//               information only, and then wait.
+////////////////////////////////////////////////////////////////////
+inline void P3DPackage::
+activate_download() {
+  _allow_data_download = true;
+
+  if (_info_ready) {
+    begin_data_download();
+  }
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: P3DPackage::get_ready
 //       Access: Public

direct/src/plugin/p3dPackage.cxx

@@ -53,6 +53,9 @@ P3DPackage(const string &package_name,
   _package_fullname += string("_") + _package_version;
   _package_dir += string("/") + _package_version;
 
+  _info_ready = false;
+  _download_size = 0;
+  _allow_data_download = false;
   _ready = false;
   _failed = false;
   _active_download = NULL;
@@ -86,26 +89,26 @@ P3DPackage::
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: P3DPackage::set_instance
+//     Function: P3DPackage::add_instance
 //       Access: Public
 //  Description: Specifies an instance that may be responsible for
 //               downloading this package.
 ////////////////////////////////////////////////////////////////////
 void P3DPackage::
-set_instance(P3DInstance *inst) {
+add_instance(P3DInstance *inst) {
   _instances.push_back(inst);
 
-  begin_download();
+  begin_info_download();
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: P3DPackage::cancel_instance
+//     Function: P3DPackage::remove_instance
 //       Access: Public
 //  Description: Indicates that the given instance will no longer be
 //               responsible for downloading this package.
 ////////////////////////////////////////////////////////////////////
 void P3DPackage::
-cancel_instance(P3DInstance *inst) {
+remove_instance(P3DInstance *inst) {
   assert(!_instances.empty());
 
   if (inst == _instances[0]) {
@@ -122,23 +125,25 @@ cancel_instance(P3DInstance *inst) {
   assert(ii != _instances.end());
   _instances.erase(ii);
 
-  begin_download();
+  begin_info_download();
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: P3DPackage::begin_download
+//     Function: P3DPackage::begin_info_download
-//       Access: Public
+//       Access: Private
-//  Description: Begins downloading and installing the package, if
+//  Description: Begins downloading and installing the information
-//               needed.
+//               about the package, including its file size and
+//               download source and such, if needed.  This is
+//               generally a very small download.
 ////////////////////////////////////////////////////////////////////
 void P3DPackage::
-begin_download() {  
+begin_info_download() {  
   if (_instances.empty()) {
     // Can't download without any instances.
     return;
   }
 
-  if (_ready) {
+  if (_info_ready) {
     // Already downloaded.
     return;
   }
@@ -341,7 +346,49 @@ got_desc_file(TiXmlDocument *doc, bool freshly_downloaded) {
     // Great, we're ready to begin.
     report_done(true);
 
-  } else if (_uncompressed_archive.quick_verify(_package_dir)) {
+  } else {
+    // We need to get the file data still, but at least we know all
+    // about it by this point.
+    if (!_allow_data_download) {
+      // Not authorized to start downloading yet; just report that
+      // we're ready.
+      report_info_ready();
+    } else {
+      // We've already been authorized to start downloading, so do it.
+      begin_data_download();
+    }
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: P3DPackage::begin_data_download
+//       Access: Private
+//  Description: Begins downloading and installing the package data
+//               itself, if needed.
+////////////////////////////////////////////////////////////////////
+void P3DPackage::
+begin_data_download() {
+  if (_instances.empty()) {
+    // Can't download without any instances.
+    return;
+  }
+
+  if (_ready) {
+    // Already downloaded.
+    return;
+  }
+
+  if (_active_download != NULL) {
+    // In the middle of downloading.
+    return;
+  }
+
+  if (!_allow_data_download) {
+    // Not authorized yet.
+    return;
+  }
+
+  if (_uncompressed_archive.quick_verify(_package_dir)) {
     // We need to re-extract the archive.
     extract_archive();
 
@@ -580,6 +627,25 @@ report_progress(double progress) {
   }
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: P3DPackage::report_info_ready
+//       Access: Private
+//  Description: Called when the package information has been
+//               successfully downloaded but activate_download() has
+//               not yet been called, and the package is now idle,
+//               waiting for activate_download() to be called.
+////////////////////////////////////////////////////////////////////
+void P3DPackage::
+report_info_ready() {
+  _info_ready = true;
+  _download_size = _compressed_archive.get_size();
+
+  Instances::iterator ii;
+  for (ii = _instances.begin(); ii != _instances.end(); ++ii) {
+    (*ii)->report_package_info_ready(this);
+  }
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: P3DPackage::report_done
 //       Access: Private
@@ -590,6 +656,7 @@ report_progress(double progress) {
 void P3DPackage::
 report_done(bool success) {
   if (success) {
+    _info_ready = true;
     _ready = true;
     _failed = false;
   } else {

direct/src/plugin/p3dPackage.h

@@ -41,6 +41,10 @@ public:
              const string &package_version);
   ~P3DPackage();
 
+  inline bool get_info_ready() const;
+  inline size_t get_download_size() const;
+
+  inline void activate_download();
   inline bool get_ready() const;
   inline bool get_failed() const;
   inline const string &get_package_dir() const;
@@ -48,8 +52,8 @@ public:
   inline const string &get_package_version() const;
   inline const string &get_package_display_name() const;
 
-  void set_instance(P3DInstance *inst);
+  void add_instance(P3DInstance *inst);
-  void cancel_instance(P3DInstance *inst);
+  void remove_instance(P3DInstance *inst);
 
 private:
   enum DownloadType {
@@ -71,7 +75,7 @@ private:
     DownloadType _dtype;
   };
 
-  void begin_download();
+  void begin_info_download();
   void download_contents_file();
   void contents_file_download_finished(bool success);
 
@@ -79,6 +83,7 @@ private:
   void desc_file_download_finished(bool success);
   void got_desc_file(TiXmlDocument *doc, bool freshly_downloaded);
 
+  void begin_data_download();
   void download_compressed_archive(bool allow_partial);
   void compressed_archive_download_progress(double progress);
   void compressed_archive_download_finished(bool success);
@@ -87,6 +92,7 @@ private:
   void extract_archive();
 
   void report_progress(double progress);
+  void report_info_ready();
   void report_done(bool success);
   void start_download(DownloadType dtype, const string &url, 
                       const string &pathname, bool allow_partial);
@@ -106,6 +112,9 @@ private:
   string _desc_file_basename;
   string _desc_file_pathname;
 
+  bool _info_ready;
+  size_t _download_size;
+  bool _allow_data_download;
   bool _ready;
   bool _failed;
   Download *_active_download;

direct/src/plugin_npapi/ppInstance.cxx

@@ -27,6 +27,10 @@
 #include <fstream>
 #include <string.h>  // strcmp()
 
+#ifndef _WIN32
+#include <sys/select.h>
+#endif
+
 PPInstance::FileDatas PPInstance::_file_datas;
 
 ////////////////////////////////////////////////////////////////////
@@ -1336,6 +1340,13 @@ thread_run() {
     // So far, so good.  Read some more.
     _file.read(buffer, buffer_size);
     count = _file.gcount();
+
+    // This is useful for development, to slow things down enough to
+    // see the progress bar move.
+    struct timeval tv;
+    tv.tv_sec = 0;
+    tv.tv_usec = 10000;
+    select(0, NULL, NULL, NULL, &tv);
   }
 
   // End of file.

direct/src/showutil/Packager.py

@@ -67,41 +67,14 @@ class Packager:
         def close(self):
             """ Writes out the contents of the current package. """
 
-            if not self.p3dApplication and not self.version:
-                # We must have a version string for packages.
-                self.version = '0.0'
-
-            self.packageBasename = self.packageName
-            packageDir = self.packageName
-            if self.platform:
-                self.packageBasename += '_' + self.platform
-                packageDir += '/' + self.platform
-            if self.version:
-                self.packageBasename += '_' + self.version
-                packageDir += '/' + self.version
-
-            self.packageDesc = self.packageBasename + '.xml'
-            self.packageImportDesc = self.packageBasename + '_import.xml'
-            if self.p3dApplication:
-                self.packageBasename += '.p3d'
-                packageDir = ''
-            else:
-                self.packageBasename += '.mf'
-                packageDir += '/'
-
-            self.packageFilename = packageDir + self.packageBasename
-            self.packageDesc = packageDir + self.packageDesc
-            self.packageImportDesc = packageDir + self.packageImportDesc
-
-            self.packageFullpath = Filename(self.packager.installDir, self.packageFilename)
-            self.packageFullpath.makeDir()
-            self.packageFullpath.unlink()
-
             if self.dryRun:
                 self.multifile = None
             else:
+                # Write the multifile to a temporary filename until we
+                # know enough to determine the output filename.
+                multifileFilename = Filename.temporary('', self.packageName)
                 self.multifile = Multifile()
-                self.multifile.openReadWrite(self.packageFullpath)
+                self.multifile.openReadWrite(multifileFilename)
 
             self.extracts = []
             self.components = []
@@ -181,6 +154,40 @@ class Packager:
                 xmodule.SetAttribute('name', moduleName)
                 self.components.append(xmodule)
 
+            # Now that we've processed all of the component files,
+            # (and set our platform if necessary), we can generate the
+            # output filename and write the output files.
+            
+            if not self.p3dApplication and not self.version:
+                # We must have a version string for packages.
+                self.version = '0.0'
+
+            self.packageBasename = self.packageName
+            packageDir = self.packageName
+            if self.platform:
+                self.packageBasename += '_' + self.platform
+                packageDir += '/' + self.platform
+            if self.version:
+                self.packageBasename += '_' + self.version
+                packageDir += '/' + self.version
+
+            self.packageDesc = self.packageBasename + '.xml'
+            self.packageImportDesc = self.packageBasename + '_import.xml'
+            if self.p3dApplication:
+                self.packageBasename += '.p3d'
+                packageDir = ''
+            else:
+                self.packageBasename += '.mf'
+                packageDir += '/'
+
+            self.packageFilename = packageDir + self.packageBasename
+            self.packageDesc = packageDir + self.packageDesc
+            self.packageImportDesc = packageDir + self.packageImportDesc
+
+            self.packageFullpath = Filename(self.packager.installDir, self.packageFilename)
+            self.packageFullpath.makeDir()
+            self.packageFullpath.unlink()
+
             if not self.dryRun:
                 self.freezer.addToMultifile(self.multifile, self.compressionLevel)
                 if self.p3dApplication:
@@ -188,6 +195,8 @@ class Packager:
                 self.multifile.repack()
                 self.multifile.close()
 
+                multifileFilename.renameTo(self.packageFullpath)
+
                 if not self.p3dApplication:
                     self.compressMultifile()
                     self.writeDescFile()