robustify download on the C++ side too

direct/src/p3d/PackageInfo.py

@@ -86,7 +86,7 @@ class PackageInfo:
         self.downloadProgress = 0
         
         # This is set true when the package file has been fully
-        # downloaded and unpackaged.
+        # downloaded and unpacked.
         self.hasPackage = False
 
         # This is set true when the package has been "installed",
@@ -270,10 +270,10 @@ class PackageInfo:
             return True
 
         # Still have to download it.
-        self.__buildInstallPlan()
+        self.__buildInstallPlans()
         return True
 
-    def __buildInstallPlan(self):
+    def __buildInstallPlans(self):
         """ Sets up self.installPlans, a list of one or more "plans"
         to download and install the package. """
 
@@ -297,8 +297,8 @@ class PackageInfo:
         # download, and build a plan (or two) to download it all.
         self.installPlans = None
 
-        # We know we will at least need to unpackage the archive at
+        # We know we will at least need to unpack the archive contents
-        # the end.
+        # at the end.
         unpackSize = 0
         for file in self.extracts:
             unpackSize += file.size

direct/src/plugin/fileSpec.cxx

@@ -229,7 +229,7 @@ check_hash(const string &pathname) const {
   MD5_CTX ctx;
   MD5_Init(&ctx);
 
-  static const int buffer_size = 1024;
+  static const int buffer_size = 4096;
   char buffer[buffer_size];
 
   stream.read(buffer, buffer_size);
@@ -245,6 +245,41 @@ check_hash(const string &pathname) const {
   return (memcmp(md, _hash, hash_size) == 0);
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: FileSpec::read_hash
+//       Access: Public
+//  Description: Computes the hash from the indicated pathname and
+//               stores it within the FileSpec.
+////////////////////////////////////////////////////////////////////
+bool FileSpec::
+read_hash(const string &pathname) {
+  memset(_hash, 0, sizeof(_hash));
+
+  ifstream stream(pathname.c_str(), ios::in | ios::binary);
+  if (!stream) {
+    //cerr << "unable to read " << pathname << "\n";
+    return false;
+  }
+
+  MD5_CTX ctx;
+  MD5_Init(&ctx);
+
+  static const int buffer_size = 4096;
+  char buffer[buffer_size];
+
+  stream.read(buffer, buffer_size);
+  size_t count = stream.gcount();
+  while (count != 0) {
+    MD5_Update(&ctx, buffer, count);
+    stream.read(buffer, buffer_size);
+    count = stream.gcount();
+  }
+
+  MD5_Final(_hash, &ctx);
+
+  return true;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: FileSpec::decode_hex
 //       Access: Private, Static

direct/src/plugin/fileSpec.h

@@ -42,6 +42,7 @@ public:
   bool full_verify(const string &package_dir) const;
   
   bool check_hash(const string &pathname) const;
+  bool read_hash(const string &pathname);
 
 private:
   static inline int decode_hexdigit(char c);

direct/src/plugin/p3dDownload.I

@@ -39,6 +39,16 @@ get_download_progress() const {
   return (double)_total_data / (double)_total_expected_data;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: P3DDownload::get_total_data
+//       Access: Public
+//  Description: Returns the total number of bytes downloaded so far.
+////////////////////////////////////////////////////////////////////
+inline size_t P3DDownload::
+get_total_data() const {
+  return _total_data;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: P3DDownload::get_download_finished
 //       Access: Public

direct/src/plugin/p3dDownload.h

@@ -40,6 +40,7 @@ public:
   inline double get_download_progress() const;
   inline bool get_download_finished() const;
   inline bool get_download_success() const;
+  inline size_t get_total_data() const;
 
   void cancel();
 

direct/src/plugin/p3dHost.I

@@ -40,15 +40,31 @@ get_host_url() const {
 //     Function: P3DHost::get_host_url_prefix
 //       Access: Public
 //  Description: Returns the root URL of this host, for constructing
-//               full URL sequences.  This is the same as
+//               the URL to download contents.xml only.  This is the
-//               get_host_url(), except it is guaranteed to end in a
+//               same as get_host_url(), except it is guaranteed to
-//               slash character.
+//               end in a slash character.
+//
+//               Also see get_download_url_prefix().
 ////////////////////////////////////////////////////////////////////
 inline const string &P3DHost::
 get_host_url_prefix() const {
   return _host_url_prefix;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: P3DHost::get_download_url_prefix
+//       Access: Public
+//  Description: Returns the root URL of this host, for downloading
+//               everything other than the contents.xml file.  This is
+//               often the same as get_host_url_prefix(), but it may
+//               be different in the case of an https server for
+//               contents.xml.
+////////////////////////////////////////////////////////////////////
+inline const string &P3DHost::
+get_download_url_prefix() const {
+  return _download_url_prefix;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: P3DHost::get_descriptive_name
 //       Access: Public
@@ -71,3 +87,27 @@ inline bool P3DHost::
 has_contents_file() const {
   return (_xcontents != NULL);
 }
+
+////////////////////////////////////////////////////////////////////
+//     Function: P3DHost::get_contents_seq
+//       Access: Public
+//  Description: Returns a number that increments whenever a new
+//               version of the contents.xml file has been read.  This
+//               can be used by packages to determine whether they
+//               need to redownload from scratch.
+////////////////////////////////////////////////////////////////////
+inline int P3DHost::
+get_contents_seq() const {
+  return _contents_seq;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: P3DHost::check_contents_hash
+//       Access: Public
+//  Description: Returns true if the indicated pathname has the same
+//               md5 hash as the contents.xml file, false otherwise.
+////////////////////////////////////////////////////////////////////
+inline bool P3DHost::
+check_contents_hash(const string &pathname) const {
+  return _contents_spec.check_hash(pathname);
+}

direct/src/plugin/p3dHost.cxx

@@ -34,8 +34,10 @@ P3DHost(const string &host_url) :
   if (!_host_url_prefix.empty() && _host_url_prefix[_host_url_prefix.size() - 1] != '/') {
     _host_url_prefix += "/";
   }
+  _download_url_prefix = _host_url_prefix;
 
   _xcontents = NULL;
+  _contents_seq = 0;
 
   determine_host_dir();
 }
@@ -137,6 +139,8 @@ read_contents_file(const string &contents_filename) {
     delete _xcontents;
   }
   _xcontents = (TiXmlElement *)xcontents->Clone();
+  ++_contents_seq;
+  _contents_spec.read_hash(contents_filename);
 
   TiXmlElement *xhost = _xcontents->FirstChildElement("host");
   if (xhost != NULL) {
@@ -463,10 +467,24 @@ determine_host_dir() {
 void P3DHost::
 read_xhost(TiXmlElement *xhost) {
   const char *descriptive_name = xhost->Attribute("descriptive_name");
-  if (descriptive_name != NULL) {
+  if (descriptive_name != NULL && _descriptive_name.empty()) {
     _descriptive_name = descriptive_name;
   }
 
+  // Get the "download" URL, which is the source from which we
+  // download everything other than the contents.xml file.
+  const char *download_url = xhost->Attribute("download_url");
+  if (download_url != NULL) {
+    _download_url_prefix = download_url;
+  }
+  if (!_download_url_prefix.empty()) {
+    if (_download_url_prefix[_download_url_prefix.size() - 1] != '/') {
+      _download_url_prefix += "/";
+    }
+  } else {
+    _download_url_prefix = _host_url_prefix;
+  }
+        
   TiXmlElement *xmirror = xhost->FirstChildElement("mirror");
   while (xmirror != NULL) {
     const char *url = xmirror->Attribute("url");

direct/src/plugin/p3dHost.h

@@ -16,7 +16,7 @@
 #define P3DHOST_H
 
 #include "p3d_plugin_common.h"
-
+#include "fileSpec.h"
 #include <map>
 
 class FileSpec;
@@ -37,11 +37,15 @@ public:
   inline const string &get_host_dir() const;
   inline const string &get_host_url() const;
   inline const string &get_host_url_prefix() const;
+  inline const string &get_download_url_prefix() const;
   inline const string &get_descriptive_name() const;
 
   P3DHost *get_alt_host(const string &alt_host);
 
   inline bool has_contents_file() const;
+  inline int get_contents_seq() const;
+  inline bool check_contents_hash(const string &pathname) const;
+
   bool read_contents_file();
   bool read_contents_file(const string &contents_filename);
 
@@ -70,8 +74,11 @@ private:
   string _host_dir;
   string _host_url;
   string _host_url_prefix;
+  string _download_url_prefix;
   string _descriptive_name;
   TiXmlElement *_xcontents;
+  int _contents_seq;
+  FileSpec _contents_spec;
 
   typedef vector<string> Mirrors;
   Mirrors _mirrors;

direct/src/plugin/p3dInstance.cxx

@@ -844,6 +844,7 @@ get_packages_failed() const {
 void P3DInstance::
 start_download(P3DDownload *download) {
   assert(download->get_download_id() == 0);
+  assert(!download->get_url().empty());
 
   P3DInstanceManager *inst_mgr = P3DInstanceManager::get_global_ptr();
 

direct/src/plugin/p3dMultifileReader.cxx

@@ -87,14 +87,12 @@ close() {
 //               directory.  Returns true on success, false on
 //               failure.
 //
-//               The parameters package, start_progress, and
+//               Upates the "step" object with the progress through
-//               progress_size are provided to make the appropriate
+//               this operation.
-//               status updates on the package's progress callbacks
-//               during this operation.
 ////////////////////////////////////////////////////////////////////
 bool P3DMultifileReader::
-extract_all(const string &to_dir,
+extract_all(const string &to_dir, P3DPackage *package, 
-            P3DPackage *package, double start_progress, double progress_size) {
+            P3DPackage::InstallStep *step) {
   assert(_is_open);
   if (_in.fail()) {
     return false;
@@ -102,12 +100,6 @@ extract_all(const string &to_dir,
 
   // Now walk through all of the files, and extract only the ones we
   // expect to encounter.
-  size_t num_processed = 0;
-  size_t num_expected = _subfiles.size();
-  if (package != NULL) {
-    num_expected = package->_extracts.size();
-  }
-
   Subfiles::iterator si;
   for (si = _subfiles.begin(); si != _subfiles.end(); ++si) {
     const Subfile &s = (*si);
@@ -141,10 +133,9 @@ extract_all(const string &to_dir,
     // program or something.
     chmod(output_pathname.c_str(), 0555);
 
-    ++num_processed;
+    if (step != NULL && package != NULL) {
-    if (package != NULL) {
+      step->_bytes_done += s._data_length;
-      double progress = (double)num_processed / (double)num_expected;
+      step->report_step_progress();
-      package->report_progress(start_progress + progress * progress_size);
     }
   }
 

direct/src/plugin/p3dMultifileReader.h

@@ -17,8 +17,7 @@
 
 #include "p3d_plugin_common.h"
 #include "p3dInstanceManager.h"  // for openssl
-
+#include "p3dPackage.h"
-class P3DPackage;
 
 ////////////////////////////////////////////////////////////////////
 //       Class : P3DMultifileReader
@@ -35,9 +34,8 @@ public:
   inline bool is_open() const;
   void close();
 
-  bool extract_all(const string &to_dir,
+  bool extract_all(const string &to_dir, P3DPackage *package, 
-                   P3DPackage *package, double start_progress, 
+                   P3DPackage::InstallStep *step);
-                   double progress_size);
 
   bool extract_one(ostream &out, const string &filename);
 

direct/src/plugin/p3dPackage.I

@@ -144,6 +144,18 @@ get_desc_file_pathname() const {
   return _desc_file_pathname;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: P3DPackage::get_desc_file_dirname
+//       Access: Public
+//  Description: Returns the relative path, on the host, of the
+//               directory that contains the desc file (and to which
+//               all of the paths in the desc file are relative).
+////////////////////////////////////////////////////////////////////
+inline const string &P3DPackage::
+get_desc_file_dirname() const {
+  return _desc_file_dirname;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: P3DPackage::get_archive_file_pathname
 //       Access: Public
@@ -156,3 +168,37 @@ get_archive_file_pathname() const {
   return _uncompressed_archive.get_pathname(_package_dir);
 }
 
+
+////////////////////////////////////////////////////////////////////
+//     Function: P3DPackage::InstallStep::get_effort
+//       Access: Public
+//  Description: Returns the relative amount of effort of this step.
+////////////////////////////////////////////////////////////////////
+inline double P3DPackage::InstallStep::
+get_effort() const {
+  return _bytes_needed * _bytes_factor;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: P3DPackage::InstallStep::get_progress
+//       Access: Public
+//  Description: Returns the progress of this step, in the range 0..1.
+////////////////////////////////////////////////////////////////////
+inline double P3DPackage::InstallStep::
+get_progress() const {
+  if (_bytes_needed == 0) {
+    return 1.0;
+  }
+  return min((double)_bytes_done / (double)_bytes_needed, 1.0);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: P3DPackage::InstallStep::report_step_progress
+//       Access: Public
+//  Description: Notifies the Package that progress has been made on
+//               this particular step.
+////////////////////////////////////////////////////////////////////
+inline void P3DPackage::InstallStep::
+report_step_progress() {
+  _package->report_progress(this);
+}

direct/src/plugin/p3dPackage.h

@@ -19,6 +19,7 @@
 #include "p3dFileDownload.h"
 #include "fileSpec.h"
 #include "get_tinyxml.h"
+#include <deque>
 
 class P3DHost;
 class P3DInstance;
@@ -61,6 +62,7 @@ public:
   inline const TiXmlElement *get_xconfig() const;
 
   inline const string &get_desc_file_pathname() const;
+  inline const string &get_desc_file_dirname() const;
   inline string get_archive_file_pathname() const;
 
   void add_instance(P3DInstance *inst);
@@ -69,10 +71,13 @@ public:
   TiXmlElement *make_xml();
 
 private:
+  typedef vector<FileSpec> Extracts;
+
   enum DownloadType {
     DT_contents_file,
+    DT_redownload_contents_file,
     DT_desc_file,
-    DT_compressed_archive
+    DT_install_step,
   };
 
   typedef vector<string> TryUrls;
@@ -87,6 +92,9 @@ private:
     virtual void download_progress();
     virtual void download_finished(bool success);
 
+  public:
+    void resume_download_finished(bool success);
+
   public:
     // URL's to try downloading from, in reverse order.
     TryUrls _try_urls;
@@ -99,33 +107,94 @@ private:
     FileSpec _file_spec;
   };
 
+  enum InstallToken {
+    IT_step_complete,
+    IT_step_failed,
+    IT_continue,
+  };
+
+  class InstallStep {
+  public:
+    InstallStep(P3DPackage *package, size_t bytes, double factor);
+    virtual ~InstallStep();
+
+    virtual InstallToken do_step(bool download_finished) = 0;
+
+    inline double get_effort() const;
+    inline double get_progress() const;
+    inline void report_step_progress();
+
+    P3DPackage *_package;
+    size_t _bytes_needed;
+    size_t _bytes_done;
+    double _bytes_factor;
+  };
+
+  class InstallStepDownloadFile : public InstallStep {
+  public:
+    InstallStepDownloadFile(P3DPackage *package, const FileSpec &file);
+    virtual ~InstallStepDownloadFile();
+
+    virtual InstallToken do_step(bool download_finished);
+
+    string _urlbase;
+    string _pathname;
+    FileSpec _file;
+    Download *_download;
+  };
+
+  class InstallStepUncompressFile : public InstallStep {
+  public:
+    InstallStepUncompressFile(P3DPackage *package, const FileSpec &source,
+                              const FileSpec &target);
+    virtual InstallToken do_step(bool download_finished);
+
+    FileSpec _source;
+    FileSpec _target;
+  };
+
+  class InstallStepUnpackArchive : public InstallStep {
+  public:
+    InstallStepUnpackArchive(P3DPackage *package, size_t unpack_size);
+    virtual InstallToken do_step(bool download_finished);
+  };
+
+  typedef deque<InstallStep *> InstallPlan;
+  typedef deque<InstallPlan> InstallPlans;
+  InstallPlans _install_plans;
+
+  double _total_plan_size;
+  double _total_plan_completed;
+  double _download_progress;
+  double _current_step_effort;
+
   void begin_info_download();
   void download_contents_file();
   void contents_file_download_finished(bool success);
+  void redownload_contents_file(Download *download);
+  void contents_file_redownload_finished(bool success);
   void host_got_contents_file();
 
   void download_desc_file();
   void desc_file_download_finished(bool success);
   void got_desc_file(TiXmlDocument *doc, bool freshly_downloaded);
 
-  void begin_data_download();
+  void clear_install_plans();
-  void download_compressed_archive();
+  void build_install_plans();
-  void compressed_archive_download_progress(double progress);
+  void follow_install_plans(bool download_finished);
-  void compressed_archive_download_finished(bool success);
 
-  void uncompress_archive();
+  class InstallStep;
-  void extract_archive();
+  void report_progress(InstallStep *step);
-
-  void report_progress(double progress);
   void report_info_ready();
   void report_done(bool success);
-  void start_download(DownloadType dtype, const string &urlbase, 
+  Download *start_download(DownloadType dtype, const string &urlbase, 
                            const string &pathname, const FileSpec &file_spec);
 
   bool is_extractable(const string &filename) const;
 
 private:
   P3DHost *_host;
+  int _host_contents_seq;
 
   string _package_name;
   string _package_version;
@@ -151,6 +220,7 @@ private:
   bool _ready;
   bool _failed;
   Download *_active_download;
+  Download *_saved_download;
 
   typedef vector<P3DInstance *> Instances;
   Instances _instances;
@@ -158,12 +228,17 @@ private:
   FileSpec _compressed_archive;
   FileSpec _uncompressed_archive;
 
-  typedef vector<FileSpec> Extracts;
+  size_t _unpack_size;
   Extracts _extracts;
 
-  friend class Download;
+  static const double _download_factor;
-  friend class P3DMultifileReader;
+  static const double _uncompress_factor;
+  static const double _unpack_factor;
+  static const double _patch_factor;
 
+  friend class Download;
+  friend class InstallStep;
+  friend class P3DMultifileReader;
   friend class P3DHost;
 };