modifications for vfs download

panda/src/downloader/downloadDb.I

@@ -183,16 +183,6 @@ set_client_multifile_extracted(string mfname) {
   write_client_db(_client_db._filename);
 }
 
-
-////////////////////////////////////////////////////////////////////
-//     Function: DownloadDb::
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-INLINE int DownloadDb::
-get_client_num_files(string mfname) const {
-  return _client_db.get_multifile_record_named(mfname)->get_num_files();
-}
 ////////////////////////////////////////////////////////////////////
 //     Function: DownloadDb::
 //       Access: Public

panda/src/downloader/downloadDb.cxx

@@ -33,6 +33,21 @@ PN_uint32 DownloadDb::_magic_number = 0xfeedfeed;
 // probably interrupted in the middle of the write.
 PN_uint32 DownloadDb::_bogus_magic_number = 0x11111111;
 
+
+static string
+back_to_front_slash(const string &str) {
+  string result = str;
+  string::iterator si;
+  for (si = result.begin(); si != result.end(); ++si) {
+    if ((*si) == '\\') {
+      (*si) = '/';
+    }
+  }
+
+  return result;
+}
+
+
 ////////////////////////////////////////////////////////////////////
 //     Function: DownloadDb::Constructor
 //       Access: Public
@@ -48,6 +63,22 @@ DownloadDb(Ramfile &server_file, Filename &client_file) {
   _server_db = read_db(server_file, 1);
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: DownloadDb::Constructor
+//       Access: Public
+//  Description: Create a download db with these client and server dbs
+////////////////////////////////////////////////////////////////////
+DownloadDb::
+DownloadDb(Filename &server_file, Filename &client_file) {
+  if (downloader_cat.is_debug())
+    downloader_cat.debug()
+      << "DownloadDb constructor called" << endl;
+  _client_db = read_db(client_file, 0);
+  _client_db._filename = client_file;
+  _server_db = read_db(server_file, 1);
+  _server_db._filename = server_file;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: DownloadDb::Constructor
 //       Access: Public
@@ -702,6 +733,11 @@ parse_mfr(uchar *start, int size) {
   mfr->_size = di.get_int32();
   mfr->_status = di.get_int32();
   mfr->_num_files = di.get_int32();
+
+  // At one time, we stored files in the database with a backslash
+  // separator.  Nowadays we use a forward slash, but we should make
+  // sure we properly convert any old records we might read.
+  mfr->_name = back_to_front_slash(mfr->_name);
   
   // Read the hash value
   HashVal hash;
@@ -740,6 +776,11 @@ parse_fr(uchar *start, int size) {
   PN_int32 fr_name_length = di.get_int32();
   fr->_name = di.extract_bytes(fr_name_length);
 
+  // At one time, we stored files in the database with a backslash
+  // separator.  Nowadays we use a forward slash, but we should make
+  // sure we properly convert any old records we might read.
+  fr->_name = back_to_front_slash(fr->_name);
+
   downloader_cat.spam()
     << "Parsed file record: " << fr->_name << endl;
 
@@ -762,6 +803,10 @@ read(istream &read_stream, bool want_server_info) {
   uchar *header_buf = new uchar[_header_length];
   // Read the header
   read_stream.read((char *)header_buf, _header_length);
+  if (read_stream.gcount() != _header_length) {
+    downloader_cat.error() << "DownloadDb::read() - Empty file" << endl;
+    return false;
+  }
 
   // Parse the header
   int num_multifiles = parse_header(header_buf, _header_length);
@@ -1028,23 +1073,13 @@ output(ostream &out) const {
   out << " FileRecord: " << _name << endl;
 }
 
-////////////////////////////////////////////////////////////////////
-//     Function: DownloadDb::add_version
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-void DownloadDb::
-add_version(const Filename &name, HashVal hash, Version version) {
-  add_version(name.get_fullpath(), hash, version);
-}
-
 ////////////////////////////////////////////////////////////////////
 //     Function: DownloadDb::add_version
 //       Access: Public
 //  Description: Note: version numbers start at 1
 ////////////////////////////////////////////////////////////////////
 void DownloadDb::
-add_version(const string &name, HashVal hash, Version version) {
+add_version(const Filename &name, HashVal hash, Version version) {
   nassertv(version >= 1);
 
   // Try to find this name in the map
@@ -1073,22 +1108,27 @@ add_version(const string &name, HashVal hash, Version version) {
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: DownloadDb::get_version
+//     Function: DownloadDb::has_version
 //       Access: Public
-//  Description:
+//  Description: Returns true if the indicated file has version
+//               information, false otherwise.  Some files recorded in
+//               the database may not bother to track versions.
 ////////////////////////////////////////////////////////////////////
-int DownloadDb::
+bool DownloadDb::
-get_version(const Filename &name, HashVal hash) {
+has_version(const Filename &name) {
-  return get_version(name.get_fullpath(), hash);
+  return (_versions.find(name) != _versions.end());
 }
 
 ////////////////////////////////////////////////////////////////////
 //     Function: DownloadDb::get_version
 //       Access: Public
-//  Description:
+//  Description: Returns the version number of this particular file,
+//               determined by looking up the hash generated from the
+//               file.  Returns -1 if the version number cannot be
+//               determined.
 ////////////////////////////////////////////////////////////////////
 int DownloadDb::
-get_version(const string &name, HashVal hash) {
+get_version(const Filename &name, HashVal hash) {
   VersionMap::const_iterator vmi = _versions.find(name);
   if (vmi == _versions.end()) {
     downloader_cat.debug()

panda/src/downloader/downloadDb.h

@@ -55,6 +55,17 @@ MultifileRecord is a Vector<FileRecord>
 typedef int Version;
 typedef float Phase;
 
+////////////////////////////////////////////////////////////////////
+//       Class : DownloadDb
+// Description : A listing of files within multifiles for management
+//               of client-side synchronization with a server-provided
+//               set of files.
+//
+//               This class manages one copy of the database for the
+//               client, representing the files on the client system,
+//               and another copy for the server, representing the
+//               files the server has available.
+////////////////////////////////////////////////////////////////////
 class EXPCL_PANDAEXPRESS DownloadDb {
 PUBLISHED:
   // Status of a multifile is stored in this enum
@@ -70,6 +81,7 @@ PUBLISHED:
 
   DownloadDb(void);
   DownloadDb(Ramfile &server_file, Filename &client_file);
+  DownloadDb(Filename &server_file, Filename &client_file);
   ~DownloadDb(void);
 
   void output(ostream &out) const;
@@ -99,11 +111,7 @@ PUBLISHED:
   INLINE void set_client_multifile_decompressed(string mfname);
   INLINE void set_client_multifile_extracted(string mfname);
 
-  INLINE int get_client_num_files(string mfname) const;
   INLINE int get_server_num_files(string mfname) const;
-
-  // The client does not store the names of all the files anymore
-  // INLINE string get_client_file_name(string mfname, int index) const;
   INLINE string get_server_file_name(string mfname, int index) const;
 
   // Queries from the Launcher
@@ -203,13 +211,12 @@ public:
   static PN_uint32 _magic_number;
   static PN_uint32 _bogus_magic_number;
   typedef pvector<HashVal> vectorHash;
-  typedef pmap<string, vectorHash> VersionMap;
+  typedef pmap<Filename, vectorHash> VersionMap;
 
 PUBLISHED:
   void add_version(const Filename &name, HashVal hash, Version version);
-  void add_version(const string &name, HashVal hash, Version version);
+  bool has_version(const Filename &name);
   int get_version(const Filename &name, HashVal hash);
-  int get_version(const string &name, HashVal hash);
 
 protected:
   void write_version_map(ofstream &write_stream);

panda/src/downloader/extractor.cxx

@@ -25,7 +25,7 @@
 
 ////////////////////////////////////////////////////////////////////
 //     Function: Extractor::Constructor
-//       Access: Public
+//       Access: Published
 //  Description:
 ////////////////////////////////////////////////////////////////////
 Extractor::
@@ -35,89 +35,137 @@ Extractor() {
 
 ////////////////////////////////////////////////////////////////////
 //     Function: Extractor::Destructor
-//       Access: Public
+//       Access: Published
 //  Description:
 ////////////////////////////////////////////////////////////////////
 Extractor::
 ~Extractor() {
-  if (downloader_cat.is_debug())
+  reset();
-    downloader_cat.debug()
-      << "Extractor destructor called" << endl;
-  if (_initiated) {
-    cleanup();
-  }
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Extractor::initiate
+//     Function: Extractor::set_multifile
-//       Access: Public
+//       Access: Published
-//  Description: Begins the extraction process.  Returns EU_success if
+//  Description: Specifies the filename of the Multifile that the
-//               successful, EU_error_abort otherwise.
+//               Extractor will read.  Returns true on success, false
-//
+//               if the mulifile name is invalid.
-//               After calling initiate(), you should repeatedly call
+////////////////////////////////////////////////////////////////////
-//               run() as a background task until the file is
+bool Extractor::
-//               completely extracted.
+set_multifile(const Filename &multifile_name) {
+  reset();
+  _multifile_name = multifile_name;
+  return _multifile.open_read(multifile_name);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Extractor::set_extract_dir
+//       Access: Published
+//  Description: Specifies the directory into which all extracted
+//               subfiles will be written.  Relative paths of subfiles
+//               within the Multifile will be written as relative
+//               paths to this directory.
+////////////////////////////////////////////////////////////////////
+void Extractor::
+set_extract_dir(const Filename &extract_dir) {
+  _extract_dir = extract_dir;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Extractor::reset
+//       Access: Published
+//  Description: Interrupts the Extractor in the middle of its
+//               business and makes it ready to accept a new list of
+//               subfiles to extract.
+////////////////////////////////////////////////////////////////////
+void Extractor::
+reset() {
+  if (_initiated) {
+    if (_read != (istream *)NULL) {
+      delete _read;
+      _read = (istream *)NULL;
+    }
+    _write.close();
+    _initiated = false;
+  }
+
+  _requests.clear();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Extractor::request_subfile
+//       Access: Published
+//  Description: Requests a particular subfile to be extracted when
+//               step() or run() is called.  Returns true if the
+//               subfile exists, false otherwise.
+////////////////////////////////////////////////////////////////////
+bool Extractor::
+request_subfile(const Filename &subfile_name) {
+  int index = _multifile.find_subfile(subfile_name);
+  if (index < 0) {
+    return false;
+  }
+  _requests.push_back(index);
+  return true;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Extractor::request_all_subfiles
+//       Access: Published
+//  Description: Requests all subfiles in the Multifile to be
+//               extracted.  Returns the number requested.
 ////////////////////////////////////////////////////////////////////
 int Extractor::
-initiate(const Filename &multifile_name, const Filename &extract_to) {
+request_all_subfiles() {
-  if (_initiated) {
+  _requests.clear();
-    downloader_cat.error()
+  int num_subfiles = _multifile.get_num_subfiles();
-      << "Extractor::initiate() - Extraction has already been initiated"
+  for (int i = 0; i < num_subfiles; i++) {
-      << endl;
+    _requests.push_back(i);
-    return EU_error_abort;
   }
-
+  return num_subfiles;
-  _multifile_name = multifile_name;
-  _extract_to = extract_to;
-
-  if (!_multifile.open_read(_multifile_name)) {
-    downloader_cat.error()
-      << "Error opening Multifile " << _multifile_name << ".\n";
-    return EU_error_abort;
-  }
-
-  _subfile_index = 0;
-  _subfile_pos = 0;
-  _subfile_length = 0;
-  _read = (istream *)NULL;
-  _initiated = true;
-
-  return EU_success;
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Extractor::run
+//     Function: Extractor::step
-//       Access: Public
+//       Access: Published
-//  Description: Extracts the next small unit of data from the
+//  Description: After all of the requests have been made via
+//               request_file() or request_all_subfiles(), call step()
+//               repeatedly until it stops returning EU_ok.
+//
+//               step() extracts the next small unit of data from the
 //               Multifile.  Returns EU_ok if progress is continuing,
 //               EU_error_abort if there is a problem, or EU_success
 //               when the last piece has been extracted.
+//
+//               Also see run().
 ////////////////////////////////////////////////////////////////////
 int Extractor::
-run() {
+step() {
   if (!_initiated) {
-    downloader_cat.error()
+    _request_index = 0;
-      << "Extractor::run() - Extraction has not been initiated"
+    _subfile_index = 0;
-      << endl;
+    _subfile_pos = 0;
-    return EU_error_abort;
+    _subfile_length = 0;
+    _read = (istream *)NULL;
+    _initiated = true;
   }
 
   if (_read == (istream *)NULL) {
     // Time to open the next subfile.
-    if (_subfile_index >= _multifile.get_num_subfiles()) {
+    if (_request_index >= (int)_requests.size()) {
       // All done!
-      cleanup();
+      reset();
       return EU_success;
     }
 
-    Filename subfile_filename(_extract_to, 
+    _subfile_index = _requests[_request_index];
+    Filename subfile_filename(_extract_dir, 
                               _multifile.get_subfile_name(_subfile_index));
     subfile_filename.set_binary();
     subfile_filename.make_dir();
     if (!subfile_filename.open_write(_write)) {
       downloader_cat.error()
         << "Unable to write to " << subfile_filename << ".\n";
-      cleanup();
+      reset();
       return EU_error_abort;
     }
 
@@ -130,7 +178,7 @@ run() {
     delete _read;
     _read = (istream *)NULL;
     _write.close();
-    _subfile_index++;
+    _request_index++;
 
   } else {
     // Read a number of bytes from the subfile and write them to the
@@ -142,7 +190,7 @@ run() {
       if (_read->eof() || _read->fail()) {
         downloader_cat.error()
           << "Unexpected EOF on multifile " << _multifile_name << ".\n";
-        cleanup();
+        reset();
         return EU_error_abort;
       }
       _write.put(byte);
@@ -154,20 +202,22 @@ run() {
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Extractor::extract
+//     Function: Extractor::run
-//       Access: Public
+//       Access: Published
 //  Description: A convenience function to extract the Multifile all
 //               at once, when you don't care about doing it in the
 //               background.
+//
+//               First, call request_file() or request_all_files() to
+//               specify the files you would like to extract, then
+//               call run() to do the extraction.  Also see step() for
+//               when you would like the extraction to happen as a
+//               background task.
 ////////////////////////////////////////////////////////////////////
 bool Extractor::
-extract(const Filename &source_file, const Filename &rel_path) {
+run() {
-  int ret = initiate(source_file, rel_path);
+  while (true) {
-  if (ret < 0) {
+    int ret = step();
-    return false;
-  }
-  for (;;) {
-    ret = run();
     if (ret == EU_success) {
       return true;
     }
@@ -177,28 +227,3 @@ extract(const Filename &source_file, const Filename &rel_path) {
   }
   return false;
 }
-
-////////////////////////////////////////////////////////////////////
-//     Function: Extractor::cleanup
-//       Access: Private
-//  Description:
-////////////////////////////////////////////////////////////////////
-void Extractor::
-cleanup() {
-  if (downloader_cat.is_debug())
-    downloader_cat.debug()
-      << "Extractor cleanup called" << endl;
-  if (!_initiated) {
-    downloader_cat.error()
-      << "Extractor::cleanup() - Extraction has not been initiated"
-      << endl;
-    return;
-  }
-
-  if (_read != (istream *)NULL) {
-    delete _read;
-    _read = (istream *)NULL;
-  }
-  _multifile.close();
-  _write.close();
-}

panda/src/downloader/extractor.h

@@ -42,21 +42,32 @@ PUBLISHED:
   Extractor();
   ~Extractor();
 
-  int initiate(const Filename &multifile_name, const Filename &extract_to = "");
+  bool set_multifile(const Filename &multifile_name);
-  int run();
+  void set_extract_dir(const Filename &extract_dir);
 
-  bool extract(const Filename &multifile_name, const Filename &extract_to = "");
+  void reset();
 
+  bool request_subfile(const Filename &subfile_name);
+  int request_all_subfiles();
+
+  int step();
   INLINE float get_progress(void) const;
 
-private:
+  bool run();
-  void cleanup();
 
-  bool _initiated;
+private:
   Filename _multifile_name;
-  Filename _extract_to;
   Multifile _multifile;
 
+  Filename _extract_dir;
+
+  typedef pvector<int> Requests;
+  Requests _requests;
+  
+  bool _initiated;
+
+  // These are used only while processing.
+  int _request_index;
   int _subfile_index;
   size_t _subfile_pos;
   size_t _subfile_length;