From 9701bfcd43a350eb56380f99cc07c5370db0a1b4 Mon Sep 17 00:00:00 2001 From: David Rose Date: Wed, 7 Aug 2002 15:54:28 +0000 Subject: [PATCH] modifications for vfs download --- panda/src/downloader/downloadDb.I | 10 -- panda/src/downloader/downloadDb.cxx | 76 ++++++++--- panda/src/downloader/downloadDb.h | 21 ++- panda/src/downloader/extractor.cxx | 199 ++++++++++++++++------------ panda/src/downloader/extractor.h | 25 +++- 5 files changed, 202 insertions(+), 129 deletions(-) diff --git a/panda/src/downloader/downloadDb.I b/panda/src/downloader/downloadDb.I index 168f661b72..77ab5ee9b4 100644 --- a/panda/src/downloader/downloadDb.I +++ b/panda/src/downloader/downloadDb.I @@ -183,16 +183,6 @@ set_client_multifile_extracted(string mfname) { write_client_db(_client_db._filename); } - -//////////////////////////////////////////////////////////////////// -// Function: DownloadDb:: -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -INLINE int DownloadDb:: -get_client_num_files(string mfname) const { - return _client_db.get_multifile_record_named(mfname)->get_num_files(); -} //////////////////////////////////////////////////////////////////// // Function: DownloadDb:: // Access: Public diff --git a/panda/src/downloader/downloadDb.cxx b/panda/src/downloader/downloadDb.cxx index 8c983a304f..c040e5c6ec 100644 --- a/panda/src/downloader/downloadDb.cxx +++ b/panda/src/downloader/downloadDb.cxx @@ -33,6 +33,21 @@ PN_uint32 DownloadDb::_magic_number = 0xfeedfeed; // probably interrupted in the middle of the write. PN_uint32 DownloadDb::_bogus_magic_number = 0x11111111; + +static string +back_to_front_slash(const string &str) { + string result = str; + string::iterator si; + for (si = result.begin(); si != result.end(); ++si) { + if ((*si) == '\\') { + (*si) = '/'; + } + } + + return result; +} + + //////////////////////////////////////////////////////////////////// // Function: DownloadDb::Constructor // Access: Public @@ -48,6 +63,22 @@ DownloadDb(Ramfile &server_file, Filename &client_file) { _server_db = read_db(server_file, 1); } +//////////////////////////////////////////////////////////////////// +// Function: DownloadDb::Constructor +// Access: Public +// Description: Create a download db with these client and server dbs +//////////////////////////////////////////////////////////////////// +DownloadDb:: +DownloadDb(Filename &server_file, Filename &client_file) { + if (downloader_cat.is_debug()) + downloader_cat.debug() + << "DownloadDb constructor called" << endl; + _client_db = read_db(client_file, 0); + _client_db._filename = client_file; + _server_db = read_db(server_file, 1); + _server_db._filename = server_file; +} + //////////////////////////////////////////////////////////////////// // Function: DownloadDb::Constructor // Access: Public @@ -702,6 +733,11 @@ parse_mfr(uchar *start, int size) { mfr->_size = di.get_int32(); mfr->_status = di.get_int32(); mfr->_num_files = di.get_int32(); + + // At one time, we stored files in the database with a backslash + // separator. Nowadays we use a forward slash, but we should make + // sure we properly convert any old records we might read. + mfr->_name = back_to_front_slash(mfr->_name); // Read the hash value HashVal hash; @@ -740,6 +776,11 @@ parse_fr(uchar *start, int size) { PN_int32 fr_name_length = di.get_int32(); fr->_name = di.extract_bytes(fr_name_length); + // At one time, we stored files in the database with a backslash + // separator. Nowadays we use a forward slash, but we should make + // sure we properly convert any old records we might read. + fr->_name = back_to_front_slash(fr->_name); + downloader_cat.spam() << "Parsed file record: " << fr->_name << endl; @@ -762,6 +803,10 @@ read(istream &read_stream, bool want_server_info) { uchar *header_buf = new uchar[_header_length]; // Read the header read_stream.read((char *)header_buf, _header_length); + if (read_stream.gcount() != _header_length) { + downloader_cat.error() << "DownloadDb::read() - Empty file" << endl; + return false; + } // Parse the header int num_multifiles = parse_header(header_buf, _header_length); @@ -1028,23 +1073,13 @@ output(ostream &out) const { out << " FileRecord: " << _name << endl; } -//////////////////////////////////////////////////////////////////// -// Function: DownloadDb::add_version -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -void DownloadDb:: -add_version(const Filename &name, HashVal hash, Version version) { - add_version(name.get_fullpath(), hash, version); -} - //////////////////////////////////////////////////////////////////// // Function: DownloadDb::add_version // Access: Public // Description: Note: version numbers start at 1 //////////////////////////////////////////////////////////////////// void DownloadDb:: -add_version(const string &name, HashVal hash, Version version) { +add_version(const Filename &name, HashVal hash, Version version) { nassertv(version >= 1); // Try to find this name in the map @@ -1073,22 +1108,27 @@ add_version(const string &name, HashVal hash, Version version) { } //////////////////////////////////////////////////////////////////// -// Function: DownloadDb::get_version +// Function: DownloadDb::has_version // Access: Public -// Description: +// Description: Returns true if the indicated file has version +// information, false otherwise. Some files recorded in +// the database may not bother to track versions. //////////////////////////////////////////////////////////////////// -int DownloadDb:: -get_version(const Filename &name, HashVal hash) { - return get_version(name.get_fullpath(), hash); +bool DownloadDb:: +has_version(const Filename &name) { + return (_versions.find(name) != _versions.end()); } //////////////////////////////////////////////////////////////////// // Function: DownloadDb::get_version // Access: Public -// Description: +// Description: Returns the version number of this particular file, +// determined by looking up the hash generated from the +// file. Returns -1 if the version number cannot be +// determined. //////////////////////////////////////////////////////////////////// int DownloadDb:: -get_version(const string &name, HashVal hash) { +get_version(const Filename &name, HashVal hash) { VersionMap::const_iterator vmi = _versions.find(name); if (vmi == _versions.end()) { downloader_cat.debug() diff --git a/panda/src/downloader/downloadDb.h b/panda/src/downloader/downloadDb.h index f363def1a8..32d1056039 100644 --- a/panda/src/downloader/downloadDb.h +++ b/panda/src/downloader/downloadDb.h @@ -55,6 +55,17 @@ MultifileRecord is a Vector typedef int Version; typedef float Phase; +//////////////////////////////////////////////////////////////////// +// Class : DownloadDb +// Description : A listing of files within multifiles for management +// of client-side synchronization with a server-provided +// set of files. +// +// This class manages one copy of the database for the +// client, representing the files on the client system, +// and another copy for the server, representing the +// files the server has available. +//////////////////////////////////////////////////////////////////// class EXPCL_PANDAEXPRESS DownloadDb { PUBLISHED: // Status of a multifile is stored in this enum @@ -70,6 +81,7 @@ PUBLISHED: DownloadDb(void); DownloadDb(Ramfile &server_file, Filename &client_file); + DownloadDb(Filename &server_file, Filename &client_file); ~DownloadDb(void); void output(ostream &out) const; @@ -99,11 +111,7 @@ PUBLISHED: INLINE void set_client_multifile_decompressed(string mfname); INLINE void set_client_multifile_extracted(string mfname); - INLINE int get_client_num_files(string mfname) const; INLINE int get_server_num_files(string mfname) const; - - // The client does not store the names of all the files anymore - // INLINE string get_client_file_name(string mfname, int index) const; INLINE string get_server_file_name(string mfname, int index) const; // Queries from the Launcher @@ -203,13 +211,12 @@ public: static PN_uint32 _magic_number; static PN_uint32 _bogus_magic_number; typedef pvector vectorHash; - typedef pmap VersionMap; + typedef pmap VersionMap; PUBLISHED: void add_version(const Filename &name, HashVal hash, Version version); - void add_version(const string &name, HashVal hash, Version version); + bool has_version(const Filename &name); int get_version(const Filename &name, HashVal hash); - int get_version(const string &name, HashVal hash); protected: void write_version_map(ofstream &write_stream); diff --git a/panda/src/downloader/extractor.cxx b/panda/src/downloader/extractor.cxx index e77db8e2c6..911ed2b459 100644 --- a/panda/src/downloader/extractor.cxx +++ b/panda/src/downloader/extractor.cxx @@ -25,7 +25,7 @@ //////////////////////////////////////////////////////////////////// // Function: Extractor::Constructor -// Access: Public +// Access: Published // Description: //////////////////////////////////////////////////////////////////// Extractor:: @@ -35,89 +35,137 @@ Extractor() { //////////////////////////////////////////////////////////////////// // Function: Extractor::Destructor -// Access: Public +// Access: Published // Description: //////////////////////////////////////////////////////////////////// Extractor:: ~Extractor() { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Extractor destructor called" << endl; - if (_initiated) { - cleanup(); - } + reset(); } //////////////////////////////////////////////////////////////////// -// Function: Extractor::initiate -// Access: Public -// Description: Begins the extraction process. Returns EU_success if -// successful, EU_error_abort otherwise. -// -// After calling initiate(), you should repeatedly call -// run() as a background task until the file is -// completely extracted. +// Function: Extractor::set_multifile +// Access: Published +// Description: Specifies the filename of the Multifile that the +// Extractor will read. Returns true on success, false +// if the mulifile name is invalid. +//////////////////////////////////////////////////////////////////// +bool Extractor:: +set_multifile(const Filename &multifile_name) { + reset(); + _multifile_name = multifile_name; + return _multifile.open_read(multifile_name); +} + +//////////////////////////////////////////////////////////////////// +// Function: Extractor::set_extract_dir +// Access: Published +// Description: Specifies the directory into which all extracted +// subfiles will be written. Relative paths of subfiles +// within the Multifile will be written as relative +// paths to this directory. +//////////////////////////////////////////////////////////////////// +void Extractor:: +set_extract_dir(const Filename &extract_dir) { + _extract_dir = extract_dir; +} + +//////////////////////////////////////////////////////////////////// +// Function: Extractor::reset +// Access: Published +// Description: Interrupts the Extractor in the middle of its +// business and makes it ready to accept a new list of +// subfiles to extract. +//////////////////////////////////////////////////////////////////// +void Extractor:: +reset() { + if (_initiated) { + if (_read != (istream *)NULL) { + delete _read; + _read = (istream *)NULL; + } + _write.close(); + _initiated = false; + } + + _requests.clear(); +} + +//////////////////////////////////////////////////////////////////// +// Function: Extractor::request_subfile +// Access: Published +// Description: Requests a particular subfile to be extracted when +// step() or run() is called. Returns true if the +// subfile exists, false otherwise. +//////////////////////////////////////////////////////////////////// +bool Extractor:: +request_subfile(const Filename &subfile_name) { + int index = _multifile.find_subfile(subfile_name); + if (index < 0) { + return false; + } + _requests.push_back(index); + return true; +} + +//////////////////////////////////////////////////////////////////// +// Function: Extractor::request_all_subfiles +// Access: Published +// Description: Requests all subfiles in the Multifile to be +// extracted. Returns the number requested. //////////////////////////////////////////////////////////////////// int Extractor:: -initiate(const Filename &multifile_name, const Filename &extract_to) { - if (_initiated) { - downloader_cat.error() - << "Extractor::initiate() - Extraction has already been initiated" - << endl; - return EU_error_abort; +request_all_subfiles() { + _requests.clear(); + int num_subfiles = _multifile.get_num_subfiles(); + for (int i = 0; i < num_subfiles; i++) { + _requests.push_back(i); } - - _multifile_name = multifile_name; - _extract_to = extract_to; - - if (!_multifile.open_read(_multifile_name)) { - downloader_cat.error() - << "Error opening Multifile " << _multifile_name << ".\n"; - return EU_error_abort; - } - - _subfile_index = 0; - _subfile_pos = 0; - _subfile_length = 0; - _read = (istream *)NULL; - _initiated = true; - - return EU_success; + return num_subfiles; } //////////////////////////////////////////////////////////////////// -// Function: Extractor::run -// Access: Public -// Description: Extracts the next small unit of data from the +// Function: Extractor::step +// Access: Published +// Description: After all of the requests have been made via +// request_file() or request_all_subfiles(), call step() +// repeatedly until it stops returning EU_ok. +// +// step() extracts the next small unit of data from the // Multifile. Returns EU_ok if progress is continuing, // EU_error_abort if there is a problem, or EU_success // when the last piece has been extracted. +// +// Also see run(). //////////////////////////////////////////////////////////////////// int Extractor:: -run() { +step() { if (!_initiated) { - downloader_cat.error() - << "Extractor::run() - Extraction has not been initiated" - << endl; - return EU_error_abort; + _request_index = 0; + _subfile_index = 0; + _subfile_pos = 0; + _subfile_length = 0; + _read = (istream *)NULL; + _initiated = true; } if (_read == (istream *)NULL) { // Time to open the next subfile. - if (_subfile_index >= _multifile.get_num_subfiles()) { + if (_request_index >= (int)_requests.size()) { // All done! - cleanup(); + reset(); return EU_success; } - Filename subfile_filename(_extract_to, + _subfile_index = _requests[_request_index]; + Filename subfile_filename(_extract_dir, _multifile.get_subfile_name(_subfile_index)); subfile_filename.set_binary(); subfile_filename.make_dir(); if (!subfile_filename.open_write(_write)) { downloader_cat.error() << "Unable to write to " << subfile_filename << ".\n"; - cleanup(); + reset(); return EU_error_abort; } @@ -130,7 +178,7 @@ run() { delete _read; _read = (istream *)NULL; _write.close(); - _subfile_index++; + _request_index++; } else { // Read a number of bytes from the subfile and write them to the @@ -142,7 +190,7 @@ run() { if (_read->eof() || _read->fail()) { downloader_cat.error() << "Unexpected EOF on multifile " << _multifile_name << ".\n"; - cleanup(); + reset(); return EU_error_abort; } _write.put(byte); @@ -154,20 +202,22 @@ run() { } //////////////////////////////////////////////////////////////////// -// Function: Extractor::extract -// Access: Public +// Function: Extractor::run +// Access: Published // Description: A convenience function to extract the Multifile all // at once, when you don't care about doing it in the // background. +// +// First, call request_file() or request_all_files() to +// specify the files you would like to extract, then +// call run() to do the extraction. Also see step() for +// when you would like the extraction to happen as a +// background task. //////////////////////////////////////////////////////////////////// bool Extractor:: -extract(const Filename &source_file, const Filename &rel_path) { - int ret = initiate(source_file, rel_path); - if (ret < 0) { - return false; - } - for (;;) { - ret = run(); +run() { + while (true) { + int ret = step(); if (ret == EU_success) { return true; } @@ -177,28 +227,3 @@ extract(const Filename &source_file, const Filename &rel_path) { } return false; } - -//////////////////////////////////////////////////////////////////// -// Function: Extractor::cleanup -// Access: Private -// Description: -//////////////////////////////////////////////////////////////////// -void Extractor:: -cleanup() { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Extractor cleanup called" << endl; - if (!_initiated) { - downloader_cat.error() - << "Extractor::cleanup() - Extraction has not been initiated" - << endl; - return; - } - - if (_read != (istream *)NULL) { - delete _read; - _read = (istream *)NULL; - } - _multifile.close(); - _write.close(); -} diff --git a/panda/src/downloader/extractor.h b/panda/src/downloader/extractor.h index 757a620c24..d6acb2069e 100644 --- a/panda/src/downloader/extractor.h +++ b/panda/src/downloader/extractor.h @@ -42,21 +42,32 @@ PUBLISHED: Extractor(); ~Extractor(); - int initiate(const Filename &multifile_name, const Filename &extract_to = ""); - int run(); + bool set_multifile(const Filename &multifile_name); + void set_extract_dir(const Filename &extract_dir); - bool extract(const Filename &multifile_name, const Filename &extract_to = ""); + void reset(); + bool request_subfile(const Filename &subfile_name); + int request_all_subfiles(); + + int step(); INLINE float get_progress(void) const; -private: - void cleanup(); + bool run(); - bool _initiated; +private: Filename _multifile_name; - Filename _extract_to; Multifile _multifile; + Filename _extract_dir; + + typedef pvector Requests; + Requests _requests; + + bool _initiated; + + // These are used only while processing. + int _request_index; int _subfile_index; size_t _subfile_pos; size_t _subfile_length;