From 9d3724a40b180780946812b61fe885a800ac3033 Mon Sep 17 00:00:00 2001 From: David Rose Date: Wed, 16 Jun 2010 21:58:39 +0000 Subject: [PATCH] optimizations to vfs.readFile(), particular for mounted multifiles --- panda/src/express/multifile.cxx | 96 +++++++++++++------ panda/src/express/multifile.h | 1 - panda/src/express/virtualFile.cxx | 41 +++----- panda/src/express/virtualFile.h | 7 +- panda/src/express/virtualFileMount.cxx | 62 ++++++++++++ panda/src/express/virtualFileMount.h | 4 + .../src/express/virtualFileMountMultifile.cxx | 30 ++++++ panda/src/express/virtualFileMountMultifile.h | 3 + panda/src/express/virtualFileSimple.cxx | 38 +++++--- panda/src/express/virtualFileSimple.h | 3 + panda/src/express/virtualFileSystem.cxx | 26 +++++ panda/src/express/virtualFileSystem.h | 4 + 12 files changed, 237 insertions(+), 78 deletions(-) diff --git a/panda/src/express/multifile.cxx b/panda/src/express/multifile.cxx index 0dde541b8c..66dd8bc1c5 100644 --- a/panda/src/express/multifile.cxx +++ b/panda/src/express/multifile.cxx @@ -21,6 +21,7 @@ #include "zStream.h" #include "encryptStream.h" #include "virtualFileSystem.h" +#include "virtualFile.h" #include #include @@ -1996,14 +1997,72 @@ read_subfile(int index, pvector &result) { nassertr(index >= 0 && index < (int)_subfiles.size(), false); result.clear(); - istream *in = open_read_subfile(index); - if (in == (istream *)NULL) { + // Now look up the particular Subfile we are reading. + nassertr(is_read_valid(), NULL); + nassertr(index >= 0 && index < (int)_subfiles.size(), NULL); + Subfile *subfile = _subfiles[index]; + + if (subfile->_source != (istream *)NULL || + !subfile->_source_filename.empty()) { + // The subfile has not yet been copied into the physical + // Multifile. Force a flush operation to incorporate it. + flush(); + + // That shouldn't change the subfile index or delete the subfile + // pointer. + nassertr(subfile == _subfiles[index], NULL); + } + + result.reserve(subfile->_uncompressed_length); + + bool success = true; + if (subfile->_flags & (SF_encrypted | SF_compressed)) { + // If the subfile is encrypted or compressed, we can't read it + // directly. Fall back to the generic implementation. + istream *in = open_read_subfile(index); + if (in == (istream *)NULL) { + return false; + } + + success = VirtualFile::simple_read_file(in, result); + close_read_subfile(in); + + } else { + // But if the subfile is just a plain file, we can just read the + // data directly from the Multifile, without paying the cost of an + // ISubStream. + static const size_t buffer_size = 4096; + char buffer[buffer_size]; + + streamsize pos = _offset + subfile->_data_start; + size_t max_bytes = subfile->_data_length; + streamsize count = 0; + bool eof = true; + + streamsize num_bytes = (streamsize)min(buffer_size, max_bytes); + _read->seek_read(pos, buffer, num_bytes, count, eof); + while (count != 0) { + thread_consider_yield(); + nassertr(count <= max_bytes, false); + result.insert(result.end(), buffer, buffer + (size_t)count); + max_bytes -= (size_t)count; + pos += count; + + num_bytes = (streamsize)min(buffer_size, max_bytes); + _read->seek_read(pos, buffer, num_bytes, count, eof); + } + + success = !eof; + } + + if (!success) { + ostringstream message; + message << "I/O error reading from " << get_multifile_name() << " at " + << get_subfile_name(index); + nassert_raise(message.str()); return false; } - bool success = read_to_pvector(result, *in); - close_read_subfile(in); - nassertr(success, false); return true; } @@ -2176,31 +2235,6 @@ standardize_subfile_name(const string &subfile_name) const { } } -//////////////////////////////////////////////////////////////////// -// Function: Multifile::read_to_pvector -// Access: Private, Static -// Description: Helper function to read the entire contents of the -// indicated stream from the current position, and -// append it onto the indicated pvector. Returns true -// on success, false on failure. -//////////////////////////////////////////////////////////////////// -bool Multifile:: -read_to_pvector(pvector &result, istream &stream) { - static const size_t buffer_size = 4096; - char buffer[buffer_size]; - - stream.read(buffer, buffer_size); - size_t count = stream.gcount(); - while (count != 0) { - result.insert(result.end(), buffer, buffer + count); - stream.read(buffer, buffer_size); - count = stream.gcount(); - } - - bool failed = stream.fail() && !stream.eof(); - return !failed; -} - //////////////////////////////////////////////////////////////////// // Function: Multifile::clear_subfiles // Access: Private @@ -2494,7 +2528,7 @@ check_signatures() { // Read the remaining buffer of certificate data. pvector buffer; - bool success = read_to_pvector(buffer, *stream); + bool success = VirtualFile::simple_read_file(stream, buffer); nassertv(success); close_read_subfile(stream); diff --git a/panda/src/express/multifile.h b/panda/src/express/multifile.h index 1623fd479d..65a73e94b8 100644 --- a/panda/src/express/multifile.h +++ b/panda/src/express/multifile.h @@ -205,7 +205,6 @@ private: void add_new_subfile(Subfile *subfile, int compression_level); istream *open_read_subfile(Subfile *subfile); string standardize_subfile_name(const string &subfile_name) const; - static bool read_to_pvector(pvector &result, istream &stream); void clear_subfiles(); bool read_index(); diff --git a/panda/src/express/virtualFile.cxx b/panda/src/express/virtualFile.cxx index 1ca48f3bea..05b34d43a0 100644 --- a/panda/src/express/virtualFile.cxx +++ b/panda/src/express/virtualFile.cxx @@ -278,35 +278,18 @@ read_file(string &result, bool auto_unwrap) const { //////////////////////////////////////////////////////////////////// // Function: VirtualFile::read_file -// Access: Public +// Access: Public, Virtual // Description: Fills up the indicated pvector with the contents of // the file, if it is a regular file. Returns true on // success, false otherwise. //////////////////////////////////////////////////////////////////// bool VirtualFile:: read_file(pvector &result, bool auto_unwrap) const { - result.clear(); - - istream *in = open_read_file(auto_unwrap); - if (in == (istream *)NULL) { - express_cat.info() - << "Unable to read " << get_filename() << "\n"; - return false; - } - - bool okflag = read_file(in, result); - - close_read_file(in); - - if (!okflag) { - express_cat.info() - << "Error while reading " << get_filename() << "\n"; - } - return okflag; + return false; } //////////////////////////////////////////////////////////////////// -// Function: VirtualFile::read_file +// Function: VirtualFile::simple_read_file // Access: Public, Static // Description: Fills up the indicated pvector with the contents of // the just-opened file. Returns true on success, false @@ -314,10 +297,8 @@ read_file(pvector &result, bool auto_unwrap) const { // data read from the file will be appended onto it. //////////////////////////////////////////////////////////////////// bool VirtualFile:: -read_file(istream *in, pvector &result) { - pvector result_vec; - - static const size_t buffer_size = 1024; +simple_read_file(istream *in, pvector &result) { + static const size_t buffer_size = 4096; char buffer[buffer_size]; in->read(buffer, buffer_size); @@ -333,14 +314,14 @@ read_file(istream *in, pvector &result) { } //////////////////////////////////////////////////////////////////// -// Function: VirtualFile::read_file -// Access: Public, Static -// Description: As in read_file() with two parameters, above, but -// only reads up to max_bytes bytes from the file. +// Function: VirtualFile::simple_read_file +// Access: Public +// Description: As in simple_read_file() with two parameters, above, +// but only reads up to max_bytes bytes from the file. //////////////////////////////////////////////////////////////////// bool VirtualFile:: -read_file(istream *in, pvector &result, size_t max_bytes) { - static const size_t buffer_size = 1024; +simple_read_file(istream *in, pvector &result, size_t max_bytes) { + static const size_t buffer_size = 4096; char buffer[buffer_size]; in->read(buffer, min(buffer_size, max_bytes)); diff --git a/panda/src/express/virtualFile.h b/panda/src/express/virtualFile.h index c5568545b7..1d21175a69 100644 --- a/panda/src/express/virtualFile.h +++ b/panda/src/express/virtualFile.h @@ -63,9 +63,10 @@ PUBLISHED: public: INLINE void set_original_filename(const Filename &filename); bool read_file(string &result, bool auto_unwrap) const; - bool read_file(pvector &result, bool auto_unwrap) const; - static bool read_file(istream *stream, pvector &result); - static bool read_file(istream *stream, pvector &result, size_t max_bytes); + virtual bool read_file(pvector &result, bool auto_unwrap) const; + + static bool simple_read_file(istream *stream, pvector &result); + static bool simple_read_file(istream *stream, pvector &result, size_t max_bytes); protected: diff --git a/panda/src/express/virtualFileMount.cxx b/panda/src/express/virtualFileMount.cxx index e0cf7daa5a..7c3721a34a 100644 --- a/panda/src/express/virtualFileMount.cxx +++ b/panda/src/express/virtualFileMount.cxx @@ -14,6 +14,7 @@ #include "virtualFileMount.h" #include "virtualFileSimple.h" +#include "zStream.h" TypeHandle VirtualFileMount::_type_handle; @@ -53,6 +54,67 @@ make_virtual_file(const Filename &local_filename, return file.p(); } +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMount::read_file +// Access: Public, Virtual +// Description: Fills up the indicated pvector with the contents of +// the file, if it is a regular file. Returns true on +// success, false otherwise. +//////////////////////////////////////////////////////////////////// +bool VirtualFileMount:: +read_file(const Filename &file, bool do_uncompress, + pvector &result) const { + result.clear(); + + istream *in = open_read_file(file, do_uncompress); + if (in == (istream *)NULL) { + express_cat.info() + << "Unable to read " << file << "\n"; + return false; + } + + off_t file_size = get_file_size(file, in); + if (file_size != 0) { + result.reserve((size_t)file_size); + } + + bool okflag = VirtualFile::simple_read_file(in, result); + + close_read_file(in); + + if (!okflag) { + express_cat.info() + << "Error while reading " << file << "\n"; + } + return okflag; +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMount::open_read_file +// Access: Published, Virtual +// Description: Opens the file for reading. Returns a newly +// allocated istream on success (which you should +// eventually delete when you are done reading). +// Returns NULL on failure. +// +// If do_uncompress is true, the file is also +// decompressed on-the-fly using zlib. +//////////////////////////////////////////////////////////////////// +istream *VirtualFileMount:: +open_read_file(const Filename &file, bool do_uncompress) const { + istream *result = open_read_file(file); + +#ifdef HAVE_ZLIB + if (result != (istream *)NULL && do_uncompress) { + // We have to slip in a layer to decompress the file on the fly. + IDecompressStream *wrapper = new IDecompressStream(result, true); + result = wrapper; + } +#endif // HAVE_ZLIB + + return result; +} + //////////////////////////////////////////////////////////////////// // Function: VirtualFileMount::close_read_file // Access: Public diff --git a/panda/src/express/virtualFileMount.h b/panda/src/express/virtualFileMount.h index bcb84cfbbc..6beb8bd236 100644 --- a/panda/src/express/virtualFileMount.h +++ b/panda/src/express/virtualFileMount.h @@ -49,7 +49,11 @@ public: virtual bool is_directory(const Filename &file) const=0; virtual bool is_regular_file(const Filename &file) const=0; + virtual bool read_file(const Filename &file, bool do_uncompress, + pvector &result) const; + virtual istream *open_read_file(const Filename &file) const=0; + istream *open_read_file(const Filename &file, bool do_uncompress) const; void close_read_file(istream *stream) const; virtual off_t get_file_size(const Filename &file, istream *stream) const=0; virtual off_t get_file_size(const Filename &file) const=0; diff --git a/panda/src/express/virtualFileMountMultifile.cxx b/panda/src/express/virtualFileMountMultifile.cxx index 71f4498781..7115292061 100644 --- a/panda/src/express/virtualFileMountMultifile.cxx +++ b/panda/src/express/virtualFileMountMultifile.cxx @@ -63,6 +63,36 @@ is_regular_file(const Filename &file) const { return (_multifile->find_subfile(file) >= 0); } +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMountMultifile::read_file +// Access: Public, Virtual +// Description: Fills up the indicated pvector with the contents of +// the file, if it is a regular file. Returns true on +// success, false otherwise. +//////////////////////////////////////////////////////////////////// +bool VirtualFileMountMultifile:: +read_file(const Filename &file, bool do_uncompress, + pvector &result) const { + if (do_uncompress) { + // If the file is to be decompressed, we'd better just use the + // higher-level implementation, which includes support for + // on-the-fly decompression. + return VirtualFileMount::read_file(file, do_uncompress, result); + } + + // But if we're just reading a straight file, let the Multifile do + // the reading, which avoids a few levels of buffer copies. + + int subfile_index = _multifile->find_subfile(file); + if (subfile_index < 0) { + express_cat.info() + << "Unable to read " << file << "\n"; + return false; + } + + return _multifile->read_subfile(subfile_index, result); +} + //////////////////////////////////////////////////////////////////// // Function: VirtualFileMountMultifile::open_read_file // Access: Public, Virtual diff --git a/panda/src/express/virtualFileMountMultifile.h b/panda/src/express/virtualFileMountMultifile.h index 2e7a0933cb..6f2e28f525 100644 --- a/panda/src/express/virtualFileMountMultifile.h +++ b/panda/src/express/virtualFileMountMultifile.h @@ -38,6 +38,9 @@ public: virtual bool is_directory(const Filename &file) const; virtual bool is_regular_file(const Filename &file) const; + virtual bool read_file(const Filename &file, bool do_uncompress, + pvector &result) const; + virtual istream *open_read_file(const Filename &file) const; virtual off_t get_file_size(const Filename &file, istream *stream) const; virtual off_t get_file_size(const Filename &file) const; diff --git a/panda/src/express/virtualFileSimple.cxx b/panda/src/express/virtualFileSimple.cxx index d01477baf8..8b9b5d9a2a 100644 --- a/panda/src/express/virtualFileSimple.cxx +++ b/panda/src/express/virtualFileSimple.cxx @@ -15,7 +15,6 @@ #include "virtualFileSimple.h" #include "virtualFileMount.h" #include "virtualFileList.h" -#include "zStream.h" TypeHandle VirtualFileSimple::_type_handle; @@ -106,24 +105,15 @@ istream *VirtualFileSimple:: open_read_file(bool auto_unwrap) const { // Will we be automatically unwrapping a .pz file? - bool do_unwrap = (_implicit_pz_file || (auto_unwrap && _local_filename.get_extension() == "pz")); + bool do_uncompress = (_implicit_pz_file || (auto_unwrap && _local_filename.get_extension() == "pz")); Filename local_filename(_local_filename); - if (do_unwrap) { + if (do_uncompress) { // .pz files are always binary, of course. local_filename.set_binary(); } - istream *result = _mount->open_read_file(local_filename); -#ifdef HAVE_ZLIB - if (result != (istream *)NULL && do_unwrap) { - // We have to slip in a layer to decompress the file on the fly. - IDecompressStream *wrapper = new IDecompressStream(result, true); - result = wrapper; - } -#endif // HAVE_ZLIB - - return result; + return _mount->open_read_file(local_filename, do_uncompress); } //////////////////////////////////////////////////////////////////// @@ -170,6 +160,28 @@ get_timestamp() const { return _mount->get_timestamp(_local_filename); } +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileSimple::read_file +// Access: Public, Virtual +// Description: Fills up the indicated pvector with the contents of +// the file, if it is a regular file. Returns true on +// success, false otherwise. +//////////////////////////////////////////////////////////////////// +bool VirtualFileSimple:: +read_file(pvector &result, bool auto_unwrap) const { + + // Will we be automatically unwrapping a .pz file? + bool do_uncompress = (_implicit_pz_file || (auto_unwrap && _local_filename.get_extension() == "pz")); + + Filename local_filename(_local_filename); + if (do_uncompress) { + // .pz files are always binary, of course. + local_filename.set_binary(); + } + + return _mount->read_file(local_filename, do_uncompress, result); +} + //////////////////////////////////////////////////////////////////// // Function: VirtualFileSimple::scan_local_directory // Access: Protected, Virtual diff --git a/panda/src/express/virtualFileSimple.h b/panda/src/express/virtualFileSimple.h index b9134d3089..59a6ad9564 100644 --- a/panda/src/express/virtualFileSimple.h +++ b/panda/src/express/virtualFileSimple.h @@ -47,6 +47,9 @@ PUBLISHED: virtual off_t get_file_size() const; virtual time_t get_timestamp() const; +public: + virtual bool read_file(pvector &result, bool auto_unwrap) const; + protected: virtual bool scan_local_directory(VirtualFileList *file_list, const ov_set &mount_points) const; diff --git a/panda/src/express/virtualFileSystem.cxx b/panda/src/express/virtualFileSystem.cxx index 696a9d0d42..afdb368ad0 100644 --- a/panda/src/express/virtualFileSystem.cxx +++ b/panda/src/express/virtualFileSystem.cxx @@ -765,6 +765,32 @@ get_global_ptr() { return _global_ptr; } +#ifdef HAVE_PYTHON +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileSystem::__py__read_file +// Access: Published +// Description: Convenience function; returns the entire contents of +// the indicated file as a string. +// +// This variant on read_file() is implemented directly +// for Python, as a small optimization, to avoid the +// double-construction of a string object that would be +// otherwise required for the return value. +//////////////////////////////////////////////////////////////////// +PyObject *VirtualFileSystem:: +__py__read_file(const Filename &filename, bool auto_unwrap) const { + pvector pv; + bool okflag = read_file(filename, pv, auto_unwrap); + nassertr(okflag, NULL); + + if (pv.empty()) { + return PyString_FromStringAndSize("", 0); + } else { + return PyString_FromStringAndSize((const char *)&pv[0], pv.size()); + } +} +#endif // HAVE_PYTHON + //////////////////////////////////////////////////////////////////// // Function: VirtualFileSystem::open_read_file // Access: Published diff --git a/panda/src/express/virtualFileSystem.h b/panda/src/express/virtualFileSystem.h index 1070b1a7b0..1168a6de89 100644 --- a/panda/src/express/virtualFileSystem.h +++ b/panda/src/express/virtualFileSystem.h @@ -91,7 +91,11 @@ PUBLISHED: static VirtualFileSystem *get_global_ptr(); +#ifdef HAVE_PYTHON + BLOCKING PyObject *__py__read_file(const Filename &filename, bool auto_unwrap) const; +#endif // HAVE_PYTHON BLOCKING INLINE string read_file(const Filename &filename, bool auto_unwrap) const; + BLOCKING istream *open_read_file(const Filename &filename, bool auto_unwrap) const; BLOCKING static void close_read_file(istream *stream);