From ce0ad5dde8b9011f295fd60b866537c094e6a01c Mon Sep 17 00:00:00 2001 From: David Rose Date: Sat, 25 Feb 2012 02:28:41 +0000 Subject: [PATCH] work-in-progress: cache-check-timestamps --- panda/src/egg/eggData.I | 29 +++++++++++++++- panda/src/egg/eggData.cxx | 13 ++++++-- panda/src/egg/eggData.h | 4 +++ panda/src/egg2pg/eggLoader.cxx | 2 +- panda/src/egg2pg/load_egg_file.cxx | 13 ++++++-- panda/src/express/datagramGenerator.cxx | 12 +++++++ panda/src/express/datagramGenerator.h | 1 + panda/src/pgraph/bamFile.cxx | 14 +++++--- panda/src/pgraph/loader.cxx | 6 ++++ panda/src/pgraph/loaderFileTypeBam.cxx | 12 ++++++- panda/src/pgraph/modelRoot.I | 44 +++++++++++++++++++++++++ panda/src/pgraph/modelRoot.h | 5 +++ panda/src/putil/bamCache.cxx | 4 +-- panda/src/putil/bamCacheRecord.I | 13 ++++++++ panda/src/putil/bamCacheRecord.cxx | 15 +++++++++ panda/src/putil/bamCacheRecord.h | 2 ++ panda/src/putil/config_util.cxx | 14 ++++++++ panda/src/putil/config_util.h | 1 + panda/src/putil/datagramInputFile.I | 1 + panda/src/putil/datagramInputFile.cxx | 15 +++++++++ panda/src/putil/datagramInputFile.h | 2 ++ 21 files changed, 207 insertions(+), 15 deletions(-) diff --git a/panda/src/egg/eggData.I b/panda/src/egg/eggData.I index a10f627f4a..2cf6853239 100644 --- a/panda/src/egg/eggData.I +++ b/panda/src/egg/eggData.I @@ -23,6 +23,7 @@ EggData() { _auto_resolve_externals = false; _had_absolute_pathnames = false; _coordsys = CS_default; + _egg_timestamp = 0; } @@ -37,7 +38,8 @@ EggData(const EggData ©) : _auto_resolve_externals(copy._auto_resolve_externals), _had_absolute_pathnames(copy._had_absolute_pathnames), _coordsys(copy._coordsys), - _egg_filename(copy._egg_filename) + _egg_filename(copy._egg_filename), + _egg_timestamp(copy._egg_timestamp) { } @@ -53,6 +55,7 @@ operator = (const EggData ©) { _had_absolute_pathnames = copy._had_absolute_pathnames; _coordsys = copy._coordsys; _egg_filename = copy._egg_filename; + _egg_timestamp = copy._egg_timestamp; return *this; } @@ -133,6 +136,30 @@ get_egg_filename() const { return _egg_filename; } +//////////////////////////////////////////////////////////////////// +// Function: EggData::set_egg_timestamp +// Access: Public +// Description: Sets the timestamp of the egg file on disk, at the +// time it was opened for reading. This is also +// implicitly set by read(). +//////////////////////////////////////////////////////////////////// +INLINE void EggData:: +set_egg_timestamp(time_t egg_timestamp) { + _egg_timestamp = egg_timestamp; +} + +//////////////////////////////////////////////////////////////////// +// Function: EggData::get_egg_timestamp +// Access: Public +// Description: Returns the timestamp of the egg file on disk, at the +// time it was opened for reading, or 0 if this +// information is not available. +//////////////////////////////////////////////////////////////////// +INLINE time_t EggData:: +get_egg_timestamp() const { + return _egg_timestamp; +} + //////////////////////////////////////////////////////////////////// // Function: EggData::recompute_vertex_normals // Access: Public diff --git a/panda/src/egg/eggData.cxx b/panda/src/egg/eggData.cxx index 69fcfedcb6..e472bb2c39 100644 --- a/panda/src/egg/eggData.cxx +++ b/panda/src/egg/eggData.cxx @@ -77,8 +77,15 @@ read(Filename filename, string display_name) { } VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); - - istream *file = vfs->open_read_file(filename, true); + + PT(VirtualFile) vfile = vfs->get_file(filename); + if (vfile == NULL) { + egg_cat.error() << "Could not find " << display_name << "\n"; + return false; + } + set_egg_timestamp(vfile->get_timestamp()); + + istream *file = vfile->open_read_file(true); if (file == (istream *)NULL) { egg_cat.error() << "Unable to open " << display_name << "\n"; return false; @@ -88,7 +95,7 @@ read(Filename filename, string display_name) { << "Reading " << display_name << "\n"; bool read_ok = read(*file); - vfs->close_read_file(file); + vfile->close_read_file(file); return read_ok; } diff --git a/panda/src/egg/eggData.h b/panda/src/egg/eggData.h index 05d783bcd7..ab299a9abb 100644 --- a/panda/src/egg/eggData.h +++ b/panda/src/egg/eggData.h @@ -69,6 +69,9 @@ PUBLISHED: INLINE void set_egg_filename(const Filename &egg_filename); INLINE const Filename &get_egg_filename() const; + INLINE void set_egg_timestamp(time_t egg_timestamp); + INLINE time_t get_egg_timestamp() const; + INLINE void recompute_vertex_normals(double threshold); INLINE void recompute_polygon_normals(); INLINE void strip_normals(); @@ -84,6 +87,7 @@ private: bool _had_absolute_pathnames; CoordinateSystem _coordsys; Filename _egg_filename; + time_t _egg_timestamp; public: static TypeHandle get_class_type() { diff --git a/panda/src/egg2pg/eggLoader.cxx b/panda/src/egg2pg/eggLoader.cxx index cbe5a9d79e..661f18385a 100644 --- a/panda/src/egg2pg/eggLoader.cxx +++ b/panda/src/egg2pg/eggLoader.cxx @@ -209,7 +209,7 @@ build_graph() { // ((EggGroupNode *)_data)->write(cerr, 0); // Now build up the scene graph. - _root = new ModelRoot(_data->get_egg_filename().get_basename()); + _root = new ModelRoot(_data->get_egg_filename(), _data->get_egg_timestamp()); make_node(_data, _root); reparent_decals(); diff --git a/panda/src/egg2pg/load_egg_file.cxx b/panda/src/egg2pg/load_egg_file.cxx index d74c28f674..1f0873f7c6 100644 --- a/panda/src/egg2pg/load_egg_file.cxx +++ b/panda/src/egg2pg/load_egg_file.cxx @@ -90,9 +90,18 @@ load_egg_file(const Filename &filename, CoordinateSystem cs, loader._data->set_coordinate_system(cs); loader._record = record; + PT(VirtualFile) vfile = vfs->get_file(egg_filename); + if (vfile == NULL) { + return NULL; + } + + loader._data->set_egg_timestamp(vfile->get_timestamp()); + bool okflag; - istream *istr = vfs->open_read_file(egg_filename, true); + istream *istr = vfile->open_read_file(true); if (istr == (istream *)NULL) { + egg2pg_cat.error() + << "Couldn't read " << egg_filename << "\n"; return NULL; } @@ -100,7 +109,7 @@ load_egg_file(const Filename &filename, CoordinateSystem cs, << "Reading " << egg_filename << "\n"; okflag = loader._data->read(*istr); - vfs->close_read_file(istr); + vfile->close_read_file(istr); if (!okflag) { egg2pg_cat.error() diff --git a/panda/src/express/datagramGenerator.cxx b/panda/src/express/datagramGenerator.cxx index 1248fb8842..8eee70b453 100644 --- a/panda/src/express/datagramGenerator.cxx +++ b/panda/src/express/datagramGenerator.cxx @@ -63,6 +63,18 @@ get_filename() { return empty_filename; } +//////////////////////////////////////////////////////////////////// +// Function: DatagramGenerator::get_timestamp +// Access: Published, Virtual +// Description: Returns the on-disk timestamp of the file that was +// read, at the time it was opened, if that is +// available, or 0 if it is not. +//////////////////////////////////////////////////////////////////// +time_t DatagramGenerator:: +get_timestamp() const { + return 0; +} + //////////////////////////////////////////////////////////////////// // Function: DatagramGenerator::get_file // Access: Published, Virtual diff --git a/panda/src/express/datagramGenerator.h b/panda/src/express/datagramGenerator.h index 9c67a7bf46..3d8361137b 100644 --- a/panda/src/express/datagramGenerator.h +++ b/panda/src/express/datagramGenerator.h @@ -41,6 +41,7 @@ PUBLISHED: virtual bool is_error() = 0; virtual const Filename &get_filename(); + virtual time_t get_timestamp() const; virtual const FileReference *get_file(); virtual VirtualFile *get_vfile(); virtual streampos get_file_pos(); diff --git a/panda/src/pgraph/bamFile.cxx b/panda/src/pgraph/bamFile.cxx index 648afa6ea7..5a438900b8 100644 --- a/panda/src/pgraph/bamFile.cxx +++ b/panda/src/pgraph/bamFile.cxx @@ -60,9 +60,6 @@ open_read(const Filename &bam_filename, bool report_errors) { return false; } - loader_cat.info() - << "Reading " << bam_filename << "\n"; - return continue_open_read(bam_filename, report_errors); } @@ -220,8 +217,6 @@ bool BamFile:: open_write(const Filename &bam_filename, bool report_errors) { close(); - loader_cat.info() << "Writing " << bam_filename << "\n"; - VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); vfs->delete_file(bam_filename); if (!_dout.open(bam_filename)) { @@ -422,6 +417,11 @@ bool BamFile:: continue_open_read(const string &bam_filename, bool report_errors) { _bam_filename = bam_filename; + if (!_bam_filename.empty()) { + loader_cat.info() + << "Reading " << _bam_filename << "\n"; + } + string head; if (!_din.read_header(head, _bam_header.size())) { if (report_errors) { @@ -457,6 +457,10 @@ bool BamFile:: continue_open_write(const string &bam_filename, bool report_errors) { _bam_filename = bam_filename; + if (!_bam_filename.empty()) { + loader_cat.info() << "Writing " << _bam_filename << "\n"; + } + if (!_dout.write_header(_bam_header)) { if (report_errors) { loader_cat.error() << "Unable to write to " << _bam_filename << "\n"; diff --git a/panda/src/pgraph/loader.cxx b/panda/src/pgraph/loader.cxx index 988a68590b..33470a4ad9 100644 --- a/panda/src/pgraph/loader.cxx +++ b/panda/src/pgraph/loader.cxx @@ -295,6 +295,12 @@ try_load_file(const Filename &pathname, const LoaderOptions &options, << "Model " << pathname << " found in disk cache.\n"; } PT(PandaNode) result = DCAST(PandaNode, record->get_data()); + if (result->is_of_type(ModelRoot::get_class_type())) { + ModelRoot *model_root = DCAST(ModelRoot, result.p()); + model_root->set_fullpath(pathname); + model_root->set_timestamp(record->get_source_timestamp()); + } + if (premunge_data) { SceneGraphReducer sgr; sgr.premunge(result, RenderState::make_empty()); diff --git a/panda/src/pgraph/loaderFileTypeBam.cxx b/panda/src/pgraph/loaderFileTypeBam.cxx index e801f934af..f11e66bbc6 100644 --- a/panda/src/pgraph/loaderFileTypeBam.cxx +++ b/panda/src/pgraph/loaderFileTypeBam.cxx @@ -76,11 +76,21 @@ load_file(const Filename &path, const LoaderOptions &options, } bool report_errors = (options.get_flags() & LoaderOptions::LF_report_errors) != 0; + BamFile bam_file; if (!bam_file.open_read(path, report_errors)) { return NULL; } bam_file.get_reader()->set_loader_options(options); - return bam_file.read_node(report_errors); + time_t timestamp = bam_file.get_reader()->get_source()->get_timestamp(); + + PT(PandaNode) node = bam_file.read_node(report_errors); + if (node->is_of_type(ModelRoot::get_class_type())) { + ModelRoot *model_root = DCAST(ModelRoot, node.p()); + model_root->set_fullpath(path); + model_root->set_timestamp(timestamp); + } + + return node; } diff --git a/panda/src/pgraph/modelRoot.I b/panda/src/pgraph/modelRoot.I index 617239d24b..dd48148130 100644 --- a/panda/src/pgraph/modelRoot.I +++ b/panda/src/pgraph/modelRoot.I @@ -22,6 +22,21 @@ INLINE ModelRoot:: ModelRoot(const string &name) : ModelNode(name), _fullpath(name), + _timestamp(0), + _reference(new ModelRoot::ModelReference) +{ +} + +//////////////////////////////////////////////////////////////////// +// Function: ModelRoot::Constructor +// Access: Published +// Description: +//////////////////////////////////////////////////////////////////// +INLINE ModelRoot:: +ModelRoot(const Filename &fullpath, time_t timestamp) : + ModelNode(fullpath.get_basename()), + _fullpath(fullpath), + _timestamp(timestamp), _reference(new ModelRoot::ModelReference) { } @@ -73,6 +88,34 @@ set_fullpath(const Filename &fullpath) { _fullpath = fullpath; } +//////////////////////////////////////////////////////////////////// +// Function: ModelRoot::get_timestamp +// Access: Published +// Description: Returns the timestamp of the file on disk that was +// read for this model, at the time it was read, if it +// is known. Returns 0 if the timestamp is not known or +// could not be provided. This can be used as a quick +// (but fallible) check to verify whether the file might +// have changed since the model was read. +//////////////////////////////////////////////////////////////////// +INLINE time_t ModelRoot:: +get_timestamp() const { + return _timestamp; +} + +//////////////////////////////////////////////////////////////////// +// Function: ModelRoot::set_timestamp +// Access: Published +// Description: Sets the timestamp of the file on disk that was read +// for this model. This is normally set automatically +// when a model is loaded, and should not be set +// directly by the user. +//////////////////////////////////////////////////////////////////// +INLINE void ModelRoot:: +set_timestamp(time_t timestamp) { + _timestamp = timestamp; +} + //////////////////////////////////////////////////////////////////// // Function: ModelRoot::get_reference // Access: Published @@ -110,6 +153,7 @@ INLINE ModelRoot:: ModelRoot(const ModelRoot ©) : ModelNode(copy), _fullpath(copy._fullpath), + _timestamp(copy._timestamp), _reference(copy._reference) { } diff --git a/panda/src/pgraph/modelRoot.h b/panda/src/pgraph/modelRoot.h index 50cc8c0d5a..e0efe88e14 100644 --- a/panda/src/pgraph/modelRoot.h +++ b/panda/src/pgraph/modelRoot.h @@ -31,12 +31,16 @@ class EXPCL_PANDA_PGRAPH ModelRoot : public ModelNode { PUBLISHED: INLINE ModelRoot(const string &name); + INLINE ModelRoot(const Filename &fulllpath, time_t timestamp); INLINE int get_model_ref_count() const; INLINE const Filename &get_fullpath() const; INLINE void set_fullpath(const Filename &fullpath); + INLINE time_t get_timestamp() const; + INLINE void set_timestamp(time_t timestamp); + // This class is used to unify references to the same model. class ModelReference : public ReferenceCount { PUBLISHED: @@ -54,6 +58,7 @@ public: private: Filename _fullpath; + time_t _timestamp; PT(ModelReference) _reference; public: diff --git a/panda/src/putil/bamCache.cxx b/panda/src/putil/bamCache.cxx index f8b9d1954d..d9b5c32eed 100644 --- a/panda/src/putil/bamCache.cxx +++ b/panda/src/putil/bamCache.cxx @@ -917,10 +917,10 @@ do_read_record(const Filename &cache_pathname, bool read_data) { } // Also get the total file size. + PT(VirtualFile) vfile = din.get_vfile(); istream &in = din.get_stream(); in.clear(); - in.seekg(0, ios::end); - record->_record_size = in.tellg(); + record->_record_size = vfile->get_file_size(&in); // And the last access time is now, duh. record->_record_access_time = time(NULL); diff --git a/panda/src/putil/bamCacheRecord.I b/panda/src/putil/bamCacheRecord.I index 76c5429307..9679a70a45 100644 --- a/panda/src/putil/bamCacheRecord.I +++ b/panda/src/putil/bamCacheRecord.I @@ -69,6 +69,19 @@ get_cache_filename() const { return _cache_filename; } +//////////////////////////////////////////////////////////////////// +// Function: BamCacheRecord::get_source_timestamp +// Access: Published +// Description: Returns the file timestamp of the original source +// file that generated this cache record, if available. +// In some cases the original file timestamp is not +// available, and this will return 0. +//////////////////////////////////////////////////////////////////// +INLINE time_t BamCacheRecord:: +get_source_timestamp() const { + return _source_timestamp; +} + //////////////////////////////////////////////////////////////////// // Function: BamCacheRecord::get_recorded_time // Access: Published diff --git a/panda/src/putil/bamCacheRecord.cxx b/panda/src/putil/bamCacheRecord.cxx index e127d21f91..90bb31e747 100644 --- a/panda/src/putil/bamCacheRecord.cxx +++ b/panda/src/putil/bamCacheRecord.cxx @@ -28,6 +28,7 @@ BamCacheRecord:: BamCacheRecord() : _recorded_time(0), _record_size(0), + _source_timestamp(0), _ptr(NULL), _ref_ptr(NULL), _record_access_time(0) @@ -46,6 +47,7 @@ BamCacheRecord(const Filename &source_pathname, _cache_filename(cache_filename), _recorded_time(0), _record_size(0), + _source_timestamp(0), _ptr(NULL), _ref_ptr(NULL), _record_access_time(0) @@ -64,6 +66,7 @@ BamCacheRecord(const BamCacheRecord ©) : _cache_filename(copy._cache_filename), _recorded_time(copy._recorded_time), _record_size(copy._record_size), + _source_timestamp(copy._source_timestamp), _ptr(NULL), _ref_ptr(NULL), _record_access_time(copy._record_access_time) @@ -152,6 +155,10 @@ add_dependent_file(const Filename &pathname) { } else { dfile._timestamp = file->get_timestamp(); dfile._size = file->get_file_size(); + + if (dfile._pathname == _source_pathname) { + _source_timestamp = dfile._timestamp; + } } } @@ -174,6 +181,8 @@ void BamCacheRecord:: write(ostream &out, int indent_level) const { indent(out, indent_level) << "BamCacheRecord " << get_source_pathname() << "\n"; + indent(out, indent_level) + << "source " << format_timestamp(_source_timestamp) << "\n"; indent(out, indent_level) << "recorded " << format_timestamp(_recorded_time) << "\n"; @@ -299,5 +308,11 @@ fillin(DatagramIterator &scan, BamReader *manager) { file._pathname = scan.get_string(); file._timestamp = scan.get_uint32(); file._size = scan.get_uint64(); + + // If we come across the original source file (we normally expect + // to), record that as its timestamp. + if (file._pathname == _source_pathname) { + _source_timestamp = file._timestamp; + } } } diff --git a/panda/src/putil/bamCacheRecord.h b/panda/src/putil/bamCacheRecord.h index 49e6c09f2a..a10601c965 100644 --- a/panda/src/putil/bamCacheRecord.h +++ b/panda/src/putil/bamCacheRecord.h @@ -52,6 +52,7 @@ PUBLISHED: INLINE const Filename &get_source_pathname() const; INLINE const Filename &get_cache_filename() const; + INLINE time_t get_source_timestamp() const; INLINE time_t get_recorded_time() const; INLINE int get_num_dependent_files() const; @@ -84,6 +85,7 @@ private: Filename _cache_filename; time_t _recorded_time; off_t _record_size; // this is accurate only in the index file. + time_t _source_timestamp; // Not record to the cache file. class DependentFile { public: diff --git a/panda/src/putil/config_util.cxx b/panda/src/putil/config_util.cxx index 1449e67805..6795d69ea8 100644 --- a/panda/src/putil/config_util.cxx +++ b/panda/src/putil/config_util.cxx @@ -146,6 +146,20 @@ ConfigVariableBool preload_simple_textures "in a sub-thread. It's not generally necessary if you are " "loading bam files that were generated via egg2bam.")); +ConfigVariableBool cache_check_timestamps +("cache-check-timestamps", true, + PRC_DESC("Set this true to check the timestamps on disk (when possible) " + "before reloading a file from the in-memory cache, e.g. via ModelPool, " + "TexturePool, etc. When this is false, a model or texture " + "that was previously loaded and is still found in the ModelPool is " + "immediately returned without consulting the disk, even if the " + "file on disk has recently changed. When this is true, the file " + "on disk is always checked to ensure its timestamp has not " + "recently changed; and if it has, the in-memory cache is automatically " + "invalidated and the file is reloaded from disk. This is not related " + "to on-disk caching via model-cache-dir, which always checks the " + "timestamps.")); + //////////////////////////////////////////////////////////////////// // Function: init_libputil // Description: Initializes the library. This must be called at diff --git a/panda/src/putil/config_util.h b/panda/src/putil/config_util.h index 795e340372..a2fe2ff06c 100644 --- a/panda/src/putil/config_util.h +++ b/panda/src/putil/config_util.h @@ -48,6 +48,7 @@ extern ConfigVariableDouble sleep_precision; extern EXPCL_PANDA_PUTIL ConfigVariableBool preload_textures; extern EXPCL_PANDA_PUTIL ConfigVariableBool preload_simple_textures; +extern EXPCL_PANDA_PUTIL ConfigVariableBool cache_check_timestamps; extern EXPCL_PANDA_PUTIL void init_libputil(); diff --git a/panda/src/putil/datagramInputFile.I b/panda/src/putil/datagramInputFile.I index 06ade2a1bf..21d241d42b 100644 --- a/panda/src/putil/datagramInputFile.I +++ b/panda/src/putil/datagramInputFile.I @@ -24,6 +24,7 @@ DatagramInputFile() { _read_first_datagram = false; _in = (istream *)NULL; _owns_in = false; + _timestamp = 0; } //////////////////////////////////////////////////////////////////// diff --git a/panda/src/putil/datagramInputFile.cxx b/panda/src/putil/datagramInputFile.cxx index 4d2864e26c..1e13bc50cb 100644 --- a/panda/src/putil/datagramInputFile.cxx +++ b/panda/src/putil/datagramInputFile.cxx @@ -45,6 +45,7 @@ open(const FileReference *file) { // No such file. return false; } + _timestamp = _vfile->get_timestamp(); _in = _vfile->open_read_file(true); _owns_in = (_in != (istream *)NULL); return _owns_in && !_in->fail(); @@ -66,6 +67,7 @@ open(istream &in, const Filename &filename) { _in = ∈ _owns_in = false; _filename = filename; + _timestamp = 0; if (!filename.empty()) { _file = new FileReference(filename); @@ -92,6 +94,7 @@ close() { _file.clear(); _filename = Filename(); + _timestamp = 0; _read_first_datagram = false; _error = false; @@ -309,6 +312,18 @@ get_filename() { return _filename; } +//////////////////////////////////////////////////////////////////// +// Function: DatagramInputFile::get_timestamp +// Access: Published, Virtual +// Description: Returns the on-disk timestamp of the file that was +// read, at the time it was opened, if that is +// available, or 0 if it is not. +//////////////////////////////////////////////////////////////////// +time_t DatagramInputFile:: +get_timestamp() const { + return _timestamp; +} + //////////////////////////////////////////////////////////////////// // Function: DatagramInputFile::get_file // Access: Published, Virtual diff --git a/panda/src/putil/datagramInputFile.h b/panda/src/putil/datagramInputFile.h index 88056d996b..9e6e900249 100644 --- a/panda/src/putil/datagramInputFile.h +++ b/panda/src/putil/datagramInputFile.h @@ -47,6 +47,7 @@ PUBLISHED: virtual bool is_error(); virtual const Filename &get_filename(); + virtual time_t get_timestamp() const; virtual const FileReference *get_file(); virtual VirtualFile *get_vfile(); virtual streampos get_file_pos(); @@ -59,6 +60,7 @@ private: istream *_in; bool _owns_in; Filename _filename; + time_t _timestamp; }; #include "datagramInputFile.I"