Better shader caching, GLSL "#pragma include" support, better error reporting

direct/src/filter/CommonFilters.py

@@ -199,7 +199,7 @@ class CommonFilters:
                 self.ssao[0].setShaderInput("depth", self.textures["depth"])
                 self.ssao[0].setShaderInput("normal", self.textures["aux"])
                 self.ssao[0].setShaderInput("random", loader.loadTexture("maps/random.rgb"))
-                self.ssao[0].setShader(Shader.make(SSAO_BODY % configuration["AmbientOcclusion"].numsamples))
+                self.ssao[0].setShader(Shader.make(SSAO_BODY % configuration["AmbientOcclusion"].numsamples, Shader.SL_Cg))
                 self.ssao[1].setShaderInput("src", ssao0)
                 self.ssao[1].setShader(self.loadShader("filter-blurx.sha"))
                 self.ssao[2].setShaderInput("src", ssao1)
@@ -339,7 +339,7 @@ class CommonFilters:
                 text += "  o_color = float4(1, 1, 1, 1) - o_color;\n"
             text += "}\n"
             
-            self.finalQuad.setShader(Shader.make(text))
+            self.finalQuad.setShader(Shader.make(text, Shader.SL_Cg))
             for tex in self.textures:
                 self.finalQuad.setShaderInput("tx"+tex, self.textures[tex])
             

panda/src/gobj/config_gobj.cxx

@@ -541,6 +541,22 @@ ConfigVariableString cg_glsl_version
           "glslv, glslf or glslg profiles.  Use this when you are having "
           "problems with these profiles.  Example values are 120 or 150."));
 
+ConfigVariableBool glsl_preprocess
+("glsl-preprocess", true,
+ PRC_DESC("If this is enabled, Panda looks for lines starting with "
+          "#pragma include when loading a GLSL shader and processes "
+          "it appropriately.  This can be useful if you have code that "
+          "is shared between multiple shaders.  Set this to false if "
+          "you have no need for this feature or if you do your own "
+          "preprocessing of GLSL shaders."));
+
+ConfigVariableInt glsl_include_recursion_limit
+("glsl-include-recursion-limit", 10,
+ PRC_DESC("This sets a limit on how many nested #pragma include "
+          "directives that Panda will follow when glsl-preprocess is "
+          "enabled.  This is used to prevent infinite recursion when "
+          "two shader files include each other."));
+
 ConfigureFn(config_gobj) {
   AnimateVerticesRequest::init_type();
   BufferContext::init_type();

panda/src/gobj/config_gobj.h

@@ -104,5 +104,7 @@ extern EXPCL_PANDA_GOBJ ConfigVariableInt lens_geom_segments;
 extern EXPCL_PANDA_GOBJ ConfigVariableBool stereo_lens_old_convergence;
 
 extern EXPCL_PANDA_GOBJ ConfigVariableString cg_glsl_version;
+extern EXPCL_PANDA_GOBJ ConfigVariableBool glsl_preprocess;
+extern EXPCL_PANDA_GOBJ ConfigVariableInt glsl_include_recursion_limit;
 
 #endif

panda/src/gobj/shader.I

@@ -21,31 +21,35 @@
 ////////////////////////////////////////////////////////////////////
 INLINE Filename Shader::
 get_filename(const ShaderType &type) const {
-  if (_filename->_separate && type == ST_none) {
+  if (_filename._separate && type != ST_none) {
     switch (type) {
       case ST_vertex:
-        return _filename->_vertex;
+        return _filename._vertex;
         break;
       case ST_fragment:
-        return _filename->_fragment;
+        return _filename._fragment;
         break;
       case ST_geometry:
-        return _filename->_geometry;
+        return _filename._geometry;
         break;
       case ST_tess_control:
-        return _text->_tess_control;
+        return _filename._tess_control;
         break;
       case ST_tess_evaluation:
-        return _text->_tess_evaluation;
+        return _filename._tess_evaluation;
         break;
       case ST_compute:
-        return _text->_compute;
+        return _filename._compute;
         break;
       default:
-        return _filename->_shared;
+        return _filename._shared;
     }
+  } else if (!_filename._shared.empty()) {
+    return _filename._shared;
+
   } else {
-    return _filename->_shared;
+    // Um, better than nothing?
+    return _filename._vertex;
   }
 }
 
@@ -56,32 +60,32 @@ get_filename(const ShaderType &type) const {
 ////////////////////////////////////////////////////////////////////
 INLINE const string &Shader::
 get_text(const ShaderType &type) const {
-  if (_text->_separate) {
+  if (_text._separate) {
-    nassertr(type != ST_none || !_text->_shared.empty(), _text->_shared);
+    nassertr(type != ST_none || !_text._shared.empty(), _text._shared);
     switch (type) {
       case ST_vertex:
-        return _text->_vertex;
+        return _text._vertex;
         break;
       case ST_fragment:
-        return _text->_fragment;
+        return _text._fragment;
         break;
       case ST_geometry:
-        return _text->_geometry;
+        return _text._geometry;
         break;
       case ST_tess_control:
-        return _text->_tess_control;
+        return _text._tess_control;
         break;
       case ST_tess_evaluation:
-        return _text->_tess_evaluation;
+        return _text._tess_evaluation;
         break;
       case ST_compute:
-        return _text->_compute;
+        return _text._compute;
         break;
       default:
-        return _text->_shared;
+        return _text._shared;
     }
   } else {
-    return _text->_shared;
+    return _text._shared;
   }
 }
 
@@ -811,3 +815,62 @@ read_datagram(DatagramIterator &scan) {
     _shared = scan.get_string();
   }
 }
+
+////////////////////////////////////////////////////////////////////
+//     Function: Shader::ShaderFile::operator <
+//       Access: Public
+//  Description: Ordering operator
+////////////////////////////////////////////////////////////////////
+INLINE bool Shader::ShaderFile::
+operator < (const Shader::ShaderFile &other) const {
+  if (_separate != other._separate) {
+    return (!_separate && other._separate);
+  }
+  if (_shared != other._shared) {
+    return (_shared < other._shared);
+  }
+  if (_vertex != other._vertex) {
+    return (_vertex < other._vertex);
+  }
+  if (_fragment != other._fragment) {
+    return (_fragment < other._fragment);
+  }
+  if (_geometry != other._geometry) {
+    return (_geometry < other._geometry);
+  }
+  if (_tess_control != other._tess_control) {
+    return (_tess_control < other._tess_control);
+  }
+  if (_tess_evaluation != other._tess_evaluation) {
+    return (_tess_evaluation < other._tess_evaluation);
+  }
+  if (_compute != other._compute) {
+    return (_compute < other._compute);
+  }
+  return false;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Shader::get_filename_from_index
+//       Access: Public
+//  Description: Returns the filename of the included shader with
+//               the given source file index (as recorded in the
+//               #line statement in r_preprocess_source).  We use
+//               this to associate error messages with included files.
+////////////////////////////////////////////////////////////////////
+INLINE Filename Shader::
+get_filename_from_index(int index, ShaderType type) const {
+  if (index == 0) {
+    Filename fn = get_filename(type);
+    if (!fn.empty()) {
+      return fn;
+    }
+  } else if (glsl_preprocess && index > 2048 &&
+             (index - 2048) < _included_files.size()) {
+    return _included_files[index - 2048];
+  }
+  // Must be a mistake.  Quietly put back the integer.
+  char str[32];
+  sprintf(str, "%d", index);
+  return Filename(str);
+}

panda/src/gobj/shader.cxx

@@ -17,6 +17,7 @@
 #include "shader.h"
 #include "preparedGraphicsObjects.h"
 #include "virtualFileSystem.h"
+#include "config_util.h"
 
 #ifdef HAVE_CG
 #include <Cg/cg.h>
@@ -1084,8 +1085,10 @@ compile_parameter(const ShaderArgId        &arg_id,
       cp_report_error(p, "Invalid type for a tex-parameter");
       return false;
     }
-    if (pieces.size()==3) {
+    if (pieces.size() == 3) {
       bind._suffix = InternalName::make(((string)"-") + pieces[2]);
+      gobj_cat.warning()
+        << "Parameter " << p._id._name << ": use of a texture suffix is deprecated.\n";
     }
     _tex_spec.push_back(bind);
     return true;
@@ -1532,7 +1535,7 @@ cg_compile_shader(const ShaderCaps &caps) {
     return false;
   }
 
-  if (!_text->_separate || !_text->_vertex.empty()) {
+  if (!_text._separate || !_text._vertex.empty()) {
     _cg_vprogram = cg_compile_entry_point("vshader", caps, ST_vertex);
     if (_cg_vprogram == 0) {
       cg_release_resources();
@@ -1541,7 +1544,7 @@ cg_compile_shader(const ShaderCaps &caps) {
     _cg_vprofile = cgGetProgramProfile(_cg_vprogram);
   }
 
-  if (!_text->_separate || !_text->_fragment.empty()) {
+  if (!_text._separate || !_text._fragment.empty()) {
     _cg_fprogram = cg_compile_entry_point("fshader", caps, ST_fragment);
     if (_cg_fprogram == 0) {
       cg_release_resources();
@@ -1550,7 +1553,7 @@ cg_compile_shader(const ShaderCaps &caps) {
     _cg_fprofile = cgGetProgramProfile(_cg_fprogram);
   }
 
-  if ((_text->_separate && !_text->_geometry.empty()) || (!_text->_separate && _text->_shared.find("gshader") != string::npos)) {
+  if ((_text._separate && !_text._geometry.empty()) || (!_text._separate && _text._shared.find("gshader") != string::npos)) {
     _cg_gprogram = cg_compile_entry_point("gshader", caps, ST_geometry);
     if (_cg_gprogram == 0) {
       cg_release_resources();
@@ -1607,7 +1610,6 @@ cg_analyze_entry_point(CGprogram prog, ShaderType type) {
   return success;
 }
 
-
 ////////////////////////////////////////////////////////////////////
 //     Function: Shader::cg_analyze_shader
 //       Access: Private
@@ -1779,26 +1781,19 @@ cg_analyze_shader(const ShaderCaps &caps) {
 ////////////////////////////////////////////////////////////////////
 CGprogram Shader::
 cg_program_from_shadertype(ShaderType type) {
-  CGprogram prog;
-
   switch (type) {
   case ST_vertex:
-      prog = _cg_vprogram;
+    return _cg_vprogram;
-      break;
 
   case ST_fragment:
-      prog = _cg_fprogram;
+    return _cg_fprogram;
-      break;
 
   case ST_geometry:
-      prog = _cg_gprogram;
+    return _cg_gprogram;
-      break;
 
   default:
-      prog = 0;
+    return 0;
-  };
+  }
-
-  return prog;
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -1915,16 +1910,15 @@ cg_compile_for(const ShaderCaps &caps,
 //     Function: Shader::Constructor
 //       Access: Private
 //  Description: Construct a Shader that will be filled in using
-//               fillin() later.
+//               fillin() or read() later.
 ////////////////////////////////////////////////////////////////////
 Shader::
-Shader() :
+Shader(ShaderLanguage lang) :
   _error_flag(false),
-  _text(NULL),
-  _filename(NULL),
   _parse(0),
   _loaded(false),
-  _language(SL_none)
+  _language(lang),
+  _last_modified(0)
 {
 #ifdef HAVE_CG
   _cg_context = 0;
@@ -1949,39 +1943,47 @@ Shader() :
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Shader::Constructor
+//     Function: Shader::read
 //       Access: Private
-//  Description: Construct a Shader.
+//  Description: Reads the shader from the given filename(s).
+//               Returns a boolean indicating success or failure.
 ////////////////////////////////////////////////////////////////////
-Shader::
+bool Shader::
-Shader(CPT(ShaderFile) filename, CPT(ShaderFile) text, const ShaderLanguage &lang) :
+read(const ShaderFile &sfile) {
-  _error_flag(false),
+  _text._separate = sfile._separate;
-  _text(text),
+
-  _filename(filename),
+  if (sfile._separate) {
-  _parse(0),
+    if (_language == SL_none) {
-  _loaded(false),
+      gobj_cat.error()
-  _language(lang)
+        << "No shader language was specified!\n";
-{
+      return false;
-#ifdef HAVE_CG
-  _cg_context = 0;
-  _cg_vprogram = 0;
-  _cg_fprogram = 0;
-  _cg_gprogram = 0;
-  _cg_vprofile = CG_PROFILE_UNKNOWN;
-  _cg_fprofile = CG_PROFILE_UNKNOWN;
-  _cg_gprofile = CG_PROFILE_UNKNOWN;
-  if (_default_caps._ultimate_vprofile == 0 || _default_caps._ultimate_vprofile == CG_PROFILE_UNKNOWN) {
-    _default_caps._active_vprofile = CG_PROFILE_UNKNOWN;
-    _default_caps._active_fprofile = CG_PROFILE_UNKNOWN;
-    _default_caps._active_gprofile = CG_PROFILE_UNKNOWN;
-    _default_caps._ultimate_vprofile = cgGetProfile("glslv");
-    _default_caps._ultimate_fprofile = cgGetProfile("glslf");
-    _default_caps._ultimate_gprofile = cgGetProfile("glslg");
-    if (_default_caps._ultimate_gprofile == CG_PROFILE_UNKNOWN) {
-      _default_caps._ultimate_gprofile = cgGetProfile("gp4gp");
     }
+
+    if (!sfile._vertex.empty() && !do_read_source(_text._vertex, sfile._vertex)) {
+      return false;
     }
-#endif
+    if (!sfile._fragment.empty() && !do_read_source(_text._fragment, sfile._fragment)) {
+      return false;
+    }
+    if (!sfile._geometry.empty() && !do_read_source(_text._geometry, sfile._geometry)) {
+      return false;
+    }
+    if (!sfile._tess_control.empty() && !do_read_source(_text._tess_control, sfile._tess_control)) {
+      return false;
+    }
+    if (!sfile._tess_evaluation.empty() && !do_read_source(_text._tess_evaluation, sfile._tess_evaluation)) {
+      return false;
+    }
+    if (!sfile._compute.empty() && !do_read_source(_text._compute, sfile._compute)) {
+      return false;
+    }
+    _filename = sfile;
+
+  } else {
+    if (!do_read_source(_text._shared, sfile._shared)) {
+      return false;
+    }
+    _filename = sfile;
 
     // Determine which language the shader is written in.
     if (_language == SL_none) {
@@ -1990,41 +1992,272 @@ Shader(CPT(ShaderFile) filename, CPT(ShaderFile) text, const ShaderLanguage &lan
       parse_line(header, true, true);
       if (header == "//Cg") {
         _language = SL_Cg;
-    } else if (header == "//GLSL") {
+      } else {
-      _language = SL_GLSL;
+        gobj_cat.error()
+          << "Unable to determine shader language of " << sfile._shared << "\n";
+        return false;
       }
+    } else if (_language == SL_GLSL) {
+      gobj_cat.error()
+        << "GLSL shaders must have separate shader bodies!\n";
+      return false;
     }
 
+    // Determine which language the shader is written in.
     if (_language == SL_Cg) {
 #ifdef HAVE_CG
-    if (!_text->_separate) {
+      if (!_text._separate) {
         cg_get_profile_from_header(_default_caps);
       }
 
       if (!cg_analyze_shader(_default_caps)) {
-      _error_flag = true;
+        gobj_cat.error()
+          << "Shader encountered an error.\n";
+        return false;
       }
 #else
       gobj_cat.error()
         << "Tried to load Cg shader, but no Cg support is enabled.\n";
 #endif
-  } else if (_language == SL_GLSL) {
-    // All of the important stuff is done in glShaderContext,
-    // to avoid gobj getting a dependency on OpenGL.
-    if (!_text->_separate) {
-      gobj_cat.error()
-        << "GLSL shaders must have separate shader bodies!\n";
-      _error_flag = true;
-    }
     } else {
       gobj_cat.error()
         << "Shader is not in a supported shader-language.\n";
-    _error_flag = true;
+      return false;
     }
-  if (_error_flag) {
+  }
+
+  _loaded = true;
+  return true;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Shader::do_read_source
+//       Access: Private
+//  Description: Reads the shader file from the given path into the
+//               given string.
+//
+//               Returns false if there was an error with this shader
+//               bad enough to consider it 'invalid'.
+////////////////////////////////////////////////////////////////////
+bool Shader::
+do_read_source(string &into, const Filename &fn) {
+  if (_language == SL_GLSL && glsl_preprocess) {
+    // Preprocess the GLSL file as we read it.
+    set<Filename> open_files;
+    ostringstream sstr;
+    if (!r_preprocess_source(sstr, fn, Filename(), open_files)) {
+      return false;
+    }
+    into = sstr.str();
+
+  } else {
+    gobj_cat.info() << "Reading shader file: " << fn << "\n";
+
+    VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
+    PT(VirtualFile) vf = vfs->find_file(fn, get_model_path());
+    if (vf == NULL) {
       gobj_cat.error()
-      << "Shader encountered an error.\n";
+        << "Could not find shader file: " << fn << "\n";
+      return false;
     }
+
+    if (!vf->read_file(into, true)) {
+      gobj_cat.error()
+        << "Could not read shader file: " << fn << "\n";
+      return false;
+    }
+
+    _last_modified = max(_last_modified, vf->get_timestamp());
+    _source_files.push_back(vf->get_filename());
+  }
+  return true;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Shader::r_preprocess_source
+//       Access: Private
+//  Description: Loads a given GLSL file line by line, and processes
+//               any #pragma include and once statements.
+//
+//               The set keeps track of which files we have already
+//               included, for checking recursive includes.
+////////////////////////////////////////////////////////////////////
+bool Shader::
+r_preprocess_source(ostream &out, const Filename &fn,
+                    const Filename &source_dir,
+                    set<Filename> &once_files, int depth) {
+
+  if (depth > glsl_include_recursion_limit) {
+    gobj_cat.error()
+      << "#pragma include nested too deeply\n";
+    return false;
+  }
+
+  DSearchPath path(get_model_path());
+  if (!source_dir.empty()) {
+    path.prepend_directory(source_dir);
+  }
+
+  VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
+  PT(VirtualFile) vf = vfs->find_file(fn, path);
+  if (vf == NULL) {
+    gobj_cat.error()
+      << "Could not find shader file: " << fn << "\n";
+    return false;
+  }
+
+  Filename full_fn = vf->get_filename();
+  if (once_files.find(full_fn) != once_files.end()) {
+    // If this file had a #pragma once, just move on.
+    return true;
+  }
+
+  istream *source = vf->open_read_file(true);
+  if (source == NULL) {
+    gobj_cat.error()
+      << "Could not open shader file: " << fn << "\n";
+    return false;
+  }
+
+  _last_modified = max(_last_modified, vf->get_timestamp());
+  _source_files.push_back(full_fn);
+
+  // We give each file an unique index.  This is so that we can identify
+  // a particular shader in the error output.  We offset them by 2048
+  // so that they are more recognizable.  GLSL doesn't give us anything
+  // more useful than that, unfortunately.
+  //
+  // Don't do this for the top-level file, though.  We don't want
+  // anything to get in before a potential #version directive.
+  int fileno = 0;
+  if (depth > 0) {
+    fileno = 2048 + _included_files.size();
+    // Write it into the vector so that we can substitute it later
+    // when we are parsing the GLSL error log.  Don't store the full
+    // filename because it would just be too long to display.
+    _included_files.push_back(fn);
+
+    out << "#line 1 " << fileno << " // " << fn << "\n";
+    if (gobj_cat.is_debug()) {
+      gobj_cat.debug()
+        << "Preprocessing shader include " << fileno << ": " << fn << "\n";
+    }
+  } else {
+    gobj_cat.info()
+      << "Preprocessing shader file: " << fn << "\n";
+  }
+
+  // Iterate over the lines for things we may need to preprocess.
+  string line;
+  bool had_include = false;
+  int lineno = 0;
+  while (getline(*source, line)) {
+    // We always forward the actual line - the GLSL compiler will
+    // silently ignore #pragma lines anyway.
+    ++lineno;
+    out << line << "\n";
+
+    // Check if this line contains a #pragma.
+    char pragma[64];
+    if (line.size() < 8 ||
+        sscanf(line.c_str(), " # pragma %63s", pragma) != 1) {
+
+      // One exception: check for an #endif after an include.  We have
+      // to restore the line number in case the include happened under
+      // an #if block.
+      int nread = 0;
+      if (had_include && sscanf(line.c_str(), " # endif %n", &nread) == 0 && nread >= 6) {
+        out << "#line " << (lineno + 1) << " " << fileno << "\n";
+      }
+      continue;
+    }
+
+    int nread = 0;
+    if (strcmp(pragma, "include") == 0) {
+      // Allow both double quotes and angle brackets.
+      Filename incfn, source_dir;
+      {
+        char incfile[2048];
+        if (sscanf(line.c_str(), " # pragma%*[ \t]include \"%2047[^\"]\" %n", incfile, &nread) == 1
+            && nread == line.size()) {
+          // A regular include, with double quotes.  Probably a local file.
+          source_dir = full_fn.get_dirname();
+          incfn = incfile;
+
+        } else if (sscanf(line.c_str(), " # pragma%*[ \t]include <%2047[^\"]> %n", incfile, &nread) == 1
+            && nread == line.size()) {
+          // Angled includes are also OK, but we don't search in the
+          // directory of the source file.
+          incfn = incfile;
+
+        } else {
+          // Couldn't parse it.
+          gobj_cat.error()
+            << "Malformed #pragma include at line " << lineno
+            << " of file " << fn << ":\n  " << line << "\n";
+          return false;
+        }
+      }
+
+      // OK, great.  Process the include.
+      if (!r_preprocess_source(out, incfn, source_dir, once_files, depth + 1)) {
+        // An error occurred.  Pass on the failure.
+        gobj_cat.error(false) << "included at line "
+          << lineno << " of file " << fn << ":\n  " << line << "\n";
+        return false;
+      }
+
+      // Restore the line counter.
+      out << "#line " << (lineno + 1) << " " << fileno << " // " << fn << "\n";
+      had_include = true;
+
+    } else if (strcmp(pragma, "once") == 0) {
+      // Do a stricter syntax check, just to be extra safe.
+      if (sscanf(line.c_str(), " # pragma%*[ \t]once %n", &nread) != 0 ||
+          nread != line.size()) {
+        gobj_cat.error()
+          << "Malformed #pragma once at line " << lineno
+          << " of file " << fn << ":\n  " << line << "\n";
+        return false;
+      }
+
+      once_files.insert(full_fn);
+
+    } else if (strcmp(pragma, "optionNV") == 0) {
+      // This is processed by NVIDIA drivers.  Don't touch it.
+
+    } else {
+      gobj_cat.warning()
+        << "Ignoring unknown pragma directive \"" << pragma << "\" at line "
+        << lineno << " of file " << fn << ":\n  " << line << "\n";
+    }
+  }
+
+  vf->close_read_file(source);
+  return true;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Shader::check_modified
+//       Access: Private
+//  Description: Checks whether the shader or any of its dependent
+//               files were modified on disk.
+////////////////////////////////////////////////////////////////////
+bool Shader::
+check_modified() const {
+  VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
+
+  pvector<Filename>::const_iterator it;
+  for (it = _source_files.begin(); it != _source_files.end(); ++it) {
+    const Filename &fn = (*it);
+
+    PT(VirtualFile) vfile = vfs->get_file(fn, true);
+    if (vfile == (VirtualFile *)NULL || vfile->get_timestamp() > _last_modified) {
+      return true;
+    }
+  }
+
+  return false;
 }
 
 #ifdef HAVE_CG
@@ -2166,11 +2399,14 @@ cg_get_profile_from_header(ShaderCaps& caps) {
 Shader::
 ~Shader() {
   release_all();
-  if (_loaded) {
+  // Note: don't try to erase ourselves from the table.  It currently
+  // keeps a reference forever, and so the only place where this
+  // constructor is called is in the destructor of the table itself.
+  /*if (_loaded) {
     _load_table.erase(_filename);
   } else {
     _make_table.erase(_text);
-  }
+  }*/
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -2179,25 +2415,28 @@ Shader::
 //  Description: Loads the shader with the given filename.
 ////////////////////////////////////////////////////////////////////
 PT(Shader) Shader::
-load(const Filename &file, const ShaderLanguage &lang) {
+load(const Filename &file, ShaderLanguage lang) {
-  PT(ShaderFile) sfile =  new ShaderFile(file);
+  ShaderFile sfile(file);
   ShaderTable::const_iterator i = _load_table.find(sfile);
   if (i != _load_table.end() && (lang == SL_none || lang == i->second->_language)) {
+    // But check that someone hasn't modified it in the meantime.
+    if (i->second->check_modified()) {
+      gobj_cat.info()
+        << "Shader " << file << " was modified on disk, reloading.\n";
+    } else {
+      gobj_cat.debug()
+        << "Shader " << file << " was found in shader cache.\n";
       return i->second;
     }
-  PT(ShaderFile) sbody = new ShaderFile("");
+  }
 
-  VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
+  PT(Shader) shader = new Shader(lang);
-  if (!vfs->read_file(file, sbody->_shared, true)) {
+  if (!shader->read(sfile)) {
-    gobj_cat.error()
-      << "Could not read shader file: " << file << "\n";
     return NULL;
   }
 
-  PT(Shader) result = new Shader(sfile, sbody, lang);
+  _load_table[sfile] = shader;
-  result->_loaded = true;
+  return shader;
-  _load_table[sfile] = result;
-  return result;
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -2207,47 +2446,30 @@ load(const Filename &file, const ShaderLanguage &lang) {
 //               programs separately.
 ////////////////////////////////////////////////////////////////////
 PT(Shader) Shader::
-load(const ShaderLanguage &lang, const Filename &vertex, 
+load(ShaderLanguage lang, const Filename &vertex,
      const Filename &fragment, const Filename &geometry,
      const Filename &tess_control, const Filename &tess_evaluation) {
-  PT(ShaderFile) sfile = new ShaderFile(vertex, fragment, geometry, tess_control, tess_evaluation);
+  ShaderFile sfile(vertex, fragment, geometry, tess_control, tess_evaluation);
   ShaderTable::const_iterator i = _load_table.find(sfile);
   if (i != _load_table.end() && (lang == SL_none || lang == i->second->_language)) {
+    // But check that someone hasn't modified it in the meantime.
+    if (i->second->check_modified()) {
+      gobj_cat.info()
+        << "Shader was modified on disk, reloading.\n";
+    } else {
+      gobj_cat.debug()
+        << "Shader was found in shader cache.\n";
       return i->second;
     }
+  }
 
-  PT(ShaderFile) sbody = new ShaderFile("", "", "", "", "");
+  PT(Shader) shader = new Shader(lang);
-  VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
+  if (!shader->read(sfile)) {
-  if (!vertex.empty() && !vfs->read_file(vertex, sbody->_vertex, true)) {
-    gobj_cat.error()
-      << "Could not read vertex shader file: " << vertex << "\n";
-    return NULL;
-  }
-  if (!fragment.empty() && !vfs->read_file(fragment, sbody->_fragment, true)) {
-    gobj_cat.error()
-      << "Could not read fragment shader file: " << fragment << "\n";
-    return NULL;
-  }
-  if (!geometry.empty() && !vfs->read_file(geometry, sbody->_geometry, true)) {
-    gobj_cat.error()
-      << "Could not read geometry shader file: " << geometry << "\n";
-    return NULL;
-  }
-  if (!tess_control.empty() && !vfs->read_file(tess_control, sbody->_tess_control, true)) {
-    gobj_cat.error()
-      << "Could not read tess_control shader file: " << tess_control << "\n";
-    return NULL;
-  }
-  if (!tess_evaluation.empty() && !vfs->read_file(tess_evaluation, sbody->_tess_evaluation, true)) {
-    gobj_cat.error()
-      << "Could not read tess_evaluation shader file: " << tess_evaluation << "\n";
     return NULL;
   }
 
-  PT(Shader) result = new Shader(sfile, sbody, lang);
+  _load_table[sfile] = shader;
-  result->_loaded = true;
+  return shader;
-  _load_table[sfile] = result;
-  return result;
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -2256,26 +2478,37 @@ load(const ShaderLanguage &lang, const Filename &vertex,
 //  Description: Loads a compute shader.
 ////////////////////////////////////////////////////////////////////
 PT(Shader) Shader::
-load_compute(const ShaderLanguage &lang, const Filename &fn) {
+load_compute(ShaderLanguage lang, const Filename &fn) {
-  PT(ShaderFile) sfile = new ShaderFile(fn);
+  if (lang != SL_GLSL) {
-  ShaderTable::const_iterator i = _load_table.find(sfile);
-  if (i != _load_table.end() && (lang == SL_none || lang == i->second->_language)) {
-    return i->second;
-  }
-
-  PT(ShaderFile) sbody = new ShaderFile;
-  sbody->_separate = true;
-  VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
-  if (!vfs->read_file(fn, sbody->_compute, true)) {
     gobj_cat.error()
-      << "Could not read compute shader file: " << fn << "\n";
+      << "Only GLSL compute shaders are currently supported.\n";
     return NULL;
   }
 
-  PT(Shader) result = new Shader(sfile, sbody, lang);
+  ShaderFile sfile;
-  result->_loaded = true;
+  sfile._separate = true;
-  _load_table[sfile] = result;
+  sfile._compute = fn;
-  return result;
+
+  ShaderTable::const_iterator i = _load_table.find(sfile);
+  if (i != _load_table.end() && (lang == SL_none || lang == i->second->_language)) {
+    // But check that someone hasn't modified it in the meantime.
+    if (i->second->check_modified()) {
+      gobj_cat.info()
+        << "Compute shader " << fn << " was modified on disk, reloading.\n";
+    } else {
+      gobj_cat.debug()
+        << "Compute shader " << fn << " was found in shader cache.\n";
+      return i->second;
+    }
+  }
+
+  PT(Shader) shader = new Shader(lang);
+  if (!shader->read(sfile)) {
+    return NULL;
+  }
+
+  _load_table[sfile] = shader;
+  return shader;
 }
 
 //////////////////////////////////////////////////////////////////////
@@ -2284,16 +2517,47 @@ load_compute(const ShaderLanguage &lang, const Filename &fn) {
 //  Description: Loads the shader, using the string as shader body.
 //////////////////////////////////////////////////////////////////////
 PT(Shader) Shader::
-make(const string &body, const ShaderLanguage &lang) {
+make(const string &body, ShaderLanguage lang) {
-  PT(ShaderFile) sbody = new ShaderFile(body);
+  if (lang == SL_GLSL) {
+    gobj_cat.error()
+      << "GLSL shaders must have separate shader bodies!\n";
+    return NULL;
+
+  } else if (lang == SL_none) {
+    gobj_cat.warning()
+      << "Shader::make() now requires an explicit shader language.  Assuming Cg.\n";
+    lang = SL_Cg;
+  }
+#ifndef HAVE_CG
+  if (lang == SL_Cg) {
+    gobj_cat.error() << "Support for Cg shaders is not enabled.\n";
+    return NULL;
+  }
+#endif
+
+  ShaderFile sbody(body);
   ShaderTable::const_iterator i = _make_table.find(sbody);
   if (i != _make_table.end() && (lang == SL_none || lang == i->second->_language)) {
     return i->second;
   }
 
-  PT(ShaderFile) sfile = new ShaderFile("created-shader");
+  PT(Shader) shader = new Shader(lang);
-  PT(Shader) result = new Shader(sfile, sbody, lang);
+  shader->_filename = ShaderFile("created-shader");
-  _make_table[sbody] = result;
+  shader->_text = sbody;
+
+#ifdef HAVE_CG
+  if (lang == SL_Cg) {
+    shader->cg_get_profile_from_header(_default_caps);
+
+    if (!shader->cg_analyze_shader(_default_caps)) {
+      gobj_cat.error()
+        << "Shader encountered an error.\n";
+      return NULL;
+    }
+  }
+#endif
+
+  _make_table[sbody] = shader;
 
   if (dump_generated_shaders) {
     ostringstream fns;
@@ -2307,7 +2571,7 @@ make(const string &body, const ShaderLanguage &lang) {
     s << body;
     s.close();
   }
-  return result;
+  return shader;
 }
 
 //////////////////////////////////////////////////////////////////////
@@ -2316,20 +2580,45 @@ make(const string &body, const ShaderLanguage &lang) {
 //  Description: Loads the shader, using the strings as shader bodies.
 //////////////////////////////////////////////////////////////////////
 PT(Shader) Shader::
-make(const ShaderLanguage &lang, const string &vertex, const string &fragment, 
+make(ShaderLanguage lang, const string &vertex, const string &fragment,
      const string &geometry, const string &tess_control,
      const string &tess_evaluation) {
-  PT(ShaderFile) sbody = new ShaderFile(vertex, fragment, geometry, tess_control, tess_evaluation);
+#ifndef HAVE_CG
+  if (lang == SL_Cg) {
+    gobj_cat.error() << "Support for Cg shaders is not enabled.\n";
+    return NULL;
+  }
+#endif
+  if (lang == SL_none) {
+    gobj_cat.error()
+      << "Shader::make() requires an explicit shader language.\n";
+    return NULL;
+  }
+
+  ShaderFile sbody(vertex, fragment, geometry, tess_control, tess_evaluation);
 
   ShaderTable::const_iterator i = _make_table.find(sbody);
   if (i != _make_table.end() && (lang == SL_none || lang == i->second->_language)) {
     return i->second;
   }
 
-  PT(ShaderFile) sfile = new ShaderFile("created-shader");
+  PT(Shader) shader = new Shader(lang);
-  PT(Shader) result = new Shader(sfile, sbody, lang);
+  shader->_filename = ShaderFile("created-shader");
-  _make_table[sbody] = result;
+  shader->_text = sbody;
-  return result;
+
+#ifdef HAVE_CG
+  if (lang == SL_Cg) {
+    if (!shader->cg_analyze_shader(_default_caps)) {
+      gobj_cat.error()
+        << "Shader encountered an error.\n";
+      return NULL;
+    }
+  }
+#endif
+
+
+  _make_table[sbody] = shader;
+  return shader;
 }
 
 //////////////////////////////////////////////////////////////////////
@@ -2338,20 +2627,28 @@ make(const ShaderLanguage &lang, const string &vertex, const string &fragment,
 //  Description: Loads the compute shader from the given string.
 //////////////////////////////////////////////////////////////////////
 PT(Shader) Shader::
-make_compute(const ShaderLanguage &lang, const string &body) {
+make_compute(ShaderLanguage lang, const string &body) {
-  PT(ShaderFile) sbody = new ShaderFile;
+  if (lang != SL_GLSL) {
-  sbody->_separate = true;
+    gobj_cat.error()
-  sbody->_compute = body;
+      << "Only GLSL compute shaders are currently supported.\n";
+    return NULL;
+  }
+
+  ShaderFile sbody;
+  sbody._separate = true;
+  sbody._compute = body;
 
   ShaderTable::const_iterator i = _make_table.find(sbody);
   if (i != _make_table.end() && (lang == SL_none || lang == i->second->_language)) {
     return i->second;
   }
 
-  PT(ShaderFile) sfile = new ShaderFile("created-shader");
+  PT(Shader) shader = new Shader(lang);
-  PT(Shader) result = new Shader(sfile, sbody, lang);
+  shader->_filename = ShaderFile("created-shader");
-  _make_table[sbody] = result;
+  shader->_text = sbody;
-  return result;
+
+  _make_table[sbody] = shader;
+  return shader;
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -2374,11 +2671,11 @@ parse_init() {
 ////////////////////////////////////////////////////////////////////
 void Shader::
 parse_line(string &result, bool lt, bool rt) {
-  nassertv(!_text->_separate);
+  nassertv(!_text._separate);
-  int len = _text->_shared.size();
+  int len = _text._shared.size();
   int head = _parse;
   int tail = head;
-  while ((tail < len) && (_text->_shared[tail] != '\n')) {
+  while ((tail < len) && (_text._shared[tail] != '\n')) {
     tail++;
   }
   if (tail < len) {
@@ -2387,10 +2684,10 @@ parse_line(string &result, bool lt, bool rt) {
     _parse = tail;
   }
   if (lt) {
-    while ((head < tail)&&(isspace(_text->_shared[head]))) head++;
+    while ((head < tail)&&(isspace(_text._shared[head]))) head++;
-    while ((tail > head)&&(isspace(_text->_shared[tail-1]))) tail--;
+    while ((tail > head)&&(isspace(_text._shared[tail-1]))) tail--;
   }
-  result = _text->_shared.substr(head, tail-head);
+  result = _text._shared.substr(head, tail-head);
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -2403,20 +2700,20 @@ parse_line(string &result, bool lt, bool rt) {
 ////////////////////////////////////////////////////////////////////
 void Shader::
 parse_upto(string &result, string pattern, bool include) {
-  nassertv(!_text->_separate);
+  nassertv(!_text._separate);
   GlobPattern endpat(pattern);
   int start = _parse;
   int last = _parse;
-  while (_parse < (int)(_text->_shared.size())) {
+  while (_parse < (int)(_text._shared.size())) {
     string t;
     parse_line(t, true, true);
     if (endpat.matches(t)) break;
     last = _parse;
   }
   if (include) {
-    result = _text->_shared.substr(start, _parse - start);
+    result = _text._shared.substr(start, _parse - start);
   } else {
-    result = _text->_shared.substr(start, last - start);
+    result = _text._shared.substr(start, last - start);
   }
 }
 
@@ -2428,8 +2725,8 @@ parse_upto(string &result, string pattern, bool include) {
 ////////////////////////////////////////////////////////////////////
 void Shader::
 parse_rest(string &result) {
-  nassertv(!_text->_separate);
+  nassertv(!_text._separate);
-  result = _text->_shared.substr(_parse, _text->_shared.size() - _parse);
+  result = _text._shared.substr(_parse, _text._shared.size() - _parse);
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -2440,7 +2737,7 @@ parse_rest(string &result) {
 ////////////////////////////////////////////////////////////////////
 bool Shader::
 parse_eof() {
-  return (int)_text->_shared.size() == _parse;
+  return (int)_text._shared.size() == _parse;
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -2629,8 +2926,8 @@ void Shader::
 write_datagram(BamWriter *manager, Datagram &dg) {
   dg.add_uint8(_language);
   dg.add_bool(_loaded);
-  _filename->write_datagram(dg);
+  _filename.write_datagram(dg);
-  _text->write_datagram(dg);
+  _text.write_datagram(dg);
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -2643,7 +2940,7 @@ write_datagram(BamWriter *manager, Datagram &dg) {
 ////////////////////////////////////////////////////////////////////
 TypedWritable *Shader::
 make_from_bam(const FactoryParams &params) {
-  Shader *attrib = new Shader;
+  Shader *attrib = new Shader(SL_none);
   DatagramIterator scan;
   BamReader *manager;
 
@@ -2663,12 +2960,6 @@ void Shader::
 fillin(DatagramIterator &scan, BamReader *manager) {
   _language = (ShaderLanguage) scan.get_uint8();
   _loaded = scan.get_bool();
-
+  _filename.read_datagram(scan);
-  PT(ShaderFile) filename = new ShaderFile;
+  _text.read_datagram(scan);
-  filename->read_datagram(scan);
-  _filename = filename;
-
-  PT(ShaderFile) text = new ShaderFile;
-  text->read_datagram(scan);
-  _text = text;
 }

panda/src/gobj/shader.h

@@ -81,20 +81,20 @@ PUBLISHED:
     bit_AutoShaderShadow = 4, // bit for AS_shadow
   };
 
-  static PT(Shader) load(const Filename &file, const ShaderLanguage &lang = SL_none);
+  static PT(Shader) load(const Filename &file, ShaderLanguage lang = SL_none);
-  static PT(Shader) make(const string &body, const ShaderLanguage &lang = SL_none);
+  static PT(Shader) make(const string &body, ShaderLanguage lang = SL_none);
-  static PT(Shader) load(const ShaderLanguage &lang, 
+  static PT(Shader) load(ShaderLanguage lang,
                          const Filename &vertex, const Filename &fragment,
                          const Filename &geometry = "",
                          const Filename &tess_control = "",
                          const Filename &tess_evaluation = "");
-  static PT(Shader) load_compute(const ShaderLanguage &lang, const Filename &fn);
+  static PT(Shader) load_compute(ShaderLanguage lang, const Filename &fn);
-  static PT(Shader) make(const ShaderLanguage &lang, 
+  static PT(Shader) make(ShaderLanguage lang,
                          const string &vertex, const string &fragment,
                          const string &geometry = "",
                          const string &tess_control = "",
                          const string &tess_evaluation = "");
-  static PT(Shader) make_compute(const ShaderLanguage &lang, const string &body);
+  static PT(Shader) make_compute(ShaderLanguage lang, const string &body);
 
   INLINE Filename get_filename(const ShaderType &type = ST_none) const;
   INLINE const string &get_text(const ShaderType &type = ST_none) const;
@@ -407,6 +407,8 @@ public:
     INLINE void write_datagram(Datagram &dg) const;
     INLINE void read_datagram(DatagramIterator &source);
 
+    INLINE bool operator < (const ShaderFile &other) const;
+
   public:
     bool _separate;
     string _shared;
@@ -464,8 +466,8 @@ public:
 
   void clear_parameters();
 
-#ifdef HAVE_CG
 private:
+#ifdef HAVE_CG
   ShaderArgClass cg_parameter_class(CGparameter p);
   ShaderArgType cg_parameter_type(CGparameter p);
   ShaderArgDir cg_parameter_dir(CGparameter p);
@@ -496,7 +498,6 @@ private:
   CGprogram cg_program_from_shadertype(ShaderType type);
 
 public:
-
   bool cg_compile_for(const ShaderCaps &caps, CGcontext &ctx,
                       CGprogram &vprogram, CGprogram &fprogram,
                       CGprogram &gprogram, pvector<CGparameter> &map);
@@ -510,19 +511,25 @@ public:
   pvector <ShaderVarSpec> _var_spec;
 
   bool _error_flag;
-  CPT(ShaderFile) _text;
+  ShaderFile _text;
 
 protected:
-  CPT(ShaderFile) _filename;
+  ShaderFile _filename;
   int _parse;
   bool _loaded;
   ShaderLanguage _language;
+  pvector<Filename> _included_files;
+
+  // Stores full paths, and includes the fullpaths of the shaders
+  // themselves as well as the includes.
+  pvector<Filename> _source_files;
+  time_t _last_modified;
 
   static ShaderCaps _default_caps;
   static ShaderUtilization _shader_utilization;
   static int _shaders_generated;
 
-  typedef pmap < CPT(ShaderFile), Shader * > ShaderTable;
+  typedef pmap<ShaderFile, PT(Shader)> ShaderTable;
 
   static ShaderTable _load_table;
   static ShaderTable _make_table;
@@ -536,12 +543,21 @@ protected:
 private:
   void clear_prepared(PreparedGraphicsObjects *prepared_objects);
 
-  Shader();
+  Shader(ShaderLanguage lang);
+
+  bool read(const ShaderFile &sfile);
+  bool do_read_source(string &into, const Filename &fn);
+  bool r_preprocess_source(ostream &out, const Filename &fn,
+                           const Filename &source_dir,
+                           set<Filename> &open_files, int depth = 0);
+
+  bool check_modified() const;
 
 public:
-  Shader(CPT(ShaderFile) name, CPT(ShaderFile) text, const ShaderLanguage &lang = SL_none);
   ~Shader();
 
+  INLINE Filename get_filename_from_index(int index, ShaderType type) const;
+
 public:
   static void register_with_read_factory();
   virtual void write_datagram(BamWriter *manager, Datagram &dg);

panda/src/pgraph/shaderPool.cxx

@@ -74,25 +74,29 @@ ns_load_shader(const Filename &orig_filename) {
     }
   }
 
-/*
+  // The shader was not found in the pool.
-  shader_cat.info()
+  gobj_cat.info()
     << "Loading shader " << filename << "\n";
-*/
 
-  CPT(Shader) shader;
+  Shader::ShaderLanguage lang = Shader::SL_none;
 
-  shader = (CPT(Shader)) NULL;
+  // Do some guesswork to see if we can figure out the shader language
-  string extension = filename.get_extension();
+  // from the file extension.  This is really just guesswork - there are
+  // no standardized extensions for shaders, especially for GLSL.
+  // These are the ones that appear to be closest to "standard".
+  string ext = downcase(filename.get_extension());
+  if (ext == "cg" || ext == "sha") {
+    // "sha" is for historical reasons.
+    lang = Shader::SL_Cg;
 
-  if (extension.empty() || extension == "cg" || extension == "hlsl") {
+  } else if (ext == "glsl" || ext == "vert" || ext == "frag" ||
-    // this does nothing for now
+             ext == "geom" || ext == "tesc" || ext == "tese" ||
+             ext == "comp") {
+    lang = Shader::SL_GLSL;
   }
 
-  if (shader == (CPT(Shader)) NULL) {
+  PT(Shader) shader = Shader::load(filename, lang);
-    shader = Shader::load (filename);
+  if (shader == (Shader *)NULL) {
-  }
-
-  if (shader == (CPT(Shader)) NULL) {
     // This shader was not found or could not be read.
     return NULL;
   }

panda/src/pgraphnodes/shaderGenerator.cxx

@@ -509,7 +509,7 @@ clear_analysis() {
 ////////////////////////////////////////////////////////////////////
 CPT(RenderAttrib) ShaderGenerator::
 create_shader_attrib(const string &txt) {
-  PT(Shader) shader = Shader::make(txt);
+  PT(Shader) shader = Shader::make(txt, Shader::SL_Cg);
   CPT(RenderAttrib) shattr = ShaderAttrib::make();
   shattr = DCAST(ShaderAttrib, shattr)->set_shader(shader);
   if (_lighting) {