ShaderGenerator: reduce combinatoric explosion of shaders

This is done by:
* Not considering the texture pointer when looking up a generated shader, only type
* Not requiring different shaders based on RescaleNormalAttrib
* Not looking at AlphaTestAttrib unless it is going to be relevant

This should dramatically reduce the number of shaders that are being generated for many scenes, especially since the only thing that differs from object to object is often just the texture.

These changes are also necessary to make b781995ef135fdaea6d2479f8b3b7e6213e9d9f3 more usable, since prepare_scene may see a slightly different state than is encountered at render time due to code in CullResult adding in an AlphaTestAttrib or RescaleNormalAttrib.

panda/src/pgraph/alphaTestAttrib.cxx

@@ -18,6 +18,7 @@
 #include "bamWriter.h"
 #include "datagram.h"
 #include "datagramIterator.h"
+#include "auxBitplaneAttrib.h"
 
 TypeHandle AlphaTestAttrib::_type_handle;
 int AlphaTestAttrib::_attrib_slot;
@@ -98,7 +99,15 @@ get_hash_impl() const {
  */
 CPT(RenderAttrib) AlphaTestAttrib::
 get_auto_shader_attrib_impl(const RenderState *state) const {
-  return this;
+  // This is only important if the shader subsumes the alpha test, which only
+  // happens if there is an AuxBitplaneAttrib with ABO_glow.
+  const AuxBitplaneAttrib *aux;
+  if (!state->get_attrib(aux) ||
+      (aux->get_outputs() & AuxBitplaneAttrib::ABO_glow) == 0) {
+    return nullptr;
+  } else {
+    return this;
+  }
 }
 
 /**

panda/src/pgraph/rescaleNormalAttrib.cxx

@@ -86,11 +86,14 @@ get_hash_impl() const {
 CPT(RenderAttrib) RescaleNormalAttrib::
 get_auto_shader_attrib_impl(const RenderState *state) const {
   // We currently only support M_normalize in the ShaderGenerator.
-  if (_mode == M_none || _mode == M_normalize) {
+  /*if (_mode == M_none || _mode == M_normalize) {
     return this;
   } else {
     return RescaleNormalAttrib::make(M_normalize);
-  }
+  }*/
+  // Actually, we currently ignore this attribute in the shader generator,
+  // and always normalize the normals.  It's too much of a bother.
+  return nullptr;
 }
 
 /**

panda/src/pgraph/shaderAttrib.cxx

@@ -523,14 +523,15 @@ void ShaderAttrib::
 output(ostream &out) const {
   out << "ShaderAttrib:";
 
-  if (_has_shader) {
-    if (_shader == NULL) {
+  if (_auto_shader) {
+    out << "auto";
+    return;
+  } else if (_has_shader) {
+    if (_shader == nullptr) {
       out << "off";
     } else {
       out << _shader->get_filename().get_basename();
     }
-  } else if (_auto_shader) {
-    out << "auto";
   }
 
   out << "," << _inputs.size() << " inputs";

panda/src/pgraph/textureAttrib.I

@@ -140,6 +140,26 @@ get_on_texture(TextureStage *stage) const {
   return NULL;
 }
 
+/**
+ * Returns the type of the texture associated with the indicated stage.  It is
+ * an error to call this if has_on_stage(stage) returns false.
+ *
+ * This method is no different than get_on_texture(stage)->get_texture_type();
+ * it merely exists to address a corner case for the shader generator.
+ */
+INLINE Texture::TextureType TextureAttrib::
+get_on_texture_type(TextureStage *stage) const {
+  Stages::const_iterator si;
+  si = _on_stages.find(StageNode(stage));
+  nassertr(si != _on_stages.end(), Texture::TT_2d_texture);
+
+  if ((*si)._texture == nullptr) {
+    return (*si)._texture_type;
+  } else {
+    return (*si)._texture->get_texture_type();
+  }
+}
+
 /**
  * Returns the sampler associated with the indicated stage, or the one
  * associated with its texture if no custom stage has been specified.  It is
@@ -215,6 +235,26 @@ is_identity() const {
   return _on_stages.empty() && _off_stages.empty() && !_off_all_stages;
 }
 
+/**
+ * Call only on a state returned by get_auto_shader_attrib.
+ */
+bool TextureAttrib::
+on_stage_affects_rgb(size_t n) const {
+  nassertr(n < _render_stages.size(), false);
+  StageNode *stage = _render_stages[n];
+  return stage->_texture_affects_rgb;
+}
+
+/**
+ * Call only on a state returned by get_auto_shader_attrib.
+ */
+bool TextureAttrib::
+on_stage_affects_alpha(size_t n) const {
+  nassertr(n < _render_stages.size(), false);
+  StageNode *stage = _render_stages[n];
+  return stage->_texture_affects_alpha;
+}
+
 /**
  * Confirms whether the _on_stages list is still sorted.  It will become
  * unsorted if someone calls TextureStage::set_sort().

panda/src/pgraph/textureAttrib.cxx

@@ -106,6 +106,8 @@ find_on_stage(const TextureStage *stage) const {
  */
 CPT(RenderAttrib) TextureAttrib::
 add_on_stage(TextureStage *stage, Texture *tex, int override) const {
+  nassertr(tex != nullptr, this);
+
   TextureAttrib *attrib = new TextureAttrib(*this);
   Stages::iterator si = attrib->_on_stages.insert(StageNode(stage)).first;
   (*si)._override = override;
@@ -127,6 +129,8 @@ add_on_stage(TextureStage *stage, Texture *tex, int override) const {
  */
 CPT(RenderAttrib) TextureAttrib::
 add_on_stage(TextureStage *stage, Texture *tex, const SamplerState &sampler, int override) const {
+  nassertr(tex != nullptr, this);
+
   TextureAttrib *attrib = new TextureAttrib(*this);
   Stages::iterator si = attrib->_on_stages.insert(StageNode(stage)).first;
   (*si)._override = override;
@@ -381,10 +385,19 @@ output(ostream &out) const {
     const StageNode &sn = *(*ri);
     TextureStage *stage = sn._stage;
     Texture *tex = sn._texture;
-    if (tex != NULL) {
+    if (tex != nullptr) {
       out << " " << stage->get_name() << ":" << tex->get_name();
     } else {
-      out << " " << stage->get_name();
+      out << " " << stage->get_name() << ":(" << sn._texture_type;
+      if (sn._texture_affects_rgb) {
+        out << " rgb";
+        if (sn._texture_affects_alpha) {
+          out << "a";
+        }
+      } else if (sn._texture_affects_alpha) {
+        out << " alpha";
+      }
+      out << ")";
     }
     if (sn._override != 0) {
       out << "^" << sn._override;
@@ -500,6 +513,19 @@ compare_to_impl(const RenderAttrib *other) const {
       }
     }
 
+    if (texture == nullptr) {
+      // This is an attribute returned by get_auto_shader_attrib_impl.
+      if ((*si)._texture_type != (*osi)._texture_type) {
+        return (*si)._texture_type < (*osi)._texture_type ? -1 : 1;
+      }
+      if ((*si)._texture_affects_rgb != (*osi)._texture_affects_rgb) {
+        return (*si)._texture_affects_rgb < (*osi)._texture_affects_rgb ? -1 : 1;
+      }
+      if ((*si)._texture_affects_alpha != (*osi)._texture_affects_alpha) {
+        return (*si)._texture_affects_alpha < (*osi)._texture_affects_alpha ? -1 : 1;
+      }
+    }
+
     ++si;
     ++osi;
   }
@@ -563,6 +589,13 @@ get_hash_impl() const {
     hash = pointer_hash::add_hash(hash, sn._texture);
     hash = int_hash::add_hash(hash, (int)sn._implicit_sort);
     hash = int_hash::add_hash(hash, sn._override);
+
+    if (sn._texture == nullptr) {
+      // This is an attribute returned by get_auto_shader_attrib_impl.
+      hash = int_hash::add_hash(hash, (int)sn._texture_type);
+      hash = int_hash::add_hash(hash, (int)sn._texture_affects_rgb);
+      hash = int_hash::add_hash(hash, (int)sn._texture_affects_alpha);
+    }
   }
 
   // This bool value goes here, between the two lists, to differentiate
@@ -739,7 +772,42 @@ invert_compose_impl(const RenderAttrib *other) const {
  */
 CPT(RenderAttrib) TextureAttrib::
 get_auto_shader_attrib_impl(const RenderState *state) const {
-  return this;
+  if (_on_stages.empty()) {
+    // Having no stages is the same as not applying a texture attribute.
+    return nullptr;
+  }
+
+  // We make a texture attribute that does not store the texture and sampler,
+  // so that we don't have to generate a shader for every possible texture.
+  // We do have to store the few texture properties that do affect the
+  // generated shader.
+  PT(TextureAttrib) attrib = new TextureAttrib;
+  attrib->_on_stages = _on_stages;
+  attrib->sort_on_stages();
+
+  Stages::iterator si;
+  for (si = attrib->_on_stages.begin(); si != attrib->_on_stages.end(); ++si) {
+    Texture::Format format = (*si)._texture->get_format();
+    (*si)._texture_type = (*si)._texture->get_texture_type();
+    (*si)._texture_affects_rgb = (format != Texture::F_alpha);
+    (*si)._texture_affects_alpha = Texture::has_alpha(format);
+    (*si)._texture = nullptr;
+    (*si)._has_sampler = false;
+
+    // We will no longer be composing or sorting this state so we can get rid
+    // of these values.
+    (*si)._override = 0;
+    (*si)._implicit_sort = 0;
+
+    // Exception to optimize a common case: if the first texture is an RGB
+    // texture, we don't care about whether it affects the alpha channel.
+    if (attrib->_render_stages[0] == &(*si) &&
+        (*si)._texture_affects_rgb && !(*si)._texture_affects_alpha) {
+      (*si)._texture_affects_alpha = true;
+    }
+  }
+
+  return return_new(attrib);
 }
 
 /**

panda/src/pgraph/textureAttrib.h

@@ -59,6 +59,7 @@ PUBLISHED:
   INLINE int get_ff_tc_index(int n) const;
   INLINE bool has_on_stage(TextureStage *stage) const;
   INLINE Texture *get_on_texture(TextureStage *stage) const;
+  INLINE Texture::TextureType get_on_texture_type(TextureStage *stage) const;
   INLINE const SamplerState &get_on_sampler(TextureStage *stage) const;
   INLINE int get_on_stage_override(TextureStage *stage) const;
 
@@ -81,6 +82,8 @@ PUBLISHED:
   CPT(RenderAttrib) unify_texture_stages(TextureStage *stage) const;
 
 public:
+  INLINE bool on_stage_affects_rgb(size_t n) const;
+  INLINE bool on_stage_affects_alpha(size_t n) const;
   CPT(TextureAttrib) filter_to_max(int max_texture_stages) const;
 
   virtual bool lower_attrib_can_override() const;
@@ -114,6 +117,11 @@ private:
     int _ff_tc_index;
     unsigned int _implicit_sort;
     int _override;
+
+    // These fields are used by the shader generator.
+    Texture::TextureType _texture_type : 16;
+    bool _texture_affects_rgb : 8;
+    bool _texture_affects_alpha : 8;
   };
 
   class CompareTextureStagePriorities {

panda/src/pgraph/transparencyAttrib.cxx

@@ -114,7 +114,14 @@ get_hash_impl() const {
  */
 CPT(RenderAttrib) TransparencyAttrib::
 get_auto_shader_attrib_impl(const RenderState *state) const {
-  return this;
+  if (_mode == TransparencyAttrib::M_alpha) {
+    return this;
+  } else if (_mode == TransparencyAttrib::M_premultiplied_alpha ||
+             _mode == TransparencyAttrib::M_dual) {
+    return return_new(new TransparencyAttrib(M_alpha));
+  } else {
+    return nullptr;
+  }
 }
 
 /**

panda/src/pgraphnodes/shaderGenerator.cxx

@@ -357,10 +357,14 @@ analyze_renderstate(const RenderState *rs) {
   }
 
   // Determine whether we should normalize the normals.
-  const RescaleNormalAttrib *rescale;
+  /*const RescaleNormalAttrib *rescale;
   rs->get_attrib_def(rescale);
 
   _normalize_normals = (rescale->get_mode() != RescaleNormalAttrib::M_none);
+  */
+  // Actually, let's always normalize the normals for now.  This helps to
+  // reduce combinatoric explosion of shaders.
+  _normalize_normals = true;
 
   // Decide which material modes need to be calculated.
 
@@ -823,11 +827,10 @@ synthesize_shader(const RenderState *rs, const GeomVertexAnimationSpec &anim) {
     text << "\t in float4 l_" << it->first->join("_") << " : " << it->second << ",\n";
   }
   const TexMatrixAttrib *tex_matrix = DCAST(TexMatrixAttrib, rs->get_attrib_def(TexMatrixAttrib::get_class_slot()));
-  for (int i=0; i<_num_textures; i++) {
+  for (int i = 0; i < _num_textures; ++i) {
     TextureStage *stage = texture->get_on_stage(i);
-    Texture *tex = texture->get_on_texture(stage);
-    nassertr(tex != NULL, NULL);
-    text << "\t uniform sampler" << texture_type_as_string(tex->get_texture_type()) << " tex_" << i << ",\n";
+    Texture::TextureType type = texture->get_on_texture_type(stage);
+    text << "\t uniform sampler" << texture_type_as_string(type) << " tex_" << i << ",\n";
     if (tex_matrix->has_stage(stage)) {
       text << "\t uniform float4x4 texmat_" << i << ",\n";
     }
@@ -935,11 +938,10 @@ synthesize_shader(const RenderState *rs, const GeomVertexAnimationSpec &anim) {
   }
   text << "\t // Fetch all textures.\n";
   if (_map_index_height >= 0 && parallax_mapping_samples > 0) {
-    Texture *tex = texture->get_on_texture(texture->get_on_stage(_map_index_height));
-    nassertr(tex != NULL, NULL);
-    text << "\t float4 tex" << _map_index_height << " = tex" << texture_type_as_string(tex->get_texture_type());
+    Texture::TextureType type = texture->get_on_texture_type(texture->get_on_stage(_map_index_height));
+    text << "\t float4 tex" << _map_index_height << " = tex" << texture_type_as_string(type);
     text << "(tex_" << _map_index_height << ", texcoord" << _map_index_height << ".";
-    switch (tex->get_texture_type()) {
+    switch (type) {
     case Texture::TT_cube_map:
     case Texture::TT_3d_texture:
     case Texture::TT_2d_texture_array:
@@ -972,18 +974,17 @@ synthesize_shader(const RenderState *rs, const GeomVertexAnimationSpec &anim) {
       text << " * 2.0 - 1.0)) * " << 0.5 * parallax_mapping_scale << ";\n";
     }
   }
-  for (int i=0; i<_num_textures; i++) {
+  for (int i = 0; i < _num_textures; ++i) {
     if (i != _map_index_height) {
-      Texture *tex = texture->get_on_texture(texture->get_on_stage(i));
-      nassertr(tex != NULL, NULL);
+      Texture::TextureType type = texture->get_on_texture_type(texture->get_on_stage(i));
       // Parallax mapping pushes the texture coordinates of the other textures
       // away from the camera.
       if (_map_index_height >= 0 && parallax_mapping_samples > 0) {
         text << "\t texcoord" << i << ".xyz -= parallax_offset;\n";
       }
-      text << "\t float4 tex" << i << " = tex" << texture_type_as_string(tex->get_texture_type());
+      text << "\t float4 tex" << i << " = tex" << texture_type_as_string(type);
       text << "(tex_" << i << ", texcoord" << i << ".";
-      switch(tex->get_texture_type()) {
+      switch (type) {
       case Texture::TT_cube_map:
       case Texture::TT_3d_texture:
       case Texture::TT_2d_texture_array:
@@ -1263,20 +1264,20 @@ synthesize_shader(const RenderState *rs, const GeomVertexAnimationSpec &anim) {
   }
 
   // Now loop through the textures to compose our magic blending formulas.
-  for (int i=0; i<_num_textures; i++) {
+  for (int i = 0; i < _num_textures; ++i) {
     TextureStage *stage = texture->get_on_stage(i);
     switch (stage->get_mode()) {
     case TextureStage::M_modulate: {
-      int num_components = texture->get_on_texture(texture->get_on_stage(i))->get_num_components();
+      bool affects_rgb = texture->on_stage_affects_rgb(i);
+      bool affects_alpha = texture->on_stage_affects_alpha(i);
 
-      if (num_components == 1) {
-        text << "\t result.a *= tex" << i << ".a;\n";
-      } else if (num_components == 3) {
-        text << "\t result.rgb *= tex" << i << ".rgb;\n";
-      } else {
+      if (affects_rgb && affects_alpha) {
         text << "\t result.rgba *= tex" << i << ".rgba;\n";
+      } else if (affects_alpha) {
+        text << "\t result.a *= tex" << i << ".a;\n";
+      } else if (affects_rgb) {
+        text << "\t result.rgb *= tex" << i << ".rgb;\n";
       }
-
       break; }
     case TextureStage::M_modulate_glow:
     case TextureStage::M_modulate_gloss: