Improve GLSL preprocessor; support GL_GOOGLE_include_directive

panda/src/gobj/shader.cxx

@@ -2385,7 +2385,7 @@ do_read_source(string &into, const Filename &fn, BamCacheRecord *record) {
 
 /**
  * Loads a given GLSL file line by line, and processes any #pragma include and
- * once statements.
+ * once statements, as well as removes any comments.
  *
  * The set keeps track of which files we have already included, for checking
  * recursive includes.
@@ -2398,7 +2398,8 @@ r_preprocess_source(ostream &out, const Filename &fn,
 
   if (depth > glsl_include_recursion_limit) {
     shader_cat.error()
-      << "#pragma include nested too deeply\n";
+      << "GLSL includes nested too deeply, raise glsl-include-recursion-limit"
+         " if necessary\n";
     return false;
   }
 
@@ -2459,56 +2460,226 @@ r_preprocess_source(ostream &out, const Filename &fn,
 
   // Iterate over the lines for things we may need to preprocess.
   string line;
+  int ext_google_include = 0; // 1 = warn, 2 = enable
+  int ext_google_line = 0;
   bool had_include = false;
   int lineno = 0;
   while (getline(*source, line)) {
     ++lineno;
 
-    // Check if this line contains a #pragma.
+    if (line.empty()) {
-    char pragma[64];
+      out.put('\n');
-    if (line.size() < 8 ||
-        sscanf(line.c_str(), " # pragma %63s", pragma) != 1) {
-      // Just pass the line through unmodified.
-      out << line << "\n";
-
-      // One exception: check for an #endif after an include.  We have to
-      // restore the line number in case the include happened under an #if
-      // block.
-      int nread = 0;
-      if (had_include && sscanf(line.c_str(), " # endif %n", &nread) == 0 && nread >= 6) {
-        out << "#line " << (lineno + 1) << " " << fileno << "\n";
-      }
       continue;
     }
 
-    int nread = 0;
+    // If the line ends with a backslash, concatenate the following line.
-    if (strcmp(pragma, "include") == 0) {
+    // Preprocessor definitions may be broken up into multiple lines.
-      // Allow both double quotes and angle brackets.
+    while (line[line.size() - 1] == '\\') {
-      Filename incfn, source_dir;
+      line.resize(line.size() - 1);
-      {
+      string line2;
-        char incfile[2048];
-        if (sscanf(line.c_str(), " # pragma%*[ \t]include \"%2047[^\"]\" %n", incfile, &nread) == 1
-            && nread == line.size()) {
-          // A regular include, with double quotes.  Probably a local file.
-          source_dir = full_fn.get_dirname();
-          incfn = incfile;
 
-        } else if (sscanf(line.c_str(), " # pragma%*[ \t]include <%2047[^\"]> %n", incfile, &nread) == 1
+      if (getline(*source, line2)) {
-            && nread == line.size()) {
+        line += line2;
-          // Angled includes are also OK, but we don't search in the directory
+        out.put('\n');
-          // of the source file.
+        ++lineno;
-          incfn = incfile;
+      } else {
+        break;
+      }
+    }
 
+    // Look for comments to strip.  This is necessary because comments may
+    // appear in the middle of or around a preprocessor definition.
+    size_t line_comment = line.find("//");
+    size_t block_comment = line.find("/*");
+    if (line_comment < block_comment) {
+      // A line comment - strip off the rest of the line.
+      line.resize(line_comment);
+
+    } else if (block_comment < line_comment) {
+      // A block comment.  Search for closing block.
+      string line2 = line.substr(block_comment + 2);
+
+      // According to the GLSL specification, a block comment is replaced with
+      // a single whitespace character.
+      line.resize(block_comment);
+      line += ' ';
+
+      size_t block_end = line2.find("*/");
+      while (block_end == string::npos) {
+        // Didn't find it - look in the next line.
+        if (getline(*source, line2)) {
+          out.put('\n');
+          ++lineno;
+          block_end = line2.find("*/");
         } else {
-          // Couldn't parse it.
           shader_cat.error()
-            << "Malformed #pragma include at line " << lineno
+            << "Expected */ before end of file " << fn << "\n";
-            << " of file " << fn << ":\n  " << line << "\n";
           return false;
         }
       }
 
+      line += line2.substr(block_end + 2);
+    }
+
+    // Check if this line contains a #directive.
+    char directive[64];
+    if (line.size() < 8 || sscanf(line.c_str(), " # %63s", directive) != 1) {
+      // Nope.  Just pass the line through unmodified.
+      out << line << "\n";
+      continue;
+    }
+
+    char pragma[64];
+    int nread = 0;
+    // What kind of directive is it?
+    if (strcmp(directive, "pragma") == 0 &&
+        sscanf(line.c_str(), " # pragma %63s", pragma) == 1) {
+      if (strcmp(pragma, "include") == 0) {
+        // Allow both double quotes and angle brackets.
+        Filename incfn, source_dir;
+        {
+          char incfile[2048];
+          if (sscanf(line.c_str(), " # pragma%*[ \t]include \"%2047[^\"]\" %n", incfile, &nread) == 1
+              && nread == line.size()) {
+            // A regular include, with double quotes.  Probably a local file.
+            source_dir = full_fn.get_dirname();
+            incfn = incfile;
+
+          } else if (sscanf(line.c_str(), " # pragma%*[ \t]include <%2047[^\"]> %n", incfile, &nread) == 1
+              && nread == line.size()) {
+            // Angled includes are also OK, but we don't search in the directory
+            // of the source file.
+            incfn = incfile;
+
+          } else {
+            // Couldn't parse it.
+            shader_cat.error()
+              << "Malformed #pragma include at line " << lineno
+              << " of file " << fn << ":\n  " << line << "\n";
+            return false;
+          }
+        }
+
+        // OK, great.  Process the include.
+        if (!r_preprocess_source(out, incfn, source_dir, once_files, record, depth + 1)) {
+          // An error occurred.  Pass on the failure.
+          shader_cat.error(false) << "included at line "
+            << lineno << " of file " << fn << ":\n  " << line << "\n";
+          return false;
+        }
+
+        // Restore the line counter.
+        out << "#line " << (lineno + 1) << " " << fileno << " // " << fn << "\n";
+        had_include = true;
+
+      } else if (strcmp(pragma, "once") == 0) {
+        // Do a stricter syntax check, just to be extra safe.
+        if (sscanf(line.c_str(), " # pragma%*[ \t]once %n", &nread) != 0 ||
+            nread != line.size()) {
+          shader_cat.error()
+            << "Malformed #pragma once at line " << lineno
+            << " of file " << fn << ":\n  " << line << "\n";
+          return false;
+        }
+
+        once_files.insert(full_fn);
+
+      } else {
+        // Forward it, the driver will ignore it if it doesn't know it.
+        out << line << "\n";
+      }
+
+    } else if (strcmp(directive, "endif") == 0) {
+      // Check for an #endif after an include.  We have to restore the line
+      // number in case the include happened under an #if block.
+      out << line << "\n";
+      int nread = 0;
+      if (had_include) {
+        out << "#line " << (lineno + 1) << " " << fileno << "\n";
+      }
+
+    } else if (strcmp(directive, "extension") == 0) {
+      // Check for special preprocessing extensions.
+      char extension[256];
+      char behavior[9];
+      if (sscanf(line.c_str(), " # extension%*[ \t]%255s : %8s", extension, behavior) == 2) {
+        // Parse the behavior string.
+        int mode;
+        if (strcmp(behavior, "require") == 0 || strcmp(behavior, "enable") == 0) {
+          mode = 2;
+        } else if (strcmp(behavior, "warn") == 0) {
+          mode = 1;
+        } else if (strcmp(behavior, "disable") == 0) {
+          mode = 0;
+        } else {
+          shader_cat.error()
+            << "Extension directive specifies invalid behavior at line "
+            << lineno << " of file " << fn << ":\n  " << line << "\n";
+          return false;
+        }
+
+        if (strcmp(extension, "all") == 0) {
+          if (mode == 2) {
+            shader_cat.error()
+              << "Extension directive for 'all' may only specify 'warn' or "
+                 "'disable' at line " << lineno << " of file " << fn
+              << ":\n  " << line << "\n";
+            return false;
+          }
+          ext_google_include = mode;
+          ext_google_line = mode;
+          out << line << "\n";
+
+        } else if (strcmp(extension, "GL_GOOGLE_include_directive") == 0) {
+          // Enable the Google extension support for #include statements.
+          // This also implicitly enables GL_GOOGLE_cpp_style_line_directive.
+          // This matches the behavior of Khronos' glslang reference compiler.
+          ext_google_include = mode;
+          ext_google_line = mode;
+
+        } else if (strcmp(extension, "GL_GOOGLE_cpp_style_line_directive") == 0) {
+          // Enables strings in #line statements.
+          ext_google_line = mode;
+
+        } else {
+          // It's an extension the driver should worry about.
+          out << line << "\n";
+        }
+      } else {
+        shader_cat.error()
+          << "Failed to parse extension directive at line "
+          << lineno << " of file " << fn << ":\n  " << line << "\n";
+        return false;
+      }
+    } else if (ext_google_include > 0 && strcmp(directive, "include") == 0) {
+      // Warn about extension use if requested.
+      if (ext_google_include == 1) {
+        shader_cat.warning()
+          << "Extension GL_GOOGLE_include_directive is being used at line "
+          << lineno << " of file " << fn
+#ifndef NDEBUG
+          << ":\n  " << line
+#endif
+          << "\n";
+      }
+
+      // This syntax allows only double quotes, not angle brackets.
+      Filename incfn;
+      {
+        char incfile[2048];
+        if (sscanf(line.c_str(), " # include%*[ \t]\"%2047[^\"]\" %n", incfile, &nread) != 1
+            || nread != line.size()) {
+          // Couldn't parse it.
+          shader_cat.error()
+            << "Malformed #include at line " << lineno
+            << " of file " << fn << ":\n  " << line << "\n";
+          return false;
+        }
+        incfn = incfile;
+      }
+
       // OK, great.  Process the include.
+      Filename source_dir = full_fn.get_dirname();
       if (!r_preprocess_source(out, incfn, source_dir, once_files, record, depth + 1)) {
         // An error occurred.  Pass on the failure.
         shader_cat.error(false) << "included at line "
@@ -2520,28 +2691,36 @@ r_preprocess_source(ostream &out, const Filename &fn,
       out << "#line " << (lineno + 1) << " " << fileno << " // " << fn << "\n";
       had_include = true;
 
-    } else if (strcmp(pragma, "once") == 0) {
+    } else if (ext_google_line > 0 && strcmp(directive, "line") == 0) {
-      // Do a stricter syntax check, just to be extra safe.
+      // It's a #line directive.  See if it uses a string instead of number.
-      if (sscanf(line.c_str(), " # pragma%*[ \t]once %n", &nread) != 0 ||
+      char filestr[2048];
-          nread != line.size()) {
+      if (sscanf(line.c_str(), " # line%*[ \t]%d%*[ \t]\"%2047[^\"]\" %n", &lineno, filestr, &nread) == 2
-        shader_cat.error()
+          && nread == line.size()) {
-          << "Malformed #pragma once at line " << lineno
+        // Warn about extension use if requested.
-          << " of file " << fn << ":\n  " << line << "\n";
+        if (ext_google_line == 1) {
-        return false;
+          shader_cat.warning()
+            << "Extension GL_GOOGLE_cpp_style_line_directive is being used at line "
+            << lineno << " of file " << fn
+#ifndef NDEBUG
+            << ":\n  " << line
+#endif
+            << "\n";
+        }
+
+        // Replace the string line number with an integer.  This is something
+        // we can substitute later when parsing the GLSL log from the driver.
+        fileno = 2048 + _included_files.size();
+        _included_files.push_back(Filename(filestr));
+
+        out << "#line " << lineno << " " << fileno << " // " << filestr << "\n";
+
+      } else {
+        // We couldn't parse the #line directive.  Pass it through unmodified.
+        out << line << "\n";
       }
-
-      once_files.insert(full_fn);
-
-    } else if (strcmp(pragma, "optionNV") == 0) {
-      // This is processed by NVIDIA drivers.  Don't touch it.
-      out << line << "\n";
-
     } else {
-      // Forward it, the driver will ignore it if it doesn't know it.
+      // Different directive (eg. #version).  Leave it untouched.
       out << line << "\n";
-      shader_cat.warning()
-        << "Ignoring unknown pragma directive \"" << pragma << "\" at line "
-        << lineno << " of file " << fn << ":\n  " << line << "\n";
     }
   }