Rewrite handling of stringify and token-paste operators to work better. Add support for variadic macros.

dtool/src/cppparser/cppManifest.cxx

@@ -24,8 +24,8 @@
 //  Description:
 ////////////////////////////////////////////////////////////////////
 CPPManifest::ExpansionNode::
-ExpansionNode(int parm_number) :
+ExpansionNode(int parm_number, bool stringify, bool paste) :
-  _parm_number(parm_number)
+  _parm_number(parm_number), _stringify(stringify), _paste(paste)
 {
 }
 
@@ -35,8 +35,8 @@ ExpansionNode(int parm_number) :
 //  Description:
 ////////////////////////////////////////////////////////////////////
 CPPManifest::ExpansionNode::
-ExpansionNode(const string &str) :
+ExpansionNode(const string &str, bool paste) :
-  _parm_number(-1), _str(str)
+  _parm_number(-1), _stringify(false), _paste(paste), _str(str)
 {
 }
 
@@ -46,7 +46,11 @@ ExpansionNode(const string &str) :
 //  Description:
 ////////////////////////////////////////////////////////////////////
 CPPManifest::
-CPPManifest(const string &args, const CPPFile &file) : _file(file) {
+CPPManifest(const string &args, const CPPFile &file) :
+  _file(file),
+  _variadic_param(-1),
+  _expr((CPPExpression *)NULL)
+{
   assert(!args.empty());
   assert(!isspace(args[0]));
 
@@ -69,6 +73,7 @@ CPPManifest(const string &args, const CPPFile &file) : _file(file) {
     _has_parameters = true;
     parse_parameters(args, p, parameter_names);
     _num_parameters = parameter_names.size();
+
     p++;
   } else {
     _has_parameters = false;
@@ -83,7 +88,6 @@ CPPManifest(const string &args, const CPPFile &file) : _file(file) {
   save_expansion(args.substr(p), parameter_names);
 }
 
-
 ////////////////////////////////////////////////////////////////////
 //     Function: CPPManifest::Destructor
 //       Access: Public
@@ -96,7 +100,58 @@ CPPManifest::
   }
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: CPPManifest::stringify
+//       Access: Public, Static
+//  Description: This implements the stringification operator, #.
+////////////////////////////////////////////////////////////////////
+string CPPManifest::
+stringify(const string &source) {
+  string result("\"");
 
+  enum {
+    S_escaped = 0x01,
+    S_single_quoted = 0x02,
+    S_double_quoted = 0x04,
+    S_quoted = 0x06,
+  };
+  int state = 0;
+
+  string::const_iterator it;
+  for (it = source.begin(); it != source.end(); ++it) {
+    char c = *it;
+
+    if ((state & S_escaped) == 0) {
+      switch (c) {
+      case '\\':
+        if (state & S_quoted) {
+          state |= S_escaped;
+          result += '\\';
+        }
+        break;
+
+      case '\'':
+        state ^= S_single_quoted;
+        break;
+
+      case '"':
+        state ^= S_double_quoted;
+        result += '\\';
+        break;
+      }
+    } else {
+      if (c == '\\' || c == '"') {
+        result += '\\';
+      }
+      state &= ~S_escaped;
+    }
+
+    result += c;
+  }
+
+  result += '"';
+  return result;
+}
 
 ////////////////////////////////////////////////////////////////////
 //     Function: CPPManifest::expand
@@ -111,14 +166,45 @@ expand(const vector_string &args) const {
   for (ei = _expansion.begin(); ei != _expansion.end(); ++ei) {
     if ((*ei)._parm_number >= 0) {
       int i = (*ei)._parm_number;
+
+      string subst;
       if (i < (int)args.size()) {
-        result += " " + args[i] + " ";
+        subst = args[i];
-      } else {
+
-        result += " ";
+        if (i == _variadic_param) {
+          for (++i; i < (int)args.size(); ++i) {
+            subst += ", " + args[i];
+          }
+        }
+        if ((*ei)._stringify) {
+          subst = stringify(subst);
+        }
+      } else if (i == _variadic_param && (*ei)._paste) {
+        // Special case GCC behavior: if __VA_ARGS__ is pasted
+        // to a comma and no arguments are passed, the comma
+        // is removed.  MSVC does this automatically.  Not sure
+        // if we should allow MSVC behavior as well.
+        if (*result.rbegin() == ',') {
+          result.resize(result.size() - 1);
+        }
+      }
+
+      if (!subst.empty()) {
+        if (result.empty() || (*ei)._paste) {
+          result += subst;
+        } else {
+          result += ' ';
+          result += subst;
+        }
       }
     }
     if (!(*ei)._str.empty()) {
-      result += (*ei)._str;
+      if (result.empty() || (*ei)._paste) {
+        result += (*ei)._str;
+      } else {
+        result += ' ';
+        result += (*ei)._str;
+      }
     }
   }
 
@@ -151,20 +237,40 @@ output(ostream &out) const {
   if (_has_parameters) {
     out << "(";
     if (_num_parameters > 0) {
-      out << "$1";
+      if (_variadic_param == 0) {
+        out << "...";
+      } else {
+        out << "$1";
+      }
+
       for (int i = 1; i < _num_parameters; ++i) {
-        out << ", $" << i + 1;
+        if (_variadic_param == i) {
+          out << ", ...";
+        } else {
+          out << ", $" << i + 1;
+        }
       }
     }
     out << ")";
   }
 
-  out << " ";
-
   Expansion::const_iterator ei;
   for (ei = _expansion.begin(); ei != _expansion.end(); ++ei) {
+    if ((*ei)._paste) {
+      out << " ## ";
+    } else {
+      out << " ";
+    }
+
     if ((*ei)._parm_number >= 0) {
-      out << " $" << (*ei)._parm_number + 1 << " ";
+      if (stringify) {
+        out << "#";
+      }
+      if ((*ei)._parm_number == _variadic_param) {
+        out << "__VA_ARGS__";
+      } else {
+        out << "$" << (*ei)._parm_number + 1;
+      }
     }
     if (!(*ei)._str.empty()) {
       out << (*ei)._str;
@@ -172,8 +278,6 @@ output(ostream &out) const {
   }
 }
 
-
-
 ////////////////////////////////////////////////////////////////////
 //     Function: CPPManifest::parse_parameters
 //       Access: Private
@@ -197,7 +301,16 @@ parse_parameters(const string &args, size_t &p,
            args[p] != ')' && args[p] != ',') {
       p++;
     }
-    parameter_names.push_back(args.substr(q, p - q));
+
+    // Check if it's a variadic parameter by checking if it ends
+    // with "...".  This picks up both C99-style variadic macros
+    // and GCC-style variadic macros.
+    if (p - q >= 3 && args.compare(p - 3, 3, "...") == 0) {
+      _variadic_param = parameter_names.size();
+      parameter_names.push_back(args.substr(q, p - q - 3));
+    } else {
+      parameter_names.push_back(args.substr(q, p - q));
+    }
 
     // Skip whitespace after the parameter name.
     while (p < args.size() && isspace(args[p])) {
@@ -221,16 +334,12 @@ parse_parameters(const string &args, size_t &p,
 ////////////////////////////////////////////////////////////////////
 void CPPManifest::
 save_expansion(const string &exp, const vector_string &parameter_names) {
-  if (parameter_names.empty()) {
-    // No parameters; this is an easy case.
-    _expansion.push_back(ExpansionNode(exp));
-    return;
-  }
-
   // Walk through the expansion string.  For each substring that is an
   // identifier, check it against parameter_names.
   size_t p = 0;
   size_t last = 0;
+  bool stringify = false;
+  bool paste = false;
   while (p < exp.size()) {
     if (isalpha(exp[p]) || exp[p] == '_') {
       // Here's the start of an identifier.  Find the end of it.
@@ -244,26 +353,67 @@ save_expansion(const string &exp, const vector_string &parameter_names) {
 
       // Is this identifier one of our parameters?
       int pnum = -1;
-      for (int i = 0; pnum == -1 && i < (int)parameter_names.size(); ++i) {
+      bool va_args = false;
-        if (parameter_names[i] == ident) {
+
-          pnum = i;
+      if (ident == "__VA_ARGS__") {
+        va_args = true;
+        // C99-style variadics, ie. #define macro(...) __VA_ARGS__
+        pnum = _variadic_param;
+
+      } else {
+        for (int i = 0; pnum == -1 && i < (int)parameter_names.size(); ++i) {
+          const string &pname = parameter_names[i];
+          if (pname == ident) {
+            pnum = i;
+          }
         }
       }
 
       if (pnum != -1) {
         // Yep!
         if (last != q) {
-          _expansion.push_back(ExpansionNode(exp.substr(last, q - last)));
+          _expansion.push_back(ExpansionNode(exp.substr(last, q - last), paste));
+          paste = false;
         }
-        _expansion.push_back(pnum);
+        _expansion.push_back(ExpansionNode(pnum, stringify, paste));
+        stringify = false;
+        paste = false;
         last = p;
       }
+    } else if (exp[p] == '#') {
+      // This may be a stringification operator.
+      if (last != p) {
+        _expansion.push_back(ExpansionNode(exp.substr(last, p - last), paste));
+        paste = false;
+      }
+
+      ++p;
+
+      if (p < exp.size() && exp[p] == '#') {
+        // Woah, this is a token-pasting operator.
+        paste = true;
+        ++p;
+      } else {
+        // Mark that the next argument should be stringified.
+        stringify = true;
+      }
+      last = p;
+
+    } else if (isspace(exp[p])) {
+      if (last != p) {
+        _expansion.push_back(ExpansionNode(exp.substr(last, p - last), paste));
+        paste = false;
+      }
+
+      ++p;
+      last = p;
+
     } else {
-      p++;
+      ++p;
     }
   }
 
   if (last != p) {
-    _expansion.push_back(exp.substr(last, p - last));
+    _expansion.push_back(ExpansionNode(exp.substr(last, p - last), paste));
   }
 }

dtool/src/cppparser/cppManifest.h

@@ -33,6 +33,8 @@ class CPPManifest {
 public:
   CPPManifest(const string &args, const CPPFile &file = CPPFile());
   ~CPPManifest();
+
+  static string stringify(const string &source);
   string expand(const vector_string &args = vector_string()) const;
 
   CPPType *determine_type() const;
@@ -42,6 +44,7 @@ public:
   string _name;
   bool _has_parameters;
   int _num_parameters;
+  int _variadic_param;
   CPPFile _file;
   CPPExpression *_expr;
 
@@ -59,9 +62,11 @@ private:
 
   class ExpansionNode {
   public:
-    ExpansionNode(int parm_number);
+    ExpansionNode(int parm_number, bool stringify, bool paste);
-    ExpansionNode(const string &str);
+    ExpansionNode(const string &str, bool paste = false);
     int _parm_number;
+    bool _stringify;
+    bool _paste;
     string _str;
   };
   typedef vector<ExpansionNode> Expansion;

dtool/src/cppparser/cppPreprocessor.cxx

@@ -315,23 +315,6 @@ get_next_token0() {
   int first_col = token._lloc.first_column;
   CPPFile first_file = token._lloc.file;
 
-  if (token._token == '#') {
-    // Stringify.
-    token = internal_get_next_token();
-    if (token._token == SIMPLE_IDENTIFIER || 
-        token._token == INTEGER ||
-        token._token == REAL ||
-        token._token == STRING) {
-      token._token = STRING;
-    } else {
-      // Stringify nothing.
-
-      _saved_tokens.push_back(token);
-      token._token = STRING;
-      token._lval.str = "";
-    }
-  }
-
   if (_resolve_identifiers &&
       (token._token == SIMPLE_IDENTIFIER || token._token == SCOPE)) {
     // We will be returning a scoped identifier, or a scoping.  Keep
@@ -366,59 +349,41 @@ get_next_token0() {
       }
     }
 
-    while (token._token == SCOPE || token._token == TOKENPASTE) {
+    while (token._token == SCOPE) {
-      if (token._token == TOKENPASTE) {
+      name += "::";
-        // The token-pasting operator creates one continuous
+      token = internal_get_next_token();
-        // identifier across whitespace.
+      string token_prefix;
-        token = internal_get_next_token();
-        if (token._token == SIMPLE_IDENTIFIER ||
-            token._token == INTEGER ||
-            token._token == REAL) {
-          name += token._lval.str;
-          ident->_names.back().append_name(token._lval.str);
-
-          token = internal_get_next_token();
-
-        } else {
-          // Token-paste with nothing.
-        }
-
-      } else { // token._token == SCOPE
-        name += "::";
-        token = internal_get_next_token();
-        string token_prefix;
-
-        if (token._token == '~') {
-          // A scoping operator followed by a tilde can only be the
-          // start of a scoped destructor name.  Make the tilde be part
-          // of the name.
-          name += "~";
-          token_prefix = "~";
-          token = internal_get_next_token();
-        }
-
-        if (token._token != SIMPLE_IDENTIFIER) {
-          // The last useful token was a SCOPE, thus this is a scoping
-          // token.
-
-          if (token._token == KW_OPERATOR) {
-            // Unless the last token we came across was the "operator"
-            // keyword.  We make a special case for this, because it's
-            // occasionally scoped in normal use.
-            token._lval = result;
-            return token;
-          }
-          _saved_tokens.push_back(token);
-          return CPPToken(SCOPING, first_line, first_col, first_file,
-                          name, result);
-        }
-
-        name += token._lval.str;
-        ident->_names.push_back(token_prefix + token._lval.str);
 
+      if (token._token == '~') {
+        // A scoping operator followed by a tilde can only be the
+        // start of a scoped destructor name.  Make the tilde be part
+        // of the name.
+        name += "~";
+        token_prefix = "~";
         token = internal_get_next_token();
       }
 
+      if (token._token != SIMPLE_IDENTIFIER) {
+        // The last useful token was a SCOPE, thus this is a scoping
+        // token.
+
+        if (token._token == KW_OPERATOR) {
+          // Unless the last token we came across was the "operator"
+          // keyword.  We make a special case for this, because it's
+          // occasionally scoped in normal use.
+          token._lval = result;
+          return token;
+        }
+        _saved_tokens.push_back(token);
+        return CPPToken(SCOPING, first_line, first_col, first_file,
+                        name, result);
+      }
+
+      name += token._lval.str;
+      ident->_names.push_back(token_prefix + token._lval.str);
+
+      token = internal_get_next_token();
+
       if (token._token == '<') {
         // If the next token is an angle bracket and the current
         // indentifier wants template instantiation, assume the angle
@@ -887,11 +852,6 @@ internal_get_next_token() {
   case '%':
     if (next_c == '=') return CPPToken(MODEQUAL, first_line, first_col, first_file);
     break;
-
-    // These are actually preprocessor operators, but it's useful to
-    // treat them as tokens.
-  case '#':
-    if (next_c == '#') return CPPToken(TOKENPASTE, first_line, first_col, first_file);
   }
 
   // It wasn't any of the two- or three-character tokens, so put back
@@ -945,11 +905,13 @@ skip_whitespace(int c) {
     c = skip_comment(c);
 
     if (c == '\\') {
-      // A backslash character is an unusual thing to encounter in the
+      // This does not usually occur in the middle of unquoted C++
-      // middle of unquoted C++ code.  But it seems to be legal, and
+      // code, except before a newline character.
-      // it seems to mean the same thing it does within quotes: to
-      // escape the following character.  We simply ignore it.
       c = get();
+      if (c != '\n') {
+        unget(c);
+        return '\\';
+      }
     }
 
     if (!isspace(c)) {
@@ -1633,7 +1595,8 @@ expand_manifest(const CPPManifest *manifest) {
 
   if (manifest->_has_parameters) {
     // Hmm, we're expecting arguments.
-    extract_manifest_args(manifest->_name, manifest->_num_parameters, args);
+    extract_manifest_args(manifest->_name, manifest->_num_parameters,
+                          manifest->_variadic_param, args);
   }
 
   string expanded = " " + manifest->expand(args) + " ";
@@ -1659,7 +1622,7 @@ expand_manifest(const CPPManifest *manifest) {
 //  Description:
 ////////////////////////////////////////////////////////////////////
 void CPPPreprocessor::
-extract_manifest_args(const string &name, int num_args,
+extract_manifest_args(const string &name, int num_args, int va_arg,
                       vector_string &args) {
   CPPFile first_file = get_file();
   int first_line = get_line_number();
@@ -1683,12 +1646,14 @@ extract_manifest_args(const string &name, int num_args,
 
   } else {
     // Skip paren.
-    c = get();
+    c = skip_whitespace(get());
+    int paren_level = 1;
     string arg;
-    while (c != EOF && c != ')') {
+    while (c != EOF) {
-      if (c == ',') {
+      if (c == ',' && paren_level == 1) {
         args.push_back(arg);
         arg = "";
+        c = get();
 
       } else if (c == '"' || c == '\'') {
         // Quoted string or character.
@@ -1706,35 +1671,55 @@ extract_manifest_args(const string &name, int num_args,
           }
         }
         arg += c;
+        c = get();
 
       } else if (c == '(') {
-        // Nested parens.
+        arg += '(';
-        int paren_level = 1;
+        ++paren_level;
-        while (c != EOF && paren_level > 0) {
+        c = get();
-          arg += c;
+
-          c = get();
+      } else if (c == ')') {
-          if (c == '(') {
+        --paren_level;
-            paren_level++;
+        if (paren_level == 0) {
-          } else if (c == ')') {
+          break;
-            paren_level--;
-          }
         }
-        if (c != EOF) {
+        arg += ')';
-          arg += c;
+        c = get();
+
+      } else if (isspace(c)) {
+        // Skip extra whitespace.
+        c = skip_whitespace(c);
+        if (!arg.empty()) {
+          arg += ' ';
+        }
+
+      } else if (c == '\\') {
+        // It could be a slash before a newline.
+        // If so, that's whitespace as well.
+        c = get();
+        if (c != '\n') {
+          arg += '\\';
+        } else if (!arg.empty()) {
+          arg += ' ';
+          c = skip_whitespace(get());
         }
 
       } else {
         arg += c;
+        c = get();
       }
-      c = get();
     }
     if (num_args != 0 || !arg.empty()) {
       args.push_back(arg);
     }
   }
 
-  if ((int)args.size() != num_args) {
+  if ((int)args.size() < num_args) {
-    warning("Wrong number of arguments for manifest " + name,
+    warning("Not enough arguments for manifest " + name,
+            first_line, first_col, first_file);
+
+  } else if (va_arg < 0 && (int)args.size() > num_args) {
+    warning("Too many arguments for manifest " + name,
             first_line, first_col, first_file);
   }
 }
@@ -1742,14 +1727,15 @@ extract_manifest_args(const string &name, int num_args,
 ////////////////////////////////////////////////////////////////////
 //     Function: CPPPreprocessor::expand_defined_function
 //       Access: Private
-//  Description:
+//  Description: Expands the defined(manifest) function to either
+//               1 or 0, depending on whether the manifest exists.
 ////////////////////////////////////////////////////////////////////
 void CPPPreprocessor::
 expand_defined_function(string &expr, size_t q, size_t &p) {
   string result;
 
   vector_string args;
-  extract_manifest_args_inline("defined", 1, args, expr, p);
+  extract_manifest_args_inline("defined", 1, -1, args, expr, p);
   if (args.size() >= 1) {
     const string &manifest_name = args[0];
     Manifests::const_iterator mi = _manifests.find(manifest_name);
@@ -1777,7 +1763,7 @@ expand_manifest_inline(string &expr, size_t q, size_t &p,
   vector_string args;
   if (manifest->_has_parameters) {
     extract_manifest_args_inline(manifest->_name, manifest->_num_parameters,
-                                 args, expr, p);
+                                 manifest->_variadic_param, args, expr, p);
   }
   string result = manifest->expand(args);
 
@@ -1792,7 +1778,7 @@ expand_manifest_inline(string &expr, size_t q, size_t &p,
 ////////////////////////////////////////////////////////////////////
 void CPPPreprocessor::
 extract_manifest_args_inline(const string &name, int num_args,
-                             vector_string &args,
+                             int va_arg, vector_string &args,
                              const string &expr, size_t &p) {
   // Skip whitespace till paren.
   while (p < expr.size() && isspace(expr[p])) {
@@ -1850,8 +1836,11 @@ extract_manifest_args_inline(const string &name, int num_args,
     }
   }
 
-  if ((int)args.size() != num_args) {
+  if ((int)args.size() < num_args) {
-    warning("Wrong number of arguments for manifest " + name);
+    warning("Not enough arguments for manifest " + name);
+
+  } else if (va_arg < 0 && (int)args.size() > num_args) {
+    warning("Too many arguments for manifest " + name);
   }
 }
 
@@ -2035,6 +2024,90 @@ check_keyword(const string &name) {
   return 0;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: CPPPreprocessor::scan_escape_sequence
+//       Access: Private
+//  Description:
+////////////////////////////////////////////////////////////////////
+int CPPPreprocessor::
+scan_escape_sequence(int c) {
+  if (c != '\\') {
+    return c;
+  }
+
+  c = get();
+  switch (c) {
+  case 'a':
+    return '\a';
+
+  case 'b':
+    return '\b';
+
+  case 'f':
+    return '\f';
+
+  case 'n':
+    return '\n';
+
+  case 'r':
+    return '\r';
+
+  case 't':
+    return '\t';
+
+  case 'v':
+    return '\v';
+
+  case 'e':
+    // \e is non-standard, buT GCC supports it.
+    return '\x1B';
+
+  case 'x':
+    // hex character.
+    c = get();
+    if (isxdigit(c)) {
+      int val = hex_val(c);
+      c = get();
+      if (isxdigit(c)) {
+        val = (val << 4) | hex_val(c);
+      } else {
+        unget(c);
+      }
+      return val;
+    }
+    break;
+
+  case '0':
+  case '1':
+  case '2':
+  case '3':
+  case '4':
+  case '5':
+  case '6':
+  case '7':
+    // Octal character.
+    {
+      int val = (c - '0');
+      c = get();
+      if (c >= '0' && c <= '7') {
+        val = (val << 3) | (c - '0');
+        c = get();
+        if (c >= '0' && c <= '7') {
+          val = (val << 3) | (c - '0');
+        } else {
+          unget(c);
+        }
+      } else {
+        unget(c);
+      }
+      return val;
+    }
+  }
+
+  // Simply output the following character.
+  return c;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: CPPPreprocessor::scan_quoted
 //       Access: Private
@@ -2049,62 +2122,7 @@ scan_quoted(int c) {
   while (c != EOF && c != '\n' && c != quote_mark) {
     if (c == '\\') {
       // Backslash means a special character follows.
-      c = get();
+      c = scan_escape_sequence(c);
-      switch (c) {
-      case 'n':
-        c = '\n';
-        break;
-
-      case 't':
-        c = '\t';
-        break;
-
-      case 'r':
-        c = '\r';
-        break;
-
-      case 'x':
-        // hex character.
-        c = get();
-        if (isxdigit(c)) {
-          int val = hex_val(c);
-          c = get();
-          if (isxdigit(c)) {
-            val = (val << 4) | hex_val(c);
-          } else {
-            unget(c);
-          }
-          c = val;
-        }
-        break;
-
-      case '0':
-      case '1':
-      case '2':
-      case '3':
-      case '4':
-      case '5':
-      case '6':
-      case '7':
-        // Octal character.
-        {
-          int val = (c - '0');
-          c = get();
-          if (c >= '0' && c <= '7') {
-            val = (val << 3) | (c - '0');
-            c = get();
-            if (c >= '0' && c <= '7') {
-              val = (val << 3) | (c - '0');
-            } else {
-              unget(c);
-            }
-          } else {
-            unget(c);
-          }
-          c = val;
-        }
-        break;
-      }
     }
 
     str += c;

dtool/src/cppparser/cppPreprocessor.h

@@ -142,16 +142,17 @@ private:
   CPPToken get_identifier(int c);
   CPPToken expand_manifest(const CPPManifest *manifest);
   void extract_manifest_args(const string &name, int num_args,
-                             vector_string &args);
+                             int va_arg, vector_string &args);
   void expand_defined_function(string &expr, size_t q, size_t &p);
   void expand_manifest_inline(string &expr, size_t q, size_t &p,
                               const CPPManifest *manifest);
   void extract_manifest_args_inline(const string &name, int num_args,
-                                    vector_string &args,
+                                    int va_arg, vector_string &args,
                                     const string &expr, size_t &p);
 
   CPPToken get_number(int c, int c2 = 0);
   static int check_keyword(const string &name);
+  int scan_escape_sequence(int c);
   string scan_quoted(int c);
 
   bool should_ignore_manifest(const CPPManifest *manifest) const;