add unicode decoding

panda/src/text/Sources.pp

@@ -18,6 +18,7 @@
     dynamicTextPage.I dynamicTextPage.h \
     geomTextGlyph.I geomTextGlyph.h \
     staticTextFont.I staticTextFont.h \
+    stringDecoder.I stringDecoder.h \
     textFont.I textFont.h \
     textGlyph.I textGlyph.h \
     textNode.I textNode.h textNode.cxx
@@ -28,6 +29,7 @@
     dynamicTextGlyph.cxx \
     dynamicTextPage.cxx \
     geomTextGlyph.cxx \
+    stringDecoder.cxx \
     staticTextFont.cxx \
     textFont.cxx textGlyph.cxx
 
@@ -38,6 +40,7 @@
     dynamicTextPage.I dynamicTextPage.h \
     geomTextGlyph.I geomTextGlyph.h \
     staticTextFont.I staticTextFont.h \
+    stringDecoder.I stringDecoder.h \
     textFont.I textFont.h \
     textGlyph.I textGlyph.h \
     textNode.I textNode.h

panda/src/text/dynamicTextFont.cxx

@@ -98,10 +98,20 @@ DynamicTextFont(const Filename &font_filename, int face_index) {
       }
       set_name(name);
 
-      text_cat.info()
+      if (!FT_IS_SCALABLE(_face)) {
-        << "Loaded font " << get_name() << "\n";
+        text_cat.error()
-      _is_valid = true;
+          << "Unable to read font " << get_name()
-      reset_scale();
+          << ": non-scalable fonts not supported.\n";
+        // Although we could support these if we wanted to, just
+        // haven't bothered to write the few lines of glue code that
+        // would do it.
+
+      } else {
+        text_cat.info()
+          << "Loaded font " << get_name() << "\n";
+        _is_valid = true;
+        reset_scale();
+      }
     }
   }
 }

panda/src/text/stringDecoder.I

@@ -0,0 +1,58 @@
+// Filename: stringDecoder.I
+// Created by:  drose (11Feb02)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
+//
+// All use of this software is subject to the terms of the Panda 3d
+// Software license.  You should have received a copy of this license
+// along with this source code; you will also find a current copy of
+// the license at http://www.panda3d.org/license.txt .
+//
+// To contact the maintainers of this program write to
+// panda3d@yahoogroups.com .
+//
+////////////////////////////////////////////////////////////////////
+
+
+////////////////////////////////////////////////////////////////////
+//     Function: StringDecoder::Constructor
+//       Access: Public
+//  Description: 
+////////////////////////////////////////////////////////////////////
+INLINE StringDecoder::
+StringDecoder(const string &input) : _input(input) {
+  _p = 0;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: StringDecoder::is_eof
+//       Access: Public
+//  Description: Returns true if the decoder has returned the last
+//               character in the string, false if there are more to
+//               go.
+////////////////////////////////////////////////////////////////////
+INLINE bool StringDecoder::
+is_eof() {
+  return (_p >= _input.size());
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: StringUtf8Decoder::Constructor
+//       Access: Public
+//  Description: 
+////////////////////////////////////////////////////////////////////
+INLINE StringUtf8Decoder::
+StringUtf8Decoder(const string &input) : StringDecoder(input) {
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: StringUnicodeDecoder::Constructor
+//       Access: Public
+//  Description: 
+////////////////////////////////////////////////////////////////////
+INLINE StringUnicodeDecoder::
+StringUnicodeDecoder(const string &input) : StringDecoder(input) {
+}

panda/src/text/stringDecoder.cxx

@@ -0,0 +1,120 @@
+// Filename: stringDecoder.cxx
+// Created by:  drose (11Feb02)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
+//
+// All use of this software is subject to the terms of the Panda 3d
+// Software license.  You should have received a copy of this license
+// along with this source code; you will also find a current copy of
+// the license at http://www.panda3d.org/license.txt .
+//
+// To contact the maintainers of this program write to
+// panda3d@yahoogroups.com .
+//
+////////////////////////////////////////////////////////////////////
+
+#include "stringDecoder.h"
+
+////////////////////////////////////////////////////////////////////
+//     Function: StringDecoder::Destructor
+//       Access: Public, Virtual
+//  Description: 
+////////////////////////////////////////////////////////////////////
+StringDecoder::
+~StringDecoder() {
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: StringDecoder::get_next_character
+//       Access: Public, Virtual
+//  Description: Returns the next character in sequence.
+////////////////////////////////////////////////////////////////////
+int StringDecoder::
+get_next_character() {
+  if (is_eof()) {
+    return -1;
+  }
+  return (unsigned char)_input[_p++];
+}
+
+/*
+In UTF-8, each 16-bit Unicode character is encoded as a sequence of
+one, two, or three 8-bit bytes, depending on the value of the
+character. The following table shows the format of such UTF-8 byte
+sequences (where the "free bits" shown by x's in the table are
+combined in the order shown, and interpreted from most significant to
+least significant):
+
+ Binary format of bytes in sequence:
+                                        Number of    Maximum expressible
+ 1st byte     2nd byte    3rd byte      free bits:      Unicode value:
+
+ 0xxxxxxx                                  7           007F hex   (127)
+ 110xxxxx     10xxxxxx                  (5+6)=11       07FF hex  (2047)
+ 1110xxxx     10xxxxxx    10xxxxxx     (4+6+6)=16      FFFF hex (65535)
+
+The value of each individual byte indicates its UTF-8 function, as follows:
+
+ 00 to 7F hex   (0 to 127):  first and only byte of a sequence.
+ 80 to BF hex (128 to 191):  continuing byte in a multi-byte sequence.
+ C2 to DF hex (194 to 223):  first byte of a two-byte sequence.
+ E0 to EF hex (224 to 239):  first byte of a three-byte sequence.
+*/
+
+////////////////////////////////////////////////////////////////////
+//     Function: StringUtf8Decoder::get_next_character
+//       Access: Public, Virtual
+//  Description: Returns the next character in sequence.
+////////////////////////////////////////////////////////////////////
+int StringUtf8Decoder::
+get_next_character() {
+  if (is_eof()) {
+    return -1;
+  }
+
+  unsigned int result = (unsigned char)_input[_p++];
+  if ((result & 0xe0) == 0xc0) {
+    // First byte of two.
+    unsigned int two = 0;
+    if (!is_eof()) {
+      two = (unsigned char)_input[_p++];
+    }
+    result = ((result & 0x1f) << 6) | (two & 0x3f);
+
+  } else if ((result & 0xf0) == 0xe0) {
+    // First byte of three.
+    unsigned int two = 0;
+    unsigned int three = 0;
+    if (!is_eof()) {
+      two = (unsigned char)_input[_p++];
+    }
+    if (!is_eof()) {
+      three = (unsigned char)_input[_p++];
+    }
+    result = ((result & 0x0f) << 12) | ((two & 0x3f) << 6) | (three & 0x3f);
+  } 
+
+  return result;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: StringUnicodeDecoder::get_next_character
+//       Access: Public, Virtual
+//  Description: Returns the next character in sequence.
+////////////////////////////////////////////////////////////////////
+int StringUnicodeDecoder::
+get_next_character() {
+  if (is_eof()) {
+    return -1;
+  }
+
+  unsigned int high = (unsigned char)_input[_p++];
+  unsigned int low = 0;
+  if (!is_eof()) {
+    low = (unsigned char)_input[_p++];
+  }
+  return ((high << 8) | low);
+}

panda/src/text/stringDecoder.h

@@ -0,0 +1,71 @@
+// Filename: stringDecoder.h
+// Created by:  drose (11Feb02)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
+//
+// All use of this software is subject to the terms of the Panda 3d
+// Software license.  You should have received a copy of this license
+// along with this source code; you will also find a current copy of
+// the license at http://www.panda3d.org/license.txt .
+//
+// To contact the maintainers of this program write to
+// panda3d@yahoogroups.com .
+//
+////////////////////////////////////////////////////////////////////
+
+#ifndef STRINGDECODER_H
+#define STRINGDECODER_H
+
+#include "pandabase.h"
+
+
+////////////////////////////////////////////////////////////////////
+//       Class : StringDecoder
+// Description : The base class to a family of classes that decode
+//               various kinds of encoded byte streams.  Give it a
+//               string, then ask it to pull the characters out one at
+//               a time.  This also serves as the plain old
+//               byte-at-a-time decoder.
+////////////////////////////////////////////////////////////////////
+class StringDecoder {
+public:
+  INLINE StringDecoder(const string &input);
+  virtual ~StringDecoder();
+
+  virtual int get_next_character();
+  INLINE bool is_eof();
+
+protected:
+  string _input;
+  size_t _p;
+};
+
+////////////////////////////////////////////////////////////////////
+//       Class : StringUtf8Decoder
+// Description : This decoder extracts utf-8 sequences.
+////////////////////////////////////////////////////////////////////
+class StringUtf8Decoder : public StringDecoder {
+public:
+  INLINE StringUtf8Decoder(const string &input);
+
+  virtual int get_next_character();
+};
+
+////////////////////////////////////////////////////////////////////
+//       Class : StringUnicodeDecoder
+// Description : This decoder extracts characters two at a time to get
+//               a plain wide character sequence.
+////////////////////////////////////////////////////////////////////
+class StringUnicodeDecoder : public StringDecoder {
+public:
+  INLINE StringUnicodeDecoder(const string &input);
+
+  virtual int get_next_character();
+};
+
+#include "stringDecoder.I"
+
+#endif

panda/src/text/textNode.I

@@ -86,22 +86,6 @@ thaw() {
   return _freeze_level;
 }
 
-/*
-////////////////////////////////////////////////////////////////////
-//     Function: TextNode::set_font
-//       Access: Published
-//  Description: Sets the font that will be used when making text.
-////////////////////////////////////////////////////////////////////
-INLINE void TextNode::
-set_font(Node *font_node) {
-  if (font_node == (Node *)NULL) {
-    set_font((TextFont *)NULL);
-  } else {
-    set_font(new TextFont(font_node));
-  }
-}
-*/
-
 ////////////////////////////////////////////////////////////////////
 //     Function: TextNode::set_font
 //       Access: Published
@@ -125,6 +109,67 @@ get_font() const {
   return _font;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: TextNode::set_encoding
+//       Access: Published
+//  Description: Specifies how the string set via set_text() is to be
+//               interpreted.  The default, E_iso8859, means a
+//               standard string with one-byte characters
+//               (i.e. ASCII).  Other encodings are possible to take
+//               advantage of character sets with more than 256
+//               characters.
+////////////////////////////////////////////////////////////////////
+INLINE void TextNode::
+set_encoding(TextNode::Encoding encoding) {
+  if (_encoding != encoding) {
+    _encoding = encoding;
+    rebuild(true);
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: TextNode::get_encoding
+//       Access: Published
+//  Description: Returns the encoding by which the string set via
+//               set_text() is to be interpreted.  See set_encoding().
+////////////////////////////////////////////////////////////////////
+INLINE TextNode::Encoding TextNode::
+get_encoding() const {
+  return _encoding;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: TextNode::set_expand_amp
+//       Access: Published
+//  Description: Sets the state of the expand_amp flag.  When this is
+//               true, embedded ampersands in the text string are
+//               expanded to special characters according to a subset
+//               of the HTML conventions.  When this is false,
+//               ampersands are treated as ordinary characters.
+////////////////////////////////////////////////////////////////////
+INLINE void TextNode::
+set_expand_amp(bool expand_amp) {
+  bool current_expand_amp = get_expand_amp();
+  if (expand_amp && !current_expand_amp) {
+    _flags |= F_expand_amp;
+    rebuild(true);
+  } else {
+    _flags &= ~F_expand_amp;
+    rebuild(true);
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: TextNode::get_expand_amp
+//       Access: Published
+//  Description: Returns the state of the expand_amp flag.  See
+//               set_expand_amp().
+////////////////////////////////////////////////////////////////////
+INLINE bool TextNode::
+get_expand_amp() const {
+  return (_flags & F_expand_amp) != 0;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: TextNode::get_line_height
 //       Access: Published
@@ -165,7 +210,7 @@ get_slant() const {
 //  Description:
 ////////////////////////////////////////////////////////////////////
 INLINE void TextNode::
-set_align(int align_type) {
+set_align(TextNode::Alignment align_type) {
   if (_align != align_type) {
     _align = align_type;
     rebuild(true);
@@ -177,7 +222,7 @@ set_align(int align_type) {
 //       Access: Published
 //  Description:
 ////////////////////////////////////////////////////////////////////
-INLINE int TextNode::
+INLINE TextNode::Alignment TextNode::
 get_align() const {
   return _align;
 }

panda/src/text/textNode.cxx

@@ -17,6 +17,7 @@
 ////////////////////////////////////////////////////////////////////
 #include "textNode.h"
 #include "textGlyph.h"
+#include "stringDecoder.h"
 #include "config_text.h"
 
 #include "compose_matrix.h"
@@ -45,10 +46,11 @@ TypeHandle TextNode::_type_handle;
 ////////////////////////////////////////////////////////////////////
 TextNode::
 TextNode(const string &name) : NamedNode(name) {
+  _encoding = E_iso8859;
   _slant = 0.0f;
 
   _flags = 0;
-  _align = TM_ALIGN_LEFT;
+  _align = A_left;
   _wordwrap_width = 1.0f;
 
   _text_color.set(1.0f, 1.0f, 1.0f, 1.0f);
@@ -112,16 +114,16 @@ write(ostream &out, int indent_level) const {
   indent(out, indent_level + 2)
     << "alignment is ";
   switch (_align) {
-  case TM_ALIGN_LEFT:
+  case A_left:
-    out << "TM_ALIGN_LEFT\n";
+    out << "A_left\n";
     break;
 
-  case TM_ALIGN_RIGHT:
+  case A_right:
-    out << "TM_ALIGN_RIGHT\n";
+    out << "A_right\n";
     break;
 
-  case TM_ALIGN_CENTER:
+  case A_center:
-    out << "TM_ALIGN_CENTER\n";
+    out << "A_center\n";
     break;
   }
 
@@ -245,13 +247,17 @@ generate() {
     text = wordwrap_to(text, _wordwrap_width, false);
   }
 
+  StringDecoder *decoder = make_decoder(text);
+
   // Assemble the text.
   LVector2f ul, lr;
   int num_rows = 0;
-  PT_Node text_root = assemble_text(text.c_str(), ul, lr, num_rows);
+  PT_Node text_root = assemble_text(decoder, ul, lr, num_rows);
   RenderRelation *text_arc =
     new RenderRelation(sub_root, text_root, _draw_order + 2);
 
+  delete decoder;
+
   if (has_text_color()) {
     text_arc->set_transition(new ColorTransition(_text_color));
     if (_text_color[3] != 1.0) {
@@ -410,9 +416,13 @@ do_measure() {
     text = wordwrap_to(text, _wordwrap_width, false);
   }
 
+  StringDecoder *decoder = make_decoder(text);
+
   LVector2f ul, lr;
   int num_rows = 0;
-  measure_text(text.c_str(), ul, lr, num_rows);
+  measure_text(decoder, ul, lr, num_rows);
+
+  delete decoder;
 
   _num_rows = num_rows;
   _ul2d = ul;
@@ -424,6 +434,29 @@ do_measure() {
   _lr3d = _lr3d * _transform;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: TextNode::make_decoder
+//       Access: Private
+//  Description: Creates and returns a new StringDecoder suitable for
+//               decoding the given input text, and corresponding to
+//               our input encoding type.  The decoder must be freed
+//               via delete later.
+////////////////////////////////////////////////////////////////////
+StringDecoder *TextNode::
+make_decoder(const string &text) {
+  switch (_encoding) {
+  case E_utf8:
+    return new StringUtf8Decoder(text);
+
+  case E_unicode:
+    return new StringUnicodeDecoder(text);
+
+  case E_iso8859:
+  default:
+    return new StringDecoder(text);
+  };
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: TextNode::assemble_row
 //       Access: Private
@@ -434,12 +467,21 @@ do_measure() {
 //               to the terminating character.
 ////////////////////////////////////////////////////////////////////
 float TextNode::
-assemble_row(const char *&source, Node *dest) {
+assemble_row(StringDecoder *decoder, Node *dest) {
   nassertr(_font != (TextFont *)NULL, 0.0f);
 
   float xpos = 0.0f;
-  while (*source != '\0' && *source != '\n') {
+  bool expand_amp = get_expand_amp();
-    int character = (unsigned char)*source;
+  if (decoder->is_eof()) {
+    return xpos;
+  }
+  int character = decoder->get_next_character();
+  while (character != '\n') {
+    if (character == '&' && expand_amp) {
+      // An ampersand in expand_amp mode is treated as an escape
+      // character.
+      character = expand_amp_sequence(decoder);
+    }
 
     if (character == ' ') {
       // A space is a special case.
@@ -481,7 +523,10 @@ assemble_row(const char *&source, Node *dest) {
         xpos += glyph->get_advance() * glyph_scale;
       }
     }
-    source++;
+    if (decoder->is_eof()) {
+      return xpos;
+    }
+    character = decoder->get_next_character();
   }
 
   return xpos;
@@ -495,7 +540,7 @@ assemble_row(const char *&source, Node *dest) {
 //               returns it.  Also sets the ul, lr corners.
 ////////////////////////////////////////////////////////////////////
 Node *TextNode::
-assemble_text(const char *source, LVector2f &ul, LVector2f &lr,
+assemble_text(StringDecoder *decoder, LVector2f &ul, LVector2f &lr,
               int &num_rows) {
   nassertr(_font != (TextFont *)NULL, (Node *)NULL);
   float line_height = get_line_height();
@@ -508,24 +553,20 @@ assemble_text(const char *source, LVector2f &ul, LVector2f &lr,
 
   float posy = 0.0f;
   int row_index = 0;
-  while (*source != '\0') {
+  while (!decoder->is_eof()) {
     char numstr[20];
     sprintf(numstr, "row%d", row_index);
     nassertr(strlen(numstr) < 20, root_node);
 
     Node *row = new NamedNode(numstr);
-    float row_width = assemble_row(source, row);
+    float row_width = assemble_row(decoder, row);
-    if (*source != '\0') {
-      // Skip past the newline.
-      source++;
-    }
 
     LMatrix4f mat = LMatrix4f::ident_mat();
-    if (_align == TM_ALIGN_LEFT) {
+    if (_align == A_left) {
       mat.set_row(3, LVector3f(0.0f, 0.0f, posy));
       lr[0] = max(lr[0], row_width);
 
-    } else if (_align == TM_ALIGN_RIGHT) {
+    } else if (_align == A_right) {
       mat.set_row(3, LVector3f(-row_width, 0.0f, posy));
       ul[0] = min(ul[0], -row_width);
 
@@ -566,10 +607,21 @@ assemble_text(const char *source, LVector2f &ul, LVector2f &lr,
 //               it.
 ////////////////////////////////////////////////////////////////////
 float TextNode::
-measure_row(const char *&source) {
+measure_row(StringDecoder *decoder) {
+  nassertr(_font != (TextFont *)NULL, 0.0f);
+
   float xpos = 0.0f;
-  while (*source != '\0' && *source != '\n') {
+  bool expand_amp = get_expand_amp();
-    int character = (unsigned char)*source;
+  if (decoder->is_eof()) {
+    return xpos;
+  }
+  int character = decoder->get_next_character();
+  while (character != '\n') {
+    if (character == '&' && expand_amp) {
+      // An ampersand in expand_amp mode is treated as an escape
+      // character.
+      character = expand_amp_sequence(decoder);
+    }
 
     if (character == ' ') {
       // A space is a special case.
@@ -584,7 +636,10 @@ measure_row(const char *&source) {
         xpos += glyph->get_advance() * glyph_scale;
       }
     }
-    source++;
+    if (decoder->is_eof()) {
+      return xpos;
+    }
+    character = decoder->get_next_character();
   }
 
   return xpos;
@@ -597,7 +652,7 @@ measure_row(const char *&source) {
 //               actually assembling it.
 ////////////////////////////////////////////////////////////////////
 void TextNode::
-measure_text(const char *source, LVector2f &ul, LVector2f &lr,
+measure_text(StringDecoder *decoder, LVector2f &ul, LVector2f &lr,
              int &num_rows) {
   nassertv(_font != (TextFont *)NULL);
   float line_height = get_line_height();
@@ -606,17 +661,13 @@ measure_text(const char *source, LVector2f &ul, LVector2f &lr,
   lr.set(0.0f, 0.0f);
 
   float posy = 0.0f;
-  while (*source != '\0') {
+  while (!decoder->is_eof()) {
-    float row_width = measure_row(source);
+    float row_width = measure_row(decoder);
-    if (*source != '\0') {
-      // Skip past the newline.
-      source++;
-    }
 
-    if (_align == TM_ALIGN_LEFT) {
+    if (_align == A_left) {
       lr[0] = max(lr[0], row_width);
 
-    } else if (_align == TM_ALIGN_RIGHT) {
+    } else if (_align == A_right) {
       ul[0] = min(ul[0], -row_width);
 
     } else {
@@ -633,6 +684,95 @@ measure_text(const char *source, LVector2f &ul, LVector2f &lr,
   lr[1] = posy + 0.8f * line_height;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: TextNode::expand_amp_sequence
+//       Access: Private
+//  Description: Given that we have just read an ampersand from the
+//               StringDecoder, and that we have expand_amp in effect
+//               and are therefore expected to expand the sequence
+//               that this ampersand begins into a single unicode
+//               character, do the expansion and return the character.
+////////////////////////////////////////////////////////////////////
+int TextNode::
+expand_amp_sequence(StringDecoder *decoder) {
+  int result = 0;
+
+  int character = decoder->get_next_character();
+  if (character == '#') {
+    // An explicit numeric sequence: &#nnn;
+    result = 0;
+    character = decoder->get_next_character();
+    while (!decoder->is_eof() && character < 128 && isdigit(character)) {
+      result = (result * 10) + (character - '0');
+      character = decoder->get_next_character();
+    }
+    if (character != ';') {
+      // Invalid sequence.
+      return 0;
+    }
+
+    return result;
+  }
+
+  string sequence;
+  
+  // Some non-numeric sequence.
+  while (!decoder->is_eof() && character < 128 && isalpha(character)) {
+    sequence += character;
+    character = decoder->get_next_character();
+  }
+  if (character != ';') {
+    // Invalid sequence.
+    return 0;
+  }
+
+  static const struct {
+    const char *name;
+    int code;
+  } tokens[] = {
+    { "amp", '&' }, { "lt", '<' }, { "gt", '>' }, { "quot", '"' },
+    { "nbsp", ' ' /* 160 */ },
+
+    { "iexcl", 161 }, { "cent", 162 }, { "pound", 163 }, { "curren", 164 },
+    { "yen", 165 }, { "brvbar", 166 }, { "brkbar", 166 }, { "sect", 167 },
+    { "uml", 168 }, { "die", 168 }, { "copy", 169 }, { "ordf", 170 },
+    { "laquo", 171 }, { "not", 172 }, { "shy", 173 }, { "reg", 174 },
+    { "macr", 175 }, { "hibar", 175 }, { "deg", 176 }, { "plusmn", 177 },
+    { "sup2", 178 }, { "sup3", 179 }, { "acute", 180 }, { "micro", 181 },
+    { "para", 182 }, { "middot", 183 }, { "cedil", 184 }, { "sup1", 185 },
+    { "ordm", 186 }, { "raquo", 187 }, { "frac14", 188 }, { "frac12", 189 },
+    { "frac34", 190 }, { "iquest", 191 }, { "Agrave", 192 }, { "Aacute", 193 },
+    { "Acirc", 194 }, { "Atilde", 195 }, { "Auml", 196 }, { "Aring", 197 },
+    { "AElig", 198 }, { "Ccedil", 199 }, { "Egrave", 200 }, { "Eacute", 201 },
+    { "Ecirc", 202 }, { "Euml", 203 }, { "Igrave", 204 }, { "Iacute", 205 },
+    { "Icirc", 206 }, { "Iuml", 207 }, { "ETH", 208 }, { "Dstrok", 208 },
+    { "Ntilde", 209 }, { "Ograve", 210 }, { "Oacute", 211 }, { "Ocirc", 212 },
+    { "Otilde", 213 }, { "Ouml", 214 }, { "times", 215 }, { "Oslash", 216 },
+    { "Ugrave", 217 }, { "Uacute", 218 }, { "Ucirc", 219 }, { "Uuml", 220 },
+    { "Yacute", 221 }, { "THORN", 222 }, { "szlig", 223 }, { "agrave", 224 },
+    { "aacute", 225 }, { "acirc", 226 }, { "atilde", 227 }, { "auml", 228 },
+    { "aring", 229 }, { "aelig", 230 }, { "ccedil", 231 }, { "egrave", 232 },
+    { "eacute", 233 }, { "ecirc", 234 }, { "euml", 235 }, { "igrave", 236 },
+    { "iacute", 237 }, { "icirc", 238 }, { "iuml", 239 }, { "eth", 240 },
+    { "ntilde", 241 }, { "ograve", 242 }, { "oacute", 243 }, { "ocirc", 244 },
+    { "otilde", 245 }, { "ouml", 246 }, { "divide", 247 }, { "oslash", 248 },
+    { "ugrave", 249 }, { "uacute", 250 }, { "ucirc", 251 }, { "uuml", 252 },
+    { "yacute", 253 }, { "thorn", 254 }, { "yuml", 255 },
+
+    { NULL, 0 },
+  };
+
+  for (int i = 0; tokens[i].name != NULL; i++) {
+    if (sequence == tokens[i].name) {
+      // Here's a match.
+      return tokens[i].code;
+    }
+  }
+
+  // Some unrecognized sequence.
+  return 0;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: TextNode::make_frame
 //       Access: Private

panda/src/text/textNode.h

@@ -15,36 +15,34 @@
 // panda3d@yahoogroups.com .
 //
 ////////////////////////////////////////////////////////////////////
+
 #ifndef TEXTNODE_H
 #define TEXTNODE_H
-//
+
-////////////////////////////////////////////////////////////////////
+#include "pandabase.h"
-// Includes
-////////////////////////////////////////////////////////////////////
-#include <pandabase.h>
 
 #include "config_text.h"
 #include "textFont.h"
 
-#include <pt_Node.h>
+#include "pt_Node.h"
-#include <namedNode.h>
+#include "namedNode.h"
-#include <luse.h>
+#include "luse.h"
-#include <geom.h>
+#include "geom.h"
-#include <geomNode.h>
+#include "geomNode.h"
-#include <renderRelation.h>
+#include "renderRelation.h"
-#include <textureTransition.h>
+#include "textureTransition.h"
-#include <transparencyTransition.h>
+#include "transparencyTransition.h"
-#include <allTransitionsWrapper.h>
+#include "allTransitionsWrapper.h"
 
-////////////////////////////////////////////////////////////////////
+// These are deprecated.  Use TextNode::Alignment instead.
-// Defines
-////////////////////////////////////////////////////////////////////
 BEGIN_PUBLISH
 #define TM_ALIGN_LEFT         1
 #define TM_ALIGN_RIGHT        2
 #define TM_ALIGN_CENTER       3
 END_PUBLISH
 
+class StringDecoder;
+
 ////////////////////////////////////////////////////////////////////
 //       Class : TextNode
 // Description : The primary interface to this module.  This class
@@ -76,21 +74,38 @@ PUBLISHED:
   TextNode(const string &name = "");
   ~TextNode();
 
+  enum Alignment {
+    A_left = TM_ALIGN_LEFT,
+    A_right = TM_ALIGN_RIGHT,
+    A_center = TM_ALIGN_CENTER,
+  };
+
+  enum Encoding {
+    E_iso8859,
+    E_utf8,
+    E_unicode
+  };
+
   INLINE int freeze();
   INLINE int get_freeze_level() const;
   INLINE int thaw();
 
-  //  INLINE void set_font(Node *font_node);
   INLINE void set_font(TextFont *font);
   INLINE TextFont *get_font() const;
 
+  INLINE void set_encoding(Encoding encoding);
+  INLINE Encoding get_encoding() const;
+
+  INLINE void set_expand_amp(bool expand_amp);
+  INLINE bool get_expand_amp() const;
+
   INLINE float get_line_height() const;
 
   INLINE void set_slant(float slant);
   INLINE float get_slant() const;
 
-  INLINE void set_align(int align_type);
+  INLINE void set_align(Alignment align_type);
-  INLINE int get_align() const;
+  INLINE Alignment get_align() const;
 
   INLINE void set_wordwrap(float width);
   INLINE void clear_wordwrap();
@@ -206,19 +221,24 @@ private:
   void do_rebuild();
   void do_measure();
 
-  float assemble_row(const char *&source, Node *dest);
+  StringDecoder *make_decoder(const string &text);
-  Node *assemble_text(const char *source, LVector2f &ul, LVector2f &lr,
+
+  float assemble_row(StringDecoder *decoder, Node *dest);
+  Node *assemble_text(StringDecoder *decoder, LVector2f &ul, LVector2f &lr,
                       int &num_rows);
-  float measure_row(const char *&source);
+  float measure_row(StringDecoder *decoder);
-  void measure_text(const char *source, LVector2f &ul, LVector2f &lr,
+  void measure_text(StringDecoder *decoder, LVector2f &ul, LVector2f &lr,
                     int &num_rows);
 
+  int expand_amp_sequence(StringDecoder *decoder);
+
   Node *make_frame();
   Node *make_card();
   Node *make_card_with_border();
 
   PT(TextFont) _font;
 
+  Encoding _encoding;
   float _slant;
 
   PT(Texture) _card_texture;
@@ -239,10 +259,11 @@ private:
     F_frame_corners    =  0x0100,
     F_card_transp      =  0x0200,
     F_has_card_border  =  0x0400,
+    F_expand_amp       =  0x0800,
   };
 
   int _flags;
-  int _align;
+  Alignment _align;
   float _wordwrap_width;
   float _frame_width;
   float _card_border_size;

panda/src/text/text_composite1.cxx

@@ -4,5 +4,6 @@
 #include "dynamicTextPage.cxx"
 #include "geomTextGlyph.cxx"
 #include "staticTextFont.cxx"
+#include "stringDecoder.cxx"
 #include "textFont.cxx"
 #include "textGlyph.cxx"