text: use HarfBuzz for text shaping; support right-to-left text

direct/src/gui/OnscreenText.py

@@ -35,7 +35,8 @@ class OnscreenText(NodePath):
                  font = None,
                  parent = None,
                  sort = 0,
-                 mayChange = True):
+                 mayChange = True,
+                 direction = None):
         """
         Make a text node from string, put it into the 2d sg and set it
         up with all the indicated parameters.
@@ -95,6 +96,9 @@ class OnscreenText(NodePath):
           mayChange: pass true if the text or its properties may need
               to be changed at runtime, false if it is static once
               created (which leads to better memory optimization).
+
+          direction: this can be set to 'ltr' or 'rtl' to override the
+              direction of the text.
         """
         if parent == None:
             parent = aspect2d
@@ -192,6 +196,17 @@ class OnscreenText(NodePath):
             textNode.setFrameColor(frame[0], frame[1], frame[2], frame[3])
             textNode.setFrameAsMargin(0.1, 0.1, 0.1, 0.1)
 
+        if direction is not None:
+            if isinstance(direction, str):
+                direction = direction.lower()
+                if direction == 'rtl':
+                    direction = TextProperties.D_rtl
+                elif direction == 'ltr':
+                    direction = TextProperties.D_ltr
+                else:
+                    raise ValueError('invalid direction')
+            textNode.setDirection(direction)
+
         # Create a transform for the text for our scale and position.
         # We'd rather do it here, on the text itself, rather than on
         # our NodePath, so we have one fewer transforms in the scene

dtool/src/parser-inc/ft2build.h

@@ -29,6 +29,7 @@ class FT_Library;
 class FT_Bitmap;
 class FT_Vector;
 class FT_Span;
+class FT_Outline;
 
 #endif
 

makepanda/makepanda.py

@@ -80,8 +80,9 @@ PkgListSet(["PYTHON", "DIRECT",                        # Python support
   "VORBIS", "FFMPEG", "SWSCALE", "SWRESAMPLE",         # Audio decoding
   "ODE", "PHYSX", "BULLET", "PANDAPHYSICS",            # Physics
   "SPEEDTREE",                                         # SpeedTree
-  "ZLIB", "PNG", "JPEG", "TIFF", "OPENEXR", "SQUISH", "FREETYPE", # 2D Formats support
+  "ZLIB", "PNG", "JPEG", "TIFF", "OPENEXR", "SQUISH",  # 2D Formats support
   ] + MAYAVERSIONS + MAXVERSIONS + [ "FCOLLADA", "ASSIMP", "EGG", # 3D Formats support
+  "FREETYPE", "HARFBUZZ",                              # Text rendering
   "VRPN", "OPENSSL",                                   # Transport
   "FFTW",                                              # Algorithm helpers
   "ARTOOLKIT", "OPENCV", "DIRECTCAM", "VISION",        # Augmented Reality
@@ -638,6 +639,7 @@ if (COMPILER == "MSVC"):
     if (PkgSkip("NVIDIACG")==0): LibName("CGDX9",    GetThirdpartyDir() + "nvidiacg/lib/cgD3D9.lib")
     if (PkgSkip("NVIDIACG")==0): LibName("NVIDIACG", GetThirdpartyDir() + "nvidiacg/lib/cg.lib")
     if (PkgSkip("FREETYPE")==0): LibName("FREETYPE", GetThirdpartyDir() + "freetype/lib/freetype.lib")
+    if (PkgSkip("HARFBUZZ")==0): LibName("HARFBUZZ", GetThirdpartyDir() + "harfbuzz/lib/harfbuzz.lib")
     if (PkgSkip("FFTW")==0):     LibName("FFTW",     GetThirdpartyDir() + "fftw/lib/rfftw.lib")
     if (PkgSkip("FFTW")==0):     LibName("FFTW",     GetThirdpartyDir() + "fftw/lib/fftw.lib")
     if (PkgSkip("ARTOOLKIT")==0):LibName("ARTOOLKIT",GetThirdpartyDir() + "artoolkit/lib/libAR.lib")
@@ -803,6 +805,7 @@ if (COMPILER=="GCC"):
         SmartPkgEnable("FFTW",      "",          ("rfftw", "fftw"), ("fftw.h", "rfftw.h"))
         SmartPkgEnable("FMODEX",    "",          ("fmodex"), ("fmodex", "fmodex/fmod.h"))
         SmartPkgEnable("FREETYPE",  "freetype2", ("freetype"), ("freetype2", "freetype2/freetype/freetype.h"))
+        SmartPkgEnable("HARFBUZZ",  "harfbuzz",  ("harfbuzz"), ("harfbuzz", "harfbuzz/hb-ft.h"))
         SmartPkgEnable("GL",        "gl",        ("GL"), ("GL/gl.h"), framework = "OpenGL")
         SmartPkgEnable("GLES",      "glesv1_cm", ("GLESv1_CM"), ("GLES/gl.h"), framework = "OpenGLES")
         SmartPkgEnable("GLES2",     "glesv2",    ("GLESv2"), ("GLES2/gl2.h")) #framework = "OpenGLES"?
@@ -3806,7 +3809,9 @@ if (PkgSkip("FREETYPE")==0 and not RUNTIME):
 #
 
 if (not RUNTIME):
-  OPTS=['DIR:panda/src/text', 'BUILDING:PANDA', 'ZLIB',  'FREETYPE']
+  DefSymbol("HARFBUZZ", "HAVE_HARFBUZZ")
+
+  OPTS=['DIR:panda/src/text', 'BUILDING:PANDA', 'ZLIB',  'FREETYPE', 'HARFBUZZ']
   TargetAdd('p3text_composite1.obj', opts=OPTS, input='p3text_composite1.cxx')
   TargetAdd('p3text_composite2.obj', opts=OPTS, input='p3text_composite2.cxx')
 
@@ -3956,7 +3961,7 @@ if (not RUNTIME):
 #
 
 if (not RUNTIME):
-  OPTS=['DIR:panda/metalibs/panda', 'BUILDING:PANDA', 'JPEG', 'PNG',
+  OPTS=['DIR:panda/metalibs/panda', 'BUILDING:PANDA', 'JPEG', 'PNG', 'HARFBUZZ',
       'TIFF', 'OPENEXR', 'ZLIB', 'OPENSSL', 'FREETYPE', 'FFTW', 'ADVAPI', 'WINSOCK2',
       'SQUISH', 'NVIDIACG', 'VORBIS', 'WINUSER', 'WINMM', 'WINGDI', 'IPHLPAPI']
 

panda/src/text/config_text.cxx

@@ -52,7 +52,13 @@ ConfigVariableBool text_kerning
 ("text-kerning", false,
  PRC_DESC("Set this true to enable kerning when the font provides kerning "
           "tables.  This can result in more aesthetically pleasing spacing "
-          "between individual glyphs."));
+          "between individual glyphs.  Has no effect when text-use-harfbuzz "
+          "is true, since HarfBuzz offers superior kerning support."));
+
+ConfigVariableBool text_use_harfbuzz
+("text-use-harfbuzz", false,
+ PRC_DESC("Set this true to enable HarfBuzz support, which offers superior "
+          "text shaping and better support for non-Latin text."));
 
 ConfigVariableInt text_anisotropic_degree
 ("text-anisotropic-degree", 1,

panda/src/text/config_text.h

@@ -31,6 +31,7 @@ NotifyCategoryDecl(text, EXPCL_PANDA_TEXT, EXPTP_PANDA_TEXT);
 extern ConfigVariableBool text_flatten;
 extern ConfigVariableBool text_dynamic_merge;
 extern ConfigVariableBool text_kerning;
+extern ConfigVariableBool text_use_harfbuzz;
 extern ConfigVariableInt text_anisotropic_degree;
 extern ConfigVariableInt text_texture_margin;
 extern ConfigVariableDouble text_poly_margin;

panda/src/text/dynamicTextFont.cxx

@@ -44,6 +44,10 @@
 #include "textureAttrib.h"
 #include "transparencyAttrib.h"
 
+#ifdef HAVE_HARFBUZZ
+#include <hb-ft.h>
+#endif
+
 TypeHandle DynamicTextFont::_type_handle;
 
 
@@ -114,7 +118,8 @@ DynamicTextFont(const DynamicTextFont &copy) :
   _has_outline(copy._has_outline),
   _tex_format(copy._tex_format),
   _needs_image_processing(copy._needs_image_processing),
-  _preferred_page(0)
+  _preferred_page(0),
+  _hb_font(nullptr)
 {
 }
 
@@ -123,6 +128,11 @@ DynamicTextFont(const DynamicTextFont &copy) :
  */
 DynamicTextFont::
 ~DynamicTextFont() {
+#ifdef HAVE_HARFBUZZ
+  if (_hb_font != nullptr) {
+    hb_font_destroy(_hb_font);
+  }
+#endif
 }
 
 /**
@@ -203,6 +213,13 @@ clear() {
   _cache.clear();
   _pages.clear();
   _empty_glyphs.clear();
+
+#ifdef HAVE_HARFBUZZ
+  if (_hb_font != nullptr) {
+    hb_font_destroy(_hb_font);
+    _hb_font = nullptr;
+  }
+#endif
 }
 
 /**
@@ -305,6 +322,55 @@ get_kerning(int first, int second) const {
   return delta.x / (_font_pixels_per_unit * 64);
 }
 
+/**
+ * Like get_glyph, but uses a glyph index.
+ */
+bool DynamicTextFont::
+get_glyph_by_index(int character, int glyph_index, CPT(TextGlyph) &glyph) {
+  if (!_is_valid) {
+    glyph = nullptr;
+    return false;
+  }
+
+  Cache::iterator ci = _cache.find(glyph_index);
+  if (ci != _cache.end()) {
+    glyph = (*ci).second;
+  } else {
+    FT_Face face = acquire_face();
+    glyph = make_glyph(character, face, glyph_index);
+    _cache.insert(Cache::value_type(glyph_index, glyph.p()));
+    release_face(face);
+  }
+
+  if (glyph.is_null()) {
+    glyph = get_invalid_glyph();
+    return false;
+  }
+
+  return true;
+}
+
+/**
+ * If Panda was compiled with HarfBuzz enabled, returns a HarfBuzz font for
+ * this font.
+ */
+hb_font_t *DynamicTextFont::
+get_hb_font() const {
+#ifdef HAVE_HARFBUZZ
+  if (_hb_font != nullptr) {
+    return _hb_font;
+  }
+
+  FT_Face face = acquire_face();
+  _hb_font = hb_ft_font_create(face, nullptr);
+  release_face(face);
+
+  return _hb_font;
+#else
+  return nullptr;
+#endif
+}
+
 /**
  * Called from both constructors to set up some initial values.
  */
@@ -328,6 +394,8 @@ initialize() {
   _winding_order = WO_default;
 
   _preferred_page = 0;
+
+  _hb_font = nullptr;
 }
 
 /**

panda/src/text/dynamicTextFont.h

@@ -31,6 +31,8 @@
 
 class NurbsCurveResult;
 
+typedef struct hb_font_t hb_font_t;
+
 /**
  * A DynamicTextFont is a special TextFont object that rasterizes its glyphs
  * from a standard font file (e.g.  a TTF file) on the fly.  It requires the
@@ -125,6 +127,9 @@ public:
   virtual bool get_glyph(int character, CPT(TextGlyph) &glyph);
   virtual PN_stdfloat get_kerning(int first, int second) const;
 
+  bool get_glyph_by_index(int character, int glyph_index, CPT(TextGlyph) &glyph);
+  hb_font_t *get_hb_font() const;
+
 private:
   void initialize();
   void update_filters();
@@ -171,6 +176,8 @@ private:
   typedef pvector< PT(TextGlyph) > EmptyGlyphs;
   EmptyGlyphs _empty_glyphs;
 
+  mutable hb_font_t *_hb_font;
+
 public:
   static TypeHandle get_class_type() {
     return _type_handle;

panda/src/text/textAssembler.cxx

@@ -31,10 +31,15 @@
 #include "geomVertexData.h"
 #include "geom.h"
 #include "modelNode.h"
+#include "dynamicTextFont.h"
 
 #include <ctype.h>
 #include <stdio.h>  // for sprintf
 
+#ifdef HAVE_HARFBUZZ
+#include <hb.h>
+#endif
+
 // This is the factor by which CT_small scales the character down.
 static const PN_stdfloat small_accent_scale = 0.6f;
 
@@ -1408,6 +1413,12 @@ assemble_row(TextAssembler::TextRow &row,
   PN_stdfloat underscore_start = 0.0f;
   const TextProperties *underscore_properties = NULL;
 
+  const ComputedProperties *prev_cprops;
+
+#ifdef HAVE_HARFBUZZ
+  hb_buffer_t *harfbuff = nullptr;
+#endif
+
   TextString::const_iterator si;
   for (si = row._string.begin(); si != row._string.end(); ++si) {
     const TextCharacter &tch = (*si);
@@ -1448,10 +1459,29 @@ assemble_row(TextAssembler::TextRow &row,
       LVecBase4 frame = graphic->get_frame();
       line_height = max(line_height, frame[3] - frame[2]);
     } else {
-      // [fabius] this is not the right place to calc line height (see below)
+      line_height = max(line_height, font->get_line_height() * properties->get_glyph_scale() * properties->get_text_scale());
-      // line_height = max(line_height, font->get_line_height());
     }
 
+#ifdef HAVE_HARFBUZZ
+    if (tch._cprops != prev_cprops || graphic != nullptr) {
+      if (harfbuff != nullptr && hb_buffer_get_length(harfbuff) > 0) {
+        // Shape the buffer accumulated so far.
+        shape_buffer(harfbuff, placed_glyphs, xpos, prev_cprops->_properties);
+        hb_buffer_reset(harfbuff);
+
+      } else if (harfbuff == nullptr && text_use_harfbuzz &&
+                 font->is_of_type(DynamicTextFont::get_class_type())) {
+        harfbuff = hb_buffer_create();
+      }
+      prev_cprops = tch._cprops;
+    }
+
+    if (graphic == nullptr && harfbuff != nullptr) {
+      hb_buffer_add(harfbuff, character, character);
+      continue;
+    }
+#endif
+
     if (character == ' ') {
       // A space is a special case.
       xpos += properties->get_glyph_scale() * properties->get_text_scale() * font->get_space_advance();
@@ -1613,10 +1643,16 @@ assemble_row(TextAssembler::TextRow &row,
       }
 
       xpos += advance * glyph_scale;
-      line_height = max(line_height, font->get_line_height() * glyph_scale);
     }
   }
 
+#ifdef HAVE_HARFBUZZ
+  if (harfbuff != nullptr && hb_buffer_get_length(harfbuff) > 0) {
+    shape_buffer(harfbuff, placed_glyphs, xpos, prev_cprops->_properties);
+  }
+  hb_buffer_destroy(harfbuff);
+#endif
+
   if (underscore && underscore_start != xpos) {
     draw_underscore(placed_glyphs, underscore_start, xpos,
                     underscore_properties);
@@ -1640,6 +1676,88 @@ assemble_row(TextAssembler::TextRow &row,
   }
 }
 
+/**
+ * Places the glyphs collected from a HarfBuzz buffer.
+ */
+void TextAssembler::
+shape_buffer(hb_buffer_t *buf, PlacedGlyphs &placed_glyphs, PN_stdfloat &xpos,
+             const TextProperties &properties) {
+
+#ifdef HAVE_HARFBUZZ
+  // If we did not specify a text direction, harfbuzz will guess it based on
+  // the script we are using.
+  hb_direction_t direction = HB_DIRECTION_INVALID;
+  if (properties.has_direction()) {
+    switch (properties.get_direction()) {
+    case TextProperties::D_ltr:
+      direction = HB_DIRECTION_LTR;
+      break;
+    case TextProperties::D_rtl:
+      direction = HB_DIRECTION_RTL;
+      break;
+    }
+  }
+  hb_buffer_set_direction(buf, direction);
+  hb_buffer_guess_segment_properties(buf);
+
+  DynamicTextFont *font = DCAST(DynamicTextFont, properties.get_font());
+  hb_font_t *hb_font = font->get_hb_font();
+  hb_shape(hb_font, buf, NULL, 0);
+
+  PN_stdfloat glyph_scale = properties.get_glyph_scale() * properties.get_text_scale();
+  PN_stdfloat scale = glyph_scale / (font->get_pixels_per_unit() * font->get_scale_factor() * 64.0);
+
+  unsigned int glyph_count;
+  hb_glyph_info_t *glyph_info = hb_buffer_get_glyph_infos(buf, &glyph_count);
+  hb_glyph_position_t *glyph_pos = hb_buffer_get_glyph_positions(buf, &glyph_count);
+
+  for (unsigned int i = 0; i < glyph_count; ++i) {
+    int character = glyph_info[i].cluster;
+    int glyph_index = glyph_info[i].codepoint;
+
+    CPT(TextGlyph) glyph;
+    if (!font->get_glyph_by_index(character, glyph_index, glyph)) {
+      char buffer[512];
+      sprintf(buffer, "U+%04x", character);
+      text_cat.warning()
+        << "No definition in " << font->get_name()
+        << " for character " << buffer;
+      if (character < 128 && isprint((unsigned int)character)) {
+        text_cat.warning(false)
+          << " ('" << (char)character << "')";
+      }
+      text_cat.warning(false)
+        << "\n";
+    }
+
+    PN_stdfloat advance = glyph_pos[i].x_advance * scale;
+    if (glyph->is_whitespace()) {
+      // A space is a special case.
+      xpos += advance;
+      continue;
+    }
+
+    PN_stdfloat x_offset = glyph_pos[i].x_offset * scale;
+    PN_stdfloat y_offset = glyph_pos[i].y_offset * scale;
+
+    // Build up a GlyphPlacement, indicating all of the Geoms that go into
+    // this character.  Normally, there is only one Geom per character, but
+    // it may involve multiple Geoms if we need to add cheesy accents or
+    // ligatures.
+    GlyphPlacement placement;
+    placement._glyph = move(glyph);
+    placement._scale = glyph_scale;
+    placement._xpos = xpos + x_offset;
+    placement._ypos = properties.get_glyph_shift() + y_offset;
+    placement._slant = properties.get_slant();
+    placement._properties = &properties;
+    placed_glyphs.push_back(placement);
+
+    xpos += advance;
+  }
+#endif
+}
+
 /**
  * Creates the geometry to render the underscore line for the indicated range
  * of glyphs in this row.

panda/src/text/textAssembler.h

@@ -28,6 +28,7 @@
 
 #include "pmap.h"
 
+typedef struct hb_buffer_t hb_buffer_t;
 
 class TextEncoder;
 class TextGraphic;
@@ -247,6 +248,9 @@ private:
                     PN_stdfloat &row_width, PN_stdfloat &line_height,
                     TextProperties::Alignment &align, PN_stdfloat &wordwrap);
 
+  void shape_buffer(hb_buffer_t *buf, PlacedGlyphs &glyphs, PN_stdfloat &xpos,
+                    const TextProperties &properties);
+
   // These interfaces are for implementing cheesy accent marks and ligatures
   // when the font doesn't support them.
   enum CheesyPosition {

panda/src/text/textProperties.I

@@ -797,3 +797,39 @@ INLINE PN_stdfloat TextProperties::
 get_text_scale() const {
   return _text_scale;
 }
+
+/**
+ * Specifies the text direction.  If none is specified, it will be guessed
+ * based on the contents of the string.
+ */
+INLINE void TextProperties::
+set_direction(Direction direction) {
+  _direction = direction;
+  _specified |= F_has_direction;
+}
+
+/**
+ * Clears the text direction setting.  If no text direction is specified, it
+ * will be guessed based on the contents of the string.
+ */
+INLINE void TextProperties::
+clear_direction() {
+  _specified &= ~F_has_direction;
+  _direction = D_ltr;
+}
+
+/**
+ *
+ */
+INLINE bool TextProperties::
+has_direction() const {
+  return (_specified & F_has_direction) != 0;
+}
+
+/**
+ * Returns the direction of the text as specified by set_direction().
+ */
+INLINE TextProperties::Direction TextProperties::
+get_direction() const {
+  return _direction;
+}

panda/src/text/textProperties.cxx

@@ -31,26 +31,27 @@ TypeHandle TextProperties::_type_handle;
  *
  */
 TextProperties::
-TextProperties() {
+TextProperties() :
-  _specified = 0;
+  _specified(0),
 
-  _small_caps = text_small_caps;
+  _small_caps(text_small_caps),
-  _small_caps_scale = text_small_caps_scale;
+  _small_caps_scale(text_small_caps_scale),
-  _slant = 0.0f;
+  _slant(0.0f),
-  _underscore = false;
+  _underscore(false),
-  _underscore_height = 0.0f;
+  _underscore_height(0.0f),
-  _align = A_left;
+  _align(A_left),
-  _indent_width = 0.0f;
+  _indent_width(0.0f),
-  _wordwrap_width = 0.0f;
+  _wordwrap_width(0.0f),
-  _preserve_trailing_whitespace = false;
+  _preserve_trailing_whitespace(false),
-  _text_color.set(1.0f, 1.0f, 1.0f, 1.0f);
+  _text_color(1.0f, 1.0f, 1.0f, 1.0f),
-  _shadow_color.set(0.0f, 0.0f, 0.0f, 1.0f);
+  _shadow_color(0.0f, 0.0f, 0.0f, 1.0f),
-  _shadow_offset.set(0.0f, 0.0f);
+  _shadow_offset(0.0f, 0.0f),
-  _draw_order = 1;
+  _draw_order(1),
-  _tab_width = text_tab_width;
+  _tab_width(text_tab_width),
-  _glyph_scale = 1.0f;
+  _glyph_scale(1.0f),
-  _glyph_shift = 0.0f;
+  _glyph_shift(0.0f),
-  _text_scale = 1.0f;
+  _text_scale(1.0f),
+  _direction(D_rtl) {
 }
 
 /**
@@ -89,6 +90,7 @@ operator = (const TextProperties &copy) {
   _glyph_scale = copy._glyph_scale;
   _glyph_shift = copy._glyph_shift;
   _text_scale = copy._text_scale;
+  _direction = copy._direction;
 
   _text_state.clear();
   _shadow_state.clear();
@@ -163,6 +165,9 @@ operator == (const TextProperties &other) const {
   if ((_specified & F_has_text_scale) && _text_scale != other._text_scale) {
     return false;
   }
+  if ((_specified & F_has_direction) && _direction != other._direction) {
+    return false;
+  }
   return true;
 }
 
@@ -238,6 +243,9 @@ add_properties(const TextProperties &other) {
   if (other.has_text_scale()) {
     set_text_scale(other.get_text_scale());
   }
+  if (other.has_direction()) {
+    set_direction(other.get_direction());
+  }
 }
 
 
@@ -361,6 +369,20 @@ write(ostream &out, int indent_level) const {
     indent(out, indent_level)
       << "text scale is " << get_text_scale() << "\n";
   }
+
+  if (has_direction()) {
+    indent(out, indent_level)
+      << "direction is ";
+    switch (get_direction()) {
+    case D_ltr:
+      out << "D_ltr\n";
+      break;
+
+    case D_rtl:
+      out << "D_rtl\n";
+      break;
+    }
+  }
 }
 
 /**

panda/src/text/textProperties.h

@@ -49,6 +49,11 @@ PUBLISHED:
     A_boxed_center
   };
 
+  enum Direction {
+    D_ltr,
+    D_rtl,
+  };
+
   TextProperties();
   TextProperties(const TextProperties &copy);
   void operator = (const TextProperties &copy);
@@ -160,6 +165,11 @@ PUBLISHED:
   INLINE bool has_text_scale() const;
   INLINE PN_stdfloat get_text_scale() const;
 
+  INLINE void set_direction(Direction direction);
+  INLINE void clear_direction();
+  INLINE bool has_direction() const;
+  INLINE Direction get_direction() const;
+
   void add_properties(const TextProperties &other);
 
   void write(ostream &out, int indent_level = 0) const;
@@ -197,6 +207,8 @@ PUBLISHED:
                               set_glyph_shift, clear_glyph_shift);
   MAKE_PROPERTY2(text_scale, has_text_scale, get_text_scale,
                              set_text_scale, clear_text_scale);
+  MAKE_PROPERTY2(direction, has_direction, get_direction,
+                            set_direction, clear_direction);
 
 public:
   const RenderState *get_text_state() const;
@@ -225,6 +237,7 @@ private:
     F_has_underscore                   = 0x00010000,
     F_has_underscore_height            = 0x00020000,
     F_has_text_scale                   = 0x00040000,
+    F_has_direction                    = 0x00080000,
   };
 
   int _specified;
@@ -248,6 +261,7 @@ private:
   PN_stdfloat _glyph_scale;
   PN_stdfloat _glyph_shift;
   PN_stdfloat _text_scale;
+  Direction _direction;
 
   mutable CPT(RenderState) _text_state;
   mutable CPT(RenderState) _shadow_state;