Support for non-BMP codepoints, take 2

src/main/java/li/cil/oc/api/internal/TextBuffer.java

@@ -301,9 +301,24 @@ public interface TextBuffer extends ManagedEnvironment, Persistable {
      * @param width  the width of the area to fill.
      * @param height the height of the area to fill.
      * @param value  the character to fill the area with.
+     * @deprecated Please use the int variant.
      */
+    @Deprecated
     void fill(int column, int row, int width, int height, char value);
 
+    /**
+     * Fill a portion of the text buffer.
+     * <p/>
+     * This will set the area's colors to the currently active ones.
+     *
+     * @param column the starting horizontal index of the area to fill.
+     * @param row    the starting vertical index of the area to fill.
+     * @param width  the width of the area to fill.
+     * @param height the height of the area to fill.
+     * @param value  the code point to fill the area with.
+     */
+    void fill(int column, int row, int width, int height, int value);
+
     /**
      * Write a string into the text buffer.
      * <br>
@@ -322,9 +337,20 @@ public interface TextBuffer extends ManagedEnvironment, Persistable {
      * @param column the horizontal index.
      * @param row    the vertical index.
      * @return the character at that index.
+     * @deprecated Please use getCodePoint going forward.
      */
+    @Deprecated
     char get(int column, int row);
 
+    /**
+     * Get the code point in the text buffer at the specified location.
+     *
+     * @param column the horizontal index.
+     * @param row    the vertical index.
+     * @return the character at that index.
+     */
+    int getCodePoint(int column, int row);
+
     /**
      * Get the foreground color of the text buffer at the specified location.
      * <br>
@@ -385,9 +411,32 @@ public interface TextBuffer extends ManagedEnvironment, Persistable {
      * @param column the horizontal index.
      * @param row    the vertical index.
      * @param text   the text to write.
+     * @deprecated Please use the int[][] variant.
      */
+    @Deprecated
     void rawSetText(int column, int row, char[][] text);
 
+    /**
+     * Overwrites a portion of the text in raw mode.
+     * <p/>
+     * This will copy the given char array into the buffer, starting at the
+     * specified column and row. The array is expected to be indexed row-
+     * first, i.e. the first dimension is the vertical axis, the second
+     * the horizontal.
+     * <p/>
+     * <em>Important</em>: this performs no checks as to whether something
+     * actually changed. It will always send the changed patch to clients.
+     * It will also not crop the specified array to the actually used range.
+     * In other words, this is not intended to be exposed as-is to user code,
+     * it should always be called with validated, and, as necessary, cropped
+     * values.
+     *
+     * @param column the horizontal index.
+     * @param row    the vertical index.
+     * @param text   the text code points to write.
+     */
+    void rawSetText(int column, int row, int[][] text);
+
     /**
      * Overwrites a portion of the foreground color information in raw mode.
      * <br>

src/main/java/li/cil/oc/util/ExtendedUnicodeHelper.java

@@ -0,0 +1,41 @@
+package li.cil.oc.util;
+
+/**
+ * Helper functions for handling strings with characters outside of the Unicode BMP.
+ */
+public final class ExtendedUnicodeHelper {
+	private ExtendedUnicodeHelper() {
+
+	}
+
+	public static int length(String s) {
+		return s.codePointCount(0, s.length());
+	}
+
+	public static String reverse(String s) {
+		StringBuilder sb = new StringBuilder();
+		for (int i = s.length() - 1; i >= 0; i--) {
+			char c = s.charAt(i);
+			if (Character.isLowSurrogate(c) && i > 0) {
+				i--;
+				char c2 = s.charAt(i);
+				if (Character.isHighSurrogate(c2)) {
+					sb.append(c2).append(c);
+				} else {
+					// Invalid surrogate pair?
+					sb.append(c).append(c2);
+				}
+			} else {
+				sb.append(c);
+			}
+		}
+		return sb.toString();
+	}
+
+	public static String substring(String s, int start, int end) {
+		return s.substring(
+				s.offsetByCodePoints(0, start),
+				s.offsetByCodePoints(0, end)
+		);
+	}
+}

src/main/scala/li/cil/oc/client/PacketHandler.scala

@@ -668,7 +668,7 @@ object PacketHandler extends CommonPacketHandler {
     val row = p.readInt()
     val w = p.readInt()
     val h = p.readInt()
-    val c = p.readChar()
+    val c = p.readMedium()
     buffer.fill(col, row, w, h, c)
   }
 
@@ -737,12 +737,12 @@ object PacketHandler extends CommonPacketHandler {
     val row = p.readInt()
 
     val rows = p.readShort()
-    val text = new Array[Array[Char]](rows)
+    val text = new Array[Array[Int]](rows)
     for (y <- 0 until rows) {
       val cols = p.readShort()
-      val line = new Array[Char](cols)
+      val line = new Array[Int](cols)
       for (x <- 0 until cols) {
-        line(x) = p.readChar()
+        line(x) = p.readMedium()
       }
       text(y) = line
     }

src/main/scala/li/cil/oc/client/renderer/font/DynamicFontRenderer.scala

@@ -24,7 +24,7 @@ class DynamicFontRenderer extends TextureFontRenderer with IResourceManagerReloa
 
   private val textures = mutable.ArrayBuffer.empty[CharTexture]
 
-  private val charMap = mutable.Map.empty[Char, DynamicFontRenderer.CharIcon]
+  private val charMap = mutable.Map.empty[Int, DynamicFontRenderer.CharIcon]
 
   private var activeTexture: CharTexture = _
 
@@ -63,18 +63,18 @@ class DynamicFontRenderer extends TextureFontRenderer with IResourceManagerReloa
     RenderState.checkError(getClass.getName + ".bindTexture")
   }
 
-  override protected def generateChar(char: Char) {
+  override protected def generateChar(char: Int) {
     charMap.getOrElseUpdate(char, createCharIcon(char))
   }
 
-  override protected def drawChar(tx: Float, ty: Float, char: Char) {
+  override protected def drawChar(tx: Float, ty: Float, char: Int) {
     charMap.get(char) match {
       case Some(icon) if icon.texture == activeTexture => icon.draw(tx, ty)
       case _ =>
     }
   }
 
-  private def createCharIcon(char: Char): DynamicFontRenderer.CharIcon = {
+  private def createCharIcon(char: Int): DynamicFontRenderer.CharIcon = {
     if (FontUtils.wcwidth(char) < 1 || glyphProvider.getGlyph(char) == null) {
       if (char == '?') null
       else charMap.getOrElseUpdate('?', createCharIcon('?'))
@@ -125,9 +125,9 @@ object DynamicFontRenderer {
       GL11.glBindTexture(GL11.GL_TEXTURE_2D, id)
     }
 
-    def isFull(char: Char) = chars + FontUtils.wcwidth(char) > capacity
+    def isFull(char: Int) = chars + FontUtils.wcwidth(char) > capacity
 
-    def add(char: Char) = {
+    def add(char: Int) = {
       val glyphWidth = FontUtils.wcwidth(char)
       val w = owner.charWidth * glyphWidth
       val h = owner.charHeight

src/main/scala/li/cil/oc/client/renderer/font/FontParserHex.java

@@ -1,5 +1,7 @@
 package li.cil.oc.client.renderer.font;
 
+import gnu.trove.map.TIntObjectMap;
+import gnu.trove.map.hash.TIntObjectHashMap;
 import li.cil.oc.OpenComputers;
 import li.cil.oc.Settings;
 import li.cil.oc.util.FontUtils;
@@ -17,13 +19,10 @@ public class FontParserHex implements IGlyphProvider {
     private static final byte[] OPAQUE = {(byte) 255, (byte) 255, (byte) 255, (byte) 255};
     private static final byte[] TRANSPARENT = {0, 0, 0, 0};
 
-    private final byte[][] glyphs = new byte[FontUtils.codepoint_limit()][];
+    private final TIntObjectMap<byte[]> glyphs = new TIntObjectHashMap<>();
 
     @Override
     public void initialize() {
-        for (int i = 0; i < glyphs.length; ++i) {
-            glyphs[i] = null;
-        }
         try {
             final InputStream font = Minecraft.getMinecraft().getResourceManager().getResource(new ResourceLocation(Settings.resourceDomain(), "font.hex")).getInputStream();
             try {
@@ -34,7 +33,10 @@ public class FontParserHex implements IGlyphProvider {
                 while ((line = input.readLine()) != null) {
                     final String[] info = line.split(":");
                     final int charCode = Integer.parseInt(info[0], 16);
-                    if (charCode < 0 || charCode >= glyphs.length) continue; // Out of bounds.
+                    if (charCode < 0 || charCode >= FontUtils.codepoint_limit()) {
+                        OpenComputers.log().warn(String.format("Unicode font contained unexpected glyph: U+%04X, ignoring", charCode));
+                        continue; // Out of bounds.
+                    }
                     final int expectedWidth = FontUtils.wcwidth(charCode);
                     if (expectedWidth < 1) continue; // Skip control characters.
                     // Two chars representing one byte represent one row of eight pixels.
@@ -44,10 +46,10 @@ public class FontParserHex implements IGlyphProvider {
                         for (int i = 0; i < glyph.length; i++) {
                             glyph[i] = (byte) Integer.parseInt(info[1].substring(i * 2, i * 2 + 2), 16);
                         }
-                        if (glyphs[charCode] == null) {
+                        if (!glyphs.containsKey(charCode)) {
                             glyphCount++;
                         }
-                        glyphs[charCode] = glyph;
+                        glyphs.put(charCode, glyph);
                     } else if (Settings.get().logHexFontErrors()) {
                         OpenComputers.log().warn(String.format("Size of glyph for code point U+%04X (%s) in font (%d) does not match expected width (%d), ignoring.", charCode, String.valueOf((char) charCode), glyphWidth, expectedWidth));
                     }
@@ -67,9 +69,11 @@ public class FontParserHex implements IGlyphProvider {
 
     @Override
     public ByteBuffer getGlyph(int charCode) {
-        if (charCode < 0 || charCode >= glyphs.length || glyphs[charCode] == null || glyphs[charCode].length == 0)
+        if (!glyphs.containsKey(charCode))
+            return null;
+        final byte[] glyph = glyphs.get(charCode);
+        if (glyph == null || glyph.length <= 0)
             return null;
-        final byte[] glyph = glyphs[charCode];
         final ByteBuffer buffer = BufferUtils.createByteBuffer(glyph.length * getGlyphWidth() * 4);
         for (byte aGlyph : glyph) {
             int c = ((int) aGlyph) & 0xFF;

src/main/scala/li/cil/oc/client/renderer/font/StaticFontRenderer.scala

@@ -50,7 +50,7 @@ class StaticFontRenderer extends TextureFontRenderer {
     }
   }
 
-  override protected def drawChar(tx: Float, ty: Float, char: Char) {
+  override protected def drawChar(tx: Float, ty: Float, char: Int) {
     val index = 1 + (chars.indexOf(char) match {
       case -1 => chars.indexOf('?')
       case i => i
@@ -69,5 +69,5 @@ class StaticFontRenderer extends TextureFontRenderer {
     GL11.glVertex3d(tx - dw, ty - dh, 0)
   }
 
-  override protected def generateChar(char: Char) {}
+  override protected def generateChar(char: Int) {}
 }

src/main/scala/li/cil/oc/client/renderer/font/TextureFontRenderer.scala

@@ -1,9 +1,7 @@
 package li.cil.oc.client.renderer.font
 
 import li.cil.oc.Settings
-import li.cil.oc.util.PackedColor
+import li.cil.oc.util.{ExtendedUnicodeHelper, PackedColor, RenderState, TextBuffer}
-import li.cil.oc.util.RenderState
-import li.cil.oc.util.TextBuffer
 import org.lwjgl.opengl.GL11
 
 /**
@@ -30,6 +28,12 @@ abstract class TextureFontRenderer {
     }
   }
 
+  def generateChars(chars: Array[Int]) {
+    for (char <- chars) {
+      generateChar(char)
+    }
+  }
+
   def drawBuffer(buffer: TextBuffer, viewportWidth: Int, viewportHeight: Int) {
     val format = buffer.format
 
@@ -113,6 +117,8 @@ abstract class TextureFontRenderer {
   }
 
   def drawString(s: String, x: Int, y: Int): Unit = {
+    val sLength = ExtendedUnicodeHelper.length(s)
+
     GL11.glPushMatrix()
     GL11.glPushAttrib(GL11.GL_ALL_ATTRIB_BITS)
 
@@ -124,13 +130,15 @@ abstract class TextureFontRenderer {
       bindTexture(i)
       GL11.glBegin(GL11.GL_QUADS)
       var tx = 0f
-      for (n <- 0 until s.length) {
+      var cx = 0
-        val ch = s.charAt(n)
+      for (n <- 0 until sLength) {
+        val ch = s.codePointAt(cx)
         // Don't render whitespace.
         if (ch != ' ') {
           drawChar(tx, 0, ch)
         }
         tx += charWidth
+        cx = s.offsetByCodePoints(cx, 1)
       }
       GL11.glEnd()
     }
@@ -147,9 +155,9 @@ abstract class TextureFontRenderer {
 
   protected def bindTexture(index: Int): Unit
 
-  protected def generateChar(char: Char): Unit
+  protected def generateChar(char: Int): Unit
 
-  protected def drawChar(tx: Float, ty: Float, char: Char): Unit
+  protected def drawChar(tx: Float, ty: Float, char: Int): Unit
 
   private def drawQuad(color: Int, x: Int, y: Int, width: Int) = if (color != 0 && width > 0) {
     val x0 = x * charWidth

src/main/scala/li/cil/oc/common/PacketBuilder.scala

@@ -58,6 +58,12 @@ abstract class PacketBuilder(stream: OutputStream) extends DataOutputStream(stre
     }
   }
 
+  def writeMedium(v: Int) = {
+    writeByte(v & 0xFF)
+    writeByte((v >> 8) & 0xFF)
+    writeByte((v >> 16) & 0xFF)
+  }
+
   def writePacketType(pt: PacketType.Value) = writeByte(pt.id)
 
   def sendToAllPlayers() = OpenComputers.channel.sendToAll(packet)

src/main/scala/li/cil/oc/common/PacketHandler.scala

@@ -132,6 +132,13 @@ abstract class PacketHandler {
       else null
     }
 
+    def readMedium(): Int = {
+      val c0 = readUnsignedByte()
+      val c1 = readUnsignedByte()
+      val c2 = readUnsignedByte()
+      (c0) | (c1 << 8) | (c2 << 16)
+    }
+
     def readPacketType() = PacketType(readByte())
   }
 

src/main/scala/li/cil/oc/common/component/GpuTextBuffer.scala

@@ -28,7 +28,7 @@ class GpuTextBuffer(val owner: String, val id: Int, val data: li.cil.oc.util.Tex
   override def onBufferSet(col: Int, row: Int, s: String, vertical: Boolean): Unit = dirty = true
   override def onBufferColorChange(): Unit = dirty = true
   override def onBufferCopy(col: Int, row: Int, w: Int, h: Int, tx: Int, ty: Int): Unit = dirty = true
-  override def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Char): Unit = dirty = true
+  override def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Int): Unit = dirty = true
 
   override def load(nbt: NBTTagCompound): Unit = {
     // the data is initially dirty because other devices don't know about it yet

src/main/scala/li/cil/oc/common/component/TextBuffer.scala

@@ -318,7 +318,7 @@ class TextBuffer(val host: EnvironmentHost) extends prefab.ManagedEnvironment wi
     proxy.onBufferCopy(col, row, w, h, tx, ty)
   }
 
-  override def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Char): Unit = {
+  override def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Int): Unit = {
     proxy.onBufferFill(col, row, w, h, c)
   }
 
@@ -338,7 +338,7 @@ class TextBuffer(val host: EnvironmentHost) extends prefab.ManagedEnvironment wi
     proxy.onBufferRamDestroy(ram)
   }
 
-  override def rawSetText(col: Int, row: Int, text: Array[Array[Char]]): Unit = {
+  override def rawSetText(col: Int, row: Int, text: Array[Array[Int]]): Unit = {
     super.rawSetText(col, row, text)
     proxy.onBufferRawSetText(col, row, text)
   }
@@ -538,7 +538,7 @@ object TextBuffer {
 
     def onBufferDepthChange(depth: api.internal.TextBuffer.ColorDepth): Unit
 
-    def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Char) {
+    def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Int) {
       owner.relativeLitArea = -1
     }
 
@@ -571,7 +571,7 @@ object TextBuffer {
       owner.relativeLitArea = -1
     }
 
-    def onBufferRawSetText(col: Int, row: Int, text: Array[Array[Char]]) {
+    def onBufferRawSetText(col: Int, row: Int, text: Array[Array[Int]]) {
       owner.relativeLitArea = -1
     }
 
@@ -630,7 +630,7 @@ object TextBuffer {
       markDirty()
     }
 
-    override def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Char) {
+    override def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Int) {
       super.onBufferFill(col, row, w, h, c)
       markDirty()
     }
@@ -732,7 +732,7 @@ object TextBuffer {
       owner.synchronized(ServerPacketSender.appendTextBufferDepthChange(owner.pendingCommands, depth))
     }
 
-    override def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Char) {
+    override def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Int) {
       super.onBufferFill(col, row, w, h, c)
       owner.host.markChanged()
       owner.synchronized(ServerPacketSender.appendTextBufferFill(owner.pendingCommands, col, row, w, h, c))
@@ -789,7 +789,7 @@ object TextBuffer {
       owner.synchronized(ServerPacketSender.appendTextBufferRamDestroy(owner.pendingCommands, ram.owner, ram.id))
     }
 
-    override def onBufferRawSetText(col: Int, row: Int, text: Array[Array[Char]]) {
+    override def onBufferRawSetText(col: Int, row: Int, text: Array[Array[Int]]) {
       super.onBufferRawSetText(col, row, text)
       owner.host.markChanged()
       owner.synchronized(ServerPacketSender.appendTextBufferRawSetText(owner.pendingCommands, col, row, text))
@@ -844,7 +844,7 @@ object TextBuffer {
         stack.getTagCompound.removeTag(Settings.namespace + "clipboard")
 
         if (line >= 0 && line < owner.getViewportHeight) {
-          val text = new String(owner.data.buffer(line)).trim
+          val text = owner.data.lineToString(line)
           if (!Strings.isNullOrEmpty(text)) {
             stack.getTagCompound.setString(Settings.namespace + "clipboard", text)
           }

src/main/scala/li/cil/oc/common/component/traits/TextBufferProxy.scala

@@ -3,7 +3,7 @@ package li.cil.oc.common.component.traits
 import li.cil.oc.util
 import li.cil.oc.api
 import li.cil.oc.api.internal.TextBuffer
-import li.cil.oc.util.PackedColor
+import li.cil.oc.util.{ExtendedUnicodeHelper, PackedColor}
 
 trait TextBufferProxy extends api.internal.TextBuffer {
   def data: util.TextBuffer
@@ -70,32 +70,47 @@ trait TextBufferProxy extends api.internal.TextBuffer {
     if (data.copy(col, row, w, h, tx, ty))
       onBufferCopy(col, row, w, h, tx, ty)
 
-  def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Char): Unit = {}
+  def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Int): Unit = {}
 
   def fill(col: Int, row: Int, w: Int, h: Int, c: Char): Unit =
+    fill(col, row, w, h, c.toInt)
+
+  def fill(col: Int, row: Int, w: Int, h: Int, c: Int): Unit =
     if (data.fill(col, row, w, h, c))
       onBufferFill(col, row, w, h, c)
 
   def onBufferSet(col: Int, row: Int, s: String, vertical: Boolean): Unit = {}
 
-  def set(col: Int, row: Int, s: String, vertical: Boolean): Unit =
+  private def truncate(s: String, sLength: Int, leftOffset: Int, maxWidth: Int): String = {
-    if (col < data.width && (col >= 0 || -col < s.length)) {
+    val subFrom = s.offsetByCodePoints(0, leftOffset)
+    val width = math.min(sLength, maxWidth)
+    if (width <= 0) ""
+    else if ((sLength - leftOffset) <= width) s
+    else s.substring(subFrom, s.offsetByCodePoints(subFrom, width))
+  }
+
+  def set(col: Int, row: Int, s: String, vertical: Boolean): Unit = {
+    val sLength = ExtendedUnicodeHelper.length(s)
+    if (col < data.width && (col >= 0 || -col < sLength)) {
       // Make sure the string isn't longer than it needs to be, in particular to
       // avoid sending too much data to our clients.
       val (x, y, truncated) =
       if (vertical) {
-        if (row < 0) (col, 0, s.substring(-row))
+        if (row < 0) (col, 0, truncate(s, sLength, -row, data.height))
-        else (col, row, s.substring(0, math.min(s.length, data.height - row)))
+        else (col, row, truncate(s, sLength, 0, data.height - row))
       }
       else {
-        if (col < 0) (0, row, s.substring(-col))
+        if (col < 0) (0, row, truncate(s, sLength, -col, data.width))
-        else (col, row, s.substring(0, math.min(s.length, data.width - col)))
+        else (col, row, truncate(s, sLength, 0, data.width - col))
       }
       if (data.set(x, y, truncated, vertical))
         onBufferSet(x, row, truncated, vertical)
     }
+  }
 
-  def get(col: Int, row: Int): Char = data.get(col, row)
+  def get(col: Int, row: Int): Char = data.get(col, row).toChar
+
+  def getCodePoint(col: Int, row: Int): Int = data.get(col, row)
 
   override def getForegroundColor(column: Int, row: Int): Int =
     if (isForegroundFromPalette(column, row)) {
@@ -126,6 +141,13 @@ trait TextBufferProxy extends api.internal.TextBuffer {
     }
   }
 
+  override def rawSetText(col: Int, row: Int, text: Array[Array[Int]]): Unit = {
+    for (y <- row until ((row + text.length) min data.height)) {
+      val line = text(y - row)
+      Array.copy(line, 0, data.buffer(y), col, line.length min data.width)
+    }
+  }
+
   override def rawSetForeground(col: Int, row: Int, color: Array[Array[Int]]): Unit = {
     for (y <- row until ((row + color.length) min data.height)) {
       val line = color(y - row)

src/main/scala/li/cil/oc/common/tileentity/Screen.scala

@@ -240,7 +240,7 @@ class Screen(var tier: Int) extends traits.TextBuffer with SidedEnvironment with
           val h = buffer.getHeight
           buffer.setForegroundColor(0xFFFFFF, false)
           buffer.setBackgroundColor(0x000000, false)
-          buffer.fill(0, 0, w, h, ' ')
+          buffer.fill(0, 0, w, h, 0x20)
         }
       })
     }

src/main/scala/li/cil/oc/server/PacketSender.scala

@@ -618,14 +618,14 @@ object PacketSender {
     pb.writeInt(value.ordinal)
   }
 
-  def appendTextBufferFill(pb: PacketBuilder, col: Int, row: Int, w: Int, h: Int, c: Char) {
+  def appendTextBufferFill(pb: PacketBuilder, col: Int, row: Int, w: Int, h: Int, c: Int) {
     pb.writePacketType(PacketType.TextBufferMultiFill)
 
     pb.writeInt(col)
     pb.writeInt(row)
     pb.writeInt(w)
     pb.writeInt(h)
-    pb.writeChar(c)
+    pb.writeMedium(c)
   }
 
   def appendTextBufferPaletteChange(pb: PacketBuilder, index: Int, color: Int) {
@@ -692,7 +692,7 @@ object PacketSender {
     pb.writeInt(id)
   }
 
-  def appendTextBufferRawSetText(pb: PacketBuilder, col: Int, row: Int, text: Array[Array[Char]]) {
+  def appendTextBufferRawSetText(pb: PacketBuilder, col: Int, row: Int, text: Array[Array[Int]]) {
     pb.writePacketType(PacketType.TextBufferMultiRawSetText)
 
     pb.writeInt(col)
@@ -702,7 +702,7 @@ object PacketSender {
       val line = text(y)
       pb.writeShort(line.length.toShort)
       for (x <- 0 until line.length.toShort) {
-        pb.writeChar(line(x))
+        pb.writeMedium(line(x))
       }
     }
   }

src/main/scala/li/cil/oc/server/component/GraphicsCard.scala

@@ -1,7 +1,6 @@
 package li.cil.oc.server.component
 
 import java.util
-
 import li.cil.oc.{Constants, Localization, Settings, api}
 import li.cil.oc.api.Network
 import li.cil.oc.api.driver.DeviceInfo
@@ -10,7 +9,7 @@ import li.cil.oc.api.driver.DeviceInfo.DeviceClass
 import li.cil.oc.api.machine.{Arguments, Callback, Context, LimitReachedException}
 import li.cil.oc.api.network._
 import li.cil.oc.api.prefab
-import li.cil.oc.util.PackedColor
+import li.cil.oc.util.{ExtendedUnicodeHelper, PackedColor}
 import net.minecraft.nbt.{NBTTagCompound, NBTTagList}
 import li.cil.oc.common.component
 import li.cil.oc.common.component.GpuTextBuffer
@@ -475,7 +474,7 @@ class GraphicsCard(val tier: Int) extends prefab.ManagedEnvironment with DeviceI
           (bgValue, Unit)
         }
 
-      result(s.get(x, y), fgColor, bgColor, fgIndex, bgIndex)
+      result(new java.lang.StringBuilder().appendCodePoint(s.getCodePoint(x, y)).toString, fgColor, bgColor, fgIndex, bgIndex)
     })
   }
 
@@ -487,7 +486,7 @@ class GraphicsCard(val tier: Int) extends prefab.ManagedEnvironment with DeviceI
     val vertical = args.optBoolean(3, false)
 
     screen(s => {
-      if (resolveInvokeCosts(bufferIndex, context, setCosts(tier), value.length, Settings.get.gpuSetCost)) {
+      if (resolveInvokeCosts(bufferIndex, context, setCosts(tier), ExtendedUnicodeHelper.length(value), Settings.get.gpuSetCost)) {
         s.set(x, y, value, vertical)
         result(true)
       } else result(Unit, "not enough energy")
@@ -518,11 +517,11 @@ class GraphicsCard(val tier: Int) extends prefab.ManagedEnvironment with DeviceI
     val w = math.max(0, args.checkInteger(2))
     val h = math.max(0, args.checkInteger(3))
     val value = args.checkString(4)
-    if (value.length == 1) screen(s => {
+    if (ExtendedUnicodeHelper.length(value) == 1) screen(s => {
-      val c = value.charAt(0)
+      val c = value.codePointAt(0)
       val cost = if (c == ' ') Settings.get.gpuClearCost else Settings.get.gpuFillCost
       if (resolveInvokeCosts(bufferIndex, context, fillCosts(tier), w * h, cost)) {
-        s.fill(x, y, w, h, value.charAt(0))
+        s.fill(x, y, w, h, c)
         result(true)
       }
       else {
@@ -559,7 +558,7 @@ class GraphicsCard(val tier: Int) extends prefab.ManagedEnvironment with DeviceI
           case machine: li.cil.oc.server.machine.Machine if machine.lastError != null =>
             if (s.getColorDepth.ordinal > api.internal.TextBuffer.ColorDepth.OneBit.ordinal) s.setBackgroundColor(0x0000FF)
             else s.setBackgroundColor(0x000000)
-            s.fill(0, 0, w, h, ' ')
+            s.fill(0, 0, w, h, 0x20)
             try {
               val wrapRegEx = s"(.{1,${math.max(1, w - 2)}})\\s".r
               val lines = wrapRegEx.replaceAllIn(Localization.localizeImmediately(machine.lastError).replace("\t", "  ") + "\n", m => Regex.quoteReplacement(m.group(1) + "\n")).lines.toArray
@@ -580,7 +579,7 @@ class GraphicsCard(val tier: Int) extends prefab.ManagedEnvironment with DeviceI
             }
           case _ =>
             s.setBackgroundColor(0x000000)
-            s.fill(0, 0, w, h, ' ')
+            s.fill(0, 0, w, h, 0x20)
         }
         null // For screen()
       })

src/main/scala/li/cil/oc/server/machine/luac/UnicodeAPI.scala

@@ -1,7 +1,8 @@
 package li.cil.oc.server.machine.luac
 
+import java.util.function.IntUnaryOperator
 import li.cil.oc.util.ExtendedLuaState.extendLuaState
-import li.cil.oc.util.FontUtils
+import li.cil.oc.util.{ExtendedUnicodeHelper, FontUtils}
 
 class UnicodeAPI(owner: NativeLuaArchitecture) extends NativeLuaAPI(owner) {
   override def initialize() {
@@ -9,13 +10,16 @@ class UnicodeAPI(owner: NativeLuaArchitecture) extends NativeLuaAPI(owner) {
     lua.newTable()
 
     lua.pushScalaFunction(lua => {
-      lua.pushString(String.valueOf((1 to lua.getTop).map(lua.checkInteger).map(_.toChar).toArray))
+      val builder = new java.lang.StringBuilder()
+      (1 to lua.getTop).map(lua.checkInteger).foreach(builder.appendCodePoint)
+      lua.pushString(builder.toString)
       1
     })
     lua.setField(-2, "char")
 
     lua.pushScalaFunction(lua => {
-      lua.pushInteger(lua.checkString(1).length)
+      val s = lua.checkString(1)
+      lua.pushInteger(ExtendedUnicodeHelper.length(s))
       1
     })
     lua.setField(-2, "len")
@@ -27,22 +31,23 @@ class UnicodeAPI(owner: NativeLuaArchitecture) extends NativeLuaAPI(owner) {
     lua.setField(-2, "lower")
 
     lua.pushScalaFunction(lua => {
-      lua.pushString(lua.checkString(1).reverse)
+      lua.pushString(ExtendedUnicodeHelper.reverse(lua.checkString(1)))
       1
     })
     lua.setField(-2, "reverse")
 
     lua.pushScalaFunction(lua => {
       val string = lua.checkString(1)
-      val start = math.max(0, lua.checkInteger(2) match {
+      val sLength = ExtendedUnicodeHelper.length(string)
-        case i if i < 0 => string.length + i
+      val start = lua.checkInteger(2) match {
-        case i => i - 1
+        case i if i < 0 => string.offsetByCodePoints(string.length, math.max(i, -sLength))
-      })
+        case i => string.offsetByCodePoints(0, math.min(i - 1, sLength))
+      }
       val end =
-        if (lua.getTop > 2) math.min(string.length, lua.checkInteger(3) match {
+        if (lua.getTop > 2) lua.checkInteger(3) match {
-          case i if i < 0 => string.length + i + 1
+          case i if i < 0 => string.offsetByCodePoints(string.length, math.max(i + 1, -sLength))
-          case i => i
+          case i => string.offsetByCodePoints(0, math.min(i, sLength))
-        })
+        }
         else string.length
       if (end <= start) lua.pushString("")
       else lua.pushString(string.substring(start, end))
@@ -70,7 +75,9 @@ class UnicodeAPI(owner: NativeLuaArchitecture) extends NativeLuaAPI(owner) {
 
     lua.pushScalaFunction(lua => {
       val value = lua.checkString(1)
-      lua.pushInteger(value.toCharArray.map(ch => math.max(1, FontUtils.wcwidth(ch))).sum)
+      lua.pushInteger(value.codePoints().map(new IntUnaryOperator {
+        override def applyAsInt(ch: Int): Int = math.max(1, FontUtils.wcwidth(ch))
+      }).sum)
       1
     })
     lua.setField(-2, "wlen")
@@ -81,8 +88,8 @@ class UnicodeAPI(owner: NativeLuaArchitecture) extends NativeLuaAPI(owner) {
       var width = 0
       var end = 0
       while (width < count) {
-        width += math.max(1, FontUtils.wcwidth(value(end)))
+        width += math.max(1, FontUtils.wcwidth(value.codePointAt(end)))
-        end += 1
+        end = value.offsetByCodePoints(end, 1)
       }
       if (end > 1) lua.pushString(value.substring(0, end - 1))
       else lua.pushString("")

src/main/scala/li/cil/oc/server/machine/luaj/UnicodeAPI.scala

@@ -1,6 +1,7 @@
 package li.cil.oc.server.machine.luaj
 
-import li.cil.oc.util.FontUtils
+import java.util.function.IntUnaryOperator
+import li.cil.oc.util.{ExtendedUnicodeHelper, FontUtils}
 import li.cil.oc.util.ScalaClosure._
 import li.cil.repack.org.luaj.vm2.LuaValue
 import li.cil.repack.org.luaj.vm2.Varargs
@@ -14,23 +15,31 @@ class UnicodeAPI(owner: LuaJLuaArchitecture) extends LuaJAPI(owner) {
 
     unicode.set("upper", (args: Varargs) => LuaValue.valueOf(args.checkjstring(1).toUpperCase))
 
-    unicode.set("char", (args: Varargs) => LuaValue.valueOf(String.valueOf((1 to args.narg).map(args.checkint).map(_.toChar).toArray)))
+    unicode.set("char", (args: Varargs) => {
+      val builder = new java.lang.StringBuilder()
+      (1 to args.narg).map(args.checkint).foreach(builder.appendCodePoint)
+      LuaValue.valueOf(builder.toString)
+    })
 
-    unicode.set("len", (args: Varargs) => LuaValue.valueOf(args.checkjstring(1).length))
+    unicode.set("len", (args: Varargs) => {
+      val s = args.checkjstring(1)
+      LuaValue.valueOf(s.codePointCount(0, s.length))
+    })
 
     unicode.set("reverse", (args: Varargs) => LuaValue.valueOf(args.checkjstring(1).reverse))
 
     unicode.set("sub", (args: Varargs) => {
       val string = args.checkjstring(1)
-      val start = math.max(0, args.checkint(2) match {
+      val sLength = ExtendedUnicodeHelper.length(string)
-        case i if i < 0 => string.length + i
+      val start = args.checkint(2) match {
-        case i => i - 1
+        case i if i < 0 => string.offsetByCodePoints(string.length, math.max(i, -sLength))
-      })
+        case i => string.offsetByCodePoints(0, math.min(i - 1, sLength))
+      }
       val end =
-        if (args.narg > 2) math.min(string.length, args.checkint(3) match {
+        if (args.narg > 2) args.checkint(3) match {
-          case i if i < 0 => string.length + i + 1
+          case i if i < 0 => string.offsetByCodePoints(string.length, math.max(i + 1, -sLength))
-          case i => i
+          case i => string.offsetByCodePoints(0, math.min(i, sLength))
-        })
+        }
         else string.length
       if (end <= start) LuaValue.valueOf("")
       else LuaValue.valueOf(string.substring(start, end))
@@ -44,7 +53,9 @@ class UnicodeAPI(owner: LuaJLuaArchitecture) extends LuaJAPI(owner) {
 
     unicode.set("wlen", (args: Varargs) => {
       val value = args.checkjstring(1)
-      LuaValue.valueOf(value.toCharArray.map(ch => math.max(1, FontUtils.wcwidth(ch))).sum)
+      LuaValue.valueOf(value.codePoints.map(new IntUnaryOperator {
+        override def applyAsInt(ch: Int): Int = math.max(1, FontUtils.wcwidth(ch))
+      }).sum)
     })
 
     unicode.set("wtrunc", (args: Varargs) => {
@@ -53,8 +64,8 @@ class UnicodeAPI(owner: LuaJLuaArchitecture) extends LuaJAPI(owner) {
       var width = 0
       var end = 0
       while (width < count) {
-        width += math.max(1, FontUtils.wcwidth(value(end)))
+        width += math.max(1, FontUtils.wcwidth(value.codePointAt(end)))
-        end += 1
+        end = value.offsetByCodePoints(end, 1)
       }
       if (end > 1) LuaValue.valueOf(value.substring(0, end - 1))
       else LuaValue.valueOf("")

src/main/scala/li/cil/oc/util/FontUtils.scala

@@ -8,14 +8,9 @@ import li.cil.oc.OpenComputers
 
 object FontUtils {
   private val defined_double_wide: BitSet = BitSet()
-  
+
-  // font.hex actually has some codepoints larger than 0x10000
+  // theoretical Unicode maximum
-  // but, UnicodeAPI.scala is using java's Integer.ToChar which only supports the utf-16 range
+  val codepoint_limit: Int = 0x110000
-  // and thus will truncate any incoming codepoint, forcing it below 0x10000
-  // I believe the solution is to use StringBuffer.appendCodePoint
-  // but that change would deserve a bit of testing first, postponing for a later update
-  // review http://www.oracle.com/us/technologies/java/supplementary-142654.html
-  val codepoint_limit: Int = 0x10000
   def wcwidth(charCode: Int): Int = if (defined_double_wide(charCode)) 2 else 1
 
   {

src/main/scala/li/cil/oc/util/TextBuffer.scala

@@ -5,6 +5,8 @@ import li.cil.oc.api
 import net.minecraft.nbt._
 import net.minecraftforge.common.util.Constants.NBT
 
+import java.lang
+
 /**
  * This stores chars in a 2D-Array and provides some manipulation functions.
  *
@@ -64,7 +66,7 @@ class TextBuffer(var width: Int, var height: Int, initialFormat: PackedColor.Col
 
   var color = Array.fill(height, width)(packed)
 
-  var buffer = Array.fill(height, width)(' ')
+  var buffer = Array.fill(height, width)(0x20)
 
   /** The current buffer size in columns by rows. */
   def size = (width, height)
@@ -80,7 +82,7 @@ class TextBuffer(var width: Int, var height: Int, initialFormat: PackedColor.Col
     val (iw, ih) = value
     val (w, h) = (math.max(iw, 1), math.max(ih, 1))
     if (width != w || height != h) {
-      val newBuffer = Array.fill(h, w)(' ')
+      val newBuffer = Array.fill(h, w)(0x20)
       val newColor = Array.fill(h, w)(packed)
       (0 until math.min(h, height)).foreach(y => {
         Array.copy(buffer(y), 0, newBuffer(y), 0, math.min(w, width))
@@ -103,17 +105,20 @@ class TextBuffer(var width: Int, var height: Int, initialFormat: PackedColor.Col
   }
 
   /** String based fill starting at a specified location. */
-  def set(col: Int, row: Int, s: String, vertical: Boolean): Boolean =
+  def set(col: Int, row: Int, s: String, vertical: Boolean): Boolean = {
+    val sLength = ExtendedUnicodeHelper.length(s)
     if (vertical) {
       if (col < 0 || col >= width) false
       else {
         var changed = false
-        for (y <- row until math.min(row + s.length, height)) if (y >= 0) {
+        var cx = 0
+        for (y <- row until math.min(row + sLength, height)) if (y >= 0) {
           val line = buffer(y)
           val lineColor = color(y)
-          val c = s(y - row)
+          val c = s.codePointAt(cx)
           changed = changed || (line(col) != c) || (lineColor(col) != packed)
           setChar(line, lineColor, col, c)
+          cx = s.offsetByCodePoints(cx, 1)
         }
         changed
       }
@@ -125,18 +130,21 @@ class TextBuffer(var width: Int, var height: Int, initialFormat: PackedColor.Col
         val line = buffer(row)
         val lineColor = color(row)
         var bx = math.max(col, 0)
-        for (x <- bx until math.min(col + s.length, width) if bx < line.length) {
+        var cx = 0
-          val c = s(x - col)
+        for (x <- bx until math.min(col + sLength, width) if bx < line.length) {
+          val c = s.codePointAt(cx)
           changed = changed || (line(bx) != c) || (lineColor(bx) != packed)
           setChar(line, lineColor, bx, c)
           bx += math.max(1, FontUtils.wcwidth(c))
+          cx = s.offsetByCodePoints(cx, 1)
         }
         changed
       }
     }
+  }
 
   /** Fills an area of the buffer with the specified character. */
-  def fill(col: Int, row: Int, w: Int, h: Int, c: Char): Boolean = {
+  def fill(col: Int, row: Int, w: Int, h: Int, c: Int): Boolean = {
     // Anything to do at all?
     if (w <= 0 || h <= 0) return false
     if (col + w < 0 || row + h < 0 || col >= width || row >= height) return false
@@ -234,7 +242,7 @@ class TextBuffer(var width: Int, var height: Int, initialFormat: PackedColor.Col
     changed
   }
 
-  private def setChar(line: Array[Char], lineColor: Array[Short], x: Int, c: Char) {
+  private def setChar(line: Array[Int], lineColor: Array[Short], x: Int, c: Int) {
     if (FontUtils.wcwidth(c) > 1 && x >= line.length - 1) {
       // Don't allow setting wide chars in right-most col.
       return
@@ -260,7 +268,12 @@ class TextBuffer(var width: Int, var height: Int, initialFormat: PackedColor.Col
     val b = nbt.getTagList("buffer", NBT.TAG_STRING)
     for (i <- 0 until math.min(h, b.tagCount)) {
       val value = b.getStringTagAt(i)
-      System.arraycopy(value.toCharArray, 0, buffer(i), 0, math.min(value.length, buffer(i).length))
+      val valueIt = value.codePoints.iterator()
+      var j = 0
+      while (j < buffer(i).length && valueIt.hasNext) {
+        buffer(i)(j) = valueIt.nextInt()
+        j += 1
+      }
     }
 
     val depth = api.internal.TextBuffer.ColorDepth.values.apply(nbt.getInteger("depth") min (api.internal.TextBuffer.ColorDepth.values.length - 1) max 0)
@@ -280,7 +293,7 @@ class TextBuffer(var width: Int, var height: Int, initialFormat: PackedColor.Col
 
     val b = new NBTTagList()
     for (i <- 0 until height) {
-      b.appendTag(new NBTTagString(String.valueOf(buffer(i))))
+      b.appendTag(new NBTTagString(lineToString(i)))
     }
     nbt.setTag("buffer", b)
 
@@ -294,14 +307,29 @@ class TextBuffer(var width: Int, var height: Int, initialFormat: PackedColor.Col
     NbtDataStream.setShortArray(nbt, "colors", color.flatten.map(_.toShort))
   }
 
-  override def toString: String = {
+  def lineToString(y: Int): String = {
-    val b = StringBuilder.newBuilder
+    val b = new lang.StringBuilder()
     if (buffer.length > 0) {
-      b.appendAll(buffer(0))
+      for (x <- 0 until width) {
-      for (y <- 1 until height) {
+        b.appendCodePoint(buffer(y)(x))
-        b.append('\n').appendAll(buffer(y))
       }
     }
-    b.toString()
+    b.toString
+  }
+
+  override def toString: String = {
+    val b = new lang.StringBuilder()
+    if (buffer.length > 0) {
+      for (x <- 0 until width) {
+        b.appendCodePoint(buffer(0)(x))
+      }
+      for (y <- 1 until height) {
+        b.append('\n')
+        for (x <- 0 until width) {
+          b.appendCodePoint(buffer(y)(x))
+        }
+      }
+    }
+    b.toString
   }
 }