From 1ed453395088493a10fb947be6def30a98c751b0 Mon Sep 17 00:00:00 2001 From: Adrian Siekierka Date: Sun, 15 Mar 2020 17:59:41 +0100 Subject: [PATCH] Support for non-BMP codepoints, take 2 --- .../li/cil/oc/api/internal/TextBuffer.java | 49 +++++++++++++++ .../li/cil/oc/util/ExtendedUnicodeHelper.java | 41 ++++++++++++ .../li/cil/oc/client/PacketHandler.scala | 8 +-- .../renderer/font/DynamicFontRenderer.scala | 12 ++-- .../client/renderer/font/FontParserHex.java | 22 ++++--- .../renderer/font/StaticFontRenderer.scala | 4 +- .../renderer/font/TextureFontRenderer.scala | 22 ++++--- .../li/cil/oc/common/PacketBuilder.scala | 6 ++ .../li/cil/oc/common/PacketHandler.scala | 7 +++ .../oc/common/component/GpuTextBuffer.scala | 2 +- .../cil/oc/common/component/TextBuffer.scala | 16 ++--- .../component/traits/TextBufferProxy.scala | 40 +++++++++--- .../li/cil/oc/common/tileentity/Screen.scala | 2 +- .../scala/li/cil/oc/server/PacketSender.scala | 8 +-- .../oc/server/component/GraphicsCard.scala | 17 +++-- .../oc/server/machine/luac/UnicodeAPI.scala | 37 ++++++----- .../oc/server/machine/luaj/UnicodeAPI.scala | 39 +++++++----- src/main/scala/li/cil/oc/util/FontUtils.scala | 11 +--- .../scala/li/cil/oc/util/TextBuffer.scala | 62 ++++++++++++++----- 19 files changed, 291 insertions(+), 114 deletions(-) create mode 100644 src/main/java/li/cil/oc/util/ExtendedUnicodeHelper.java diff --git a/src/main/java/li/cil/oc/api/internal/TextBuffer.java b/src/main/java/li/cil/oc/api/internal/TextBuffer.java index 0d21340fc..aa1d27642 100644 --- a/src/main/java/li/cil/oc/api/internal/TextBuffer.java +++ b/src/main/java/li/cil/oc/api/internal/TextBuffer.java @@ -301,9 +301,24 @@ public interface TextBuffer extends ManagedEnvironment, Persistable { * @param width the width of the area to fill. * @param height the height of the area to fill. * @param value the character to fill the area with. + * @deprecated Please use the int variant. */ + @Deprecated void fill(int column, int row, int width, int height, char value); + /** + * Fill a portion of the text buffer. + *

+ * This will set the area's colors to the currently active ones. + * + * @param column the starting horizontal index of the area to fill. + * @param row the starting vertical index of the area to fill. + * @param width the width of the area to fill. + * @param height the height of the area to fill. + * @param value the code point to fill the area with. + */ + void fill(int column, int row, int width, int height, int value); + /** * Write a string into the text buffer. *
@@ -322,9 +337,20 @@ public interface TextBuffer extends ManagedEnvironment, Persistable { * @param column the horizontal index. * @param row the vertical index. * @return the character at that index. + * @deprecated Please use getCodePoint going forward. */ + @Deprecated char get(int column, int row); + /** + * Get the code point in the text buffer at the specified location. + * + * @param column the horizontal index. + * @param row the vertical index. + * @return the character at that index. + */ + int getCodePoint(int column, int row); + /** * Get the foreground color of the text buffer at the specified location. *
@@ -385,9 +411,32 @@ public interface TextBuffer extends ManagedEnvironment, Persistable { * @param column the horizontal index. * @param row the vertical index. * @param text the text to write. + * @deprecated Please use the int[][] variant. */ + @Deprecated void rawSetText(int column, int row, char[][] text); + /** + * Overwrites a portion of the text in raw mode. + *

+ * This will copy the given char array into the buffer, starting at the + * specified column and row. The array is expected to be indexed row- + * first, i.e. the first dimension is the vertical axis, the second + * the horizontal. + *

+ * Important: this performs no checks as to whether something + * actually changed. It will always send the changed patch to clients. + * It will also not crop the specified array to the actually used range. + * In other words, this is not intended to be exposed as-is to user code, + * it should always be called with validated, and, as necessary, cropped + * values. + * + * @param column the horizontal index. + * @param row the vertical index. + * @param text the text code points to write. + */ + void rawSetText(int column, int row, int[][] text); + /** * Overwrites a portion of the foreground color information in raw mode. *
diff --git a/src/main/java/li/cil/oc/util/ExtendedUnicodeHelper.java b/src/main/java/li/cil/oc/util/ExtendedUnicodeHelper.java new file mode 100644 index 000000000..9ea4ce20a --- /dev/null +++ b/src/main/java/li/cil/oc/util/ExtendedUnicodeHelper.java @@ -0,0 +1,41 @@ +package li.cil.oc.util; + +/** + * Helper functions for handling strings with characters outside of the Unicode BMP. + */ +public final class ExtendedUnicodeHelper { + private ExtendedUnicodeHelper() { + + } + + public static int length(String s) { + return s.codePointCount(0, s.length()); + } + + public static String reverse(String s) { + StringBuilder sb = new StringBuilder(); + for (int i = s.length() - 1; i >= 0; i--) { + char c = s.charAt(i); + if (Character.isLowSurrogate(c) && i > 0) { + i--; + char c2 = s.charAt(i); + if (Character.isHighSurrogate(c2)) { + sb.append(c2).append(c); + } else { + // Invalid surrogate pair? + sb.append(c).append(c2); + } + } else { + sb.append(c); + } + } + return sb.toString(); + } + + public static String substring(String s, int start, int end) { + return s.substring( + s.offsetByCodePoints(0, start), + s.offsetByCodePoints(0, end) + ); + } +} diff --git a/src/main/scala/li/cil/oc/client/PacketHandler.scala b/src/main/scala/li/cil/oc/client/PacketHandler.scala index fd40c0dde..713f7b540 100644 --- a/src/main/scala/li/cil/oc/client/PacketHandler.scala +++ b/src/main/scala/li/cil/oc/client/PacketHandler.scala @@ -668,7 +668,7 @@ object PacketHandler extends CommonPacketHandler { val row = p.readInt() val w = p.readInt() val h = p.readInt() - val c = p.readChar() + val c = p.readMedium() buffer.fill(col, row, w, h, c) } @@ -737,12 +737,12 @@ object PacketHandler extends CommonPacketHandler { val row = p.readInt() val rows = p.readShort() - val text = new Array[Array[Char]](rows) + val text = new Array[Array[Int]](rows) for (y <- 0 until rows) { val cols = p.readShort() - val line = new Array[Char](cols) + val line = new Array[Int](cols) for (x <- 0 until cols) { - line(x) = p.readChar() + line(x) = p.readMedium() } text(y) = line } diff --git a/src/main/scala/li/cil/oc/client/renderer/font/DynamicFontRenderer.scala b/src/main/scala/li/cil/oc/client/renderer/font/DynamicFontRenderer.scala index 0d3df8a69..8df2448b3 100644 --- a/src/main/scala/li/cil/oc/client/renderer/font/DynamicFontRenderer.scala +++ b/src/main/scala/li/cil/oc/client/renderer/font/DynamicFontRenderer.scala @@ -24,7 +24,7 @@ class DynamicFontRenderer extends TextureFontRenderer with IResourceManagerReloa private val textures = mutable.ArrayBuffer.empty[CharTexture] - private val charMap = mutable.Map.empty[Char, DynamicFontRenderer.CharIcon] + private val charMap = mutable.Map.empty[Int, DynamicFontRenderer.CharIcon] private var activeTexture: CharTexture = _ @@ -63,18 +63,18 @@ class DynamicFontRenderer extends TextureFontRenderer with IResourceManagerReloa RenderState.checkError(getClass.getName + ".bindTexture") } - override protected def generateChar(char: Char) { + override protected def generateChar(char: Int) { charMap.getOrElseUpdate(char, createCharIcon(char)) } - override protected def drawChar(tx: Float, ty: Float, char: Char) { + override protected def drawChar(tx: Float, ty: Float, char: Int) { charMap.get(char) match { case Some(icon) if icon.texture == activeTexture => icon.draw(tx, ty) case _ => } } - private def createCharIcon(char: Char): DynamicFontRenderer.CharIcon = { + private def createCharIcon(char: Int): DynamicFontRenderer.CharIcon = { if (FontUtils.wcwidth(char) < 1 || glyphProvider.getGlyph(char) == null) { if (char == '?') null else charMap.getOrElseUpdate('?', createCharIcon('?')) @@ -125,9 +125,9 @@ object DynamicFontRenderer { GL11.glBindTexture(GL11.GL_TEXTURE_2D, id) } - def isFull(char: Char) = chars + FontUtils.wcwidth(char) > capacity + def isFull(char: Int) = chars + FontUtils.wcwidth(char) > capacity - def add(char: Char) = { + def add(char: Int) = { val glyphWidth = FontUtils.wcwidth(char) val w = owner.charWidth * glyphWidth val h = owner.charHeight diff --git a/src/main/scala/li/cil/oc/client/renderer/font/FontParserHex.java b/src/main/scala/li/cil/oc/client/renderer/font/FontParserHex.java index 78eee0844..681c7352f 100644 --- a/src/main/scala/li/cil/oc/client/renderer/font/FontParserHex.java +++ b/src/main/scala/li/cil/oc/client/renderer/font/FontParserHex.java @@ -1,5 +1,7 @@ package li.cil.oc.client.renderer.font; +import gnu.trove.map.TIntObjectMap; +import gnu.trove.map.hash.TIntObjectHashMap; import li.cil.oc.OpenComputers; import li.cil.oc.Settings; import li.cil.oc.util.FontUtils; @@ -17,13 +19,10 @@ public class FontParserHex implements IGlyphProvider { private static final byte[] OPAQUE = {(byte) 255, (byte) 255, (byte) 255, (byte) 255}; private static final byte[] TRANSPARENT = {0, 0, 0, 0}; - private final byte[][] glyphs = new byte[FontUtils.codepoint_limit()][]; + private final TIntObjectMap glyphs = new TIntObjectHashMap<>(); @Override public void initialize() { - for (int i = 0; i < glyphs.length; ++i) { - glyphs[i] = null; - } try { final InputStream font = Minecraft.getMinecraft().getResourceManager().getResource(new ResourceLocation(Settings.resourceDomain(), "font.hex")).getInputStream(); try { @@ -34,7 +33,10 @@ public class FontParserHex implements IGlyphProvider { while ((line = input.readLine()) != null) { final String[] info = line.split(":"); final int charCode = Integer.parseInt(info[0], 16); - if (charCode < 0 || charCode >= glyphs.length) continue; // Out of bounds. + if (charCode < 0 || charCode >= FontUtils.codepoint_limit()) { + OpenComputers.log().warn(String.format("Unicode font contained unexpected glyph: U+%04X, ignoring", charCode)); + continue; // Out of bounds. + } final int expectedWidth = FontUtils.wcwidth(charCode); if (expectedWidth < 1) continue; // Skip control characters. // Two chars representing one byte represent one row of eight pixels. @@ -44,10 +46,10 @@ public class FontParserHex implements IGlyphProvider { for (int i = 0; i < glyph.length; i++) { glyph[i] = (byte) Integer.parseInt(info[1].substring(i * 2, i * 2 + 2), 16); } - if (glyphs[charCode] == null) { + if (!glyphs.containsKey(charCode)) { glyphCount++; } - glyphs[charCode] = glyph; + glyphs.put(charCode, glyph); } else if (Settings.get().logHexFontErrors()) { OpenComputers.log().warn(String.format("Size of glyph for code point U+%04X (%s) in font (%d) does not match expected width (%d), ignoring.", charCode, String.valueOf((char) charCode), glyphWidth, expectedWidth)); } @@ -67,9 +69,11 @@ public class FontParserHex implements IGlyphProvider { @Override public ByteBuffer getGlyph(int charCode) { - if (charCode < 0 || charCode >= glyphs.length || glyphs[charCode] == null || glyphs[charCode].length == 0) + if (!glyphs.containsKey(charCode)) + return null; + final byte[] glyph = glyphs.get(charCode); + if (glyph == null || glyph.length <= 0) return null; - final byte[] glyph = glyphs[charCode]; final ByteBuffer buffer = BufferUtils.createByteBuffer(glyph.length * getGlyphWidth() * 4); for (byte aGlyph : glyph) { int c = ((int) aGlyph) & 0xFF; diff --git a/src/main/scala/li/cil/oc/client/renderer/font/StaticFontRenderer.scala b/src/main/scala/li/cil/oc/client/renderer/font/StaticFontRenderer.scala index 3a1944b03..2e5fa93c4 100644 --- a/src/main/scala/li/cil/oc/client/renderer/font/StaticFontRenderer.scala +++ b/src/main/scala/li/cil/oc/client/renderer/font/StaticFontRenderer.scala @@ -50,7 +50,7 @@ class StaticFontRenderer extends TextureFontRenderer { } } - override protected def drawChar(tx: Float, ty: Float, char: Char) { + override protected def drawChar(tx: Float, ty: Float, char: Int) { val index = 1 + (chars.indexOf(char) match { case -1 => chars.indexOf('?') case i => i @@ -69,5 +69,5 @@ class StaticFontRenderer extends TextureFontRenderer { GL11.glVertex3d(tx - dw, ty - dh, 0) } - override protected def generateChar(char: Char) {} + override protected def generateChar(char: Int) {} } diff --git a/src/main/scala/li/cil/oc/client/renderer/font/TextureFontRenderer.scala b/src/main/scala/li/cil/oc/client/renderer/font/TextureFontRenderer.scala index 97c290a2a..6194a0b9b 100644 --- a/src/main/scala/li/cil/oc/client/renderer/font/TextureFontRenderer.scala +++ b/src/main/scala/li/cil/oc/client/renderer/font/TextureFontRenderer.scala @@ -1,9 +1,7 @@ package li.cil.oc.client.renderer.font import li.cil.oc.Settings -import li.cil.oc.util.PackedColor -import li.cil.oc.util.RenderState -import li.cil.oc.util.TextBuffer +import li.cil.oc.util.{ExtendedUnicodeHelper, PackedColor, RenderState, TextBuffer} import org.lwjgl.opengl.GL11 /** @@ -30,6 +28,12 @@ abstract class TextureFontRenderer { } } + def generateChars(chars: Array[Int]) { + for (char <- chars) { + generateChar(char) + } + } + def drawBuffer(buffer: TextBuffer, viewportWidth: Int, viewportHeight: Int) { val format = buffer.format @@ -113,6 +117,8 @@ abstract class TextureFontRenderer { } def drawString(s: String, x: Int, y: Int): Unit = { + val sLength = ExtendedUnicodeHelper.length(s) + GL11.glPushMatrix() GL11.glPushAttrib(GL11.GL_ALL_ATTRIB_BITS) @@ -124,13 +130,15 @@ abstract class TextureFontRenderer { bindTexture(i) GL11.glBegin(GL11.GL_QUADS) var tx = 0f - for (n <- 0 until s.length) { - val ch = s.charAt(n) + var cx = 0 + for (n <- 0 until sLength) { + val ch = s.codePointAt(cx) // Don't render whitespace. if (ch != ' ') { drawChar(tx, 0, ch) } tx += charWidth + cx = s.offsetByCodePoints(cx, 1) } GL11.glEnd() } @@ -147,9 +155,9 @@ abstract class TextureFontRenderer { protected def bindTexture(index: Int): Unit - protected def generateChar(char: Char): Unit + protected def generateChar(char: Int): Unit - protected def drawChar(tx: Float, ty: Float, char: Char): Unit + protected def drawChar(tx: Float, ty: Float, char: Int): Unit private def drawQuad(color: Int, x: Int, y: Int, width: Int) = if (color != 0 && width > 0) { val x0 = x * charWidth diff --git a/src/main/scala/li/cil/oc/common/PacketBuilder.scala b/src/main/scala/li/cil/oc/common/PacketBuilder.scala index aa66535e8..c1713b5b6 100644 --- a/src/main/scala/li/cil/oc/common/PacketBuilder.scala +++ b/src/main/scala/li/cil/oc/common/PacketBuilder.scala @@ -58,6 +58,12 @@ abstract class PacketBuilder(stream: OutputStream) extends DataOutputStream(stre } } + def writeMedium(v: Int) = { + writeByte(v & 0xFF) + writeByte((v >> 8) & 0xFF) + writeByte((v >> 16) & 0xFF) + } + def writePacketType(pt: PacketType.Value) = writeByte(pt.id) def sendToAllPlayers() = OpenComputers.channel.sendToAll(packet) diff --git a/src/main/scala/li/cil/oc/common/PacketHandler.scala b/src/main/scala/li/cil/oc/common/PacketHandler.scala index ca9df6ec2..3a5cb3607 100644 --- a/src/main/scala/li/cil/oc/common/PacketHandler.scala +++ b/src/main/scala/li/cil/oc/common/PacketHandler.scala @@ -132,6 +132,13 @@ abstract class PacketHandler { else null } + def readMedium(): Int = { + val c0 = readUnsignedByte() + val c1 = readUnsignedByte() + val c2 = readUnsignedByte() + (c0) | (c1 << 8) | (c2 << 16) + } + def readPacketType() = PacketType(readByte()) } diff --git a/src/main/scala/li/cil/oc/common/component/GpuTextBuffer.scala b/src/main/scala/li/cil/oc/common/component/GpuTextBuffer.scala index 2d9f8f4df..72526e6e5 100644 --- a/src/main/scala/li/cil/oc/common/component/GpuTextBuffer.scala +++ b/src/main/scala/li/cil/oc/common/component/GpuTextBuffer.scala @@ -28,7 +28,7 @@ class GpuTextBuffer(val owner: String, val id: Int, val data: li.cil.oc.util.Tex override def onBufferSet(col: Int, row: Int, s: String, vertical: Boolean): Unit = dirty = true override def onBufferColorChange(): Unit = dirty = true override def onBufferCopy(col: Int, row: Int, w: Int, h: Int, tx: Int, ty: Int): Unit = dirty = true - override def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Char): Unit = dirty = true + override def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Int): Unit = dirty = true override def load(nbt: NBTTagCompound): Unit = { // the data is initially dirty because other devices don't know about it yet diff --git a/src/main/scala/li/cil/oc/common/component/TextBuffer.scala b/src/main/scala/li/cil/oc/common/component/TextBuffer.scala index f3127270f..a1df4080c 100644 --- a/src/main/scala/li/cil/oc/common/component/TextBuffer.scala +++ b/src/main/scala/li/cil/oc/common/component/TextBuffer.scala @@ -318,7 +318,7 @@ class TextBuffer(val host: EnvironmentHost) extends prefab.ManagedEnvironment wi proxy.onBufferCopy(col, row, w, h, tx, ty) } - override def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Char): Unit = { + override def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Int): Unit = { proxy.onBufferFill(col, row, w, h, c) } @@ -338,7 +338,7 @@ class TextBuffer(val host: EnvironmentHost) extends prefab.ManagedEnvironment wi proxy.onBufferRamDestroy(ram) } - override def rawSetText(col: Int, row: Int, text: Array[Array[Char]]): Unit = { + override def rawSetText(col: Int, row: Int, text: Array[Array[Int]]): Unit = { super.rawSetText(col, row, text) proxy.onBufferRawSetText(col, row, text) } @@ -538,7 +538,7 @@ object TextBuffer { def onBufferDepthChange(depth: api.internal.TextBuffer.ColorDepth): Unit - def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Char) { + def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Int) { owner.relativeLitArea = -1 } @@ -571,7 +571,7 @@ object TextBuffer { owner.relativeLitArea = -1 } - def onBufferRawSetText(col: Int, row: Int, text: Array[Array[Char]]) { + def onBufferRawSetText(col: Int, row: Int, text: Array[Array[Int]]) { owner.relativeLitArea = -1 } @@ -630,7 +630,7 @@ object TextBuffer { markDirty() } - override def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Char) { + override def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Int) { super.onBufferFill(col, row, w, h, c) markDirty() } @@ -732,7 +732,7 @@ object TextBuffer { owner.synchronized(ServerPacketSender.appendTextBufferDepthChange(owner.pendingCommands, depth)) } - override def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Char) { + override def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Int) { super.onBufferFill(col, row, w, h, c) owner.host.markChanged() owner.synchronized(ServerPacketSender.appendTextBufferFill(owner.pendingCommands, col, row, w, h, c)) @@ -789,7 +789,7 @@ object TextBuffer { owner.synchronized(ServerPacketSender.appendTextBufferRamDestroy(owner.pendingCommands, ram.owner, ram.id)) } - override def onBufferRawSetText(col: Int, row: Int, text: Array[Array[Char]]) { + override def onBufferRawSetText(col: Int, row: Int, text: Array[Array[Int]]) { super.onBufferRawSetText(col, row, text) owner.host.markChanged() owner.synchronized(ServerPacketSender.appendTextBufferRawSetText(owner.pendingCommands, col, row, text)) @@ -844,7 +844,7 @@ object TextBuffer { stack.getTagCompound.removeTag(Settings.namespace + "clipboard") if (line >= 0 && line < owner.getViewportHeight) { - val text = new String(owner.data.buffer(line)).trim + val text = owner.data.lineToString(line) if (!Strings.isNullOrEmpty(text)) { stack.getTagCompound.setString(Settings.namespace + "clipboard", text) } diff --git a/src/main/scala/li/cil/oc/common/component/traits/TextBufferProxy.scala b/src/main/scala/li/cil/oc/common/component/traits/TextBufferProxy.scala index 41a9ea286..2bc5d0a64 100644 --- a/src/main/scala/li/cil/oc/common/component/traits/TextBufferProxy.scala +++ b/src/main/scala/li/cil/oc/common/component/traits/TextBufferProxy.scala @@ -3,7 +3,7 @@ package li.cil.oc.common.component.traits import li.cil.oc.util import li.cil.oc.api import li.cil.oc.api.internal.TextBuffer -import li.cil.oc.util.PackedColor +import li.cil.oc.util.{ExtendedUnicodeHelper, PackedColor} trait TextBufferProxy extends api.internal.TextBuffer { def data: util.TextBuffer @@ -70,32 +70,47 @@ trait TextBufferProxy extends api.internal.TextBuffer { if (data.copy(col, row, w, h, tx, ty)) onBufferCopy(col, row, w, h, tx, ty) - def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Char): Unit = {} + def onBufferFill(col: Int, row: Int, w: Int, h: Int, c: Int): Unit = {} def fill(col: Int, row: Int, w: Int, h: Int, c: Char): Unit = + fill(col, row, w, h, c.toInt) + + def fill(col: Int, row: Int, w: Int, h: Int, c: Int): Unit = if (data.fill(col, row, w, h, c)) onBufferFill(col, row, w, h, c) def onBufferSet(col: Int, row: Int, s: String, vertical: Boolean): Unit = {} - def set(col: Int, row: Int, s: String, vertical: Boolean): Unit = - if (col < data.width && (col >= 0 || -col < s.length)) { + private def truncate(s: String, sLength: Int, leftOffset: Int, maxWidth: Int): String = { + val subFrom = s.offsetByCodePoints(0, leftOffset) + val width = math.min(sLength, maxWidth) + if (width <= 0) "" + else if ((sLength - leftOffset) <= width) s + else s.substring(subFrom, s.offsetByCodePoints(subFrom, width)) + } + + def set(col: Int, row: Int, s: String, vertical: Boolean): Unit = { + val sLength = ExtendedUnicodeHelper.length(s) + if (col < data.width && (col >= 0 || -col < sLength)) { // Make sure the string isn't longer than it needs to be, in particular to // avoid sending too much data to our clients. val (x, y, truncated) = if (vertical) { - if (row < 0) (col, 0, s.substring(-row)) - else (col, row, s.substring(0, math.min(s.length, data.height - row))) + if (row < 0) (col, 0, truncate(s, sLength, -row, data.height)) + else (col, row, truncate(s, sLength, 0, data.height - row)) } else { - if (col < 0) (0, row, s.substring(-col)) - else (col, row, s.substring(0, math.min(s.length, data.width - col))) + if (col < 0) (0, row, truncate(s, sLength, -col, data.width)) + else (col, row, truncate(s, sLength, 0, data.width - col)) } if (data.set(x, y, truncated, vertical)) onBufferSet(x, row, truncated, vertical) } + } - def get(col: Int, row: Int): Char = data.get(col, row) + def get(col: Int, row: Int): Char = data.get(col, row).toChar + + def getCodePoint(col: Int, row: Int): Int = data.get(col, row) override def getForegroundColor(column: Int, row: Int): Int = if (isForegroundFromPalette(column, row)) { @@ -126,6 +141,13 @@ trait TextBufferProxy extends api.internal.TextBuffer { } } + override def rawSetText(col: Int, row: Int, text: Array[Array[Int]]): Unit = { + for (y <- row until ((row + text.length) min data.height)) { + val line = text(y - row) + Array.copy(line, 0, data.buffer(y), col, line.length min data.width) + } + } + override def rawSetForeground(col: Int, row: Int, color: Array[Array[Int]]): Unit = { for (y <- row until ((row + color.length) min data.height)) { val line = color(y - row) diff --git a/src/main/scala/li/cil/oc/common/tileentity/Screen.scala b/src/main/scala/li/cil/oc/common/tileentity/Screen.scala index 8347a17be..e6f8939af 100644 --- a/src/main/scala/li/cil/oc/common/tileentity/Screen.scala +++ b/src/main/scala/li/cil/oc/common/tileentity/Screen.scala @@ -240,7 +240,7 @@ class Screen(var tier: Int) extends traits.TextBuffer with SidedEnvironment with val h = buffer.getHeight buffer.setForegroundColor(0xFFFFFF, false) buffer.setBackgroundColor(0x000000, false) - buffer.fill(0, 0, w, h, ' ') + buffer.fill(0, 0, w, h, 0x20) } }) } diff --git a/src/main/scala/li/cil/oc/server/PacketSender.scala b/src/main/scala/li/cil/oc/server/PacketSender.scala index eaead252f..bf4ec0d47 100644 --- a/src/main/scala/li/cil/oc/server/PacketSender.scala +++ b/src/main/scala/li/cil/oc/server/PacketSender.scala @@ -618,14 +618,14 @@ object PacketSender { pb.writeInt(value.ordinal) } - def appendTextBufferFill(pb: PacketBuilder, col: Int, row: Int, w: Int, h: Int, c: Char) { + def appendTextBufferFill(pb: PacketBuilder, col: Int, row: Int, w: Int, h: Int, c: Int) { pb.writePacketType(PacketType.TextBufferMultiFill) pb.writeInt(col) pb.writeInt(row) pb.writeInt(w) pb.writeInt(h) - pb.writeChar(c) + pb.writeMedium(c) } def appendTextBufferPaletteChange(pb: PacketBuilder, index: Int, color: Int) { @@ -692,7 +692,7 @@ object PacketSender { pb.writeInt(id) } - def appendTextBufferRawSetText(pb: PacketBuilder, col: Int, row: Int, text: Array[Array[Char]]) { + def appendTextBufferRawSetText(pb: PacketBuilder, col: Int, row: Int, text: Array[Array[Int]]) { pb.writePacketType(PacketType.TextBufferMultiRawSetText) pb.writeInt(col) @@ -702,7 +702,7 @@ object PacketSender { val line = text(y) pb.writeShort(line.length.toShort) for (x <- 0 until line.length.toShort) { - pb.writeChar(line(x)) + pb.writeMedium(line(x)) } } } diff --git a/src/main/scala/li/cil/oc/server/component/GraphicsCard.scala b/src/main/scala/li/cil/oc/server/component/GraphicsCard.scala index ce64575fe..c17b8a34c 100644 --- a/src/main/scala/li/cil/oc/server/component/GraphicsCard.scala +++ b/src/main/scala/li/cil/oc/server/component/GraphicsCard.scala @@ -1,7 +1,6 @@ package li.cil.oc.server.component import java.util - import li.cil.oc.{Constants, Localization, Settings, api} import li.cil.oc.api.Network import li.cil.oc.api.driver.DeviceInfo @@ -10,7 +9,7 @@ import li.cil.oc.api.driver.DeviceInfo.DeviceClass import li.cil.oc.api.machine.{Arguments, Callback, Context, LimitReachedException} import li.cil.oc.api.network._ import li.cil.oc.api.prefab -import li.cil.oc.util.PackedColor +import li.cil.oc.util.{ExtendedUnicodeHelper, PackedColor} import net.minecraft.nbt.{NBTTagCompound, NBTTagList} import li.cil.oc.common.component import li.cil.oc.common.component.GpuTextBuffer @@ -475,7 +474,7 @@ class GraphicsCard(val tier: Int) extends prefab.ManagedEnvironment with DeviceI (bgValue, Unit) } - result(s.get(x, y), fgColor, bgColor, fgIndex, bgIndex) + result(new java.lang.StringBuilder().appendCodePoint(s.getCodePoint(x, y)).toString, fgColor, bgColor, fgIndex, bgIndex) }) } @@ -487,7 +486,7 @@ class GraphicsCard(val tier: Int) extends prefab.ManagedEnvironment with DeviceI val vertical = args.optBoolean(3, false) screen(s => { - if (resolveInvokeCosts(bufferIndex, context, setCosts(tier), value.length, Settings.get.gpuSetCost)) { + if (resolveInvokeCosts(bufferIndex, context, setCosts(tier), ExtendedUnicodeHelper.length(value), Settings.get.gpuSetCost)) { s.set(x, y, value, vertical) result(true) } else result(Unit, "not enough energy") @@ -518,11 +517,11 @@ class GraphicsCard(val tier: Int) extends prefab.ManagedEnvironment with DeviceI val w = math.max(0, args.checkInteger(2)) val h = math.max(0, args.checkInteger(3)) val value = args.checkString(4) - if (value.length == 1) screen(s => { - val c = value.charAt(0) + if (ExtendedUnicodeHelper.length(value) == 1) screen(s => { + val c = value.codePointAt(0) val cost = if (c == ' ') Settings.get.gpuClearCost else Settings.get.gpuFillCost if (resolveInvokeCosts(bufferIndex, context, fillCosts(tier), w * h, cost)) { - s.fill(x, y, w, h, value.charAt(0)) + s.fill(x, y, w, h, c) result(true) } else { @@ -559,7 +558,7 @@ class GraphicsCard(val tier: Int) extends prefab.ManagedEnvironment with DeviceI case machine: li.cil.oc.server.machine.Machine if machine.lastError != null => if (s.getColorDepth.ordinal > api.internal.TextBuffer.ColorDepth.OneBit.ordinal) s.setBackgroundColor(0x0000FF) else s.setBackgroundColor(0x000000) - s.fill(0, 0, w, h, ' ') + s.fill(0, 0, w, h, 0x20) try { val wrapRegEx = s"(.{1,${math.max(1, w - 2)}})\\s".r val lines = wrapRegEx.replaceAllIn(Localization.localizeImmediately(machine.lastError).replace("\t", " ") + "\n", m => Regex.quoteReplacement(m.group(1) + "\n")).lines.toArray @@ -580,7 +579,7 @@ class GraphicsCard(val tier: Int) extends prefab.ManagedEnvironment with DeviceI } case _ => s.setBackgroundColor(0x000000) - s.fill(0, 0, w, h, ' ') + s.fill(0, 0, w, h, 0x20) } null // For screen() }) diff --git a/src/main/scala/li/cil/oc/server/machine/luac/UnicodeAPI.scala b/src/main/scala/li/cil/oc/server/machine/luac/UnicodeAPI.scala index c327815c4..4f403fd1c 100644 --- a/src/main/scala/li/cil/oc/server/machine/luac/UnicodeAPI.scala +++ b/src/main/scala/li/cil/oc/server/machine/luac/UnicodeAPI.scala @@ -1,7 +1,8 @@ package li.cil.oc.server.machine.luac +import java.util.function.IntUnaryOperator import li.cil.oc.util.ExtendedLuaState.extendLuaState -import li.cil.oc.util.FontUtils +import li.cil.oc.util.{ExtendedUnicodeHelper, FontUtils} class UnicodeAPI(owner: NativeLuaArchitecture) extends NativeLuaAPI(owner) { override def initialize() { @@ -9,13 +10,16 @@ class UnicodeAPI(owner: NativeLuaArchitecture) extends NativeLuaAPI(owner) { lua.newTable() lua.pushScalaFunction(lua => { - lua.pushString(String.valueOf((1 to lua.getTop).map(lua.checkInteger).map(_.toChar).toArray)) + val builder = new java.lang.StringBuilder() + (1 to lua.getTop).map(lua.checkInteger).foreach(builder.appendCodePoint) + lua.pushString(builder.toString) 1 }) lua.setField(-2, "char") lua.pushScalaFunction(lua => { - lua.pushInteger(lua.checkString(1).length) + val s = lua.checkString(1) + lua.pushInteger(ExtendedUnicodeHelper.length(s)) 1 }) lua.setField(-2, "len") @@ -27,22 +31,23 @@ class UnicodeAPI(owner: NativeLuaArchitecture) extends NativeLuaAPI(owner) { lua.setField(-2, "lower") lua.pushScalaFunction(lua => { - lua.pushString(lua.checkString(1).reverse) + lua.pushString(ExtendedUnicodeHelper.reverse(lua.checkString(1))) 1 }) lua.setField(-2, "reverse") lua.pushScalaFunction(lua => { val string = lua.checkString(1) - val start = math.max(0, lua.checkInteger(2) match { - case i if i < 0 => string.length + i - case i => i - 1 - }) + val sLength = ExtendedUnicodeHelper.length(string) + val start = lua.checkInteger(2) match { + case i if i < 0 => string.offsetByCodePoints(string.length, math.max(i, -sLength)) + case i => string.offsetByCodePoints(0, math.min(i - 1, sLength)) + } val end = - if (lua.getTop > 2) math.min(string.length, lua.checkInteger(3) match { - case i if i < 0 => string.length + i + 1 - case i => i - }) + if (lua.getTop > 2) lua.checkInteger(3) match { + case i if i < 0 => string.offsetByCodePoints(string.length, math.max(i + 1, -sLength)) + case i => string.offsetByCodePoints(0, math.min(i, sLength)) + } else string.length if (end <= start) lua.pushString("") else lua.pushString(string.substring(start, end)) @@ -70,7 +75,9 @@ class UnicodeAPI(owner: NativeLuaArchitecture) extends NativeLuaAPI(owner) { lua.pushScalaFunction(lua => { val value = lua.checkString(1) - lua.pushInteger(value.toCharArray.map(ch => math.max(1, FontUtils.wcwidth(ch))).sum) + lua.pushInteger(value.codePoints().map(new IntUnaryOperator { + override def applyAsInt(ch: Int): Int = math.max(1, FontUtils.wcwidth(ch)) + }).sum) 1 }) lua.setField(-2, "wlen") @@ -81,8 +88,8 @@ class UnicodeAPI(owner: NativeLuaArchitecture) extends NativeLuaAPI(owner) { var width = 0 var end = 0 while (width < count) { - width += math.max(1, FontUtils.wcwidth(value(end))) - end += 1 + width += math.max(1, FontUtils.wcwidth(value.codePointAt(end))) + end = value.offsetByCodePoints(end, 1) } if (end > 1) lua.pushString(value.substring(0, end - 1)) else lua.pushString("") diff --git a/src/main/scala/li/cil/oc/server/machine/luaj/UnicodeAPI.scala b/src/main/scala/li/cil/oc/server/machine/luaj/UnicodeAPI.scala index 2f008bddb..d8ff46b4e 100644 --- a/src/main/scala/li/cil/oc/server/machine/luaj/UnicodeAPI.scala +++ b/src/main/scala/li/cil/oc/server/machine/luaj/UnicodeAPI.scala @@ -1,6 +1,7 @@ package li.cil.oc.server.machine.luaj -import li.cil.oc.util.FontUtils +import java.util.function.IntUnaryOperator +import li.cil.oc.util.{ExtendedUnicodeHelper, FontUtils} import li.cil.oc.util.ScalaClosure._ import li.cil.repack.org.luaj.vm2.LuaValue import li.cil.repack.org.luaj.vm2.Varargs @@ -14,23 +15,31 @@ class UnicodeAPI(owner: LuaJLuaArchitecture) extends LuaJAPI(owner) { unicode.set("upper", (args: Varargs) => LuaValue.valueOf(args.checkjstring(1).toUpperCase)) - unicode.set("char", (args: Varargs) => LuaValue.valueOf(String.valueOf((1 to args.narg).map(args.checkint).map(_.toChar).toArray))) + unicode.set("char", (args: Varargs) => { + val builder = new java.lang.StringBuilder() + (1 to args.narg).map(args.checkint).foreach(builder.appendCodePoint) + LuaValue.valueOf(builder.toString) + }) - unicode.set("len", (args: Varargs) => LuaValue.valueOf(args.checkjstring(1).length)) + unicode.set("len", (args: Varargs) => { + val s = args.checkjstring(1) + LuaValue.valueOf(s.codePointCount(0, s.length)) + }) unicode.set("reverse", (args: Varargs) => LuaValue.valueOf(args.checkjstring(1).reverse)) unicode.set("sub", (args: Varargs) => { val string = args.checkjstring(1) - val start = math.max(0, args.checkint(2) match { - case i if i < 0 => string.length + i - case i => i - 1 - }) + val sLength = ExtendedUnicodeHelper.length(string) + val start = args.checkint(2) match { + case i if i < 0 => string.offsetByCodePoints(string.length, math.max(i, -sLength)) + case i => string.offsetByCodePoints(0, math.min(i - 1, sLength)) + } val end = - if (args.narg > 2) math.min(string.length, args.checkint(3) match { - case i if i < 0 => string.length + i + 1 - case i => i - }) + if (args.narg > 2) args.checkint(3) match { + case i if i < 0 => string.offsetByCodePoints(string.length, math.max(i + 1, -sLength)) + case i => string.offsetByCodePoints(0, math.min(i, sLength)) + } else string.length if (end <= start) LuaValue.valueOf("") else LuaValue.valueOf(string.substring(start, end)) @@ -44,7 +53,9 @@ class UnicodeAPI(owner: LuaJLuaArchitecture) extends LuaJAPI(owner) { unicode.set("wlen", (args: Varargs) => { val value = args.checkjstring(1) - LuaValue.valueOf(value.toCharArray.map(ch => math.max(1, FontUtils.wcwidth(ch))).sum) + LuaValue.valueOf(value.codePoints.map(new IntUnaryOperator { + override def applyAsInt(ch: Int): Int = math.max(1, FontUtils.wcwidth(ch)) + }).sum) }) unicode.set("wtrunc", (args: Varargs) => { @@ -53,8 +64,8 @@ class UnicodeAPI(owner: LuaJLuaArchitecture) extends LuaJAPI(owner) { var width = 0 var end = 0 while (width < count) { - width += math.max(1, FontUtils.wcwidth(value(end))) - end += 1 + width += math.max(1, FontUtils.wcwidth(value.codePointAt(end))) + end = value.offsetByCodePoints(end, 1) } if (end > 1) LuaValue.valueOf(value.substring(0, end - 1)) else LuaValue.valueOf("") diff --git a/src/main/scala/li/cil/oc/util/FontUtils.scala b/src/main/scala/li/cil/oc/util/FontUtils.scala index 3c5ae83bc..a2e8796d5 100644 --- a/src/main/scala/li/cil/oc/util/FontUtils.scala +++ b/src/main/scala/li/cil/oc/util/FontUtils.scala @@ -8,14 +8,9 @@ import li.cil.oc.OpenComputers object FontUtils { private val defined_double_wide: BitSet = BitSet() - - // font.hex actually has some codepoints larger than 0x10000 - // but, UnicodeAPI.scala is using java's Integer.ToChar which only supports the utf-16 range - // and thus will truncate any incoming codepoint, forcing it below 0x10000 - // I believe the solution is to use StringBuffer.appendCodePoint - // but that change would deserve a bit of testing first, postponing for a later update - // review http://www.oracle.com/us/technologies/java/supplementary-142654.html - val codepoint_limit: Int = 0x10000 + + // theoretical Unicode maximum + val codepoint_limit: Int = 0x110000 def wcwidth(charCode: Int): Int = if (defined_double_wide(charCode)) 2 else 1 { diff --git a/src/main/scala/li/cil/oc/util/TextBuffer.scala b/src/main/scala/li/cil/oc/util/TextBuffer.scala index ce4393a47..6479f018f 100644 --- a/src/main/scala/li/cil/oc/util/TextBuffer.scala +++ b/src/main/scala/li/cil/oc/util/TextBuffer.scala @@ -5,6 +5,8 @@ import li.cil.oc.api import net.minecraft.nbt._ import net.minecraftforge.common.util.Constants.NBT +import java.lang + /** * This stores chars in a 2D-Array and provides some manipulation functions. * @@ -64,7 +66,7 @@ class TextBuffer(var width: Int, var height: Int, initialFormat: PackedColor.Col var color = Array.fill(height, width)(packed) - var buffer = Array.fill(height, width)(' ') + var buffer = Array.fill(height, width)(0x20) /** The current buffer size in columns by rows. */ def size = (width, height) @@ -80,7 +82,7 @@ class TextBuffer(var width: Int, var height: Int, initialFormat: PackedColor.Col val (iw, ih) = value val (w, h) = (math.max(iw, 1), math.max(ih, 1)) if (width != w || height != h) { - val newBuffer = Array.fill(h, w)(' ') + val newBuffer = Array.fill(h, w)(0x20) val newColor = Array.fill(h, w)(packed) (0 until math.min(h, height)).foreach(y => { Array.copy(buffer(y), 0, newBuffer(y), 0, math.min(w, width)) @@ -103,17 +105,20 @@ class TextBuffer(var width: Int, var height: Int, initialFormat: PackedColor.Col } /** String based fill starting at a specified location. */ - def set(col: Int, row: Int, s: String, vertical: Boolean): Boolean = + def set(col: Int, row: Int, s: String, vertical: Boolean): Boolean = { + val sLength = ExtendedUnicodeHelper.length(s) if (vertical) { if (col < 0 || col >= width) false else { var changed = false - for (y <- row until math.min(row + s.length, height)) if (y >= 0) { + var cx = 0 + for (y <- row until math.min(row + sLength, height)) if (y >= 0) { val line = buffer(y) val lineColor = color(y) - val c = s(y - row) + val c = s.codePointAt(cx) changed = changed || (line(col) != c) || (lineColor(col) != packed) setChar(line, lineColor, col, c) + cx = s.offsetByCodePoints(cx, 1) } changed } @@ -125,18 +130,21 @@ class TextBuffer(var width: Int, var height: Int, initialFormat: PackedColor.Col val line = buffer(row) val lineColor = color(row) var bx = math.max(col, 0) - for (x <- bx until math.min(col + s.length, width) if bx < line.length) { - val c = s(x - col) + var cx = 0 + for (x <- bx until math.min(col + sLength, width) if bx < line.length) { + val c = s.codePointAt(cx) changed = changed || (line(bx) != c) || (lineColor(bx) != packed) setChar(line, lineColor, bx, c) bx += math.max(1, FontUtils.wcwidth(c)) + cx = s.offsetByCodePoints(cx, 1) } changed } } + } /** Fills an area of the buffer with the specified character. */ - def fill(col: Int, row: Int, w: Int, h: Int, c: Char): Boolean = { + def fill(col: Int, row: Int, w: Int, h: Int, c: Int): Boolean = { // Anything to do at all? if (w <= 0 || h <= 0) return false if (col + w < 0 || row + h < 0 || col >= width || row >= height) return false @@ -234,7 +242,7 @@ class TextBuffer(var width: Int, var height: Int, initialFormat: PackedColor.Col changed } - private def setChar(line: Array[Char], lineColor: Array[Short], x: Int, c: Char) { + private def setChar(line: Array[Int], lineColor: Array[Short], x: Int, c: Int) { if (FontUtils.wcwidth(c) > 1 && x >= line.length - 1) { // Don't allow setting wide chars in right-most col. return @@ -260,7 +268,12 @@ class TextBuffer(var width: Int, var height: Int, initialFormat: PackedColor.Col val b = nbt.getTagList("buffer", NBT.TAG_STRING) for (i <- 0 until math.min(h, b.tagCount)) { val value = b.getStringTagAt(i) - System.arraycopy(value.toCharArray, 0, buffer(i), 0, math.min(value.length, buffer(i).length)) + val valueIt = value.codePoints.iterator() + var j = 0 + while (j < buffer(i).length && valueIt.hasNext) { + buffer(i)(j) = valueIt.nextInt() + j += 1 + } } val depth = api.internal.TextBuffer.ColorDepth.values.apply(nbt.getInteger("depth") min (api.internal.TextBuffer.ColorDepth.values.length - 1) max 0) @@ -280,7 +293,7 @@ class TextBuffer(var width: Int, var height: Int, initialFormat: PackedColor.Col val b = new NBTTagList() for (i <- 0 until height) { - b.appendTag(new NBTTagString(String.valueOf(buffer(i)))) + b.appendTag(new NBTTagString(lineToString(i))) } nbt.setTag("buffer", b) @@ -294,14 +307,29 @@ class TextBuffer(var width: Int, var height: Int, initialFormat: PackedColor.Col NbtDataStream.setShortArray(nbt, "colors", color.flatten.map(_.toShort)) } - override def toString: String = { - val b = StringBuilder.newBuilder + def lineToString(y: Int): String = { + val b = new lang.StringBuilder() if (buffer.length > 0) { - b.appendAll(buffer(0)) - for (y <- 1 until height) { - b.append('\n').appendAll(buffer(y)) + for (x <- 0 until width) { + b.appendCodePoint(buffer(y)(x)) } } - b.toString() + b.toString + } + + override def toString: String = { + val b = new lang.StringBuilder() + if (buffer.length > 0) { + for (x <- 0 until width) { + b.appendCodePoint(buffer(0)(x)) + } + for (y <- 1 until height) { + b.append('\n') + for (x <- 0 until width) { + b.appendCodePoint(buffer(y)(x)) + } + } + } + b.toString } } \ No newline at end of file