slightly improved vhdl parser

src/main/java/de/neemann/digital/core/extern/ApplicationVHDLStdIO.java

@@ -12,7 +12,8 @@ import java.io.*;
 import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.NoSuchElementException;
-import java.util.StringTokenizer;
+
+import static de.neemann.digital.core.extern.VHDLTokenizer.Token.*;
 
 /**
  * Base class of applications which are able to interprete VHDL-Code.
@@ -160,90 +161,94 @@ public abstract class ApplicationVHDLStdIO implements Application {
     @Override
     public boolean ensureConsistency(ElementAttributes attributes) {
         String code = attributes.get(Keys.EXTERNAL_CODE);
-        StringTokenizer st = new StringTokenizer(code, "(), :;\t\n\r");
+        VHDLTokenizer st = new VHDLTokenizer(new StringReader(code));
         try {
-            while (st.hasMoreTokens()) {
+            while (!st.value().equalsIgnoreCase("entity"))
-                if (st.nextToken().toLowerCase().equals("entity"))
+                st.next();
+
+            String label = st.consumeIdent();
+
+            st.consumeIdent("is");
+            st.consumeIdent("port");
+            st.consume(OPEN);
+
+            PortDefinition in = new PortDefinition("");
+            PortDefinition out = new PortDefinition("");
+            while (true) {
+                scanPorts(st, in, out);
+                if (st.peek() != SEMICOLON)
                     break;
+                st.consume(SEMICOLON);
             }
+            st.consume(CLOSE);
 
-            String label = st.nextToken();
+            if (in.size() > 0 && out.size() > 0) {
+                attributes.set(Keys.LABEL, label);
+                attributes.set(Keys.EXTERNAL_INPUTS, in.toString());
+                attributes.set(Keys.EXTERNAL_OUTPUTS, out.toString());
+                return true;
+            } else
+                return false;
 
-            while (st.hasMoreTokens()) {
+        } catch (NoSuchElementException | ParseException | VHDLTokenizer.TokenizerException | IOException e) {
-                String tok = st.nextToken().toLowerCase();
-                if (tok.equals("end"))
-                    return false;
-                else if (tok.equals("port")) {
-                    PortDefinition in = new PortDefinition("");
-                    PortDefinition out = new PortDefinition("");
-                    scanPorts(st, in, out);
-                    if (in.size() > 0 && out.size() > 0) {
-                        attributes.set(Keys.LABEL, label);
-                        attributes.set(Keys.EXTERNAL_INPUTS, in.toString());
-                        attributes.set(Keys.EXTERNAL_OUTPUTS, out.toString());
-                        return true;
-                    } else
-                        return false;
-                }
-            }
-            return false;
-        } catch (NoSuchElementException | ParseException e) {
             return false;
         }
     }
 
-    private void scanPorts(StringTokenizer st, PortDefinition in, PortDefinition out) throws ParseException {
+    private void scanPorts(VHDLTokenizer st, PortDefinition in, PortDefinition out) throws ParseException, IOException, VHDLTokenizer.TokenizerException {
         ArrayList<String> vars = new ArrayList<>();
-        while (st.hasMoreTokens()) {
+        vars.add(st.consumeIdent());
-            String tok = st.nextToken();
+        while (true) {
-            switch (tok.toLowerCase()) {
+            switch (st.next()) {
-                case "in":
+                case COLON:
-                    scanPort(st, vars, in);
+                    switch (st.consumeIdent().toLowerCase()) {
-                    vars.clear();
+                        case "in":
-                    break;
+                            scanPort(st, vars, in);
-                case "out":
+                            break;
-                    scanPort(st, vars, out);
+                        case "out":
-                    vars.clear();
+                            scanPort(st, vars, out);
-                    break;
+                            break;
-                case "end":
+                        default:
+                            throw new ParseException("unexpected token " + st);
+                    }
                     return;
+                case COMMA:
+                    vars.add(st.consumeIdent());
+                    break;
                 default:
-                    vars.add(tok);
+                    throw new ParseException("unexpected token " + st);
             }
         }
     }
 
-    private void scanPort(StringTokenizer st, ArrayList<String> vars, PortDefinition port) throws ParseException {
+    private void scanPort(VHDLTokenizer st, ArrayList<String> vars, PortDefinition port) throws ParseException, IOException, VHDLTokenizer.TokenizerException {
-        switch (st.nextToken().toLowerCase()) {
+        switch (st.consumeIdent().toLowerCase()) {
             case "std_logic":
                 for (String var : vars)
                     port.addPort(var, 1);
                 break;
             case "std_logic_vector":
-                int upper = getNumber(st);
+                st.consume(OPEN);
-                if (!st.nextToken().toLowerCase().equals("downto"))
+                int upper = st.consumeNumber();
-                    throw new ParseException();
+                st.consumeIdent("downto");
-                int lower = getNumber(st);
+                int lower = st.consumeNumber();
+                st.consume(CLOSE);
 
                 if (lower != 0)
-                    throw new ParseException();
+                    throw new ParseException("lower is not zero");
 
                 for (String var : vars)
                     port.addPort(var, upper + 1);
                 break;
             default:
-                throw new ParseException();
+                throw new ParseException("unexpected token " + st);
-        }
-    }
-
-    private int getNumber(StringTokenizer st) throws ParseException {
-        try {
-            return Integer.parseInt(st.nextToken());
-        } catch (NumberFormatException e) {
-            throw new ParseException();
         }
     }
 
     private static final class ParseException extends Exception {
+        private ParseException(String message) {
+            super(message);
+        }
     }
+
 }

src/main/java/de/neemann/digital/core/extern/VHDLTokenizer.java

@@ -0,0 +1,249 @@
+/*
+ * Copyright (c) 2016 Helmut Neemann
+ * Use of this source code is governed by the GPL v3 license
+ * that can be found in the LICENSE file.
+ */
+package de.neemann.digital.core.extern;
+
+import de.neemann.digital.core.ExceptionWithOrigin;
+
+import java.io.IOException;
+import java.io.Reader;
+
+import static de.neemann.digital.core.extern.VHDLTokenizer.Token.IDENT;
+import static de.neemann.digital.core.extern.VHDLTokenizer.Token.NUMBER;
+
+/**
+ * Simple tokenizer to tokenize boolean expressions.
+ */
+public class VHDLTokenizer {
+
+    enum Token {UNKNOWN, IDENT, OPEN, CLOSE, NUMBER, COMMA, COLON, SEMICOLON}
+
+    private final Reader in;
+    private Token token;
+    private boolean isToken;
+    private StringBuilder builder;
+    private boolean isUnreadChar = false;
+    private int unreadChar;
+
+    /**
+     * Creates a new instance
+     *
+     * @param in the reader
+     */
+    public VHDLTokenizer(Reader in) {
+        this.in = in;
+        token = Token.UNKNOWN;
+        isToken = false;
+        builder = new StringBuilder();
+    }
+
+    /**
+     * Reads the next token
+     *
+     * @return the token
+     * @throws IOException        IOException
+     * @throws TokenizerException TokenizerException
+     */
+    public Token next() throws IOException, TokenizerException {
+        Token token = peek();
+        consume();
+        return token;
+    }
+
+    /**
+     * Consumes the token after a peek call
+     */
+    public void consume() {
+        isToken = false;
+    }
+
+    /**
+     * Consumes the given token
+     *
+     * @param t the token to consume
+     * @throws IOException        IOException
+     * @throws TokenizerException TokenizerException
+     */
+    public void consume(Token t) throws TokenizerException, IOException {
+        if (next() != t)
+            throw new TokenizerException("ident expected");
+    }
+
+    /**
+     * Peeks the next token.
+     * The token is kept in the stream, so next() or peek() will return this token again!
+     *
+     * @return the token
+     * @throws IOException        IOException
+     * @throws TokenizerException TokenizerException
+     */
+    public Token peek() throws IOException, TokenizerException {
+        if (isToken)
+            return token;
+
+        int c;
+        do {
+            c = readChar();
+            if (c == '-') {
+                int cc = readChar();
+                if (cc == '-') {
+                    do {
+                        c = readChar();
+                    } while (c != '\n');
+                } else
+                    unreadChar(cc);
+            }
+        } while (isWhiteSpace(c));
+
+        switch (c) {
+            case -1:
+                throw new TokenizerException("unexpected EOF");
+            case '(':
+                token = Token.OPEN;
+                break;
+            case ')':
+                token = Token.CLOSE;
+                break;
+            case ';':
+                token = Token.SEMICOLON;
+                break;
+            case ',':
+                token = Token.COMMA;
+                break;
+            case ':':
+                token = Token.COLON;
+                break;
+            default:
+                if (isIdentChar(c)) {
+                    token = IDENT;
+                    builder.setLength(0);
+                    builder.append((char) c);
+                    boolean wasChar = true;
+                    do {
+                        c = readChar();
+                        if (isIdentChar(c) || isNumberChar(c)) {
+                            builder.append((char) c);
+                        } else {
+                            unreadChar(c);
+                            wasChar = false;
+                        }
+                    } while (wasChar);
+                } else if (isNumberChar(c)) {
+                    token = NUMBER;
+                    builder.setLength(0);
+                    builder.append((char) c);
+                    boolean wasChar = true;
+                    do {
+                        c = readChar();
+                        if (isNumberChar(c)) {
+                            builder.append((char) c);
+                        } else {
+                            unreadChar(c);
+                            wasChar = false;
+                        }
+                    } while (wasChar);
+                } else {
+                    token = Token.UNKNOWN;
+                    builder.setLength(0);
+                    builder.append((char) c);
+                }
+        }
+
+        isToken = true;
+        return token;
+    }
+
+    /**
+     * @return the identifier
+     * @throws IOException        IOException
+     * @throws TokenizerException TokenizerException
+     */
+    public String consumeIdent() throws IOException, TokenizerException {
+        if (next() != IDENT)
+            throw new TokenizerException("ident expected");
+        return builder.toString();
+    }
+
+    /**
+     * Consumes an identifier
+     *
+     * @param ident the identifier to consume
+     * @throws IOException        IOException
+     * @throws TokenizerException TokenizerException
+     */
+    public void consumeIdent(String ident) throws IOException, TokenizerException {
+        if (next() != IDENT)
+            throw new TokenizerException("ident expected");
+        if (builder.toString().equalsIgnoreCase(ident))
+            return;
+        throw new TokenizerException("ident " + ident + " expected");
+    }
+
+    /**
+     * @return the identifier
+     * @throws IOException        IOException
+     * @throws TokenizerException TokenizerException
+     */
+    public int consumeNumber() throws TokenizerException, IOException {
+        if (next() != NUMBER)
+            throw new TokenizerException("ident expected");
+        try {
+            return Integer.parseInt(builder.toString());
+        } catch (NumberFormatException e) {
+            throw new TokenizerException("not a number " + builder.toString());
+        }
+    }
+
+    /**
+     * @return the value of the last parsed token
+     */
+    public String value() {
+        return builder.toString();
+    }
+
+    private int readChar() throws IOException {
+        if (isUnreadChar) {
+            isUnreadChar = false;
+            return unreadChar;
+        } else
+            return in.read();
+    }
+
+    private void unreadChar(int c) {
+        unreadChar = c;
+        isUnreadChar = true;
+    }
+
+    private boolean isIdentChar(int c) {
+        return (c >= 'a' && c <= 'z')
+                || (c >= 'A' && c <= 'Z')
+                || (c == '_');
+    }
+
+    private boolean isNumberChar(int c) {
+        return (c >= '0' && c <= '9');
+    }
+
+    private boolean isWhiteSpace(int c) {
+        return c == ' ' || c == '\n' || c == '\r' || c == '\t';
+    }
+
+    @Override
+    public String toString() {
+        if (token == NUMBER || token == IDENT || token == Token.UNKNOWN)
+            return builder.toString();
+        else
+            return token.name();
+    }
+
+    /**
+     * The tokenizer exception
+     */
+    public static final class TokenizerException extends ExceptionWithOrigin {
+        private TokenizerException(String message) {
+            super(message);
+        }
+    }
+}

src/main/java/de/neemann/digital/gui/components/EditorFactory.java

@@ -201,7 +201,8 @@ public final class EditorFactory {
 
         @Override
         public void setValue(String value) {
-            text.setText(value);
+            if (!text.getText().equals(value))
+                text.setText(value);
         }
 
         public JTextComponent getTextComponent() {

src/test/java/de/neemann/digital/core/extern/ApplicationVHDLStdIOTest.java

@@ -93,7 +93,6 @@ public class ApplicationVHDLStdIOTest extends TestCase {
                 "\tsignal ALUOut : unsigned(7 downto 0);  -- internal\n" +
                 "\tsignal ALUIn : unsigned(7 downto 0);  -- internal\n" +
                 "begin\n" +
-                "\tend process;\n" +
                 "end nBitZaehlerRTL;", true);
 
         assertEquals("nBitZaehler", attr.getCleanLabel());
@@ -101,6 +100,28 @@ public class ApplicationVHDLStdIOTest extends TestCase {
         assertEquals("CountOut:8", attr.get(Keys.EXTERNAL_OUTPUTS));
     }
 
+    public void testExtractionComment() {
+        ElementAttributes attr = extractParameters("-- comment at start\n"+
+                "library IEEE;\n" +
+                "use IEEE.std_logic_1164.all;\n" +
+                "use IEEE.numeric_std.all;\n" +
+                "\n" +
+                "\n" +
+                "entity nBitZaehler is -- commnet\n" +
+                "\tport (LoadIn : in std_logic_vector (7 downto 0);--comment \n" +
+                "\tload,reset,clk : in std_logic; CountOut : out std_logic_vector (7 downto 0));--comment\n" +
+                "end nBitZaehler;\n" +
+                "\n" +
+                "architecture nBitZaehlerRTL of nBitZaehler is\n" +
+                "begin\n" +
+                "end nBitZaehlerRTL;", true);
+
+        assertEquals("nBitZaehler", attr.getCleanLabel());
+        assertEquals("LoadIn:8,load,reset,clk", attr.get(Keys.EXTERNAL_INPUTS));
+        assertEquals("CountOut:8", attr.get(Keys.EXTERNAL_OUTPUTS));
+    }
+
+
     public void testExtractionFail() {
         extractParameters("library IEEE;\n" +
                 "use IEEE.std_logic_1164.all;\n" +

src/test/java/de/neemann/digital/core/extern/VHDLTokenizerTest.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2018 Helmut Neemann.
+ * Use of this source code is governed by the GPL v3 license
+ * that can be found in the LICENSE file.
+ */
+package de.neemann.digital.core.extern;
+
+import junit.framework.TestCase;
+
+import java.io.IOException;
+import java.io.StringReader;
+
+public class VHDLTokenizerTest extends TestCase {
+
+    public void testTokenizer() throws IOException, VHDLTokenizer.TokenizerException {
+        VHDLTokenizer tok = new VHDLTokenizer(new StringReader("aa-bb"));
+        assertEquals(VHDLTokenizer.Token.IDENT, tok.next());
+        assertEquals("aa", tok.value());
+        assertEquals(VHDLTokenizer.Token.UNKNOWN, tok.next());
+        assertEquals("-", tok.value());
+        assertEquals(VHDLTokenizer.Token.IDENT, tok.next());
+        assertEquals("bb", tok.value());
+    }
+
+    public void testTokenizerComment() throws IOException, VHDLTokenizer.TokenizerException {
+        VHDLTokenizer tok = new VHDLTokenizer(new StringReader("aa--gfgfg\nbb"));
+        assertEquals(VHDLTokenizer.Token.IDENT, tok.next());
+        assertEquals("aa", tok.value());
+        assertEquals(VHDLTokenizer.Token.IDENT, tok.next());
+        assertEquals("bb", tok.value());
+    }
+
+    public void testTokenizerBracket() throws IOException, VHDLTokenizer.TokenizerException {
+        VHDLTokenizer tok = new VHDLTokenizer(new StringReader("(aa)"));
+        assertEquals(VHDLTokenizer.Token.OPEN, tok.next());
+        assertEquals(VHDLTokenizer.Token.IDENT, tok.next());
+        assertEquals("aa", tok.value());
+        assertEquals(VHDLTokenizer.Token.CLOSE, tok.next());
+    }
+
+}