cleanup of test case parser, see #708

src/main/java/de/neemann/digital/testing/parser/OperatorPrecedence.java

@@ -0,0 +1,16 @@
+/*
+ * Copyright (c) 2021 Helmut Neemann.
+ * Use of this source code is governed by the GPL v3 license
+ * that can be found in the LICENSE file.
+ */
+package de.neemann.digital.testing.parser;
+
+/**
+ * Defines the operator precedences
+ */
+public enum OperatorPrecedence {
+    /**
+     * the operator precedences
+     */
+    MUL, ADD, SHIFT, COMPARE, EQUAL, AND, XOR, OR
+}

src/main/java/de/neemann/digital/testing/parser/Parser.java

@@ -318,14 +318,6 @@ public class Parser {
         return emitter;
     }
 
-    private boolean isToken(Tokenizer.Token t) throws IOException {
-        if (tok.peek() == t) {
-            tok.next();
-            return true;
-        }
-        return false;
-    }
-
     /**
      * Returns the value of the expression
      *
@@ -351,172 +343,47 @@ public class Parser {
         return value;
     }
 
-    /**
+    private Expression parseExpression(OperatorPrecedence oppr, Next next) throws IOException, ParserException {
-     * Parses a string to a simple expression
+        Expression ac = next.next();
-     *
+        while (tok.peek().getPrecedence() == oppr) {
-     * @return the expression
+            Tokenizer.Binary op = tok.next().getFunction();
-     * @throws IOException     IOException
+            Expression a = ac;
-     * @throws ParserException IOException
+            Expression b = next.next();
-     */
+            ac = (c) -> op.op(a.value(c), b.value(c));
+        }
+        return ac;
+    }
+
     private Expression parseExpression() throws IOException, ParserException {
-        Expression ac = parseSmalerEqual();
+        return parseExpression(OperatorPrecedence.OR, this::parseXOR);
-        while (isToken(Tokenizer.Token.SMALER)) {
-            Expression a = ac;
-            Expression b = parseGreater();
-            ac = (c) -> a.value(c) < b.value(c) ? 1 : 0;
-        }
-        return ac;
-    }
-
-    private Expression parseSmalerEqual() throws IOException, ParserException {
-        Expression ac = parseGreater();
-        while (isToken(Tokenizer.Token.SMALEREQUAL)) {
-            Expression a = ac;
-            Expression b = parseGreater();
-            ac = (c) -> a.value(c) <= b.value(c) ? 1 : 0;
-        }
-        return ac;
-    }
-
-    private Expression parseGreater() throws IOException, ParserException {
-        Expression ac = parseGreaterEqual();
-        while (isToken(Tokenizer.Token.GREATER)) {
-            Expression a = ac;
-            Expression b = parseEquals();
-            ac = (c) -> a.value(c) > b.value(c) ? 1 : 0;
-        }
-        return ac;
-    }
-
-    private Expression parseGreaterEqual() throws IOException, ParserException {
-        Expression ac = parseEquals();
-        while (isToken(Tokenizer.Token.GREATEREQUAL)) {
-            Expression a = ac;
-            Expression b = parseEquals();
-            ac = (c) -> a.value(c) >= b.value(c) ? 1 : 0;
-        }
-        return ac;
-    }
-
-    private Expression parseEquals() throws IOException, ParserException {
-        Expression ac = parseNotEquals();
-        while (isToken(Tokenizer.Token.EQUAL)) {
-            Expression a = ac;
-            Expression b = parseNotEquals();
-            ac = (c) -> a.value(c) == b.value(c) ? 1 : 0;
-        }
-        return ac;
-    }
-
-    private Expression parseNotEquals() throws IOException, ParserException {
-        Expression ac = parseOR();
-        while (isToken(Tokenizer.Token.LOG_NOT)) {
-            expect(Tokenizer.Token.EQUAL);
-            Expression a = ac;
-            Expression b = parseOR();
-            ac = (c) -> a.value(c) == b.value(c) ? 0 : 1;
-        }
-        return ac;
-    }
-
-    private Expression parseOR() throws IOException, ParserException {
-        Expression ac = parseXOR();
-        while (isToken(Tokenizer.Token.OR)) {
-            Expression a = ac;
-            Expression b = parseXOR();
-            ac = (c) -> a.value(c) | b.value(c);
-        }
-        return ac;
     }
 
     private Expression parseXOR() throws IOException, ParserException {
-        Expression ac = parseAND();
+        return parseExpression(OperatorPrecedence.XOR, this::parseAND);
-        while (isToken(Tokenizer.Token.XOR)) {
-            Expression a = ac;
-            Expression b = parseAND();
-            ac = (c) -> a.value(c) ^ b.value(c);
-        }
-        return ac;
     }
 
     private Expression parseAND() throws IOException, ParserException {
-        Expression ac = parseShiftRight();
+        return parseExpression(OperatorPrecedence.AND, this::parseEquals);
-        while (isToken(Tokenizer.Token.AND)) {
-            Expression a = ac;
-            Expression b = parseShiftRight();
-            ac = (c) -> a.value(c) & b.value(c);
-        }
-        return ac;
     }
 
-    private Expression parseShiftRight() throws IOException, ParserException {
+    private Expression parseEquals() throws IOException, ParserException {
-        Expression ac = parseShiftLeft();
+        return parseExpression(OperatorPrecedence.EQUAL, this::parseCompare);
-        while (isToken(Tokenizer.Token.SHIFTRIGHT)) {
-            Expression a = ac;
-            Expression b = parseShiftLeft();
-            ac = (c) -> a.value(c) >>> b.value(c);
-        }
-        return ac;
     }
 
-    private Expression parseShiftLeft() throws IOException, ParserException {
+    private Expression parseCompare() throws IOException, ParserException {
-        Expression ac = parseAdd();
+        return parseExpression(OperatorPrecedence.COMPARE, this::parseShift);
-        while (isToken(Tokenizer.Token.SHIFTLEFT)) {
+    }
-            Expression a = ac;
+
-            Expression b = parseAdd();
+    private Expression parseShift() throws IOException, ParserException {
-            ac = (c) -> a.value(c) << b.value(c);
+        return parseExpression(OperatorPrecedence.SHIFT, this::parseAdd);
-        }
-        return ac;
     }
 
     private Expression parseAdd() throws IOException, ParserException {
-        Expression ac = parseSub();
+        return parseExpression(OperatorPrecedence.ADD, this::parseMul);
-        while (isToken(Tokenizer.Token.ADD)) {
-            Expression a = ac;
-            Expression b = parseSub();
-            ac = (c) -> a.value(c) + b.value(c);
-        }
-        return ac;
-    }
-
-    private Expression parseSub() throws IOException, ParserException {
-        Expression ac = parseMul();
-        while (isToken(Tokenizer.Token.SUB)) {
-            Expression a = ac;
-            Expression b = parseMul();
-            ac = (c) -> a.value(c) - b.value(c);
-        }
-        return ac;
     }
 
     private Expression parseMul() throws IOException, ParserException {
-        Expression ac = parseDiv();
+        return parseExpression(OperatorPrecedence.MUL, this::parseIdent);
-        while (isToken(Tokenizer.Token.MUL)) {
-            Expression a = ac;
-            Expression b = parseDiv();
-            ac = (c) -> a.value(c) * b.value(c);
-        }
-        return ac;
-    }
-
-    private Expression parseDiv() throws IOException, ParserException {
-        Expression ac = parseMod();
-        while (isToken(Tokenizer.Token.DIV)) {
-            Expression a = ac;
-            Expression b = parseMod();
-            ac = (c) -> a.value(c) / b.value(c);
-        }
-        return ac;
-    }
-
-    private Expression parseMod() throws IOException, ParserException {
-        Expression ac = parseIdent();
-        while (isToken(Tokenizer.Token.MOD)) {
-            Expression a = ac;
-            Expression b = parseIdent();
-            ac = (c) -> a.value(c) % b.value(c);
-        }
-        return ac;
     }
 
     private Expression parseIdent() throws IOException, ParserException {
@@ -564,4 +431,8 @@ public class Parser {
 
         return (c) -> f.calcValue(c, args);
     }
+
+    private interface Next {
+        Expression next() throws IOException, ParserException;
+    }
 }

src/main/java/de/neemann/digital/testing/parser/Tokenizer.java

@@ -14,10 +14,50 @@ import java.util.HashMap;
  */
 public class Tokenizer {
 
+    interface Binary {
+        long op(long a, long b);
+    }
+
     enum Token {
-        UNKNOWN, IDENT, AND, OR, XOR, BIN_NOT, OPEN, CLOSE, NUMBER, EOL, EOF, SHIFTLEFT, SHIFTRIGHT, COMMA, EQUAL,
+        UNKNOWN, IDENT, BIN_NOT, LOG_NOT, OPEN, CLOSE, NUMBER, EOL, EOF, COMMA,
-        ADD, SUB, MUL, GREATER, GREATEREQUAL, SMALER, SMALEREQUAL, DIV, MOD, END, LOOP, REPEAT, BITS, SEMICOLON,
+        AND(OperatorPrecedence.AND, (a, b) -> a & b),
-        LET, LOG_NOT, DECLARE, PROGRAM, INIT, MEMORY, WHILE
+        OR(OperatorPrecedence.OR, (a, b) -> a | b),
+        XOR(OperatorPrecedence.XOR, (a, b) -> a ^ b),
+        SHIFT_LEFT(OperatorPrecedence.SHIFT, (a, b) -> a << b),
+        SHIFT_RIGHT(OperatorPrecedence.SHIFT, (a, b) -> a >> b),
+        EQUAL(OperatorPrecedence.EQUAL, (a, b) -> (a == b) ? 1 : 0),
+        NOT_EQUAL(OperatorPrecedence.EQUAL, (a, b) -> (a != b) ? 1 : 0),
+        ADD(OperatorPrecedence.ADD, (a, b) -> a + b),
+        SUB(OperatorPrecedence.ADD, (a, b) -> a - b),
+        MUL(OperatorPrecedence.MUL, (a, b) -> a * b),
+        DIV(OperatorPrecedence.MUL, (a, b) -> a / b),
+        MOD(OperatorPrecedence.MUL, (a, b) -> a % b),
+        GREATER(OperatorPrecedence.COMPARE, (a, b) -> (a > b) ? 1 : 0),
+        GREATER_EQUAL(OperatorPrecedence.COMPARE, (a, b) -> (a >= b) ? 1 : 0),
+        SMALLER(OperatorPrecedence.COMPARE, (a, b) -> (a < b) ? 1 : 0),
+        SMALLER_EQUAL(OperatorPrecedence.COMPARE, (a, b) -> (a <= b) ? 1 : 0),
+        END, LOOP, REPEAT, BITS, SEMICOLON,
+        LET, DECLARE, PROGRAM, INIT, MEMORY, WHILE;
+
+        private final OperatorPrecedence precedence;
+        private final Binary function;
+
+        Token() {
+            this(null, null);
+        }
+
+        Token(OperatorPrecedence precedence, Binary function) {
+            this.precedence = precedence;
+            this.function = function;
+        }
+
+        public OperatorPrecedence getPrecedence() {
+            return precedence;
+        }
+
+        public Binary getFunction() {
+            return function;
+        }
     }
 
     private final static HashMap<String, Token> STATEMENT_MAP = new HashMap<>();
@@ -132,18 +172,18 @@ public class Tokenizer {
                 break;
             case '<':
                 if (isNextChar('<')) {
-                    token = Token.SHIFTLEFT;
+                    token = Token.SHIFT_LEFT;
                 } else if (isNextChar('=')) {
-                    token = Token.SMALEREQUAL;
+                    token = Token.SMALLER_EQUAL;
                 } else {
-                    token = Token.SMALER;
+                    token = Token.SMALLER;
                 }
                 break;
             case '>':
                 if (isNextChar('>')) {
-                    token = Token.SHIFTRIGHT;
+                    token = Token.SHIFT_RIGHT;
                 } else if (isNextChar('=')) {
-                    token = Token.GREATEREQUAL;
+                    token = Token.GREATER_EQUAL;
                 } else {
                     token = Token.GREATER;
                 }
@@ -152,7 +192,11 @@ public class Tokenizer {
                 token = Token.BIN_NOT;
                 break;
             case '!':
-                token = Token.LOG_NOT;
+                if (isNextChar('=')) {
+                    token = Token.NOT_EQUAL;
+                } else {
+                    token = Token.LOG_NOT;
+                }
                 break;
             case ',':
                 token = Token.COMMA;