TheCloud/Digital
1
0
Fork 0
mirror of https://github.com/hneemann/Digital.git synced 2025-09-17 00:44:40 -04:00
Code Issues Packages Projects Releases Wiki Activity

slightly improved vhdl parser

Browse Source
This commit is contained in:
hneemann 2018-03-10 13:36:29 +01:00
parent 043a0ef882
commit 9d2f42aa63
5 changed files with 373 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

113
src/main/java/de/neemann/digital/core/extern/ApplicationVHDLStdIO.java vendored
View File

@ -12,7 +12,8 @@ import java.io.*;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.NoSuchElementException;
import java.util.StringTokenizer;
import static de.neemann.digital.core.extern.VHDLTokenizer.Token.*;
/**
* Base class of applications which are able to interprete VHDL-Code.
@ -160,90 +161,94 @@ public abstract class ApplicationVHDLStdIO implements Application {
@Override
public boolean ensureConsistency(ElementAttributes attributes) {
String code = attributes.get(Keys.EXTERNAL_CODE);
StringTokenizer st = new StringTokenizer(code, "(), :;\t\n\r");
VHDLTokenizer st = new VHDLTokenizer(new StringReader(code));
try {
while (st.hasMoreTokens()) {
if (st.nextToken().toLowerCase().equals("entity"))
while (!st.value().equalsIgnoreCase("entity"))
st.next();
String label = st.consumeIdent();
st.consumeIdent("is");
st.consumeIdent("port");
st.consume(OPEN);
PortDefinition in = new PortDefinition("");
PortDefinition out = new PortDefinition("");
while (true) {
scanPorts(st, in, out);
if (st.peek() != SEMICOLON)
break;
st.consume(SEMICOLON);
}
st.consume(CLOSE);
String label = st.nextToken();
if (in.size() > 0 && out.size() > 0) {
attributes.set(Keys.LABEL, label);
attributes.set(Keys.EXTERNAL_INPUTS, in.toString());
attributes.set(Keys.EXTERNAL_OUTPUTS, out.toString());
return true;
} else
return false;
while (st.hasMoreTokens()) {
String tok = st.nextToken().toLowerCase();
if (tok.equals("end"))
return false;
else if (tok.equals("port")) {
PortDefinition in = new PortDefinition("");
PortDefinition out = new PortDefinition("");
scanPorts(st, in, out);
if (in.size() > 0 && out.size() > 0) {
attributes.set(Keys.LABEL, label);
attributes.set(Keys.EXTERNAL_INPUTS, in.toString());
attributes.set(Keys.EXTERNAL_OUTPUTS, out.toString());
return true;
} else
return false;
}
}
return false;
} catch (NoSuchElementException | ParseException e) {
} catch (NoSuchElementException | ParseException | VHDLTokenizer.TokenizerException | IOException e) {
return false;
}
}
private void scanPorts(StringTokenizer st, PortDefinition in, PortDefinition out) throws ParseException {
private void scanPorts(VHDLTokenizer st, PortDefinition in, PortDefinition out) throws ParseException, IOException, VHDLTokenizer.TokenizerException {
ArrayList<String> vars = new ArrayList<>();
while (st.hasMoreTokens()) {
String tok = st.nextToken();
switch (tok.toLowerCase()) {
case "in":
scanPort(st, vars, in);
vars.clear();
break;
case "out":
scanPort(st, vars, out);
vars.clear();
break;
case "end":
vars.add(st.consumeIdent());
while (true) {
switch (st.next()) {
case COLON:
switch (st.consumeIdent().toLowerCase()) {
case "in":
scanPort(st, vars, in);
break;
case "out":
scanPort(st, vars, out);
break;
default:
throw new ParseException("unexpected token " + st);
}
return;
case COMMA:
vars.add(st.consumeIdent());
break;
default:
vars.add(tok);
throw new ParseException("unexpected token " + st);
}
}
}
private void scanPort(StringTokenizer st, ArrayList<String> vars, PortDefinition port) throws ParseException {
switch (st.nextToken().toLowerCase()) {
private void scanPort(VHDLTokenizer st, ArrayList<String> vars, PortDefinition port) throws ParseException, IOException, VHDLTokenizer.TokenizerException {
switch (st.consumeIdent().toLowerCase()) {
case "std_logic":
for (String var : vars)
port.addPort(var, 1);
break;
case "std_logic_vector":
int upper = getNumber(st);
if (!st.nextToken().toLowerCase().equals("downto"))
throw new ParseException();
int lower = getNumber(st);
st.consume(OPEN);
int upper = st.consumeNumber();
st.consumeIdent("downto");
int lower = st.consumeNumber();
st.consume(CLOSE);
if (lower != 0)
throw new ParseException();
throw new ParseException("lower is not zero");
for (String var : vars)
port.addPort(var, upper + 1);
break;
default:
throw new ParseException();
}
}
private int getNumber(StringTokenizer st) throws ParseException {
try {
return Integer.parseInt(st.nextToken());
} catch (NumberFormatException e) {
throw new ParseException();
throw new ParseException("unexpected token " + st);
}
}
private static final class ParseException extends Exception {
private ParseException(String message) {
super(message);
}
}
}

249
src/main/java/de/neemann/digital/core/extern/VHDLTokenizer.java vendored Normal file
View File

@ -0,0 +1,249 @@
/*
* Copyright (c) 2016 Helmut Neemann
* Use of this source code is governed by the GPL v3 license
* that can be found in the LICENSE file.
*/
package de.neemann.digital.core.extern;
import de.neemann.digital.core.ExceptionWithOrigin;
import java.io.IOException;
import java.io.Reader;
import static de.neemann.digital.core.extern.VHDLTokenizer.Token.IDENT;
import static de.neemann.digital.core.extern.VHDLTokenizer.Token.NUMBER;
/**
* Simple tokenizer to tokenize boolean expressions.
*/
public class VHDLTokenizer {
enum Token {UNKNOWN, IDENT, OPEN, CLOSE, NUMBER, COMMA, COLON, SEMICOLON}
private final Reader in;
private Token token;
private boolean isToken;
private StringBuilder builder;
private boolean isUnreadChar = false;
private int unreadChar;
/**
* Creates a new instance
*
* @param in the reader
*/
public VHDLTokenizer(Reader in) {
this.in = in;
token = Token.UNKNOWN;
isToken = false;
builder = new StringBuilder();
}
/**
* Reads the next token
*
* @return the token
* @throws IOException IOException
* @throws TokenizerException TokenizerException
*/
public Token next() throws IOException, TokenizerException {
Token token = peek();
consume();
return token;
}
/**
* Consumes the token after a peek call
*/
public void consume() {
isToken = false;
}
/**
* Consumes the given token
*
* @param t the token to consume
* @throws IOException IOException
* @throws TokenizerException TokenizerException
*/
public void consume(Token t) throws TokenizerException, IOException {
if (next() != t)
throw new TokenizerException("ident expected");
}
/**
* Peeks the next token.
* The token is kept in the stream, so next() or peek() will return this token again!
*
* @return the token
* @throws IOException IOException
* @throws TokenizerException TokenizerException
*/
public Token peek() throws IOException, TokenizerException {
if (isToken)
return token;
int c;
do {
c = readChar();
if (c == '-') {
int cc = readChar();
if (cc == '-') {
do {
c = readChar();
} while (c != '\n');
} else
unreadChar(cc);
}
} while (isWhiteSpace(c));
switch (c) {
case -1:
throw new TokenizerException("unexpected EOF");
case '(':
token = Token.OPEN;
break;
case ')':
token = Token.CLOSE;
break;
case ';':
token = Token.SEMICOLON;
break;
case ',':
token = Token.COMMA;
break;
case ':':
token = Token.COLON;
break;
default:
if (isIdentChar(c)) {
token = IDENT;
builder.setLength(0);
builder.append((char) c);
boolean wasChar = true;
do {
c = readChar();
if (isIdentChar(c) || isNumberChar(c)) {
builder.append((char) c);
} else {
unreadChar(c);
wasChar = false;
}
} while (wasChar);
} else if (isNumberChar(c)) {
token = NUMBER;
builder.setLength(0);
builder.append((char) c);
boolean wasChar = true;
do {
c = readChar();
if (isNumberChar(c)) {
builder.append((char) c);
} else {
unreadChar(c);
wasChar = false;
}
} while (wasChar);
} else {
token = Token.UNKNOWN;
builder.setLength(0);
builder.append((char) c);
}
}
isToken = true;
return token;
}
/**
* @return the identifier
* @throws IOException IOException
* @throws TokenizerException TokenizerException
*/
public String consumeIdent() throws IOException, TokenizerException {
if (next() != IDENT)
throw new TokenizerException("ident expected");
return builder.toString();
}
/**
* Consumes an identifier
*
* @param ident the identifier to consume
* @throws IOException IOException
* @throws TokenizerException TokenizerException
*/
public void consumeIdent(String ident) throws IOException, TokenizerException {
if (next() != IDENT)
throw new TokenizerException("ident expected");
if (builder.toString().equalsIgnoreCase(ident))
return;
throw new TokenizerException("ident " + ident + " expected");
}
/**
* @return the identifier
* @throws IOException IOException
* @throws TokenizerException TokenizerException
*/
public int consumeNumber() throws TokenizerException, IOException {
if (next() != NUMBER)
throw new TokenizerException("ident expected");
try {
return Integer.parseInt(builder.toString());
} catch (NumberFormatException e) {
throw new TokenizerException("not a number " + builder.toString());
}
}
/**
* @return the value of the last parsed token
*/
public String value() {
return builder.toString();
}
private int readChar() throws IOException {
if (isUnreadChar) {
isUnreadChar = false;
return unreadChar;
} else
return in.read();
}
private void unreadChar(int c) {
unreadChar = c;
isUnreadChar = true;
}
private boolean isIdentChar(int c) {
return (c >= 'a' && c <= 'z')
|| (c >= 'A' && c <= 'Z')
|| (c == '_');
}
private boolean isNumberChar(int c) {
return (c >= '0' && c <= '9');
}
private boolean isWhiteSpace(int c) {
return c == ' ' || c == '\n' || c == '\r' || c == '\t';
}
@Override
public String toString() {
if (token == NUMBER || token == IDENT || token == Token.UNKNOWN)
return builder.toString();
else
return token.name();
}
/**
* The tokenizer exception
*/
public static final class TokenizerException extends ExceptionWithOrigin {
private TokenizerException(String message) {
super(message);
}
}
}

3
src/main/java/de/neemann/digital/gui/components/EditorFactory.java
View File

@ -201,7 +201,8 @@ public final class EditorFactory {
@Override
public void setValue(String value) {
text.setText(value);
if (!text.getText().equals(value))
text.setText(value);
}
public JTextComponent getTextComponent() {

23
src/test/java/de/neemann/digital/core/extern/ApplicationVHDLStdIOTest.java vendored
View File

@ -93,7 +93,6 @@ public class ApplicationVHDLStdIOTest extends TestCase {
"\tsignal ALUOut : unsigned(7 downto 0); -- internal\n" +
"\tsignal ALUIn : unsigned(7 downto 0); -- internal\n" +
"begin\n" +
"\tend process;\n" +
"end nBitZaehlerRTL;", true);
assertEquals("nBitZaehler", attr.getCleanLabel());
@ -101,6 +100,28 @@ public class ApplicationVHDLStdIOTest extends TestCase {
assertEquals("CountOut:8", attr.get(Keys.EXTERNAL_OUTPUTS));
}
public void testExtractionComment() {
ElementAttributes attr = extractParameters("-- comment at start\n"+
"library IEEE;\n" +
"use IEEE.std_logic_1164.all;\n" +
"use IEEE.numeric_std.all;\n" +
"\n" +
"\n" +
"entity nBitZaehler is -- commnet\n" +
"\tport (LoadIn : in std_logic_vector (7 downto 0);--comment \n" +
"\tload,reset,clk : in std_logic; CountOut : out std_logic_vector (7 downto 0));--comment\n" +
"end nBitZaehler;\n" +
"\n" +
"architecture nBitZaehlerRTL of nBitZaehler is\n" +
"begin\n" +
"end nBitZaehlerRTL;", true);
assertEquals("nBitZaehler", attr.getCleanLabel());
assertEquals("LoadIn:8,load,reset,clk", attr.get(Keys.EXTERNAL_INPUTS));
assertEquals("CountOut:8", attr.get(Keys.EXTERNAL_OUTPUTS));
}
public void testExtractionFail() {
extractParameters("library IEEE;\n" +
"use IEEE.std_logic_1164.all;\n" +

41
src/test/java/de/neemann/digital/core/extern/VHDLTokenizerTest.java vendored Normal file
View File

@ -0,0 +1,41 @@
/*
* Copyright (c) 2018 Helmut Neemann.
* Use of this source code is governed by the GPL v3 license
* that can be found in the LICENSE file.
*/
package de.neemann.digital.core.extern;
import junit.framework.TestCase;
import java.io.IOException;
import java.io.StringReader;
public class VHDLTokenizerTest extends TestCase {
public void testTokenizer() throws IOException, VHDLTokenizer.TokenizerException {
VHDLTokenizer tok = new VHDLTokenizer(new StringReader("aa-bb"));
assertEquals(VHDLTokenizer.Token.IDENT, tok.next());
assertEquals("aa", tok.value());
assertEquals(VHDLTokenizer.Token.UNKNOWN, tok.next());
assertEquals("-", tok.value());
assertEquals(VHDLTokenizer.Token.IDENT, tok.next());
assertEquals("bb", tok.value());
}
public void testTokenizerComment() throws IOException, VHDLTokenizer.TokenizerException {
VHDLTokenizer tok = new VHDLTokenizer(new StringReader("aa--gfgfg\nbb"));
assertEquals(VHDLTokenizer.Token.IDENT, tok.next());
assertEquals("aa", tok.value());
assertEquals(VHDLTokenizer.Token.IDENT, tok.next());
assertEquals("bb", tok.value());
}
public void testTokenizerBracket() throws IOException, VHDLTokenizer.TokenizerException {
VHDLTokenizer tok = new VHDLTokenizer(new StringReader("(aa)"));
assertEquals(VHDLTokenizer.Token.OPEN, tok.next());
assertEquals(VHDLTokenizer.Token.IDENT, tok.next());
assertEquals("aa", tok.value());
assertEquals(VHDLTokenizer.Token.CLOSE, tok.next());
}
}