first working parameterized test cases, see #543

This commit is contained in:
hneemann 2020-11-07 14:41:37 +01:00
parent 72558db65f
commit 2cd5eb71bb
15 changed files with 276 additions and 175 deletions

View File

@ -35,6 +35,10 @@
<visualElement>
<elementName>Testcase</elementName>
<elementAttributes>
<entry>
<string>Label</string>
<string>3 bits</string>
</entry>
<entry>
<string>Testdata</string>
<testData>
@ -49,14 +53,7 @@ C 0
</entry>
<entry>
<string>generic</string>
<string>?&gt;C G
0 0
loop (n,(1&lt;&lt; &lt;?=args.bits?&gt; )-1)
C ((n+1) ^ ((n+1)&gt;&gt;1))
end loop
C 0&lt;?
this.Testdata=output();</string>
<string>bits := 3;</string>
</entry>
</elementAttributes>
<pos x="680" y="40"/>
@ -140,6 +137,10 @@ ist als drei.}}</string>
<string>Bits</string>
<int>3</int>
</entry>
<entry>
<string>generic</string>
<string>this.Bits=int(args.bits);</string>
</entry>
</elementAttributes>
<pos x="680" y="340"/>
</visualElement>
@ -164,7 +165,59 @@ if (args.bits&gt;3) {
<string>bits := 3;</string>
</entry>
</elementAttributes>
<pos x="680" y="100"/>
<pos x="680" y="200"/>
</visualElement>
<visualElement>
<elementName>Testcase</elementName>
<elementAttributes>
<entry>
<string>Label</string>
<string>4 bits</string>
</entry>
<entry>
<string>Testdata</string>
<testData>
<dataString>C G
0 0
loop (n,(1&lt;&lt;4)-1)
C ((n+1) ^ ((n+1)&gt;&gt;1))
end loop
C 0
</dataString>
</testData>
</entry>
<entry>
<string>generic</string>
<string>bits := 4;</string>
</entry>
</elementAttributes>
<pos x="780" y="40"/>
</visualElement>
<visualElement>
<elementName>Testcase</elementName>
<elementAttributes>
<entry>
<string>Label</string>
<string>5 bits</string>
</entry>
<entry>
<string>Testdata</string>
<testData>
<dataString>C G
0 0
loop (n,(1&lt;&lt;5)-1)
C ((n+1) ^ ((n+1)&gt;&gt;1))
end loop
C 0
</dataString>
</testData>
</entry>
<entry>
<string>generic</string>
<string>bits := 5;</string>
</entry>
</elementAttributes>
<pos x="680" y="120"/>
</visualElement>
</visualElements>
<wires>

View File

@ -9,7 +9,6 @@ import de.neemann.digital.cli.cli.Argument;
import de.neemann.digital.cli.cli.BasicCommand;
import de.neemann.digital.cli.cli.CLIException;
import de.neemann.digital.core.ErrorDetector;
import de.neemann.digital.core.Model;
import de.neemann.digital.draw.elements.Circuit;
import de.neemann.digital.lang.Lang;
import de.neemann.digital.testing.TestExecutor;
@ -74,12 +73,11 @@ public class CommandLineTester {
label = "unnamed";
try {
Model model = circuitLoader.createModel();
ErrorDetector errorDetector = new ErrorDetector();
model.addObserver(errorDetector);
TestExecutor te = new TestExecutor(t.getTestCaseDescription())
TestExecutor te = new TestExecutor(t, circuitLoader.getCircuit(), circuitLoader.getLibrary())
.setAllowMissingInputs(allowMissingInputs)
.create(model);
.addObserver(errorDetector)
.create();
if (te.allPassed()) {
out.println(label + ": passed");

View File

@ -351,9 +351,7 @@ public class Circuit implements Copyable<Circuit> {
public List<TestCase> getTestCases() {
ArrayList<TestCase> tc = new ArrayList<>();
for (VisualElement ve : getElements(v -> v.equalsDescription(TestCaseElement.TESTCASEDESCRIPTION) && v.getElementAttributes().get(Keys.ENABLED))) {
tc.add(new TestCase(
ve.getElementAttributes().getLabel(),
new TestCaseDescription(ve.getElementAttributes().get(Keys.TESTDATA))));
tc.add(new TestCase(ve));
}
return tc;
}
@ -361,13 +359,29 @@ public class Circuit implements Copyable<Circuit> {
/**
* A simple java bean to encapsulate a test case description
*/
public static final class TestCase {
public static final class TestCase implements Comparable<TestCase> {
private final String label;
private final TestCaseDescription testCaseDescription;
private final boolean hasGenericCode;
private final VisualElement visualElement;
private TestCase(String label, TestCaseDescription testCaseDescription) {
this.label = label;
this.testCaseDescription = testCaseDescription;
/**
* Used in some test cases.
* Don't use this constructor in production code!
*
* @param testCaseDescription the test case description
*/
public TestCase(TestCaseDescription testCaseDescription) {
this(new VisualElement(TestCaseElement.TESTCASEDESCRIPTION.getName())
.setAttribute(Keys.TESTDATA, testCaseDescription));
}
private TestCase(VisualElement visualElement) {
this.visualElement = visualElement;
ElementAttributes attr = visualElement.getElementAttributes();
this.label = attr.getLabel();
this.testCaseDescription = attr.get(Keys.TESTDATA);
this.hasGenericCode = !attr.get(Keys.GENERIC).isEmpty();
}
/**
@ -383,6 +397,42 @@ public class Circuit implements Copyable<Circuit> {
public TestCaseDescription getTestCaseDescription() {
return testCaseDescription;
}
/**
* @return true if the test case has generic code
*/
public boolean hasGenericCode() {
return hasGenericCode;
}
/**
* @return the visual element which contains the test case, maybe null
*/
public VisualElement getVisualElement() {
return visualElement;
}
@Override
public int compareTo(TestCase o) {
return label.compareTo(o.label);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
TestCase testCase = (TestCase) o;
return label.equals(testCase.label);
}
@Override
public int hashCode() {
return label.hashCode();
}
}
/**

View File

@ -1127,9 +1127,7 @@ public final class Main extends JFrame implements ClosingWindowListener.ConfirmS
*/
public void startTests() {
try {
ArrayList<ValueTableDialog.TestSet> tsl = new ArrayList<>();
for (Circuit.TestCase tc : circuitComponent.getCircuit().getTestCases())
tsl.add(new ValueTableDialog.TestSet(tc.getTestCaseDescription(), tc.getLabel()));
List<Circuit.TestCase> tsl = circuitComponent.getCircuit().getTestCases();
if (tsl.isEmpty())
throw new TestingDataException(Lang.get("err_noTestData"));

View File

@ -6,7 +6,6 @@
package de.neemann.digital.gui.components.testing;
import de.neemann.digital.core.ErrorDetector;
import de.neemann.digital.core.Model;
import de.neemann.digital.core.NodeException;
import de.neemann.digital.data.Value;
import de.neemann.digital.data.ValueTable;
@ -15,11 +14,9 @@ import de.neemann.digital.draw.elements.Circuit;
import de.neemann.digital.draw.elements.PinException;
import de.neemann.digital.draw.library.ElementLibrary;
import de.neemann.digital.draw.library.ElementNotFoundException;
import de.neemann.digital.draw.model.ModelCreator;
import de.neemann.digital.gui.SaveAsHelper;
import de.neemann.digital.gui.components.data.GraphDialog;
import de.neemann.digital.lang.Lang;
import de.neemann.digital.testing.TestCaseDescription;
import de.neemann.digital.testing.TestExecutor;
import de.neemann.digital.testing.TestingDataException;
import de.neemann.gui.IconCreator;
@ -121,24 +118,24 @@ public class ValueTableDialog extends JDialog {
* @throws PinException PinException
* @throws ElementNotFoundException ElementNotFoundException
*/
public ValueTableDialog addTestResult(ArrayList<TestSet> tsl, Circuit circuit, ElementLibrary library) throws TestingDataException, ElementNotFoundException, PinException, NodeException {
public ValueTableDialog addTestResult(java.util.List<Circuit.TestCase> tsl, Circuit circuit, ElementLibrary library) throws TestingDataException, ElementNotFoundException, PinException, NodeException {
Collections.sort(tsl);
int i = 0;
int errorTabIndex = -1;
for (TestSet ts : tsl) {
Model model = new ModelCreator(circuit, library).createModel(false);
for (Circuit.TestCase ts : tsl) {
ErrorDetector errorDetector = new ErrorDetector();
model.addObserver(errorDetector);
try {
TestExecutor testExecutor = new TestExecutor(ts.data).create(model);
TestExecutor testExecutor = new TestExecutor(ts, circuit, library)
.addObserver(errorDetector)
.create();
String tabName;
Icon tabIcon;
if (testExecutor.allPassed()) {
tabName = Lang.get("msg_test_N_Passed", ts.name);
tabName = Lang.get("msg_test_N_Passed", ts.getLabel());
tabIcon = ICON_PASSED;
} else {
tabName = Lang.get("msg_test_N_Failed", ts.name);
tabName = Lang.get("msg_test_N_Failed", ts.getLabel());
tabIcon = ICON_FAILED;
errorTabIndex = i;
}
@ -152,9 +149,7 @@ public class ValueTableDialog extends JDialog {
i++;
errorDetector.check();
} catch (Exception e) {
throw new TestingDataException(Lang.get("err_whileExecutingTests_N0", ts.name), e);
} finally {
model.close();
throw new TestingDataException(Lang.get("err_whileExecutingTests_N0", ts.getLabel()), e);
}
}
if (errorTabIndex >= 0)
@ -200,48 +195,6 @@ public class ValueTableDialog extends JDialog {
return this;
}
/**
* A TestSet contains the {@link TestCaseDescription} and the name of the TestData.
* Is only a value bean
*/
public static class TestSet implements Comparable<TestSet> {
private final TestCaseDescription data;
private final String name;
/**
* Creates a new instance
*
* @param data the TestData
* @param name the name of the data, eg. the used label
*/
public TestSet(TestCaseDescription data, String name) {
this.data = data;
this.name = name;
}
@Override
public int compareTo(TestSet o) {
return name.compareTo(o.name);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
TestSet testSet = (TestSet) o;
return name.equals(testSet.name);
}
@Override
public int hashCode() {
return name.hashCode();
}
}
private static class ValueRenderer extends DefaultTableCellRenderer {
@Override

View File

@ -70,6 +70,9 @@ public class VerilogTestBenchCreator {
filename = filename.substring(0, p);
for (Circuit.TestCase tc : testCases) {
if (tc.hasGenericCode())
throw new HDLException(Lang.get("err_hdlTestCaseHasGenericCode"));
String testName = tc.getLabel();
if (testName.length() > 0)
testName = filename + "_" + testName + "_tb";

View File

@ -64,6 +64,9 @@ public class VHDLTestBenchCreator {
VHDLRenaming renaming = new VHDLRenaming();
for (Circuit.TestCase tc : testCases) {
if (tc.hasGenericCode())
throw new HDLException(Lang.get("err_hdlTestCaseHasGenericCode"));
String testName = tc.getLabel();
if (testName.length() > 0) {
testName = filename + "_" + renaming.checkName(testName) + "_tb";

View File

@ -5,7 +5,6 @@
*/
package de.neemann.digital.testing;
import de.neemann.digital.core.Model;
import de.neemann.digital.core.NodeException;
import de.neemann.digital.draw.elements.Circuit;
import de.neemann.digital.draw.elements.PinException;
@ -199,9 +198,8 @@ public class FolderTestRunner {
StringBuilder sb = new StringBuilder();
int rowCount = 0;
for (Circuit.TestCase tc : testCases) {
Model model = new ModelCreator(circuit, library).createModel(false);
try {
TestExecutor te = new TestExecutor(tc.getTestCaseDescription()).create(model);
TestExecutor te = new TestExecutor(tc, circuit, library).create();
if (te.allPassed()) {
rowCount += te.getResult().getRows();
} else {
@ -222,7 +220,7 @@ public class FolderTestRunner {
setMessage(f, i, sb.toString(), FileToTest.Status.failed);
}
} catch (IOException | NodeException | ElementNotFoundException | PinException | ParserException | RuntimeException e) {
} catch (IOException | ElementNotFoundException | PinException | ParserException | RuntimeException e) {
setMessage(f, i, e.getMessage(), FileToTest.Status.error);
}
}

View File

@ -63,7 +63,7 @@ public class TestCaseDescription {
}
private void check() throws TestingDataException {
if (lines == null) {
if (lines == null || names == null) {
try {
Parser tdp = new Parser(dataString).parse();
lines = tdp.getLines();

View File

@ -6,9 +6,16 @@
package de.neemann.digital.testing;
import de.neemann.digital.core.*;
import de.neemann.digital.core.element.Keys;
import de.neemann.digital.core.wiring.Clock;
import de.neemann.digital.data.Value;
import de.neemann.digital.data.ValueTable;
import de.neemann.digital.draw.elements.Circuit;
import de.neemann.digital.draw.elements.PinException;
import de.neemann.digital.draw.library.ElementLibrary;
import de.neemann.digital.draw.library.ElementNotFoundException;
import de.neemann.digital.draw.library.ResolveGenerics;
import de.neemann.digital.draw.model.ModelCreator;
import de.neemann.digital.lang.Lang;
import de.neemann.digital.testing.parser.Context;
import de.neemann.digital.testing.parser.LineEmitter;
@ -26,6 +33,7 @@ public class TestExecutor {
private static final int ERR_RESULTS = MAX_RESULTS * 2;
private final ArrayList<String> names;
private final Model model;
private final LineEmitter lines;
private final ValueTable results;
private boolean errorOccurred;
@ -38,30 +46,57 @@ public class TestExecutor {
private boolean allowMissingInputs;
/**
* Creates a new testing result
* Creates a new testing result.
*
* @param testCaseDescription the testing data
* @param testCase the testing data
* @param circuit the circuit
* @param library the library
* @throws TestingDataException DataException
* @throws ElementNotFoundException ElementNotFoundException
* @throws PinException PinException
* @throws NodeException NodeException
*/
public TestExecutor(TestCaseDescription testCaseDescription) throws TestingDataException {
names = testCaseDescription.getNames();
public TestExecutor(Circuit.TestCase testCase, Circuit circuit, ElementLibrary library) throws TestingDataException, NodeException, ElementNotFoundException, PinException {
this(testCase, createModel(testCase, circuit, library));
}
static private Model createModel(Circuit.TestCase testCase, Circuit circuit, ElementLibrary library) throws NodeException, ElementNotFoundException, PinException {
final Model model;
if (circuit != null && circuit.getAttributes().get(Keys.IS_GENERIC) && testCase.hasGenericCode()) {
Circuit c = new ResolveGenerics().resolveCircuit(testCase.getVisualElement(), circuit, library).getCircuit();
model = new ModelCreator(c, library, false).createModel(false);
} else
model = new ModelCreator(circuit, library).createModel(false);
return model;
}
/**
* Use for tests only! Do'nt use this constructor with a model you have created from a circuit.
* If a circuit is available use the other constructor.
*
* @param testCase the test case
* @param model the model
* @throws TestingDataException TestingDataException
*/
public TestExecutor(Circuit.TestCase testCase, Model model) throws TestingDataException {
names = testCase.getTestCaseDescription().getNames();
this.model = model;
results = new ValueTable(names);
visibleRows = 0;
lines = testCaseDescription.getLines();
lines = testCase.getTestCaseDescription().getLines();
}
/**
* Creates the result by comparing the testing vector with the given model-
*
* @param model the model to check
* @return this for chained calls
* @throws TestingDataException DataException
* @throws NodeException NodeException
* @throws ParserException ParserException
*/
public TestExecutor create(Model model) throws TestingDataException, NodeException, ParserException {
public TestExecutor create() throws TestingDataException, NodeException, ParserException {
try {
HashSet<String> usedSignals = new HashSet<>();
inputs = new ArrayList<>();
outputs = new ArrayList<>();
for (Signal s : model.getInputs()) {
@ -119,6 +154,9 @@ public class TestExecutor {
lines.emitLines(new LineListenerResolveDontCare(values -> checkRow(model, values), inputs), new Context().setModel(model));
return this;
} finally {
model.close();
}
}
private void addTo(HashSet<String> signals, String name) throws TestingDataException {
@ -271,6 +309,17 @@ public class TestExecutor {
return this;
}
/**
* Adds a observer to the model of this test executor
*
* @param observer the observer to add
* @return this for chained calls
*/
public TestExecutor addObserver(ModelStateObserverTyped observer) {
model.addObserver(observer);
return this;
}
/**
* A test signal
*/

View File

@ -1047,6 +1047,9 @@ Sind evtl. die Namen der Variablen nicht eindeutig?</string>
<string name="err_vhdlANameIsMissing">Es fehlt ein Name. Sind z.B. alle Pins benannt?</string>
<string name="err_hdlMultipleOutputsConnectedToNet_N_N_N">Es sind mehrere Ausgänge miteinander verbunden.
Diese Art der Verschaltung wird beim HDL-Export nicht unterstützt ({0}, {1}, {2}).</string>
<string name="err_hdlTestCaseHasGenericCode">Testfälle mit generischer Parametrisierung werden im HDL-Export nicht
unterstützt!
</string>
<string name="err_unnamedNet">unbenanntes Netz</string>
<string name="err_toManyVars">Zu viele Variablen!</string>
<string name="err_invalidExpression">Ungültiger Ausdruck!</string>

View File

@ -1057,6 +1057,9 @@
<string name="err_vhdlANameIsMissing">A name is missing. Have e.g. all pins a label set?</string>
<string name="err_hdlMultipleOutputsConnectedToNet_N_N_N">Several outputs are connected to each other.
This type of interconnection is not supported for HDL export. ({0}, {1}, {2}).</string>
<string name="err_hdlTestCaseHasGenericCode">Test cases with generic parameterization are not supported in HDL
export!
</string>
<string name="err_unnamedNet">unnamed net</string>
<string name="err_toManyVars">Too many variables!</string>
<string name="err_invalidExpression">Invalid expression!</string>

View File

@ -5,13 +5,9 @@
*/
package de.neemann.digital.draw.library;
import de.neemann.digital.core.element.Keys;
import de.neemann.digital.draw.elements.Circuit;
import de.neemann.digital.draw.elements.VisualElement;
import de.neemann.digital.integration.Resources;
import de.neemann.digital.integration.ToBreakRunner;
import de.neemann.digital.testing.TestCaseDescription;
import de.neemann.digital.testing.TestCaseElement;
import de.neemann.digital.testing.TestExecutor;
import junit.framework.TestCase;
@ -38,6 +34,6 @@ public class JarComponentManagerTest extends TestCase {
};
for (Circuit.TestCase tc : br.getCircuit().getTestCases())
assertTrue(new TestExecutor(tc.getTestCaseDescription()).create(br.getModel()).allPassed());
assertTrue(new TestExecutor(tc, br.getCircuit(), br.getLibrary()).create().allPassed());
}
}

View File

@ -17,7 +17,6 @@ import de.neemann.digital.draw.library.ElementNotFoundException;
import de.neemann.digital.draw.library.GenericInitCode;
import de.neemann.digital.draw.library.ResolveGenerics;
import de.neemann.digital.draw.model.ModelCreator;
import de.neemann.digital.testing.TestCaseDescription;
import de.neemann.digital.testing.TestExecutor;
import junit.framework.TestCase;
@ -41,7 +40,7 @@ public class TestExamples extends TestCase {
public void testDistExamples() throws Exception {
File examples = new File(Resources.getRoot().getParentFile().getParentFile(), "/main/dig");
assertEquals(302, new FileScanner(this::check).scan(examples));
assertEquals(198, testCasesInFiles);
assertEquals(200, testCasesInFiles);
}
/**
@ -82,22 +81,17 @@ public class TestExamples extends TestCase {
testCasesInFiles++;
String label = tc.getLabel();
TestCaseDescription td = tc.getTestCaseDescription();
Model model = new ModelCreator(br.getCircuit(), br.getLibrary()).createModel(false);
ErrorDetector ed = new ErrorDetector();
model.addObserver(ed);
try {
TestExecutor tr = new TestExecutor(td).create(model);
TestExecutor tr = new TestExecutor(tc, br.getCircuit(), br.getLibrary())
.addObserver(ed)
.create();
if (label.contains("Failing"))
assertFalse(dig.getName() + ":" + label, tr.allPassed());
else
assertTrue(dig.getName() + ":" + label, tr.allPassed());
} finally {
model.close();
}
ed.check();
}
} catch (Exception e) {

View File

@ -47,8 +47,8 @@ public class TestResultTest extends TestCase {
+ "0 1 1\n"
+ "1 0 1\n"
+ "1 1 0\n");
TestExecutor tr = new TestExecutor(data).create(model);
assertEquals(4,tr.getResult().getRows());
TestExecutor tr = new TestExecutor(new Circuit.TestCase(data), model).create();
assertEquals(4, tr.getResult().getRows());
assertTrue(tr.allPassed());
}
@ -60,9 +60,9 @@ public class TestResultTest extends TestCase {
+ "0 1 1\n"
+ "1 0 1\n"
+ "1 1 0\n");
TestExecutor te = new TestExecutor(data).create(model);
TestExecutor te = new TestExecutor(new Circuit.TestCase(data), model).create();
ValueTable tr = te.getResult();
assertEquals(4,tr.getRows());
assertEquals(4, tr.getRows());
assertFalse(te.allPassed());
assertEquals(true, ((MatchedValue) tr.getValue(0, 2)).isPassed());
assertEquals(true, ((MatchedValue) tr.getValue(1, 2)).isPassed());
@ -78,9 +78,9 @@ public class TestResultTest extends TestCase {
+ "0 1 1\n"
+ "1 0 1\n"
+ "1 1 x\n");
TestExecutor te = new TestExecutor(data).create(model);
TestExecutor te = new TestExecutor(new Circuit.TestCase(data), model).create();
ValueTable tr = te.getResult();
assertEquals(4,tr.getRows());
assertEquals(4, tr.getRows());
assertTrue(te.allPassed());
}
@ -92,9 +92,9 @@ public class TestResultTest extends TestCase {
+ "0 1 1\n"
+ "1 0 1\n"
+ "1 1 1\n");
TestExecutor te = new TestExecutor(data).create(model);
TestExecutor te = new TestExecutor(new Circuit.TestCase(data), model).create();
ValueTable tr = te.getResult();
assertEquals(4,tr.getRows());
assertEquals(4, tr.getRows());
assertTrue(te.allPassed());
}
@ -104,9 +104,9 @@ public class TestResultTest extends TestCase {
"A B Y\n"
+ "x 0 0\n"
+ "x 1 1\n");
TestExecutor te = new TestExecutor(data).create(model);
TestExecutor te = new TestExecutor(new Circuit.TestCase(data), model).create();
ValueTable tr = te.getResult();
assertEquals(4,tr.getRows());
assertEquals(4, tr.getRows());
assertTrue(te.allPassed());
}
@ -116,9 +116,9 @@ public class TestResultTest extends TestCase {
"A B C Y\n"
+ "x x 0 0\n"
+ "x x 1 1\n");
TestExecutor te = new TestExecutor(data).create(model);
TestExecutor te = new TestExecutor(new Circuit.TestCase(data), model).create();
ValueTable tr = te.getResult();
assertEquals(8,tr.getRows());
assertEquals(8, tr.getRows());
assertTrue(te.allPassed());
}