From 7db57bec15c58747732e8ef769f9f22147d0d73d Mon Sep 17 00:00:00 2001
From: hneemann <helmut.neemann@arcor.de>
Date: Wed, 5 Aug 2020 14:34:51 +0200
Subject: [PATCH] cli test executor outputs more details, see #501

---
 .../digital/cli/CommandLineTester.java        |  7 +++-
 .../neemann/digital/testing/TestExecutor.java | 38 +++++++++++++++----
 2 files changed, 36 insertions(+), 9 deletions(-)

diff --git a/src/main/java/de/neemann/digital/cli/CommandLineTester.java b/src/main/java/de/neemann/digital/cli/CommandLineTester.java
index c20abe160..9a718358a 100644
--- a/src/main/java/de/neemann/digital/cli/CommandLineTester.java
+++ b/src/main/java/de/neemann/digital/cli/CommandLineTester.java
@@ -97,7 +97,12 @@ public class CommandLineTester {
                         out.println(label + ": passed");
                         testsPassed++;
                     } else {
-                        out.println(label + ": failed");
+                        String message = label + ": failed";
+                        if (te.isErrorOccurred())
+                            message += " due to an error";
+                        else
+                            message += " (" + te.failedPercent() + "%)";
+                        out.println(message);
                         errorCount++;
                     }
                     errorDetector.check();
diff --git a/src/main/java/de/neemann/digital/testing/TestExecutor.java b/src/main/java/de/neemann/digital/testing/TestExecutor.java
index b190c39e5..8b8f4beda 100644
--- a/src/main/java/de/neemann/digital/testing/TestExecutor.java
+++ b/src/main/java/de/neemann/digital/testing/TestExecutor.java
@@ -28,7 +28,9 @@ public class TestExecutor {
     private final ArrayList<String> names;
     private final LineEmitter lines;
     private final ValueTable results;
-    private boolean allPassed;
+    private boolean errorOccurred;
+    private int failedCount;
+    private int passedCount;
     private boolean toManyResults = false;
     private ArrayList<TestSignal> inputs;
     private ArrayList<TestSignal> outputs;
@@ -58,7 +60,6 @@ public class TestExecutor {
      * @throws ParserException      ParserException
      */
     public TestExecutor create(Model model) throws TestingDataException, NodeException, ParserException {
-        allPassed = true;
         HashSet<String> usedSignals = new HashSet<>();
 
         inputs = new ArrayList<>();
@@ -112,7 +113,7 @@ public class TestExecutor {
         model.init();
         model.addObserver(event -> {
             if (event.getType() == ModelEventType.ERROR_OCCURRED)
-                allPassed = false;
+                errorOccurred = true;
         }, ModelEventType.ERROR_OCCURRED);
 
         lines.emitLines(new LineListenerResolveDontCare(values -> checkRow(model, values), inputs), new Context().setModel(model));
@@ -168,7 +169,7 @@ public class TestExecutor {
 
             model.doStep();
         } catch (RuntimeException e) {
-            allPassed = false;
+            errorOccurred = true;
             throw e;
         }
 
@@ -176,12 +177,15 @@ public class TestExecutor {
         for (TestSignal out : outputs) {
             MatchedValue matchedValue = new MatchedValue(values[out.index], out.value);
             res[out.index] = matchedValue;
-            if (!matchedValue.isPassed()) {
-                allPassed = false;
+            if (!matchedValue.isPassed())
                 ok = false;
-            }
         }
 
+        if (ok)
+            passedCount++;
+        else
+            failedCount++;
+
         if (visibleRows < (ok ? MAX_RESULTS : ERR_RESULTS)) {
             visibleRows++;
             results.add(new TestRow(res, testRow.getDescription()));
@@ -205,9 +209,27 @@ public class TestExecutor {
      * @return true if all tests have passed
      */
     public boolean allPassed() {
-        return allPassed;
+        return !errorOccurred && failedCount == 0 && passedCount > 0;
     }
 
+    /**
+     * @return true if the test failed due to an error
+     */
+    public boolean isErrorOccurred() {
+        return errorOccurred;
+    }
+
+    /**
+     * @return the percentage of failed test rows
+     */
+    public int failedPercent() {
+        if (passedCount == 0)
+            return 100;
+        int p = 100 * failedCount / passedCount;
+        if (p == 0 && failedCount > 0)
+            p = 1;
+        return p;
+    }
 
     /**
      * Indicates if there are to many entries in the table to show.