Merge pull request #9204 from gilles-peskine-arm/ssl-test-no-skip-all-2.28

Backport 2.28: ssl-opt.sh, compat.sh: Error out if not executing any tests

tests/compat.sh

@@ -98,6 +98,7 @@ FILTER=""
 EXCLUDE='NULL\|DES\|RC4\|ARCFOUR\|ARIA\|CHACHA20-POLY1305'
 VERBOSE=""
 MEMCHECK=0
+MIN_TESTS=1
 PRESERVE_LOGS=0
 PEERS="OpenSSL$PEER_GNUTLS mbedTLS"
 
@@ -118,6 +119,7 @@ print_usage() {
     printf "  -M|--memcheck\tCheck memory leaks and errors.\n"
     printf "  -v|--verbose\tSet verbose output.\n"
     printf "     --list-test-cases\tList all potential test cases (No Execution)\n"
+    printf "     --min      \tMinimum number of non-skipped tests (default 1)\n"
     printf "     --outcome-file\tFile where test outcomes are written\n"
     printf "                   \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n"
     printf "     --preserve-logs\tPreserve logs of successful tests as well\n"
@@ -213,6 +215,9 @@ get_options() {
                 list_test_cases
                 exit $?
                 ;;
+            --min)
+                shift; MIN_TESTS=$1
+                ;;
             --outcome-file)
                 shift; MBEDTLS_TEST_OUTCOME_FILE=$1
                 ;;
@@ -1018,7 +1023,7 @@ setup_arguments()
     M_SERVER_ARGS="server_port=$PORT server_addr=0.0.0.0 force_version=$MODE arc4=1"
     O_SERVER_ARGS="-accept $PORT -cipher ALL,COMPLEMENTOFALL -$O_MODE"
     G_SERVER_ARGS="-p $PORT --http $G_MODE"
-    G_SERVER_PRIO="NORMAL:${G_PRIO_CCM}+ARCFOUR-128:+NULL:+MD5:+PSK:+DHE-PSK:+ECDHE-PSK:+SHA256:+SHA384:+RSA-PSK:-VERS-TLS-ALL:$G_PRIO_MODE"
+    G_SERVER_PRIO="NORMAL:${G_PRIO_CCM}+ARCFOUR-128:+3DES-CBC:+NULL:+MD5:+PSK:+DHE-PSK:+ECDHE-PSK:+SHA256:+SHA384:+RSA-PSK:-VERS-TLS-ALL:$G_PRIO_MODE"
 
     # The default prime for `openssl s_server` depends on the version:
     # * OpenSSL <= 1.0.2a: 512-bit
@@ -1627,6 +1632,16 @@ fi
 PASSED=$(( $TESTS - $FAILED ))
 echo " ($PASSED / $TESTS tests ($SKIPPED skipped$MEMREPORT))"
 
+if [ $((TESTS - SKIPPED)) -lt $MIN_TESTS ]; then
+    cat <<EOF
+Error: Expected to run at least $MIN_TESTS, but only ran $((TESTS - SKIPPED)).
+Maybe a bad filter ('$FILTER' excluding '$EXCLUDE') or a bad configuration?
+EOF
+    if [ $FAILED -eq 0 ]; then
+        FAILED=1
+    fi
+fi
+
 FAILED=$(( $FAILED + $SRVMEM ))
 if [ $FAILED -gt 255 ]; then
     # Clamp at 255 as caller gets exit code & 0xFF

tests/scripts/test-ref-configs.pl

@@ -27,7 +27,8 @@ my %configs = (
         'test_again_with_use_psa' => 1
     },
     'config-mini-tls1_1.h' => {
-        'compat' => '-m tls1_1 -f \'^DES-CBC3-SHA$\|^TLS-RSA-WITH-3DES-EDE-CBC-SHA$\'', #',
+         # Include DES; exclude (EC)DH; only pure-RSA key exchanges
+        'compat' => '-m tls1_1 -e \'NULL\|RC4\|ARCFOUR\|ARIA\|CAMELLIA\|DH\|PSK\' -t RSA',
         ## Skip ssl-opt testing for now because ssl-opt.sh is missing a lot
         ## of requires_xxx so it would try to run tests that don't apply.
         # 'opt' => ' ',

tests/ssl-opt.sh

@@ -129,6 +129,7 @@ SHOW_TEST_NUMBER=0
 LIST_TESTS=0
 RUN_TEST_NUMBER=''
 
+MIN_TESTS=1
 PRESERVE_LOGS=0
 
 # Pick a "unique" server port in the range 10000-19999, and a proxy
@@ -147,6 +148,7 @@ print_usage() {
     printf "  -s|--show-numbers\tShow test numbers in front of test names\n"
     printf "  -p|--preserve-logs\tPreserve logs of successful tests as well\n"
     printf "     --list-test-cases\tList all potential test cases (No Execution)\n"
+    printf "     --min      \tMinimum number of non-skipped tests (default 1)\n"
     printf "     --outcome-file\tFile where test outcomes are written\n"
     printf "                \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n"
     printf "     --port     \tTCP/UDP port (default: randomish 1xxxx)\n"
@@ -178,6 +180,9 @@ get_options() {
             -p|--preserve-logs)
                 PRESERVE_LOGS=1
                 ;;
+            --min)
+                shift; MIN_TESTS=$1
+                ;;
             --outcome-file)
                 shift; MBEDTLS_TEST_OUTCOME_FILE=$1
                 ;;
@@ -10617,6 +10622,16 @@ if [ "$LIST_TESTS" -eq 0 ]; then
     fi
     PASSES=$(( $TESTS - $FAILS ))
     echo " ($PASSES / $TESTS tests ($SKIPS skipped))"
+
+    if [ $((TESTS - SKIPS)) -lt $MIN_TESTS ]; then
+        cat <<EOF
+Error: Expected to run at least $MIN_TESTS, but only ran $((TESTS - SKIPS)).
+Maybe a bad filter ('$FILTER') or a bad configuration?
+EOF
+        if [ $FAILS -eq 0 ]; then
+            FAILS=1
+        fi
+    fi
 fi
 
 if [ $FAILS -gt 255 ]; then