From 0dd6ca4175ca5e8955dc93a223e590870d26cd94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Wed, 10 Apr 2024 12:26:24 +0200 Subject: [PATCH 1/4] compat.sh: properly report skipped tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don't just silently continue. Signed-off-by: Manuel Pégourié-Gonnard --- tests/compat.sh | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/tests/compat.sh b/tests/compat.sh index d7a91b47e..320d0c57c 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -588,7 +588,22 @@ add_mbedtls_ciphersuites() # o_check_ciphersuite STANDARD_CIPHER_SUITE o_check_ciphersuite() { - if [ "${O_SUPPORT_ECDH}" = "NO" ]; then + # skip DTLS when lack of support was declared + if test "$OSSL_NO_DTLS" -gt 0 && is_dtls "$MODE"; then + SKIP_NEXT_="YES" + fi + + # OpenSSL <1.0.2 doesn't support DTLS 1.2. Check if OpenSSL + # supports $O_MODE from the s_server help. (The s_client + # help isn't accurate as of 1.0.2g: it supports DTLS 1.2 + # but doesn't list it. But the s_server help seems to be + # accurate.) + if ! $OPENSSL s_server -help 2>&1 | grep -q "^ *-$O_MODE "; then + SKIP_NEXT_="YES" + fi + + # skip static ECDH when OpenSSL doesn't support it + if [ "${O_SUPPORT_STATIC_ECDH}" = "NO" ]; then case "$1" in *ECDH_*) SKIP_NEXT="YES" esac @@ -665,8 +680,8 @@ setup_arguments() esac case $($OPENSSL ciphers ALL) in - *ECDH-ECDSA*|*ECDH-RSA*) O_SUPPORT_ECDH="YES";; - *) O_SUPPORT_ECDH="NO";; + *ECDH-ECDSA*|*ECDH-RSA*) O_SUPPORT_STATIC_ECDH="YES";; + *) O_SUPPORT_STATIC_ECDH="NO";; esac if [ "X$VERIFY" = "XYES" ]; @@ -1109,19 +1124,6 @@ for MODE in $MODES; do [Oo]pen*) - if test "$OSSL_NO_DTLS" -gt 0 && is_dtls "$MODE"; then - continue; - fi - - # OpenSSL <1.0.2 doesn't support DTLS 1.2. Check if OpenSSL - # supports $O_MODE from the s_server help. (The s_client - # help isn't accurate as of 1.0.2g: it supports DTLS 1.2 - # but doesn't list it. But the s_server help seems to be - # accurate.) - if ! $OPENSSL s_server -help 2>&1 | grep -q "^ *-$O_MODE "; then - continue; - fi - reset_ciphersuites add_common_ciphersuites add_openssl_ciphersuites From cb424097be3b779cb50dfeb741b1ac0c5f2d55dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Wed, 10 Apr 2024 22:11:20 +0200 Subject: [PATCH 2/4] compat.sh: properly skip unsupported DTLS 1.2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Skipping DTLS 1.2 with old versions was already done, but now properly test support only once and use the results. Note that historically, this script's policy was that it's the user's job to find the right value of -e (EXCLUDE) for their version for OpenSSL & config. Now it's a weird mix of that and the script doing some detection and skipping. Signed-off-by: Manuel Pégourié-Gonnard --- tests/compat.sh | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/compat.sh b/tests/compat.sh index 320d0c57c..20f2dbda6 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -593,13 +593,9 @@ o_check_ciphersuite() SKIP_NEXT_="YES" fi - # OpenSSL <1.0.2 doesn't support DTLS 1.2. Check if OpenSSL - # supports $O_MODE from the s_server help. (The s_client - # help isn't accurate as of 1.0.2g: it supports DTLS 1.2 - # but doesn't list it. But the s_server help seems to be - # accurate.) - if ! $OPENSSL s_server -help 2>&1 | grep -q "^ *-$O_MODE "; then - SKIP_NEXT_="YES" + # skip DTLS 1.2 is support was not detected + if [ "$O_SUPPORT_DTLS12" = "NO" -a "$MODE" = "dtls12" ]; then + SKIP_NEXT="YES" fi # skip static ECDH when OpenSSL doesn't support it @@ -684,6 +680,21 @@ setup_arguments() *) O_SUPPORT_STATIC_ECDH="NO";; esac + case $($OPENSSL ciphers ALL) in + *DES-CBC-*) O_SUPPORT_SINGLE_DES="YES";; + *) O_SUPPORT_SINGLE_DES="NO";; + esac + + # OpenSSL <1.0.2 doesn't support DTLS 1.2. Check if OpenSSL + # supports -dtls1_2 from the s_server help. (The s_client + # help isn't accurate as of 1.0.2g: it supports DTLS 1.2 + # but doesn't list it. But the s_server help seems to be + # accurate.) + O_SUPPORT_DTLS12="NO" + if $OPENSSL s_server -help 2>&1 | grep -q "^ *-dtls1_2 "; then + O_SUPPORT_DTLS12="YES" + fi + if [ "X$VERIFY" = "XYES" ]; then M_SERVER_ARGS="$M_SERVER_ARGS ca_file=data_files/test-ca_cat12.crt auth_mode=required" From 62d0bb8f2cb4d78acee2f518610d45665e65fa22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Wed, 17 Apr 2024 12:30:05 +0200 Subject: [PATCH 3/4] Simplify full invocation of compat.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We actually only need two invocations. This also moves all the default tests to OPENSSL_NEXT, which is good because OPENSSL is ancient. I have no idea why NULL doesn't work with OPENSSL_NEXT (1.1.1a) server, because according to the manpage [1], "ALL,COMPLEMENTOFALL" (which is what we are using) should do it, and indeed $OPENSSL_NEXT ciphers "ALL,COMPLEMENTOFALL" | tr ':' '\n' lists NULL ciphersuites, and also they work client-side with OPENSSL_NEXT... [1] https://www.openssl.org/docs/man1.1.1/man1/ciphers.html Also, while at it, remove partial invocation (only non-default) from one component, as we already have a full invocation in the same config (plus ASan) in another component. Signed-off-by: Manuel Pégourié-Gonnard --- tests/scripts/all.sh | 29 +++++++++++++---------------- tests/scripts/basic-build-test.sh | 10 +++------- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 3aabec41d..3f7ad957d 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -1218,8 +1218,11 @@ component_test_full_cmake_gcc_asan () { msg "test: ssl-opt.sh (full config, ASan build)" tests/ssl-opt.sh - msg "test: compat.sh (full config, ASan build)" - tests/compat.sh + msg "test: compat.sh: NULL (full config, ASan build)" + tests/compat.sh -f 'NULL' + + msg "test: compat.sh next: all except NULL (full config, ASan build)" + env OPENSSL="$OPENSSL_NEXT" tests/compat.sh -e 'NULL' msg "test: context-info.sh (full config, ASan build)" # ~ 15 sec tests/context-info.sh @@ -1242,8 +1245,11 @@ component_test_full_cmake_gcc_asan_new_bignum () { msg "test: ssl-opt.sh (full config, ASan build)" tests/ssl-opt.sh - msg "test: compat.sh (full config, ASan build)" - tests/compat.sh + msg "test: compat.sh: NULL (full config, ASan build)" + tests/compat.sh -f 'NULL' + + msg "test: compat.sh next: all except NULL (full config, ASan build)" + env OPENSSL="$OPENSSL_NEXT" tests/compat.sh -e 'NULL' msg "test: context-info.sh (full config, ASan build)" # ~ 15 sec tests/context-info.sh @@ -2161,12 +2167,6 @@ component_test_full_cmake_clang () { msg "test: ssl-opt.sh default, ECJPAKE, SSL async (full config)" # ~ 1s tests/ssl-opt.sh -f 'Default\|ECJPAKE\|SSL async private' - - msg "test: compat.sh NULL (full config)" # ~ 2 min - tests/compat.sh -e '^$' -f 'NULL' - - msg "test: compat.sh ARIA + ChachaPoly" - env OPENSSL="$OPENSSL_NEXT" tests/compat.sh -e '^$' -f 'ARIA\|CHACHA' } skip_suites_without_constant_flow () { @@ -2610,14 +2610,11 @@ component_test_no_psa_crypto_full_cmake_asan() { msg "test: ssl-opt.sh (full minus PSA crypto)" tests/ssl-opt.sh - msg "test: compat.sh default (full minus PSA crypto)" - tests/compat.sh - - msg "test: compat.sh NULL (full minus PSA crypto)" + msg "test: compat.sh: NULL (full minus PSA crypto)" tests/compat.sh -f 'NULL' - msg "test: compat.sh ARIA + ChachaPoly (full minus PSA crypto)" - env OPENSSL="$OPENSSL_NEXT" tests/compat.sh -e '^$' -f 'ARIA\|CHACHA' + msg "test: compat.sh next: all except NULL (full minus PSA crypto)" + env OPENSSL="$OPENSSL_NEXT" tests/compat.sh -e 'NULL' } component_test_psa_crypto_config_accel_ecdsa () { diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index 52617541d..e365eeddc 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -102,16 +102,12 @@ echo # Step 2c - Compatibility tests (keep going even if some tests fail) echo '################ compat.sh ################' { - echo '#### compat.sh: Default versions' - sh compat.sh - echo - - echo '#### compat.sh: null cipher' + echo '#### compat.sh: NULL ciphersuites' sh compat.sh -e '^$' -f 'NULL' echo - echo '#### compat.sh: next (ARIA, ChaCha)' - OPENSSL="$OPENSSL_NEXT" sh compat.sh -e '^$' -f 'ARIA\|CHACHA' + echo '#### compat.sh: next (all except NULL)' + OPENSSL="$OPENSSL_NEXT" sh compat.sh -e 'NULL' echo } | tee compat-test-$TEST_OUTPUT echo '^^^^^^^^^^^^^^^^ compat.sh ^^^^^^^^^^^^^^^^' From eb86b906d719c89bf19a3abb1ea92cadc7ab7289 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Mon, 22 Apr 2024 10:25:09 +0200 Subject: [PATCH 4/4] Fix full invocation of ssl-opt.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit had: - one obvious mistake (-f NULL with default -e runs nothing) - one unforeseen issue: OPENSSL_NEXT skips static ECDH - arguably scope creep: the stated goal was to simplify the full invocation (in particular, make it obvious that everything is run without having to remember the default value of EXCLUDE), but it also made an unrelated change: running most tests with OPENSSL_NEXT (hence the previous point). This commit should fix all this, in particular it switches back to running most tests with OPENSSL and using OPENSSL_NEXT only when needed. Hopefully in the future we'll do the opposite: most tests will run with a recent OpenSSL, and only those that need an older one will use something older. But that will be another PR. Signed-off-by: Manuel Pégourié-Gonnard --- tests/scripts/all.sh | 40 ++++++++++++++++++------------- tests/scripts/basic-build-test.sh | 8 +++---- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 3f7ad957d..9a674ed73 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -1212,17 +1212,19 @@ component_test_full_cmake_gcc_asan () { msg "test: main suites (inc. selftests) (full config, ASan build)" make test - msg "test: selftest (ASan build)" # ~ 10s + msg "test: selftest (full config, ASan build)" # ~ 10s programs/test/selftest msg "test: ssl-opt.sh (full config, ASan build)" tests/ssl-opt.sh - msg "test: compat.sh: NULL (full config, ASan build)" - tests/compat.sh -f 'NULL' + # Note: the next two invocations cover all compat.sh test cases. + # We should use the same here and in basic-build-test.sh. + msg "test: compat.sh: default version (full config, ASan build)" + tests/compat.sh -e 'ARIA\|CHACHA' - msg "test: compat.sh next: all except NULL (full config, ASan build)" - env OPENSSL="$OPENSSL_NEXT" tests/compat.sh -e 'NULL' + msg "test: compat.sh: next: ARIA, Chacha (full config, ASan build)" + env OPENSSL="$OPENSSL_NEXT" tests/compat.sh -e '^$' -f 'ARIA\|CHACHA' msg "test: context-info.sh (full config, ASan build)" # ~ 15 sec tests/context-info.sh @@ -1236,22 +1238,24 @@ component_test_full_cmake_gcc_asan_new_bignum () { CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make - msg "test: main suites (inc. selftests) (full config, ASan build)" + msg "test: main suites (inc. selftests) (full config, new bignum, ASan)" make test - msg "test: selftest (ASan build)" # ~ 10s + msg "test: selftest (full config, new bignum, ASan)" # ~ 10s programs/test/selftest - msg "test: ssl-opt.sh (full config, ASan build)" + msg "test: ssl-opt.sh (full config, new bignum, ASan)" tests/ssl-opt.sh - msg "test: compat.sh: NULL (full config, ASan build)" - tests/compat.sh -f 'NULL' + # Note: the next two invocations cover all compat.sh test cases. + # We should use the same here and in basic-build-test.sh. + msg "test: compat.sh: default version (full config, new bignum, ASan)" + tests/compat.sh -e 'ARIA\|CHACHA' - msg "test: compat.sh next: all except NULL (full config, ASan build)" - env OPENSSL="$OPENSSL_NEXT" tests/compat.sh -e 'NULL' + msg "test: compat.sh: next: ARIA, Chacha (full config, new bignum, ASan)" + env OPENSSL="$OPENSSL_NEXT" tests/compat.sh -e '^$' -f 'ARIA\|CHACHA' - msg "test: context-info.sh (full config, ASan build)" # ~ 15 sec + msg "test: context-info.sh (full config, new bignum, ASan)" # ~ 15 sec tests/context-info.sh } @@ -2610,11 +2614,13 @@ component_test_no_psa_crypto_full_cmake_asan() { msg "test: ssl-opt.sh (full minus PSA crypto)" tests/ssl-opt.sh - msg "test: compat.sh: NULL (full minus PSA crypto)" - tests/compat.sh -f 'NULL' + # Note: the next two invocations cover all compat.sh test cases. + # We should use the same here and in basic-build-test.sh. + msg "test: compat.sh: default version (full minus PSA crypto)" + tests/compat.sh -e 'ARIA\|CHACHA' - msg "test: compat.sh next: all except NULL (full minus PSA crypto)" - env OPENSSL="$OPENSSL_NEXT" tests/compat.sh -e 'NULL' + msg "test: compat.sh: next: ARIA, Chacha (full minus PSA crypto)" + env OPENSSL="$OPENSSL_NEXT" tests/compat.sh -e '^$' -f 'ARIA\|CHACHA' } component_test_psa_crypto_config_accel_ecdsa () { diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index e365eeddc..d2e955f1e 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -102,12 +102,12 @@ echo # Step 2c - Compatibility tests (keep going even if some tests fail) echo '################ compat.sh ################' { - echo '#### compat.sh: NULL ciphersuites' - sh compat.sh -e '^$' -f 'NULL' + echo '#### compat.sh: Default versions' + sh compat.sh -e 'ARIA\|CHACHA' echo - echo '#### compat.sh: next (all except NULL)' - OPENSSL="$OPENSSL_NEXT" sh compat.sh -e 'NULL' + echo '#### compat.sh: next (ARIA, ChaCha)' + OPENSSL="$OPENSSL_NEXT" sh compat.sh -e '^$' -f 'ARIA\|CHACHA' echo } | tee compat-test-$TEST_OUTPUT echo '^^^^^^^^^^^^^^^^ compat.sh ^^^^^^^^^^^^^^^^'