From 2a71fac470424bd0400a03defd12ce032e710d35 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 17 Sep 2024 15:07:22 +0200 Subject: [PATCH 01/14] In PSA generated tests, ignore mechanisms that are not implemented This clears more than half of the test cases that are not executed. This also captures a few negative test cases that are executed. Subsequent commits will refine the filtering. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 62 +++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 72dba99f7..698358dbb 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -7,6 +7,7 @@ less likely to be useful. """ import re +import typing import scripts_path # pylint: disable=unused-import from mbedtls_framework import outcome_analysis @@ -17,6 +18,67 @@ class CoverageTask(outcome_analysis.CoverageTask): # is just a warning. outcome_analysis.FULL_COVERAGE_BY_DEFAULT = False + @staticmethod + def _has_word_re(words: typing.Iterable[str]) -> typing.Pattern: + """Construct a regex that matches if any of the words appears. + + The occurrence must start and end at a word boundary. + """ + return re.compile(r'.*\b(?:' + r'|'.join(words) + r')\b.*') + + # generate_psa_tests.py generates test cases involving cryptographic + # mechanisms (key types, families, algorithms) that are declared but + # not implemented. Until we improve the Python scripts, ignore those + # test cases in the analysis. + # https://github.com/Mbed-TLS/mbedtls/issues/9572 + _PSA_MECHANISMS_NOT_IMPLEMENTED = [ + r'CBC_MAC', + r'DETERMINISTIC_DSA', + r'DET_DSA', + r'DSA', + r'ECC_KEY_PAIR\(BRAINPOOL_P_R1\) (?:160|192|224|320)-bit', + r'ECC_KEY_PAIR\(SECP_K1\) 225-bit', + r'ECC_PAIR\(BP_R1\) (?:160|192|224|320)-bit', + r'ECC_PAIR\(SECP_K1\) 225-bit', + r'ECC_PUBLIC_KEY\(BRAINPOOL_P_R1\) (?:160|192|224|320)-bit', + r'ECC_PUBLIC_KEY\(SECP_K1\) 225-bit', + r'ECC_PUB\(BP_R1\) (?:160|192|224|320)-bit', + r'ECC_PUB\(SECP_K1\) 225-bit', + r'ED25519PH', + r'ED448PH', + r'PEPPER', + r'PURE_EDDSA', + r'SECP_R2', + r'SECT_K1', + r'SECT_R1', + r'SECT_R2', + r'SHAKE256_512', + r'SHA_512_224', + r'SHA_512_256', + r'TWISTED_EDWARDS', + r'XTS', + ] + PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE = \ + _has_word_re(_PSA_MECHANISMS_NOT_IMPLEMENTED) + + IGNORED_TESTS = { + 'test_suite_psa_crypto_generate_key.generated': [ + PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, + ], + 'test_suite_psa_crypto_not_supported.generated': [ + PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, + ], + 'test_suite_psa_crypto_op_fail.generated': [ + PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, + ], + 'test_suite_psa_crypto_storage_format.current': [ + PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, + ], + 'test_suite_psa_crypto_storage_format.v0': [ + PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, + ], + } + # The names that we give to classes derived from DriverVSReference do not # follow the usual naming convention, because it's more readable to use From 5872c0d4c0b9f8d32a4344e62f280c3b28e2bb01 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 17 Sep 2024 17:15:29 +0200 Subject: [PATCH 02/14] In PSA generated tests, don't ignore not-implemented in some negative tests Some negative tests involving unsupported mechanisms are executed, because they're testing what happens if the mechanism is unsupported. Refine the ignore list for `test_suite_psa_crypto_generate_key.generated` and `test_suite_psa_crypto_op_fail.generated` accordingly. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 698358dbb..b0b16c54d 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -19,12 +19,21 @@ class CoverageTask(outcome_analysis.CoverageTask): outcome_analysis.FULL_COVERAGE_BY_DEFAULT = False @staticmethod - def _has_word_re(words: typing.Iterable[str]) -> typing.Pattern: + def _has_word_re(words: typing.Iterable[str], + exclude: typing.Optional[str] = None) -> typing.Pattern: """Construct a regex that matches if any of the words appears. The occurrence must start and end at a word boundary. + + If exclude is specified, strings containing a match for that + regular expression will not match the returned pattern. """ - return re.compile(r'.*\b(?:' + r'|'.join(words) + r')\b.*') + exclude_clause = r'' + if exclude: + exclude_clause = r'(?!.*' + exclude + ')' + return re.compile(exclude_clause + + r'.*\b(?:' + r'|'.join(words) + r')\b.*', + re.S) # generate_psa_tests.py generates test cases involving cryptographic # mechanisms (key types, families, algorithms) that are declared but @@ -63,13 +72,25 @@ class CoverageTask(outcome_analysis.CoverageTask): IGNORED_TESTS = { 'test_suite_psa_crypto_generate_key.generated': [ - PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, + # Ignore mechanisms that are not implemented, except + # for public keys for which we always test that + # psa_generate_key() returns PSA_ERROR_INVALID_ARGUMENT + # regardless of whether the specific key type is supported. + _has_word_re((mech + for mech in _PSA_MECHANISMS_NOT_IMPLEMENTED + if not mech.startswith('ECC_PUB')), + exclude=r'ECC_PUB'), ], 'test_suite_psa_crypto_not_supported.generated': [ PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, ], 'test_suite_psa_crypto_op_fail.generated': [ - PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, + # Ignore mechanisms that are not implemented, except + # for test cases that assume the mechanism is not supported. + _has_word_re(_PSA_MECHANISMS_NOT_IMPLEMENTED, + exclude=(r'.*: !(?:' + + r'|'.join(_PSA_MECHANISMS_NOT_IMPLEMENTED) + + r')\b')), ], 'test_suite_psa_crypto_storage_format.current': [ PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, From ab5cc9bd3023a75a2b5940ce660ef844f1fc7bd4 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 17 Sep 2024 17:57:11 +0200 Subject: [PATCH 03/14] In PSA generated tests, ignore patterns for which an issue has been raised Ignore certain test cases which either should not be generated or should be executed. For each ignore list entry, link to a GitHub issue whose definition of done includes removing the entry. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 41 +++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index b0b16c54d..4ee0c40cf 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -82,7 +82,21 @@ class CoverageTask(outcome_analysis.CoverageTask): exclude=r'ECC_PUB'), ], 'test_suite_psa_crypto_not_supported.generated': [ + # It is a bug that not-supported test cases aren't getting + # run for never-implemented key types. + # https://github.com/Mbed-TLS/mbedtls/issues/7915 PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, + # We mever test with DH key support disabled but support + # for a DH group enabled. The dependencies of these test + # cases don't really make sense. + # https://github.com/Mbed-TLS/mbedtls/issues/9574 + re.compile(r'PSA \w+ DH_.*type not supported'), + # We only test partial support for DH with the 2048-bit group + # enabled and the other groups disabled. + # https://github.com/Mbed-TLS/mbedtls/issues/9575 + 'PSA generate DH_KEY_PAIR(RFC7919) 2048-bit group not supported', + 'PSA import DH_KEY_PAIR(RFC7919) 2048-bit group not supported', + 'PSA import DH_PUBLIC_KEY(RFC7919) 2048-bit group not supported', ], 'test_suite_psa_crypto_op_fail.generated': [ # Ignore mechanisms that are not implemented, except @@ -91,6 +105,33 @@ class CoverageTask(outcome_analysis.CoverageTask): exclude=(r'.*: !(?:' + r'|'.join(_PSA_MECHANISMS_NOT_IMPLEMENTED) + r')\b')), + # Incorrect dependency generation. To be fixed as part of the + # resolution of https://github.com/Mbed-TLS/mbedtls/issues/9167 + # by forward-porting the commit + # "PSA test case generation: dependency inference class: operation fail" + # from https://github.com/Mbed-TLS/mbedtls/pull/9025 . + re.compile(r'.* with (?:DH|ECC)_(?:KEY_PAIR|PUBLIC_KEY)\(.*'), + # PBKDF2_HMAC is not in the default configuration, so we don't + # enable it in depends.py where we remove hashes. + # https://github.com/Mbed-TLS/mbedtls/issues/9576 + re.compile(r'PSA key_derivation PBKDF2_HMAC\(\w+\): !(?!PBKDF2_HMAC\Z).*'), + # We never test with TLS12_PRF or TLS12_PSK_TO_MS disabled + # but certain other things enabled. + # https://github.com/Mbed-TLS/mbedtls/issues/9577 + re.compile(r'PSA key_derivation TLS12_PRF\(\w+\): !TLS12_PRF'), + re.compile(r'PSA key_derivation TLS12_PSK_TO_MS' + r'\((?!SHA_256|SHA_384|SHA_512)\w+\): !TLS12_PSK_TO_MS'), + 'PSA key_derivation KEY_AGREEMENT(ECDH,TLS12_PRF(SHA_256)): !TLS12_PRF', + 'PSA key_derivation KEY_AGREEMENT(ECDH,TLS12_PRF(SHA_384)): !TLS12_PRF', + + # We never test with the HMAC algorithm enabled but the HMAC + # key type disabled. Those dependencies don't really make sense. + # https://github.com/Mbed-TLS/mbedtls/issues/9573 + re.compile(r'.* !HMAC with HMAC'), + # There's something wrong with PSA_WANT_ALG_RSA_PSS_ANY_SALT + # differing from PSA_WANT_ALG_RSA_PSS. + # https://github.com/Mbed-TLS/mbedtls/issues/9578 + re.compile(r'PSA sign RSA_PSS_ANY_SALT.*!(?:MD|RIPEMD|SHA).*'), ], 'test_suite_psa_crypto_storage_format.current': [ PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, From e3c64c3f26fd9646c6ccd24961b3df5b91ae24f6 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 17 Sep 2024 18:18:57 +0200 Subject: [PATCH 04/14] Fix typo in dependency Signed-off-by: Gilles Peskine --- tests/suites/test_suite_ssl.data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/suites/test_suite_ssl.data b/tests/suites/test_suite_ssl.data index 358d7c267..c522459bf 100644 --- a/tests/suites/test_suite_ssl.data +++ b/tests/suites/test_suite_ssl.data @@ -565,7 +565,7 @@ depends_on:PSA_WANT_ALG_SHA_384:PSA_WANT_KEY_TYPE_CAMELLIA:PSA_WANT_ALG_CBC_NO_P handshake_ciphersuite_select:"TLS-ECDH-ECDSA-WITH-CAMELLIA-256-CBC-SHA384":MBEDTLS_PK_ECDSA:"":PSA_ALG_ECDSA(PSA_ALG_ANY_HASH):PSA_ALG_ECDH:PSA_KEY_USAGE_SIGN_HASH|PSA_KEY_USAGE_DERIVE:0:MBEDTLS_TLS_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 Handshake, select ECDH-ECDSA-WITH-CAMELLIA-256-CBC-SHA384, opaque, PSA_ALG_SHA_384 -depends_on:PSA_WANT_ALG_SHA_384:PSA_WANT_KEY_TYPE_CAMELLIA:PSA_WANT_ALG_CBC_NO_PADDING:PSA_WANT_ECC_SECP_R1_256:PSA_WANT_ECC_SECP_R1_384:MBEDTLS_KEY_EXCHANGE_ECDH_ECDSA_ENABLED:MBEDTLS_USE_PSA_CRYPTO:MBEDTSL_PSA_CRYPTO_C +depends_on:PSA_WANT_ALG_SHA_384:PSA_WANT_KEY_TYPE_CAMELLIA:PSA_WANT_ALG_CBC_NO_PADDING:PSA_WANT_ECC_SECP_R1_256:PSA_WANT_ECC_SECP_R1_384:MBEDTLS_KEY_EXCHANGE_ECDH_ECDSA_ENABLED:MBEDTLS_USE_PSA_CRYPTO:MBEDTLS_PSA_CRYPTO_C handshake_ciphersuite_select:"TLS-ECDH-ECDSA-WITH-CAMELLIA-256-CBC-SHA384":MBEDTLS_PK_ECDSA:"":PSA_ALG_ECDSA(PSA_ALG_SHA_384):PSA_ALG_ECDH:PSA_KEY_USAGE_SIGN_HASH|PSA_KEY_USAGE_DERIVE:0:MBEDTLS_TLS_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 Handshake, select ECDH-ECDSA-WITH-CAMELLIA-256-CBC-SHA384, opaque, missing alg From b061ecd4e81c446d4e14943db666a8de69e3e2a0 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 17 Sep 2024 18:24:44 +0200 Subject: [PATCH 05/14] Remove test case involving SECP224K1 in PSA Our PSA crypto implementation does not and will not support SECP224K1. Signed-off-by: Gilles Peskine --- tf-psa-crypto/tests/suites/test_suite_psa_crypto.data | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tf-psa-crypto/tests/suites/test_suite_psa_crypto.data b/tf-psa-crypto/tests/suites/test_suite_psa_crypto.data index c2deaa525..e921c112d 100644 --- a/tf-psa-crypto/tests/suites/test_suite_psa_crypto.data +++ b/tf-psa-crypto/tests/suites/test_suite_psa_crypto.data @@ -7820,9 +7820,7 @@ ECP group ID <-> PSA family - SECP192K1 depends_on:PSA_WANT_ECC_SECP_K1_192 ecc_conversion_functions:MBEDTLS_ECP_DP_SECP192K1:PSA_ECC_FAMILY_SECP_K1:192 -ECP group ID <-> PSA family - SECP224K1 -depends_on:PSA_WANT_ECC_SECP_K1_224 -ecc_conversion_functions:MBEDTLS_ECP_DP_SECP224K1:PSA_ECC_FAMILY_SECP_K1:224 +# No test case for SECP224K1, which is not implemented in the PSA API. ECP group ID <-> PSA family - SECP256K1 depends_on:PSA_WANT_ECC_SECP_K1_256 From de2316b8ea1c60f244ef38e6e66957622e14010f Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 17 Sep 2024 18:32:05 +0200 Subject: [PATCH 06/14] Add ignore list entries for TLS tests that are not executed For each ignore list entry, link to a GitHub issue for its resolution, except for ssl-opt Valgrind tests which we never intend to run on the CI. Signed-off-by: Gilles Peskine --- tests/opt-testcases/tls13-misc.sh | 1 + tests/scripts/analyze_outcomes.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/tests/opt-testcases/tls13-misc.sh b/tests/opt-testcases/tls13-misc.sh index 2bd47d58e..cc6a31d79 100644 --- a/tests/opt-testcases/tls13-misc.sh +++ b/tests/opt-testcases/tls13-misc.sh @@ -922,6 +922,7 @@ run_test "TLS 1.3 m->O: resumption fails, no ticket support" \ # ephemeral then ticket based scenario we use for early data testing the first # handshake fails. The following skipped test is here to illustrate the kind # of testing we would like to do. +# https://github.com/Mbed-TLS/mbedtls/issues/9582 skip_next_test requires_openssl_tls1_3_with_compatible_ephemeral requires_config_enabled MBEDTLS_SSL_CLI_C diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 4ee0c40cf..8cd0c160e 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -71,6 +71,27 @@ class CoverageTask(outcome_analysis.CoverageTask): _has_word_re(_PSA_MECHANISMS_NOT_IMPLEMENTED) IGNORED_TESTS = { + 'ssl-opt': [ + # We don't run ssl-opt.sh with Valgrind on the CI because + # it's extremely slow. We don't intend to change this. + 'DTLS client reconnect from same port: reconnect, nbio, valgrind', + + # We don't have IPv6 in our CI environment. + # https://github.com/Mbed-TLS/mbedtls-test/issues/176 + 'DTLS cookie: enabled, IPv6', + # Disabled due to OpenSSL bug. + # https://github.com/openssl/openssl/issues/18887 + 'DTLS fragmenting: 3d, openssl client, DTLS 1.2', + # We don't run ssl-opt.sh with Valgrind on the CI because + # it's extremely slow. We don't intend to change this. + 'DTLS fragmenting: proxy MTU: auto-reduction (with valgrind)', + # It seems that we don't run `ssl-opt.sh` with + # `MBEDTLS_USE_PSA_CRYPTO` enabled but `MBEDTLS_SSL_ASYNC_PRIVATE` + # disabled. + # https://github.com/Mbed-TLS/mbedtls/issues/9581 + 'Opaque key for server authentication: invalid key: decrypt with ECC key, no async', + 'Opaque key for server authentication: invalid key: ecdh with RSA key, no async', + ], 'test_suite_psa_crypto_generate_key.generated': [ # Ignore mechanisms that are not implemented, except # for public keys for which we always test that @@ -139,6 +160,14 @@ class CoverageTask(outcome_analysis.CoverageTask): 'test_suite_psa_crypto_storage_format.v0': [ PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, ], + 'tls13-misc': [ + # Disabled due to OpenSSL bug. + # https://github.com/openssl/openssl/issues/10714 + 'TLS 1.3 O->m: resumption', + # Disabled due to OpenSSL command line limitation. + # https://github.com/Mbed-TLS/mbedtls/issues/9582 + 'TLS 1.3 m->O: resumption with early data', + ], } From b0ec85dcc64037c2648b52b2fd23abadba392089 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 17 Sep 2024 18:33:29 +0200 Subject: [PATCH 07/14] Add ignore list entries for crypto tests that are not executed For each ignore list entry, link to a GitHub issue for its resolution. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 8cd0c160e..8dd812f1d 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -92,6 +92,19 @@ class CoverageTask(outcome_analysis.CoverageTask): 'Opaque key for server authentication: invalid key: decrypt with ECC key, no async', 'Opaque key for server authentication: invalid key: ecdh with RSA key, no async', ], + 'test_suite_pkcs12': [ + # Probably a missing configuration in the CI. + # https://github.com/Mbed-TLS/mbedtls/issues/9580 + 'PBE Decrypt, (Invalid padding & PKCS7 padding disabled)', + 'PBE Encrypt, pad = 8 (PKCS7 padding disabled)', + ], + 'test_suite_pkcs5': [ + # Probably a missing configuration in the CI. + # https://github.com/Mbed-TLS/mbedtls/issues/9580 + 'PBES2 Decrypt (Invalid padding & PKCS7 padding disabled)', + 'PBES2 Encrypt, pad=6 (PKCS7 padding disabled)', + 'PBES2 Encrypt, pad=8 (PKCS7 padding disabled)', + ], 'test_suite_psa_crypto_generate_key.generated': [ # Ignore mechanisms that are not implemented, except # for public keys for which we always test that @@ -102,6 +115,18 @@ class CoverageTask(outcome_analysis.CoverageTask): if not mech.startswith('ECC_PUB')), exclude=r'ECC_PUB'), ], + 'test_suite_psa_crypto_metadata': [ + # Algorithms declared but not supported. + # https://github.com/Mbed-TLS/mbedtls/issues/9579 + 'Asymmetric signature: Ed25519ph', + 'Asymmetric signature: Ed448ph', + 'Asymmetric signature: pure EdDSA', + 'Cipher: XTS', + 'MAC: CBC_MAC-3DES', + 'MAC: CBC_MAC-AES-128', + 'MAC: CBC_MAC-AES-192', + 'MAC: CBC_MAC-AES-256', + ], 'test_suite_psa_crypto_not_supported.generated': [ # It is a bug that not-supported test cases aren't getting # run for never-implemented key types. From 2fd25bb2a589053cfb5c5db91bd1f38b960baca9 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 17 Sep 2024 19:46:18 +0200 Subject: [PATCH 08/14] Add ignore list entries for configurations that are not tested For each ignore list entry, link to a GitHub issue for its resolution, except for a few configurations which there is a good reason to leave uncovered. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 104 +++++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 2 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 8dd812f1d..75dc41d6f 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -92,14 +92,114 @@ class CoverageTask(outcome_analysis.CoverageTask): 'Opaque key for server authentication: invalid key: decrypt with ECC key, no async', 'Opaque key for server authentication: invalid key: ecdh with RSA key, no async', ], + 'test_suite_config.mbedtls_boolean': [ + # We never test with CBC/PKCS5/PKCS12 enabled but + # PKCS7 padding disabled. + # https://github.com/Mbed-TLS/mbedtls/issues/9580 + 'Config: !MBEDTLS_CIPHER_PADDING_PKCS7', + # https://github.com/Mbed-TLS/mbedtls/issues/9583 + 'Config: !MBEDTLS_ECP_NIST_OPTIM', + # Missing coverage of test configurations. + # https://github.com/Mbed-TLS/mbedtls/issues/9585 + 'Config: !MBEDTLS_SSL_DTLS_ANTI_REPLAY', + # Missing coverage of test configurations. + # https://github.com/Mbed-TLS/mbedtls/issues/9585 + 'Config: !MBEDTLS_SSL_DTLS_HELLO_VERIFY', + # We don't run test_suite_config when we test this. + # https://github.com/Mbed-TLS/mbedtls/issues/9586 + 'Config: !MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_ENABLED', + # We only test multithreading with pthreads. + # https://github.com/Mbed-TLS/mbedtls/issues/9584 + 'Config: !MBEDTLS_THREADING_PTHREAD', + # Built but not tested. + # https://github.com/Mbed-TLS/mbedtls/issues/9587 + 'Config: MBEDTLS_AES_USE_HARDWARE_ONLY', + # Untested platform-specific optimizations. + # https://github.com/Mbed-TLS/mbedtls/issues/9588 + 'Config: MBEDTLS_HAVE_SSE2', + # Obsolete configuration option, to be replaced by + # PSA entropy drivers. + # https://github.com/Mbed-TLS/mbedtls/issues/8150 + 'Config: MBEDTLS_NO_PLATFORM_ENTROPY', + # Untested aspect of the platform interface. + # https://github.com/Mbed-TLS/mbedtls/issues/9589 + 'Config: MBEDTLS_PLATFORM_NO_STD_FUNCTIONS', + # In a client-server build, test_suite_config runs in the + # client configuration, so it will never report + # MBEDTLS_PSA_CRYPTO_SPM as enabled. That's ok. + 'Config: MBEDTLS_PSA_CRYPTO_SPM', + # We don't test on armv8 yet. + 'Config: MBEDTLS_SHA256_USE_A64_CRYPTO_IF_PRESENT', + 'Config: MBEDTLS_SHA256_USE_A64_CRYPTO_ONLY', + 'Config: MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY', + 'Config: MBEDTLS_SHA512_USE_A64_CRYPTO_ONLY', + # We don't run test_suite_config when we test this. + # https://github.com/Mbed-TLS/mbedtls/issues/9586 + 'Config: MBEDTLS_TEST_CONSTANT_FLOW_VALGRIND', + ], + 'test_suite_config.psa_boolean': [ + # We don't test with HMAC disabled. + # https://github.com/Mbed-TLS/mbedtls/issues/9591 + 'Config: !PSA_WANT_ALG_HMAC', + # We don't test with HMAC disabled. + # https://github.com/Mbed-TLS/mbedtls/issues/9591 + 'Config: !PSA_WANT_ALG_TLS12_PRF', + # The DERIVE key type is always enabled. + 'Config: !PSA_WANT_KEY_TYPE_DERIVE', + # More granularity of key pair type enablement macros + # than we care to test. + # https://github.com/Mbed-TLS/mbedtls/issues/9590 + 'Config: !PSA_WANT_KEY_TYPE_DH_KEY_PAIR_EXPORT', + 'Config: !PSA_WANT_KEY_TYPE_DH_KEY_PAIR_GENERATE', + 'Config: !PSA_WANT_KEY_TYPE_DH_KEY_PAIR_IMPORT', + # More granularity of key pair type enablement macros + # than we care to test. + # https://github.com/Mbed-TLS/mbedtls/issues/9590 + 'Config: !PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_EXPORT', + 'Config: !PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_IMPORT', + # We don't test with HMAC disabled. + # https://github.com/Mbed-TLS/mbedtls/issues/9591 + 'Config: !PSA_WANT_KEY_TYPE_HMAC', + # The PASSWORD key type is always enabled. + 'Config: !PSA_WANT_KEY_TYPE_PASSWORD', + # The PASSWORD_HASH key type is always enabled. + 'Config: !PSA_WANT_KEY_TYPE_PASSWORD_HASH', + # The RAW_DATA key type is always enabled. + 'Config: !PSA_WANT_KEY_TYPE_RAW_DATA', + # More granularity of key pair type enablement macros + # than we care to test. + # https://github.com/Mbed-TLS/mbedtls/issues/9590 + 'Config: !PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_EXPORT', + 'Config: !PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_IMPORT', + # Algorithm declared but not supported. + 'Config: PSA_WANT_ALG_CBC_MAC', + # Algorithm declared but not supported. + 'Config: PSA_WANT_ALG_XTS', + # Family declared but not supported. + 'Config: PSA_WANT_ECC_SECP_K1_224', + # More granularity of key pair type enablement macros + # than we care to test. + # https://github.com/Mbed-TLS/mbedtls/issues/9590 + 'Config: PSA_WANT_KEY_TYPE_DH_KEY_PAIR_DERIVE', + 'Config: PSA_WANT_KEY_TYPE_ECC_KEY_PAIR', + 'Config: PSA_WANT_KEY_TYPE_RSA_KEY_PAIR', + 'Config: PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_DERIVE', + ], + 'test_suite_config.psa_combinations': [ + # We don't test this unusual, but sensible configuration. + # https://github.com/Mbed-TLS/mbedtls/issues/9592 + 'Config: PSA_WANT_ALG_DETERMINSTIC_ECDSA without PSA_WANT_ALG_ECDSA', + ], 'test_suite_pkcs12': [ - # Probably a missing configuration in the CI. + # We never test with CBC/PKCS5/PKCS12 enabled but + # PKCS7 padding disabled. # https://github.com/Mbed-TLS/mbedtls/issues/9580 'PBE Decrypt, (Invalid padding & PKCS7 padding disabled)', 'PBE Encrypt, pad = 8 (PKCS7 padding disabled)', ], 'test_suite_pkcs5': [ - # Probably a missing configuration in the CI. + # We never test with CBC/PKCS5/PKCS12 enabled but + # PKCS7 padding disabled. # https://github.com/Mbed-TLS/mbedtls/issues/9580 'PBES2 Decrypt (Invalid padding & PKCS7 padding disabled)', 'PBES2 Encrypt, pad=6 (PKCS7 padding disabled)', From 24b03d8b72ba47b2a53635baf64554661a4d83f1 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 4 Oct 2024 16:22:24 +0200 Subject: [PATCH 09/14] Add ignore list for restartable ECDH in TLS TLS only supports actual restartable ECDH with the legacy code that's going away, not with the MBEDTLS_USE_PSA_CRYPTO code that's becoming the only variant. This leaves a few test cases that validate restartable ECDH in TLS as desirable, but not currently able to pass. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 75dc41d6f..0962cfe90 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -85,6 +85,9 @@ class CoverageTask(outcome_analysis.CoverageTask): # We don't run ssl-opt.sh with Valgrind on the CI because # it's extremely slow. We don't intend to change this. 'DTLS fragmenting: proxy MTU: auto-reduction (with valgrind)', + # TLS doesn't use restartable ECDH yet. + # https://github.com/Mbed-TLS/mbedtls/issues/7294 + re.compile(r'EC restart:.*no USE_PSA.*'), # It seems that we don't run `ssl-opt.sh` with # `MBEDTLS_USE_PSA_CRYPTO` enabled but `MBEDTLS_SSL_ASYNC_PRIVATE` # disabled. From d9c40f538dd5dab83106b765a9412353aa6eccf7 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 4 Oct 2024 16:24:02 +0200 Subject: [PATCH 10/14] We never test without the PSA client code. Should we? To be decided later. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 0962cfe90..ee14826d0 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -102,6 +102,9 @@ class CoverageTask(outcome_analysis.CoverageTask): 'Config: !MBEDTLS_CIPHER_PADDING_PKCS7', # https://github.com/Mbed-TLS/mbedtls/issues/9583 'Config: !MBEDTLS_ECP_NIST_OPTIM', + # We never test without the PSA client code. Should we? + # https://github.com/Mbed-TLS/TF-PSA-Crypto/issues/112 + 'Config: !MBEDTLS_PSA_CRYPTO_CLIENT', # Missing coverage of test configurations. # https://github.com/Mbed-TLS/mbedtls/issues/9585 'Config: !MBEDTLS_SSL_DTLS_ANTI_REPLAY', From 095561cca1c5dfef966390b204fa4be12ce77768 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 4 Oct 2024 16:24:26 +0200 Subject: [PATCH 11/14] Switch outcome analysis to enforcing that all tests are executed Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index ee14826d0..58e21e3e4 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -14,9 +14,7 @@ from mbedtls_framework import outcome_analysis class CoverageTask(outcome_analysis.CoverageTask): - # We'll populate IGNORED_TESTS soon. In the meantime, lack of coverage - # is just a warning. - outcome_analysis.FULL_COVERAGE_BY_DEFAULT = False + """Justify test cases that are never executed.""" @staticmethod def _has_word_re(words: typing.Iterable[str], From ec2278bdd460e487ad0cdb13d8a7bfadc803d3bc Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 10 Oct 2024 21:39:58 +0200 Subject: [PATCH 12/14] Update framework like 3.6 backport The framework change is not needed in development, but should be harmless. Signed-off-by: Gilles Peskine --- framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework b/framework index 1de0641e7..239c0d8d1 160000 --- a/framework +++ b/framework @@ -1 +1 @@ -Subproject commit 1de0641e789d3c38b3ce99d7922002992cbe816c +Subproject commit 239c0d8d17884d192654519ce0de4d4c5ee4ae09 From 5e3ed3f8a0e05de5ef4325d34fd34b7173a6c500 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 11 Oct 2024 12:00:44 +0200 Subject: [PATCH 13/14] Minor readability improvements Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 58e21e3e4..2acf77a95 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -31,7 +31,7 @@ class CoverageTask(outcome_analysis.CoverageTask): exclude_clause = r'(?!.*' + exclude + ')' return re.compile(exclude_clause + r'.*\b(?:' + r'|'.join(words) + r')\b.*', - re.S) + re.DOTALL) # generate_psa_tests.py generates test cases involving cryptographic # mechanisms (key types, families, algorithms) that are declared but @@ -73,7 +73,6 @@ class CoverageTask(outcome_analysis.CoverageTask): # We don't run ssl-opt.sh with Valgrind on the CI because # it's extremely slow. We don't intend to change this. 'DTLS client reconnect from same port: reconnect, nbio, valgrind', - # We don't have IPv6 in our CI environment. # https://github.com/Mbed-TLS/mbedtls-test/issues/176 'DTLS cookie: enabled, IPv6', @@ -236,7 +235,7 @@ class CoverageTask(outcome_analysis.CoverageTask): # run for never-implemented key types. # https://github.com/Mbed-TLS/mbedtls/issues/7915 PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, - # We mever test with DH key support disabled but support + # We never test with DH key support disabled but support # for a DH group enabled. The dependencies of these test # cases don't really make sense. # https://github.com/Mbed-TLS/mbedtls/issues/9574 From 86421f3e877934e40be0026730c2613c095d712c Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 16 Oct 2024 19:34:30 +0200 Subject: [PATCH 14/14] Update framework to main Signed-off-by: Gilles Peskine --- framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework b/framework index 239c0d8d1..3eafac12a 160000 --- a/framework +++ b/framework @@ -1 +1 @@ -Subproject commit 239c0d8d17884d192654519ce0de4d4c5ee4ae09 +Subproject commit 3eafac12ae1ddc68cc1f0aefdff540d6d3d5a2fb