From d893837d017086aec55dccfee287f35bd40452d7 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Thu, 23 Feb 2023 10:04:58 +0800 Subject: [PATCH 01/10] compat.sh: add --list-test-case The option --list-test-case lists all potential test cases without executing them. The test case description is identical with $TITLE during test case execution. Signed-off-by: Yanray Wang --- tests/compat.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/compat.sh b/tests/compat.sh index 8f7d72c7b..440e87a2a 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -110,6 +110,36 @@ print_usage() { printf " \tAlso available: GnuTLS (needs v3.2.15 or higher)\n" printf " -M|--memcheck\tCheck memory leaks and errors.\n" printf " -v|--verbose\tSet verbose output.\n" + printf " --list-test-case\tList all potential test cases (No Execution)\n" +} + +# print_test_title +print_test_title() { + for i in $3; do + TITLE="$1->$2 $MODE,$VERIF $i" + echo "$TITLE" + done +} + +list_test_case() { + reset_ciphersuites + for TYPE in $TYPES; do + add_common_ciphersuites + add_openssl_ciphersuites + add_gnutls_ciphersuites + add_mbedtls_ciphersuites + done + + for VERIFY in $VERIFIES; do + VERIF=$(echo $VERIFY | tr '[:upper:]' '[:lower:]') + for MODE in $MODES; do + print_test_title m o "$O_CIPHERS" + print_test_title o m "$O_CIPHERS" + print_test_title m g "$G_CIPHERS" + print_test_title g m "$G_CIPHERS" + print_test_title m m "$M_CIPHERS" + done + done } get_options() { @@ -139,6 +169,10 @@ get_options() { -M|--memcheck) MEMCHECK=1 ;; + --list-test-case) + list_test_case + exit 0 + ;; -h|--help) print_usage exit 0 From 235469302d592873dac98230f7ede501d0150315 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Fri, 24 Feb 2023 14:53:29 +0800 Subject: [PATCH 02/10] check_test_cases.py: support checking test coverage in compat.sh Test case description in compat.sh is in format of [ogm]->[ogm] TLSmode, VERIFY CIPHERSUITE_NAME This program calls compat.sh to list all potential test case descriptions then checks test case duplication. Signed-off-by: Yanray Wang --- tests/scripts/check_test_cases.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index d84ed042c..dd167672f 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -25,6 +25,7 @@ import argparse import glob import os import re +import subprocess import sys class Results: @@ -111,6 +112,24 @@ state may override this method. self.process_test_case(descriptions, file_name, line_number, description) + def walk_compat_sh(self, file_name): + """Iterate over the test cases compat.sh with a similar format.""" + descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none + compat_cmd = ['sh', file_name, '--list-test-case'] + result = subprocess.run(compat_cmd, + stdout=subprocess.PIPE, + check=False) + if result.returncode != 0: + print(*compat_cmd, 'returned', str(result.returncode)) + return + else: + # Pattern: g->m dtls12,no TLS_DHE_PSK_WITH_AES_128_CBC_SHA\n + m = re.findall(br'[^ogm]*((?:[ogm]->[ogm]\s*\w*.\w*\s\w*)*)\n', + result.stdout) + if m: + for i in m: + self.process_test_case(descriptions, file_name, 1, i) + @staticmethod def collect_test_directories(): """Get the relative path for the TLS and Crypto test directories.""" @@ -136,6 +155,9 @@ state may override this method. for ssl_opt_file_name in glob.glob(os.path.join(directory, 'opt-testcases', '*.sh')): self.walk_ssl_opt_sh(ssl_opt_file_name) + compat_sh = os.path.join(directory, 'compat.sh') + if os.path.exists(compat_sh): + self.walk_compat_sh(compat_sh) class TestDescriptions(TestDescriptionExplorer): """Collect the available test cases.""" From 3fcd3a73c97394a52b82badf67e2357c31ffd0cf Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Fri, 24 Feb 2023 17:07:47 +0800 Subject: [PATCH 03/10] compat.sh: uniform test description Test case description is printed by different block of code. This causes code maintenance harder since we need to maintain two parts of code with same functionality. print_test_title is used to control test case description in compat.sh Signed-off-by: Yanray Wang --- tests/compat.sh | 25 +++++++++++++------------ tests/scripts/check_test_cases.py | 4 ++-- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/tests/compat.sh b/tests/compat.sh index 440e87a2a..9a435c312 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -35,6 +35,7 @@ TESTS=0 FAILED=0 SKIPPED=0 SRVMEM=0 +LIST_TEST_CASE=0 # default commands, can be overridden by the environment : ${M_SRV:=../programs/ssl/ssl_server2} @@ -113,14 +114,6 @@ print_usage() { printf " --list-test-case\tList all potential test cases (No Execution)\n" } -# print_test_title -print_test_title() { - for i in $3; do - TITLE="$1->$2 $MODE,$VERIF $i" - echo "$TITLE" - done -} - list_test_case() { reset_ciphersuites for TYPE in $TYPES; do @@ -170,6 +163,7 @@ get_options() { MEMCHECK=1 ;; --list-test-case) + LIST_TEST_CASE=1 list_test_case exit 0 ;; @@ -824,14 +818,21 @@ wait_client_done() { echo "EXIT: $EXIT" >> $CLI_OUT } +# print_test_title +print_test_title() { + for i in $3; do + TITLE="$1->$2 $MODE,$VERIF $i" + DOTS72="........................................................................" + printf "%s %.*s " "$TITLE" "$((71 - ${#TITLE}))" "$DOTS72" + [ $LIST_TEST_CASE -eq 1 ] && printf "\n" + done +} + # run_client PROGRAM_NAME STANDARD_CIPHER_SUITE PROGRAM_CIPHER_SUITE run_client() { # announce what we're going to do TESTS=$(( $TESTS + 1 )) - TITLE="${1%"${1#?}"}->${SERVER_NAME%"${SERVER_NAME#?}"}" - TITLE="$TITLE $MODE,$VERIF $2" - DOTS72="........................................................................" - printf "%s %.*s " "$TITLE" "$((71 - ${#TITLE}))" "$DOTS72" + print_test_title "${1%"${1#?}"}" "${SERVER_NAME%"${SERVER_NAME#?}"}" $2 # should we skip? if [ "X$SKIP_NEXT" = "XYES" ]; then diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index dd167672f..b75d51743 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -123,8 +123,8 @@ state may override this method. print(*compat_cmd, 'returned', str(result.returncode)) return else: - # Pattern: g->m dtls12,no TLS_DHE_PSK_WITH_AES_128_CBC_SHA\n - m = re.findall(br'[^ogm]*((?:[ogm]->[ogm]\s*\w*.\w*\s\w*)*)\n', + # Pattern: g->m dtls12,no TLS_DHE_PSK_WITH_AES_128_CBC_SHA .......... \n + m = re.findall(br'[^ogm]*((?:[ogm]->[ogm]\s*\w*.\w*\s\w*)*)\s*\.*\s*\n', result.stdout) if m: for i in m: From 7b394da7381aa58f009da0995f8ae9cc92488796 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Thu, 2 Mar 2023 11:41:26 +0800 Subject: [PATCH 04/10] compat.sh: fix uncompatiable name of peers in --list-test-case Signed-off-by: Yanray Wang --- tests/compat.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/compat.sh b/tests/compat.sh index 9a435c312..37df940ce 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -126,10 +126,10 @@ list_test_case() { for VERIFY in $VERIFIES; do VERIF=$(echo $VERIFY | tr '[:upper:]' '[:lower:]') for MODE in $MODES; do - print_test_title m o "$O_CIPHERS" - print_test_title o m "$O_CIPHERS" - print_test_title m g "$G_CIPHERS" - print_test_title g m "$G_CIPHERS" + print_test_title m O "$O_CIPHERS" + print_test_title O m "$O_CIPHERS" + print_test_title m G "$G_CIPHERS" + print_test_title G m "$G_CIPHERS" print_test_title m m "$M_CIPHERS" done done From fb784b26d29730a69842f9cb11b1cb400e896282 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Thu, 2 Mar 2023 11:55:55 +0800 Subject: [PATCH 05/10] compat.sh: uniform TITLE format for --list-test-case and run_client uniform_title is used to print identical format of $TITLE between --list-test-case and run_client. In such way, no matter how $TITLE is developed, --list-test-case will in the same format of test case description as stored in OUTCOME.CSV. Signed-off-by: Yanray Wang --- tests/compat.sh | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/tests/compat.sh b/tests/compat.sh index 37df940ce..d63fc063b 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -35,7 +35,6 @@ TESTS=0 FAILED=0 SKIPPED=0 SRVMEM=0 -LIST_TEST_CASE=0 # default commands, can be overridden by the environment : ${M_SRV:=../programs/ssl/ssl_server2} @@ -114,6 +113,14 @@ print_usage() { printf " --list-test-case\tList all potential test cases (No Execution)\n" } +# print_test_case +print_test_case() { + for i in $3; do + uniform_title $1 $2 $i + echo $TITLE + done +} + list_test_case() { reset_ciphersuites for TYPE in $TYPES; do @@ -126,11 +133,11 @@ list_test_case() { for VERIFY in $VERIFIES; do VERIF=$(echo $VERIFY | tr '[:upper:]' '[:lower:]') for MODE in $MODES; do - print_test_title m O "$O_CIPHERS" - print_test_title O m "$O_CIPHERS" - print_test_title m G "$G_CIPHERS" - print_test_title G m "$G_CIPHERS" - print_test_title m m "$M_CIPHERS" + print_test_case m O "$O_CIPHERS" + print_test_case O m "$O_CIPHERS" + print_test_case m G "$G_CIPHERS" + print_test_case G m "$G_CIPHERS" + print_test_case m m "$M_CIPHERS" done done } @@ -163,7 +170,6 @@ get_options() { MEMCHECK=1 ;; --list-test-case) - LIST_TEST_CASE=1 list_test_case exit 0 ;; @@ -818,21 +824,21 @@ wait_client_done() { echo "EXIT: $EXIT" >> $CLI_OUT } -# print_test_title -print_test_title() { - for i in $3; do - TITLE="$1->$2 $MODE,$VERIF $i" - DOTS72="........................................................................" - printf "%s %.*s " "$TITLE" "$((71 - ${#TITLE}))" "$DOTS72" - [ $LIST_TEST_CASE -eq 1 ] && printf "\n" - done +# uniform_title +# $TITLE is considered as test case description for both --list-test-case and +# MBEDTLS_TEST_OUTCOME_FILE. This function aims to control the format of +# each test case description. +uniform_title() { + TITLE="$1->$2 $MODE,$VERIF $3" } # run_client PROGRAM_NAME STANDARD_CIPHER_SUITE PROGRAM_CIPHER_SUITE run_client() { # announce what we're going to do TESTS=$(( $TESTS + 1 )) - print_test_title "${1%"${1#?}"}" "${SERVER_NAME%"${SERVER_NAME#?}"}" $2 + uniform_title "${1%"${1#?}"}" "${SERVER_NAME%"${SERVER_NAME#?}"}" $2 + DOTS72="........................................................................" + printf "%s %.*s " "$TITLE" "$((71 - ${#TITLE}))" "$DOTS72" # should we skip? if [ "X$SKIP_NEXT" = "XYES" ]; then From 521710e91dadbfb41a033aa2e39426f2b139af89 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Thu, 2 Mar 2023 14:45:01 +0800 Subject: [PATCH 06/10] check_test_cases.py: simplify how to store test case description Signed-off-by: Yanray Wang --- tests/scripts/check_test_cases.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index b75d51743..b0b84213a 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -123,12 +123,11 @@ state may override this method. print(*compat_cmd, 'returned', str(result.returncode)) return else: - # Pattern: g->m dtls12,no TLS_DHE_PSK_WITH_AES_128_CBC_SHA .......... \n - m = re.findall(br'[^ogm]*((?:[ogm]->[ogm]\s*\w*.\w*\s\w*)*)\s*\.*\s*\n', - result.stdout) - if m: - for i in m: - self.process_test_case(descriptions, file_name, 1, i) + # Assume compat.sh is responsible for printing identical format of + # test case description between --list-test-case and its OUTCOME.CSV + description = result.stdout.strip().split(b'\n') + for idx, descrip in enumerate(description): + self.process_test_case(descriptions, file_name, idx, descrip) @staticmethod def collect_test_directories(): From cdc07083345eef2b2404d7a4a9c55c2546a7899f Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Fri, 3 Mar 2023 17:12:29 +0800 Subject: [PATCH 07/10] check_test_cases.py: use check_output to capture error and return This commit includes: - use subprocess.check_output to report error and capture return value - add comment as a reminder for option --list-test-case Signed-off-by: Yanray Wang --- tests/compat.sh | 3 +++ tests/scripts/check_test_cases.py | 19 +++++++------------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/tests/compat.sh b/tests/compat.sh index d63fc063b..ff621fc75 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -121,6 +121,7 @@ print_test_case() { done } +# list_test_case lists all potential test cases in compat.sh without execution list_test_case() { reset_ciphersuites for TYPE in $TYPES; do @@ -169,6 +170,8 @@ get_options() { -M|--memcheck) MEMCHECK=1 ;; + # Please check scripts/check_test_cases.py correspondingly + # if you have to modify option, --list-test-case --list-test-case) list_test_case exit 0 diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index b0b84213a..ae2bfdfe0 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -116,18 +116,13 @@ state may override this method. """Iterate over the test cases compat.sh with a similar format.""" descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none compat_cmd = ['sh', file_name, '--list-test-case'] - result = subprocess.run(compat_cmd, - stdout=subprocess.PIPE, - check=False) - if result.returncode != 0: - print(*compat_cmd, 'returned', str(result.returncode)) - return - else: - # Assume compat.sh is responsible for printing identical format of - # test case description between --list-test-case and its OUTCOME.CSV - description = result.stdout.strip().split(b'\n') - for idx, descrip in enumerate(description): - self.process_test_case(descriptions, file_name, idx, descrip) + compat_output = subprocess.check_output(compat_cmd, + stderr=subprocess.STDOUT) + # Assume compat.sh is responsible for printing identical format of + # test case description between --list-test-case and its OUTCOME.CSV + description = compat_output.strip().split(b'\n') + for idx, descrip in enumerate(description): + self.process_test_case(descriptions, file_name, idx, descrip) @staticmethod def collect_test_directories(): From 2b50c651da1cf13a06956438508c58b36c534aff Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Mon, 6 Mar 2023 19:35:04 +0800 Subject: [PATCH 08/10] check_test_cases.py: do not redirect stderr to stdout Signed-off-by: Yanray Wang --- tests/scripts/check_test_cases.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index ae2bfdfe0..2692371ab 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -116,8 +116,7 @@ state may override this method. """Iterate over the test cases compat.sh with a similar format.""" descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none compat_cmd = ['sh', file_name, '--list-test-case'] - compat_output = subprocess.check_output(compat_cmd, - stderr=subprocess.STDOUT) + compat_output = subprocess.check_output(compat_cmd) # Assume compat.sh is responsible for printing identical format of # test case description between --list-test-case and its OUTCOME.CSV description = compat_output.strip().split(b'\n') From 14e052fd7cca9a4165bd45b6ad0fcc10cdafbb28 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Mon, 6 Mar 2023 19:37:07 +0800 Subject: [PATCH 09/10] compat.sh: return $? in option --list-test-case to handle error case Signed-off-by: Yanray Wang --- tests/compat.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/compat.sh b/tests/compat.sh index ff621fc75..68c10c3cc 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -174,7 +174,7 @@ get_options() { # if you have to modify option, --list-test-case --list-test-case) list_test_case - exit 0 + exit $? ;; -h|--help) print_usage From 63f0abe2265ab12e31affcacc30c5658fb38ffc1 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Wed, 30 Aug 2023 18:31:35 +0800 Subject: [PATCH 10/10] check_test_cases: add a comment to explain idx in walk_compat_sh Signed-off-by: Yanray Wang --- tests/scripts/check_test_cases.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index 2692371ab..1395d4d90 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -120,6 +120,8 @@ state may override this method. # Assume compat.sh is responsible for printing identical format of # test case description between --list-test-case and its OUTCOME.CSV description = compat_output.strip().split(b'\n') + # idx indicates the number of test case since there is no line number + # in `compat.sh` for each test case. for idx, descrip in enumerate(description): self.process_test_case(descriptions, file_name, idx, descrip)