From 7ad603e662cd67ed675aff1870e2db7eb32c6ade Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 10 Dec 2017 23:22:20 +0100 Subject: [PATCH 1/6] all.sh: indent --- tests/scripts/all.sh | 128 +++++++++++++++++++++---------------------- 1 file changed, 64 insertions(+), 64 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 77deecbc9..4decbb1cb 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -252,9 +252,9 @@ export GNUTLS_SERV="$GNUTLS_SERV" # Make sure the tools we need are available. check_tools "$OPENSSL" "$OPENSSL_LEGACY" "$GNUTLS_CLI" "$GNUTLS_SERV" \ - "$GNUTLS_LEGACY_CLI" "$GNUTLS_LEGACY_SERV" "doxygen" "dot" \ - "arm-none-eabi-gcc" "$ARMC5_CC" "$ARMC5_AR" "$ARMC6_CC" "$ARMC6_AR" \ - "i686-w64-mingw32-gcc" + "$GNUTLS_LEGACY_CLI" "$GNUTLS_LEGACY_SERV" "doxygen" "dot" \ + "arm-none-eabi-gcc" "$ARMC5_CC" "$ARMC5_AR" "$ARMC6_CC" "$ARMC6_AR" \ + "i686-w64-mingw32-gcc" # # Test Suites to be executed @@ -461,42 +461,42 @@ msg "test: MBEDTLS_TEST_NULL_ENTROPY - main suites (inc. selftests) (ASan build) make test if uname -a | grep -F Linux >/dev/null; then -msg "build/test: make shared" # ~ 40s -cleanup -make SHARED=1 all check + msg "build/test: make shared" # ~ 40s + cleanup + make SHARED=1 all check fi if uname -a | grep -F x86_64 >/dev/null; then -msg "build: i386, make, gcc" # ~ 30s -cleanup -CC=gcc CFLAGS='-Werror -Wall -Wextra -m32' make + msg "build: i386, make, gcc" # ~ 30s + cleanup + CC=gcc CFLAGS='-Werror -Wall -Wextra -m32' make -msg "build: gcc, force 32-bit compilation" -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl unset MBEDTLS_HAVE_ASM -scripts/config.pl unset MBEDTLS_AESNI_C -scripts/config.pl unset MBEDTLS_PADLOCK_C -CC=gcc CFLAGS='-Werror -Wall -Wextra -DMBEDTLS_HAVE_INT32' make + msg "build: gcc, force 32-bit compilation" + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl unset MBEDTLS_HAVE_ASM + scripts/config.pl unset MBEDTLS_AESNI_C + scripts/config.pl unset MBEDTLS_PADLOCK_C + CC=gcc CFLAGS='-Werror -Wall -Wextra -DMBEDTLS_HAVE_INT32' make -msg "build: gcc, force 64-bit compilation" -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl unset MBEDTLS_HAVE_ASM -scripts/config.pl unset MBEDTLS_AESNI_C -scripts/config.pl unset MBEDTLS_PADLOCK_C -CC=gcc CFLAGS='-Werror -Wall -Wextra -DMBEDTLS_HAVE_INT64' make + msg "build: gcc, force 64-bit compilation" + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl unset MBEDTLS_HAVE_ASM + scripts/config.pl unset MBEDTLS_AESNI_C + scripts/config.pl unset MBEDTLS_PADLOCK_C + CC=gcc CFLAGS='-Werror -Wall -Wextra -DMBEDTLS_HAVE_INT64' make -msg "test: gcc, force 64-bit compilation" -make test + msg "test: gcc, force 64-bit compilation" + make test -msg "build: gcc, force 64-bit compilation" -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl unset MBEDTLS_HAVE_ASM -scripts/config.pl unset MBEDTLS_AESNI_C -scripts/config.pl unset MBEDTLS_PADLOCK_C -CC=gcc CFLAGS='-Werror -Wall -Wextra -DMBEDTLS_HAVE_INT64' make + msg "build: gcc, force 64-bit compilation" + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl unset MBEDTLS_HAVE_ASM + scripts/config.pl unset MBEDTLS_AESNI_C + scripts/config.pl unset MBEDTLS_PADLOCK_C + CC=gcc CFLAGS='-Werror -Wall -Wextra -DMBEDTLS_HAVE_INT64' make fi # x86_64 msg "build: arm-none-eabi-gcc, make" # ~ 10s @@ -599,49 +599,49 @@ WINDOWS_BUILD=1 make clean # MemSan currently only available on Linux 64 bits if uname -a | grep 'Linux.*x86_64' >/dev/null; then -msg "build: MSan (clang)" # ~ 1 min 20s -cleanup -cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl unset MBEDTLS_AESNI_C # memsan doesn't grok asm -CC=clang cmake -D CMAKE_BUILD_TYPE:String=MemSan . -make + msg "build: MSan (clang)" # ~ 1 min 20s + cleanup + cp "$CONFIG_H" "$CONFIG_BAK" + scripts/config.pl unset MBEDTLS_AESNI_C # memsan doesn't grok asm + CC=clang cmake -D CMAKE_BUILD_TYPE:String=MemSan . + make -msg "test: main suites (MSan)" # ~ 10s -make test + msg "test: main suites (MSan)" # ~ 10s + make test -msg "test: ssl-opt.sh (MSan)" # ~ 1 min -tests/ssl-opt.sh + msg "test: ssl-opt.sh (MSan)" # ~ 1 min + tests/ssl-opt.sh -# Optional part(s) + # Optional part(s) -if [ "$MEMORY" -gt 0 ]; then - msg "test: compat.sh (MSan)" # ~ 6 min 20s - tests/compat.sh -fi + if [ "$MEMORY" -gt 0 ]; then + msg "test: compat.sh (MSan)" # ~ 6 min 20s + tests/compat.sh + fi else # no MemSan -msg "build: Release (clang)" -cleanup -CC=clang cmake -D CMAKE_BUILD_TYPE:String=Release . -make + msg "build: Release (clang)" + cleanup + CC=clang cmake -D CMAKE_BUILD_TYPE:String=Release . + make -msg "test: main suites valgrind (Release)" -make memcheck + msg "test: main suites valgrind (Release)" + make memcheck -# Optional part(s) -# Currently broken, programs don't seem to receive signals -# under valgrind on OS X + # Optional part(s) + # Currently broken, programs don't seem to receive signals + # under valgrind on OS X -if [ "$MEMORY" -gt 0 ]; then - msg "test: ssl-opt.sh --memcheck (Release)" - tests/ssl-opt.sh --memcheck -fi + if [ "$MEMORY" -gt 0 ]; then + msg "test: ssl-opt.sh --memcheck (Release)" + tests/ssl-opt.sh --memcheck + fi -if [ "$MEMORY" -gt 1 ]; then - msg "test: compat.sh --memcheck (Release)" - tests/compat.sh --memcheck -fi + if [ "$MEMORY" -gt 1 ]; then + msg "test: compat.sh --memcheck (Release)" + tests/compat.sh --memcheck + fi fi # MemSan From 709346aed8d9683c2a4fffa1110d1fa877eb4616 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 10 Dec 2017 23:43:39 +0100 Subject: [PATCH 2/6] all.sh: cleaned up usage output --- tests/scripts/all.sh | 115 ++++++++++++++++++++++++------------------- 1 file changed, 63 insertions(+), 52 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 4decbb1cb..0931012a4 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -56,22 +56,28 @@ fi usage() { - printf "Usage: $0\n" - printf " -h|--help\t\tPrint this help.\n" - printf " -m|--memory\t\tAdditional optional memory tests.\n" - printf " -f|--force\t\tForce the tests to overwrite any modified files.\n" - printf " -s|--seed\t\tInteger seed value to use for this test run.\n" - printf " -r|--release-test\t\tRun this script in release mode. This fixes the seed value to 1.\n" - printf " --no-yotta\t\tSkip yotta build\n" - printf " --out-of-source-dir=\t\tDirectory used for CMake out-of-source build tests." - printf " --openssl=\t\tPath to OpenSSL executable to use for most tests.\n" - printf " --openssl-legacy=\t\tPath to OpenSSL executable to use for legacy tests e.g. SSLv3.\n" - printf " --gnutls-cli=\t\tPath to GnuTLS client executable to use for most tests.\n" - printf " --gnutls-serv=\t\tPath to GnuTLS server executable to use for most tests.\n" - printf " --gnutls-legacy-cli=\t\tPath to GnuTLS client executable to use for legacy tests.\n" - printf " --gnutls-legacy-serv=\t\tPath to GnuTLS server executable to use for legacy tests.\n" - printf " --armc5-bin-dir=\t\tPath to the ARM Compiler 5 bin directory.\n" - printf " --armc6-bin-dir=\t\tPath to the ARM Compiler 6 bin directory.\n" + cat < Directory used for CMake out-of-source build tests. + -r|--release-test Run this script in release mode. This fixes the seed value to 1. + -s|--seed Integer seed value to use for this test run. + +Tool path options: + --armc5-bin-dir= ARM Compiler 5 bin directory. + --armc6-bin-dir= ARM Compiler 6 bin directory. + --gnutls-cli= GnuTLS client executable to use for most tests. + --gnutls-serv= GnuTLS server executable to use for most tests. + --gnutls-legacy-cli= GnuTLS client executable to use for legacy tests. + --gnutls-legacy-serv= GnuTLS server executable to use for legacy tests. + --openssl= OpenSSL executable to use for most tests. + --openssl-legacy= OpenSSL executable to use for legacy tests e.g. SSLv3. +EOF } # remove built files as well as the cmake cache/config @@ -127,42 +133,21 @@ check_tools() while [ $# -gt 0 ]; do case "$1" in - --memory|-m*) - MEMORY=${1#-m} + --armc5-bin-dir) + shift + ARMC5_BIN_DIR="$1" + ;; + --armc6-bin-dir) + shift + ARMC6_BIN_DIR="$1" ;; --force|-f) FORCE=1 ;; - --seed|-s) - shift - SEED="$1" - ;; - --release-test|-r) - RELEASE=1 - ;; - --no-yotta) - YOTTA=0 - ;; - --out-of-source-dir) - shift - OUT_OF_SOURCE_DIR="$1" - ;; - --openssl) - shift - OPENSSL="$1" - ;; - --openssl-legacy) - shift - OPENSSL_LEGACY="$1" - ;; --gnutls-cli) shift GNUTLS_CLI="$1" ;; - --gnutls-serv) - shift - GNUTLS_SERV="$1" - ;; --gnutls-legacy-cli) shift GNUTLS_LEGACY_CLI="$1" @@ -171,17 +156,43 @@ while [ $# -gt 0 ]; do shift GNUTLS_LEGACY_SERV="$1" ;; - --armc5-bin-dir) + --gnutls-serv) shift - ARMC5_BIN_DIR="$1" + GNUTLS_SERV="$1" ;; - --armc6-bin-dir) - shift - ARMC6_BIN_DIR="$1" - ;; - --help|-h|*) + --help|-h) usage - exit 1 + exit + ;; + --memory|-m) + MEMORY=1 + ;; + --no-yotta) + YOTTA=0 + ;; + --openssl) + shift + OPENSSL="$1" + ;; + --openssl-legacy) + shift + OPENSSL_LEGACY="$1" + ;; + --out-of-source-dir) + shift + OUT_OF_SOURCE_DIR="$1" + ;; + --release-test|-r) + RELEASE=1 + ;; + --seed|-s) + shift + SEED="$1" + ;; + *) + echo >&2 "Unknown option: $1" + echo >&2 "Run $0 --help for usage." + exit 120 ;; esac shift From 7c6521688ac2af9af845f329ede5fb8fffa7f22d Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 11 Dec 2017 00:01:40 +0100 Subject: [PATCH 3/6] all.sh: --keep-going mode Add --keep-going mode to all.sh. In this mode, if a test fails, keep running the subsequent tests. If a build fails, skip any tests of this build and move on to the next tests. Errors in infrastructure, such as git or cmake runs, remain fatal. Print an error summary at the end of the run, and return a nonzero code if there was any failure. In known terminal types, use color to highlight errors. On a fatal signal, interrupt the run and report the errors so far. --- tests/scripts/all.sh | 174 ++++++++++++++++++++++++++++++++----------- 1 file changed, 130 insertions(+), 44 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 0931012a4..923932f21 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -35,6 +35,7 @@ CONFIG_BAK="$CONFIG_H.bak" MEMORY=0 FORCE=0 +KEEP_GOING=0 RELEASE=0 YOTTA=1 @@ -62,6 +63,7 @@ Usage: $0 [OPTION]... General options: -f|--force Force the tests to overwrite any modified files. + -k|--keep-going Run all tests and report errors at the end. -m|--memory Additional optional memory tests. --no-yotta Skip yotta build. --out-of-source-dir= Directory used for CMake out-of-source build tests. @@ -83,7 +85,7 @@ EOF # remove built files as well as the cmake cache/config cleanup() { - make clean + command make clean find . -name yotta -prune -o -iname '*cmake*' -not -name CMakeLists.txt -exec rm -rf {} \+ rm -f include/Makefile include/mbedtls/Makefile programs/*/Makefile @@ -95,7 +97,21 @@ cleanup() fi } -trap cleanup INT TERM HUP +# Executed on exit. May be redefined depending on command line options. +final_report () { + : +} + +fatal_signal () { + cleanup + final_report $1 + trap - $1 + kill -$1 $$ +} + +trap 'fatal_signal HUP' HUP +trap 'fatal_signal INT' INT +trap 'fatal_signal TERM' TERM msg() { @@ -104,6 +120,7 @@ msg() echo "* $1 " printf "* "; date echo "******************************************************************" + current_section=$1 } armc6_build_test() @@ -164,6 +181,9 @@ while [ $# -gt 0 ]; do usage exit ;; + --keep-going|-k) + KEEP_GOING=1 + ;; --memory|-m) MEMORY=1 ;; @@ -221,7 +241,6 @@ else fi if ! git diff-files --quiet include/mbedtls/config.h; then - echo $? err_msg "Warning - the configuration file 'include/mbedtls/config.h' has been edited. " echo "You can either delete or preserve your work, or force the test by rerunning the" echo "script as: $0 --force" @@ -229,6 +248,71 @@ else fi fi +build_status=0 +if [ $KEEP_GOING -eq 1 ]; then + failure_summary= + failure_count=0 + start_red= + end_color= + if [ -t 1 ]; then + case "$TERM" in + *color*|cygwin|linux|rxvt*|screen|[Eex]term*) + start_red=$(printf '\033[31m') + end_color=$(printf '\033[0m') + ;; + esac + fi + record_status () { + if "$@"; then + last_status=0 + else + last_status=$? + text="$current_section: $* -> $last_status" + failure_summary="$failure_summary +$text" + failure_count=$((failure_count + 1)) + echo "${start_red}^^^^$text^^^^${end_color}" + fi + } + make () { + case "$*" in + *test|*check) + if [ $build_status -eq 0 ]; then + record_status command make "$@" + else + echo "(skipped because the build failed)" + fi + ;; + *) + record_status command make "$@" + build_status=$last_status + ;; + esac + } + final_report () { + if [ $failure_count -gt 0 ]; then + echo + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo "${start_red}FAILED: $failure_count${end_color}$failure_summary" + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + elif [ -z "${1-}" ]; then + echo "SUCCESS :)" + fi + if [ -n "${1-}" ]; then + echo "Killed by SIG$1." + fi + } +else + record_status () { + "$@" + } +fi +if_build_succeeded () { + if [ $build_status -eq 0 ]; then + record_status "$@" + fi +} + if [ $RELEASE -eq 1 ]; then # Fix the seed value to 1 to ensure that the tests are deterministic. SEED=1 @@ -306,7 +390,7 @@ if [ $YOTTA -ne 0 ]; then # on the path, and uses whatever version of armcc it finds there. msg "build: create and build yotta module" # ~ 30s cleanup - tests/scripts/yotta-build.sh + record_status tests/scripts/yotta-build.sh fi msg "build: cmake, gcc, ASan" # ~ 1 min 50s @@ -318,16 +402,16 @@ msg "test: main suites (inc. selftests) (ASan build)" # ~ 50s make test msg "test: ssl-opt.sh (ASan build)" # ~ 1 min -tests/ssl-opt.sh +if_build_succeeded tests/ssl-opt.sh msg "test/build: ref-configs (ASan build)" # ~ 6 min 20s -tests/scripts/test-ref-configs.pl +if_build_succeeded tests/scripts/test-ref-configs.pl msg "build: with ASan (rebuild after ref-configs)" # ~ 1 min make msg "test: compat.sh (ASan build)" # ~ 6 min -tests/compat.sh +if_build_succeeded tests/compat.sh msg "build: Default + SSLv3 (ASan build)" # ~ 6 min cleanup @@ -340,11 +424,11 @@ msg "test: SSLv3 - main suites (inc. selftests) (ASan build)" # ~ 50s make test msg "build: SSLv3 - compat.sh (ASan build)" # ~ 6 min -tests/compat.sh -m 'tls1 tls1_1 tls1_2 dtls1 dtls1_2' -OPENSSL_CMD="$OPENSSL_LEGACY" tests/compat.sh -m 'ssl3' +if_build_succeeded tests/compat.sh -m 'tls1 tls1_1 tls1_2 dtls1 dtls1_2' +if_build_succeeded env OPENSSL_CMD="$OPENSSL_LEGACY" tests/compat.sh -m 'ssl3' msg "build: SSLv3 - ssl-opt.sh (ASan build)" # ~ 6 min -tests/ssl-opt.sh +if_build_succeeded tests/ssl-opt.sh msg "build: Default + !MBEDTLS_SSL_RENEGOTIATION (ASan build)" # ~ 6 min cleanup @@ -357,7 +441,7 @@ msg "test: !MBEDTLS_SSL_RENEGOTIATION - main suites (inc. selftests) (ASan build make test msg "test: !MBEDTLS_SSL_RENEGOTIATION - ssl-opt.sh (ASan build)" # ~ 6 min -tests/ssl-opt.sh +if_build_succeeded tests/ssl-opt.sh msg "build: cmake, full config, clang, C99" # ~ 50s cleanup @@ -365,30 +449,30 @@ cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # too slow for tests CC=clang cmake -D CMAKE_BUILD_TYPE:String=Check -D ENABLE_TESTING=On . -CFLAGS='-Werror -Wall -Wextra -std=c99 -pedantic' make +make CFLAGS='-Werror -Wall -Wextra -std=c99 -pedantic' msg "test: main suites (full config)" # ~ 5s -CFLAGS='-Werror -Wall -Wextra' make test +make CFLAGS='-Werror -Wall -Wextra' test msg "test: ssl-opt.sh default (full config)" # ~ 1s -tests/ssl-opt.sh -f Default +if_build_succeeded tests/ssl-opt.sh -f Default msg "test: compat.sh RC4, DES & NULL (full config)" # ~ 2 min -OPENSSL_CMD="$OPENSSL_LEGACY" GNUTLS_CLI="$GNUTLS_LEGACY_CLI" GNUTLS_SERV="$GNUTLS_LEGACY_SERV" tests/compat.sh -e '3DES\|DES-CBC3' -f 'NULL\|DES\|RC4\|ARCFOUR' +if_build_succeeded env OPENSSL_CMD="$OPENSSL_LEGACY" GNUTLS_CLI="$GNUTLS_LEGACY_CLI" GNUTLS_SERV="$GNUTLS_LEGACY_SERV" tests/compat.sh -e '3DES\|DES-CBC3' -f 'NULL\|DES\|RC4\|ARCFOUR' msg "test/build: curves.pl (gcc)" # ~ 4 min cleanup cmake -D CMAKE_BUILD_TYPE:String=Debug . -tests/scripts/curves.pl +if_build_succeeded tests/scripts/curves.pl msg "test/build: key-exchanges (gcc)" # ~ 1 min cleanup cmake -D CMAKE_BUILD_TYPE:String=Check . -tests/scripts/key-exchanges.pl +if_build_succeeded tests/scripts/key-exchanges.pl msg "build: Unix make, -Os (gcc)" # ~ 30s cleanup -CC=gcc CFLAGS='-Werror -Wall -Wextra -Os' make +make CC=gcc CFLAGS='-Werror -Wall -Wextra -Os' # Full configuration build, without platform support, file IO and net sockets. # This should catch missing mbedtls_printf definitions, and by disabling file @@ -410,8 +494,8 @@ scripts/config.pl unset MBEDTLS_MEMORY_BUFFER_ALLOC_C scripts/config.pl unset MBEDTLS_FS_IO # Note, _DEFAULT_SOURCE needs to be defined for platforms using glibc version >2.19, # to re-enable platform integration features otherwise disabled in C99 builds -CC=gcc CFLAGS='-Werror -Wall -Wextra -std=c99 -pedantic -O0 -D_DEFAULT_SOURCE' make lib programs -CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' make test +make CC=gcc CFLAGS='-Werror -Wall -Wextra -std=c99 -pedantic -O0 -D_DEFAULT_SOURCE' lib programs +make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' test # catch compile bugs in _uninit functions msg "build: full config with NO_STD_FUNCTION, make, gcc" # ~ 30s @@ -420,21 +504,21 @@ cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl set MBEDTLS_PLATFORM_NO_STD_FUNCTIONS scripts/config.pl unset MBEDTLS_ENTROPY_NV_SEED -CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' make +make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' msg "build: full config except ssl_srv.c, make, gcc" # ~ 30s cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl unset MBEDTLS_SSL_SRV_C -CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' make +make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' msg "build: full config except ssl_cli.c, make, gcc" # ~ 30s cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl unset MBEDTLS_SSL_CLI_C -CC=gcc CFLAGS='-Werror -Wall -Werror -O0' make +make CC=gcc CFLAGS='-Werror -Wall -Werror -O0' # Note, C99 compliance can also be tested with the sockets support disabled, # as that requires a POSIX platform (which isn't the same as C99). @@ -444,7 +528,7 @@ cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl unset MBEDTLS_NET_C # getaddrinfo() undeclared, etc. scripts/config.pl set MBEDTLS_NO_PLATFORM_ENTROPY # uses syscall() on GNU/Linux -CC=gcc CFLAGS='-Werror -Wall -Wextra -O0 -std=c99 -pedantic' make lib +make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0 -std=c99 -pedantic' lib msg "build: default config except MFL extension (ASan build)" # ~ 30s cleanup @@ -454,7 +538,7 @@ CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make msg "test: ssl-opt.sh, MFL-related tests" -tests/ssl-opt.sh -f "Max fragment length" +if_build_succeeded tests/ssl-opt.sh -f "Max fragment length" msg "build: default config with MBEDTLS_TEST_NULL_ENTROPY (ASan build)" cleanup @@ -480,7 +564,7 @@ fi if uname -a | grep -F x86_64 >/dev/null; then msg "build: i386, make, gcc" # ~ 30s cleanup - CC=gcc CFLAGS='-Werror -Wall -Wextra -m32' make + make CC=gcc CFLAGS='-Werror -Wall -Wextra -m32' msg "build: gcc, force 32-bit compilation" cleanup @@ -488,7 +572,7 @@ if uname -a | grep -F x86_64 >/dev/null; then scripts/config.pl unset MBEDTLS_HAVE_ASM scripts/config.pl unset MBEDTLS_AESNI_C scripts/config.pl unset MBEDTLS_PADLOCK_C - CC=gcc CFLAGS='-Werror -Wall -Wextra -DMBEDTLS_HAVE_INT32' make + make CC=gcc CFLAGS='-Werror -Wall -Wextra -DMBEDTLS_HAVE_INT32' msg "build: gcc, force 64-bit compilation" cleanup @@ -496,7 +580,7 @@ if uname -a | grep -F x86_64 >/dev/null; then scripts/config.pl unset MBEDTLS_HAVE_ASM scripts/config.pl unset MBEDTLS_AESNI_C scripts/config.pl unset MBEDTLS_PADLOCK_C - CC=gcc CFLAGS='-Werror -Wall -Wextra -DMBEDTLS_HAVE_INT64' make + make CC=gcc CFLAGS='-Werror -Wall -Wextra -DMBEDTLS_HAVE_INT64' msg "test: gcc, force 64-bit compilation" make test @@ -507,7 +591,7 @@ if uname -a | grep -F x86_64 >/dev/null; then scripts/config.pl unset MBEDTLS_HAVE_ASM scripts/config.pl unset MBEDTLS_AESNI_C scripts/config.pl unset MBEDTLS_PADLOCK_C - CC=gcc CFLAGS='-Werror -Wall -Wextra -DMBEDTLS_HAVE_INT64' make + make CC=gcc CFLAGS='-Werror -Wall -Wextra -DMBEDTLS_HAVE_INT64' fi # x86_64 msg "build: arm-none-eabi-gcc, make" # ~ 10s @@ -525,7 +609,7 @@ scripts/config.pl unset MBEDTLS_THREADING_PTHREAD scripts/config.pl unset MBEDTLS_THREADING_C scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # execinfo.h scripts/config.pl unset MBEDTLS_MEMORY_BUFFER_ALLOC_C # calls exit -CC=arm-none-eabi-gcc AR=arm-none-eabi-ar LD=arm-none-eabi-ld CFLAGS='-Werror -Wall -Wextra' make lib +make CC=arm-none-eabi-gcc AR=arm-none-eabi-ar LD=arm-none-eabi-ld CFLAGS='-Werror -Wall -Wextra' lib msg "build: arm-none-eabi-gcc -DMBEDTLS_NO_UDBL_DIVISION, make" # ~ 10s cleanup @@ -543,7 +627,7 @@ scripts/config.pl unset MBEDTLS_THREADING_C scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # execinfo.h scripts/config.pl unset MBEDTLS_MEMORY_BUFFER_ALLOC_C # calls exit scripts/config.pl set MBEDTLS_NO_UDBL_DIVISION -CC=arm-none-eabi-gcc AR=arm-none-eabi-ar LD=arm-none-eabi-ld CFLAGS='-Werror -Wall -Wextra' make lib +make CC=arm-none-eabi-gcc AR=arm-none-eabi-ar LD=arm-none-eabi-ld CFLAGS='-Werror -Wall -Wextra' lib echo "Checking that software 64-bit division is not required" ! grep __aeabi_uldiv library/*.o @@ -567,7 +651,7 @@ scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # execinfo.h scripts/config.pl unset MBEDTLS_MEMORY_BUFFER_ALLOC_C # calls exit scripts/config.pl unset MBEDTLS_PLATFORM_TIME_ALT # depends on MBEDTLS_HAVE_TIME -CC="$ARMC5_CC" AR="$ARMC5_AR" WARNING_CFLAGS='--strict --c99' make lib +make CC="$ARMC5_CC" AR="$ARMC5_AR" WARNING_CFLAGS='--strict --c99' lib make clean # ARM Compiler 6 - Target ARMv7-A @@ -589,23 +673,23 @@ msg "build: allow SHA1 in certificates by default" cleanup cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl set MBEDTLS_TLS_DEFAULT_ALLOW_SHA1_IN_CERTIFICATES -CFLAGS='-Werror -Wall -Wextra' make +make CFLAGS='-Werror -Wall -Wextra' msg "test: allow SHA1 in certificates by default" make test -tests/ssl-opt.sh -f SHA-1 +if_build_succeeded tests/ssl-opt.sh -f SHA-1 msg "build: Windows cross build - mingw64, make (Link Library)" # ~ 30s cleanup -CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 make lib programs +make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 lib programs # note Make tests only builds the tests, but doesn't run them -CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror' WINDOWS_BUILD=1 make tests -WINDOWS_BUILD=1 make clean +make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror' WINDOWS_BUILD=1 tests +make WINDOWS_BUILD=1 clean msg "build: Windows cross build - mingw64, make (DLL)" # ~ 30s -CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 SHARED=1 make lib programs -CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 SHARED=1 make tests -WINDOWS_BUILD=1 make clean +make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 SHARED=1 lib programs +make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 SHARED=1 tests +make WINDOWS_BUILD=1 clean # MemSan currently only available on Linux 64 bits if uname -a | grep 'Linux.*x86_64' >/dev/null; then @@ -621,13 +705,13 @@ if uname -a | grep 'Linux.*x86_64' >/dev/null; then make test msg "test: ssl-opt.sh (MSan)" # ~ 1 min - tests/ssl-opt.sh + if_build_succeeded tests/ssl-opt.sh # Optional part(s) if [ "$MEMORY" -gt 0 ]; then msg "test: compat.sh (MSan)" # ~ 6 min 20s - tests/compat.sh + if_build_succeeded tests/compat.sh fi else # no MemSan @@ -646,12 +730,12 @@ else # no MemSan if [ "$MEMORY" -gt 0 ]; then msg "test: ssl-opt.sh --memcheck (Release)" - tests/ssl-opt.sh --memcheck + if_build_succeeded tests/ssl-opt.sh --memcheck fi if [ "$MEMORY" -gt 1 ]; then msg "test: compat.sh --memcheck (Release)" - tests/compat.sh --memcheck + if_build_succeeded tests/compat.sh --memcheck fi fi # MemSan @@ -671,3 +755,5 @@ rm -rf "$OUT_OF_SOURCE_DIR" msg "Done, cleaning up" cleanup + +final_report From 2a22a8041c4ec02774269a0944c4af1739f7984e Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 21 Dec 2017 15:19:00 +0100 Subject: [PATCH 4/6] all.sh: add --yotta to go with --no-yotta Add --yotta which is currently a no-op but may not remain so if we decide to make no-yotta the default in the future. --- tests/scripts/all.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 923932f21..ae991f13d 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -65,10 +65,11 @@ General options: -f|--force Force the tests to overwrite any modified files. -k|--keep-going Run all tests and report errors at the end. -m|--memory Additional optional memory tests. - --no-yotta Skip yotta build. + --no-yotta Skip yotta module build. --out-of-source-dir= Directory used for CMake out-of-source build tests. -r|--release-test Run this script in release mode. This fixes the seed value to 1. -s|--seed Integer seed value to use for this test run. + --yotta Build yotta module (on by default). Tool path options: --armc5-bin-dir= ARM Compiler 5 bin directory. @@ -209,6 +210,9 @@ while [ $# -gt 0 ]; do shift SEED="$1" ;; + --yotta) + YOTTA=1 + ;; *) echo >&2 "Unknown option: $1" echo >&2 "Run $0 --help for usage." @@ -226,7 +230,7 @@ if [ $FORCE -eq 1 ]; then cleanup else - if [ $YOTTA -eq 1 ] && [ -d yotta/module ]; then + if [ $YOTTA -ne 0 ] && [ -d yotta/module ]; then err_msg "Warning - there is an existing yotta module in the directory 'yotta/module'" echo "You can either delete your work and retry, or force the test to overwrite the" echo "test by rerunning the script as: $0 --force" From bca6ab9d38dccbcf024c43d3979c0feda8751281 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 19 Dec 2017 18:24:31 +0100 Subject: [PATCH 5/6] all.sh: new option --no-armcc With this option, don't run anything that requires armcc or yotta, so the script can run offline. --- tests/scripts/all.sh | 65 +++++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 25 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index ae991f13d..c9c790c80 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -37,6 +37,7 @@ MEMORY=0 FORCE=0 KEEP_GOING=0 RELEASE=0 +RUN_ARMCC=1 YOTTA=1 # Default commands, can be overriden by the environment @@ -65,6 +66,8 @@ General options: -f|--force Force the tests to overwrite any modified files. -k|--keep-going Run all tests and report errors at the end. -m|--memory Additional optional memory tests. + --armcc Run ARM Compiler builds (on by default). + --no-armcc Skip ARM Compiler builds. --no-yotta Skip yotta module build. --out-of-source-dir= Directory used for CMake out-of-source build tests. -r|--release-test Run this script in release mode. This fixes the seed value to 1. @@ -124,15 +127,17 @@ msg() current_section=$1 } -armc6_build_test() -{ - FLAGS="$1" +if [ $RUN_ARMCC -ne 0 ]; then + armc6_build_test() + { + FLAGS="$1" - msg "build: ARM Compiler 6 ($FLAGS), make" - ARM_TOOL_VARIANT="ult" CC="$ARMC6_CC" AR="$ARMC6_AR" CFLAGS="$FLAGS" \ - WARNING_CFLAGS='-xc -std=c99' make lib - make clean -} + msg "build: ARM Compiler 6 ($FLAGS), make" + ARM_TOOL_VARIANT="ult" CC="$ARMC6_CC" AR="$ARMC6_AR" CFLAGS="$FLAGS" \ + WARNING_CFLAGS='-xc -std=c99' make lib + make clean + } +fi err_msg() { @@ -151,6 +156,9 @@ check_tools() while [ $# -gt 0 ]; do case "$1" in + --armcc) + RUN_ARMCC=1 + ;; --armc5-bin-dir) shift ARMC5_BIN_DIR="$1" @@ -188,6 +196,9 @@ while [ $# -gt 0 ]; do --memory|-m) MEMORY=1 ;; + --no-armcc) + RUN_ARMCC=0 + ;; --no-yotta) YOTTA=0 ;; @@ -352,8 +363,10 @@ export GNUTLS_SERV="$GNUTLS_SERV" # Make sure the tools we need are available. check_tools "$OPENSSL" "$OPENSSL_LEGACY" "$GNUTLS_CLI" "$GNUTLS_SERV" \ "$GNUTLS_LEGACY_CLI" "$GNUTLS_LEGACY_SERV" "doxygen" "dot" \ - "arm-none-eabi-gcc" "$ARMC5_CC" "$ARMC5_AR" "$ARMC6_CC" "$ARMC6_AR" \ - "i686-w64-mingw32-gcc" + "arm-none-eabi-gcc" "i686-w64-mingw32-gcc" +if [ $RUN_ARMCC -ne 0 ]; then + check_tools "$ARMC5_CC" "$ARMC5_AR" "$ARMC6_CC" "$ARMC6_AR" +fi # # Test Suites to be executed @@ -389,9 +402,9 @@ msg "test: doxygen warnings" # ~ 3s cleanup tests/scripts/doxygen.sh -if [ $YOTTA -ne 0 ]; then - # Note - use of yotta is deprecated, and yotta also requires armcc to be - # on the path, and uses whatever version of armcc it finds there. +if [ $RUN_ARMCC -ne 0 ] && [ $YOTTA -ne 0 ]; then + # Note - use of yotta is deprecated, and yotta also requires armcc to be on the + # path, and uses whatever version of armcc it finds there. msg "build: create and build yotta module" # ~ 30s cleanup record_status tests/scripts/yotta-build.sh @@ -655,23 +668,25 @@ scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # execinfo.h scripts/config.pl unset MBEDTLS_MEMORY_BUFFER_ALLOC_C # calls exit scripts/config.pl unset MBEDTLS_PLATFORM_TIME_ALT # depends on MBEDTLS_HAVE_TIME -make CC="$ARMC5_CC" AR="$ARMC5_AR" WARNING_CFLAGS='--strict --c99' lib -make clean +if [ $RUN_ARMCC -ne 0 ]; then + make CC="$ARMC5_CC" AR="$ARMC5_AR" WARNING_CFLAGS='--strict --c99' lib + make clean -# ARM Compiler 6 - Target ARMv7-A -armc6_build_test "--target=arm-arm-none-eabi -march=armv7-a" + # ARM Compiler 6 - Target ARMv7-A + armc6_build_test "--target=arm-arm-none-eabi -march=armv7-a" -# ARM Compiler 6 - Target ARMv7-M -armc6_build_test "--target=arm-arm-none-eabi -march=armv7-m" + # ARM Compiler 6 - Target ARMv7-M + armc6_build_test "--target=arm-arm-none-eabi -march=armv7-m" -# ARM Compiler 6 - Target ARMv8-A - AArch32 -armc6_build_test "--target=arm-arm-none-eabi -march=armv8.2-a" + # ARM Compiler 6 - Target ARMv8-A - AArch32 + armc6_build_test "--target=arm-arm-none-eabi -march=armv8.2-a" -# ARM Compiler 6 - Target ARMv8-M -armc6_build_test "--target=arm-arm-none-eabi -march=armv8-m.main" + # ARM Compiler 6 - Target ARMv8-M + armc6_build_test "--target=arm-arm-none-eabi -march=armv8-m.main" -# ARM Compiler 6 - Target ARMv8-A - AArch64 -armc6_build_test "--target=aarch64-arm-none-eabi -march=armv8.2-a" + # ARM Compiler 6 - Target ARMv8-A - AArch64 + armc6_build_test "--target=aarch64-arm-none-eabi -march=armv8.2-a" +fi msg "build: allow SHA1 in certificates by default" cleanup From 192c72f7a12cac724150c0bebd0f224722c1ff63 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 21 Dec 2017 15:59:21 +0100 Subject: [PATCH 6/6] all.sh: add some documentation --- tests/scripts/all.sh | 90 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 83 insertions(+), 7 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index c9c790c80..2ea31dbc2 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -4,20 +4,78 @@ # # This file is part of mbed TLS (https://tls.mbed.org) # -# Copyright (c) 2014-2016, ARM Limited, All Rights Reserved -# +# Copyright (c) 2014-2017, ARM Limited, All Rights Reserved + + + +################################################################ +#### Documentation +################################################################ + # Purpose +# ------- # # To run all tests possible or available on the platform. # +# Notes for users +# --------------- +# # Warning: the test is destructive. It includes various build modes and # configurations, and can and will arbitrarily change the current CMake -# configuration. After this script has been run, the CMake cache will be lost -# and CMake will no longer be initialised. +# configuration. The following files must be committed into git: +# * include/mbedtls/config.h +# * Makefile, library/Makefile, programs/Makefile, tests/Makefile +# After running this script, the CMake cache will be lost and CMake +# will no longer be initialised. # -# The script assumes the presence of gcc and clang (recent enough for using -# ASan with gcc and MemSan with clang, or valgrind) are available, as well as -# cmake and a "good" find. +# The script assumes the presence of a number of tools: +# * Basic Unix tools (Windows users note: a Unix-style find must be before +# the Windows find in the PATH) +# * Perl +# * GNU Make +# * CMake +# * GCC and Clang (recent enough for using ASan with gcc and MemSan with clang, or valgrind) +# * arm-gcc and mingw-gcc +# * ArmCC 5 and ArmCC 6, unless invoked with --no-armcc +# * Yotta build dependencies, unless invoked with --no-yotta +# * OpenSSL and GnuTLS command line tools, recent enough for the +# interoperability tests. If they don't support SSLv3 then a legacy +# version of these tools must be present as well (search for LEGACY +# below). +# See the invocation of check_tools below for details. +# +# This script must be invoked from the toplevel directory of a git +# working copy of Mbed TLS. +# +# Note that the output is not saved. You may want to run +# script -c tests/scripts/all.sh +# or +# tests/scripts/all.sh >all.log 2>&1 +# +# Notes for maintainers +# --------------------- +# +# The tests are roughly in order from fastest to slowest. This doesn't +# have to be exact, but in general you should add slower tests towards +# the end and fast checks near the beginning. +# +# Sanity checks have the following form: +# 1. msg "short description of what is about to be done" +# 2. run sanity check (failure stops the script) +# +# Build or build-and-test steps have the following form: +# 1. msg "short description of what is about to be done" +# 2. cleanup +# 3. preparation (config.pl, cmake, ...) (failure stops the script) +# 4. make +# 5. Run tests if relevant. All tests must be prefixed with +# if_build_successful for the sake of --keep-going. + + + +################################################################ +#### Initialization and command line parsing +################################################################ # Abort on errors (and uninitialised variables) set -eu @@ -368,6 +426,12 @@ if [ $RUN_ARMCC -ne 0 ]; then check_tools "$ARMC5_CC" "$ARMC5_AR" "$ARMC6_CC" "$ARMC6_AR" fi + + +################################################################ +#### Basic checks +################################################################ + # # Test Suites to be executed # @@ -402,6 +466,12 @@ msg "test: doxygen warnings" # ~ 3s cleanup tests/scripts/doxygen.sh + + +################################################################ +#### Build and test many configurations and targets +################################################################ + if [ $RUN_ARMCC -ne 0 ] && [ $YOTTA -ne 0 ]; then # Note - use of yotta is deprecated, and yotta also requires armcc to be on the # path, and uses whatever version of armcc it finds there. @@ -772,6 +842,12 @@ make test cd "$MBEDTLS_ROOT_DIR" rm -rf "$OUT_OF_SOURCE_DIR" + + +################################################################ +#### Termination +################################################################ + msg "Done, cleaning up" cleanup