From 146d91c03f7d26dcbb60debe1fbd401270c7fc6b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 15 Oct 2016 13:09:45 -0700 Subject: [PATCH] Test script improvements --- .gitignore | 1 + tools/{android_test.sh => android_build.sh} | 44 ++-- tools/exec_tests.sh | 30 +++ tools/mips_build.sh | 9 + tools/mips_test.sh | 12 -- tools/run_tests.sh | 219 ++++++++++++++++++++ tools/windows_build.sh | 10 + tools/windows_test.sh | 10 - 8 files changed, 284 insertions(+), 51 deletions(-) rename tools/{android_test.sh => android_build.sh} (63%) create mode 100644 tools/exec_tests.sh create mode 100755 tools/mips_build.sh delete mode 100755 tools/mips_test.sh create mode 100755 tools/run_tests.sh create mode 100755 tools/windows_build.sh delete mode 100755 tools/windows_test.sh diff --git a/.gitignore b/.gitignore index 14046dd..e2ac930 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ /benchmark /gzip /gunzip +/run_tests.log /test_checksums tags cscope* diff --git a/tools/android_test.sh b/tools/android_build.sh similarity index 63% rename from tools/android_test.sh rename to tools/android_build.sh index 9fec24e..e58daba 100755 --- a/tools/android_test.sh +++ b/tools/android_build.sh @@ -1,33 +1,26 @@ #!/bin/bash -# -# android_test.sh -# -# Build and run the libdeflate benchmark program on Android -# -set -e +set -eu -FILE="$HOME/data/testdata" ARCH="arm32" COMPILER="gcc" NDKDIR="/opt/android-ndk" -DATADIR="/data/local/tmp" +DISABLE_NEON= usage() { cat << EOF -Usage: $0 [OPTION]... -- [BENCHMARK_ARG]... -Build and run the libdeflate benchmark program on Android +Usage: $0 [OPTION]... -- [BENCHMARK_PROGRAM_ARG]... +Build the libdeflate test programs for Android - --file=FILE Input data file (default: $FILE) --arch=ARCH Architecture: arm32|arm64 (default: $ARCH) --compiler=COMPILER Compiler: gcc|clang (default: $COMPILER) --ndkdir=NDKDIR Android NDK directory (default: $NDKDIR) - --datadir=DATADIR Data directory on Android (default: $DATADIR) + --disable-neon Disable NEON instructions EOF } if ! options=$(getopt -o '' \ - -l 'file:,arch:,compiler:,ndkdir:,datadir:,help' -- "$@"); then + -l 'arch:,compiler:,ndkdir:,disable-neon,help' -- "$@"); then usage exit 1 fi @@ -36,10 +29,6 @@ eval set -- "$options" while [ $# -gt 0 ]; do case "$1" in - --file) - FILE="$2" - shift - ;; --arch) ARCH="$2" shift @@ -52,9 +41,8 @@ while [ $# -gt 0 ]; do NDKDIR="$2" shift ;; - --datadir) - DATADIR="$2" - shift + --disable-neon) + DISABLE_NEON=1 ;; --help) usage @@ -78,7 +66,11 @@ case "$ARCH" in arm|arm32|aarch32) GCC_TOOLCHAIN="arm-linux-androideabi-4.9" CLANG_TARGET="armv7-none-linux-androideabi" - CFLAGS+=" -march=armv7-a -mfpu=neon -mfloat-abi=softfp" + if [ -n "$DISABLE_NEON" ]; then + CFLAGS+=" -march=armv6" + else + CFLAGS+=" -march=armv7-a -mfpu=neon -mfloat-abi=softfp" + fi CFLAGS+=" --sysroot=\"$NDKDIR/platforms/android-12/arch-arm\"" ;; arm64|aarch64) @@ -108,11 +100,5 @@ clang) exit 1 esac -make -j$(grep -c processor /proc/cpuinfo) benchmark CC="$CC" CFLAGS="$CFLAGS" -adb push benchmark "$DATADIR" - -FILENAME="$(basename "$FILE")" -if [ -z "$(adb shell "[ -e \"$DATADIR/$FILENAME\" ] && echo 1")" ]; then - adb push "$FILE" "$DATADIR/$FILENAME" -fi -adb shell "$DATADIR/benchmark" "$@" "$DATADIR/$FILENAME" +make -j$(grep -c processor /proc/cpuinfo) test_programs \ + CC="$CC" CFLAGS="$CFLAGS" diff --git a/tools/exec_tests.sh b/tools/exec_tests.sh new file mode 100644 index 0000000..6d06b6e --- /dev/null +++ b/tools/exec_tests.sh @@ -0,0 +1,30 @@ +# +# Helper script used by run_tests.sh, not intended to be run directly +# + +set -eu + +run_cmd() { + echo "$WRAPPER $@" + $WRAPPER "$@" > /dev/null +} + +run_cmd ./test_checksums + +for format in '' '-g' '-z'; do + for ref_impl in '' '-Y' '-Z'; do + run_cmd ./benchmark $format $ref_impl $SMOKEDATA + done +done +for level in 1 3 7 9; do + for ref_impl in '' '-Y'; do + run_cmd ./benchmark -$level $ref_impl $SMOKEDATA + done +done +for level in 1 3 7 9 12; do + for ref_impl in '' '-Z'; do + run_cmd ./benchmark -$level $ref_impl $SMOKEDATA + done +done + +echo "exec_tests finished successfully" # Needed for 'adb shell' diff --git a/tools/mips_build.sh b/tools/mips_build.sh new file mode 100755 index 0000000..3cbfd02 --- /dev/null +++ b/tools/mips_build.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -eu + +TOOLCHAIN_DIR=$HOME/src/ddwrt-toolchains/toolchain-mips_34kc_gcc-5.1.0_musl-1.1.9 + +make -j$(grep -c processor /proc/cpuinfo) test_programs \ + CC="$TOOLCHAIN_DIR/bin/mips-openwrt-linux-musl-gcc" \ + CFLAGS="-DNEED_PRINTF -Werror" diff --git a/tools/mips_test.sh b/tools/mips_test.sh deleted file mode 100755 index 9194304..0000000 --- a/tools/mips_test.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh - -set -e - -TOOLCHAIN_DIR=$HOME/src/ddwrt-toolchains/toolchain-mips_34kc_gcc-5.1.0_musl-1.1.9 - -make -j benchmark \ - CC="$TOOLCHAIN_DIR/bin/mips-openwrt-linux-musl-gcc" \ - CFLAGS="-DNEED_PRINTF" - -scp benchmark $HOME/data/test root@dd-wrt: -ssh root@dd-wrt ./benchmark "$@" test diff --git a/tools/run_tests.sh b/tools/run_tests.sh new file mode 100755 index 0000000..4ff4ab8 --- /dev/null +++ b/tools/run_tests.sh @@ -0,0 +1,219 @@ +#!/bin/bash +# +# Test script for libdeflate +# +# Usage: ./tools/run_tests.sh [TESTGROUP]... [-TESTGROUP]... +# +# By default all tests are run, but it is possible to explicitly include or +# exclude specific test groups. +# + +set -eu +cd "$(dirname "$0")/.." + +TESTGROUPS="$@" +if [ ${#TESTGROUPS} -eq 0 ]; then + TESTGROUPS=(all) +fi + +SMOKEDATA="${SMOKEDATA:=$HOME/data/smokedata}" +if [ ! -e "$SMOKEDATA" ]; then + echo "SMOKEDATA (value: $SMOKEDATA) does not exist. Set the" \ + "environmental variable SMOKEDATA to a file to use in" \ + "compression/decompression tests." 1>&2 + exit 1 +fi + +NDKDIR="${NDKDIR:=/opt/android-ndk}" + +FILES=("$SMOKEDATA" ./tools/exec_tests.sh benchmark test_checksums) +EXEC_TESTS_CMD="WRAPPER= SMOKEDATA=\"$(basename $SMOKEDATA)\" sh exec_tests.sh" +NPROC=$(grep -c processor /proc/cpuinfo) + +############################################################################### + +rm -f run_tests.log +exec > >(tee -ia run_tests.log) +exec 2> >(tee -ia run_tests.log >&2) + +TESTS_SKIPPED= +log_skip() { + log "[WARNING, TEST SKIPPED]: $@" + TESTS_SKIPPED=1 +} + +log() { + echo "[$(date)] $@" +} + +run_cmd() { + log "$@" + "$@" > /dev/null +} + +test_group_enabled() { + local status=1 group + for group in "${TESTGROUPS[@]}"; do + if [ $group = $1 ]; then + status=0 # explicitly included + break + fi + if [ $group = -$1 ]; then + status=1 # explicitly excluded + break + fi + if [ $group = all ]; then # implicitly included + status=0 + fi + done + if [ $status -eq 0 ]; then + log "Starting test group: $1" + fi + return $status +} + +############################################################################### + +native_build_and_test() { + make "$@" -j$NPROC all test_programs > /dev/null + WRAPPER="$WRAPPER" SMOKEDATA="$SMOKEDATA" sh ./tools/exec_tests.sh \ + > /dev/null +} + +native_tests() { + test_group_enabled native || return 0 + local compiler cflags compilers=(gcc clang) + shopt -s nullglob + compilers+=(/usr/bin/gcc-[0-9]*) + compilers+=(/opt/gcc*/bin/gcc) + shopt -u nullglob + for compiler in ${compilers[@]}; do + for cflags in "" "-march=native" "-m32"; do + if [ "$compiler" = "/usr/bin/gcc-4.8" -a \ + "$cflags" = "-m32" ]; then + continue + fi + log "Running tests with CC=$compiler," \ + "CFLAGS=$cflags" + WRAPPER= native_build_and_test \ + CC=$compiler CFLAGS="$cflags -Werror" + done + done + + log "Running tests with Valgrind" + WRAPPER="valgrind --error-exitcode=100 --quiet" native_build_and_test + + log "Running tests with undefined behavior sanitizer" + WRAPPER= native_build_and_test CC=clang CFLAGS="-fsanitize=undefined" +} + +############################################################################### + +android_build() { + run_cmd ./tools/android_build.sh --ndkdir="$NDKDIR" "$@" +} + +android_build_and_test() { + android_build "$@" + run_cmd adb push "${FILES[@]}" /data/local/tmp/ + + # Note: adb shell always returns 0, even if the shell command fails... + log "adb shell \"cd /data/local/tmp && $EXEC_TESTS_CMD\"" + adb shell "cd /data/local/tmp && $EXEC_TESTS_CMD" | \ + grep -q "exec_tests finished successfully" +} + +android_tests() { + local compiler + + test_group_enabled android || return 0 + if [ ! -e $NDKDIR ]; then + log_skip "Android NDK was not found in NDKDIR=$NDKDIR!" \ + "If you want to run the Android tests, set the" \ + "environmental variable NDKDIR to the location of" \ + "your Android NDK installation" + return 0 + fi + + if ! type -P adb > /dev/null; then + log_skip "adb (android-tools) is not installed" + return 0 + fi + + if (( $(adb devices | wc -l) < 3)); then + log_skip "No Android device is currently attached" + return 0; + fi + + for compiler in gcc clang; do + android_build_and_test --arch=arm --compiler=$compiler + + android_build_and_test --arch=arm --compiler=$compiler \ + --disable-neon + + # arm64: currently compiled but not run + android_build --arch=arm64 --compiler=$compiler + done +} + +############################################################################### + +mips_tests() { + test_group_enabled mips || return 0 + if ! ping -c 1 dd-wrt > /dev/null; then + log_skip "Can't run MIPS tests: dd-wrt system not available" + return 0 + fi + run_cmd ./tools/mips_build.sh + run_cmd scp "${FILES[@]}" root@dd-wrt: + run_cmd ssh root@dd-wrt "$EXEC_TESTS_CMD" +} + +############################################################################### + +windows_tests() { + local arch + + test_group_enabled windows || return 0 + + # Windows: currently compiled but not run + for arch in i686 x86_64; do + local compiler=${arch}-w64-mingw32-gcc + if ! type -P $compiler > /dev/null; then + log_skip "$compiler not found" + continue + fi + run_cmd make CC=$compiler CFLAGS=-Werror -j$NPROC \ + all test_programs + done +} + +############################################################################### + +static_analysis_tests() { + test_group_enabled static_analysis || return 0 + if ! type -P scan-build > /dev/null; then + log_skip "clang static analyzer (scan-build) not found" + return 0 + fi + run_cmd scan-build --status-bugs make -j$NPROC +} + +############################################################################### + +log "Starting libdeflate tests" +log " TESTGROUPS=(${TESTGROUPS[@]})" +log " SMOKEDATA=$SMOKEDATA" +log " NDKDIR=$NDKDIR" + +native_tests +android_tests +mips_tests +windows_tests +static_analysis_tests + +if [ -n "$TESTS_SKIPPED" ]; then + log "No tests failed, but some tests were skipped. See above." +else + log "All tests passed!" +fi diff --git a/tools/windows_build.sh b/tools/windows_build.sh new file mode 100755 index 0000000..a91ed65 --- /dev/null +++ b/tools/windows_build.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -eu + +make -j CC=i686-w64-mingw32-gcc all test_programs +cp -vf *.exe /j/exe/ +make -j CC=x86_64-w64-mingw32-gcc all test_programs +cp -vf *.exe /j/exe64/ + +sudo $HOME/bin/sudo/restart-smbd diff --git a/tools/windows_test.sh b/tools/windows_test.sh deleted file mode 100755 index 595872a..0000000 --- a/tools/windows_test.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -set -e - -make -j CC=i686-w64-mingw32-gcc -cp -vf *.exe /j/exe/ -make -j CC=x86_64-w64-mingw32-gcc -cp -vf *.exe /j/exe64/ - -sudo $HOME/bin/sudo/restart-smbd