From a89aba54b7f0b545c5d6f96de2486b5b6d10552a Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sun, 18 Oct 2020 14:21:26 -0700 Subject: [PATCH] scripts: move litrunlen overflow test into a C test program This removes the ad-hoc "edge_case" group of tests, and it removes the dependency on Python. --- .travis.yml | 12 ++--- programs/test_litrunlen_overflow.c | 72 ++++++++++++++++++++++++++++++ scripts/run_tests.sh | 40 ----------------- 3 files changed, 78 insertions(+), 46 deletions(-) create mode 100644 programs/test_litrunlen_overflow.c diff --git a/.travis.yml b/.travis.yml index 832fee5..894bf58 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,14 +15,14 @@ matrix: script: - scripts/run_tests.sh native freestanding - - name: Checksum, static analysis, and edge case tests (Linux) + - name: Checksum and static analysis tests (Linux) os: linux dist: bionic before_install: - sudo apt-get install -y libz-dev gcc-multilib libz-dev:i386 - libc6-dev-i386 clang python3 + libc6-dev-i386 clang script: - - scripts/run_tests.sh checksum_benchmarks static_analysis edge_case + - scripts/run_tests.sh checksum_benchmarks static_analysis - name: Native tests (Linux aarch64) os: linux @@ -33,14 +33,14 @@ matrix: script: - scripts/run_tests.sh native freestanding - - name: Checksum, static analysis, and edge case tests (Linux aarch64) + - name: Checksum and static analysis tests (Linux aarch64) os: linux dist: bionic arch: arm64 before_install: - - sudo apt-get install -y libz-dev clang python3 + - sudo apt-get install -y libz-dev clang script: - - scripts/run_tests.sh checksum_benchmarks static_analysis edge_case + - scripts/run_tests.sh checksum_benchmarks static_analysis - name: gzip and cross-compile-for-Windows tests (Linux) os: linux diff --git a/programs/test_litrunlen_overflow.c b/programs/test_litrunlen_overflow.c new file mode 100644 index 0000000..7a9d5b1 --- /dev/null +++ b/programs/test_litrunlen_overflow.c @@ -0,0 +1,72 @@ +/* + * test_litrunlen_overflow.c + * + * Regression test for commit f2f0df727444 ("deflate_compress: fix corruption + * with long literal run"). Try to compress a file longer than 65535 bytes + * where no 2-byte sequence (3 would be sufficient) is repeated <= 32768 bytes + * apart, and the distribution of bytes remains constant throughout, and yet not + * all bytes are used so the data is still slightly compressible. There will be + * no matches in this data, but the compressor should still output a compressed + * block, and this block should contain more than 65535 consecutive literals, + * which triggered the bug. + * + * Note: on random data, this situation is extremely unlikely if the compressor + * uses all matches it finds, since random data will on average have a 3-byte + * match every (256**3)/32768 = 512 bytes. + */ + +#include "test_util.h" + +int +tmain(int argc, tchar *argv[]) +{ + const size_t data_size = 2 * 250 * 251; + u8 *orig_data, *compressed_data, *decompressed_data; + int i, stride, multiple, j = 0; + struct libdeflate_decompressor *d; + static const int levels[] = { 3, 6, 12 }; + + begin_program(argv); + + orig_data = xmalloc(data_size); + compressed_data = xmalloc(data_size); + decompressed_data = xmalloc(data_size); + + for (i = 0; i < 2; i++) { + for (stride = 1; stride < 251; stride++) { + for (multiple = 0; multiple < 251; multiple++) + orig_data[j++] = (stride * multiple) % 251; + } + } + ASSERT(j == data_size); + + d = libdeflate_alloc_decompressor(); + ASSERT(d != NULL); + + for (i = 0; i < ARRAY_LEN(levels); i++) { + struct libdeflate_compressor *c; + size_t csize; + enum libdeflate_result res; + + c = libdeflate_alloc_compressor(levels[i]); + ASSERT(c != NULL); + + csize = libdeflate_deflate_compress(c, orig_data, data_size, + compressed_data, data_size); + ASSERT(csize > 0 && csize < data_size); + + res = libdeflate_deflate_decompress(d, compressed_data, csize, + decompressed_data, + data_size, NULL); + ASSERT(res == LIBDEFLATE_SUCCESS); + ASSERT(memcmp(orig_data, decompressed_data, data_size) == 0); + + libdeflate_free_compressor(c); + } + + libdeflate_free_decompressor(d); + free(orig_data); + free(compressed_data); + free(decompressed_data); + return 0; +} diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 1b13664..7dea4da 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -118,13 +118,6 @@ have_ubsan() { fi } -have_python() { - if ! type -P python3 > /dev/null; then - log_skip "Python not found" - return 1 - fi -} - ############################################################################### native_build_and_test() { @@ -351,38 +344,6 @@ gzip_tests() { ############################################################################### -edge_case_tests() { - test_group_included edge_case || return 0 - - # Regression test for "deflate_compress: fix corruption with long - # literal run". Try to compress a file longer than 65535 bytes where no - # 2-byte sequence (3 would be sufficient) is repeated <= 32768 bytes - # apart, and the distribution of bytes remains constant throughout, and - # yet not all bytes are used so the data is still slightly compressible. - # There will be no matches in this data, but the compressor should still - # output a compressed block, and this block should contain more than - # 65535 consecutive literals, which triggered the bug. - # - # Note: on random data, this situation is extremely unlikely if the - # compressor uses all matches it finds, since random data will on - # average have a 3-byte match every (256**3)/32768 = 512 bytes. - if have_python; then - python3 > "$TMPFILE" << EOF -import sys -for i in range(2): - for stride in range(1,251): - b = bytes(stride*multiple % 251 for multiple in range(251)) - sys.stdout.buffer.write(b) -EOF - run_cmd make -j$NPROC benchmark - run_cmd ./benchmark -3 "$TMPFILE" - run_cmd ./benchmark -6 "$TMPFILE" - run_cmd ./benchmark -12 "$TMPFILE" - fi -} - -############################################################################### - log "Starting libdeflate tests" log " TESTGROUPS=(${TESTGROUPS[@]})" log " SMOKEDATA=$SMOKEDATA" @@ -395,7 +356,6 @@ android_tests windows_tests static_analysis_tests gzip_tests -edge_case_tests if (( TESTS_SKIPPED )); then log "No tests failed, but some tests were skipped. See above."