From fb1de37c0ec2f739ac6e9b320711fc7acc48d70d Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 30 Jan 2016 15:13:29 -0600 Subject: [PATCH] Support for Microsoft C / Visual Studio --- Makefile | 29 +++++---- Makefile.msc | 48 ++++++++++++++ README.md | 15 ++--- libdeflate.h | 39 +++++++---- src/bt_matchfinder.h | 6 +- src/compiler-gcc.h | 4 +- src/compiler-msc.h | 56 ++++++++++++++++ src/compiler.h | 4 +- src/decompress_impl.h | 2 + src/deflate_compress.c | 10 +-- src/deflate_decompress.c | 97 ++++++++++------------------ src/gzip_compress.c | 4 +- src/gzip_decompress.c | 4 +- src/hc_matchfinder.h | 6 +- src/zlib_compress.c | 4 +- src/zlib_decompress.c | 4 +- tools/benchmark.c | 121 ++++++++++------------------------- tools/gzip.c | 103 ++--------------------------- tools/msc_test.bat | 23 +++++++ tools/util.h | 135 +++++++++++++++++++++++++++++++++++++++ tools/wgetopt.c | 62 ++++++++++++++++++ tools/wgetopt.h | 4 ++ tools/windows_test.sh | 4 +- 23 files changed, 478 insertions(+), 306 deletions(-) create mode 100644 Makefile.msc create mode 100644 src/compiler-msc.h create mode 100755 tools/msc_test.bat create mode 100644 tools/util.h create mode 100644 tools/wgetopt.c create mode 100644 tools/wgetopt.h diff --git a/Makefile b/Makefile index 7aee03a..bafd9d0 100644 --- a/Makefile +++ b/Makefile @@ -53,19 +53,19 @@ ifneq ($(findstring -mingw,$(CC)),) AR := $(patsubst %-gcc,%-ar,$(CC)) endif WINDOWS := yes - LIB_SUFFIX := .a + LIB_SUFFIX := .lib SHLIB_SUFFIX := .dll PROG_SUFFIX := .exe - PROG_CFLAGS := -static - GZIP_CFLAGS := -municode + PROG_SRC := tools/wgetopt.c + PROG_CFLAGS := -static -municode SHLIB_IS_PIC := no else WINDOWS := no LIB_SUFFIX := .a SHLIB_SUFFIX := .so PROG_SUFFIX := + PROG_SRC := PROG_CFLAGS := - GZIP_CFLAGS := SHLIB_IS_PIC := yes endif @@ -102,6 +102,9 @@ ifneq ($(RUNTIME_CPU_DETECTION),yes) override CFLAGS += -DRUNTIME_CPU_DETECTION=0 endif +PROG_CFLAGS += $(CFLAGS) +PROG_SRC += libdeflate$(LIB_SUFFIX) + SRC := src/aligned_malloc.c ifeq ($(SUPPORT_COMPRESSION),yes) SRC += src/deflate_compress.c @@ -151,15 +154,15 @@ libdeflate$(SHLIB_SUFFIX):$(SHLIB_OBJ) libdeflate$(LIB_SUFFIX):$(OBJ) $(AR) cr $@ $+ -benchmark$(PROG_SUFFIX):tools/benchmark.c libdeflate$(LIB_SUFFIX) - $(CC) -o $@ $(CFLAGS) -L. $+ libdeflate$(LIB_SUFFIX) -lz +benchmark$(PROG_SUFFIX):tools/benchmark.c $(PROG_SRC) + $(CC) -o $@ $(PROG_CFLAGS) $+ -lz -gzip$(PROG_SUFFIX):tools/gzip.c libdeflate$(LIB_SUFFIX) - $(CC) -o $@ $(CFLAGS) $(GZIP_CFLAGS) $(PROG_CFLAGS) -L. $+ libdeflate$(LIB_SUFFIX) +gzip$(PROG_SUFFIX):tools/gzip.c $(PROG_SRC) + $(CC) -o $@ $(PROG_CFLAGS) $+ ifeq ($(WINDOWS),yes) -gunzip$(PROG_SUFFIX):tools/gzip.c libdeflate$(LIB_SUFFIX) - $(CC) -o $@ $(CFLAGS) $(GZIP_CFLAGS) $(PROG_CFLAGS) -L. $+ libdeflate$(LIB_SUFFIX) +gunzip$(PROG_SUFFIX):tools/gzip.c $(PROG_SRC) + $(CC) -o $@ $(PROG_CFLAGS) $+ else gunzip$(PROG_SUFFIX):gzip$(PROG_SUFFIX) ln -f gzip$(PROG_SUFFIX) $@ @@ -187,10 +190,8 @@ endif all:$(TARGETS) clean: - rm -f libdeflate.a libdeflate.so libdeflate.dll src/*.o \ - benchmark benchmark.exe \ - gzip gzip.exe \ - gunzip gunzip.exe + rm -f libdeflate.a libdeflate.so libdeflate.dll src/*.o src/*.obj \ + benchmark gzip gunzip *.exe *.lib *.obj *.exp .PHONY: all clean diff --git a/Makefile.msc b/Makefile.msc new file mode 100644 index 0000000..c1b7091 --- /dev/null +++ b/Makefile.msc @@ -0,0 +1,48 @@ +# +# Makefile for the Microsoft toolchain. It builds a static library and a shared +# library (DLL) and its corresponding import library. +# +# Usage: +# nmake /f Makefile.msc +# + +CC = cl +LD = link +AR = lib +CFLAGS = /MD /O2 -I. + +STATICLIB = libdeflatestatic.lib +SHAREDLIB = libdeflate.dll +IMPLIB = libdeflate.lib + +OBJS = src/adler32.obj \ + src/aligned_malloc.obj \ + src/crc32.obj \ + src/deflate_compress.obj \ + src/deflate_decompress.obj \ + src/gzip_compress.obj \ + src/gzip_decompress.obj \ + src/x86_cpu_features.obj \ + src/zlib_compress.obj \ + src/zlib_decompress.obj + +all: $(STATICLIB) $(SHAREDLIB) $(IMPLIB) + +$(STATICLIB): $(OBJS) + $(AR) $(ARFLAGS) -out:$@ $(OBJS) + +$(IMPLIB): $(SHAREDLIB) + +$(SHAREDLIB): $(OBJS) + $(LD) $(LDFLAGS) -out:$@ -dll -implib:$(IMPLIB) $(OBJS) + +.c.obj: + $(CC) -c /Fo:$@ $(CFLAGS) $< + +clean: + -del *.lib + -del *.dll + -del *.exe + -del *.exp + -del *.obj + -del src\*.obj diff --git a/README.md b/README.md index 7404d54..8252bb9 100644 --- a/README.md +++ b/README.md @@ -21,19 +21,18 @@ on it, and anyone is free to use it for any reason. Building ======== -Currently, the build system is very bare-bones. On a UNIX-like system, just run -`make`. You need GNU Make and either GCC or Clang. - +On a UNIX-like system, just run `make`. You need GNU Make and either GCC or +Clang. There is no `make install` yet; just copy the file(s) to where you want. There are various options which can be set on the `make` command line; see the Makefile for details. As an example, you can run `make SUPPORT_COMPRESSION=no` to build a decompression-only library. -There is no `make install` yet; just copy the file(s) to where you want. - -It's possible to build Windows binaries using MinGW with a command like this: - - $ make CC=x86_64-w64-mingw32-gcc BUILD_PROGRAMS=yes +On Windows, a separate Makefile, `Makefile.msc`, is provided for the tools that +come with Visual Studio. However, using MinGW (GCC) instead is recommended +because it is a superior compiler that produces better-performing binaries. You +can build Windows binaries using MinGW with a command like this: + $ make CC=x86_64-w64-mingw32-gcc API === diff --git a/libdeflate.h b/libdeflate.h index be57f7b..476dffb 100644 --- a/libdeflate.h +++ b/libdeflate.h @@ -11,6 +11,19 @@ extern "C" { #include +/* Microsoft C / Visual Studio garbage. If you want to link to the DLL version + * of libdeflate, then #define LIBDEFLATE_DLL. */ +#ifdef _MSC_VER +# ifdef BUILDING_LIBDEFLATE +# define LIBDEFLATEAPI __declspec(dllexport) +# elif defined(LIBDEFLATE_DLL) +# define LIBDEFLATEAPI __declspec(dllimport) +# endif +#endif +#ifndef LIBDEFLATEAPI +# define LIBDEFLATEAPI +#endif + /* ========================================================================== */ /* Compression */ /* ========================================================================== */ @@ -28,7 +41,7 @@ struct deflate_compressor; * to 32768, the largest size permissible in the DEFLATE format. It cannot be * changed at runtime. */ -extern struct deflate_compressor * +LIBDEFLATEAPI struct deflate_compressor * deflate_alloc_compressor(unsigned int compression_level); /* @@ -38,7 +51,7 @@ deflate_alloc_compressor(unsigned int compression_level); * The return value is the compressed size in bytes, or 0 if the data could not * be compressed to 'out_nbytes_avail' bytes or fewer. */ -extern size_t +LIBDEFLATEAPI size_t deflate_compress(struct deflate_compressor *compressor, const void *in, size_t in_nbytes, void *out, size_t out_nbytes_avail); @@ -62,13 +75,13 @@ deflate_compress(struct deflate_compressor *compressor, * and store the data uncompressed if deflate_compress() returns 0, indicating * that the compressed data did not fit into the provided output buffer. */ -extern size_t +LIBDEFLATEAPI size_t deflate_compress_bound(struct deflate_compressor *compressor, size_t in_nbytes); /* * Like deflate_compress(), but stores the data in the zlib wrapper format. */ -extern size_t +LIBDEFLATEAPI size_t zlib_compress(struct deflate_compressor *compressor, const void *in, size_t in_nbytes, void *out, size_t out_nbytes_avail); @@ -77,13 +90,13 @@ zlib_compress(struct deflate_compressor *compressor, * Like deflate_compress_bound(), but assumes the data will be compressed with * zlib_compress() rather than with deflate_compress(). */ -extern size_t +LIBDEFLATEAPI size_t zlib_compress_bound(struct deflate_compressor *compressor, size_t in_nbytes); /* * Like deflate_compress(), but stores the data in the gzip wrapper format. */ -extern size_t +LIBDEFLATEAPI size_t gzip_compress(struct deflate_compressor *compressor, const void *in, size_t in_nbytes, void *out, size_t out_nbytes_avail); @@ -92,7 +105,7 @@ gzip_compress(struct deflate_compressor *compressor, * Like deflate_compress_bound(), but assumes the data will be compressed with * gzip_compress() rather than with deflate_compress(). */ -extern size_t +LIBDEFLATEAPI size_t gzip_compress_bound(struct deflate_compressor *compressor, size_t in_nbytes); /* @@ -100,7 +113,7 @@ gzip_compress_bound(struct deflate_compressor *compressor, size_t in_nbytes); * deflate_alloc_compressor(). If a NULL pointer is passed in, no action is * taken. */ -extern void +LIBDEFLATEAPI void deflate_free_compressor(struct deflate_compressor *compressor); /* ========================================================================== */ @@ -120,7 +133,7 @@ struct deflate_decompressor; * DEFLATE, zlib, or gzip); however, the appropriate decompression function must * be called. */ -extern struct deflate_decompressor * +LIBDEFLATEAPI struct deflate_decompressor * deflate_alloc_decompressor(void); /* @@ -172,7 +185,7 @@ enum decompress_result { * but no other problems were encountered, or another nonzero result code if * decompression failed for another reason. */ -extern enum decompress_result +LIBDEFLATEAPI enum decompress_result deflate_decompress(struct deflate_decompressor *decompressor, const void *in, size_t in_nbytes, void *out, size_t out_nbytes_avail, @@ -182,7 +195,7 @@ deflate_decompress(struct deflate_decompressor *decompressor, * Like deflate_decompress(), but assumes the zlib wrapper format instead of raw * DEFLATE. */ -extern enum decompress_result +LIBDEFLATEAPI enum decompress_result zlib_decompress(struct deflate_decompressor *decompressor, const void *in, size_t in_nbytes, void *out, size_t out_nbytes_avail, @@ -192,7 +205,7 @@ zlib_decompress(struct deflate_decompressor *decompressor, * Like deflate_decompress(), but assumes the gzip wrapper format instead of raw * DEFLATE. */ -extern enum decompress_result +LIBDEFLATEAPI enum decompress_result gzip_decompress(struct deflate_decompressor *decompressor, const void *in, size_t in_nbytes, void *out, size_t out_nbytes_avail, @@ -203,7 +216,7 @@ gzip_decompress(struct deflate_decompressor *decompressor, * with deflate_alloc_decompressor(). If a NULL pointer is passed in, no action * is taken. */ -extern void +LIBDEFLATEAPI void deflate_free_decompressor(struct deflate_decompressor *decompressor); diff --git a/src/bt_matchfinder.h b/src/bt_matchfinder.h index 62ca635..6a224c3 100644 --- a/src/bt_matchfinder.h +++ b/src/bt_matchfinder.h @@ -126,7 +126,7 @@ bt_matchfinder_advance_one_byte(struct bt_matchfinder * const restrict mf, const u32 max_len, const u32 nice_len, const u32 max_search_depth, - u32 next_hashes[const restrict static 2], + u32 next_hashes[restrict 2], u32 * const restrict best_len_ret, struct lz_match * restrict lz_matchptr, const bool record_matches) @@ -294,7 +294,7 @@ bt_matchfinder_get_matches(struct bt_matchfinder *mf, u32 max_len, u32 nice_len, u32 max_search_depth, - u32 next_hashes[static 2], + u32 next_hashes[2], u32 *best_len_ret, struct lz_match *lz_matchptr) { @@ -323,7 +323,7 @@ bt_matchfinder_skip_position(struct bt_matchfinder *mf, u32 max_len, u32 nice_len, u32 max_search_depth, - u32 next_hashes[static 2]) + u32 next_hashes[2]) { u32 best_len; bt_matchfinder_advance_one_byte(mf, diff --git a/src/compiler-gcc.h b/src/compiler-gcc.h index 2258a8c..3383d56 100644 --- a/src/compiler-gcc.h +++ b/src/compiler-gcc.h @@ -2,9 +2,7 @@ * compiler-gcc.h - definitions for the GNU C compiler (and for clang) */ -#pragma once - -#ifdef __WIN32__ +#ifdef _WIN32 # define LIBEXPORT __declspec(dllexport) #else # define LIBEXPORT __attribute__((visibility("default"))) diff --git a/src/compiler-msc.h b/src/compiler-msc.h new file mode 100644 index 0000000..1035ef5 --- /dev/null +++ b/src/compiler-msc.h @@ -0,0 +1,56 @@ +/* + * compiler-msc.h - definitions for the Microsoft C Compiler + */ + +#include +#include + +#define BUILDING_LIBDEFLATE + +/* MSC __restrict has nonstandard behavior. Don't use it. */ +#define restrict + +#define LIBEXPORT __declspec(dllexport) +#define forceinline __forceinline + +/* Assume a little endian architecture with fast unaligned access. */ +#define CPU_IS_LITTLE_ENDIAN() 1 +#define UNALIGNED_ACCESS_IS_FAST 1 + +#define compiler_bswap16 _byteswap_ushort +#define compiler_bswap32 _byteswap_ulong +#define compiler_bswap64 _byteswap_uint64 + +static forceinline unsigned +compiler_fls32(uint32_t n) +{ + _BitScanReverse(&n, n); + return n; +} +#define compiler_fls32 compiler_fls32 + +static forceinline unsigned +compiler_ffs32(uint32_t n) +{ + _BitScanForward(&n, n); + return n; +} +#define compiler_ffs32 compiler_ffs32 + +#ifdef _M_X64 +static forceinline unsigned +compiler_fls64(uint64_t n) +{ + _BitScanReverse64(&n, n); + return n; +} +#define compiler_fls64 compiler_fls64 + +static forceinline unsigned +compiler_ffs64(uint64_t n) +{ + _BitScanForward64(&n, n); + return n; +} +#define compiler_ffs64 compiler_ffs64 +#endif /* _M_X64 */ diff --git a/src/compiler.h b/src/compiler.h index 8507571..cb88cdf 100644 --- a/src/compiler.h +++ b/src/compiler.h @@ -8,8 +8,10 @@ * define as many of them as possible. */ #ifdef __GNUC__ # include "compiler-gcc.h" +#elif defined(_MSC_VER) +# include "compiler-msc.h" #else -# warning "Unrecognized compiler. Please add a header file for your compiler. Compilation will proceed, but performance may suffer!" +# pragma message("Unrecognized compiler. Please add a header file for your compiler. Compilation will proceed, but performance may suffer!") #endif /* forceinline - force a function to be inlined */ diff --git a/src/decompress_impl.h b/src/decompress_impl.h index 33d5973..d703fa3 100644 --- a/src/decompress_impl.h +++ b/src/decompress_impl.h @@ -26,6 +26,8 @@ FUNCNAME(struct deflate_decompressor * restrict d, u16 nlen; unsigned num_litlen_syms; unsigned num_offset_syms; + u16 tmp16; + u32 tmp32; next_block: /* Starting to read the next block. */ diff --git a/src/deflate_compress.c b/src/deflate_compress.c index 04ad4d7..a271fed 100644 --- a/src/deflate_compress.c +++ b/src/deflate_compress.c @@ -11,13 +11,13 @@ #include #include -#include "libdeflate.h" - #include "aligned_malloc.h" #include "deflate_compress.h" #include "deflate_constants.h" #include "unaligned.h" +#include "libdeflate.h" + /* * Note: when compiling this file, SUPPORT_NEAR_OPTIMAL_PARSING should be * defined to either 0 or 1. When defined to 1, the near-optimal parsing @@ -1565,7 +1565,7 @@ deflate_compress_greedy(struct deflate_compressor * restrict c, struct deflate_sequence *next_seq = c->sequences; u32 litrunlen = 0; u32 items_remaining = MAX_ITEMS_PER_BLOCK; - u32 next_hashes[2] = {}; + u32 next_hashes[2] = {0, 0}; deflate_init_output(&os, out, out_nbytes_avail); deflate_reset_symbol_frequencies(c); @@ -1662,7 +1662,7 @@ deflate_compress_lazy(struct deflate_compressor * restrict c, struct deflate_sequence *next_seq = c->sequences; u32 litrunlen = 0; u32 items_remaining = MAX_ITEMS_PER_BLOCK; - u32 next_hashes[2] = {}; + u32 next_hashes[2] = {0, 0}; deflate_init_output(&os, out, out_nbytes_avail); deflate_reset_symbol_frequencies(c); @@ -2171,7 +2171,7 @@ deflate_compress_near_optimal(struct deflate_compressor * restrict c, struct lz_match *cache_end; const u8 *in_block_begin; const u8 *in_block_end; - u32 next_hashes[2] = {}; + u32 next_hashes[2] = {0, 0}; deflate_init_output(&os, out, out_nbytes_avail); deflate_reset_symbol_frequencies(c); diff --git a/src/deflate_decompress.c b/src/deflate_decompress.c index 5dee7b5..8ea1415 100644 --- a/src/deflate_decompress.c +++ b/src/deflate_decompress.c @@ -9,10 +9,10 @@ * * --------------------------------------------------------------------------- * - * This is a highly optimized DEFLATE decompressor. On x86_64 it decompresses - * data in about 52% of the time of zlib (48% if BMI2 instructions are - * available). On other architectures it should still be significantly faster - * than zlib, but the difference may be smaller. + * This is a highly optimized DEFLATE decompressor. When compiled with gcc on + * x86_64, it decompresses data in about 52% of the time of zlib (48% if BMI2 + * instructions are available). On other architectures it should still be + * significantly faster than zlib, but the difference may be smaller. * * Why this is faster than zlib's implementation: * @@ -30,19 +30,19 @@ #include #include -#include "libdeflate.h" - #include "deflate_constants.h" #include "unaligned.h" #include "x86_cpu_features.h" +#include "libdeflate.h" + /* By default, if the expression passed to SAFETY_CHECK() evaluates to false, * then deflate_decompress() immediately returns DECOMPRESS_BAD_DATA as the * compressed data is invalid. But if unsafe decompression is enabled, then the * value of the expression is ignored, allowing the compiler to optimize out * some code. */ #if UNSAFE_DECOMPRESSION -# warning "UNSAFE DECOMPRESSION IS ENABLED. THIS MUST ONLY BE USED IF THE DECOMPRESSOR INPUT WILL ALWAYS BE TRUSTED!" +# pragma message("UNSAFE DECOMPRESSION IS ENABLED. THIS MUST ONLY BE USED IF THE DECOMPRESSOR INPUT WILL ALWAYS BE TRUSTED!") # define SAFETY_CHECK(expr) (void)(expr) #else # define SAFETY_CHECK(expr) if (unlikely(!(expr))) return DECOMPRESS_BAD_DATA @@ -194,16 +194,14 @@ typedef machine_word_t bitbuf_t; * corrupted in such a way that fully retains its validity. Users should run a * checksum against the uncompressed data if they wish to detect corruptions. */ -#define FILL_BITS_BYTEWISE() \ -({ \ - do { \ - if (likely(in_next != in_end)) \ - bitbuf |= (bitbuf_t)*in_next++ << bitsleft; \ - else \ - overrun_count++; \ - bitsleft += 8; \ - } while (bitsleft <= BITBUF_NBITS - 8); \ -}) +#define FILL_BITS_BYTEWISE() \ +do { \ + if (likely(in_next != in_end)) \ + bitbuf |= (bitbuf_t)*in_next++ << bitsleft; \ + else \ + overrun_count++; \ + bitsleft += 8; \ +} while (bitsleft <= BITBUF_NBITS - 8) /* * Fill the bitbuffer variable by reading the next word from the input buffer. @@ -213,12 +211,12 @@ typedef machine_word_t bitbuf_t; * most efficient on little-endian architectures that support fast unaligned * access, such as x86 and x86_64. */ -#define FILL_BITS_WORDWISE() \ -({ \ - bitbuf |= get_unaligned_leword(in_next) << bitsleft; \ - in_next += (BITBUF_NBITS - bitsleft) >> 3; \ - bitsleft += (BITBUF_NBITS - bitsleft) & ~7; \ -}) +#define FILL_BITS_WORDWISE() \ +do { \ + bitbuf |= get_unaligned_leword(in_next) << bitsleft; \ + in_next += (BITBUF_NBITS - bitsleft) >> 3; \ + bitsleft += (BITBUF_NBITS - bitsleft) & ~7; \ +} while (0) /* * Does the bitbuffer variable currently contain at least 'n' bits? @@ -226,56 +224,34 @@ typedef machine_word_t bitbuf_t; #define HAVE_BITS(n) (bitsleft >= (n)) /* - * Raw form of ENSURE_BITS(): the bitbuffer variable must not already contain - * the requested number of bits. + * Load more bits from the input buffer until the specified number of bits is + * present in the bitbuffer variable. 'n' cannot be too large; see MAX_ENSURE + * and CAN_ENSURE(). */ -#define DO_ENSURE_BITS(n) \ -({ \ +#define ENSURE_BITS(n) \ +if (!HAVE_BITS(n)) { \ if (CPU_IS_LITTLE_ENDIAN() && \ UNALIGNED_ACCESS_IS_FAST && \ likely(in_end - in_next >= sizeof(bitbuf_t))) \ FILL_BITS_WORDWISE(); \ else \ FILL_BITS_BYTEWISE(); \ -}) - -/* - * Load more bits from the input buffer until the specified number of bits is - * present in the bitbuffer variable. 'n' cannot be too large; see MAX_ENSURE - * and CAN_ENSURE(). - */ -#define ENSURE_BITS(n) \ -({ \ - if (!HAVE_BITS(n)) \ - DO_ENSURE_BITS(n); \ -}) +} /* * Return the next 'n' bits from the bitbuffer variable without removing them. */ -#define BITS(n) \ -({ \ - (u32)bitbuf & (((u32)1 << (n)) - 1); \ -}) +#define BITS(n) ((u32)bitbuf & (((u32)1 << (n)) - 1)) /* * Remove the next 'n' bits from the bitbuffer variable. */ -#define REMOVE_BITS(n) \ -({ \ - bitbuf >>= (n); \ - bitsleft -= (n); \ -}) +#define REMOVE_BITS(n) (bitbuf >>= (n), bitsleft -= (n)) /* * Remove and return the next 'n' bits from the bitbuffer variable. */ -#define POP_BITS(n) \ -({ \ - u32 bits = BITS(n); \ - REMOVE_BITS(n); \ - bits; \ -}) +#define POP_BITS(n) (tmp32 = BITS(n), REMOVE_BITS(n), tmp32) /* * Align the input to the next byte boundary, discarding any remaining bits in @@ -287,24 +263,17 @@ typedef machine_word_t bitbuf_t; * be actually discarded. */ #define ALIGN_INPUT() \ -({ \ +do { \ in_next -= (bitsleft >> 3) - MIN(overrun_count, bitsleft >> 3); \ bitbuf = 0; \ bitsleft = 0; \ -}) +} while(0) /* * Read a 16-bit value from the input. This must have been preceded by a call * to ALIGN_INPUT(), and the caller must have already checked for overrun. */ -#define READ_U16() \ -({ \ - u16 v; \ - \ - v = get_unaligned_le16(in_next); \ - in_next += 2; \ - v; \ -}) +#define READ_U16() (tmp16 = get_unaligned_le16(in_next), in_next += 2, tmp16) /***************************************************************************** * Huffman decoding * diff --git a/src/gzip_compress.c b/src/gzip_compress.c index cb60448..e66fa3c 100644 --- a/src/gzip_compress.c +++ b/src/gzip_compress.c @@ -8,13 +8,13 @@ * You can do whatever you want with this file. */ -#include "libdeflate.h" - #include "crc32.h" #include "deflate_compress.h" #include "gzip_constants.h" #include "unaligned.h" +#include "libdeflate.h" + LIBEXPORT size_t gzip_compress(struct deflate_compressor *c, const void *in, size_t in_size, void *out, size_t out_nbytes_avail) diff --git a/src/gzip_decompress.c b/src/gzip_decompress.c index a2748a5..7237c5a 100644 --- a/src/gzip_decompress.c +++ b/src/gzip_decompress.c @@ -8,12 +8,12 @@ * You can do whatever you want with this file. */ -#include "libdeflate.h" - #include "crc32.h" #include "gzip_constants.h" #include "unaligned.h" +#include "libdeflate.h" + LIBEXPORT enum decompress_result gzip_decompress(struct deflate_decompressor *d, const void *in, size_t in_nbytes, diff --git a/src/hc_matchfinder.h b/src/hc_matchfinder.h index 9b53477..edc9cea 100644 --- a/src/hc_matchfinder.h +++ b/src/hc_matchfinder.h @@ -167,12 +167,12 @@ hc_matchfinder_longest_match(struct hc_matchfinder * const restrict mf, const u32 max_len, const u32 nice_len, const u32 max_search_depth, - u32 next_hashes[const restrict static 2], + u32 next_hashes[restrict 2], u32 * const restrict offset_ret) { const u8 *in_next = in_base + cur_pos; u32 depth_remaining = max_search_depth; - const u8 *best_matchptr = best_matchptr; /* uninitialized */ + const u8 *best_matchptr = in_next; const mf_pos_t cutoff = cur_pos - MATCHFINDER_WINDOW_SIZE; mf_pos_t cur_node3, cur_node4; u32 hash3, hash4; @@ -335,7 +335,7 @@ hc_matchfinder_skip_positions(struct hc_matchfinder * const restrict mf, const ptrdiff_t cur_pos, const ptrdiff_t end_pos, const u32 count, - u32 next_hashes[const restrict static 2]) + u32 next_hashes[restrict 2]) { const u8 *in_next = in_base + cur_pos; const u8 * const stop_ptr = in_next + count; diff --git a/src/zlib_compress.c b/src/zlib_compress.c index 79765cf..f8ab689 100644 --- a/src/zlib_compress.c +++ b/src/zlib_compress.c @@ -8,13 +8,13 @@ * You can do whatever you want with this file. */ -#include "libdeflate.h" - #include "adler32.h" #include "deflate_compress.h" #include "unaligned.h" #include "zlib_constants.h" +#include "libdeflate.h" + LIBEXPORT size_t zlib_compress(struct deflate_compressor *c, const void *in, size_t in_size, void *out, size_t out_nbytes_avail) diff --git a/src/zlib_decompress.c b/src/zlib_decompress.c index eca05dd..25024f5 100644 --- a/src/zlib_decompress.c +++ b/src/zlib_decompress.c @@ -8,12 +8,12 @@ * You can do whatever you want with this file. */ -#include "libdeflate.h" - #include "adler32.h" #include "unaligned.h" #include "zlib_constants.h" +#include "libdeflate.h" + LIBEXPORT enum decompress_result zlib_decompress(struct deflate_decompressor *d, const void *in, size_t in_nbytes, diff --git a/tools/benchmark.c b/tools/benchmark.c index bc88fa0..7e3c2fd 100644 --- a/tools/benchmark.c +++ b/tools/benchmark.c @@ -8,70 +8,37 @@ * You can do whatever you want with this file. */ -#define _FILE_OFFSET_BITS 64 -#undef _ANSI_SOURCE -#define _POSIX_C_SOURCE 199309L -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "util.h" -#include #include +static const tchar *const optstring = T("s:l:123456789zgYZh"); + static void usage(FILE *fp) { static const char * const str = -"Usage: benchmark [FILE...]\n" +"Usage: benchmark [OPTION]... [FILE]...\n" "\n" "A compression and decompression benchmark and testing program.\n" "Benchmarks are run on each FILE specified, or stdin if no file is specified.\n" "\n" "Options:\n" -" -s, --chunk-size=SIZE chunk size\n" -" -l, --level=LEVEL compression level [1-12]\n" -" -1 fastest\n" -" -9 slow\n" -" -z, --zlib use zlib wrapper\n" -" -g, --gzip use gzip wrapper\n" -" -Y, --compress-with-libz compress with libz, not libdeflate\n" -" -Z, --decompress-with-libz decompress with libz, not libdeflate\n" -" -h, --help print this help\n" +" -s SIZE chunk size\n" +" -l LEVEL compression level [1-12]\n" +" -1 fastest\n" +" -9 slow\n" +" -z use zlib wrapper\n" +" -g use gzip wrapper\n" +" -Y compress with libz, not libdeflate\n" +" -Z decompress with libz, not libdeflate\n" +" -h print this help\n" ; fputs(str, fp); } -static void -fatal_error(const char *fmt, ...) -{ - va_list va; - - va_start(va, fmt); - fprintf(stderr, "ERROR: "); - vfprintf(stderr, fmt, va); - fprintf(stderr, "\n"); - va_end(va); - - exit(1); -} - -#define ASSERT(expr, fmt, ...) \ -{ \ - if (!(expr)) \ - fatal_error((fmt), ## __VA_ARGS__); \ -} - enum wrapper { NO_WRAPPER, ZLIB_WRAPPER, @@ -293,15 +260,6 @@ decompressor_destroy(struct decompressor *d) (*d->free_private)(d->private); } -#define TIME_UNIT_PER_MS 1000000 -static uint64_t -current_time(void) -{ - struct timespec ts; - clock_gettime(CLOCK_MONOTONIC, &ts); - return ((uint64_t)1000000000 * ts.tv_sec) + ts.tv_nsec; -} - static void do_benchmark(int fd, char *ubuf1, char *ubuf2, char *cbuf, uint32_t max_chunk_size, @@ -312,13 +270,9 @@ do_benchmark(int fd, char *ubuf1, char *ubuf2, uint64_t compress_time_total = 0; uint64_t decompress_time_total = 0; -#ifdef __WIN32__ - _setmode(fd, O_BINARY); -#endif - for (;;) { char *p = ubuf1; - ssize_t bytes_read; + int32_t bytes_read; size_t usize; size_t csize; uint64_t start_time; @@ -381,27 +335,15 @@ do_benchmark(int fd, char *ubuf1, char *ubuf2, (unsigned int)(csize_total * 100 / usize_total), (unsigned int)(csize_total * 100000 / usize_total % 1000)); printf("\tCompression time: %"PRIu64" ms (%"PRIu64" MB/s)\n", - compress_time_total / TIME_UNIT_PER_MS, + compress_time_total / 1000000, 1000 * usize_total / compress_time_total); printf("\tDecompression time: %"PRIu64" ms (%"PRIu64" MB/s)\n", - decompress_time_total / TIME_UNIT_PER_MS, + decompress_time_total / 1000000, 1000 * usize_total / decompress_time_total); } -static const char *const optstring = "s:l:0123456789gzYZh"; -static const struct option longopts[] = { - {"chunk-size", required_argument, NULL, 's'}, - {"level", required_argument, NULL, 'l'}, - {"zlib", no_argument, NULL, 'z'}, - {"gzip", no_argument, NULL, 'g'}, - {"compress-with-libz", no_argument, NULL, 'Y'}, - {"decompress-with-libz", no_argument, NULL, 'Z'}, - {"help", no_argument, NULL, 'h'}, - {NULL, 0, NULL, 0}, -}; - int -main(int argc, char **argv) +main(int argc, tchar *argv[]) { uint32_t chunk_size = 32768; int level = 6; @@ -415,17 +357,23 @@ main(int argc, char **argv) struct decompressor d; int opt_char; - while ((opt_char = - getopt_long(argc, argv, optstring, longopts, NULL)) != -1) - { + while ((opt_char = tgetopt(argc, argv, optstring)) != -1) { switch (opt_char) { case 's': - chunk_size = strtoul(optarg, NULL, 10); + chunk_size = tstrtoul(toptarg, NULL, 10); break; case 'l': - level = strtoul(optarg, NULL, 10); + level = tstrtoul(toptarg, NULL, 10); break; - case '1' ... '9': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': level = opt_char - '0'; break; case 'z': @@ -449,8 +397,8 @@ main(int argc, char **argv) } } - argc -= optind; - argv += optind; + argc -= toptind; + argv += toptind; printf("Benchmarking DEFLATE compression:\n"); printf("\tCompression level: %d\n", level); @@ -475,14 +423,15 @@ main(int argc, char **argv) if (argc == 0) { printf("Reading from stdin...\n"); + set_binary_mode(STDIN_FILENO); do_benchmark(STDIN_FILENO, ubuf1, ubuf2, cbuf, chunk_size, &c, &d); } else { for (int i = 0; i < argc; i++) { - printf("Processing \"%s\"...\n", argv[i]); - int fd = open(argv[i], O_RDONLY); + printf("Processing \"%"TS"\"...\n", argv[i]); + int fd = topen(argv[i], O_RDONLY | O_BINARY); ASSERT(fd >= 0, - "Can't open \"%s\" for reading: %s\n", + "Can't open \"%"TS"\" for reading: %s\n", argv[i], strerror(errno)); do_benchmark(fd, ubuf1, ubuf2, cbuf, chunk_size, &c, &d); close(fd); diff --git a/tools/gzip.c b/tools/gzip.c index e8b3c7e..fa974e0 100644 --- a/tools/gzip.c +++ b/tools/gzip.c @@ -8,76 +8,7 @@ * You can do whatever you want with this file. */ -#define _FILE_OFFSET_BITS 64 - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __WIN32__ -# include -# include -#else -# include -#endif - -#include "libdeflate.h" - -#ifdef __WIN32__ -# define main wmain -# define tchar wchar_t -# define _T(text) L##text -# define T(text) _T(text) -# define TS "ls" -# define tstrcmp wcscmp -# define tstrlen wcslen -# define tmemcpy wmemcpy -# define tstrrchr wcsrchr -# define tstrtol wcstol -# define topen _wopen -# define topen _wopen -# define tstat _wstati64 -# define tunlink _wunlink -#else -# define main main -# define tchar char -# define T(text) text -# define TS "s" -# define tstrcmp strcmp -# define tstrlen strlen -# define tmemcpy memcpy -# define tstrrchr strrchr -# define tstrtol strtol -# define topen open -# define tstat stat -# define tunlink unlink -#endif - -#ifndef O_BINARY -# define O_BINARY 0 -#endif - -static const tchar * -basename(const tchar *argv0) -{ - const tchar *p = tstrrchr(argv0, '/'); -#ifdef __WIN32__ - const tchar *p2 = tstrrchr(argv0, '\\'); - if (p2 && (!p || p2 > p)) - p = p2; -#endif - if (p) - return p + 1; - return argv0; -} +#include "util.h" static bool is_gunzip(const tchar *argv0) @@ -86,7 +17,7 @@ is_gunzip(const tchar *argv0) if (!tstrcmp(name, T("gunzip"))) return true; -#ifdef __WIN32__ +#ifdef _WIN32 if (!tstrcmp(name, T("gunzip.exe"))) return true; #endif @@ -104,26 +35,6 @@ usage(void) exit(1); } -static void -fatal_error(const char *fmt, ...) -{ - va_list va; - - va_start(va, fmt); - fprintf(stderr, "ERROR: "); - vfprintf(stderr, fmt, va); - fprintf(stderr, "\n"); - va_end(va); - - exit(1); -} - -#define ASSERT(expr, fmt, ...) \ -{ \ - if (!(expr)) \ - fatal_error((fmt), ## __VA_ARGS__); \ -} - static void * map_file_contents(const tchar *path, size_t *size_ret, void **token_ret) { @@ -148,8 +59,8 @@ map_file_contents(const tchar *path, size_t *size_ret, void **token_ret) "File \"%"TS"\" cannot be processed by " "this program because it is too large.", path); -#ifdef __WIN32__ - HANDLE h = CreateFileMapping((HANDLE)_get_osfhandle(fd), +#ifdef _WIN32 + HANDLE h = CreateFileMapping((HANDLE)(intptr_t)_get_osfhandle(fd), NULL, PAGE_READONLY, 0, 0, NULL); ASSERT(h != NULL, "Unable create file mapping for \"%"TS"\": " "Windows error %u", path, (unsigned int )GetLastError()); @@ -178,7 +89,7 @@ map_file_contents(const tchar *path, size_t *size_ret, void **token_ret) static void unmap_file_contents(void *token, void *map, size_t size) { -#ifdef __WIN32__ +#ifdef _WIN32 UnmapViewOfFile(map); CloseHandle((HANDLE)token); #else @@ -207,7 +118,7 @@ write_file(const tchar *path, const void *contents, size_t size) ASSERT(ret > 0, "Error writing data to \"%"TS"\": %s", path, strerror(errno)); size -= ret; - contents += ret; + contents = (const uint8_t *)contents + ret; } ASSERT(!close(fd), "Error writing data to \"%"TS"\"", @@ -253,7 +164,7 @@ decompress_file(struct deflate_decompressor *d, const tchar *path, bool force) ASSERT(compressed_size >= sizeof(uint32_t), "File \"%"TS"\" is not a gzip file.", path); - uncompressed_size = load_u32_gzip(compressed_data + + uncompressed_size = load_u32_gzip((const uint8_t *)compressed_data + (compressed_size - sizeof(uint32_t))); uncompressed_data = malloc(uncompressed_size); ASSERT(uncompressed_data != NULL, diff --git a/tools/msc_test.bat b/tools/msc_test.bat new file mode 100755 index 0000000..3ec07ac --- /dev/null +++ b/tools/msc_test.bat @@ -0,0 +1,23 @@ +call d:\VS2015\vc\vcvarsall +nmake /f Makefile.msc clean +nmake /f Makefile.msc +cl /MD /O2 /Fe:benchmark.exe -I. -Itools -I..\zlib-msc tools\benchmark.c tools\wgetopt.c libdeflatestatic.lib ..\zlib-msc\zlib.lib +cl /MD /O2 /Fe:gzip.exe -I. -Itools tools\gzip.c tools\wgetopt.c libdeflatestatic.lib +del j:\exe\gzip.exe 2> nul +del j:\exe\gunzip.exe 2> nul +copy gzip.exe j:\exe\gzip.exe +copy gzip.exe j:\exe\gunzip.exe +del j:\exe\benchmark.exe 2> nul +copy benchmark.exe j:\exe\benchmark.exe + +call d:\VS2015\vc\vcvarsall x86_amd64 +nmake /f Makefile.msc clean +nmake /f Makefile.msc +cl /MD /O2 /Fe:benchmark.exe -I. -Itools -I..\zlib-msc64 tools\benchmark.c tools\wgetopt.c libdeflatestatic.lib ..\zlib-msc64\zlib.lib +cl /MD /O2 /Fe:gzip.exe -I. -Itools tools\gzip.c tools\wgetopt.c libdeflatestatic.lib +del j:\exe64\gzip.exe 2> nul +del j:\exe64\gunzip.exe 2> nul +copy gzip.exe j:\exe64\gzip.exe +copy gzip.exe j:\exe64\gunzip.exe +del j:\exe64\benchmark.exe 2> nul +copy benchmark.exe j:\exe64\benchmark.exe diff --git a/tools/util.h b/tools/util.h new file mode 100644 index 0000000..15058fd --- /dev/null +++ b/tools/util.h @@ -0,0 +1,135 @@ +#define _FILE_OFFSET_BITS 64 +#undef _ANSI_SOURCE +#define _POSIX_C_SOURCE 199309L + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +# include +# include +# include "wgetopt.h" +#else +# include +# include +# include +# include +#endif + +#include + +#ifdef _WIN32 +# define main wmain +# define tchar wchar_t +# define _T(text) L##text +# define T(text) _T(text) +# define TS "ls" +# define tgetopt wgetopt +# define toptarg woptarg +# define toptind woptind +# define tstrcmp wcscmp +# define tstrlen wcslen +# define tmemcpy wmemcpy +# define tstrrchr wcsrchr +# define tstrtol wcstol +# define tstrtoul wcstoul +# define topen _wopen +# define tunlink _wunlink +# ifdef _MSC_VER +# define fstat _fstat64 +# define stat _stat64 +# define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) +# endif +# define tstat _wstati64 +#else +# define main main +# define tchar char +# define T(text) text +# define TS "s" +# define tgetopt getopt +# define toptarg optarg +# define toptind optind +# define tstrcmp strcmp +# define tstrlen strlen +# define tmemcpy memcpy +# define tstrrchr strrchr +# define tstrtol strtol +# define tstrtoul strtoul +# define topen open +# define tunlink unlink +# define tstat stat +#endif + +#ifndef O_BINARY +# define O_BINARY 0 +#endif + +#ifndef STDIN_FILENO +# define STDIN_FILENO 0 +#endif + +static void +fatal_error(const char *fmt, ...) +{ + va_list va; + + va_start(va, fmt); + fprintf(stderr, "ERROR: "); + vfprintf(stderr, fmt, va); + fprintf(stderr, "\n"); + va_end(va); + + exit(1); +} + +#define ASSERT(expr, fmt, ...) \ +{ \ + if (!(expr)) \ + fatal_error((fmt), ## __VA_ARGS__); \ +} + +static inline const tchar * +basename(const tchar *argv0) +{ + const tchar *p = tstrrchr(argv0, '/'); +#ifdef _WIN32 + const tchar *p2 = tstrrchr(argv0, '\\'); + if (p2 && (!p || p2 > p)) + p = p2; +#endif + if (p) + return p + 1; + return argv0; +} + +static inline void +set_binary_mode(int fd) +{ +#ifdef _WIN32 + _setmode(fd, O_BINARY); +#endif +} + +static inline uint64_t +current_time(void) +{ +#ifdef _WIN32 + FILETIME ft; + GetSystemTimeAsFileTime(&ft); + return 100 * (((uint64_t)ft.dwHighDateTime << 32) | ft.dwLowDateTime); +#else + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ((uint64_t)1000000000 * ts.tv_sec) + ts.tv_nsec; +#endif +} diff --git a/tools/wgetopt.c b/tools/wgetopt.c new file mode 100644 index 0000000..1b0dc71 --- /dev/null +++ b/tools/wgetopt.c @@ -0,0 +1,62 @@ +/* + * wide-character getopt for Windows + * + * Public domain + */ + +#include +#include + +#include "wgetopt.h" + +wchar_t *woptarg; /* Global argument pointer */ +int woptind = 0; /* Global argv index */ + +static wchar_t *scan = NULL; /* Private scan pointer */ + +int wgetopt(int argc, wchar_t * const argv[], const wchar_t *optstring) +{ + wchar_t c; + wchar_t *place; + + woptarg = NULL; + + if (!scan || *scan == '\0') { + if (woptind == 0) + woptind++; + if (woptind >= argc || argv[woptind][0] != '-' + || argv[woptind][1] == '\0') + return EOF; + if (argv[woptind][1] == '-' && argv[woptind][2] == '\0') { + woptind++; + return EOF; + } + scan = argv[woptind] + 1; + woptind++; + } + + c = *scan++; + place = wcschr(optstring, c); + + if (!place || c == ':') { + fprintf(stderr, "%ls: unknown option -%lc\n", argv[0], c); + return '?'; + } + + place++; + if (*place == ':') { + if (*scan != '\0') { + woptarg = scan; + scan = NULL; + } else if (woptind < argc) { + woptarg = argv[woptind]; + woptind++; + } else { + fprintf(stderr, "%ls: option requires argument -%lc\n", + argv[0], c); + return ':'; + } + } + + return c; +} diff --git a/tools/wgetopt.h b/tools/wgetopt.h new file mode 100644 index 0000000..e8fa68f --- /dev/null +++ b/tools/wgetopt.h @@ -0,0 +1,4 @@ +extern wchar_t *woptarg; +extern int woptind; + +int wgetopt(int argc, wchar_t * const argv[], const wchar_t *optstring); diff --git a/tools/windows_test.sh b/tools/windows_test.sh index add30a0..0c97eeb 100755 --- a/tools/windows_test.sh +++ b/tools/windows_test.sh @@ -4,10 +4,10 @@ set -e make clean make -j4 CC=i686-w64-mingw32-gcc BUILD_PROGRAMS=yes -cp *.exe /j/exe +cp -vf *.exe /j/exe/ make clean make -j4 CC=x86_64-w64-mingw32-gcc BUILD_PROGRAMS=yes -cp *.exe /j/exe64 +cp -vf *.exe /j/exe64/ sudo systemctl restart smbd