cuberite/libdeflate
1
0
Fork 0
mirror of https://github.com/cuberite/libdeflate.git synced 2025-09-10 04:41:45 -04:00
Code Issues Packages Projects Releases Wiki Activity

Fix check for match cache overflow

Browse Source
This commit is contained in:
Eric Biggers 2016-05-23 07:47:57 -05:00
parent 3d52ac1195
commit 4f7d134b39
1 changed files with 52 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
lib/deflate_compress.c
View File

@ -50,27 +50,33 @@
# include "bt_matchfinder.h" # include "bt_matchfinder.h"
#endif #endif
/* The minimum and maximum block lengths, in bytes of source data, which the /*
* parsing algorithms may choose. */ * The minimum and maximum block lengths, in bytes of source data, which the
* parsing algorithms may choose.
*/
#define MIN_BLOCK_LENGTH 10000 #define MIN_BLOCK_LENGTH 10000
#define MAX_BLOCK_LENGTH 300000 #define MAX_BLOCK_LENGTH 300000
#if SUPPORT_NEAR_OPTIMAL_PARSING #if SUPPORT_NEAR_OPTIMAL_PARSING
/* Constants specific to the near-optimal parsing algorithm. */ /* Constants specific to the near-optimal parsing algorithm */
/* The maximum number of matches the matchfinder can find at a single position. /*
* The maximum number of matches the matchfinder can find at a single position.
* Since the matchfinder never finds more than one match for the same length, * Since the matchfinder never finds more than one match for the same length,
* presuming one of each possible length is sufficient for an upper bound. * presuming one of each possible length is sufficient for an upper bound.
* (This says nothing about whether it is worthwhile to consider so many * (This says nothing about whether it is worthwhile to consider so many
* matches; this is just defining the worst case.) */ * matches; this is just defining the worst case.)
*/
# define MAX_MATCHES_PER_POS (DEFLATE_MAX_MATCH_LEN - DEFLATE_MIN_MATCH_LEN + 1) # define MAX_MATCHES_PER_POS (DEFLATE_MAX_MATCH_LEN - DEFLATE_MIN_MATCH_LEN + 1)
/* The number of array spaces to reserve for a single block's matches. This /*
* value should be high enough so that virtually the time, all matches found in * The number of lz_match structures in the match cache, excluding the extra
* MAX_BLOCK_LENGTH consecutive positions can fit in this array. However, this * "overflow" entries. This value should be high enough so that nearly the
* is *not* the true upper bound on the number of matches that can possibly be * time, all matches found in a given block can fit in the match cache.
* found. Therefore, checks for overflow are still required. */ * However, fallback behavior (immediately terminating the block) on cache
# define CACHE_LEN ((MAX_BLOCK_LENGTH * 5) + (MAX_MATCHES_PER_POS + 1)) * overflow is still required.
*/
# define CACHE_LENGTH (MAX_BLOCK_LENGTH * 5)
#endif /* SUPPORT_NEAR_OPTIMAL_PARSING */ #endif /* SUPPORT_NEAR_OPTIMAL_PARSING */
@ -251,8 +257,8 @@ struct deflate_sequence {
* each outgoing edge from this node is labeled with a literal or a match that * each outgoing edge from this node is labeled with a literal or a match that
* can be taken to advance from this position to a later position. * can be taken to advance from this position to a later position.
* *
* But these "edges" are actually stored elsewhere (in 'cached_matches'). * But these "edges" are actually stored elsewhere (in 'match_cache'). Here we
* Here we associate with each node just two pieces of information: * associate with each node just two pieces of information:
* *
* 'cost_to_end' is the minimum cost to reach the end of the block from * 'cost_to_end' is the minimum cost to reach the end of the block from
* this position. * this position.
@ -362,11 +368,33 @@ struct deflate_compressor {
/* Binary tree matchfinder */ /* Binary tree matchfinder */
struct bt_matchfinder bt_mf; struct bt_matchfinder bt_mf;
/* Matches found using the matchfinder are cached in /*
* this array so that later optimization of the block * Cached matches for the current block. This array
* has the matches easily available. The cached matches * contains the matches that were found at each position
* are cleared when a new block is started. */ * in the block. Specifically, for each position, there
struct lz_match cached_matches[CACHE_LEN]; * is a list of matches found at that position, if any,
* sorted by strictly increasing length. In addition,
* following the matches for each position, there is a
* special 'struct lz_match' whose 'length' member
* contains the number of matches found at that
* position, and whose 'offset' member contains the
* literal at that position.
*
* Note: in rare cases, there will be a very high number
* of matches in the block and this array will overflow.
* If this happens, we force the end of the current
* block. CACHE_LENGTH is the length at which we
* actually check for overflow. The extra slots beyond
* this are enough to absorb the worst case overflow,
* which occurs if starting at &match_cache[CACHE_LENGTH
* - 1], we write the match count header, then write
* MAX_MATCHES_PER_POS matches, then skip searching for
* matches at 'DEFLATE_MAX_MATCH_LEN - 1' positions and
* write the match count header for each.
*/
struct lz_match match_cache[CACHE_LENGTH +
MAX_MATCHES_PER_POS +
DEFLATE_MAX_MATCH_LEN - 1];
/* Array of structures, one per position, for running /* Array of structures, one per position, for running
* the minimum-cost path algorithm. */ * the minimum-cost path algorithm. */
@ -2268,7 +2296,7 @@ deflate_optimize_and_write_block(struct deflate_compressor *c,
struct deflate_output_bitstream *os, struct deflate_output_bitstream *os,
const u8 * const block_begin, const u8 * const block_begin,
const u32 block_length, const u32 block_length,
struct lz_match * const end_cache_ptr, const struct lz_match * const end_cache_ptr,
const bool is_final_block) const bool is_final_block)
{ {
struct deflate_optimum_node * const end_node = c->optimum + block_length; struct deflate_optimum_node * const end_node = c->optimum + block_length;
@ -2298,7 +2326,7 @@ deflate_optimize_and_write_block(struct deflate_compressor *c,
* and the match/literal choice is saved. * and the match/literal choice is saved.
*/ */
struct deflate_optimum_node *cur_node = end_node; struct deflate_optimum_node *cur_node = end_node;
struct lz_match *cache_ptr = end_cache_ptr; const struct lz_match *cache_ptr = end_cache_ptr;
cur_node->cost_to_end = 0; cur_node->cost_to_end = 0;
do { do {
@ -2320,7 +2348,7 @@ deflate_optimize_and_write_block(struct deflate_compressor *c,
/* Also consider matches if there are any. */ /* Also consider matches if there are any. */
if (num_matches) { if (num_matches) {
struct lz_match *match; const struct lz_match *match;
unsigned len; unsigned len;
unsigned offset; unsigned offset;
unsigned offset_slot; unsigned offset_slot;
@ -2410,8 +2438,7 @@ deflate_compress_near_optimal(struct deflate_compressor * restrict c,
do { do {
/* Starting a new DEFLATE block. */ /* Starting a new DEFLATE block. */
struct lz_match *cache_ptr = c->cached_matches; struct lz_match *cache_ptr = c->match_cache;
struct lz_match * const cache_end = &c->cached_matches[CACHE_LEN - (MAX_MATCHES_PER_POS + 1)];
const u8 * const in_block_begin = in_next; const u8 * const in_block_begin = in_next;
const u8 * const in_max_block_end = in_next + MIN(in_end - in_next, MAX_BLOCK_LENGTH); const u8 * const in_max_block_end = in_next + MIN(in_end - in_next, MAX_BLOCK_LENGTH);
struct block_split_stats split_stats; struct block_split_stats split_stats;
@ -2450,7 +2477,7 @@ deflate_compress_near_optimal(struct deflate_compressor * restrict c,
/* /*
* Find matches with the current position using the * Find matches with the current position using the
* binary tree matchfinder and save them in * binary tree matchfinder and save them in
* 'cached_matches'. * 'match_cache'.
* *
* Note: the binary tree matchfinder is more suited for * Note: the binary tree matchfinder is more suited for
* optimal parsing than the hash chain matchfinder. The * optimal parsing than the hash chain matchfinder. The
@ -2534,7 +2561,7 @@ deflate_compress_near_optimal(struct deflate_compressor * restrict c,
} while (--best_len); } while (--best_len);
} }
} while (in_next < in_max_block_end && } while (in_next < in_max_block_end &&
cache_ptr <= cache_end && cache_ptr < &c->match_cache[CACHE_LENGTH] &&
!should_end_block(&split_stats, in_block_begin, in_next, in_end)); !should_end_block(&split_stats, in_block_begin, in_next, in_end));
/* All the matches for this block have been cached. Now compute /* All the matches for this block have been cached. Now compute