diff --git a/vlib/compress/compress.c.v b/vlib/compress/compress.c.v index a5f0ffe6ff..f2baf98b70 100644 --- a/vlib/compress/compress.c.v +++ b/vlib/compress/compress.c.v @@ -51,3 +51,51 @@ pub fn decompress(data []u8, flags int) ![]u8 { return ret } } + +// ChunkCallback is used to receive decompressed chunks of maximum 32768 bytes. +// After processing the chunk this function should return the chunk's length to indicate +// the decompressor to send more chunks, otherwise the decompression stops. +// The userdata parameter comes from the call to decompress_with_callback/4, and can be used +// to pass arbitrary data, without having to create a closure. +pub type ChunkCallback = fn (chunk []u8, userdata voidptr) int + +// decompress_with_callback decompresses an array of bytes based on the provided flags, +// and a V fn callback to receive decompressed chunks, of at most 32 kilobytes each. +// It returns the total decompressed length, or a decompression error. +// NB: this is a low level api, a high level implementation like zlib/gzip should be preferred. +pub fn decompress_with_callback(data []u8, cb ChunkCallback, userdata voidptr, flags int) !u64 { + cbdata := DecompressionCallBackData{ + data: data.data + size: usize(data.len) + cb: cb + userdata: userdata + } + status := C.tinfl_decompress_mem_to_callback(cbdata.data, &cbdata.size, c_cb_for_decompress_mem, + &cbdata, flags) + if status == 0 { + return error('decompression error') + } + return cbdata.decompressed_size +} + +struct DecompressionCallBackData { +mut: + data voidptr + size usize + decompressed_size u64 + userdata voidptr + cb ChunkCallback = unsafe { nil } +} + +fn c_cb_for_decompress_mem(buf &char, len int, pdcbd voidptr) int { + mut cbdata := unsafe { &DecompressionCallBackData(pdcbd) } + if cbdata.cb(unsafe { voidptr(buf).vbytes(len) }, cbdata.userdata) == len { + cbdata.decompressed_size += u64(len) + return 1 // continue decompressing + } + return 0 // stop decompressing +} + +type DecompressCallback = fn (const_buffer voidptr, len int, userdata voidptr) int + +fn C.tinfl_decompress_mem_to_callback(const_input_buffer voidptr, psize &usize, put_buf_cb DecompressCallback, userdata voidptr, flags int) int diff --git a/vlib/compress/gzip/gzip.v b/vlib/compress/gzip/gzip.v index 8ac9c7cb95..96ad3432bd 100644 --- a/vlib/compress/gzip/gzip.v +++ b/vlib/compress/gzip/gzip.v @@ -203,7 +203,7 @@ pub fn validate(data []u8, params DecompressParams) !GzipHeader { return header } -// decompresses an array of bytes using zlib and returns the decompressed bytes in a new array +// decompress an array of bytes using zlib and returns the decompressed bytes in a new array // Example: decompressed := gzip.decompress(b)! pub fn decompress(data []u8, params DecompressParams) ![]u8 { gzip_header := validate(data, params)! @@ -221,3 +221,20 @@ pub fn decompress(data []u8, params DecompressParams) ![]u8 { } return decompressed } + +// decompress_with_callback decompresses the given `data`, using zlib. It calls `cb` with each chunk of decompressed bytes. +// A chunk is usually 32 KB or less. Note: the chunk data received by `cb` should be cloned, if you need to store it for later, +// and not process it right away. +// The callback function should return the chunk length, if it wants to continue decompressing, or 0, if it wants to abort the decompression early. +// See also compress.ChunkCallback for more details. +pub fn decompress_with_callback(data []u8, cb compr.ChunkCallback, userdata voidptr, params DecompressParams) !int { + gzip_header := validate(data, params)! + header_len := gzip_header.length + expected_len := int((u32(data[data.len - 1]) << 24) | (u32(data[data.len - 2]) << 16) | (u32(data[data.len - 3]) << 8) | data[data.len - 4]) + body := data[header_len..data.len - 8] + chunks_len := int(compr.decompress_with_callback(body, cb, userdata, 0)!) + if params.verify_length && expected_len != chunks_len { + return error('Decompress error: expected length:${expected_len}, got:${chunks_len}') + } + return chunks_len +} diff --git a/vlib/compress/gzip/gzip_test.v b/vlib/compress/gzip/gzip_test.v index f785fe0f5a..b4aab3cf4f 100644 --- a/vlib/compress/gzip/gzip_test.v +++ b/vlib/compress/gzip/gzip_test.v @@ -132,3 +132,18 @@ fn test_gzip_with_invalid_flags() { compressed[3] |= 0b1000_0000 assert_decompress_error(compressed, 'reserved flags are set, unsupported field detected')! } + +fn test_gzip_decompress_callback() { + uncompressed := '321323'.repeat(10_000) + gz := compress(uncompressed.bytes())! + mut size := 0 + mut ref := &size + decoded := decompress_with_callback(gz, fn (chunk []u8, ref &int) int { + unsafe { + *ref += chunk.len + } + return chunk.len + }, ref)! + assert decoded == size + assert decoded == uncompressed.len +}