compress.gzip: change the endianness for validation to conform to the gzip file specification (fix #19839) (#19849)

This commit is contained in:
shove 2023-11-12 17:30:05 +08:00 committed by GitHub
parent c494b63d75
commit 14e53f9a75
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 50 additions and 9 deletions

View File

@ -28,14 +28,14 @@ pub fn compress(data []u8) ![]u8 {
checksum := crc32.sum(data)
length := data.len
result << [
u8(checksum >> 24),
u8(checksum >> 16),
u8(checksum >> 8),
u8(checksum),
u8(length >> 24),
u8(length >> 16),
u8(length >> 8),
u8(checksum >> 8),
u8(checksum >> 16),
u8(checksum >> 24),
u8(length),
u8(length >> 8),
u8(length >> 16),
u8(length >> 24),
] // 8 bytes
return result
}
@ -140,12 +140,12 @@ pub fn decompress(data []u8, params DecompressParams) ![]u8 {
header_length := gzip_header.length
decompressed := compr.decompress(data[header_length..data.len - 8], 0)!
length_expected := (u32(data[data.len - 4]) << 24) | (u32(data[data.len - 3]) << 16) | (u32(data[data.len - 2]) << 8) | data[data.len - 1]
length_expected := (u32(data[data.len - 1]) << 24) | (u32(data[data.len - 2]) << 16) | (u32(data[data.len - 3]) << 8) | data[data.len - 4]
if params.verify_length && decompressed.len != length_expected {
return error('length verification failed, got ${decompressed.len}, expected ${length_expected}')
}
checksum := crc32.sum(decompressed)
checksum_expected := (u32(data[data.len - 8]) << 24) | (u32(data[data.len - 7]) << 16) | (u32(data[data.len - 6]) << 8) | data[data.len - 5]
checksum_expected := (u32(data[data.len - 5]) << 24) | (u32(data[data.len - 6]) << 16) | (u32(data[data.len - 7]) << 8) | data[data.len - 8]
if params.verify_checksum && checksum != checksum_expected {
return error('checksum verification failed')
}

View File

@ -123,7 +123,7 @@ fn test_gzip_with_invalid_length() {
uncompressed := 'Hello world!'
mut compressed := compress(uncompressed.bytes())!
compressed[compressed.len - 1] += 1
assert_decompress_error(compressed, 'length verification failed, got 12, expected 13')!
assert_decompress_error(compressed, 'length verification failed, got 12, expected 16777228')!
}
fn test_gzip_with_invalid_flags() {

View File

@ -0,0 +1,41 @@
import os
import compress.gzip
const samples_folder = os.join_path(os.dir(@FILE), 'samples')
fn s(fname string) string {
return os.join_path(samples_folder, fname)
}
fn read_and_decode_file(fpath string) !([]u8, string) {
compressed := os.read_bytes(fpath)!
decoded := gzip.decompress(compressed)!
content := decoded.bytestr()
return compressed, content
}
fn test_reading_and_decoding_a_known_gziped_file() {
compressed, content := read_and_decode_file(s('known.gz'))!
assert compressed#[0..3] == [u8(31), 139, 8]
assert compressed#[-5..] == [u8(127), 115, 1, 0, 0]
assert content.contains('## Description:')
assert content.contains('## Examples:')
assert content.ends_with('```\n')
}
fn test_decoding_all_samples_files() {
for gz_file in os.walk_ext(samples_folder, '.gz') {
_, content := read_and_decode_file(gz_file)!
assert content.len > 0, 'decoded content should not be empty: `${content}`'
}
}
fn test_reading_gzip_files_compressed_with_different_options() {
_, content1 := read_and_decode_file(s('readme_level_1.gz'))!
_, content5 := read_and_decode_file(s('readme_level_5.gz'))!
_, content9 := read_and_decode_file(s('readme_level_9.gz'))!
_, content9_rsyncable := read_and_decode_file(s('readme_level_9_rsyncable.gz'))!
assert content9_rsyncable == content9
assert content9 == content5
assert content5 == content1
}

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.