From acf2df1a09361de7bf85895ed6433ada092523e5 Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Tue, 20 May 2025 18:45:36 +0200 Subject: [PATCH] wip: compressor memory usage --- include/dwarfs/block_compressor.h | 6 + src/compression/brotli.cpp | 508 ++++++++++++++++++++++++++++++ src/compression/flac.cpp | 5 + src/compression/lz4.cpp | 8 + src/compression/lzma.cpp | 7 + src/compression/null.cpp | 5 + src/compression/ricepp.cpp | 5 + src/compression/zstd.cpp | 69 ++++ 8 files changed, 613 insertions(+) diff --git a/include/dwarfs/block_compressor.h b/include/dwarfs/block_compressor.h index 26492ba1..b0831d0f 100644 --- a/include/dwarfs/block_compressor.h +++ b/include/dwarfs/block_compressor.h @@ -78,6 +78,10 @@ class block_compressor { return impl_->get_compression_constraints(metadata); } + size_t estimate_memory_usage(size_t data_size) const { + return impl_->estimate_memory_usage(data_size); + } + explicit operator bool() const { return static_cast(impl_); } class impl { @@ -96,6 +100,8 @@ class block_compressor { virtual compression_constraints get_compression_constraints(std::string const& metadata) const = 0; + + virtual size_t estimate_memory_usage(size_t data_size) const = 0; }; private: diff --git a/src/compression/brotli.cpp b/src/compression/brotli.cpp index c0cbfe6f..618f75a0 100644 --- a/src/compression/brotli.cpp +++ b/src/compression/brotli.cpp @@ -26,6 +26,7 @@ * SPDX-License-Identifier: MIT */ +#include #include #include @@ -47,6 +48,495 @@ namespace dwarfs { namespace { +#ifndef BROTLI_BUILD_ENC_EXTRA_API +constexpr size_t kBrotliMemUsageGranularity = 262144; + +constexpr std::array brotli_memory_usage{ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, + 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 1, 1, 1, 1, 1, 2, + 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 1, 1, 1, 1, 1, 2, 3, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 1, 1, 1, 1, 1, 2, 3, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 1, 1, 1, 1, 1, 2, 3, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 1, 1, 1, 1, 1, 2, 3, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 1, 1, 1, 1, 1, 2, 3, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 1, + 1, 1, 1, 1, 2, 3, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 1, 1, + 1, 1, 1, 2, 3, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 1, 1, 1, + 1, 1, 2, 3, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 1, 1, 1, 1, + 1, 2, 3, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 1, 1, 1, 1, 1, + 2, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 1, 1, 1, 1, 1, 2, + 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 1, 1, 1, 1, 1, 2, 3, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 1, 1, 1, 1, 1, 2, 3, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 2, 2, 2, 2, 2, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, + 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, + 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, + 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, + 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 2, 2, 2, 2, 3, + 3, 4, 5, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 2, 2, 2, 2, 4, 4, + 5, 6, 7, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 2, 2, 2, 2, 6, 6, 7, + 8, 9, 11, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 2, 2, 2, 2, 10, 10, 11, 12, + 13, 15, 19, 27, 27, 27, 27, 27, 27, 27, 27, + 27, 27, 2, 2, 2, 2, 18, 18, 19, 20, 21, + 23, 27, 35, 51, 51, 51, 51, 51, 51, 51, 51, + 51, 2, 2, 2, 2, 34, 34, 35, 36, 37, 39, + 43, 51, 67, 99, 99, 99, 99, 99, 99, 99, 99, + 2, 2, 2, 2, 66, 66, 67, 68, 69, 71, 75, + 83, 99, 131, 195, 195, 195, 195, 195, 195, 195, 2, + 2, 2, 2, 130, 130, 131, 132, 133, 135, 139, 147, + 163, 195, 259, 259, 259, 259, 259, 259, 259, 2, 2, + 2, 2, 130, 130, 131, 132, 133, 135, 139, 147, 163, + 195, 259, 259, 259, 259, 259, 259, 259, 2, 2, 2, + 2, 130, 130, 131, 132, 133, 135, 139, 147, 163, 195, + 259, 259, 259, 259, 259, 259, 259, 2, 2, 2, 2, + 130, 130, 131, 132, 133, 135, 139, 147, 163, 195, 259, + 259, 259, 259, 259, 259, 259, 2, 2, 2, 2, 130, + 130, 131, 132, 133, 135, 139, 147, 163, 195, 259, 259, + 259, 259, 259, 259, 259, 2, 2, 2, 2, 130, 130, + 131, 132, 133, 135, 139, 147, 163, 195, 259, 259, 259, + 259, 259, 259, 259, 2, 2, 2, 2, 130, 130, 131, + 132, 133, 135, 139, 147, 163, 195, 259, 259, 259, 259, + 259, 259, 259, 2, 2, 2, 2, 2, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 2, 2, 2, 2, 2, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, + 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, + 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, + 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 2, 2, 2, 2, + 3, 3, 4, 5, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 2, 2, 2, 2, 4, + 4, 5, 6, 7, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 2, 2, 2, 2, 6, 6, + 7, 8, 9, 11, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 2, 2, 2, 2, 10, 10, 11, + 12, 13, 15, 19, 27, 27, 27, 27, 27, 27, 27, + 27, 27, 27, 2, 2, 2, 2, 18, 18, 19, 20, + 21, 23, 27, 35, 51, 51, 51, 51, 51, 51, 51, + 51, 51, 2, 2, 2, 2, 34, 34, 35, 36, 37, + 39, 43, 51, 67, 99, 99, 99, 99, 99, 99, 99, + 99, 2, 2, 2, 2, 66, 66, 67, 68, 69, 71, + 75, 83, 99, 131, 195, 195, 195, 195, 195, 195, 195, + 2, 2, 2, 2, 130, 130, 131, 132, 133, 135, 139, + 147, 163, 195, 259, 259, 259, 259, 259, 259, 259, 258, + 258, 258, 258, 514, 514, 515, 516, 517, 519, 523, 531, + 547, 579, 643, 643, 643, 643, 643, 643, 643, 258, 258, + 258, 258, 770, 770, 771, 772, 773, 775, 779, 787, 803, + 835, 899, 899, 899, 899, 899, 899, 899, 258, 258, 258, + 258, 1282, 1282, 1283, 1284, 1285, 1287, 1291, 1299, 1315, 1347, + 1411, 1411, 1411, 1411, 1411, 1411, 1411, 258, 258, 258, 258, + 2306, 2306, 2307, 2308, 2309, 2311, 2315, 2323, 2339, 2371, 2435, + 2435, 2435, 2435, 2435, 2435, 2435, 258, 258, 258, 258, 4354, + 4354, 4355, 4356, 4357, 4359, 4363, 4371, 4387, 4419, 4483, 4483, + 4483, 4483, 4483, 4483, 4483, 258, 258, 258, 258, 8450, 8450, + 8451, 8452, 8453, 8455, 8459, 8467, 8483, 8515, 8579, 8579, 8579, + 8579, 8579, 8579, 8579, 3, 3, 3, 3, 3, 4, 6, + 7, 7, 7, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 3, 3, 3, 3, 3, 4, 6, 7, + 7, 7, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 3, 3, 3, 3, 3, 4, 6, 7, 7, + 7, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 3, 3, 3, 3, 3, 4, 6, 7, 7, 7, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 3, 3, 3, 3, 3, 4, 6, 7, 7, 7, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 3, + 3, 3, 3, 3, 4, 6, 7, 7, 7, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 3, 3, + 3, 3, 3, 4, 6, 7, 7, 7, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 3, 3, 3, + 3, 3, 4, 6, 8, 10, 10, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 3, 3, 3, 3, + 3, 4, 7, 9, 11, 16, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 3, 3, 3, 3, 3, + 4, 9, 11, 13, 18, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 3, 3, 3, 3, 3, 4, + 13, 15, 17, 22, 46, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 3, 3, 3, 3, 3, 4, 21, + 23, 25, 30, 54, 72, 104, 104, 104, 104, 104, 104, + 104, 104, 104, 3, 3, 3, 3, 3, 4, 37, 39, + 41, 46, 70, 88, 120, 184, 184, 184, 184, 184, 184, + 184, 184, 3, 3, 3, 3, 3, 4, 69, 71, 73, + 78, 102, 120, 152, 216, 344, 344, 344, 344, 344, 344, + 344, 3, 3, 3, 3, 3, 4, 133, 135, 137, 142, + 166, 184, 216, 280, 408, 408, 408, 408, 408, 408, 408, + 3, 3, 3, 3, 3, 4, 261, 263, 265, 270, 550, + 568, 600, 664, 792, 792, 792, 792, 792, 792, 792, 3, + 3, 3, 3, 3, 4, 517, 519, 521, 526, 806, 824, + 856, 920, 1048, 1048, 1048, 1048, 1048, 1048, 1048, 3, 3, + 3, 3, 3, 4, 1029, 1031, 1033, 1038, 1318, 1336, 1368, + 1432, 1560, 1560, 1560, 1560, 1560, 1560, 1560, 3, 3, 3, + 3, 3, 4, 2053, 2055, 2057, 2062, 2342, 2360, 2392, 2456, + 2584, 2584, 2584, 2584, 2584, 2584, 2584, 3, 3, 3, 3, + 3, 4, 4101, 4103, 4105, 4110, 4390, 4408, 4440, 4504, 4632, + 4632, 4632, 4632, 4632, 4632, 4632, 3, 3, 3, 3, 3, + 4, 8197, 8199, 8201, 8206, 8486, 8504, 8536, 8600, 8728, 8728, + 8728, 8728, 8728, 8728, 8728, 3, 3, 3, 3, 3, 4, + 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 3, 3, 3, 3, 3, 4, 6, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 3, 3, 3, 3, 3, 4, 6, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 3, 3, 3, 3, 3, 4, 6, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 3, 3, 3, 3, 3, 4, 6, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 3, 3, 3, 3, 3, 4, 6, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 3, + 3, 3, 3, 3, 4, 6, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 5, 5, + 5, 5, 5, 6, 9, 10, 12, 12, 12, 12, 12, + 12, 12, 12, 12, 12, 12, 12, 12, 5, 5, 5, + 5, 5, 6, 10, 11, 13, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 5, 5, 5, 5, + 5, 6, 12, 13, 15, 20, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 5, 5, 5, 5, 5, + 6, 16, 17, 19, 24, 38, 56, 56, 56, 56, 56, + 56, 56, 56, 56, 56, 5, 5, 5, 5, 5, 6, + 24, 25, 27, 32, 46, 64, 96, 96, 96, 96, 96, + 96, 96, 96, 96, 5, 5, 5, 5, 5, 6, 40, + 41, 43, 48, 62, 80, 112, 176, 176, 176, 176, 176, + 176, 176, 176, 5, 5, 5, 5, 5, 6, 72, 73, + 75, 80, 94, 112, 144, 208, 336, 336, 336, 336, 336, + 336, 336, 5, 5, 5, 5, 5, 6, 136, 137, 139, + 144, 158, 176, 208, 272, 400, 400, 400, 400, 400, 400, + 400, 5, 5, 5, 5, 5, 6, 264, 265, 267, 272, + 542, 560, 592, 656, 784, 784, 784, 784, 784, 784, 784, + 5, 5, 5, 5, 5, 6, 520, 521, 523, 528, 798, + 816, 848, 912, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 5, + 5, 5, 5, 5, 6, 1032, 1033, 1035, 1040, 1310, 1328, + 1360, 1424, 1552, 1552, 1552, 1552, 1552, 1552, 1552, 5, 5, + 5, 5, 5, 6, 2056, 2057, 2059, 2064, 2334, 2352, 2384, + 2448, 2576, 2576, 2576, 2576, 2576, 2576, 2576, 5, 5, 5, + 5, 5, 6, 4104, 4105, 4107, 4112, 4382, 4400, 4432, 4496, + 4624, 4624, 4624, 4624, 4624, 4624, 4624, 5, 5, 5, 5, + 5, 6, 8200, 8201, 8203, 8208, 8478, 8496, 8528, 8592, 8720, + 8720, 8720, 8720, 8720, 8720, 8720, 3, 3, 3, 3, 3, + 4, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 3, 3, 3, 3, 3, 4, + 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 3, 3, 3, 3, 3, 4, 6, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 3, 3, 3, 3, 3, 4, 6, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 3, 3, 3, 3, 3, 4, 6, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 3, 3, 3, 3, 3, 4, 6, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 3, 3, 3, 3, 3, 4, 6, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 9, + 9, 9, 9, 9, 10, 13, 14, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 9, 9, + 9, 9, 9, 10, 14, 15, 17, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 9, 9, 9, + 9, 9, 10, 16, 17, 19, 24, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 9, 9, 9, 9, + 9, 10, 20, 21, 23, 28, 46, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 9, 9, 9, 9, 9, + 10, 28, 29, 31, 36, 54, 72, 104, 104, 104, 104, + 104, 104, 104, 104, 104, 9, 9, 9, 9, 9, 10, + 44, 45, 47, 52, 70, 88, 120, 184, 184, 184, 184, + 184, 184, 184, 184, 9, 9, 9, 9, 9, 10, 76, + 77, 79, 84, 102, 120, 152, 216, 344, 344, 344, 344, + 344, 344, 344, 9, 9, 9, 9, 9, 10, 140, 141, + 143, 148, 166, 184, 216, 280, 408, 408, 408, 408, 408, + 408, 408, 9, 9, 9, 9, 9, 10, 268, 269, 271, + 276, 550, 568, 600, 664, 792, 792, 792, 792, 792, 792, + 792, 9, 9, 9, 9, 9, 10, 524, 525, 527, 532, + 806, 824, 856, 920, 1048, 1048, 1048, 1048, 1048, 1048, 1048, + 9, 9, 9, 9, 9, 10, 1036, 1037, 1039, 1044, 1318, + 1336, 1368, 1432, 1560, 1560, 1560, 1560, 1560, 1560, 1560, 9, + 9, 9, 9, 9, 10, 2060, 2061, 2063, 2068, 2342, 2360, + 2392, 2456, 2584, 2584, 2584, 2584, 2584, 2584, 2584, 9, 9, + 9, 9, 9, 10, 4108, 4109, 4111, 4116, 4390, 4408, 4440, + 4504, 4632, 4632, 4632, 4632, 4632, 4632, 4632, 9, 9, 9, + 9, 9, 10, 8204, 8205, 8207, 8212, 8486, 8504, 8536, 8600, + 8728, 8728, 8728, 8728, 8728, 8728, 8728, 3, 3, 3, 3, + 3, 4, 6, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 3, 3, 3, 3, 3, + 4, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 3, 3, 3, 3, 3, 4, + 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 3, 3, 3, 3, 3, 4, 6, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 3, 3, 3, 3, 3, 4, 6, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 3, 3, 3, 3, 3, 4, 6, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 3, 3, 3, 3, 3, 4, 6, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 33, 33, 33, 33, 33, 34, 37, 38, 40, 40, 40, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 33, + 33, 33, 33, 33, 34, 38, 39, 41, 46, 46, 46, + 46, 46, 46, 46, 46, 46, 46, 46, 46, 33, 33, + 33, 33, 33, 34, 40, 41, 43, 48, 58, 58, 58, + 58, 58, 58, 58, 58, 58, 58, 58, 33, 33, 33, + 33, 33, 34, 44, 45, 47, 52, 62, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 33, 33, 33, 33, + 33, 34, 52, 53, 55, 60, 70, 88, 120, 120, 120, + 120, 120, 120, 120, 120, 120, 33, 33, 33, 33, 33, + 34, 68, 69, 71, 76, 86, 104, 136, 200, 200, 200, + 200, 200, 200, 200, 200, 33, 33, 33, 33, 33, 34, + 100, 101, 103, 108, 118, 136, 168, 232, 360, 360, 360, + 360, 360, 360, 360, 33, 33, 33, 33, 33, 34, 164, + 165, 167, 172, 182, 200, 232, 296, 424, 424, 424, 424, + 424, 424, 424, 33, 33, 33, 33, 33, 34, 292, 293, + 295, 300, 566, 584, 616, 680, 808, 808, 808, 808, 808, + 808, 808, 33, 33, 33, 33, 33, 34, 548, 549, 551, + 556, 822, 840, 872, 936, 1064, 1064, 1064, 1064, 1064, 1064, + 1064, 33, 33, 33, 33, 33, 34, 1060, 1061, 1063, 1068, + 1334, 1352, 1384, 1448, 1576, 1576, 1576, 1576, 1576, 1576, 1576, + 33, 33, 33, 33, 33, 34, 2084, 2085, 2087, 2092, 2358, + 2376, 2408, 2472, 2600, 2600, 2600, 2600, 2600, 2600, 2600, 33, + 33, 33, 33, 33, 34, 4132, 4133, 4135, 4140, 4406, 4424, + 4456, 4520, 4648, 4648, 4648, 4648, 4648, 4648, 4648, 33, 33, + 33, 33, 33, 34, 8228, 8229, 8231, 8236, 8502, 8520, 8552, + 8616, 8744, 8744, 8744, 8744, 8744, 8744, 8744, 3, 3, 3, + 3, 3, 4, 6, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 3, 3, 3, 3, + 3, 4, 6, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 3, 3, 3, 3, 3, + 4, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 3, 3, 3, 3, 3, 4, + 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 3, 3, 3, 3, 3, 4, 6, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 3, 3, 3, 3, 3, 4, 6, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 3, 3, 3, 3, 3, 4, 6, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 65, 65, 65, 65, 65, 66, 69, 70, 72, 72, + 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, + 65, 65, 65, 65, 65, 66, 70, 71, 73, 78, 78, + 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 65, + 65, 65, 65, 65, 66, 72, 73, 75, 80, 90, 90, + 90, 90, 90, 90, 90, 90, 90, 90, 90, 65, 65, + 65, 65, 65, 66, 76, 77, 79, 84, 94, 112, 112, + 112, 112, 112, 112, 112, 112, 112, 112, 65, 65, 65, + 65, 65, 66, 84, 85, 87, 92, 102, 120, 152, 152, + 152, 152, 152, 152, 152, 152, 152, 65, 65, 65, 65, + 65, 66, 100, 101, 103, 108, 118, 136, 168, 232, 232, + 232, 232, 232, 232, 232, 232, 65, 65, 65, 65, 65, + 66, 132, 133, 135, 140, 150, 168, 200, 264, 392, 392, + 392, 392, 392, 392, 392, 65, 65, 65, 65, 65, 66, + 196, 197, 199, 204, 214, 232, 264, 328, 456, 456, 456, + 456, 456, 456, 456, 65, 65, 65, 65, 65, 66, 324, + 325, 327, 332, 598, 616, 648, 712, 840, 840, 840, 840, + 840, 840, 840, 65, 65, 65, 65, 65, 66, 580, 581, + 583, 588, 854, 872, 904, 968, 1096, 1096, 1096, 1096, 1096, + 1096, 1096, 65, 65, 65, 65, 65, 66, 1092, 1093, 1095, + 1100, 1366, 1384, 1416, 1480, 1608, 1608, 1608, 1608, 1608, 1608, + 1608, 65, 65, 65, 65, 65, 66, 2116, 2117, 2119, 2124, + 2390, 2408, 2440, 2504, 2632, 2632, 2632, 2632, 2632, 2632, 2632, + 65, 65, 65, 65, 65, 66, 4164, 4165, 4167, 4172, 4438, + 4456, 4488, 4552, 4680, 4680, 4680, 4680, 4680, 4680, 4680, 65, + 65, 65, 65, 65, 66, 8260, 8261, 8263, 8268, 8534, 8552, + 8584, 8648, 8776, 8776, 8776, 8776, 8776, 8776, 8776, 6, 6, + 6, 6, 6, 7, 9, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 10, 10, 6, 6, 6, + 6, 6, 7, 9, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 10, 6, 6, 6, 6, + 6, 7, 9, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 6, 6, 6, 6, 6, + 7, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 10, 10, 6, 6, 6, 6, 6, 7, + 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 10, 6, 6, 6, 6, 6, 7, 9, + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 6, 6, 6, 6, 6, 7, 9, 10, + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 129, 129, 129, 129, 129, 130, 132, 136, 138, + 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, + 138, 129, 129, 129, 129, 129, 130, 132, 135, 143, 148, + 148, 148, 148, 148, 148, 148, 148, 148, 148, 148, 148, + 129, 129, 129, 129, 129, 130, 132, 135, 145, 150, 160, + 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 129, + 129, 129, 129, 129, 130, 132, 135, 149, 154, 164, 182, + 182, 182, 182, 182, 182, 182, 182, 182, 182, 129, 129, + 129, 129, 129, 130, 132, 135, 157, 162, 172, 190, 222, + 222, 222, 222, 222, 222, 222, 222, 222, 129, 129, 129, + 129, 129, 130, 132, 135, 173, 178, 188, 206, 238, 302, + 302, 302, 302, 302, 302, 302, 302, 129, 129, 129, 129, + 129, 130, 132, 135, 205, 210, 220, 238, 270, 334, 462, + 462, 462, 462, 462, 462, 462, 129, 129, 129, 129, 129, + 130, 132, 135, 269, 274, 284, 302, 334, 398, 526, 526, + 526, 526, 526, 526, 526, 129, 129, 129, 129, 129, 130, + 132, 135, 397, 402, 668, 686, 718, 782, 910, 910, 910, + 910, 910, 910, 910, 129, 129, 129, 129, 129, 130, 132, + 135, 653, 658, 924, 942, 974, 1038, 1166, 1166, 1166, 1166, + 1166, 1166, 1166, 129, 129, 129, 129, 129, 130, 132, 135, + 1165, 1170, 1436, 1454, 1486, 1550, 1678, 1678, 1678, 1678, 1678, + 1678, 1678, 129, 129, 129, 129, 129, 130, 132, 135, 2189, + 2194, 2460, 2478, 2510, 2574, 2702, 2702, 2702, 2702, 2702, 2702, + 2702, 129, 129, 129, 129, 129, 130, 132, 135, 4237, 4242, + 4508, 4526, 4558, 4622, 4750, 4750, 4750, 4750, 4750, 4750, 4750, + 129, 129, 129, 129, 129, 130, 132, 135, 8333, 8338, 8604, + 8622, 8654, 8718, 8846, 8846, 8846, 8846, 8846, 8846, 8846, 803, + 803, 803, 803, 803, 804, 806, 807, 807, 807, 807, 807, + 807, 807, 807, 807, 807, 807, 807, 807, 807, 803, 803, + 803, 803, 803, 804, 806, 807, 807, 807, 807, 807, 807, + 807, 807, 807, 807, 807, 807, 807, 807, 803, 803, 803, + 803, 803, 804, 806, 807, 807, 807, 807, 807, 807, 807, + 807, 807, 807, 807, 807, 807, 807, 803, 803, 803, 803, + 803, 804, 806, 807, 807, 807, 807, 807, 807, 807, 807, + 807, 807, 807, 807, 807, 807, 803, 803, 803, 803, 804, + 804, 806, 807, 807, 807, 807, 807, 807, 807, 807, 807, + 807, 807, 807, 807, 807, 803, 803, 803, 803, 804, 805, + 807, 808, 808, 808, 808, 808, 808, 808, 808, 808, 808, + 808, 808, 808, 808, 803, 803, 803, 803, 804, 805, 808, + 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, + 809, 809, 809, 803, 803, 803, 803, 804, 805, 807, 813, + 815, 815, 815, 815, 815, 815, 815, 815, 815, 815, 815, + 815, 815, 803, 803, 803, 803, 804, 805, 807, 812, 824, + 828, 828, 828, 828, 828, 828, 828, 828, 828, 828, 828, + 828, 803, 803, 803, 803, 804, 805, 807, 812, 826, 838, + 846, 846, 846, 846, 846, 846, 846, 846, 846, 846, 846, + 803, 803, 803, 803, 804, 805, 807, 812, 830, 842, 866, + 882, 882, 882, 882, 882, 882, 882, 882, 882, 882, 803, + 803, 803, 803, 804, 805, 807, 812, 838, 850, 874, 922, + 954, 954, 954, 954, 954, 954, 954, 954, 954, 803, 803, + 803, 803, 804, 805, 807, 812, 854, 866, 890, 938, 1034, + 1098, 1098, 1098, 1098, 1098, 1098, 1098, 1098, 803, 803, 803, + 803, 804, 805, 807, 812, 886, 898, 922, 970, 1066, 1258, + 1386, 1386, 1386, 1386, 1386, 1386, 1386, 803, 803, 803, 803, + 804, 805, 807, 812, 950, 962, 986, 1034, 1130, 1322, 1706, + 1706, 1706, 1706, 1706, 1706, 1706, 803, 803, 803, 803, 804, + 805, 807, 812, 1078, 1090, 1114, 1162, 1258, 1450, 1834, 2346, + 2346, 2346, 2346, 2346, 2346, 803, 803, 803, 803, 804, 805, + 807, 812, 1334, 1346, 1370, 1418, 1514, 1706, 2090, 2602, 3626, + 3626, 3626, 3626, 3626, 803, 803, 803, 803, 804, 805, 807, + 812, 1846, 1858, 1882, 1930, 2026, 2218, 2602, 3114, 4138, 6186, + 6186, 6186, 6186, 803, 803, 803, 803, 804, 805, 807, 812, + 2870, 2882, 2906, 2954, 3050, 3242, 3626, 4138, 5162, 7210, 11306, + 11306, 11306, 803, 803, 803, 803, 804, 805, 807, 812, 4918, + 4930, 4954, 5002, 5098, 5290, 5674, 6186, 7210, 9258, 13354, 21546, + 21546, 803, 803, 803, 803, 804, 805, 807, 812, 9014, 9026, + 9050, 9098, 9194, 9386, 9770, 10282, 11306, 13354, 17450, 25642, 42026, + 803, 803, 803, 803, 803, 804, 806, 807, 807, 807, 807, + 807, 807, 807, 807, 807, 807, 807, 807, 807, 807, 803, + 803, 803, 803, 803, 804, 806, 807, 807, 807, 807, 807, + 807, 807, 807, 807, 807, 807, 807, 807, 807, 803, 803, + 803, 803, 803, 804, 806, 807, 807, 807, 807, 807, 807, + 807, 807, 807, 807, 807, 807, 807, 807, 803, 803, 803, + 803, 803, 804, 806, 807, 807, 807, 807, 807, 807, 807, + 807, 807, 807, 807, 807, 807, 807, 803, 803, 803, 803, + 804, 804, 806, 807, 807, 807, 807, 807, 807, 807, 807, + 807, 807, 807, 807, 807, 807, 803, 803, 803, 803, 804, + 805, 807, 808, 808, 808, 808, 808, 808, 808, 808, 808, + 808, 808, 808, 808, 808, 803, 803, 803, 803, 804, 805, + 808, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, + 809, 809, 809, 809, 803, 803, 803, 803, 804, 805, 807, + 813, 815, 815, 815, 815, 815, 815, 815, 815, 815, 815, + 815, 815, 815, 803, 803, 803, 803, 804, 805, 807, 812, + 824, 828, 828, 828, 828, 828, 828, 828, 828, 828, 828, + 828, 828, 803, 803, 803, 803, 804, 805, 807, 812, 826, + 838, 846, 846, 846, 846, 846, 846, 846, 846, 846, 846, + 846, 803, 803, 803, 803, 804, 805, 807, 812, 830, 842, + 866, 882, 882, 882, 882, 882, 882, 882, 882, 882, 882, + 803, 803, 803, 803, 804, 805, 807, 812, 838, 850, 874, + 922, 954, 954, 954, 954, 954, 954, 954, 954, 954, 803, + 803, 803, 803, 804, 805, 807, 812, 854, 866, 890, 938, + 1034, 1098, 1098, 1098, 1098, 1098, 1098, 1098, 1098, 803, 803, + 803, 803, 804, 805, 807, 812, 886, 898, 922, 970, 1066, + 1258, 1386, 1386, 1386, 1386, 1386, 1386, 1386, 803, 803, 803, + 803, 804, 805, 807, 812, 950, 962, 986, 1034, 1130, 1322, + 1706, 1706, 1706, 1706, 1706, 1706, 1706, 803, 803, 803, 803, + 804, 805, 807, 812, 1078, 1090, 1114, 1162, 1258, 1450, 1834, + 2346, 2346, 2346, 2346, 2346, 2346, 803, 803, 803, 803, 804, + 805, 807, 812, 1334, 1346, 1370, 1418, 1514, 1706, 2090, 2602, + 3626, 3626, 3626, 3626, 3626, 803, 803, 803, 803, 804, 805, + 807, 812, 1846, 1858, 1882, 1930, 2026, 2218, 2602, 3114, 4138, + 6186, 6186, 6186, 6186, 803, 803, 803, 803, 804, 805, 807, + 812, 2870, 2882, 2906, 2954, 3050, 3242, 3626, 4138, 5162, 7210, + 11306, 11306, 11306, 803, 803, 803, 803, 804, 805, 807, 812, + 4918, 4930, 4954, 5002, 5098, 5290, 5674, 6186, 7210, 9258, 13354, + 21546, 21546, 803, 803, 803, 803, 804, 805, 807, 812, 9014, + 9026, 9050, 9098, 9194, 9386, 9770, 10282, 11306, 13354, 17450, 25642, + 42026, +}; +#endif + class brotli_block_compressor final : public block_compressor::impl { public: brotli_block_compressor(uint32_t quality, uint32_t window_bits) @@ -92,6 +582,24 @@ class brotli_block_compressor final : public block_compressor::impl { return {}; } + size_t estimate_memory_usage(size_t data_size) const override { +#ifdef BROTLI_BUILD_ENC_EXTRA_API + return ::BrotliEncoderEstimatePeakMemoryUsage(quality_, window_bits_, + data_size) + + data_size; +#else + auto const q = std::clamp(quality_, 0, 11); + auto const lg_win = std::clamp(window_bits_, 10, 30) - 10; + auto const lg_data_size = + std::clamp(static_cast(std::ceil(std::log2(data_size))), 10, + 30) - + 10; + auto const index = (q * 21 + lg_win) * 21 + lg_data_size; + return kBrotliMemUsageGranularity * brotli_memory_usage.at(index) + + data_size; +#endif + } + private: uint32_t const quality_; uint32_t const window_bits_; diff --git a/src/compression/flac.cpp b/src/compression/flac.cpp index e854fbe6..1bbd62d2 100644 --- a/src/compression/flac.cpp +++ b/src/compression/flac.cpp @@ -393,6 +393,11 @@ class flac_block_compressor final : public block_compressor::impl { return cc; } + size_t estimate_memory_usage(size_t data_size) const override { + // TODO: can we estimate the FLAC encoder state size? + return data_size; + } + private: uint32_t const level_; bool const exhaustive_; diff --git a/src/compression/lz4.cpp b/src/compression/lz4.cpp index 74a6ce7d..c23d53c3 100644 --- a/src/compression/lz4.cpp +++ b/src/compression/lz4.cpp @@ -54,6 +54,8 @@ struct lz4_compression_policy { } static std::string describe(int /*level*/) { return "lz4"; } + + static size_t state_size(size_t /*data_size*/) { return LZ4_sizeofState(); } }; struct lz4hc_compression_policy { @@ -67,6 +69,8 @@ struct lz4hc_compression_policy { static std::string describe(int level) { return fmt::format("lz4hc [level={}]", level); } + + static size_t state_size(size_t /*data_size*/) { return LZ4_sizeofStateHC(); } }; template @@ -113,6 +117,10 @@ class lz4_block_compressor final : public block_compressor::impl { return {}; } + size_t estimate_memory_usage(size_t data_size) const override { + return Policy::state_size(data_size) + data_size; + } + private: int const level_; }; diff --git a/src/compression/lzma.cpp b/src/compression/lzma.cpp index d13dc673..61149639 100644 --- a/src/compression/lzma.cpp +++ b/src/compression/lzma.cpp @@ -136,6 +136,13 @@ class lzma_block_compressor final : public block_compressor::impl { return {}; } + size_t estimate_memory_usage(size_t data_size) const override { + auto lzma_opts = opt_lzma_; + std::array filters{ + {{LZMA_FILTER_LZMA2, &lzma_opts}, {LZMA_VLI_UNKNOWN, nullptr}}}; + return lzma_raw_encoder_memusage(filters.data()) + data_size; + } + private: shared_byte_buffer compress(shared_byte_buffer const& data, lzma_filter const* filters) const; diff --git a/src/compression/null.cpp b/src/compression/null.cpp index 4ec7ab2e..137e7367 100644 --- a/src/compression/null.cpp +++ b/src/compression/null.cpp @@ -66,6 +66,11 @@ class null_block_compressor final : public block_compressor::impl { get_compression_constraints(std::string const&) const override { return {}; } + + size_t estimate_memory_usage(size_t /*data_size*/) const override { + // we're re-using the input buffer, so we don't need any extra memory + return 0; + } }; class null_block_decompressor final : public block_decompressor_base { diff --git a/src/compression/ricepp.cpp b/src/compression/ricepp.cpp index f78e4600..7260d8b1 100644 --- a/src/compression/ricepp.cpp +++ b/src/compression/ricepp.cpp @@ -171,6 +171,11 @@ class ricepp_block_compressor final : public block_compressor::impl { return cc; } + size_t estimate_memory_usage(size_t data_size) const override { + // ricepp encoder basically has no memory overhead by itself + return data_size; + } + private: size_t const block_size_; }; diff --git a/src/compression/zstd.cpp b/src/compression/zstd.cpp index feeae7b8..93820709 100644 --- a/src/compression/zstd.cpp +++ b/src/compression/zstd.cpp @@ -50,6 +50,59 @@ namespace dwarfs { namespace { +#ifndef ZSTD_STATIC_LINKING_ONLY + +constexpr size_t kZstdMemUsageGranularity = 16384; + +constexpr std::array zstd_memory_usage{ + 3, 4, 6, 10, 18, 34, 41, 56, 64, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 3, + 4, 5, 8, 14, 12, 19, 34, 36, 36, 36, 36, + 36, 36, 36, 36, 36, 36, 36, 36, 36, 3, 4, + 5, 8, 14, 18, 25, 40, 40, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 3, 4, 6, + 10, 18, 34, 41, 56, 64, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 3, 4, 6, 10, + 14, 34, 65, 96, 80, 160, 160, 160, 160, 160, 160, + 160, 160, 160, 160, 160, 160, 3, 4, 6, 10, 14, + 34, 65, 80, 128, 224, 224, 224, 224, 224, 224, 224, + 224, 224, 224, 224, 224, 3, 4, 6, 10, 14, 34, + 65, 80, 224, 224, 224, 224, 224, 224, 224, 224, 224, + 224, 224, 224, 224, 3, 4, 6, 10, 14, 34, 65, + 80, 224, 416, 416, 416, 416, 416, 416, 416, 416, 416, + 416, 416, 416, 3, 4, 6, 10, 14, 34, 65, 80, + 224, 416, 416, 416, 416, 416, 416, 416, 416, 416, 416, + 416, 416, 3, 5, 7, 12, 18, 34, 65, 80, 224, + 416, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, + 800, 3, 5, 7, 12, 18, 34, 65, 80, 224, 416, + 800, 1568, 1568, 1568, 1568, 1568, 1568, 1568, 1568, 1568, 1568, + 12, 14, 16, 21, 27, 42, 81, 96, 224, 416, 800, + 1568, 1568, 1568, 1568, 1568, 1568, 1568, 1568, 1568, 1568, 13, + 14, 17, 24, 32, 42, 81, 128, 288, 416, 800, 1568, + 3104, 3104, 3104, 3104, 3104, 3104, 3104, 3104, 3104, 13, 14, + 17, 24, 32, 51, 90, 137, 233, 544, 1056, 2080, 2080, + 2080, 2080, 2080, 2080, 2080, 2080, 2080, 2080, 13, 14, 17, + 24, 36, 61, 110, 177, 273, 544, 1056, 2080, 3104, 3104, + 3104, 3104, 3104, 3104, 3104, 3104, 3104, 13, 14, 17, 24, + 36, 61, 110, 177, 273, 544, 1056, 2080, 4128, 4128, 4128, + 4128, 4128, 4128, 4128, 4128, 4128, 13, 14, 17, 24, 36, + 61, 110, 177, 337, 553, 1065, 2089, 2089, 2089, 2089, 2089, + 2089, 2089, 2089, 2089, 2089, 13, 14, 17, 24, 36, 61, + 110, 177, 337, 553, 1065, 2089, 3113, 3113, 3113, 3113, 3113, + 3113, 3113, 3113, 3113, 13, 14, 17, 24, 36, 61, 110, + 177, 337, 593, 1105, 2129, 3153, 3153, 3153, 3153, 3153, 3153, + 3153, 3153, 3153, 13, 14, 17, 24, 36, 61, 110, 177, + 337, 593, 1105, 2129, 3153, 5201, 5201, 5201, 5201, 5201, 5201, + 5201, 5201, 13, 14, 17, 24, 36, 61, 110, 177, 337, + 593, 1105, 2129, 4177, 6225, 10321, 10321, 10321, 10321, 10321, 10321, + 10321, 13, 14, 17, 24, 36, 61, 110, 177, 337, 593, + 1105, 2129, 4177, 8273, 12369, 20561, 20561, 20561, 20561, 20561, 20561, + 13, 14, 17, 24, 36, 61, 110, 177, 337, 593, 1105, + 2129, 4177, 8273, 16465, 24657, 41041, 41562, 41562, 41562, 41562, +}; + +#endif + class zstd_block_compressor final : public block_compressor::impl { public: explicit zstd_block_compressor(int level) @@ -77,6 +130,22 @@ class zstd_block_compressor final : public block_compressor::impl { return {}; } + size_t estimate_memory_usage(size_t data_size) const override { +#ifdef ZSTD_STATIC_LINKING_ONLY + // TODO: check if dictSize == 0 is correct + auto params = ZSTD_getCParams(level_, data_size, 0); + return ZSTD_estimateCCtxSize_usingCParams(params) + data_size; +#else + auto const l = std::clamp(level_, 0, 22); + auto const lg_data_size = + std::clamp(static_cast(std::ceil(std::log2(data_size))), 10, + 30) - + 10; + auto const index = l * 21 + lg_data_size; + return kZstdMemUsageGranularity * zstd_memory_usage.at(index) + data_size; +#endif + } + private: int const level_; };