mirror of
https://github.com/mhx/dwarfs.git
synced 2025-09-09 12:28:13 -04:00
wip: compressor memory usage
This commit is contained in:
Marcus Holland-Moritz
parent
2ea6c2420b
commit
acf2df1a09
@ -78,6 +78,10 @@ class block_compressor {
|
||||
return impl_->get_compression_constraints(metadata);
|
||||
}
|
||||
|
||||
size_t estimate_memory_usage(size_t data_size) const {
|
||||
return impl_->estimate_memory_usage(data_size);
|
||||
}
|
||||
|
||||
explicit operator bool() const { return static_cast<bool>(impl_); }
|
||||
|
||||
class impl {
|
||||
@ -96,6 +100,8 @@ class block_compressor {
|
||||
|
||||
virtual compression_constraints
|
||||
get_compression_constraints(std::string const& metadata) const = 0;
|
||||
|
||||
virtual size_t estimate_memory_usage(size_t data_size) const = 0;
|
||||
};
|
||||
|
||||
private:
|
||||
|
||||
@ -26,6 +26,7 @@
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
|
||||
#include <algorithm>
|
||||
#include <cassert>
|
||||
|
||||
#include <brotli/decode.h>
|
||||
@ -47,6 +48,495 @@ namespace dwarfs {
|
||||
|
||||
namespace {
|
||||
|
||||
#ifndef BROTLI_BUILD_ENC_EXTRA_API
|
||||
constexpr size_t kBrotliMemUsageGranularity = 262144;
|
||||
|
||||
constexpr std::array<uint16_t, 12 * 21 * 21> brotli_memory_usage{
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1,
|
||||
2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
||||
3, 3, 3, 3, 3, 1, 1, 1, 1, 1, 2,
|
||||
3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
|
||||
5, 5, 5, 5, 1, 1, 1, 1, 1, 2, 3,
|
||||
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
|
||||
5, 5, 5, 1, 1, 1, 1, 1, 2, 3, 5,
|
||||
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
|
||||
5, 5, 1, 1, 1, 1, 1, 2, 3, 5, 5,
|
||||
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
|
||||
5, 1, 1, 1, 1, 1, 2, 3, 5, 5, 5,
|
||||
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
|
||||
1, 1, 1, 1, 1, 2, 3, 5, 5, 5, 5,
|
||||
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 1,
|
||||
1, 1, 1, 1, 2, 3, 5, 5, 5, 5, 5,
|
||||
5, 5, 5, 5, 5, 5, 5, 5, 5, 1, 1,
|
||||
1, 1, 1, 2, 3, 5, 5, 5, 5, 5, 5,
|
||||
5, 5, 5, 5, 5, 5, 5, 5, 1, 1, 1,
|
||||
1, 1, 2, 3, 5, 5, 5, 5, 5, 5, 5,
|
||||
5, 5, 5, 5, 5, 5, 5, 1, 1, 1, 1,
|
||||
1, 2, 3, 5, 5, 5, 5, 5, 5, 5, 5,
|
||||
5, 5, 5, 5, 5, 5, 1, 1, 1, 1, 1,
|
||||
2, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5,
|
||||
5, 5, 5, 5, 5, 1, 1, 1, 1, 1, 2,
|
||||
3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
|
||||
5, 5, 5, 5, 1, 1, 1, 1, 1, 2, 3,
|
||||
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
|
||||
5, 5, 5, 1, 1, 1, 1, 1, 2, 3, 5,
|
||||
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
|
||||
5, 5, 2, 2, 2, 2, 2, 3, 3, 3, 3,
|
||||
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
||||
3, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3,
|
||||
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
||||
2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3,
|
||||
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2,
|
||||
2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3,
|
||||
3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2,
|
||||
2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3,
|
||||
3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2,
|
||||
2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
||||
3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2,
|
||||
3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4,
|
||||
4, 4, 4, 4, 4, 4, 2, 2, 2, 2, 3,
|
||||
3, 4, 5, 6, 6, 6, 6, 6, 6, 6, 6,
|
||||
6, 6, 6, 6, 6, 2, 2, 2, 2, 4, 4,
|
||||
5, 6, 7, 9, 9, 9, 9, 9, 9, 9, 9,
|
||||
9, 9, 9, 9, 2, 2, 2, 2, 6, 6, 7,
|
||||
8, 9, 11, 15, 15, 15, 15, 15, 15, 15, 15,
|
||||
15, 15, 15, 2, 2, 2, 2, 10, 10, 11, 12,
|
||||
13, 15, 19, 27, 27, 27, 27, 27, 27, 27, 27,
|
||||
27, 27, 2, 2, 2, 2, 18, 18, 19, 20, 21,
|
||||
23, 27, 35, 51, 51, 51, 51, 51, 51, 51, 51,
|
||||
51, 2, 2, 2, 2, 34, 34, 35, 36, 37, 39,
|
||||
43, 51, 67, 99, 99, 99, 99, 99, 99, 99, 99,
|
||||
2, 2, 2, 2, 66, 66, 67, 68, 69, 71, 75,
|
||||
83, 99, 131, 195, 195, 195, 195, 195, 195, 195, 2,
|
||||
2, 2, 2, 130, 130, 131, 132, 133, 135, 139, 147,
|
||||
163, 195, 259, 259, 259, 259, 259, 259, 259, 2, 2,
|
||||
2, 2, 130, 130, 131, 132, 133, 135, 139, 147, 163,
|
||||
195, 259, 259, 259, 259, 259, 259, 259, 2, 2, 2,
|
||||
2, 130, 130, 131, 132, 133, 135, 139, 147, 163, 195,
|
||||
259, 259, 259, 259, 259, 259, 259, 2, 2, 2, 2,
|
||||
130, 130, 131, 132, 133, 135, 139, 147, 163, 195, 259,
|
||||
259, 259, 259, 259, 259, 259, 2, 2, 2, 2, 130,
|
||||
130, 131, 132, 133, 135, 139, 147, 163, 195, 259, 259,
|
||||
259, 259, 259, 259, 259, 2, 2, 2, 2, 130, 130,
|
||||
131, 132, 133, 135, 139, 147, 163, 195, 259, 259, 259,
|
||||
259, 259, 259, 259, 2, 2, 2, 2, 130, 130, 131,
|
||||
132, 133, 135, 139, 147, 163, 195, 259, 259, 259, 259,
|
||||
259, 259, 259, 2, 2, 2, 2, 2, 3, 3, 3,
|
||||
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
||||
3, 3, 2, 2, 2, 2, 2, 3, 3, 3, 3,
|
||||
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
||||
3, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3,
|
||||
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
||||
2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3,
|
||||
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2,
|
||||
2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3,
|
||||
3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2,
|
||||
2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3,
|
||||
3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2,
|
||||
2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4,
|
||||
4, 4, 4, 4, 4, 4, 4, 2, 2, 2, 2,
|
||||
3, 3, 4, 5, 6, 6, 6, 6, 6, 6, 6,
|
||||
6, 6, 6, 6, 6, 6, 2, 2, 2, 2, 4,
|
||||
4, 5, 6, 7, 9, 9, 9, 9, 9, 9, 9,
|
||||
9, 9, 9, 9, 9, 2, 2, 2, 2, 6, 6,
|
||||
7, 8, 9, 11, 15, 15, 15, 15, 15, 15, 15,
|
||||
15, 15, 15, 15, 2, 2, 2, 2, 10, 10, 11,
|
||||
12, 13, 15, 19, 27, 27, 27, 27, 27, 27, 27,
|
||||
27, 27, 27, 2, 2, 2, 2, 18, 18, 19, 20,
|
||||
21, 23, 27, 35, 51, 51, 51, 51, 51, 51, 51,
|
||||
51, 51, 2, 2, 2, 2, 34, 34, 35, 36, 37,
|
||||
39, 43, 51, 67, 99, 99, 99, 99, 99, 99, 99,
|
||||
99, 2, 2, 2, 2, 66, 66, 67, 68, 69, 71,
|
||||
75, 83, 99, 131, 195, 195, 195, 195, 195, 195, 195,
|
||||
2, 2, 2, 2, 130, 130, 131, 132, 133, 135, 139,
|
||||
147, 163, 195, 259, 259, 259, 259, 259, 259, 259, 258,
|
||||
258, 258, 258, 514, 514, 515, 516, 517, 519, 523, 531,
|
||||
547, 579, 643, 643, 643, 643, 643, 643, 643, 258, 258,
|
||||
258, 258, 770, 770, 771, 772, 773, 775, 779, 787, 803,
|
||||
835, 899, 899, 899, 899, 899, 899, 899, 258, 258, 258,
|
||||
258, 1282, 1282, 1283, 1284, 1285, 1287, 1291, 1299, 1315, 1347,
|
||||
1411, 1411, 1411, 1411, 1411, 1411, 1411, 258, 258, 258, 258,
|
||||
2306, 2306, 2307, 2308, 2309, 2311, 2315, 2323, 2339, 2371, 2435,
|
||||
2435, 2435, 2435, 2435, 2435, 2435, 258, 258, 258, 258, 4354,
|
||||
4354, 4355, 4356, 4357, 4359, 4363, 4371, 4387, 4419, 4483, 4483,
|
||||
4483, 4483, 4483, 4483, 4483, 258, 258, 258, 258, 8450, 8450,
|
||||
8451, 8452, 8453, 8455, 8459, 8467, 8483, 8515, 8579, 8579, 8579,
|
||||
8579, 8579, 8579, 8579, 3, 3, 3, 3, 3, 4, 6,
|
||||
7, 7, 7, 21, 21, 21, 21, 21, 21, 21, 21,
|
||||
21, 21, 21, 3, 3, 3, 3, 3, 4, 6, 7,
|
||||
7, 7, 21, 21, 21, 21, 21, 21, 21, 21, 21,
|
||||
21, 21, 3, 3, 3, 3, 3, 4, 6, 7, 7,
|
||||
7, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
|
||||
21, 3, 3, 3, 3, 3, 4, 6, 7, 7, 7,
|
||||
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
|
||||
3, 3, 3, 3, 3, 4, 6, 7, 7, 7, 21,
|
||||
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 3,
|
||||
3, 3, 3, 3, 4, 6, 7, 7, 7, 21, 21,
|
||||
21, 21, 21, 21, 21, 21, 21, 21, 21, 3, 3,
|
||||
3, 3, 3, 4, 6, 7, 7, 7, 21, 21, 21,
|
||||
21, 21, 21, 21, 21, 21, 21, 21, 3, 3, 3,
|
||||
3, 3, 4, 6, 8, 10, 10, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 3, 3, 3, 3,
|
||||
3, 4, 7, 9, 11, 16, 30, 30, 30, 30, 30,
|
||||
30, 30, 30, 30, 30, 30, 3, 3, 3, 3, 3,
|
||||
4, 9, 11, 13, 18, 42, 42, 42, 42, 42, 42,
|
||||
42, 42, 42, 42, 42, 3, 3, 3, 3, 3, 4,
|
||||
13, 15, 17, 22, 46, 64, 64, 64, 64, 64, 64,
|
||||
64, 64, 64, 64, 3, 3, 3, 3, 3, 4, 21,
|
||||
23, 25, 30, 54, 72, 104, 104, 104, 104, 104, 104,
|
||||
104, 104, 104, 3, 3, 3, 3, 3, 4, 37, 39,
|
||||
41, 46, 70, 88, 120, 184, 184, 184, 184, 184, 184,
|
||||
184, 184, 3, 3, 3, 3, 3, 4, 69, 71, 73,
|
||||
78, 102, 120, 152, 216, 344, 344, 344, 344, 344, 344,
|
||||
344, 3, 3, 3, 3, 3, 4, 133, 135, 137, 142,
|
||||
166, 184, 216, 280, 408, 408, 408, 408, 408, 408, 408,
|
||||
3, 3, 3, 3, 3, 4, 261, 263, 265, 270, 550,
|
||||
568, 600, 664, 792, 792, 792, 792, 792, 792, 792, 3,
|
||||
3, 3, 3, 3, 4, 517, 519, 521, 526, 806, 824,
|
||||
856, 920, 1048, 1048, 1048, 1048, 1048, 1048, 1048, 3, 3,
|
||||
3, 3, 3, 4, 1029, 1031, 1033, 1038, 1318, 1336, 1368,
|
||||
1432, 1560, 1560, 1560, 1560, 1560, 1560, 1560, 3, 3, 3,
|
||||
3, 3, 4, 2053, 2055, 2057, 2062, 2342, 2360, 2392, 2456,
|
||||
2584, 2584, 2584, 2584, 2584, 2584, 2584, 3, 3, 3, 3,
|
||||
3, 4, 4101, 4103, 4105, 4110, 4390, 4408, 4440, 4504, 4632,
|
||||
4632, 4632, 4632, 4632, 4632, 4632, 3, 3, 3, 3, 3,
|
||||
4, 8197, 8199, 8201, 8206, 8486, 8504, 8536, 8600, 8728, 8728,
|
||||
8728, 8728, 8728, 8728, 8728, 3, 3, 3, 3, 3, 4,
|
||||
6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 7, 7, 3, 3, 3, 3, 3, 4, 6,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 7, 3, 3, 3, 3, 3, 4, 6, 7,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 3, 3, 3, 3, 3, 4, 6, 7, 7,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 3, 3, 3, 3, 3, 4, 6, 7, 7, 7,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
3, 3, 3, 3, 3, 4, 6, 7, 7, 7, 7,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 3,
|
||||
3, 3, 3, 3, 4, 6, 7, 7, 7, 7, 7,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 5, 5,
|
||||
5, 5, 5, 6, 9, 10, 12, 12, 12, 12, 12,
|
||||
12, 12, 12, 12, 12, 12, 12, 12, 5, 5, 5,
|
||||
5, 5, 6, 10, 11, 13, 18, 18, 18, 18, 18,
|
||||
18, 18, 18, 18, 18, 18, 18, 5, 5, 5, 5,
|
||||
5, 6, 12, 13, 15, 20, 34, 34, 34, 34, 34,
|
||||
34, 34, 34, 34, 34, 34, 5, 5, 5, 5, 5,
|
||||
6, 16, 17, 19, 24, 38, 56, 56, 56, 56, 56,
|
||||
56, 56, 56, 56, 56, 5, 5, 5, 5, 5, 6,
|
||||
24, 25, 27, 32, 46, 64, 96, 96, 96, 96, 96,
|
||||
96, 96, 96, 96, 5, 5, 5, 5, 5, 6, 40,
|
||||
41, 43, 48, 62, 80, 112, 176, 176, 176, 176, 176,
|
||||
176, 176, 176, 5, 5, 5, 5, 5, 6, 72, 73,
|
||||
75, 80, 94, 112, 144, 208, 336, 336, 336, 336, 336,
|
||||
336, 336, 5, 5, 5, 5, 5, 6, 136, 137, 139,
|
||||
144, 158, 176, 208, 272, 400, 400, 400, 400, 400, 400,
|
||||
400, 5, 5, 5, 5, 5, 6, 264, 265, 267, 272,
|
||||
542, 560, 592, 656, 784, 784, 784, 784, 784, 784, 784,
|
||||
5, 5, 5, 5, 5, 6, 520, 521, 523, 528, 798,
|
||||
816, 848, 912, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 5,
|
||||
5, 5, 5, 5, 6, 1032, 1033, 1035, 1040, 1310, 1328,
|
||||
1360, 1424, 1552, 1552, 1552, 1552, 1552, 1552, 1552, 5, 5,
|
||||
5, 5, 5, 6, 2056, 2057, 2059, 2064, 2334, 2352, 2384,
|
||||
2448, 2576, 2576, 2576, 2576, 2576, 2576, 2576, 5, 5, 5,
|
||||
5, 5, 6, 4104, 4105, 4107, 4112, 4382, 4400, 4432, 4496,
|
||||
4624, 4624, 4624, 4624, 4624, 4624, 4624, 5, 5, 5, 5,
|
||||
5, 6, 8200, 8201, 8203, 8208, 8478, 8496, 8528, 8592, 8720,
|
||||
8720, 8720, 8720, 8720, 8720, 8720, 3, 3, 3, 3, 3,
|
||||
4, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 7, 7, 7, 3, 3, 3, 3, 3, 4,
|
||||
6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 7, 7, 3, 3, 3, 3, 3, 4, 6,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 7, 3, 3, 3, 3, 3, 4, 6, 7,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 3, 3, 3, 3, 3, 4, 6, 7, 7,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 3, 3, 3, 3, 3, 4, 6, 7, 7, 7,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
3, 3, 3, 3, 3, 4, 6, 7, 7, 7, 7,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 9,
|
||||
9, 9, 9, 9, 10, 13, 14, 16, 16, 16, 16,
|
||||
16, 16, 16, 16, 16, 16, 16, 16, 16, 9, 9,
|
||||
9, 9, 9, 10, 14, 15, 17, 22, 22, 22, 22,
|
||||
22, 22, 22, 22, 22, 22, 22, 22, 9, 9, 9,
|
||||
9, 9, 10, 16, 17, 19, 24, 42, 42, 42, 42,
|
||||
42, 42, 42, 42, 42, 42, 42, 9, 9, 9, 9,
|
||||
9, 10, 20, 21, 23, 28, 46, 64, 64, 64, 64,
|
||||
64, 64, 64, 64, 64, 64, 9, 9, 9, 9, 9,
|
||||
10, 28, 29, 31, 36, 54, 72, 104, 104, 104, 104,
|
||||
104, 104, 104, 104, 104, 9, 9, 9, 9, 9, 10,
|
||||
44, 45, 47, 52, 70, 88, 120, 184, 184, 184, 184,
|
||||
184, 184, 184, 184, 9, 9, 9, 9, 9, 10, 76,
|
||||
77, 79, 84, 102, 120, 152, 216, 344, 344, 344, 344,
|
||||
344, 344, 344, 9, 9, 9, 9, 9, 10, 140, 141,
|
||||
143, 148, 166, 184, 216, 280, 408, 408, 408, 408, 408,
|
||||
408, 408, 9, 9, 9, 9, 9, 10, 268, 269, 271,
|
||||
276, 550, 568, 600, 664, 792, 792, 792, 792, 792, 792,
|
||||
792, 9, 9, 9, 9, 9, 10, 524, 525, 527, 532,
|
||||
806, 824, 856, 920, 1048, 1048, 1048, 1048, 1048, 1048, 1048,
|
||||
9, 9, 9, 9, 9, 10, 1036, 1037, 1039, 1044, 1318,
|
||||
1336, 1368, 1432, 1560, 1560, 1560, 1560, 1560, 1560, 1560, 9,
|
||||
9, 9, 9, 9, 10, 2060, 2061, 2063, 2068, 2342, 2360,
|
||||
2392, 2456, 2584, 2584, 2584, 2584, 2584, 2584, 2584, 9, 9,
|
||||
9, 9, 9, 10, 4108, 4109, 4111, 4116, 4390, 4408, 4440,
|
||||
4504, 4632, 4632, 4632, 4632, 4632, 4632, 4632, 9, 9, 9,
|
||||
9, 9, 10, 8204, 8205, 8207, 8212, 8486, 8504, 8536, 8600,
|
||||
8728, 8728, 8728, 8728, 8728, 8728, 8728, 3, 3, 3, 3,
|
||||
3, 4, 6, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 7, 7, 7, 7, 3, 3, 3, 3, 3,
|
||||
4, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 7, 7, 7, 3, 3, 3, 3, 3, 4,
|
||||
6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 7, 7, 3, 3, 3, 3, 3, 4, 6,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 7, 3, 3, 3, 3, 3, 4, 6, 7,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 3, 3, 3, 3, 3, 4, 6, 7, 7,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 3, 3, 3, 3, 3, 4, 6, 7, 7, 7,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
33, 33, 33, 33, 33, 34, 37, 38, 40, 40, 40,
|
||||
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 33,
|
||||
33, 33, 33, 33, 34, 38, 39, 41, 46, 46, 46,
|
||||
46, 46, 46, 46, 46, 46, 46, 46, 46, 33, 33,
|
||||
33, 33, 33, 34, 40, 41, 43, 48, 58, 58, 58,
|
||||
58, 58, 58, 58, 58, 58, 58, 58, 33, 33, 33,
|
||||
33, 33, 34, 44, 45, 47, 52, 62, 80, 80, 80,
|
||||
80, 80, 80, 80, 80, 80, 80, 33, 33, 33, 33,
|
||||
33, 34, 52, 53, 55, 60, 70, 88, 120, 120, 120,
|
||||
120, 120, 120, 120, 120, 120, 33, 33, 33, 33, 33,
|
||||
34, 68, 69, 71, 76, 86, 104, 136, 200, 200, 200,
|
||||
200, 200, 200, 200, 200, 33, 33, 33, 33, 33, 34,
|
||||
100, 101, 103, 108, 118, 136, 168, 232, 360, 360, 360,
|
||||
360, 360, 360, 360, 33, 33, 33, 33, 33, 34, 164,
|
||||
165, 167, 172, 182, 200, 232, 296, 424, 424, 424, 424,
|
||||
424, 424, 424, 33, 33, 33, 33, 33, 34, 292, 293,
|
||||
295, 300, 566, 584, 616, 680, 808, 808, 808, 808, 808,
|
||||
808, 808, 33, 33, 33, 33, 33, 34, 548, 549, 551,
|
||||
556, 822, 840, 872, 936, 1064, 1064, 1064, 1064, 1064, 1064,
|
||||
1064, 33, 33, 33, 33, 33, 34, 1060, 1061, 1063, 1068,
|
||||
1334, 1352, 1384, 1448, 1576, 1576, 1576, 1576, 1576, 1576, 1576,
|
||||
33, 33, 33, 33, 33, 34, 2084, 2085, 2087, 2092, 2358,
|
||||
2376, 2408, 2472, 2600, 2600, 2600, 2600, 2600, 2600, 2600, 33,
|
||||
33, 33, 33, 33, 34, 4132, 4133, 4135, 4140, 4406, 4424,
|
||||
4456, 4520, 4648, 4648, 4648, 4648, 4648, 4648, 4648, 33, 33,
|
||||
33, 33, 33, 34, 8228, 8229, 8231, 8236, 8502, 8520, 8552,
|
||||
8616, 8744, 8744, 8744, 8744, 8744, 8744, 8744, 3, 3, 3,
|
||||
3, 3, 4, 6, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 7, 7, 7, 7, 7, 3, 3, 3, 3,
|
||||
3, 4, 6, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 7, 7, 7, 7, 3, 3, 3, 3, 3,
|
||||
4, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 7, 7, 7, 3, 3, 3, 3, 3, 4,
|
||||
6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 7, 7, 3, 3, 3, 3, 3, 4, 6,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 7, 3, 3, 3, 3, 3, 4, 6, 7,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 3, 3, 3, 3, 3, 4, 6, 7, 7,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 65, 65, 65, 65, 65, 66, 69, 70, 72, 72,
|
||||
72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72,
|
||||
65, 65, 65, 65, 65, 66, 70, 71, 73, 78, 78,
|
||||
78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 65,
|
||||
65, 65, 65, 65, 66, 72, 73, 75, 80, 90, 90,
|
||||
90, 90, 90, 90, 90, 90, 90, 90, 90, 65, 65,
|
||||
65, 65, 65, 66, 76, 77, 79, 84, 94, 112, 112,
|
||||
112, 112, 112, 112, 112, 112, 112, 112, 65, 65, 65,
|
||||
65, 65, 66, 84, 85, 87, 92, 102, 120, 152, 152,
|
||||
152, 152, 152, 152, 152, 152, 152, 65, 65, 65, 65,
|
||||
65, 66, 100, 101, 103, 108, 118, 136, 168, 232, 232,
|
||||
232, 232, 232, 232, 232, 232, 65, 65, 65, 65, 65,
|
||||
66, 132, 133, 135, 140, 150, 168, 200, 264, 392, 392,
|
||||
392, 392, 392, 392, 392, 65, 65, 65, 65, 65, 66,
|
||||
196, 197, 199, 204, 214, 232, 264, 328, 456, 456, 456,
|
||||
456, 456, 456, 456, 65, 65, 65, 65, 65, 66, 324,
|
||||
325, 327, 332, 598, 616, 648, 712, 840, 840, 840, 840,
|
||||
840, 840, 840, 65, 65, 65, 65, 65, 66, 580, 581,
|
||||
583, 588, 854, 872, 904, 968, 1096, 1096, 1096, 1096, 1096,
|
||||
1096, 1096, 65, 65, 65, 65, 65, 66, 1092, 1093, 1095,
|
||||
1100, 1366, 1384, 1416, 1480, 1608, 1608, 1608, 1608, 1608, 1608,
|
||||
1608, 65, 65, 65, 65, 65, 66, 2116, 2117, 2119, 2124,
|
||||
2390, 2408, 2440, 2504, 2632, 2632, 2632, 2632, 2632, 2632, 2632,
|
||||
65, 65, 65, 65, 65, 66, 4164, 4165, 4167, 4172, 4438,
|
||||
4456, 4488, 4552, 4680, 4680, 4680, 4680, 4680, 4680, 4680, 65,
|
||||
65, 65, 65, 65, 66, 8260, 8261, 8263, 8268, 8534, 8552,
|
||||
8584, 8648, 8776, 8776, 8776, 8776, 8776, 8776, 8776, 6, 6,
|
||||
6, 6, 6, 7, 9, 10, 10, 10, 10, 10, 10,
|
||||
10, 10, 10, 10, 10, 10, 10, 10, 6, 6, 6,
|
||||
6, 6, 7, 9, 10, 10, 10, 10, 10, 10, 10,
|
||||
10, 10, 10, 10, 10, 10, 10, 6, 6, 6, 6,
|
||||
6, 7, 9, 10, 10, 10, 10, 10, 10, 10, 10,
|
||||
10, 10, 10, 10, 10, 10, 6, 6, 6, 6, 6,
|
||||
7, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10,
|
||||
10, 10, 10, 10, 10, 6, 6, 6, 6, 6, 7,
|
||||
9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
|
||||
10, 10, 10, 10, 6, 6, 6, 6, 6, 7, 9,
|
||||
10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
|
||||
10, 10, 10, 6, 6, 6, 6, 6, 7, 9, 10,
|
||||
10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
|
||||
10, 10, 129, 129, 129, 129, 129, 130, 132, 136, 138,
|
||||
138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138,
|
||||
138, 129, 129, 129, 129, 129, 130, 132, 135, 143, 148,
|
||||
148, 148, 148, 148, 148, 148, 148, 148, 148, 148, 148,
|
||||
129, 129, 129, 129, 129, 130, 132, 135, 145, 150, 160,
|
||||
160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 129,
|
||||
129, 129, 129, 129, 130, 132, 135, 149, 154, 164, 182,
|
||||
182, 182, 182, 182, 182, 182, 182, 182, 182, 129, 129,
|
||||
129, 129, 129, 130, 132, 135, 157, 162, 172, 190, 222,
|
||||
222, 222, 222, 222, 222, 222, 222, 222, 129, 129, 129,
|
||||
129, 129, 130, 132, 135, 173, 178, 188, 206, 238, 302,
|
||||
302, 302, 302, 302, 302, 302, 302, 129, 129, 129, 129,
|
||||
129, 130, 132, 135, 205, 210, 220, 238, 270, 334, 462,
|
||||
462, 462, 462, 462, 462, 462, 129, 129, 129, 129, 129,
|
||||
130, 132, 135, 269, 274, 284, 302, 334, 398, 526, 526,
|
||||
526, 526, 526, 526, 526, 129, 129, 129, 129, 129, 130,
|
||||
132, 135, 397, 402, 668, 686, 718, 782, 910, 910, 910,
|
||||
910, 910, 910, 910, 129, 129, 129, 129, 129, 130, 132,
|
||||
135, 653, 658, 924, 942, 974, 1038, 1166, 1166, 1166, 1166,
|
||||
1166, 1166, 1166, 129, 129, 129, 129, 129, 130, 132, 135,
|
||||
1165, 1170, 1436, 1454, 1486, 1550, 1678, 1678, 1678, 1678, 1678,
|
||||
1678, 1678, 129, 129, 129, 129, 129, 130, 132, 135, 2189,
|
||||
2194, 2460, 2478, 2510, 2574, 2702, 2702, 2702, 2702, 2702, 2702,
|
||||
2702, 129, 129, 129, 129, 129, 130, 132, 135, 4237, 4242,
|
||||
4508, 4526, 4558, 4622, 4750, 4750, 4750, 4750, 4750, 4750, 4750,
|
||||
129, 129, 129, 129, 129, 130, 132, 135, 8333, 8338, 8604,
|
||||
8622, 8654, 8718, 8846, 8846, 8846, 8846, 8846, 8846, 8846, 803,
|
||||
803, 803, 803, 803, 804, 806, 807, 807, 807, 807, 807,
|
||||
807, 807, 807, 807, 807, 807, 807, 807, 807, 803, 803,
|
||||
803, 803, 803, 804, 806, 807, 807, 807, 807, 807, 807,
|
||||
807, 807, 807, 807, 807, 807, 807, 807, 803, 803, 803,
|
||||
803, 803, 804, 806, 807, 807, 807, 807, 807, 807, 807,
|
||||
807, 807, 807, 807, 807, 807, 807, 803, 803, 803, 803,
|
||||
803, 804, 806, 807, 807, 807, 807, 807, 807, 807, 807,
|
||||
807, 807, 807, 807, 807, 807, 803, 803, 803, 803, 804,
|
||||
804, 806, 807, 807, 807, 807, 807, 807, 807, 807, 807,
|
||||
807, 807, 807, 807, 807, 803, 803, 803, 803, 804, 805,
|
||||
807, 808, 808, 808, 808, 808, 808, 808, 808, 808, 808,
|
||||
808, 808, 808, 808, 803, 803, 803, 803, 804, 805, 808,
|
||||
809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809,
|
||||
809, 809, 809, 803, 803, 803, 803, 804, 805, 807, 813,
|
||||
815, 815, 815, 815, 815, 815, 815, 815, 815, 815, 815,
|
||||
815, 815, 803, 803, 803, 803, 804, 805, 807, 812, 824,
|
||||
828, 828, 828, 828, 828, 828, 828, 828, 828, 828, 828,
|
||||
828, 803, 803, 803, 803, 804, 805, 807, 812, 826, 838,
|
||||
846, 846, 846, 846, 846, 846, 846, 846, 846, 846, 846,
|
||||
803, 803, 803, 803, 804, 805, 807, 812, 830, 842, 866,
|
||||
882, 882, 882, 882, 882, 882, 882, 882, 882, 882, 803,
|
||||
803, 803, 803, 804, 805, 807, 812, 838, 850, 874, 922,
|
||||
954, 954, 954, 954, 954, 954, 954, 954, 954, 803, 803,
|
||||
803, 803, 804, 805, 807, 812, 854, 866, 890, 938, 1034,
|
||||
1098, 1098, 1098, 1098, 1098, 1098, 1098, 1098, 803, 803, 803,
|
||||
803, 804, 805, 807, 812, 886, 898, 922, 970, 1066, 1258,
|
||||
1386, 1386, 1386, 1386, 1386, 1386, 1386, 803, 803, 803, 803,
|
||||
804, 805, 807, 812, 950, 962, 986, 1034, 1130, 1322, 1706,
|
||||
1706, 1706, 1706, 1706, 1706, 1706, 803, 803, 803, 803, 804,
|
||||
805, 807, 812, 1078, 1090, 1114, 1162, 1258, 1450, 1834, 2346,
|
||||
2346, 2346, 2346, 2346, 2346, 803, 803, 803, 803, 804, 805,
|
||||
807, 812, 1334, 1346, 1370, 1418, 1514, 1706, 2090, 2602, 3626,
|
||||
3626, 3626, 3626, 3626, 803, 803, 803, 803, 804, 805, 807,
|
||||
812, 1846, 1858, 1882, 1930, 2026, 2218, 2602, 3114, 4138, 6186,
|
||||
6186, 6186, 6186, 803, 803, 803, 803, 804, 805, 807, 812,
|
||||
2870, 2882, 2906, 2954, 3050, 3242, 3626, 4138, 5162, 7210, 11306,
|
||||
11306, 11306, 803, 803, 803, 803, 804, 805, 807, 812, 4918,
|
||||
4930, 4954, 5002, 5098, 5290, 5674, 6186, 7210, 9258, 13354, 21546,
|
||||
21546, 803, 803, 803, 803, 804, 805, 807, 812, 9014, 9026,
|
||||
9050, 9098, 9194, 9386, 9770, 10282, 11306, 13354, 17450, 25642, 42026,
|
||||
803, 803, 803, 803, 803, 804, 806, 807, 807, 807, 807,
|
||||
807, 807, 807, 807, 807, 807, 807, 807, 807, 807, 803,
|
||||
803, 803, 803, 803, 804, 806, 807, 807, 807, 807, 807,
|
||||
807, 807, 807, 807, 807, 807, 807, 807, 807, 803, 803,
|
||||
803, 803, 803, 804, 806, 807, 807, 807, 807, 807, 807,
|
||||
807, 807, 807, 807, 807, 807, 807, 807, 803, 803, 803,
|
||||
803, 803, 804, 806, 807, 807, 807, 807, 807, 807, 807,
|
||||
807, 807, 807, 807, 807, 807, 807, 803, 803, 803, 803,
|
||||
804, 804, 806, 807, 807, 807, 807, 807, 807, 807, 807,
|
||||
807, 807, 807, 807, 807, 807, 803, 803, 803, 803, 804,
|
||||
805, 807, 808, 808, 808, 808, 808, 808, 808, 808, 808,
|
||||
808, 808, 808, 808, 808, 803, 803, 803, 803, 804, 805,
|
||||
808, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809,
|
||||
809, 809, 809, 809, 803, 803, 803, 803, 804, 805, 807,
|
||||
813, 815, 815, 815, 815, 815, 815, 815, 815, 815, 815,
|
||||
815, 815, 815, 803, 803, 803, 803, 804, 805, 807, 812,
|
||||
824, 828, 828, 828, 828, 828, 828, 828, 828, 828, 828,
|
||||
828, 828, 803, 803, 803, 803, 804, 805, 807, 812, 826,
|
||||
838, 846, 846, 846, 846, 846, 846, 846, 846, 846, 846,
|
||||
846, 803, 803, 803, 803, 804, 805, 807, 812, 830, 842,
|
||||
866, 882, 882, 882, 882, 882, 882, 882, 882, 882, 882,
|
||||
803, 803, 803, 803, 804, 805, 807, 812, 838, 850, 874,
|
||||
922, 954, 954, 954, 954, 954, 954, 954, 954, 954, 803,
|
||||
803, 803, 803, 804, 805, 807, 812, 854, 866, 890, 938,
|
||||
1034, 1098, 1098, 1098, 1098, 1098, 1098, 1098, 1098, 803, 803,
|
||||
803, 803, 804, 805, 807, 812, 886, 898, 922, 970, 1066,
|
||||
1258, 1386, 1386, 1386, 1386, 1386, 1386, 1386, 803, 803, 803,
|
||||
803, 804, 805, 807, 812, 950, 962, 986, 1034, 1130, 1322,
|
||||
1706, 1706, 1706, 1706, 1706, 1706, 1706, 803, 803, 803, 803,
|
||||
804, 805, 807, 812, 1078, 1090, 1114, 1162, 1258, 1450, 1834,
|
||||
2346, 2346, 2346, 2346, 2346, 2346, 803, 803, 803, 803, 804,
|
||||
805, 807, 812, 1334, 1346, 1370, 1418, 1514, 1706, 2090, 2602,
|
||||
3626, 3626, 3626, 3626, 3626, 803, 803, 803, 803, 804, 805,
|
||||
807, 812, 1846, 1858, 1882, 1930, 2026, 2218, 2602, 3114, 4138,
|
||||
6186, 6186, 6186, 6186, 803, 803, 803, 803, 804, 805, 807,
|
||||
812, 2870, 2882, 2906, 2954, 3050, 3242, 3626, 4138, 5162, 7210,
|
||||
11306, 11306, 11306, 803, 803, 803, 803, 804, 805, 807, 812,
|
||||
4918, 4930, 4954, 5002, 5098, 5290, 5674, 6186, 7210, 9258, 13354,
|
||||
21546, 21546, 803, 803, 803, 803, 804, 805, 807, 812, 9014,
|
||||
9026, 9050, 9098, 9194, 9386, 9770, 10282, 11306, 13354, 17450, 25642,
|
||||
42026,
|
||||
};
|
||||
#endif
|
||||
|
||||
class brotli_block_compressor final : public block_compressor::impl {
|
||||
public:
|
||||
brotli_block_compressor(uint32_t quality, uint32_t window_bits)
|
||||
@ -92,6 +582,24 @@ class brotli_block_compressor final : public block_compressor::impl {
|
||||
return {};
|
||||
}
|
||||
|
||||
size_t estimate_memory_usage(size_t data_size) const override {
|
||||
#ifdef BROTLI_BUILD_ENC_EXTRA_API
|
||||
return ::BrotliEncoderEstimatePeakMemoryUsage(quality_, window_bits_,
|
||||
data_size) +
|
||||
data_size;
|
||||
#else
|
||||
auto const q = std::clamp<int>(quality_, 0, 11);
|
||||
auto const lg_win = std::clamp<int>(window_bits_, 10, 30) - 10;
|
||||
auto const lg_data_size =
|
||||
std::clamp<int>(static_cast<int>(std::ceil(std::log2(data_size))), 10,
|
||||
30) -
|
||||
10;
|
||||
auto const index = (q * 21 + lg_win) * 21 + lg_data_size;
|
||||
return kBrotliMemUsageGranularity * brotli_memory_usage.at(index) +
|
||||
data_size;
|
||||
#endif
|
||||
}
|
||||
|
||||
private:
|
||||
uint32_t const quality_;
|
||||
uint32_t const window_bits_;
|
||||
|
||||
@ -393,6 +393,11 @@ class flac_block_compressor final : public block_compressor::impl {
|
||||
return cc;
|
||||
}
|
||||
|
||||
size_t estimate_memory_usage(size_t data_size) const override {
|
||||
// TODO: can we estimate the FLAC encoder state size?
|
||||
return data_size;
|
||||
}
|
||||
|
||||
private:
|
||||
uint32_t const level_;
|
||||
bool const exhaustive_;
|
||||
|
||||
@ -54,6 +54,8 @@ struct lz4_compression_policy {
|
||||
}
|
||||
|
||||
static std::string describe(int /*level*/) { return "lz4"; }
|
||||
|
||||
static size_t state_size(size_t /*data_size*/) { return LZ4_sizeofState(); }
|
||||
};
|
||||
|
||||
struct lz4hc_compression_policy {
|
||||
@ -67,6 +69,8 @@ struct lz4hc_compression_policy {
|
||||
static std::string describe(int level) {
|
||||
return fmt::format("lz4hc [level={}]", level);
|
||||
}
|
||||
|
||||
static size_t state_size(size_t /*data_size*/) { return LZ4_sizeofStateHC(); }
|
||||
};
|
||||
|
||||
template <typename Policy>
|
||||
@ -113,6 +117,10 @@ class lz4_block_compressor final : public block_compressor::impl {
|
||||
return {};
|
||||
}
|
||||
|
||||
size_t estimate_memory_usage(size_t data_size) const override {
|
||||
return Policy::state_size(data_size) + data_size;
|
||||
}
|
||||
|
||||
private:
|
||||
int const level_;
|
||||
};
|
||||
|
||||
@ -136,6 +136,13 @@ class lzma_block_compressor final : public block_compressor::impl {
|
||||
return {};
|
||||
}
|
||||
|
||||
size_t estimate_memory_usage(size_t data_size) const override {
|
||||
auto lzma_opts = opt_lzma_;
|
||||
std::array<lzma_filter, 2> filters{
|
||||
{{LZMA_FILTER_LZMA2, &lzma_opts}, {LZMA_VLI_UNKNOWN, nullptr}}};
|
||||
return lzma_raw_encoder_memusage(filters.data()) + data_size;
|
||||
}
|
||||
|
||||
private:
|
||||
shared_byte_buffer
|
||||
compress(shared_byte_buffer const& data, lzma_filter const* filters) const;
|
||||
|
||||
@ -66,6 +66,11 @@ class null_block_compressor final : public block_compressor::impl {
|
||||
get_compression_constraints(std::string const&) const override {
|
||||
return {};
|
||||
}
|
||||
|
||||
size_t estimate_memory_usage(size_t /*data_size*/) const override {
|
||||
// we're re-using the input buffer, so we don't need any extra memory
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
class null_block_decompressor final : public block_decompressor_base {
|
||||
|
||||
@ -171,6 +171,11 @@ class ricepp_block_compressor final : public block_compressor::impl {
|
||||
return cc;
|
||||
}
|
||||
|
||||
size_t estimate_memory_usage(size_t data_size) const override {
|
||||
// ricepp encoder basically has no memory overhead by itself
|
||||
return data_size;
|
||||
}
|
||||
|
||||
private:
|
||||
size_t const block_size_;
|
||||
};
|
||||
|
||||
@ -50,6 +50,59 @@ namespace dwarfs {
|
||||
|
||||
namespace {
|
||||
|
||||
#ifndef ZSTD_STATIC_LINKING_ONLY
|
||||
|
||||
constexpr size_t kZstdMemUsageGranularity = 16384;
|
||||
|
||||
constexpr std::array<uint16_t, 23 * 21> zstd_memory_usage{
|
||||
3, 4, 6, 10, 18, 34, 41, 56, 64, 80, 80,
|
||||
80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 3,
|
||||
4, 5, 8, 14, 12, 19, 34, 36, 36, 36, 36,
|
||||
36, 36, 36, 36, 36, 36, 36, 36, 36, 3, 4,
|
||||
5, 8, 14, 18, 25, 40, 40, 48, 48, 48, 48,
|
||||
48, 48, 48, 48, 48, 48, 48, 48, 3, 4, 6,
|
||||
10, 18, 34, 41, 56, 64, 80, 80, 80, 80, 80,
|
||||
80, 80, 80, 80, 80, 80, 80, 3, 4, 6, 10,
|
||||
14, 34, 65, 96, 80, 160, 160, 160, 160, 160, 160,
|
||||
160, 160, 160, 160, 160, 160, 3, 4, 6, 10, 14,
|
||||
34, 65, 80, 128, 224, 224, 224, 224, 224, 224, 224,
|
||||
224, 224, 224, 224, 224, 3, 4, 6, 10, 14, 34,
|
||||
65, 80, 224, 224, 224, 224, 224, 224, 224, 224, 224,
|
||||
224, 224, 224, 224, 3, 4, 6, 10, 14, 34, 65,
|
||||
80, 224, 416, 416, 416, 416, 416, 416, 416, 416, 416,
|
||||
416, 416, 416, 3, 4, 6, 10, 14, 34, 65, 80,
|
||||
224, 416, 416, 416, 416, 416, 416, 416, 416, 416, 416,
|
||||
416, 416, 3, 5, 7, 12, 18, 34, 65, 80, 224,
|
||||
416, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800,
|
||||
800, 3, 5, 7, 12, 18, 34, 65, 80, 224, 416,
|
||||
800, 1568, 1568, 1568, 1568, 1568, 1568, 1568, 1568, 1568, 1568,
|
||||
12, 14, 16, 21, 27, 42, 81, 96, 224, 416, 800,
|
||||
1568, 1568, 1568, 1568, 1568, 1568, 1568, 1568, 1568, 1568, 13,
|
||||
14, 17, 24, 32, 42, 81, 128, 288, 416, 800, 1568,
|
||||
3104, 3104, 3104, 3104, 3104, 3104, 3104, 3104, 3104, 13, 14,
|
||||
17, 24, 32, 51, 90, 137, 233, 544, 1056, 2080, 2080,
|
||||
2080, 2080, 2080, 2080, 2080, 2080, 2080, 2080, 13, 14, 17,
|
||||
24, 36, 61, 110, 177, 273, 544, 1056, 2080, 3104, 3104,
|
||||
3104, 3104, 3104, 3104, 3104, 3104, 3104, 13, 14, 17, 24,
|
||||
36, 61, 110, 177, 273, 544, 1056, 2080, 4128, 4128, 4128,
|
||||
4128, 4128, 4128, 4128, 4128, 4128, 13, 14, 17, 24, 36,
|
||||
61, 110, 177, 337, 553, 1065, 2089, 2089, 2089, 2089, 2089,
|
||||
2089, 2089, 2089, 2089, 2089, 13, 14, 17, 24, 36, 61,
|
||||
110, 177, 337, 553, 1065, 2089, 3113, 3113, 3113, 3113, 3113,
|
||||
3113, 3113, 3113, 3113, 13, 14, 17, 24, 36, 61, 110,
|
||||
177, 337, 593, 1105, 2129, 3153, 3153, 3153, 3153, 3153, 3153,
|
||||
3153, 3153, 3153, 13, 14, 17, 24, 36, 61, 110, 177,
|
||||
337, 593, 1105, 2129, 3153, 5201, 5201, 5201, 5201, 5201, 5201,
|
||||
5201, 5201, 13, 14, 17, 24, 36, 61, 110, 177, 337,
|
||||
593, 1105, 2129, 4177, 6225, 10321, 10321, 10321, 10321, 10321, 10321,
|
||||
10321, 13, 14, 17, 24, 36, 61, 110, 177, 337, 593,
|
||||
1105, 2129, 4177, 8273, 12369, 20561, 20561, 20561, 20561, 20561, 20561,
|
||||
13, 14, 17, 24, 36, 61, 110, 177, 337, 593, 1105,
|
||||
2129, 4177, 8273, 16465, 24657, 41041, 41562, 41562, 41562, 41562,
|
||||
};
|
||||
|
||||
#endif
|
||||
|
||||
class zstd_block_compressor final : public block_compressor::impl {
|
||||
public:
|
||||
explicit zstd_block_compressor(int level)
|
||||
@ -77,6 +130,22 @@ class zstd_block_compressor final : public block_compressor::impl {
|
||||
return {};
|
||||
}
|
||||
|
||||
size_t estimate_memory_usage(size_t data_size) const override {
|
||||
#ifdef ZSTD_STATIC_LINKING_ONLY
|
||||
// TODO: check if dictSize == 0 is correct
|
||||
auto params = ZSTD_getCParams(level_, data_size, 0);
|
||||
return ZSTD_estimateCCtxSize_usingCParams(params) + data_size;
|
||||
#else
|
||||
auto const l = std::clamp<int>(level_, 0, 22);
|
||||
auto const lg_data_size =
|
||||
std::clamp<int>(static_cast<int>(std::ceil(std::log2(data_size))), 10,
|
||||
30) -
|
||||
10;
|
||||
auto const index = l * 21 + lg_data_size;
|
||||
return kZstdMemUsageGranularity * zstd_memory_usage.at(index) + data_size;
|
||||
#endif
|
||||
}
|
||||
|
||||
private:
|
||||
int const level_;
|
||||
};
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user