TheCloud/dwarfs
1
0
Fork 0
mirror of https://github.com/mhx/dwarfs.git synced 2025-09-11 13:30:47 -04:00
Code Issues Packages Projects Releases Wiki Activity

Fix #86: block size bits config issues

Browse Source 
- limit block size bits to between 10 and 30
- slightly tweak help for --block-size-bits
- fix math to make sure we *could* use block sizes above 30
This commit is contained in:
Marcus Holland-Moritz 2022-06-10 19:07:33 +02:00
parent 3d2059a457
commit d2133354b7
1 changed files with 14 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/mkdwarfs.cpp
View File

@ -114,6 +114,9 @@ const std::map<std::string, uint32_t> time_resolutions{
{"day", 86400}, {"day", 86400},
}; };
constexpr size_t min_block_size_bits{10};
constexpr size_t max_block_size_bits{30};
} // namespace } // namespace
namespace dwarfs { namespace dwarfs {
@ -364,7 +367,7 @@ int mkdwarfs(int argc, char** argv) {
"compression level (0=fast, 9=best, please see man page for details)") "compression level (0=fast, 9=best, please see man page for details)")
("block-size-bits,S", ("block-size-bits,S",
po::value<unsigned>(&cfg.block_size_bits), po::value<unsigned>(&cfg.block_size_bits),
"block size bits (size = 2^bits)") "block size bits (size = 2^arg bits)")
("num-workers,N", ("num-workers,N",
po::value<size_t>(&num_workers)->default_value(num_cpu), po::value<size_t>(&num_workers)->default_value(num_cpu),
"number of scanner/writer worker threads") "number of scanner/writer worker threads")
@ -573,6 +576,13 @@ int mkdwarfs(int argc, char** argv) {
order = defaults.order; order = defaults.order;
} }
if (cfg.block_size_bits < min_block_size_bits ||
cfg.block_size_bits > max_block_size_bits) {
std::cerr << "error: block size must be between " << min_block_size_bits
<< " and " << max_block_size_bits << std::endl;
return 1;
}
bool recompress = vm.count("recompress"); bool recompress = vm.count("recompress");
rewrite_options rw_opts; rewrite_options rw_opts;
if (recompress) { if (recompress) {
@ -829,13 +839,13 @@ int mkdwarfs(int argc, char** argv) {
block_compressor schema_bc(schema_compression); block_compressor schema_bc(schema_compression);
block_compressor metadata_bc(metadata_compression); block_compressor metadata_bc(metadata_compression);
size_t min_memory_req = num_workers * (1 << cfg.block_size_bits); auto min_memory_req = num_workers * (UINT64_C(1) << cfg.block_size_bits);
if (mem_limit < min_memory_req && compression != "null") { if (mem_limit < min_memory_req && compression != "null") {
LOG_WARN << "low memory limit (" << size_with_unit(mem_limit) << "), need " LOG_WARN << "low memory limit (" << size_with_unit(mem_limit) << "), need "
<< size_with_unit(min_memory_req) << " to efficiently compress " << size_with_unit(min_memory_req) << " to efficiently compress "
<< size_with_unit(1 << cfg.block_size_bits) << " blocks with " << size_with_unit(UINT64_C(1) << cfg.block_size_bits)
<< num_workers << " threads"; << " blocks with " << num_workers << " threads";
} }
filesystem_writer fsw(ofs, lgr, wg_compress, prog, bc, schema_bc, metadata_bc, filesystem_writer fsw(ofs, lgr, wg_compress, prog, bc, schema_bc, metadata_bc,