diff --git a/include/dwarfs/options.h b/include/dwarfs/options.h index abdde8bc..f32273d3 100644 --- a/include/dwarfs/options.h +++ b/include/dwarfs/options.h @@ -73,7 +73,7 @@ struct filesystem_options { struct filesystem_writer_options { size_t max_queue_size{64 << 20}; - size_t max_queued_blocks{8}; + size_t worst_case_block_size{4 << 20}; bool remove_header{false}; bool no_section_index{false}; }; diff --git a/src/dwarfs/filesystem_writer.cpp b/src/dwarfs/filesystem_writer.cpp index c3ff8522..c932f7c1 100644 --- a/src/dwarfs/filesystem_writer.cpp +++ b/src/dwarfs/filesystem_writer.cpp @@ -124,6 +124,23 @@ class fsblock { std::unique_ptr impl_; }; +class fsblock_merger_policy { + public: + fsblock_merger_policy(size_t worst_case_block_size) + : worst_case_block_size_{worst_case_block_size} {} + + static size_t block_size(std::unique_ptr const& fsb) { + return fsb->size(); + } + + size_t worst_case_source_block_size(fragment_category /*source_id*/) const { + return worst_case_block_size_; + } + + private: + size_t worst_case_block_size_; +}; + class raw_fsblock : public fsblock::impl { public: raw_fsblock(section_type type, const block_compressor& bc, @@ -347,7 +364,8 @@ class filesystem_writer_ final : public filesystem_writer::impl { private: using block_merger_type = - multi_queue_block_merger>; + multi_queue_block_merger, + fsblock_merger_policy>; using block_holder_type = block_merger_type::block_holder_type; block_compressor const& @@ -671,8 +689,9 @@ void filesystem_writer_::configure( } merger_ = std::make_unique( - max_active_slots, options_.max_queued_blocks, expected_categories, - [this](auto&& holder) { on_block_merged(std::move(holder)); }); + max_active_slots, options_.max_queue_size, expected_categories, + [this](auto&& holder) { on_block_merged(std::move(holder)); }, + fsblock_merger_policy{options_.worst_case_block_size}); } template diff --git a/src/mkdwarfs_main.cpp b/src/mkdwarfs_main.cpp index 005c8508..865c47cc 100644 --- a/src/mkdwarfs_main.cpp +++ b/src/mkdwarfs_main.cpp @@ -888,11 +888,9 @@ int mkdwarfs_main(int argc, sys_char** argv) { ? 2000 : 200; - // TODO: clean up the use of both max_queued_blocks and max_queue_size filesystem_writer_options fswopts; fswopts.max_queue_size = mem_limit; - fswopts.max_queued_blocks = - mem_limit / (UINT64_C(1) << sf_config.block_size_bits); + fswopts.worst_case_block_size = UINT64_C(1) << sf_config.block_size_bits; fswopts.remove_header = remove_header; fswopts.no_section_index = no_section_index;