TheCloud/dwarfs
1
0
Fork 0
mirror of https://github.com/mhx/dwarfs.git synced 2025-09-08 03:49:44 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(filesystem_writer): use fsblock policy for block merger

Browse Source
This commit is contained in:
Marcus Holland-Moritz 2023-11-27 08:47:30 +01:00
parent c2ceeee9bd
commit 071da719cb
3 changed files with 24 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/dwarfs/options.h
View File

@ -73,7 +73,7 @@ struct filesystem_options {
struct filesystem_writer_options {
size_t max_queue_size{64 << 20};
size_t max_queued_blocks{8};
size_t worst_case_block_size{4 << 20};
bool remove_header{false};
bool no_section_index{false};
};

25
src/dwarfs/filesystem_writer.cpp
View File

@ -124,6 +124,23 @@ class fsblock {
std::unique_ptr<impl> impl_;
};
class fsblock_merger_policy {
public:
fsblock_merger_policy(size_t worst_case_block_size)
: worst_case_block_size_{worst_case_block_size} {}
static size_t block_size(std::unique_ptr<fsblock> const& fsb) {
return fsb->size();
}
size_t worst_case_source_block_size(fragment_category /*source_id*/) const {
return worst_case_block_size_;
}
private:
size_t worst_case_block_size_;
};
class raw_fsblock : public fsblock::impl {
public:
raw_fsblock(section_type type, const block_compressor& bc,
@ -347,7 +364,8 @@ class filesystem_writer_ final : public filesystem_writer::impl {
private:
using block_merger_type =
multi_queue_block_merger<fragment_category, std::unique_ptr<fsblock>>;
multi_queue_block_merger<fragment_category, std::unique_ptr<fsblock>,
fsblock_merger_policy>;
using block_holder_type = block_merger_type::block_holder_type;
block_compressor const&
@ -671,8 +689,9 @@ void filesystem_writer_<LoggerPolicy>::configure(
}
merger_ = std::make_unique<block_merger_type>(
max_active_slots, options_.max_queued_blocks, expected_categories,
[this](auto&& holder) { on_block_merged(std::move(holder)); });
max_active_slots, options_.max_queue_size, expected_categories,
[this](auto&& holder) { on_block_merged(std::move(holder)); },
fsblock_merger_policy{options_.worst_case_block_size});
}
template <typename LoggerPolicy>

4
src/mkdwarfs_main.cpp
View File

@ -888,11 +888,9 @@ int mkdwarfs_main(int argc, sys_char** argv) {
? 2000
: 200;
// TODO: clean up the use of both max_queued_blocks and max_queue_size
filesystem_writer_options fswopts;
fswopts.max_queue_size = mem_limit;
fswopts.max_queued_blocks =
mem_limit / (UINT64_C(1) << sf_config.block_size_bits);
fswopts.worst_case_block_size = UINT64_C(1) << sf_config.block_size_bits;
fswopts.remove_header = remove_header;
fswopts.no_section_index = no_section_index;