TheCloud/dwarfs
1
0
Fork 0
mirror of https://github.com/mhx/dwarfs.git synced 2025-09-09 20:41:04 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(filesystem_writer): use fsblock policy for block merger

Browse Source
This commit is contained in:
Marcus Holland-Moritz 2023-11-27 08:47:30 +01:00
parent c2ceeee9bd
commit 071da719cb
3 changed files with 24 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/dwarfs/options.h
View File

@ -73,7 +73,7 @@ struct filesystem_options {
struct filesystem_writer_options { struct filesystem_writer_options {
size_t max_queue_size{64 << 20}; size_t max_queue_size{64 << 20};
size_t max_queued_blocks{8}; size_t worst_case_block_size{4 << 20};
bool remove_header{false}; bool remove_header{false};
bool no_section_index{false}; bool no_section_index{false};
}; };

25
src/dwarfs/filesystem_writer.cpp
View File

@ -124,6 +124,23 @@ class fsblock {
std::unique_ptr<impl> impl_; std::unique_ptr<impl> impl_;
}; };
class fsblock_merger_policy {
public:
fsblock_merger_policy(size_t worst_case_block_size)
: worst_case_block_size_{worst_case_block_size} {}
static size_t block_size(std::unique_ptr<fsblock> const& fsb) {
return fsb->size();
}
size_t worst_case_source_block_size(fragment_category /*source_id*/) const {
return worst_case_block_size_;
}
private:
size_t worst_case_block_size_;
};
class raw_fsblock : public fsblock::impl { class raw_fsblock : public fsblock::impl {
public: public:
raw_fsblock(section_type type, const block_compressor& bc, raw_fsblock(section_type type, const block_compressor& bc,
@ -347,7 +364,8 @@ class filesystem_writer_ final : public filesystem_writer::impl {
private: private:
using block_merger_type = using block_merger_type =
multi_queue_block_merger<fragment_category, std::unique_ptr<fsblock>>; multi_queue_block_merger<fragment_category, std::unique_ptr<fsblock>,
fsblock_merger_policy>;
using block_holder_type = block_merger_type::block_holder_type; using block_holder_type = block_merger_type::block_holder_type;
block_compressor const& block_compressor const&
@ -671,8 +689,9 @@ void filesystem_writer_<LoggerPolicy>::configure(
} }
merger_ = std::make_unique<block_merger_type>( merger_ = std::make_unique<block_merger_type>(
max_active_slots, options_.max_queued_blocks, expected_categories, max_active_slots, options_.max_queue_size, expected_categories,
[this](auto&& holder) { on_block_merged(std::move(holder)); }); [this](auto&& holder) { on_block_merged(std::move(holder)); },
fsblock_merger_policy{options_.worst_case_block_size});
} }
template <typename LoggerPolicy> template <typename LoggerPolicy>

4
src/mkdwarfs_main.cpp
View File

@ -888,11 +888,9 @@ int mkdwarfs_main(int argc, sys_char** argv) {
? 2000 ? 2000
: 200; : 200;
// TODO: clean up the use of both max_queued_blocks and max_queue_size
filesystem_writer_options fswopts; filesystem_writer_options fswopts;
fswopts.max_queue_size = mem_limit; fswopts.max_queue_size = mem_limit;
fswopts.max_queued_blocks = fswopts.worst_case_block_size = UINT64_C(1) << sf_config.block_size_bits;
mem_limit / (UINT64_C(1) << sf_config.block_size_bits);
fswopts.remove_header = remove_header; fswopts.remove_header = remove_header;
fswopts.no_section_index = no_section_index; fswopts.no_section_index = no_section_index;