diff --git a/include/dwarfs/writer/internal/filesystem_writer_detail.h b/include/dwarfs/writer/internal/filesystem_writer_detail.h index 0d783be2..41227553 100644 --- a/include/dwarfs/writer/internal/filesystem_writer_detail.h +++ b/include/dwarfs/writer/internal/filesystem_writer_detail.h @@ -94,7 +94,7 @@ class filesystem_writer_detail { compression_type compression, std::span data, std::optional cat = std::nullopt, block_compression_info* info = nullptr) = 0; - virtual void write_section( + virtual void rewrite_section( section_type type, compression_type compression, std::span data, std::optional cat = std::nullopt) = 0; diff --git a/src/utility/rewrite_filesystem.cpp b/src/utility/rewrite_filesystem.cpp index ba101a76..6c55c547 100644 --- a/src/utility/rewrite_filesystem.cpp +++ b/src/utility/rewrite_filesystem.cpp @@ -453,8 +453,8 @@ void rewrite_filesystem(logger& lgr, dwarfs::reader::filesystem_v2 const& fs, if (recompress_block) { log_recompress(s, cat); - writer.write_section(section_type::BLOCK, s->compression(), - parser->section_data(*s), cat); + writer.rewrite_section(section_type::BLOCK, s->compression(), + parser->section_data(*s), cat); } else { copy_compressed(s, cat); } @@ -491,8 +491,8 @@ void rewrite_filesystem(logger& lgr, dwarfs::reader::filesystem_v2 const& fs, } else { if (opts.recompress_metadata && !from_none_to_none(s)) { log_recompress(s); - writer.write_section(s->type(), s->compression(), - parser->section_data(*s)); + writer.rewrite_section(s->type(), s->compression(), + parser->section_data(*s)); } else { copy_compressed(s); } diff --git a/src/writer/filesystem_writer.cpp b/src/writer/filesystem_writer.cpp index f3726b77..cb37ea97 100644 --- a/src/writer/filesystem_writer.cpp +++ b/src/writer/filesystem_writer.cpp @@ -595,9 +595,10 @@ class filesystem_writer_ final : public filesystem_writer_detail { std::span data, std::optional cat, block_compression_info* info) override; - void write_section(section_type type, compression_type compression, - std::span data, - std::optional cat) override; + void + rewrite_section(section_type type, compression_type compression, + std::span data, + std::optional cat) override; void rewrite_block(delayed_data_fn_type data, size_t uncompressed_size, std::optional cat) override; void write_compressed_section(fs_section const& sec, @@ -617,10 +618,9 @@ class filesystem_writer_ final : public filesystem_writer_detail { write_block_impl(fragment_category cat, shared_byte_buffer data, block_compressor const& bc, std::optional meta, physical_block_cb_type physical_block_cb); - void - write_section_delayed_data(section_type type, delayed_data_fn_type data, - size_t uncompressed_size, - std::optional cat); + void rewrite_section_delayed_data( + section_type type, delayed_data_fn_type data, size_t uncompressed_size, + std::optional cat); void on_block_merged(block_holder_type holder); void write_section_impl(section_type type, shared_byte_buffer data); void write(fsblock const& fsb); @@ -744,6 +744,8 @@ size_t filesystem_writer_::mem_used() const { s += holder.value()->capacity(); } + LOG_VERBOSE << "mem_used: " << s; + return s; } @@ -921,7 +923,7 @@ void filesystem_writer_::check_block_compression( } template -void filesystem_writer_::write_section_delayed_data( +void filesystem_writer_::rewrite_section_delayed_data( section_type type, delayed_data_fn_type data, size_t uncompressed_size, std::optional cat) { { @@ -931,8 +933,9 @@ void filesystem_writer_::write_section_delayed_data( pctx_ = prog_.create_context(); } - // TODO: do we still need this with the merger in place? + // TODO: this isn't currently working while (mem_used() > options_.max_queue_size) { + LOG_VERBOSE << "waiting for queue to drain"; cond_.wait(lock); } @@ -951,14 +954,14 @@ void filesystem_writer_::write_section_delayed_data( } template -void filesystem_writer_::write_section( +void filesystem_writer_::rewrite_section( section_type type, compression_type compression, std::span data, std::optional cat) { auto bd = block_decompressor(compression, data); auto uncompressed_size = bd.uncompressed_size(); - write_section_delayed_data( + rewrite_section_delayed_data( type, [bd = std::move(bd)]() mutable { auto block = bd.start_decompression(malloc_byte_buffer::create()); @@ -972,8 +975,8 @@ template void filesystem_writer_::rewrite_block( delayed_data_fn_type data, size_t uncompressed_size, std::optional cat) { - write_section_delayed_data(section_type::BLOCK, std::move(data), - uncompressed_size, cat); + rewrite_section_delayed_data(section_type::BLOCK, std::move(data), + uncompressed_size, cat); } template