diff --git a/include/dwarfs/filesystem_writer.h b/include/dwarfs/filesystem_writer.h index 59a1b5a6..0dde7657 100644 --- a/include/dwarfs/filesystem_writer.h +++ b/include/dwarfs/filesystem_writer.h @@ -38,8 +38,13 @@ namespace dwarfs { -class block_compressor; +namespace internal { + class block_data; + +} // namespace internal + +class block_compressor; class logger; class progress; class thread_pool; @@ -87,7 +92,8 @@ class filesystem_writer { // TODO: check which write_block() API is actually used - void write_block(fragment_category cat, std::shared_ptr&& data, + void write_block(fragment_category cat, + std::shared_ptr&& data, physical_block_cb_type physical_block_cb, std::optional meta = std::nullopt) { impl_->write_block(cat, std::move(data), std::move(physical_block_cb), @@ -97,20 +103,20 @@ class filesystem_writer { void finish_category(fragment_category cat) { impl_->finish_category(cat); } void write_block(fragment_category::value_type cat, - std::shared_ptr&& data, + std::shared_ptr&& data, std::optional meta = std::nullopt) { impl_->write_block(cat, std::move(data), std::move(meta)); } - void write_metadata_v2_schema(std::shared_ptr&& data) { + void write_metadata_v2_schema(std::shared_ptr&& data) { impl_->write_metadata_v2_schema(std::move(data)); } - void write_metadata_v2(std::shared_ptr&& data) { + void write_metadata_v2(std::shared_ptr&& data) { impl_->write_metadata_v2(std::move(data)); } - void write_history(std::shared_ptr&& data) { + void write_history(std::shared_ptr&& data) { impl_->write_history(std::move(data)); } @@ -152,18 +158,20 @@ class filesystem_writer { configure(std::vector const& expected_categories, size_t max_active_slots) = 0; virtual void copy_header(std::span header) = 0; - virtual void - write_block(fragment_category cat, std::shared_ptr&& data, - physical_block_cb_type physical_block_cb, - std::optional meta) = 0; + virtual void write_block(fragment_category cat, + std::shared_ptr&& data, + physical_block_cb_type physical_block_cb, + std::optional meta) = 0; virtual void finish_category(fragment_category cat) = 0; virtual void write_block(fragment_category::value_type cat, - std::shared_ptr&& data, + std::shared_ptr&& data, std::optional meta) = 0; virtual void - write_metadata_v2_schema(std::shared_ptr&& data) = 0; - virtual void write_metadata_v2(std::shared_ptr&& data) = 0; - virtual void write_history(std::shared_ptr&& data) = 0; + write_metadata_v2_schema(std::shared_ptr&& data) = 0; + virtual void + write_metadata_v2(std::shared_ptr&& data) = 0; + virtual void + write_history(std::shared_ptr&& data) = 0; virtual void check_block_compression( compression_type compression, std::span data, std::optional cat) = 0; diff --git a/include/dwarfs/block_data.h b/include/dwarfs/internal/block_data.h similarity index 96% rename from include/dwarfs/block_data.h rename to include/dwarfs/internal/block_data.h index adb848c0..5127bc16 100644 --- a/include/dwarfs/block_data.h +++ b/include/dwarfs/internal/block_data.h @@ -25,7 +25,7 @@ #include #include -namespace dwarfs { +namespace dwarfs::internal { class block_data { public: @@ -51,4 +51,4 @@ class block_data { std::vector vec_; }; -} // namespace dwarfs +} // namespace dwarfs::internal diff --git a/include/dwarfs/segmenter.h b/include/dwarfs/segmenter.h index 7c22fe39..8e158bab 100644 --- a/include/dwarfs/segmenter.h +++ b/include/dwarfs/segmenter.h @@ -28,7 +28,12 @@ namespace dwarfs { +namespace internal { + class block_data; + +} // namespace internal + class block_manager; class chunkable; class logger; @@ -47,8 +52,8 @@ class segmenter { unsigned block_size_bits{22}; }; - using block_ready_cb = std::function, - size_t logical_block_num)>; + using block_ready_cb = std::function, size_t logical_block_num)>; segmenter(logger& lgr, progress& prog, std::shared_ptr blkmgr, config const& cfg, compression_constraints const& cc, diff --git a/src/dwarfs/filesystem_v2.cpp b/src/dwarfs/filesystem_v2.cpp index abe52c92..d1e9f2fb 100644 --- a/src/dwarfs/filesystem_v2.cpp +++ b/src/dwarfs/filesystem_v2.cpp @@ -33,7 +33,6 @@ #include #include -#include #include #include #include @@ -43,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -820,7 +820,8 @@ void filesystem_::rewrite(progress& prog, << "), compressing using '" << writer.get_compressor(s->type()).describe() << "'"; - writer.write_history(std::make_shared(hist.serialize())); + writer.write_history( + std::make_shared(hist.serialize())); } else { LOG_VERBOSE << "removing " << get_section_name(s->type()); } diff --git a/src/dwarfs/filesystem_writer.cpp b/src/dwarfs/filesystem_writer.cpp index 13603934..8134a3fd 100644 --- a/src/dwarfs/filesystem_writer.cpp +++ b/src/dwarfs/filesystem_writer.cpp @@ -34,11 +34,11 @@ #include #include -#include #include #include #include #include +#include #include #include #include @@ -94,7 +94,7 @@ class compression_progress : public progress::context { class fsblock { public: fsblock(section_type type, block_compressor const& bc, - std::shared_ptr&& data, + std::shared_ptr&& data, std::shared_ptr pctx, folly::Function set_block_cb = nullptr); @@ -170,7 +170,7 @@ class fsblock_merger_policy { class raw_fsblock : public fsblock::impl { public: raw_fsblock(section_type type, const block_compressor& bc, - std::shared_ptr&& data, + std::shared_ptr&& data, std::shared_ptr pctx, folly::Function set_block_cb) : type_{type} @@ -186,30 +186,32 @@ class raw_fsblock : public fsblock::impl { std::promise prom; future_ = prom.get_future(); - wg.add_job([this, prom = std::move(prom), - meta = std::move(meta)]() mutable { - try { - std::shared_ptr tmp; + wg.add_job( + [this, prom = std::move(prom), meta = std::move(meta)]() mutable { + try { + std::shared_ptr tmp; - if (meta) { - tmp = std::make_shared(bc_.compress(data_->vec(), *meta)); - } else { - tmp = std::make_shared(bc_.compress(data_->vec())); - } + if (meta) { + tmp = std::make_shared( + bc_.compress(data_->vec(), *meta)); + } else { + tmp = std::make_shared( + bc_.compress(data_->vec())); + } - pctx_->bytes_in += data_->vec().size(); - pctx_->bytes_out += tmp->vec().size(); + pctx_->bytes_in += data_->vec().size(); + pctx_->bytes_out += tmp->vec().size(); - { - std::lock_guard lock(mx_); - data_.swap(tmp); - } - } catch (bad_compression_ratio_error const&) { - comp_type_ = compression_type::NONE; - } + { + std::lock_guard lock(mx_); + data_.swap(tmp); + } + } catch (bad_compression_ratio_error const&) { + comp_type_ = compression_type::NONE; + } - prom.set_value(); - }); + prom.set_value(); + }); } void wait_until_compressed() override { future_.wait(); } @@ -262,7 +264,7 @@ class raw_fsblock : public fsblock::impl { block_compressor const& bc_; const size_t uncompressed_size_; mutable std::recursive_mutex mx_; - std::shared_ptr data_; + std::shared_ptr data_; std::future future_; std::optional number_; std::optional mutable header_; @@ -444,7 +446,7 @@ class rewritten_fsblock : public fsblock::impl { }; fsblock::fsblock(section_type type, block_compressor const& bc, - std::shared_ptr&& data, + std::shared_ptr&& data, std::shared_ptr pctx, folly::Function set_block_cb) : impl_(std::make_unique(type, bc, std::move(data), @@ -537,16 +539,18 @@ class filesystem_writer_ final : public filesystem_writer::impl { void configure(std::vector const& expected_categories, size_t max_active_slots) override; void copy_header(std::span header) override; - void write_block(fragment_category cat, std::shared_ptr&& data, + void write_block(fragment_category cat, + std::shared_ptr&& data, physical_block_cb_type physical_block_cb, std::optional meta) override; void finish_category(fragment_category cat) override; void write_block(fragment_category::value_type cat, - std::shared_ptr&& data, + std::shared_ptr&& data, std::optional meta) override; - void write_metadata_v2_schema(std::shared_ptr&& data) override; - void write_metadata_v2(std::shared_ptr&& data) override; - void write_history(std::shared_ptr&& data) override; + void write_metadata_v2_schema( + std::shared_ptr&& data) override; + void write_metadata_v2(std::shared_ptr&& data) override; + void write_history(std::shared_ptr&& data) override; void check_block_compression( compression_type compression, std::span data, std::optional cat) override; @@ -566,11 +570,13 @@ class filesystem_writer_ final : public filesystem_writer::impl { block_compressor const& compressor_for_category(fragment_category::value_type cat) const; void - write_block_impl(fragment_category cat, std::shared_ptr&& data, + write_block_impl(fragment_category cat, + std::shared_ptr&& data, block_compressor const& bc, std::optional meta, physical_block_cb_type physical_block_cb); void on_block_merged(block_holder_type holder); - void write_section_impl(section_type type, std::shared_ptr&& data, + void write_section_impl(section_type type, + std::shared_ptr&& data, block_compressor const& bc, std::optional meta = std::nullopt); void write(fsblock const& fsb); @@ -770,7 +776,7 @@ filesystem_writer_::compressor_for_category( template void filesystem_writer_::write_block_impl( - fragment_category cat, std::shared_ptr&& data, + fragment_category cat, std::shared_ptr&& data, block_compressor const& bc, std::optional meta, physical_block_cb_type physical_block_cb) { if (!merger_) { @@ -831,7 +837,7 @@ void filesystem_writer_::finish_category(fragment_category cat) { template void filesystem_writer_::write_section_impl( - section_type type, std::shared_ptr&& data, + section_type type, std::shared_ptr&& data, block_compressor const& bc, std::optional meta) { uint32_t number; @@ -1017,7 +1023,7 @@ void filesystem_writer_::copy_header( template void filesystem_writer_::write_block( - fragment_category cat, std::shared_ptr&& data, + fragment_category cat, std::shared_ptr&& data, physical_block_cb_type physical_block_cb, std::optional meta) { write_block_impl(cat, std::move(data), compressor_for_category(cat.value()), std::move(meta), std::move(physical_block_cb)); @@ -1025,7 +1031,8 @@ void filesystem_writer_::write_block( template void filesystem_writer_::write_block( - fragment_category::value_type cat, std::shared_ptr&& data, + fragment_category::value_type cat, + std::shared_ptr&& data, std::optional meta) { write_section_impl(section_type::BLOCK, std::move(data), compressor_for_category(cat), std::move(meta)); @@ -1033,20 +1040,20 @@ void filesystem_writer_::write_block( template void filesystem_writer_::write_metadata_v2_schema( - std::shared_ptr&& data) { + std::shared_ptr&& data) { write_section_impl(section_type::METADATA_V2_SCHEMA, std::move(data), schema_bc_); } template void filesystem_writer_::write_metadata_v2( - std::shared_ptr&& data) { + std::shared_ptr&& data) { write_section_impl(section_type::METADATA_V2, std::move(data), metadata_bc_); } template void filesystem_writer_::write_history( - std::shared_ptr&& data) { + std::shared_ptr&& data) { write_section_impl(section_type::HISTORY, std::move(data), history_bc_); } diff --git a/src/dwarfs/scanner.cpp b/src/dwarfs/scanner.cpp index 7144834f..2c2fc945 100644 --- a/src/dwarfs/scanner.cpp +++ b/src/dwarfs/scanner.cpp @@ -38,7 +38,6 @@ #include -#include #include #include #include @@ -52,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -989,13 +989,15 @@ void scanner_::scan( LOG_VERBOSE << "uncompressed metadata size: " << size_with_unit(data.size()); - fsw.write_metadata_v2_schema(std::make_shared(std::move(schema))); - fsw.write_metadata_v2(std::make_shared(std::move(data))); + fsw.write_metadata_v2_schema( + std::make_shared(std::move(schema))); + fsw.write_metadata_v2( + std::make_shared(std::move(data))); if (options_.enable_history) { history hist(options_.history); hist.append(options_.command_line_arguments); - fsw.write_history(std::make_shared(hist.serialize())); + fsw.write_history(std::make_shared(hist.serialize())); } LOG_INFO << "waiting for compression to finish..."; diff --git a/src/dwarfs/segmenter.cpp b/src/dwarfs/segmenter.cpp index 42ae495b..b693f13b 100644 --- a/src/dwarfs/segmenter.cpp +++ b/src/dwarfs/segmenter.cpp @@ -41,7 +41,6 @@ #include #include -#include #include #include #include @@ -49,6 +48,7 @@ #include #include #include +#include #include #include #include @@ -567,7 +567,7 @@ class active_block : private GranularityPolicy { , filter_(bloom_filter_size) , repseqmap_{repseqmap} , repeating_collisions_{repcoll} - , data_{std::make_shared()} { + , data_{std::make_shared()} { DWARFS_CHECK((window_step & window_step_mask_) == 0, "window step size not a power of two"); data_->reserve(this->frames_to_bytes(capacity_in_frames_)); @@ -583,7 +583,9 @@ class active_block : private GranularityPolicy { return size_in_frames() == capacity_in_frames_; } - DWARFS_FORCE_INLINE std::shared_ptr data() const { return data_; } + DWARFS_FORCE_INLINE std::shared_ptr data() const { + return data_; + } DWARFS_FORCE_INLINE void append_bytes(std::span data, bloom_filter& global_filter); @@ -628,7 +630,7 @@ class active_block : private GranularityPolicy { fast_multimap offsets_; repeating_sequence_map_type const& repseqmap_; repeating_collisions_map_type& repeating_collisions_; - std::shared_ptr data_; + std::shared_ptr data_; }; class segmenter_progress : public progress::context { diff --git a/test/segmenter_benchmark.cpp b/test/segmenter_benchmark.cpp index 0278a98a..53a61950 100644 --- a/test/segmenter_benchmark.cpp +++ b/test/segmenter_benchmark.cpp @@ -24,10 +24,10 @@ #include -#include #include #include #include +#include #include #include @@ -142,11 +142,11 @@ void run_segmenter_test(unsigned iters, unsigned granularity, dwarfs::progress prog; auto blkmgr = std::make_shared(); - std::vector> written; + std::vector> written; dwarfs::segmenter seg( lgr, prog, blkmgr, cfg, cc, total_size, - [&written, blkmgr](std::shared_ptr blk, + [&written, blkmgr](std::shared_ptr blk, auto logical_block_num) { auto physical_block_num = written.size(); written.push_back(blk);