From c8bfc821405d94313bffa038d5759f9bf88372b7 Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Sun, 28 Jul 2024 12:18:29 +0200 Subject: [PATCH] refactor: internal namespace cleanup --- .../detail/multi_queue_block_merger_impl.h | 3 +- include/dwarfs/internal/inode_reader_v2.h | 3 +- include/dwarfs/internal/metadata_v2.h | 3 +- src/dwarfs/block_range.cpp | 1 + src/dwarfs/builtin_script.cpp | 8 +- src/dwarfs/categorizer.cpp | 128 ++++++++--------- src/dwarfs/conv.cpp | 6 +- src/dwarfs/entry.cpp | 3 +- src/dwarfs/file_scanner.cpp | 3 +- src/dwarfs/filesystem_extractor.cpp | 13 +- src/dwarfs/filesystem_v2.cpp | 49 ++++--- src/dwarfs/filesystem_writer.cpp | 131 +++++++++--------- src/dwarfs/inode_manager.cpp | 31 +++-- src/dwarfs/inode_ordering.cpp | 25 ++-- src/dwarfs/internal/block_cache.cpp | 7 +- src/dwarfs/internal/cached_block.cpp | 3 +- src/dwarfs/internal/global_entry_data.cpp | 3 +- src/dwarfs/internal/inode_reader_v2.cpp | 7 +- src/dwarfs/internal/metadata_types.cpp | 3 +- src/dwarfs/internal/metadata_v2.cpp | 3 +- src/dwarfs/internal/worker_group.cpp | 3 +- src/dwarfs/metadata_types.cpp | 3 +- src/dwarfs/performance_monitor.cpp | 6 +- src/dwarfs/scanner.cpp | 52 +++---- src/dwarfs/segmenter.cpp | 30 ++-- src/dwarfs/segmenter_factory.cpp | 10 +- src/dwarfs/similarity_ordering.cpp | 13 +- src/dwarfs/thread_pool.cpp | 3 +- test/block_cache_test.cpp | 3 +- test/segmenter_benchmark.cpp | 5 +- test/utils_test.cpp | 3 +- 31 files changed, 307 insertions(+), 257 deletions(-) diff --git a/include/dwarfs/internal/detail/multi_queue_block_merger_impl.h b/include/dwarfs/internal/detail/multi_queue_block_merger_impl.h index 435fbb1f..445a9912 100644 --- a/include/dwarfs/internal/detail/multi_queue_block_merger_impl.h +++ b/include/dwarfs/internal/detail/multi_queue_block_merger_impl.h @@ -40,9 +40,10 @@ #include #include -#include #include +#include + namespace dwarfs::internal::detail { template diff --git a/include/dwarfs/internal/inode_reader_v2.h b/include/dwarfs/internal/inode_reader_v2.h index 2a3af18c..6bf80b20 100644 --- a/include/dwarfs/internal/inode_reader_v2.h +++ b/include/dwarfs/internal/inode_reader_v2.h @@ -29,9 +29,10 @@ #include #include -#include #include +#include + namespace dwarfs { struct cache_tidy_config; diff --git a/include/dwarfs/internal/metadata_v2.h b/include/dwarfs/internal/metadata_v2.h index d4ac5202..807c7c51 100644 --- a/include/dwarfs/internal/metadata_v2.h +++ b/include/dwarfs/internal/metadata_v2.h @@ -34,9 +34,10 @@ #include -#include #include +#include + namespace dwarfs { class logger; diff --git a/src/dwarfs/block_range.cpp b/src/dwarfs/block_range.cpp index 0aa3c1a8..f0ee3889 100644 --- a/src/dwarfs/block_range.cpp +++ b/src/dwarfs/block_range.cpp @@ -23,6 +23,7 @@ #include #include + #include namespace dwarfs { diff --git a/src/dwarfs/builtin_script.cpp b/src/dwarfs/builtin_script.cpp index fb83f623..bec7c42f 100644 --- a/src/dwarfs/builtin_script.cpp +++ b/src/dwarfs/builtin_script.cpp @@ -34,6 +34,8 @@ namespace dwarfs { +namespace internal { + struct filter_rule { enum class rule_type { include, @@ -280,10 +282,12 @@ void builtin_script_::transform(entry_interface& ei) { } } +} // namespace internal + builtin_script::builtin_script(logger& lgr, std::shared_ptr fa) - : impl_(make_unique_logging_object( - lgr, std::move(fa))) {} + : impl_(make_unique_logging_object(lgr, std::move(fa))) {} builtin_script::~builtin_script() = default; diff --git a/src/dwarfs/categorizer.cpp b/src/dwarfs/categorizer.cpp index a09f04c2..5840adbd 100644 --- a/src/dwarfs/categorizer.cpp +++ b/src/dwarfs/categorizer.cpp @@ -39,59 +39,10 @@ namespace dwarfs { +namespace internal { + using namespace std::placeholders; -namespace po = boost::program_options; - -std::string category_prefix(std::shared_ptr const& mgr, - fragment_category cat) { - return category_prefix(mgr.get(), cat); -} - -std::string category_prefix(std::unique_ptr const& mgr, - fragment_category cat) { - return category_prefix(mgr.get(), cat); -} - -std::string category_prefix(std::shared_ptr const& mgr, - fragment_category::value_type cat) { - return category_prefix(mgr.get(), cat); -} - -std::string category_prefix(std::unique_ptr const& mgr, - fragment_category::value_type cat) { - return category_prefix(mgr.get(), cat); -} - -std::string category_prefix(categorizer_manager const* mgr, - fragment_category::value_type cat) { - return category_prefix(mgr, fragment_category(cat)); -} - -std::string -category_prefix(categorizer_manager const* mgr, fragment_category cat) { - std::string prefix; - if (mgr) { - prefix = fmt::format( - "[{}{}] ", mgr->category_name(cat.value()), - cat.has_subcategory() ? fmt::format("/{}", cat.subcategory()) : ""); - } - return prefix; -} - -std::string -categorizer::category_metadata(std::string_view, fragment_category) const { - return std::string(); -} - -void categorizer::set_metadata_requirements(std::string_view, - std::string requirements) { - if (!requirements.empty()) { - compression_metadata_requirements().parse( - nlohmann::json::parse(requirements)); - } -} - class categorizer_manager_private : public categorizer_manager::impl { public: virtual std::vector> const& @@ -213,11 +164,6 @@ bool categorizer_job_::best_result_found() const { return is_global_best_; } -categorizer_job::categorizer_job() = default; - -categorizer_job::categorizer_job(std::unique_ptr impl) - : impl_{std::move(impl)} {} - template class categorizer_manager_ final : public categorizer_manager_private { public: @@ -278,10 +224,6 @@ class categorizer_manager_ final : public categorizer_manager_private { std::unordered_map catmap_; }; -fragment_category categorizer_manager::default_category() { - return fragment_category(0); -} - template void categorizer_manager_::add(std::shared_ptr c) { for (auto const& cat : c->categories()) { @@ -354,10 +296,72 @@ bool categorizer_manager_::deterministic_less( return categorizer->subcategory_less(a, b); } +} // namespace internal + +namespace po = boost::program_options; + +std::string category_prefix(std::shared_ptr const& mgr, + fragment_category cat) { + return category_prefix(mgr.get(), cat); +} + +std::string category_prefix(std::unique_ptr const& mgr, + fragment_category cat) { + return category_prefix(mgr.get(), cat); +} + +std::string category_prefix(std::shared_ptr const& mgr, + fragment_category::value_type cat) { + return category_prefix(mgr.get(), cat); +} + +std::string category_prefix(std::unique_ptr const& mgr, + fragment_category::value_type cat) { + return category_prefix(mgr.get(), cat); +} + +std::string category_prefix(categorizer_manager const* mgr, + fragment_category::value_type cat) { + return category_prefix(mgr, fragment_category(cat)); +} + +std::string +category_prefix(categorizer_manager const* mgr, fragment_category cat) { + std::string prefix; + if (mgr) { + prefix = fmt::format( + "[{}{}] ", mgr->category_name(cat.value()), + cat.has_subcategory() ? fmt::format("/{}", cat.subcategory()) : ""); + } + return prefix; +} + +std::string +categorizer::category_metadata(std::string_view, fragment_category) const { + return std::string(); +} + +void categorizer::set_metadata_requirements(std::string_view, + std::string requirements) { + if (!requirements.empty()) { + compression_metadata_requirements().parse( + nlohmann::json::parse(requirements)); + } +} + +categorizer_job::categorizer_job() = default; + +categorizer_job::categorizer_job(std::unique_ptr impl) + : impl_{std::move(impl)} {} + categorizer_manager::categorizer_manager(logger& lgr) - : impl_(make_unique_logging_object(lgr)) {} +fragment_category categorizer_manager::default_category() { + return fragment_category(0); +} + categorizer_registry& categorizer_registry::instance() { static categorizer_registry the_instance; return the_instance; @@ -369,7 +373,7 @@ void categorizer_registry::register_factory( if (!factories_.emplace(name, std::move(factory)).second) { std::cerr << "categorizer factory name conflict (" << name << "\n"; - ::abort(); + std::abort(); } } diff --git a/src/dwarfs/conv.cpp b/src/dwarfs/conv.cpp index d58a9ffb..8786675c 100644 --- a/src/dwarfs/conv.cpp +++ b/src/dwarfs/conv.cpp @@ -23,8 +23,7 @@ #include -namespace dwarfs { -namespace detail { +namespace dwarfs::detail { std::optional str_to_bool(std::string_view s) { if (auto r = folly::tryTo(s)) { @@ -33,5 +32,4 @@ std::optional str_to_bool(std::string_view s) { return std::nullopt; } -} // namespace detail -} // namespace dwarfs +} // namespace dwarfs::detail diff --git a/src/dwarfs/entry.cpp b/src/dwarfs/entry.cpp index 2cc8be8d..2b870aa0 100644 --- a/src/dwarfs/entry.cpp +++ b/src/dwarfs/entry.cpp @@ -30,7 +30,6 @@ #include #include #include -#include #include #include #include @@ -39,6 +38,8 @@ #include #include +#include + #include namespace dwarfs { diff --git a/src/dwarfs/file_scanner.cpp b/src/dwarfs/file_scanner.cpp index b2085acc..9bdb1b67 100644 --- a/src/dwarfs/file_scanner.cpp +++ b/src/dwarfs/file_scanner.cpp @@ -38,7 +38,6 @@ #include #include #include -#include #include #include #include @@ -46,6 +45,8 @@ #include #include +#include + namespace dwarfs::detail { namespace { diff --git a/src/dwarfs/filesystem_extractor.cpp b/src/dwarfs/filesystem_extractor.cpp index 109ebb20..bafb9190 100644 --- a/src/dwarfs/filesystem_extractor.cpp +++ b/src/dwarfs/filesystem_extractor.cpp @@ -43,15 +43,18 @@ #include #include #include -#include #include #include #include #include #include +#include + namespace dwarfs { +namespace internal { + namespace { class cache_semaphore { @@ -242,7 +245,7 @@ bool filesystem_extractor_::extract( ::archive_entry* spare = nullptr; - internal::worker_group archiver(LOG_GET_LOGGER, os_, "archiver", 1); + worker_group archiver(LOG_GET_LOGGER, os_, "archiver", 1); cache_semaphore sem; LOG_DEBUG << "extractor semaphore size: " << opts.max_queued_bytes @@ -423,9 +426,11 @@ bool filesystem_extractor_::extract( return true; } +} // namespace internal + filesystem_extractor::filesystem_extractor(logger& lgr, os_access const& os) : impl_(make_unique_logging_object( - lgr, os)) {} + internal::filesystem_extractor_, + logger_policies>(lgr, os)) {} } // namespace dwarfs diff --git a/src/dwarfs/filesystem_v2.cpp b/src/dwarfs/filesystem_v2.cpp index d1e9f2fb..7da18596 100644 --- a/src/dwarfs/filesystem_v2.cpp +++ b/src/dwarfs/filesystem_v2.cpp @@ -41,11 +41,6 @@ #include #include #include -#include -#include -#include -#include -#include #include #include #include @@ -53,8 +48,16 @@ #include #include +#include +#include +#include +#include +#include + namespace dwarfs { +namespace internal { + namespace { void check_section_logger(logger& lgr, fs_section const& section) { @@ -357,7 +360,7 @@ get_section_data(std::shared_ptr mm, fs_section const& section, return buffer; } -internal::metadata_v2 +metadata_v2 make_metadata(logger& lgr, std::shared_ptr mm, section_map const& sections, std::vector& schema_buffer, std::vector& meta_buffer, @@ -408,13 +411,15 @@ make_metadata(logger& lgr, std::shared_ptr mm, } } - return internal::metadata_v2(lgr, - get_section_data(mm, schema_it->second.front(), - schema_buffer, force_buffers), - meta_section_range, options, inode_offset, - force_consistency_check, perfmon); + return metadata_v2(lgr, + get_section_data(mm, schema_it->second.front(), + schema_buffer, force_buffers), + meta_section_range, options, inode_offset, + force_consistency_check, perfmon); } +} // namespace + template class filesystem_ final : public filesystem_v2::impl { public: @@ -497,8 +502,8 @@ class filesystem_ final : public filesystem_v2::impl { LOG_PROXY_DECL(LoggerPolicy); os_access const& os_; std::shared_ptr mm_; - internal::metadata_v2 meta_; - internal::inode_reader_v2 ir_; + metadata_v2 meta_; + inode_reader_v2 ir_; mutable std::mutex mx_; std::vector meta_buffer_; std::optional> header_; @@ -610,7 +615,7 @@ filesystem_::filesystem_( PERFMON_CLS_TIMER_INIT(readv_future) PERFMON_CLS_TIMER_INIT(readv_future_throw) // clang-format on { - internal::block_cache cache(lgr, os_, mm_, options.block_cache, perfmon); + block_cache cache(lgr, os_, mm_, options.block_cache, perfmon); filesystem_parser parser(mm_, image_offset_); if (parser.has_index()) { @@ -662,8 +667,7 @@ filesystem_::filesystem_( cache.set_block_size(meta_.block_size()); - ir_ = internal::inode_reader_v2(lgr, std::move(cache), options.inode_reader, - perfmon); + ir_ = inode_reader_v2(lgr, std::move(cache), options.inode_reader, perfmon); if (auto it = sections.find(section_type::HISTORY); it != sections.end()) { for (auto& section : it->second) { @@ -820,8 +824,7 @@ void filesystem_::rewrite(progress& prog, << "), compressing using '" << writer.get_compressor(s->type()).describe() << "'"; - writer.write_history( - std::make_shared(hist.serialize())); + writer.write_history(std::make_shared(hist.serialize())); } else { LOG_VERBOSE << "removing " << get_section_name(s->type()); } @@ -846,7 +849,7 @@ int filesystem_::check(filesystem_check_level level, size_t num_threads) const { filesystem_parser parser(mm_, image_offset_); - internal::worker_group wg(LOG_GET_LOGGER, os_, "fscheck", num_threads); + worker_group wg(LOG_GET_LOGGER, os_, "fscheck", num_threads); std::vector> sections; while (auto sp = parser.next_section()) { @@ -1231,7 +1234,7 @@ filesystem_::header() const { return header_; } -} // namespace +} // namespace internal filesystem_v2::filesystem_v2(logger& lgr, os_access const& os, std::shared_ptr mm) @@ -1241,8 +1244,8 @@ filesystem_v2::filesystem_v2(logger& lgr, os_access const& os, std::shared_ptr mm, const filesystem_options& options, std::shared_ptr perfmon) - : impl_(make_unique_logging_object( + : impl_(make_unique_logging_object( lgr, os, std::move(mm), options, std::move(perfmon))) {} int filesystem_v2::identify(logger& lgr, os_access const& os, @@ -1270,7 +1273,7 @@ filesystem_v2::header(std::shared_ptr mm) { std::optional> filesystem_v2::header(std::shared_ptr mm, file_off_t image_offset) { - return filesystem_parser(mm, image_offset).header(); + return internal::filesystem_parser(mm, image_offset).header(); } } // namespace dwarfs diff --git a/src/dwarfs/filesystem_writer.cpp b/src/dwarfs/filesystem_writer.cpp index 8134a3fd..2164d126 100644 --- a/src/dwarfs/filesystem_writer.cpp +++ b/src/dwarfs/filesystem_writer.cpp @@ -38,16 +38,19 @@ #include #include #include -#include -#include -#include #include #include #include #include +#include +#include +#include + namespace dwarfs { +namespace internal { + namespace { size_t copy_stream(std::istream& is, std::ostream& os) { @@ -94,7 +97,7 @@ class compression_progress : public progress::context { class fsblock { public: fsblock(section_type type, block_compressor const& bc, - std::shared_ptr&& data, + std::shared_ptr&& data, std::shared_ptr pctx, folly::Function set_block_cb = nullptr); @@ -108,8 +111,8 @@ class fsblock { std::span data, compression_type data_comp_type, std::shared_ptr pctx); - void compress(internal::worker_group& wg, - std::optional meta = std::nullopt) { + void + compress(worker_group& wg, std::optional meta = std::nullopt) { impl_->compress(wg, std::move(meta)); } void wait_until_compressed() { impl_->wait_until_compressed(); } @@ -128,7 +131,7 @@ class fsblock { virtual ~impl() = default; virtual void - compress(internal::worker_group& wg, std::optional meta) = 0; + compress(worker_group& wg, std::optional meta) = 0; virtual void wait_until_compressed() = 0; virtual section_type type() const = 0; virtual compression_type compression() const = 0; @@ -170,7 +173,7 @@ class fsblock_merger_policy { class raw_fsblock : public fsblock::impl { public: raw_fsblock(section_type type, const block_compressor& bc, - std::shared_ptr&& data, + std::shared_ptr&& data, std::shared_ptr pctx, folly::Function set_block_cb) : type_{type} @@ -181,37 +184,34 @@ class raw_fsblock : public fsblock::impl { , pctx_{std::move(pctx)} , set_block_cb_{std::move(set_block_cb)} {} - void compress(internal::worker_group& wg, - std::optional meta) override { + void compress(worker_group& wg, std::optional meta) override { std::promise prom; future_ = prom.get_future(); - wg.add_job( - [this, prom = std::move(prom), meta = std::move(meta)]() mutable { - try { - std::shared_ptr tmp; + wg.add_job([this, prom = std::move(prom), + meta = std::move(meta)]() mutable { + try { + std::shared_ptr tmp; - if (meta) { - tmp = std::make_shared( - bc_.compress(data_->vec(), *meta)); - } else { - tmp = std::make_shared( - bc_.compress(data_->vec())); - } + if (meta) { + tmp = std::make_shared(bc_.compress(data_->vec(), *meta)); + } else { + tmp = std::make_shared(bc_.compress(data_->vec())); + } - pctx_->bytes_in += data_->vec().size(); - pctx_->bytes_out += tmp->vec().size(); + pctx_->bytes_in += data_->vec().size(); + pctx_->bytes_out += tmp->vec().size(); - { - std::lock_guard lock(mx_); - data_.swap(tmp); - } - } catch (bad_compression_ratio_error const&) { - comp_type_ = compression_type::NONE; - } + { + std::lock_guard lock(mx_); + data_.swap(tmp); + } + } catch (bad_compression_ratio_error const&) { + comp_type_ = compression_type::NONE; + } - prom.set_value(); - }); + prom.set_value(); + }); } void wait_until_compressed() override { future_.wait(); } @@ -264,7 +264,7 @@ class raw_fsblock : public fsblock::impl { block_compressor const& bc_; const size_t uncompressed_size_; mutable std::recursive_mutex mx_; - std::shared_ptr data_; + std::shared_ptr data_; std::future future_; std::optional number_; std::optional mutable header_; @@ -289,8 +289,8 @@ class compressed_fsblock : public fsblock::impl { , pctx_{std::move(pctx)} , sec_{std::move(sec)} {} - void compress(internal::worker_group& wg, - std::optional /* meta */) override { + void + compress(worker_group& wg, std::optional /* meta */) override { std::promise prom; future_ = prom.get_future(); @@ -346,8 +346,7 @@ class rewritten_fsblock : public fsblock::impl { , pctx_{std::move(pctx)} , data_comp_type_{data_comp_type} {} - void compress(internal::worker_group& wg, - std::optional meta) override { + void compress(worker_group& wg, std::optional meta) override { std::promise prom; future_ = prom.get_future(); @@ -446,7 +445,7 @@ class rewritten_fsblock : public fsblock::impl { }; fsblock::fsblock(section_type type, block_compressor const& bc, - std::shared_ptr&& data, + std::shared_ptr&& data, std::shared_ptr pctx, folly::Function set_block_cb) : impl_(std::make_unique(type, bc, std::move(data), @@ -514,12 +513,14 @@ void fsblock::build_section_header(section_header_v2& sh, DWARFS_CHECK(sha.finalize(&sh.sha2_512_256), "SHA512/256 checksum failed"); } +} // namespace + template class filesystem_writer_ final : public filesystem_writer::impl { public: using physical_block_cb_type = filesystem_writer::physical_block_cb_type; - filesystem_writer_(logger& lgr, std::ostream& os, internal::worker_group& wg, + filesystem_writer_(logger& lgr, std::ostream& os, worker_group& wg, progress& prog, block_compressor const& schema_bc, block_compressor const& metadata_bc, block_compressor const& history_bc, @@ -539,18 +540,16 @@ class filesystem_writer_ final : public filesystem_writer::impl { void configure(std::vector const& expected_categories, size_t max_active_slots) override; void copy_header(std::span header) override; - void write_block(fragment_category cat, - std::shared_ptr&& data, + void write_block(fragment_category cat, std::shared_ptr&& data, physical_block_cb_type physical_block_cb, std::optional meta) override; void finish_category(fragment_category cat) override; void write_block(fragment_category::value_type cat, - std::shared_ptr&& data, + std::shared_ptr&& data, std::optional meta) override; - void write_metadata_v2_schema( - std::shared_ptr&& data) override; - void write_metadata_v2(std::shared_ptr&& data) override; - void write_history(std::shared_ptr&& data) override; + void write_metadata_v2_schema(std::shared_ptr&& data) override; + void write_metadata_v2(std::shared_ptr&& data) override; + void write_history(std::shared_ptr&& data) override; void check_block_compression( compression_type compression, std::span data, std::optional cat) override; @@ -563,20 +562,19 @@ class filesystem_writer_ final : public filesystem_writer::impl { size_t size() const override { return image_size_; } private: - using block_merger_type = internal::multi_queue_block_merger< - fragment_category, std::unique_ptr, fsblock_merger_policy>; + using block_merger_type = + multi_queue_block_merger, + fsblock_merger_policy>; using block_holder_type = block_merger_type::block_holder_type; block_compressor const& compressor_for_category(fragment_category::value_type cat) const; void - write_block_impl(fragment_category cat, - std::shared_ptr&& data, + write_block_impl(fragment_category cat, std::shared_ptr&& data, block_compressor const& bc, std::optional meta, physical_block_cb_type physical_block_cb); void on_block_merged(block_holder_type holder); - void write_section_impl(section_type type, - std::shared_ptr&& data, + void write_section_impl(section_type type, std::shared_ptr&& data, block_compressor const& bc, std::optional meta = std::nullopt); void write(fsblock const& fsb); @@ -592,7 +590,7 @@ class filesystem_writer_ final : public filesystem_writer::impl { std::ostream& os_; size_t image_size_{0}; std::istream* header_; - internal::worker_group& wg_; + worker_group& wg_; progress& prog_; std::optional default_bc_; std::unordered_map bc_; @@ -617,7 +615,7 @@ class filesystem_writer_ final : public filesystem_writer::impl { // into something that gets passed a (section_type, category) pair? template filesystem_writer_::filesystem_writer_( - logger& lgr, std::ostream& os, internal::worker_group& wg, progress& prog, + logger& lgr, std::ostream& os, worker_group& wg, progress& prog, block_compressor const& schema_bc, block_compressor const& metadata_bc, block_compressor const& history_bc, filesystem_writer_options const& options, std::istream* header) @@ -776,7 +774,7 @@ filesystem_writer_::compressor_for_category( template void filesystem_writer_::write_block_impl( - fragment_category cat, std::shared_ptr&& data, + fragment_category cat, std::shared_ptr&& data, block_compressor const& bc, std::optional meta, physical_block_cb_type physical_block_cb) { if (!merger_) { @@ -837,7 +835,7 @@ void filesystem_writer_::finish_category(fragment_category cat) { template void filesystem_writer_::write_section_impl( - section_type type, std::shared_ptr&& data, + section_type type, std::shared_ptr&& data, block_compressor const& bc, std::optional meta) { uint32_t number; @@ -1023,7 +1021,7 @@ void filesystem_writer_::copy_header( template void filesystem_writer_::write_block( - fragment_category cat, std::shared_ptr&& data, + fragment_category cat, std::shared_ptr&& data, physical_block_cb_type physical_block_cb, std::optional meta) { write_block_impl(cat, std::move(data), compressor_for_category(cat.value()), std::move(meta), std::move(physical_block_cb)); @@ -1031,8 +1029,7 @@ void filesystem_writer_::write_block( template void filesystem_writer_::write_block( - fragment_category::value_type cat, - std::shared_ptr&& data, + fragment_category::value_type cat, std::shared_ptr&& data, std::optional meta) { write_section_impl(section_type::BLOCK, std::move(data), compressor_for_category(cat), std::move(meta)); @@ -1040,20 +1037,20 @@ void filesystem_writer_::write_block( template void filesystem_writer_::write_metadata_v2_schema( - std::shared_ptr&& data) { + std::shared_ptr&& data) { write_section_impl(section_type::METADATA_V2_SCHEMA, std::move(data), schema_bc_); } template void filesystem_writer_::write_metadata_v2( - std::shared_ptr&& data) { + std::shared_ptr&& data) { write_section_impl(section_type::METADATA_V2, std::move(data), metadata_bc_); } template void filesystem_writer_::write_history( - std::shared_ptr&& data) { + std::shared_ptr&& data) { write_section_impl(section_type::HISTORY, std::move(data), history_bc_); } @@ -1099,7 +1096,7 @@ void filesystem_writer_::write_section_index() { write(fsb); } -} // namespace +} // namespace internal filesystem_writer::filesystem_writer(std::ostream& os, logger& lgr, thread_pool& pool, progress& prog, @@ -1108,9 +1105,9 @@ filesystem_writer::filesystem_writer(std::ostream& os, logger& lgr, block_compressor const& history_bc, filesystem_writer_options const& options, std::istream* header) - : impl_( - make_unique_logging_object( - lgr, os, pool.get_worker_group(), prog, schema_bc, metadata_bc, - history_bc, options, header)) {} + : impl_(make_unique_logging_object( + lgr, os, pool.get_worker_group(), prog, schema_bc, metadata_bc, + history_bc, options, header)) {} } // namespace dwarfs diff --git a/src/dwarfs/inode_manager.cpp b/src/dwarfs/inode_manager.cpp index d7c2e8c9..4670e416 100644 --- a/src/dwarfs/inode_manager.cpp +++ b/src/dwarfs/inode_manager.cpp @@ -46,7 +46,6 @@ #include #include #include -#include #include #include #include @@ -61,15 +60,21 @@ #include #include +#include + #include namespace dwarfs { +namespace internal { + namespace { constexpr std::string_view const kScanContext{"[scanning] "}; constexpr std::string_view const kCategorizeContext{"[categorizing] "}; +} // namespace + class inode_ : public inode { public: using chunk_type = thrift::metadata::chunk; @@ -529,8 +534,6 @@ class inode_ : public inode { similarity_; }; -} // namespace - template class inode_manager_ final : public inode_manager::impl { public: @@ -607,13 +610,12 @@ class inode_manager_ final : public inode_manager::impl { return rv; } - void scan_background(internal::worker_group& wg, os_access const& os, + void scan_background(worker_group& wg, os_access const& os, std::shared_ptr ino, file* p) const override; bool has_invalid_inodes() const override; - void - try_scan_invalid(internal::worker_group& wg, os_access const& os) override; + void try_scan_invalid(worker_group& wg, os_access const& os) override; void dump(std::ostream& os) const override; @@ -621,8 +623,8 @@ class inode_manager_ final : public inode_manager::impl { return sortable_inode_span(inodes_); } - sortable_inode_span ordered_span(fragment_category cat, - internal::worker_group& wg) const override; + sortable_inode_span + ordered_span(fragment_category cat, worker_group& wg) const override; private: void update_prog(std::shared_ptr const& ino, file const* p) const { @@ -653,7 +655,7 @@ class inode_manager_ final : public inode_manager::impl { }; template -void inode_manager_::scan_background(internal::worker_group& wg, +void inode_manager_::scan_background(worker_group& wg, os_access const& os, std::shared_ptr ino, file* p) const { @@ -698,7 +700,7 @@ bool inode_manager_::has_invalid_inodes() const { } template -void inode_manager_::try_scan_invalid(internal::worker_group& wg, +void inode_manager_::try_scan_invalid(worker_group& wg, os_access const& os) { LOG_VERBOSE << "trying to scan " << num_invalid_inodes_.load() << " invalid inodes..."; @@ -752,8 +754,7 @@ void inode_manager_::dump(std::ostream& os) const { template auto inode_manager_::ordered_span( - fragment_category cat, - internal::worker_group& wg) const -> sortable_inode_span { + fragment_category cat, worker_group& wg) const -> sortable_inode_span { auto prefix = category_prefix(opts_.categorizer_mgr, cat); auto opts = opts_.fragment_order.get(cat); @@ -811,9 +812,11 @@ auto inode_manager_::ordered_span( return span; } +} // namespace internal + inode_manager::inode_manager(logger& lgr, progress& prog, inode_options const& opts) - : impl_(make_unique_logging_object( - lgr, prog, opts)) {} + : impl_(make_unique_logging_object(lgr, prog, opts)) {} } // namespace dwarfs diff --git a/src/dwarfs/inode_ordering.cpp b/src/dwarfs/inode_ordering.cpp index 6a4a35fa..000c0e41 100644 --- a/src/dwarfs/inode_ordering.cpp +++ b/src/dwarfs/inode_ordering.cpp @@ -24,14 +24,17 @@ #include #include #include -#include #include #include #include #include +#include + namespace dwarfs { +namespace internal { + namespace { bool inode_less_by_size(inode const* a, inode const* b) { @@ -40,6 +43,8 @@ bool inode_less_by_size(inode const* a, inode const* b) { return sa > sb || (sa == sb && a->any()->less_revpath(*b->any())); } +} // namespace + template class inode_ordering_ final : public inode_ordering::impl { public: @@ -54,14 +59,12 @@ class inode_ordering_ final : public inode_ordering::impl { void by_similarity(sortable_inode_span& sp, fragment_category cat) const override; void - by_nilsimsa(internal::worker_group& wg, - similarity_ordering_options const& opts, sortable_inode_span& sp, - fragment_category cat) const override; + by_nilsimsa(worker_group& wg, similarity_ordering_options const& opts, + sortable_inode_span& sp, fragment_category cat) const override; private: void - by_nilsimsa_impl(internal::worker_group& wg, - similarity_ordering_options const& opts, + by_nilsimsa_impl(worker_group& wg, similarity_ordering_options const& opts, std::span const> inodes, std::vector& index, fragment_category cat) const; @@ -160,7 +163,7 @@ void inode_ordering_::by_similarity(sortable_inode_span& sp, template void inode_ordering_::by_nilsimsa( - internal::worker_group& wg, similarity_ordering_options const& opts, + worker_group& wg, similarity_ordering_options const& opts, sortable_inode_span& sp, fragment_category cat) const { auto raw = sp.raw(); auto& index = sp.index(); @@ -190,7 +193,7 @@ void inode_ordering_::by_nilsimsa( template void inode_ordering_::by_nilsimsa_impl( - internal::worker_group& wg, similarity_ordering_options const& opts, + worker_group& wg, similarity_ordering_options const& opts, std::span const> inodes, std::vector& index, fragment_category cat) const { auto ev = inode_element_view(inodes, index, cat); @@ -202,11 +205,11 @@ void inode_ordering_::by_nilsimsa_impl( future.get().swap(index); } -} // namespace +} // namespace internal inode_ordering::inode_ordering(logger& lgr, progress& prog, inode_options const& opts) - : impl_(make_unique_logging_object( - lgr, prog, opts)) {} + : impl_(make_unique_logging_object(lgr, prog, opts)) {} } // namespace dwarfs diff --git a/src/dwarfs/internal/block_cache.cpp b/src/dwarfs/internal/block_cache.cpp index 2a3a49d2..d47e419d 100644 --- a/src/dwarfs/internal/block_cache.cpp +++ b/src/dwarfs/internal/block_cache.cpp @@ -43,15 +43,16 @@ #include #include -#include -#include -#include #include #include #include #include #include +#include +#include +#include + namespace dwarfs::internal { namespace { diff --git a/src/dwarfs/internal/cached_block.cpp b/src/dwarfs/internal/cached_block.cpp index eee19d36..ab53bcf0 100644 --- a/src/dwarfs/internal/cached_block.cpp +++ b/src/dwarfs/internal/cached_block.cpp @@ -28,10 +28,11 @@ #include #include #include -#include #include #include +#include + namespace dwarfs::internal { namespace { diff --git a/src/dwarfs/internal/global_entry_data.cpp b/src/dwarfs/internal/global_entry_data.cpp index cacfc078..7ae6822f 100644 --- a/src/dwarfs/internal/global_entry_data.cpp +++ b/src/dwarfs/internal/global_entry_data.cpp @@ -22,9 +22,10 @@ #include #include -#include #include +#include + namespace dwarfs::internal { template diff --git a/src/dwarfs/internal/inode_reader_v2.cpp b/src/dwarfs/internal/inode_reader_v2.cpp index 91d5785a..76ed74a1 100644 --- a/src/dwarfs/internal/inode_reader_v2.cpp +++ b/src/dwarfs/internal/inode_reader_v2.cpp @@ -33,15 +33,16 @@ #include #include -#include -#include -#include #include #include #include #include #include +#include +#include +#include + namespace dwarfs::internal { namespace { diff --git a/src/dwarfs/internal/metadata_types.cpp b/src/dwarfs/internal/metadata_types.cpp index c7d93095..c919e4a9 100644 --- a/src/dwarfs/internal/metadata_types.cpp +++ b/src/dwarfs/internal/metadata_types.cpp @@ -27,11 +27,12 @@ #include #include -#include #include #include #include +#include + #include namespace dwarfs::internal { diff --git a/src/dwarfs/internal/metadata_v2.cpp b/src/dwarfs/internal/metadata_v2.cpp index b24cac82..a37b97ec 100644 --- a/src/dwarfs/internal/metadata_v2.cpp +++ b/src/dwarfs/internal/metadata_v2.cpp @@ -48,7 +48,6 @@ #include #include #include -#include #include #include #include @@ -56,6 +55,8 @@ #include #include +#include + #include #include diff --git a/src/dwarfs/internal/worker_group.cpp b/src/dwarfs/internal/worker_group.cpp index 8e0ad9da..669ef154 100644 --- a/src/dwarfs/internal/worker_group.cpp +++ b/src/dwarfs/internal/worker_group.cpp @@ -41,11 +41,12 @@ #include #include -#include #include #include #include +#include + namespace dwarfs::internal { namespace { diff --git a/src/dwarfs/metadata_types.cpp b/src/dwarfs/metadata_types.cpp index f6b81fe0..0482158c 100644 --- a/src/dwarfs/metadata_types.cpp +++ b/src/dwarfs/metadata_types.cpp @@ -27,12 +27,13 @@ #include #include -#include #include #include #include #include +#include + namespace dwarfs { inode_view::mode_type inode_view::mode() const { return iv_->mode(); } diff --git a/src/dwarfs/performance_monitor.cpp b/src/dwarfs/performance_monitor.cpp index 1ef1ff76..b3fab79c 100644 --- a/src/dwarfs/performance_monitor.cpp +++ b/src/dwarfs/performance_monitor.cpp @@ -49,6 +49,8 @@ namespace dwarfs { +namespace internal { + namespace { class single_timer { @@ -367,6 +369,8 @@ class performance_monitor_impl : public performance_monitor { std::shared_ptr fa_; }; +} // namespace internal + performance_monitor_proxy::performance_monitor_proxy( std::shared_ptr mon, std::string const& mon_namespace) @@ -379,7 +383,7 @@ std::unique_ptr performance_monitor::create( std::optional trace_file) { return enabled_namespaces.empty() ? nullptr - : std::make_unique( + : std::make_unique( std::move(enabled_namespaces), std::move(fa), std::move(trace_file)); } diff --git a/src/dwarfs/scanner.cpp b/src/dwarfs/scanner.cpp index 696674a9..2c242a1a 100644 --- a/src/dwarfs/scanner.cpp +++ b/src/dwarfs/scanner.cpp @@ -50,10 +50,6 @@ #include #include #include -#include -#include -#include -#include #include #include #include @@ -68,10 +64,17 @@ #include #include +#include +#include +#include +#include + #include namespace dwarfs { +namespace internal { + using namespace std::chrono_literals; namespace { @@ -149,7 +152,7 @@ class pipe_set_inode_visitor : public visitor_base { class names_and_symlinks_visitor : public visitor_base { public: - explicit names_and_symlinks_visitor(internal::global_entry_data& data) + explicit names_and_symlinks_visitor(global_entry_data& data) : data_(data) {} void visit(file* p) override { data_.add_name(p->name()); } @@ -168,7 +171,7 @@ class names_and_symlinks_visitor : public visitor_base { } private: - internal::global_entry_data& data_; + global_entry_data& data_; }; class save_directories_visitor : public visitor_base { @@ -179,8 +182,7 @@ class save_directories_visitor : public visitor_base { void visit(dir* p) override { directories_.at(p->inode_num().value()) = p; } - void - pack(thrift::metadata::metadata& mv2, internal::global_entry_data& ge_data) { + void pack(thrift::metadata::metadata& mv2, global_entry_data& ge_data) { for (auto p : directories_) { if (!p->has_parent()) { p->set_entry_index(mv2.dir_entries()->size()); @@ -285,8 +287,7 @@ std::string status_string(progress const& p, size_t width) { template class scanner_ final : public scanner::impl { public: - scanner_(logger& lgr, internal::worker_group& wg, - std::shared_ptr sf, + scanner_(logger& lgr, worker_group& wg, std::shared_ptr sf, std::shared_ptr ef, std::shared_ptr os, std::shared_ptr