TheCloud/dwarfs
1
0
Fork 0
mirror of https://github.com/mhx/dwarfs.git synced 2025-09-10 04:50:31 -04:00
Code Issues Packages Projects Releases Wiki Activity

refactor: internal namespace cleanup

Browse Source
This commit is contained in:
Marcus Holland-Moritz 2024-07-28 12:18:29 +02:00
parent f700ff5cb3
commit c8bfc82140
31 changed files with 307 additions and 257 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/dwarfs/internal/detail/multi_queue_block_merger_impl.h
View File

@ -40,9 +40,10 @@
#include <range/v3/view/join.hpp>
#include <range/v3/view/transform.hpp>
#include <dwarfs/internal/block_merger.h>
#include <dwarfs/terminal.h>
#include <dwarfs/internal/block_merger.h>
namespace dwarfs::internal::detail {
template <typename SourceT, typename BlockT, typename BlockPolicy>

3
include/dwarfs/internal/inode_reader_v2.h
View File

@ -29,9 +29,10 @@
#include <folly/Expected.h>
#include <dwarfs/block_range.h>
#include <dwarfs/internal/metadata_types.h>
#include <dwarfs/types.h>
#include <dwarfs/internal/metadata_types.h>
namespace dwarfs {
struct cache_tidy_config;

3
include/dwarfs/internal/metadata_v2.h
View File

@ -34,9 +34,10 @@
#include <nlohmann/json.hpp>
#include <dwarfs/internal/metadata_types.h>
#include <dwarfs/metadata_types.h>
#include <dwarfs/internal/metadata_types.h>
namespace dwarfs {
class logger;

1
src/dwarfs/block_range.cpp
View File

@ -23,6 +23,7 @@
#include <dwarfs/block_range.h>
#include <dwarfs/error.h>
#include <dwarfs/internal/cached_block.h>
namespace dwarfs {

8
src/dwarfs/builtin_script.cpp
View File

@ -34,6 +34,8 @@
namespace dwarfs {
namespace internal {
struct filter_rule {
enum class rule_type {
include,
@ -280,10 +282,12 @@ void builtin_script_<LoggerPolicy>::transform(entry_interface& ei) {
}
}
} // namespace internal
builtin_script::builtin_script(logger& lgr,
std::shared_ptr<file_access const> fa)
: impl_(make_unique_logging_object<impl, builtin_script_, logger_policies>(
lgr, std::move(fa))) {}
: impl_(make_unique_logging_object<impl, internal::builtin_script_,
logger_policies>(lgr, std::move(fa))) {}
builtin_script::~builtin_script() = default;

128
src/dwarfs/categorizer.cpp
View File

@ -39,59 +39,10 @@
namespace dwarfs {
namespace internal {
using namespace std::placeholders;
namespace po = boost::program_options;
std::string category_prefix(std::shared_ptr<categorizer_manager> const& mgr,
fragment_category cat) {
return category_prefix(mgr.get(), cat);
}
std::string category_prefix(std::unique_ptr<categorizer_manager> const& mgr,
fragment_category cat) {
return category_prefix(mgr.get(), cat);
}
std::string category_prefix(std::shared_ptr<categorizer_manager> const& mgr,
fragment_category::value_type cat) {
return category_prefix(mgr.get(), cat);
}
std::string category_prefix(std::unique_ptr<categorizer_manager> const& mgr,
fragment_category::value_type cat) {
return category_prefix(mgr.get(), cat);
}
std::string category_prefix(categorizer_manager const* mgr,
fragment_category::value_type cat) {
return category_prefix(mgr, fragment_category(cat));
}
std::string
category_prefix(categorizer_manager const* mgr, fragment_category cat) {
std::string prefix;
if (mgr) {
prefix = fmt::format(
"[{}{}] ", mgr->category_name(cat.value()),
cat.has_subcategory() ? fmt::format("/{}", cat.subcategory()) : "");
}
return prefix;
}
std::string
categorizer::category_metadata(std::string_view, fragment_category) const {
return std::string();
}
void categorizer::set_metadata_requirements(std::string_view,
std::string requirements) {
if (!requirements.empty()) {
compression_metadata_requirements().parse(
nlohmann::json::parse(requirements));
}
}
class categorizer_manager_private : public categorizer_manager::impl {
public:
virtual std::vector<std::shared_ptr<categorizer>> const&
@ -213,11 +164,6 @@ bool categorizer_job_<LoggerPolicy>::best_result_found() const {
return is_global_best_;
}
categorizer_job::categorizer_job() = default;
categorizer_job::categorizer_job(std::unique_ptr<impl> impl)
: impl_{std::move(impl)} {}
template <typename LoggerPolicy>
class categorizer_manager_ final : public categorizer_manager_private {
public:
@ -278,10 +224,6 @@ class categorizer_manager_ final : public categorizer_manager_private {
std::unordered_map<std::string_view, fragment_category::value_type> catmap_;
};
fragment_category categorizer_manager::default_category() {
return fragment_category(0);
}
template <typename LoggerPolicy>
void categorizer_manager_<LoggerPolicy>::add(std::shared_ptr<categorizer> c) {
for (auto const& cat : c->categories()) {
@ -354,10 +296,72 @@ bool categorizer_manager_<LoggerPolicy>::deterministic_less(
return categorizer->subcategory_less(a, b);
}
} // namespace internal
namespace po = boost::program_options;
std::string category_prefix(std::shared_ptr<categorizer_manager> const& mgr,
fragment_category cat) {
return category_prefix(mgr.get(), cat);
}
std::string category_prefix(std::unique_ptr<categorizer_manager> const& mgr,
fragment_category cat) {
return category_prefix(mgr.get(), cat);
}
std::string category_prefix(std::shared_ptr<categorizer_manager> const& mgr,
fragment_category::value_type cat) {
return category_prefix(mgr.get(), cat);
}
std::string category_prefix(std::unique_ptr<categorizer_manager> const& mgr,
fragment_category::value_type cat) {
return category_prefix(mgr.get(), cat);
}
std::string category_prefix(categorizer_manager const* mgr,
fragment_category::value_type cat) {
return category_prefix(mgr, fragment_category(cat));
}
std::string
category_prefix(categorizer_manager const* mgr, fragment_category cat) {
std::string prefix;
if (mgr) {
prefix = fmt::format(
"[{}{}] ", mgr->category_name(cat.value()),
cat.has_subcategory() ? fmt::format("/{}", cat.subcategory()) : "");
}
return prefix;
}
std::string
categorizer::category_metadata(std::string_view, fragment_category) const {
return std::string();
}
void categorizer::set_metadata_requirements(std::string_view,
std::string requirements) {
if (!requirements.empty()) {
compression_metadata_requirements().parse(
nlohmann::json::parse(requirements));
}
}
categorizer_job::categorizer_job() = default;
categorizer_job::categorizer_job(std::unique_ptr<impl> impl)
: impl_{std::move(impl)} {}
categorizer_manager::categorizer_manager(logger& lgr)
: impl_(make_unique_logging_object<impl, categorizer_manager_,
: impl_(make_unique_logging_object<impl, internal::categorizer_manager_,
logger_policies>(lgr)) {}
fragment_category categorizer_manager::default_category() {
return fragment_category(0);
}
categorizer_registry& categorizer_registry::instance() {
static categorizer_registry the_instance;
return the_instance;
@ -369,7 +373,7 @@ void categorizer_registry::register_factory(
if (!factories_.emplace(name, std::move(factory)).second) {
std::cerr << "categorizer factory name conflict (" << name << "\n";
::abort();
std::abort();
}
}

6
src/dwarfs/conv.cpp
View File

@ -23,8 +23,7 @@
#include <dwarfs/conv.h>
namespace dwarfs {
namespace detail {
namespace dwarfs::detail {
std::optional<bool> str_to_bool(std::string_view s) {
if (auto r = folly::tryTo<bool>(s)) {
@ -33,5 +32,4 @@ std::optional<bool> str_to_bool(std::string_view s) {
return std::nullopt;
}
} // namespace detail
} // namespace dwarfs
} // namespace dwarfs::detail

3
src/dwarfs/entry.cpp
View File

@ -30,7 +30,6 @@
#include <dwarfs/error.h>
#include <dwarfs/file_type.h>
#include <dwarfs/inode.h>
#include <dwarfs/internal/global_entry_data.h>
#include <dwarfs/mmif.h>
#include <dwarfs/nilsimsa.h>
#include <dwarfs/options.h>
@ -39,6 +38,8 @@
#include <dwarfs/scanner_progress.h>
#include <dwarfs/util.h>
#include <dwarfs/internal/global_entry_data.h>
#include <dwarfs/gen-cpp2/metadata_types.h>
namespace dwarfs {

3
src/dwarfs/file_scanner.cpp
View File

@ -38,7 +38,6 @@
#include <dwarfs/format.h>
#include <dwarfs/inode.h>
#include <dwarfs/inode_manager.h>
#include <dwarfs/internal/worker_group.h>
#include <dwarfs/logger.h>
#include <dwarfs/mmif.h>
#include <dwarfs/options.h>
@ -46,6 +45,8 @@
#include <dwarfs/progress.h>
#include <dwarfs/util.h>
#include <dwarfs/internal/worker_group.h>
namespace dwarfs::detail {
namespace {

13
src/dwarfs/filesystem_extractor.cpp
View File

@ -43,15 +43,18 @@
#include <dwarfs/filesystem_extractor.h>
#include <dwarfs/filesystem_v2.h>
#include <dwarfs/fstypes.h>
#include <dwarfs/internal/worker_group.h>
#include <dwarfs/logger.h>
#include <dwarfs/options.h>
#include <dwarfs/os_access.h>
#include <dwarfs/util.h>
#include <dwarfs/vfs_stat.h>
#include <dwarfs/internal/worker_group.h>
namespace dwarfs {
namespace internal {
namespace {
class cache_semaphore {
@ -242,7 +245,7 @@ bool filesystem_extractor_<LoggerPolicy>::extract(
::archive_entry* spare = nullptr;
internal::worker_group archiver(LOG_GET_LOGGER, os_, "archiver", 1);
worker_group archiver(LOG_GET_LOGGER, os_, "archiver", 1);
cache_semaphore sem;
LOG_DEBUG << "extractor semaphore size: " << opts.max_queued_bytes
@ -423,9 +426,11 @@ bool filesystem_extractor_<LoggerPolicy>::extract(
return true;
}
} // namespace internal
filesystem_extractor::filesystem_extractor(logger& lgr, os_access const& os)
: impl_(make_unique_logging_object<filesystem_extractor::impl,
filesystem_extractor_, logger_policies>(
lgr, os)) {}
internal::filesystem_extractor_,
logger_policies>(lgr, os)) {}
} // namespace dwarfs

41
src/dwarfs/filesystem_v2.cpp
View File

@ -41,11 +41,6 @@
#include <dwarfs/fs_section.h>
#include <dwarfs/fstypes.h>
#include <dwarfs/history.h>
#include <dwarfs/internal/block_cache.h>
#include <dwarfs/internal/block_data.h>
#include <dwarfs/internal/inode_reader_v2.h>
#include <dwarfs/internal/metadata_v2.h>
#include <dwarfs/internal/worker_group.h>
#include <dwarfs/logger.h>
#include <dwarfs/mmif.h>
#include <dwarfs/options.h>
@ -53,8 +48,16 @@
#include <dwarfs/progress.h>
#include <dwarfs/util.h>
#include <dwarfs/internal/block_cache.h>
#include <dwarfs/internal/block_data.h>
#include <dwarfs/internal/inode_reader_v2.h>
#include <dwarfs/internal/metadata_v2.h>
#include <dwarfs/internal/worker_group.h>
namespace dwarfs {
namespace internal {
namespace {
void check_section_logger(logger& lgr, fs_section const& section) {
@ -357,7 +360,7 @@ get_section_data(std::shared_ptr<mmif> mm, fs_section const& section,
return buffer;
}
internal::metadata_v2
metadata_v2
make_metadata(logger& lgr, std::shared_ptr<mmif> mm,
section_map const& sections, std::vector<uint8_t>& schema_buffer,
std::vector<uint8_t>& meta_buffer,
@ -408,13 +411,15 @@ make_metadata(logger& lgr, std::shared_ptr<mmif> mm,
}
}
return internal::metadata_v2(lgr,
return metadata_v2(lgr,
get_section_data(mm, schema_it->second.front(),
schema_buffer, force_buffers),
meta_section_range, options, inode_offset,
force_consistency_check, perfmon);
}
} // namespace
template <typename LoggerPolicy>
class filesystem_ final : public filesystem_v2::impl {
public:
@ -497,8 +502,8 @@ class filesystem_ final : public filesystem_v2::impl {
LOG_PROXY_DECL(LoggerPolicy);
os_access const& os_;
std::shared_ptr<mmif> mm_;
internal::metadata_v2 meta_;
internal::inode_reader_v2 ir_;
metadata_v2 meta_;
inode_reader_v2 ir_;
mutable std::mutex mx_;
std::vector<uint8_t> meta_buffer_;
std::optional<std::span<uint8_t const>> header_;
@ -610,7 +615,7 @@ filesystem_<LoggerPolicy>::filesystem_(
PERFMON_CLS_TIMER_INIT(readv_future)
PERFMON_CLS_TIMER_INIT(readv_future_throw) // clang-format on
{
internal::block_cache cache(lgr, os_, mm_, options.block_cache, perfmon);
block_cache cache(lgr, os_, mm_, options.block_cache, perfmon);
filesystem_parser parser(mm_, image_offset_);
if (parser.has_index()) {
@ -662,8 +667,7 @@ filesystem_<LoggerPolicy>::filesystem_(
cache.set_block_size(meta_.block_size());
ir_ = internal::inode_reader_v2(lgr, std::move(cache), options.inode_reader,
perfmon);
ir_ = inode_reader_v2(lgr, std::move(cache), options.inode_reader, perfmon);
if (auto it = sections.find(section_type::HISTORY); it != sections.end()) {
for (auto& section : it->second) {
@ -820,8 +824,7 @@ void filesystem_<LoggerPolicy>::rewrite(progress& prog,
<< "), compressing using '"
<< writer.get_compressor(s->type()).describe() << "'";
writer.write_history(
std::make_shared<internal::block_data>(hist.serialize()));
writer.write_history(std::make_shared<block_data>(hist.serialize()));
} else {
LOG_VERBOSE << "removing " << get_section_name(s->type());
}
@ -846,7 +849,7 @@ int filesystem_<LoggerPolicy>::check(filesystem_check_level level,
size_t num_threads) const {
filesystem_parser parser(mm_, image_offset_);
internal::worker_group wg(LOG_GET_LOGGER, os_, "fscheck", num_threads);
worker_group wg(LOG_GET_LOGGER, os_, "fscheck", num_threads);
std::vector<std::future<fs_section>> sections;
while (auto sp = parser.next_section()) {
@ -1231,7 +1234,7 @@ filesystem_<LoggerPolicy>::header() const {
return header_;
}
} // namespace
} // namespace internal
filesystem_v2::filesystem_v2(logger& lgr, os_access const& os,
std::shared_ptr<mmif> mm)
@ -1241,8 +1244,8 @@ filesystem_v2::filesystem_v2(logger& lgr, os_access const& os,
std::shared_ptr<mmif> mm,
const filesystem_options& options,
std::shared_ptr<performance_monitor const> perfmon)
: impl_(make_unique_logging_object<filesystem_v2::impl, filesystem_,
logger_policies>(
: impl_(make_unique_logging_object<filesystem_v2::impl,
internal::filesystem_, logger_policies>(
lgr, os, std::move(mm), options, std::move(perfmon))) {}
int filesystem_v2::identify(logger& lgr, os_access const& os,
@ -1270,7 +1273,7 @@ filesystem_v2::header(std::shared_ptr<mmif> mm) {
std::optional<std::span<uint8_t const>>
filesystem_v2::header(std::shared_ptr<mmif> mm, file_off_t image_offset) {
return filesystem_parser(mm, image_offset).header();
return internal::filesystem_parser(mm, image_offset).header();
}
} // namespace dwarfs

97
src/dwarfs/filesystem_writer.cpp
View File

@ -38,16 +38,19 @@
#include <dwarfs/compression_metadata_requirements.h>
#include <dwarfs/filesystem_writer.h>
#include <dwarfs/fstypes.h>
#include <dwarfs/internal/block_data.h>
#include <dwarfs/internal/multi_queue_block_merger.h>
#include <dwarfs/internal/worker_group.h>
#include <dwarfs/logger.h>
#include <dwarfs/progress.h>
#include <dwarfs/thread_pool.h>
#include <dwarfs/util.h>
#include <dwarfs/internal/block_data.h>
#include <dwarfs/internal/multi_queue_block_merger.h>
#include <dwarfs/internal/worker_group.h>
namespace dwarfs {
namespace internal {
namespace {
size_t copy_stream(std::istream& is, std::ostream& os) {
@ -94,7 +97,7 @@ class compression_progress : public progress::context {
class fsblock {
public:
fsblock(section_type type, block_compressor const& bc,
std::shared_ptr<internal::block_data>&& data,
std::shared_ptr<block_data>&& data,
std::shared_ptr<compression_progress> pctx,
folly::Function<void(size_t)> set_block_cb = nullptr);
@ -108,8 +111,8 @@ class fsblock {
std::span<uint8_t const> data, compression_type data_comp_type,
std::shared_ptr<compression_progress> pctx);
void compress(internal::worker_group& wg,
std::optional<std::string> meta = std::nullopt) {
void
compress(worker_group& wg, std::optional<std::string> meta = std::nullopt) {
impl_->compress(wg, std::move(meta));
}
void wait_until_compressed() { impl_->wait_until_compressed(); }
@ -128,7 +131,7 @@ class fsblock {
virtual ~impl() = default;
virtual void
compress(internal::worker_group& wg, std::optional<std::string> meta) = 0;
compress(worker_group& wg, std::optional<std::string> meta) = 0;
virtual void wait_until_compressed() = 0;
virtual section_type type() const = 0;
virtual compression_type compression() const = 0;
@ -170,7 +173,7 @@ class fsblock_merger_policy {
class raw_fsblock : public fsblock::impl {
public:
raw_fsblock(section_type type, const block_compressor& bc,
std::shared_ptr<internal::block_data>&& data,
std::shared_ptr<block_data>&& data,
std::shared_ptr<compression_progress> pctx,
folly::Function<void(size_t)> set_block_cb)
: type_{type}
@ -181,22 +184,19 @@ class raw_fsblock : public fsblock::impl {
, pctx_{std::move(pctx)}
, set_block_cb_{std::move(set_block_cb)} {}
void compress(internal::worker_group& wg,
std::optional<std::string> meta) override {
void compress(worker_group& wg, std::optional<std::string> meta) override {
std::promise<void> prom;
future_ = prom.get_future();
wg.add_job(
[this, prom = std::move(prom), meta = std::move(meta)]() mutable {
wg.add_job([this, prom = std::move(prom),
meta = std::move(meta)]() mutable {
try {
std::shared_ptr<internal::block_data> tmp;
std::shared_ptr<block_data> tmp;
if (meta) {
tmp = std::make_shared<internal::block_data>(
bc_.compress(data_->vec(), *meta));
tmp = std::make_shared<block_data>(bc_.compress(data_->vec(), *meta));
} else {
tmp = std::make_shared<internal::block_data>(
bc_.compress(data_->vec()));
tmp = std::make_shared<block_data>(bc_.compress(data_->vec()));
}
pctx_->bytes_in += data_->vec().size();
@ -264,7 +264,7 @@ class raw_fsblock : public fsblock::impl {
block_compressor const& bc_;
const size_t uncompressed_size_;
mutable std::recursive_mutex mx_;
std::shared_ptr<internal::block_data> data_;
std::shared_ptr<block_data> data_;
std::future<void> future_;
std::optional<uint32_t> number_;
std::optional<section_header_v2> mutable header_;
@ -289,8 +289,8 @@ class compressed_fsblock : public fsblock::impl {
, pctx_{std::move(pctx)}
, sec_{std::move(sec)} {}
void compress(internal::worker_group& wg,
std::optional<std::string> /* meta */) override {
void
compress(worker_group& wg, std::optional<std::string> /* meta */) override {
std::promise<void> prom;
future_ = prom.get_future();
@ -346,8 +346,7 @@ class rewritten_fsblock : public fsblock::impl {
, pctx_{std::move(pctx)}
, data_comp_type_{data_comp_type} {}
void compress(internal::worker_group& wg,
std::optional<std::string> meta) override {
void compress(worker_group& wg, std::optional<std::string> meta) override {
std::promise<void> prom;
future_ = prom.get_future();
@ -446,7 +445,7 @@ class rewritten_fsblock : public fsblock::impl {
};
fsblock::fsblock(section_type type, block_compressor const& bc,
std::shared_ptr<internal::block_data>&& data,
std::shared_ptr<block_data>&& data,
std::shared_ptr<compression_progress> pctx,
folly::Function<void(size_t)> set_block_cb)
: impl_(std::make_unique<raw_fsblock>(type, bc, std::move(data),
@ -514,12 +513,14 @@ void fsblock::build_section_header(section_header_v2& sh,
DWARFS_CHECK(sha.finalize(&sh.sha2_512_256), "SHA512/256 checksum failed");
}
} // namespace
template <typename LoggerPolicy>
class filesystem_writer_ final : public filesystem_writer::impl {
public:
using physical_block_cb_type = filesystem_writer::physical_block_cb_type;
filesystem_writer_(logger& lgr, std::ostream& os, internal::worker_group& wg,
filesystem_writer_(logger& lgr, std::ostream& os, worker_group& wg,
progress& prog, block_compressor const& schema_bc,
block_compressor const& metadata_bc,
block_compressor const& history_bc,
@ -539,18 +540,16 @@ class filesystem_writer_ final : public filesystem_writer::impl {
void configure(std::vector<fragment_category> const& expected_categories,
size_t max_active_slots) override;
void copy_header(std::span<uint8_t const> header) override;
void write_block(fragment_category cat,
std::shared_ptr<internal::block_data>&& data,
void write_block(fragment_category cat, std::shared_ptr<block_data>&& data,
physical_block_cb_type physical_block_cb,
std::optional<std::string> meta) override;
void finish_category(fragment_category cat) override;
void write_block(fragment_category::value_type cat,
std::shared_ptr<internal::block_data>&& data,
std::shared_ptr<block_data>&& data,
std::optional<std::string> meta) override;
void write_metadata_v2_schema(
std::shared_ptr<internal::block_data>&& data) override;
void write_metadata_v2(std::shared_ptr<internal::block_data>&& data) override;
void write_history(std::shared_ptr<internal::block_data>&& data) override;
void write_metadata_v2_schema(std::shared_ptr<block_data>&& data) override;
void write_metadata_v2(std::shared_ptr<block_data>&& data) override;
void write_history(std::shared_ptr<block_data>&& data) override;
void check_block_compression(
compression_type compression, std::span<uint8_t const> data,
std::optional<fragment_category::value_type> cat) override;
@ -563,20 +562,19 @@ class filesystem_writer_ final : public filesystem_writer::impl {
size_t size() const override { return image_size_; }
private:
using block_merger_type = internal::multi_queue_block_merger<
fragment_category, std::unique_ptr<fsblock>, fsblock_merger_policy>;
using block_merger_type =
multi_queue_block_merger<fragment_category, std::unique_ptr<fsblock>,
fsblock_merger_policy>;
using block_holder_type = block_merger_type::block_holder_type;
block_compressor const&
compressor_for_category(fragment_category::value_type cat) const;
void
write_block_impl(fragment_category cat,
std::shared_ptr<internal::block_data>&& data,
write_block_impl(fragment_category cat, std::shared_ptr<block_data>&& data,
block_compressor const& bc, std::optional<std::string> meta,
physical_block_cb_type physical_block_cb);
void on_block_merged(block_holder_type holder);
void write_section_impl(section_type type,
std::shared_ptr<internal::block_data>&& data,
void write_section_impl(section_type type, std::shared_ptr<block_data>&& data,
block_compressor const& bc,
std::optional<std::string> meta = std::nullopt);
void write(fsblock const& fsb);
@ -592,7 +590,7 @@ class filesystem_writer_ final : public filesystem_writer::impl {
std::ostream& os_;
size_t image_size_{0};
std::istream* header_;
internal::worker_group& wg_;
worker_group& wg_;
progress& prog_;
std::optional<block_compressor> default_bc_;
std::unordered_map<fragment_category::value_type, block_compressor> bc_;
@ -617,7 +615,7 @@ class filesystem_writer_ final : public filesystem_writer::impl {
// into something that gets passed a (section_type, category) pair?
template <typename LoggerPolicy>
filesystem_writer_<LoggerPolicy>::filesystem_writer_(
logger& lgr, std::ostream& os, internal::worker_group& wg, progress& prog,
logger& lgr, std::ostream& os, worker_group& wg, progress& prog,
block_compressor const& schema_bc, block_compressor const& metadata_bc,
block_compressor const& history_bc,
filesystem_writer_options const& options, std::istream* header)
@ -776,7 +774,7 @@ filesystem_writer_<LoggerPolicy>::compressor_for_category(
template <typename LoggerPolicy>
void filesystem_writer_<LoggerPolicy>::write_block_impl(
fragment_category cat, std::shared_ptr<internal::block_data>&& data,
fragment_category cat, std::shared_ptr<block_data>&& data,
block_compressor const& bc, std::optional<std::string> meta,
physical_block_cb_type physical_block_cb) {
if (!merger_) {
@ -837,7 +835,7 @@ void filesystem_writer_<LoggerPolicy>::finish_category(fragment_category cat) {
template <typename LoggerPolicy>
void filesystem_writer_<LoggerPolicy>::write_section_impl(
section_type type, std::shared_ptr<internal::block_data>&& data,
section_type type, std::shared_ptr<block_data>&& data,
block_compressor const& bc, std::optional<std::string> meta) {
uint32_t number;
@ -1023,7 +1021,7 @@ void filesystem_writer_<LoggerPolicy>::copy_header(
template <typename LoggerPolicy>
void filesystem_writer_<LoggerPolicy>::write_block(
fragment_category cat, std::shared_ptr<internal::block_data>&& data,
fragment_category cat, std::shared_ptr<block_data>&& data,
physical_block_cb_type physical_block_cb, std::optional<std::string> meta) {
write_block_impl(cat, std::move(data), compressor_for_category(cat.value()),
std::move(meta), std::move(physical_block_cb));
@ -1031,8 +1029,7 @@ void filesystem_writer_<LoggerPolicy>::write_block(
template <typename LoggerPolicy>
void filesystem_writer_<LoggerPolicy>::write_block(
fragment_category::value_type cat,
std::shared_ptr<internal::block_data>&& data,
fragment_category::value_type cat, std::shared_ptr<block_data>&& data,
std::optional<std::string> meta) {
write_section_impl(section_type::BLOCK, std::move(data),
compressor_for_category(cat), std::move(meta));
@ -1040,20 +1037,20 @@ void filesystem_writer_<LoggerPolicy>::write_block(
template <typename LoggerPolicy>
void filesystem_writer_<LoggerPolicy>::write_metadata_v2_schema(
std::shared_ptr<internal::block_data>&& data) {
std::shared_ptr<block_data>&& data) {
write_section_impl(section_type::METADATA_V2_SCHEMA, std::move(data),
schema_bc_);
}
template <typename LoggerPolicy>
void filesystem_writer_<LoggerPolicy>::write_metadata_v2(
std::shared_ptr<internal::block_data>&& data) {
std::shared_ptr<block_data>&& data) {
write_section_impl(section_type::METADATA_V2, std::move(data), metadata_bc_);
}
template <typename LoggerPolicy>
void filesystem_writer_<LoggerPolicy>::write_history(
std::shared_ptr<internal::block_data>&& data) {
std::shared_ptr<block_data>&& data) {
write_section_impl(section_type::HISTORY, std::move(data), history_bc_);
}
@ -1099,7 +1096,7 @@ void filesystem_writer_<LoggerPolicy>::write_section_index() {
write(fsb);
}
} // namespace
} // namespace internal
filesystem_writer::filesystem_writer(std::ostream& os, logger& lgr,
thread_pool& pool, progress& prog,
@ -1108,8 +1105,8 @@ filesystem_writer::filesystem_writer(std::ostream& os, logger& lgr,
block_compressor const& history_bc,
filesystem_writer_options const& options,
std::istream* header)
: impl_(
make_unique_logging_object<impl, filesystem_writer_, logger_policies>(
: impl_(make_unique_logging_object<impl, internal::filesystem_writer_,
logger_policies>(
lgr, os, pool.get_worker_group(), prog, schema_bc, metadata_bc,
history_bc, options, header)) {}

31
src/dwarfs/inode_manager.cpp
View File

@ -46,7 +46,6 @@
#include <dwarfs/error.h>
#include <dwarfs/inode_manager.h>
#include <dwarfs/inode_ordering.h>
#include <dwarfs/internal/worker_group.h>
#include <dwarfs/logger.h>
#include <dwarfs/match.h>
#include <dwarfs/mmif.h>
@ -61,15 +60,21 @@
#include <dwarfs/similarity_ordering.h>
#include <dwarfs/util.h>
#include <dwarfs/internal/worker_group.h>
#include <dwarfs/gen-cpp2/metadata_types.h>
namespace dwarfs {
namespace internal {
namespace {
constexpr std::string_view const kScanContext{"[scanning] "};
constexpr std::string_view const kCategorizeContext{"[categorizing] "};
} // namespace
class inode_ : public inode {
public:
using chunk_type = thrift::metadata::chunk;
@ -529,8 +534,6 @@ class inode_ : public inode {
similarity_;
};
} // namespace
template <typename LoggerPolicy>
class inode_manager_ final : public inode_manager::impl {
public:
@ -607,13 +610,12 @@ class inode_manager_ final : public inode_manager::impl {
return rv;
}
void scan_background(internal::worker_group& wg, os_access const& os,
void scan_background(worker_group& wg, os_access const& os,
std::shared_ptr<inode> ino, file* p) const override;
bool has_invalid_inodes() const override;
void
try_scan_invalid(internal::worker_group& wg, os_access const& os) override;
void try_scan_invalid(worker_group& wg, os_access const& os) override;
void dump(std::ostream& os) const override;
@ -621,8 +623,8 @@ class inode_manager_ final : public inode_manager::impl {
return sortable_inode_span(inodes_);
}
sortable_inode_span ordered_span(fragment_category cat,
internal::worker_group& wg) const override;
sortable_inode_span
ordered_span(fragment_category cat, worker_group& wg) const override;
private:
void update_prog(std::shared_ptr<inode> const& ino, file const* p) const {
@ -653,7 +655,7 @@ class inode_manager_ final : public inode_manager::impl {
};
template <typename LoggerPolicy>
void inode_manager_<LoggerPolicy>::scan_background(internal::worker_group& wg,
void inode_manager_<LoggerPolicy>::scan_background(worker_group& wg,
os_access const& os,
std::shared_ptr<inode> ino,
file* p) const {
@ -698,7 +700,7 @@ bool inode_manager_<LoggerPolicy>::has_invalid_inodes() const {
}
template <typename LoggerPolicy>
void inode_manager_<LoggerPolicy>::try_scan_invalid(internal::worker_group& wg,
void inode_manager_<LoggerPolicy>::try_scan_invalid(worker_group& wg,
os_access const& os) {
LOG_VERBOSE << "trying to scan " << num_invalid_inodes_.load()
<< " invalid inodes...";
@ -752,8 +754,7 @@ void inode_manager_<LoggerPolicy>::dump(std::ostream& os) const {
template <typename LoggerPolicy>
auto inode_manager_<LoggerPolicy>::ordered_span(
fragment_category cat,
internal::worker_group& wg) const -> sortable_inode_span {
fragment_category cat, worker_group& wg) const -> sortable_inode_span {
auto prefix = category_prefix(opts_.categorizer_mgr, cat);
auto opts = opts_.fragment_order.get(cat);
@ -811,9 +812,11 @@ auto inode_manager_<LoggerPolicy>::ordered_span(
return span;
}
} // namespace internal
inode_manager::inode_manager(logger& lgr, progress& prog,
inode_options const& opts)
: impl_(make_unique_logging_object<impl, inode_manager_, logger_policies>(
lgr, prog, opts)) {}
: impl_(make_unique_logging_object<impl, internal::inode_manager_,
logger_policies>(lgr, prog, opts)) {}
} // namespace dwarfs

25
src/dwarfs/inode_ordering.cpp
View File

@ -24,14 +24,17 @@
#include <dwarfs/entry.h>
#include <dwarfs/inode_element_view.h>
#include <dwarfs/inode_ordering.h>
#include <dwarfs/internal/worker_group.h>
#include <dwarfs/logger.h>
#include <dwarfs/options.h>
#include <dwarfs/promise_receiver.h>
#include <dwarfs/similarity_ordering.h>
#include <dwarfs/internal/worker_group.h>
namespace dwarfs {
namespace internal {
namespace {
bool inode_less_by_size(inode const* a, inode const* b) {
@ -40,6 +43,8 @@ bool inode_less_by_size(inode const* a, inode const* b) {
return sa > sb || (sa == sb && a->any()->less_revpath(*b->any()));
}
} // namespace
template <typename LoggerPolicy>
class inode_ordering_ final : public inode_ordering::impl {
public:
@ -54,14 +59,12 @@ class inode_ordering_ final : public inode_ordering::impl {
void
by_similarity(sortable_inode_span& sp, fragment_category cat) const override;
void
by_nilsimsa(internal::worker_group& wg,
similarity_ordering_options const& opts, sortable_inode_span& sp,
fragment_category cat) const override;
by_nilsimsa(worker_group& wg, similarity_ordering_options const& opts,
sortable_inode_span& sp, fragment_category cat) const override;
private:
void
by_nilsimsa_impl(internal::worker_group& wg,
similarity_ordering_options const& opts,
by_nilsimsa_impl(worker_group& wg, similarity_ordering_options const& opts,
std::span<std::shared_ptr<inode> const> inodes,
std::vector<uint32_t>& index, fragment_category cat) const;
@ -160,7 +163,7 @@ void inode_ordering_<LoggerPolicy>::by_similarity(sortable_inode_span& sp,
template <typename LoggerPolicy>
void inode_ordering_<LoggerPolicy>::by_nilsimsa(
internal::worker_group& wg, similarity_ordering_options const& opts,
worker_group& wg, similarity_ordering_options const& opts,
sortable_inode_span& sp, fragment_category cat) const {
auto raw = sp.raw();
auto& index = sp.index();
@ -190,7 +193,7 @@ void inode_ordering_<LoggerPolicy>::by_nilsimsa(
template <typename LoggerPolicy>
void inode_ordering_<LoggerPolicy>::by_nilsimsa_impl(
internal::worker_group& wg, similarity_ordering_options const& opts,
worker_group& wg, similarity_ordering_options const& opts,
std::span<std::shared_ptr<inode> const> inodes,
std::vector<uint32_t>& index, fragment_category cat) const {
auto ev = inode_element_view(inodes, index, cat);
@ -202,11 +205,11 @@ void inode_ordering_<LoggerPolicy>::by_nilsimsa_impl(
future.get().swap(index);
}
} // namespace
} // namespace internal
inode_ordering::inode_ordering(logger& lgr, progress& prog,
inode_options const& opts)
: impl_(make_unique_logging_object<impl, inode_ordering_, logger_policies>(
lgr, prog, opts)) {}
: impl_(make_unique_logging_object<impl, internal::inode_ordering_,
logger_policies>(lgr, prog, opts)) {}
} // namespace dwarfs

7
src/dwarfs/internal/block_cache.cpp
View File

@ -43,15 +43,16 @@
#include <folly/system/ThreadName.h>
#include <dwarfs/fs_section.h>
#include <dwarfs/internal/block_cache.h>
#include <dwarfs/internal/cached_block.h>
#include <dwarfs/internal/worker_group.h>
#include <dwarfs/logger.h>
#include <dwarfs/mmif.h>
#include <dwarfs/options.h>
#include <dwarfs/performance_monitor.h>
#include <dwarfs/util.h>
#include <dwarfs/internal/block_cache.h>
#include <dwarfs/internal/cached_block.h>
#include <dwarfs/internal/worker_group.h>
namespace dwarfs::internal {
namespace {

3
src/dwarfs/internal/cached_block.cpp
View File

@ -28,10 +28,11 @@
#include <dwarfs/block_compressor.h>
#include <dwarfs/error.h>
#include <dwarfs/fs_section.h>
#include <dwarfs/internal/cached_block.h>
#include <dwarfs/logger.h>
#include <dwarfs/mmif.h>
#include <dwarfs/internal/cached_block.h>
namespace dwarfs::internal {
namespace {

3
src/dwarfs/internal/global_entry_data.cpp
View File

@ -22,9 +22,10 @@
#include <ranges>
#include <dwarfs/error.h>
#include <dwarfs/internal/global_entry_data.h>
#include <dwarfs/options.h>
#include <dwarfs/internal/global_entry_data.h>
namespace dwarfs::internal {
template <typename T, typename U>

7
src/dwarfs/internal/inode_reader_v2.cpp
View File

@ -33,15 +33,16 @@
#include <folly/stats/Histogram.h>
#include <dwarfs/fstypes.h>
#include <dwarfs/internal/block_cache.h>
#include <dwarfs/internal/inode_reader_v2.h>
#include <dwarfs/internal/offset_cache.h>
#include <dwarfs/iovec_read_buf.h>
#include <dwarfs/logger.h>
#include <dwarfs/options.h>
#include <dwarfs/performance_monitor.h>
#include <dwarfs/util.h>
#include <dwarfs/internal/block_cache.h>
#include <dwarfs/internal/inode_reader_v2.h>
#include <dwarfs/internal/offset_cache.h>
namespace dwarfs::internal {
namespace {

3
src/dwarfs/internal/metadata_types.cpp
View File

@ -27,11 +27,12 @@
#include <fmt/format.h>
#include <dwarfs/error.h>
#include <dwarfs/internal/metadata_types.h>
#include <dwarfs/logger.h>
#include <dwarfs/match.h>
#include <dwarfs/util.h>
#include <dwarfs/internal/metadata_types.h>
#include <dwarfs/gen-cpp2/metadata_types_custom_protocol.h>
namespace dwarfs::internal {

3
src/dwarfs/internal/metadata_v2.cpp
View File

@ -48,7 +48,6 @@
#include <dwarfs/features.h>
#include <dwarfs/file_stat.h>
#include <dwarfs/fstypes.h>
#include <dwarfs/internal/metadata_v2.h>
#include <dwarfs/logger.h>
#include <dwarfs/options.h>
#include <dwarfs/performance_monitor.h>
@ -56,6 +55,8 @@
#include <dwarfs/util.h>
#include <dwarfs/vfs_stat.h>
#include <dwarfs/internal/metadata_v2.h>
#include <dwarfs/gen-cpp2/metadata_layouts.h>
#include <dwarfs/gen-cpp2/metadata_types_custom_protocol.h>

3
src/dwarfs/internal/worker_group.cpp
View File

@ -41,11 +41,12 @@
#include <folly/system/ThreadName.h>
#include <dwarfs/error.h>
#include <dwarfs/internal/worker_group.h>
#include <dwarfs/logger.h>
#include <dwarfs/os_access.h>
#include <dwarfs/util.h>
#include <dwarfs/internal/worker_group.h>
namespace dwarfs::internal {
namespace {

3
src/dwarfs/metadata_types.cpp
View File

@ -27,12 +27,13 @@
#include <fmt/format.h>
#include <dwarfs/error.h>
#include <dwarfs/internal/metadata_types.h>
#include <dwarfs/logger.h>
#include <dwarfs/match.h>
#include <dwarfs/metadata_types.h>
#include <dwarfs/util.h>
#include <dwarfs/internal/metadata_types.h>
namespace dwarfs {
inode_view::mode_type inode_view::mode() const { return iv_->mode(); }

6
src/dwarfs/performance_monitor.cpp
View File

@ -49,6 +49,8 @@
namespace dwarfs {
namespace internal {
namespace {
class single_timer {
@ -367,6 +369,8 @@ class performance_monitor_impl : public performance_monitor {
std::shared_ptr<file_access const> fa_;
};
} // namespace internal
performance_monitor_proxy::performance_monitor_proxy(
std::shared_ptr<performance_monitor const> mon,
std::string const& mon_namespace)
@ -379,7 +383,7 @@ std::unique_ptr<performance_monitor> performance_monitor::create(
std::optional<std::filesystem::path> trace_file) {
return enabled_namespaces.empty()
? nullptr
: std::make_unique<performance_monitor_impl>(
: std::make_unique<internal::performance_monitor_impl>(
std::move(enabled_namespaces), std::move(fa),
std::move(trace_file));
}

48
src/dwarfs/scanner.cpp
View File

@ -50,10 +50,6 @@
#include <dwarfs/inode.h>
#include <dwarfs/inode_manager.h>
#include <dwarfs/inode_ordering.h>
#include <dwarfs/internal/block_data.h>
#include <dwarfs/internal/block_manager.h>
#include <dwarfs/internal/global_entry_data.h>
#include <dwarfs/internal/worker_group.h>
#include <dwarfs/logger.h>
#include <dwarfs/metadata_freezer.h>
#include <dwarfs/mmif.h>
@ -68,10 +64,17 @@
#include <dwarfs/util.h>
#include <dwarfs/version.h>
#include <dwarfs/internal/block_data.h>
#include <dwarfs/internal/block_manager.h>
#include <dwarfs/internal/global_entry_data.h>
#include <dwarfs/internal/worker_group.h>
#include <dwarfs/gen-cpp2/metadata_types.h>
namespace dwarfs {
namespace internal {
using namespace std::chrono_literals;
namespace {
@ -149,7 +152,7 @@ class pipe_set_inode_visitor : public visitor_base {
class names_and_symlinks_visitor : public visitor_base {
public:
explicit names_and_symlinks_visitor(internal::global_entry_data& data)
explicit names_and_symlinks_visitor(global_entry_data& data)
: data_(data) {}
void visit(file* p) override { data_.add_name(p->name()); }
@ -168,7 +171,7 @@ class names_and_symlinks_visitor : public visitor_base {
}
private:
internal::global_entry_data& data_;
global_entry_data& data_;
};
class save_directories_visitor : public visitor_base {
@ -179,8 +182,7 @@ class save_directories_visitor : public visitor_base {
void visit(dir* p) override { directories_.at(p->inode_num().value()) = p; }
void
pack(thrift::metadata::metadata& mv2, internal::global_entry_data& ge_data) {
void pack(thrift::metadata::metadata& mv2, global_entry_data& ge_data) {
for (auto p : directories_) {
if (!p->has_parent()) {
p->set_entry_index(mv2.dir_entries()->size());
@ -285,8 +287,7 @@ std::string status_string(progress const& p, size_t width) {
template <typename LoggerPolicy>
class scanner_ final : public scanner::impl {
public:
scanner_(logger& lgr, internal::worker_group& wg,
std::shared_ptr<segmenter_factory> sf,
scanner_(logger& lgr, worker_group& wg, std::shared_ptr<segmenter_factory> sf,
std::shared_ptr<entry_factory> ef,
std::shared_ptr<os_access const> os, std::shared_ptr<script> scr,
const scanner_options& options);
@ -314,7 +315,7 @@ class scanner_ final : public scanner::impl {
std::function<void(std::ostream&)> dumper) const;
LOG_PROXY_DECL(LoggerPolicy);
internal::worker_group& wg_;
worker_group& wg_;
scanner_options const& options_;
std::shared_ptr<segmenter_factory> segmenter_factory_;
std::shared_ptr<entry_factory> entry_factory_;
@ -323,7 +324,7 @@ class scanner_ final : public scanner::impl {
};
template <typename LoggerPolicy>
scanner_<LoggerPolicy>::scanner_(logger& lgr, internal::worker_group& wg,
scanner_<LoggerPolicy>::scanner_(logger& lgr, worker_group& wg,
std::shared_ptr<segmenter_factory> sf,
std::shared_ptr<entry_factory> ef,
std::shared_ptr<os_access const> os,
@ -683,7 +684,7 @@ void scanner_<LoggerPolicy>::scan(
}
}
internal::global_entry_data ge_data(options_);
global_entry_data ge_data(options_);
thrift::metadata::metadata mv2;
feature_set features;
@ -730,14 +731,12 @@ void scanner_<LoggerPolicy>::scan(
// which gets run on a worker groups; each batch keeps track of
// its CPU time and affects thread naming
auto blockmgr = std::make_shared<internal::block_manager>();
auto blockmgr = std::make_shared<block_manager>();
{
size_t const num_threads = options_.num_segmenter_workers;
internal::worker_group wg_ordering(LOG_GET_LOGGER, *os_, "ordering",
num_threads);
internal::worker_group wg_blockify(LOG_GET_LOGGER, *os_, "blockify",
num_threads);
worker_group wg_ordering(LOG_GET_LOGGER, *os_, "ordering", num_threads);
worker_group wg_blockify(LOG_GET_LOGGER, *os_, "blockify", num_threads);
fsw.configure(frag_info.categories, num_threads);
@ -989,15 +988,13 @@ void scanner_<LoggerPolicy>::scan(
LOG_VERBOSE << "uncompressed metadata size: " << size_with_unit(data.size());
fsw.write_metadata_v2_schema(
std::make_shared<internal::block_data>(std::move(schema)));
fsw.write_metadata_v2(
std::make_shared<internal::block_data>(std::move(data)));
fsw.write_metadata_v2_schema(std::make_shared<block_data>(std::move(schema)));
fsw.write_metadata_v2(std::make_shared<block_data>(std::move(data)));
if (options_.enable_history) {
history hist(options_.history);
hist.append(options_.command_line_arguments);
fsw.write_history(std::make_shared<internal::block_data>(hist.serialize()));
fsw.write_history(std::make_shared<block_data>(hist.serialize()));
}
LOG_INFO << "waiting for compression to finish...";
@ -1010,12 +1007,15 @@ void scanner_<LoggerPolicy>::scan(
<< ")";
}
} // namespace internal
scanner::scanner(logger& lgr, thread_pool& pool,
std::shared_ptr<segmenter_factory> sf,
std::shared_ptr<entry_factory> ef,
std::shared_ptr<os_access const> os,
std::shared_ptr<script> scr, const scanner_options& options)
: impl_(make_unique_logging_object<impl, scanner_, logger_policies>(
: impl_(
make_unique_logging_object<impl, internal::scanner_, logger_policies>(
lgr, pool.get_worker_group(), std::move(sf), std::move(ef),
std::move(os), std::move(scr), options)) {}

30
src/dwarfs/segmenter.cpp
View File

@ -47,15 +47,18 @@
#include <dwarfs/cyclic_hash.h>
#include <dwarfs/entry.h>
#include <dwarfs/error.h>
#include <dwarfs/internal/block_data.h>
#include <dwarfs/internal/block_manager.h>
#include <dwarfs/logger.h>
#include <dwarfs/progress.h>
#include <dwarfs/segmenter.h>
#include <dwarfs/util.h>
#include <dwarfs/internal/block_data.h>
#include <dwarfs/internal/block_manager.h>
namespace dwarfs {
namespace internal {
namespace {
/**
@ -567,7 +570,7 @@ class active_block : private GranularityPolicy {
, filter_(bloom_filter_size)
, repseqmap_{repseqmap}
, repeating_collisions_{repcoll}
, data_{std::make_shared<internal::block_data>()} {
, data_{std::make_shared<block_data>()} {
DWARFS_CHECK((window_step & window_step_mask_) == 0,
"window step size not a power of two");
data_->reserve(this->frames_to_bytes(capacity_in_frames_));
@ -583,9 +586,7 @@ class active_block : private GranularityPolicy {
return size_in_frames() == capacity_in_frames_;
}
DWARFS_FORCE_INLINE std::shared_ptr<internal::block_data> data() const {
return data_;
}
DWARFS_FORCE_INLINE std::shared_ptr<block_data> data() const { return data_; }
DWARFS_FORCE_INLINE void
append_bytes(std::span<uint8_t const> data, bloom_filter& global_filter);
@ -630,7 +631,7 @@ class active_block : private GranularityPolicy {
fast_multimap<hash_t, offset_t, num_inline_offsets> offsets_;
repeating_sequence_map_type const& repseqmap_;
repeating_collisions_map_type& repeating_collisions_;
std::shared_ptr<internal::block_data> data_;
std::shared_ptr<block_data> data_;
};
class segmenter_progress : public progress::context {
@ -678,8 +679,7 @@ class segmenter_ final : public segmenter::impl, private SegmentingPolicy {
public:
template <typename... PolicyArgs>
segmenter_(logger& lgr, progress& prog,
std::shared_ptr<internal::block_manager> blkmgr,
segmenter_(logger& lgr, progress& prog, std::shared_ptr<block_manager> blkmgr,
segmenter::config const& cfg, size_t total_size,
segmenter::block_ready_cb block_ready, PolicyArgs&&... args)
: SegmentingPolicy(std::forward<PolicyArgs>(args)...)
@ -764,7 +764,7 @@ class segmenter_ final : public segmenter::impl, private SegmentingPolicy {
LOG_PROXY_DECL(LoggerPolicy);
progress& prog_;
std::shared_ptr<internal::block_manager> blkmgr_;
std::shared_ptr<block_manager> blkmgr_;
segmenter::config const cfg_;
segmenter::block_ready_cb block_ready_;
std::shared_ptr<segmenter_progress> pctx_;
@ -1289,7 +1289,7 @@ struct variable_granularity_segmenter_ {
template <template <typename> typename SegmentingPolicy>
std::unique_ptr<segmenter::impl>
create_segmenter2(logger& lgr, progress& prog,
std::shared_ptr<internal::block_manager> blkmgr,
std::shared_ptr<block_manager> blkmgr,
segmenter::config const& cfg,
compression_constraints const& cc, size_t total_size,
segmenter::block_ready_cb block_ready) {
@ -1328,7 +1328,7 @@ create_segmenter2(logger& lgr, progress& prog,
std::unique_ptr<segmenter::impl>
create_segmenter(logger& lgr, progress& prog,
std::shared_ptr<internal::block_manager> blkmgr,
std::shared_ptr<block_manager> blkmgr,
segmenter::config const& cfg,
compression_constraints const& cc, size_t total_size,
segmenter::block_ready_cb block_ready) {
@ -1351,11 +1351,13 @@ create_segmenter(logger& lgr, progress& prog,
} // namespace
} // namespace internal
segmenter::segmenter(logger& lgr, progress& prog,
std::shared_ptr<internal::block_manager> blkmgr,
config const& cfg, compression_constraints const& cc,
size_t total_size, block_ready_cb block_ready)
: impl_(create_segmenter(lgr, prog, std::move(blkmgr), cfg, cc, total_size,
std::move(block_ready))) {}
: impl_(internal::create_segmenter(lgr, prog, std::move(blkmgr), cfg, cc,
total_size, std::move(block_ready))) {}
} // namespace dwarfs

10
src/dwarfs/segmenter_factory.cpp
View File

@ -24,6 +24,8 @@
namespace dwarfs {
namespace internal {
class segmenter_factory_ final : public segmenter_factory::impl {
public:
segmenter_factory_(logger& lgr, progress& prog,
@ -36,7 +38,7 @@ class segmenter_factory_ final : public segmenter_factory::impl {
segmenter create(fragment_category cat, size_t cat_size,
compression_constraints const& cc,
std::shared_ptr<internal::block_manager> blkmgr,
std::shared_ptr<block_manager> blkmgr,
segmenter::block_ready_cb block_ready) const override {
segmenter::config cfg;
@ -65,11 +67,13 @@ class segmenter_factory_ final : public segmenter_factory::impl {
segmenter_factory::config cfg_;
};
} // namespace internal
segmenter_factory::segmenter_factory(
logger& lgr, progress& prog, std::shared_ptr<categorizer_manager> catmgr,
config const& cfg)
: impl_(std::make_unique<segmenter_factory_>(lgr, prog, std::move(catmgr),
cfg)) {}
: impl_(std::make_unique<internal::segmenter_factory_>(
lgr, prog, std::move(catmgr), cfg)) {}
segmenter_factory::segmenter_factory(logger& lgr, progress& prog,
config const& cfg)

13
src/dwarfs/similarity_ordering.cpp
View File

@ -31,13 +31,16 @@
#include <folly/experimental/Bits.h>
#include <dwarfs/compiler.h>
#include <dwarfs/internal/worker_group.h>
#include <dwarfs/logger.h>
#include <dwarfs/progress.h>
#include <dwarfs/similarity_ordering.h>
#include <dwarfs/internal/worker_group.h>
namespace dwarfs {
namespace internal {
namespace {
// TODO: move out of here
@ -231,7 +234,7 @@ class similarity_ordering_ final : public similarity_ordering::impl {
basic_cluster<256, uint64_t, uint32_t, index_value_type>;
using nilsimsa_cluster_tree_node = basic_cluster_tree_node<nilsimsa_cluster>;
similarity_ordering_(logger& lgr, progress& prog, internal::worker_group& wg,
similarity_ordering_(logger& lgr, progress& prog, worker_group& wg,
similarity_ordering_options const& opts)
: LOG_PROXY_INIT(lgr)
, prog_{prog}
@ -297,7 +300,7 @@ class similarity_ordering_ final : public similarity_ordering::impl {
LOG_PROXY_DECL(LoggerPolicy);
progress& prog_;
internal::worker_group& wg_;
worker_group& wg_;
similarity_ordering_options const opts_;
};
@ -650,10 +653,12 @@ void similarity_ordering_<LoggerPolicy>::order_nilsimsa(
});
}
} // namespace internal
similarity_ordering::similarity_ordering(
logger& lgr, progress& prog, internal::worker_group& wg,
similarity_ordering_options const& opts)
: impl_(make_unique_logging_object<impl, similarity_ordering_,
: impl_(make_unique_logging_object<impl, internal::similarity_ordering_,
logger_policies>(lgr, prog, wg, opts)) {}
} // namespace dwarfs

3
src/dwarfs/thread_pool.cpp
View File

@ -19,9 +19,10 @@
* along with dwarfs. If not, see <https://www.gnu.org/licenses/>.
*/
#include <dwarfs/internal/worker_group.h>
#include <dwarfs/thread_pool.h>
#include <dwarfs/internal/worker_group.h>
namespace dwarfs {
thread_pool::thread_pool() = default;

3
test/block_cache_test.cpp
View File

@ -35,9 +35,10 @@
#include <dwarfs/block_range.h>
#include <dwarfs/error.h>
#include <dwarfs/filesystem_v2.h>
#include <dwarfs/internal/cached_block.h>
#include <dwarfs_tool_main.h>
#include <dwarfs/internal/cached_block.h>
#include "mmap_mock.h"
#include "test_helpers.h"
#include "test_logger.h"

5
test/segmenter_benchmark.cpp
View File

@ -26,11 +26,12 @@
#include <dwarfs/chunkable.h>
#include <dwarfs/compression_constraints.h>
#include <dwarfs/internal/block_data.h>
#include <dwarfs/internal/block_manager.h>
#include <dwarfs/progress.h>
#include <dwarfs/segmenter.h>
#include <dwarfs/internal/block_data.h>
#include <dwarfs/internal/block_manager.h>
#include "loremipsum.h"
#include "test_logger.h"

3
test/utils_test.cpp
View File

@ -28,10 +28,11 @@
#include <vector>
#include <dwarfs/error.h>
#include <dwarfs/internal/offset_cache.h>
#include <dwarfs/options.h>
#include <dwarfs/util.h>
#include <dwarfs/internal/offset_cache.h>
using namespace dwarfs;
TEST(utils, utf8_display_width) {