TheCloud/dwarfs
1
0
Fork 0
mirror of https://github.com/mhx/dwarfs.git synced 2025-08-04 02:06:22 -04:00
Code Issues Packages Projects Releases Wiki Activity

refactor(block_data): move to internal namespace

Browse Source
This commit is contained in:
Marcus Holland-Moritz 2024-07-28 10:47:24 +02:00
parent 08266788af
commit 9c12efda18
8 changed files with 95 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
include/dwarfs/filesystem_writer.h
View File

@ -38,8 +38,13 @@
namespace dwarfs {
class block_compressor;
namespace internal {
class block_data;
} // namespace internal
class block_compressor;
class logger;
class progress;
class thread_pool;
@ -87,7 +92,8 @@ class filesystem_writer {
// TODO: check which write_block() API is actually used
void write_block(fragment_category cat, std::shared_ptr<block_data>&& data,
void write_block(fragment_category cat,
std::shared_ptr<internal::block_data>&& data,
physical_block_cb_type physical_block_cb,
std::optional<std::string> meta = std::nullopt) {
impl_->write_block(cat, std::move(data), std::move(physical_block_cb),
@ -97,20 +103,20 @@ class filesystem_writer {
void finish_category(fragment_category cat) { impl_->finish_category(cat); }
void write_block(fragment_category::value_type cat,
std::shared_ptr<block_data>&& data,
std::shared_ptr<internal::block_data>&& data,
std::optional<std::string> meta = std::nullopt) {
impl_->write_block(cat, std::move(data), std::move(meta));
}
void write_metadata_v2_schema(std::shared_ptr<block_data>&& data) {
void write_metadata_v2_schema(std::shared_ptr<internal::block_data>&& data) {
impl_->write_metadata_v2_schema(std::move(data));
}
void write_metadata_v2(std::shared_ptr<block_data>&& data) {
void write_metadata_v2(std::shared_ptr<internal::block_data>&& data) {
impl_->write_metadata_v2(std::move(data));
}
void write_history(std::shared_ptr<block_data>&& data) {
void write_history(std::shared_ptr<internal::block_data>&& data) {
impl_->write_history(std::move(data));
}
@ -152,18 +158,20 @@ class filesystem_writer {
configure(std::vector<fragment_category> const& expected_categories,
size_t max_active_slots) = 0;
virtual void copy_header(std::span<uint8_t const> header) = 0;
virtual void
write_block(fragment_category cat, std::shared_ptr<block_data>&& data,
physical_block_cb_type physical_block_cb,
std::optional<std::string> meta) = 0;
virtual void write_block(fragment_category cat,
std::shared_ptr<internal::block_data>&& data,
physical_block_cb_type physical_block_cb,
std::optional<std::string> meta) = 0;
virtual void finish_category(fragment_category cat) = 0;
virtual void write_block(fragment_category::value_type cat,
std::shared_ptr<block_data>&& data,
std::shared_ptr<internal::block_data>&& data,
std::optional<std::string> meta) = 0;
virtual void
write_metadata_v2_schema(std::shared_ptr<block_data>&& data) = 0;
virtual void write_metadata_v2(std::shared_ptr<block_data>&& data) = 0;
virtual void write_history(std::shared_ptr<block_data>&& data) = 0;
write_metadata_v2_schema(std::shared_ptr<internal::block_data>&& data) = 0;
virtual void
write_metadata_v2(std::shared_ptr<internal::block_data>&& data) = 0;
virtual void
write_history(std::shared_ptr<internal::block_data>&& data) = 0;
virtual void check_block_compression(
compression_type compression, std::span<uint8_t const> data,
std::optional<fragment_category::value_type> cat) = 0;

4
include/dwarfs/block_data.h → include/dwarfs/internal/block_data.h
View File

@ -25,7 +25,7 @@
#include <string_view>
#include <vector>
namespace dwarfs {
namespace dwarfs::internal {
class block_data {
public:
@ -51,4 +51,4 @@ class block_data {
std::vector<uint8_t> vec_;
};
} // namespace dwarfs
} // namespace dwarfs::internal

9
include/dwarfs/segmenter.h
View File

@ -28,7 +28,12 @@
namespace dwarfs {
namespace internal {
class block_data;
} // namespace internal
class block_manager;
class chunkable;
class logger;
@ -47,8 +52,8 @@ class segmenter {
unsigned block_size_bits{22};
};
using block_ready_cb = std::function<void(std::shared_ptr<block_data>,
size_t logical_block_num)>;
using block_ready_cb = std::function<void(
std::shared_ptr<internal::block_data>, size_t logical_block_num)>;
segmenter(logger& lgr, progress& prog, std::shared_ptr<block_manager> blkmgr,
config const& cfg, compression_constraints const& cc,

5
src/dwarfs/filesystem_v2.cpp
View File

@ -33,7 +33,6 @@
#include <fmt/format.h>
#include <dwarfs/block_compressor.h>
#include <dwarfs/block_data.h>
#include <dwarfs/categorizer.h>
#include <dwarfs/category_resolver.h>
#include <dwarfs/error.h>
@ -43,6 +42,7 @@
#include <dwarfs/fstypes.h>
#include <dwarfs/history.h>
#include <dwarfs/internal/block_cache.h>
#include <dwarfs/internal/block_data.h>
#include <dwarfs/internal/inode_reader_v2.h>
#include <dwarfs/internal/metadata_v2.h>
#include <dwarfs/internal/worker_group.h>
@ -820,7 +820,8 @@ void filesystem_<LoggerPolicy>::rewrite(progress& prog,
<< "), compressing using '"
<< writer.get_compressor(s->type()).describe() << "'";
writer.write_history(std::make_shared<block_data>(hist.serialize()));
writer.write_history(
std::make_shared<internal::block_data>(hist.serialize()));
} else {
LOG_VERBOSE << "removing " << get_section_name(s->type());
}

85
src/dwarfs/filesystem_writer.cpp
View File

@ -34,11 +34,11 @@
#include <folly/system/ThreadName.h>
#include <dwarfs/block_compressor.h>
#include <dwarfs/block_data.h>
#include <dwarfs/checksum.h>
#include <dwarfs/compression_metadata_requirements.h>
#include <dwarfs/filesystem_writer.h>
#include <dwarfs/fstypes.h>
#include <dwarfs/internal/block_data.h>
#include <dwarfs/internal/multi_queue_block_merger.h>
#include <dwarfs/internal/worker_group.h>
#include <dwarfs/logger.h>
@ -94,7 +94,7 @@ class compression_progress : public progress::context {
class fsblock {
public:
fsblock(section_type type, block_compressor const& bc,
std::shared_ptr<block_data>&& data,
std::shared_ptr<internal::block_data>&& data,
std::shared_ptr<compression_progress> pctx,
folly::Function<void(size_t)> set_block_cb = nullptr);
@ -170,7 +170,7 @@ class fsblock_merger_policy {
class raw_fsblock : public fsblock::impl {
public:
raw_fsblock(section_type type, const block_compressor& bc,
std::shared_ptr<block_data>&& data,
std::shared_ptr<internal::block_data>&& data,
std::shared_ptr<compression_progress> pctx,
folly::Function<void(size_t)> set_block_cb)
: type_{type}
@ -186,30 +186,32 @@ class raw_fsblock : public fsblock::impl {
std::promise<void> prom;
future_ = prom.get_future();
wg.add_job([this, prom = std::move(prom),
meta = std::move(meta)]() mutable {
try {
std::shared_ptr<block_data> tmp;
wg.add_job(
[this, prom = std::move(prom), meta = std::move(meta)]() mutable {
try {
std::shared_ptr<internal::block_data> tmp;
if (meta) {
tmp = std::make_shared<block_data>(bc_.compress(data_->vec(), *meta));
} else {
tmp = std::make_shared<block_data>(bc_.compress(data_->vec()));
}
if (meta) {
tmp = std::make_shared<internal::block_data>(
bc_.compress(data_->vec(), *meta));
} else {
tmp = std::make_shared<internal::block_data>(
bc_.compress(data_->vec()));
}
pctx_->bytes_in += data_->vec().size();
pctx_->bytes_out += tmp->vec().size();
pctx_->bytes_in += data_->vec().size();
pctx_->bytes_out += tmp->vec().size();
{
std::lock_guard lock(mx_);
data_.swap(tmp);
}
} catch (bad_compression_ratio_error const&) {
comp_type_ = compression_type::NONE;
}
{
std::lock_guard lock(mx_);
data_.swap(tmp);
}
} catch (bad_compression_ratio_error const&) {
comp_type_ = compression_type::NONE;
}
prom.set_value();
});
prom.set_value();
});
}
void wait_until_compressed() override { future_.wait(); }
@ -262,7 +264,7 @@ class raw_fsblock : public fsblock::impl {
block_compressor const& bc_;
const size_t uncompressed_size_;
mutable std::recursive_mutex mx_;
std::shared_ptr<block_data> data_;
std::shared_ptr<internal::block_data> data_;
std::future<void> future_;
std::optional<uint32_t> number_;
std::optional<section_header_v2> mutable header_;
@ -444,7 +446,7 @@ class rewritten_fsblock : public fsblock::impl {
};
fsblock::fsblock(section_type type, block_compressor const& bc,
std::shared_ptr<block_data>&& data,
std::shared_ptr<internal::block_data>&& data,
std::shared_ptr<compression_progress> pctx,
folly::Function<void(size_t)> set_block_cb)
: impl_(std::make_unique<raw_fsblock>(type, bc, std::move(data),
@ -537,16 +539,18 @@ class filesystem_writer_ final : public filesystem_writer::impl {
void configure(std::vector<fragment_category> const& expected_categories,
size_t max_active_slots) override;
void copy_header(std::span<uint8_t const> header) override;
void write_block(fragment_category cat, std::shared_ptr<block_data>&& data,
void write_block(fragment_category cat,
std::shared_ptr<internal::block_data>&& data,
physical_block_cb_type physical_block_cb,
std::optional<std::string> meta) override;
void finish_category(fragment_category cat) override;
void write_block(fragment_category::value_type cat,
std::shared_ptr<block_data>&& data,
std::shared_ptr<internal::block_data>&& data,
std::optional<std::string> meta) override;
void write_metadata_v2_schema(std::shared_ptr<block_data>&& data) override;
void write_metadata_v2(std::shared_ptr<block_data>&& data) override;
void write_history(std::shared_ptr<block_data>&& data) override;
void write_metadata_v2_schema(
std::shared_ptr<internal::block_data>&& data) override;
void write_metadata_v2(std::shared_ptr<internal::block_data>&& data) override;
void write_history(std::shared_ptr<internal::block_data>&& data) override;
void check_block_compression(
compression_type compression, std::span<uint8_t const> data,
std::optional<fragment_category::value_type> cat) override;
@ -566,11 +570,13 @@ class filesystem_writer_ final : public filesystem_writer::impl {
block_compressor const&
compressor_for_category(fragment_category::value_type cat) const;
void
write_block_impl(fragment_category cat, std::shared_ptr<block_data>&& data,
write_block_impl(fragment_category cat,
std::shared_ptr<internal::block_data>&& data,
block_compressor const& bc, std::optional<std::string> meta,
physical_block_cb_type physical_block_cb);
void on_block_merged(block_holder_type holder);
void write_section_impl(section_type type, std::shared_ptr<block_data>&& data,
void write_section_impl(section_type type,
std::shared_ptr<internal::block_data>&& data,
block_compressor const& bc,
std::optional<std::string> meta = std::nullopt);
void write(fsblock const& fsb);
@ -770,7 +776,7 @@ filesystem_writer_<LoggerPolicy>::compressor_for_category(
template <typename LoggerPolicy>
void filesystem_writer_<LoggerPolicy>::write_block_impl(
fragment_category cat, std::shared_ptr<block_data>&& data,
fragment_category cat, std::shared_ptr<internal::block_data>&& data,
block_compressor const& bc, std::optional<std::string> meta,
physical_block_cb_type physical_block_cb) {
if (!merger_) {
@ -831,7 +837,7 @@ void filesystem_writer_<LoggerPolicy>::finish_category(fragment_category cat) {
template <typename LoggerPolicy>
void filesystem_writer_<LoggerPolicy>::write_section_impl(
section_type type, std::shared_ptr<block_data>&& data,
section_type type, std::shared_ptr<internal::block_data>&& data,
block_compressor const& bc, std::optional<std::string> meta) {
uint32_t number;
@ -1017,7 +1023,7 @@ void filesystem_writer_<LoggerPolicy>::copy_header(
template <typename LoggerPolicy>
void filesystem_writer_<LoggerPolicy>::write_block(
fragment_category cat, std::shared_ptr<block_data>&& data,
fragment_category cat, std::shared_ptr<internal::block_data>&& data,
physical_block_cb_type physical_block_cb, std::optional<std::string> meta) {
write_block_impl(cat, std::move(data), compressor_for_category(cat.value()),
std::move(meta), std::move(physical_block_cb));
@ -1025,7 +1031,8 @@ void filesystem_writer_<LoggerPolicy>::write_block(
template <typename LoggerPolicy>
void filesystem_writer_<LoggerPolicy>::write_block(
fragment_category::value_type cat, std::shared_ptr<block_data>&& data,
fragment_category::value_type cat,
std::shared_ptr<internal::block_data>&& data,
std::optional<std::string> meta) {
write_section_impl(section_type::BLOCK, std::move(data),
compressor_for_category(cat), std::move(meta));
@ -1033,20 +1040,20 @@ void filesystem_writer_<LoggerPolicy>::write_block(
template <typename LoggerPolicy>
void filesystem_writer_<LoggerPolicy>::write_metadata_v2_schema(
std::shared_ptr<block_data>&& data) {
std::shared_ptr<internal::block_data>&& data) {
write_section_impl(section_type::METADATA_V2_SCHEMA, std::move(data),
schema_bc_);
}
template <typename LoggerPolicy>
void filesystem_writer_<LoggerPolicy>::write_metadata_v2(
std::shared_ptr<block_data>&& data) {
std::shared_ptr<internal::block_data>&& data) {
write_section_impl(section_type::METADATA_V2, std::move(data), metadata_bc_);
}
template <typename LoggerPolicy>
void filesystem_writer_<LoggerPolicy>::write_history(
std::shared_ptr<block_data>&& data) {
std::shared_ptr<internal::block_data>&& data) {
write_section_impl(section_type::HISTORY, std::move(data), history_bc_);
}

10
src/dwarfs/scanner.cpp
View File

@ -38,7 +38,6 @@
#include <fmt/format.h>
#include <dwarfs/block_data.h>
#include <dwarfs/block_manager.h>
#include <dwarfs/categorizer.h>
#include <dwarfs/entry.h>
@ -52,6 +51,7 @@
#include <dwarfs/inode.h>
#include <dwarfs/inode_manager.h>
#include <dwarfs/inode_ordering.h>
#include <dwarfs/internal/block_data.h>
#include <dwarfs/internal/global_entry_data.h>
#include <dwarfs/internal/worker_group.h>
#include <dwarfs/logger.h>
@ -989,13 +989,15 @@ void scanner_<LoggerPolicy>::scan(
LOG_VERBOSE << "uncompressed metadata size: " << size_with_unit(data.size());
fsw.write_metadata_v2_schema(std::make_shared<block_data>(std::move(schema)));
fsw.write_metadata_v2(std::make_shared<block_data>(std::move(data)));
fsw.write_metadata_v2_schema(
std::make_shared<internal::block_data>(std::move(schema)));
fsw.write_metadata_v2(
std::make_shared<internal::block_data>(std::move(data)));
if (options_.enable_history) {
history hist(options_.history);
hist.append(options_.command_line_arguments);
fsw.write_history(std::make_shared<block_data>(hist.serialize()));
fsw.write_history(std::make_shared<internal::block_data>(hist.serialize()));
}
LOG_INFO << "waiting for compression to finish...";

10
src/dwarfs/segmenter.cpp
View File

@ -41,7 +41,6 @@
#include <folly/sorted_vector_types.h>
#include <folly/stats/Histogram.h>
#include <dwarfs/block_data.h>
#include <dwarfs/block_manager.h>
#include <dwarfs/chunkable.h>
#include <dwarfs/compiler.h>
@ -49,6 +48,7 @@
#include <dwarfs/cyclic_hash.h>
#include <dwarfs/entry.h>
#include <dwarfs/error.h>
#include <dwarfs/internal/block_data.h>
#include <dwarfs/logger.h>
#include <dwarfs/progress.h>
#include <dwarfs/segmenter.h>
@ -567,7 +567,7 @@ class active_block : private GranularityPolicy {
, filter_(bloom_filter_size)
, repseqmap_{repseqmap}
, repeating_collisions_{repcoll}
, data_{std::make_shared<block_data>()} {
, data_{std::make_shared<internal::block_data>()} {
DWARFS_CHECK((window_step & window_step_mask_) == 0,
"window step size not a power of two");
data_->reserve(this->frames_to_bytes(capacity_in_frames_));
@ -583,7 +583,9 @@ class active_block : private GranularityPolicy {
return size_in_frames() == capacity_in_frames_;
}
DWARFS_FORCE_INLINE std::shared_ptr<block_data> data() const { return data_; }
DWARFS_FORCE_INLINE std::shared_ptr<internal::block_data> data() const {
return data_;
}
DWARFS_FORCE_INLINE void
append_bytes(std::span<uint8_t const> data, bloom_filter& global_filter);
@ -628,7 +630,7 @@ class active_block : private GranularityPolicy {
fast_multimap<hash_t, offset_t, num_inline_offsets> offsets_;
repeating_sequence_map_type const& repseqmap_;
repeating_collisions_map_type& repeating_collisions_;
std::shared_ptr<block_data> data_;
std::shared_ptr<internal::block_data> data_;
};
class segmenter_progress : public progress::context {

6
test/segmenter_benchmark.cpp
View File

@ -24,10 +24,10 @@
#include <folly/Benchmark.h>
#include <dwarfs/block_data.h>
#include <dwarfs/block_manager.h>
#include <dwarfs/chunkable.h>
#include <dwarfs/compression_constraints.h>
#include <dwarfs/internal/block_data.h>
#include <dwarfs/progress.h>
#include <dwarfs/segmenter.h>
@ -142,11 +142,11 @@ void run_segmenter_test(unsigned iters, unsigned granularity,
dwarfs::progress prog;
auto blkmgr = std::make_shared<dwarfs::block_manager>();
std::vector<std::shared_ptr<dwarfs::block_data>> written;
std::vector<std::shared_ptr<dwarfs::internal::block_data>> written;
dwarfs::segmenter seg(
lgr, prog, blkmgr, cfg, cc, total_size,
[&written, blkmgr](std::shared_ptr<dwarfs::block_data> blk,
[&written, blkmgr](std::shared_ptr<dwarfs::internal::block_data> blk,
auto logical_block_num) {
auto physical_block_num = written.size();
written.push_back(blk);