TheCloud/dwarfs
1
0
Fork 0
mirror of https://github.com/mhx/dwarfs.git synced 2025-09-14 06:48:39 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat: pass category metadata to compressors when re-writing filesystem

Browse Source
This commit is contained in:
Marcus Holland-Moritz 2025-08-26 23:26:01 +02:00
parent 276745674f
commit e6d848bcb4
3 changed files with 36 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/dwarfs/writer/internal/filesystem_writer_detail.h
View File

@ -93,11 +93,13 @@ class filesystem_writer_detail {
virtual void check_block_compression(
compression_type compression, std::span<uint8_t const> data,
std::optional<fragment_category::value_type> cat = std::nullopt,
std::optional<std::string> cat_metadata = std::nullopt,
block_compression_info* info = nullptr) = 0;
virtual void rewrite_section(
section_type type, compression_type compression,
std::span<uint8_t const> data,
std::optional<fragment_category::value_type> cat = std::nullopt) = 0;
std::optional<fragment_category::value_type> cat = std::nullopt,
std::optional<std::string> cat_metadata = std::nullopt) = 0;
virtual void rewrite_block(
delayed_data_fn_type data, size_t uncompressed_size,
std::optional<fragment_category::value_type> cat = std::nullopt) = 0;

15
src/utility/rewrite_filesystem.cpp
View File

@ -288,13 +288,18 @@ void rewrite_filesystem(logger& lgr, dwarfs::reader::filesystem_v2 const& fs,
dwarfs::writer::internal::block_compression_info bci;
auto catstr = fs.get_block_category(block_no);
std::optional<fragment_category::value_type> cat;
std::optional<std::string> cat_metadata;
if (catstr) {
cat = cat_resolver.category_value(catstr.value());
}
if (auto cm = fs.get_block_category_metadata(block_no)) {
cat_metadata = cm->dump();
}
writer.check_block_compression(
s->compression(), parser->section_data(*s), cat,
s->compression(), parser->section_data(*s), cat, cat_metadata,
opts.change_block_size ? &bci : nullptr);
if (opts.change_block_size) {
@ -453,8 +458,14 @@ void rewrite_filesystem(logger& lgr, dwarfs::reader::filesystem_v2 const& fs,
if (recompress_block) {
log_recompress(s, cat);
std::optional<std::string> cat_metadata;
if (auto cm = fs.get_block_category_metadata(block_no)) {
cat_metadata = cm->dump();
}
writer.rewrite_section(section_type::BLOCK, s->compression(),
parser->section_data(*s), cat);
parser->section_data(*s), cat, cat_metadata);
} else {
copy_compressed(s, cat);
}

30
src/writer/filesystem_writer.cpp
View File

@ -594,11 +594,12 @@ class filesystem_writer_ final : public filesystem_writer_detail {
void check_block_compression(compression_type compression,
std::span<uint8_t const> data,
std::optional<fragment_category::value_type> cat,
std::optional<std::string> cat_metadata,
block_compression_info* info) override;
void
rewrite_section(section_type type, compression_type compression,
std::span<uint8_t const> data,
std::optional<fragment_category::value_type> cat) override;
void rewrite_section(section_type type, compression_type compression,
std::span<uint8_t const> data,
std::optional<fragment_category::value_type> cat,
std::optional<std::string> cat_metadata) override;
void rewrite_block(delayed_data_fn_type data, size_t uncompressed_size,
std::optional<fragment_category::value_type> cat) override;
void write_compressed_section(fs_section const& sec,
@ -892,7 +893,7 @@ template <typename LoggerPolicy>
void filesystem_writer_<LoggerPolicy>::check_block_compression(
compression_type compression, std::span<uint8_t const> data,
std::optional<fragment_category::value_type> cat,
block_compression_info* info) {
std::optional<std::string> cat_metadata, block_compression_info* info) {
block_compressor const* bc{nullptr};
if (cat) {
@ -903,11 +904,15 @@ void filesystem_writer_<LoggerPolicy>::check_block_compression(
block_decompressor bd(compression, data);
if (!cat_metadata) {
cat_metadata = bd.metadata();
}
if (auto reqstr = bc->metadata_requirements(); !reqstr.empty()) {
auto req = compression_metadata_requirements<nlohmann::json>{reqstr};
try {
req.check(bd.metadata());
req.check(cat_metadata);
} catch (std::exception const& e) {
auto msg = fmt::format(
"cannot compress {} compressed block with compressor '{}' because "
@ -919,7 +924,7 @@ void filesystem_writer_<LoggerPolicy>::check_block_compression(
if (info) {
info->uncompressed_size = bd.uncompressed_size();
info->metadata = bd.metadata();
info->metadata = cat_metadata;
if (info->metadata) {
info->constraints = bc->get_compression_constraints(*info->metadata);
}
@ -961,16 +966,21 @@ template <typename LoggerPolicy>
void filesystem_writer_<LoggerPolicy>::rewrite_section(
section_type type, compression_type compression,
std::span<uint8_t const> data,
std::optional<fragment_category::value_type> cat) {
std::optional<fragment_category::value_type> cat,
std::optional<std::string> cat_metadata) {
auto bd = block_decompressor(compression, data);
auto uncompressed_size = bd.uncompressed_size();
if (!cat_metadata) {
cat_metadata = bd.metadata();
}
rewrite_section_delayed_data(
type,
[bd = std::move(bd)]() mutable {
[bd = std::move(bd), meta = std::move(cat_metadata)]() mutable {
auto block = bd.start_decompression(malloc_byte_buffer::create());
bd.decompress_frame(bd.uncompressed_size());
return std::pair{std::move(block), bd.metadata()};
return std::pair{std::move(block), meta};
},
uncompressed_size, cat);
}