TheCloud/dwarfs
1
0
Fork 0
mirror of https://github.com/mhx/dwarfs.git synced 2025-09-14 14:59:52 -04:00
Code Issues Packages Projects Releases Wiki Activity

refactor(metadata): cleanup and use references where possible

Browse Source
This commit is contained in:
Marcus Holland-Moritz 2023-12-19 19:03:03 +01:00
parent 866712bb3c
commit 366846caea
4 changed files with 165 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
include/dwarfs/metadata_types.h
View File

@ -58,9 +58,12 @@ class global_metadata {
using Meta = using Meta =
::apache::thrift::frozen::MappedFrozen<thrift::metadata::metadata>; ::apache::thrift::frozen::MappedFrozen<thrift::metadata::metadata>;
global_metadata(logger& lgr, Meta const* meta, bool check_consistency); global_metadata(logger& lgr, Meta const& meta);
Meta const* meta() const { return meta_; } static void check_consistency(logger& lgr, Meta const& meta);
void check_consistency(logger& lgr) const;
Meta const& meta() const { return meta_; }
uint32_t first_dir_entry(uint32_t ino) const; uint32_t first_dir_entry(uint32_t ino) const;
uint32_t parent_dir_entry(uint32_t ino) const; uint32_t parent_dir_entry(uint32_t ino) const;
@ -72,7 +75,7 @@ class global_metadata {
} }
private: private:
Meta const* const meta_; Meta const& meta_;
std::vector<thrift::metadata::directory> const directories_storage_; std::vector<thrift::metadata::directory> const directories_storage_;
thrift::metadata::directory const* const directories_; thrift::metadata::directory const* const directories_;
string_table const names_; string_table const names_;
@ -107,10 +110,10 @@ class inode_view
uint32_t inode_num() const { return inode_num_; } uint32_t inode_num() const { return inode_num_; }
private: private:
inode_view(InodeView iv, uint32_t inode_num_, Meta const* meta) inode_view(InodeView iv, uint32_t inode_num_, Meta const& meta)
: InodeView{iv} : InodeView{iv}
, inode_num_{inode_num_} , inode_num_{inode_num_}
, meta_{meta} {} , meta_{&meta} {}
uint32_t inode_num_; uint32_t inode_num_;
Meta const* meta_; Meta const* meta_;
@ -136,9 +139,9 @@ class directory_view {
boost::integer_range<uint32_t> entry_range() const; boost::integer_range<uint32_t> entry_range() const;
private: private:
directory_view(uint32_t inode, global_metadata const* g) directory_view(uint32_t inode, global_metadata const& g)
: inode_{inode} : inode_{inode}
, g_{g} {} , g_{&g} {}
uint32_t first_entry(uint32_t ino) const; uint32_t first_entry(uint32_t ino) const;
uint32_t parent_entry(uint32_t ino) const; uint32_t parent_entry(uint32_t ino) const;
@ -175,29 +178,29 @@ class dir_entry_view {
private: private:
dir_entry_view(DirEntryView v, uint32_t self_index, uint32_t parent_index, dir_entry_view(DirEntryView v, uint32_t self_index, uint32_t parent_index,
global_metadata const* g) global_metadata const& g)
: v_{v} : v_{v}
, self_index_{self_index} , self_index_{self_index}
, parent_index_{parent_index} , parent_index_{parent_index}
, g_{g} {} , g_{&g} {}
dir_entry_view(InodeView v, uint32_t self_index, uint32_t parent_index, dir_entry_view(InodeView v, uint32_t self_index, uint32_t parent_index,
global_metadata const* g) global_metadata const& g)
: v_{v} : v_{v}
, self_index_{self_index} , self_index_{self_index}
, parent_index_{parent_index} , parent_index_{parent_index}
, g_{g} {} , g_{&g} {}
static dir_entry_view static dir_entry_view
from_dir_entry_index(uint32_t self_index, uint32_t parent_index, from_dir_entry_index(uint32_t self_index, uint32_t parent_index,
global_metadata const* g); global_metadata const& g);
static dir_entry_view static dir_entry_view
from_dir_entry_index(uint32_t self_index, global_metadata const* g); from_dir_entry_index(uint32_t self_index, global_metadata const& g);
// TODO: this works, but it's strange; a limited version of dir_entry_view // TODO: this works, but it's strange; a limited version of dir_entry_view
// should work without a parent for these use cases // should work without a parent for these use cases
static std::string name(uint32_t index, global_metadata const* g); static std::string name(uint32_t index, global_metadata const& g);
static inode_view inode(uint32_t index, global_metadata const* g); static inode_view inode(uint32_t index, global_metadata const& g);
std::variant<DirEntryView, InodeView> v_; std::variant<DirEntryView, InodeView> v_;
uint32_t self_index_, parent_index_; uint32_t self_index_, parent_index_;
@ -229,8 +232,8 @@ class chunk_range {
friend class chunk_range; friend class chunk_range;
iterator(Meta const* meta, uint32_t it) iterator(Meta const* meta, uint32_t it)
: meta_(meta) : meta_{meta}
, it_(it) {} , it_{it} {}
bool equal(iterator const& other) const { bool equal(iterator const& other) const {
return meta_ == other.meta_ && it_ == other.it_; return meta_ == other.meta_ && it_ == other.it_;
@ -268,8 +271,8 @@ class chunk_range {
chunk_view operator[](uint32_t index) const { return meta_->chunks()[index]; } chunk_view operator[](uint32_t index) const { return meta_->chunks()[index]; }
private: private:
chunk_range(Meta const* meta, uint32_t begin, uint32_t end) chunk_range(Meta const& meta, uint32_t begin, uint32_t end)
: meta_(meta) : meta_(&meta)
, begin_(begin) , begin_(begin)
, end_(end) {} , end_(end) {}

4
include/dwarfs/metadata_v2.h
View File

@ -53,13 +53,13 @@ class metadata;
class metadata_v2 { class metadata_v2 {
public: public:
metadata_v2() = default; metadata_v2() = default;
metadata_v2(metadata_v2&&) = default;
metadata_v2& operator=(metadata_v2&&) = default;
metadata_v2(logger& lgr, std::span<uint8_t const> schema, metadata_v2(logger& lgr, std::span<uint8_t const> schema,
std::span<uint8_t const> data, metadata_options const& options, std::span<uint8_t const> data, metadata_options const& options,
int inode_offset = 0, bool force_consistency_check = false); int inode_offset = 0, bool force_consistency_check = false);
metadata_v2& operator=(metadata_v2&&) = default;
void void
dump(std::ostream& os, int detail_level, filesystem_info const& fsinfo, dump(std::ostream& os, int detail_level, filesystem_info const& fsinfo,
std::function<void(const std::string&, uint32_t)> const& icb) const { std::function<void(const std::string&, uint32_t)> const& icb) const {

233
src/dwarfs/metadata_types.cpp
View File

@ -38,16 +38,16 @@ namespace dwarfs {
namespace { namespace {
std::vector<thrift::metadata::directory> std::vector<thrift::metadata::directory>
unpack_directories(logger& lgr, global_metadata::Meta const* meta) { unpack_directories(logger& lgr, global_metadata::Meta const& meta) {
std::vector<thrift::metadata::directory> directories; std::vector<thrift::metadata::directory> directories;
if (auto opts = meta->options(); opts and opts->packed_directories()) { if (auto opts = meta.options(); opts and opts->packed_directories()) {
LOG_PROXY(debug_logger_policy, lgr); LOG_PROXY(debug_logger_policy, lgr);
auto ti = LOG_TIMED_DEBUG; auto ti = LOG_TIMED_DEBUG;
auto dirent = *meta->dir_entries(); auto dirent = *meta.dir_entries();
auto metadir = meta->directories(); auto metadir = meta.directories();
directories.resize(metadir.size()); directories.resize(metadir.size());
@ -104,41 +104,41 @@ int mode_rank(uint16_t mode) {
} }
} }
void check_empty_tables(global_metadata::Meta const* meta) { void check_empty_tables(global_metadata::Meta const& meta) {
if (meta->inodes().empty()) { if (meta.inodes().empty()) {
DWARFS_THROW(runtime_error, "empty inodes table"); DWARFS_THROW(runtime_error, "empty inodes table");
} }
if (meta->directories().empty()) { if (meta.directories().empty()) {
DWARFS_THROW(runtime_error, "empty directories table"); DWARFS_THROW(runtime_error, "empty directories table");
} }
if (meta->chunk_table().empty()) { if (meta.chunk_table().empty()) {
DWARFS_THROW(runtime_error, "empty chunk_table table"); DWARFS_THROW(runtime_error, "empty chunk_table table");
} }
if (auto de = meta->dir_entries()) { if (auto de = meta.dir_entries()) {
if (de->empty()) { if (de->empty()) {
DWARFS_THROW(runtime_error, "empty dir_entries table"); DWARFS_THROW(runtime_error, "empty dir_entries table");
} }
} else { } else {
if (meta->entry_table_v2_2().empty()) { if (meta.entry_table_v2_2().empty()) {
DWARFS_THROW(runtime_error, "empty entry_table_v2_2 table"); DWARFS_THROW(runtime_error, "empty entry_table_v2_2 table");
} }
} }
if (meta->modes().empty()) { if (meta.modes().empty()) {
DWARFS_THROW(runtime_error, "empty modes table"); DWARFS_THROW(runtime_error, "empty modes table");
} }
} }
void check_index_range(global_metadata::Meta const* meta) { void check_index_range(global_metadata::Meta const& meta) {
auto num_modes = meta->modes().size(); auto num_modes = meta.modes().size();
auto num_uids = meta->uids().size(); auto num_uids = meta.uids().size();
auto num_gids = meta->gids().size(); auto num_gids = meta.gids().size();
auto num_names = meta->names().size(); auto num_names = meta.names().size();
auto num_inodes = meta->inodes().size(); auto num_inodes = meta.inodes().size();
bool v2_2 = !static_cast<bool>(meta->dir_entries()); bool v2_2 = !static_cast<bool>(meta.dir_entries());
if (num_modes >= std::numeric_limits<uint16_t>::max()) { if (num_modes >= std::numeric_limits<uint16_t>::max()) {
DWARFS_THROW(runtime_error, "invalid number of modes"); DWARFS_THROW(runtime_error, "invalid number of modes");
@ -160,7 +160,7 @@ void check_index_range(global_metadata::Meta const* meta) {
DWARFS_THROW(runtime_error, "invalid number of inodes"); DWARFS_THROW(runtime_error, "invalid number of inodes");
} }
for (auto ino : meta->inodes()) { for (auto ino : meta.inodes()) {
if (ino.mode_index() >= num_modes) { if (ino.mode_index() >= num_modes) {
DWARFS_THROW(runtime_error, "mode_index out of range"); DWARFS_THROW(runtime_error, "mode_index out of range");
} }
@ -177,12 +177,12 @@ void check_index_range(global_metadata::Meta const* meta) {
} }
} }
if (auto dep = meta->dir_entries()) { if (auto dep = meta.dir_entries()) {
if (dep->size() >= std::numeric_limits<uint32_t>::max()) { if (dep->size() >= std::numeric_limits<uint32_t>::max()) {
DWARFS_THROW(runtime_error, "invalid number of dir_entries"); DWARFS_THROW(runtime_error, "invalid number of dir_entries");
} }
if (auto cn = meta->compact_names()) { if (auto cn = meta.compact_names()) {
num_names = cn->index().size(); num_names = cn->index().size();
if (!cn->packed_index()) { if (!cn->packed_index()) {
if (num_names == 0) { if (num_names == 0) {
@ -201,12 +201,12 @@ void check_index_range(global_metadata::Meta const* meta) {
} }
} }
} else { } else {
if (meta->entry_table_v2_2().size() >= if (meta.entry_table_v2_2().size() >=
std::numeric_limits<uint32_t>::max()) { std::numeric_limits<uint32_t>::max()) {
DWARFS_THROW(runtime_error, "invalid number of entries"); DWARFS_THROW(runtime_error, "invalid number of entries");
} }
for (auto ent : meta->entry_table_v2_2()) { for (auto ent : meta.entry_table_v2_2()) {
if (ent >= num_inodes) { if (ent >= num_inodes) {
DWARFS_THROW(runtime_error, "entry_table_v2_2 value out of range"); DWARFS_THROW(runtime_error, "entry_table_v2_2 value out of range");
} }
@ -214,38 +214,38 @@ void check_index_range(global_metadata::Meta const* meta) {
} }
} }
void check_packed_tables(global_metadata::Meta const* meta) { void check_packed_tables(global_metadata::Meta const& meta) {
if (meta->directories().size() >= std::numeric_limits<uint32_t>::max()) { if (meta.directories().size() >= std::numeric_limits<uint32_t>::max()) {
DWARFS_THROW(runtime_error, "invalid number of directories"); DWARFS_THROW(runtime_error, "invalid number of directories");
} }
if (meta->chunk_table().size() >= std::numeric_limits<uint32_t>::max()) { if (meta.chunk_table().size() >= std::numeric_limits<uint32_t>::max()) {
DWARFS_THROW(runtime_error, "invalid number of chunk_table entries"); DWARFS_THROW(runtime_error, "invalid number of chunk_table entries");
} }
if (auto opt = meta->options(); opt and opt->packed_directories()) { if (auto opt = meta.options(); opt and opt->packed_directories()) {
if (std::any_of(meta->directories().begin(), meta->directories().end(), if (std::any_of(meta.directories().begin(), meta.directories().end(),
[](auto i) { return i.parent_entry() != 0; })) { [](auto i) { return i.parent_entry() != 0; })) {
DWARFS_THROW(runtime_error, "parent_entry set in packed directory"); DWARFS_THROW(runtime_error, "parent_entry set in packed directory");
} }
if (std::accumulate(meta->directories().begin(), meta->directories().end(), if (std::accumulate(meta.directories().begin(), meta.directories().end(),
static_cast<size_t>(0), [](auto n, auto d) { static_cast<size_t>(0), [](auto n, auto d) {
return n + d.first_entry(); return n + d.first_entry();
}) != meta->dir_entries()->size()) { }) != meta.dir_entries()->size()) {
DWARFS_THROW(runtime_error, DWARFS_THROW(runtime_error,
"first_entry inconsistency in packed directories"); "first_entry inconsistency in packed directories");
} }
} else { } else {
size_t num_entries = meta->dir_entries() ? meta->dir_entries()->size() size_t num_entries =
: meta->inodes().size(); meta.dir_entries() ? meta.dir_entries()->size() : meta.inodes().size();
if (!std::is_sorted( if (!std::is_sorted(
meta->directories().begin(), meta->directories().end(), meta.directories().begin(), meta.directories().end(),
[](auto a, auto b) { return a.first_entry() < b.first_entry(); })) { [](auto a, auto b) { return a.first_entry() < b.first_entry(); })) {
DWARFS_THROW(runtime_error, "first_entry inconsistency"); DWARFS_THROW(runtime_error, "first_entry inconsistency");
} }
for (auto d : meta->directories()) { for (auto d : meta.directories()) {
if (auto i = d.first_entry(); i > num_entries) { if (auto i = d.first_entry(); i > num_entries) {
DWARFS_THROW(runtime_error, "first_entry out of range"); DWARFS_THROW(runtime_error, "first_entry out of range");
} }
@ -255,15 +255,14 @@ void check_packed_tables(global_metadata::Meta const* meta) {
} }
} }
if (auto opt = meta->options(); opt and opt->packed_chunk_table()) { if (auto opt = meta.options(); opt and opt->packed_chunk_table()) {
if (std::accumulate(meta->chunk_table().begin(), meta->chunk_table().end(), if (std::accumulate(meta.chunk_table().begin(), meta.chunk_table().end(),
static_cast<size_t>(0)) != meta->chunks().size()) { static_cast<size_t>(0)) != meta.chunks().size()) {
DWARFS_THROW(runtime_error, "packed chunk_table inconsistency"); DWARFS_THROW(runtime_error, "packed chunk_table inconsistency");
} }
} else { } else {
if (!std::is_sorted(meta->chunk_table().begin(), if (!std::is_sorted(meta.chunk_table().begin(), meta.chunk_table().end()) or
meta->chunk_table().end()) or meta.chunk_table().back() != meta.chunks().size()) {
meta->chunk_table().back() != meta->chunks().size()) {
DWARFS_THROW(runtime_error, "chunk_table inconsistency"); DWARFS_THROW(runtime_error, "chunk_table inconsistency");
} }
} }
@ -331,9 +330,9 @@ void check_plain_strings(
} }
} }
void check_string_tables(global_metadata::Meta const* meta) { void check_string_tables(global_metadata::Meta const& meta) {
size_t num_names = 0; size_t num_names = 0;
if (auto dep = meta->dir_entries()) { if (auto dep = meta.dir_entries()) {
if (dep->size() > 1) { if (dep->size() > 1) {
num_names = std::max_element(dep->begin(), dep->end(), num_names = std::max_element(dep->begin(), dep->end(),
[](auto const& a, auto const& b) { [](auto const& a, auto const& b) {
@ -343,9 +342,9 @@ void check_string_tables(global_metadata::Meta const* meta) {
1; 1;
} }
} else { } else {
if (meta->inodes().size() > 1) { if (meta.inodes().size() > 1) {
num_names = num_names =
std::max_element(meta->inodes().begin(), meta->inodes().end(), std::max_element(meta.inodes().begin(), meta.inodes().end(),
[](auto const& a, auto const& b) { [](auto const& a, auto const& b) {
return a.name_index_v2_2() < b.name_index_v2_2(); return a.name_index_v2_2() < b.name_index_v2_2();
}) })
@ -359,40 +358,40 @@ void check_string_tables(global_metadata::Meta const* meta) {
constexpr size_t max_name_len = 512; constexpr size_t max_name_len = 512;
constexpr size_t max_symlink_len = 4096; constexpr size_t max_symlink_len = 4096;
if (auto cn = meta->compact_names()) { if (auto cn = meta.compact_names()) {
check_compact_strings(*cn, num_names, max_name_len, "names"); check_compact_strings(*cn, num_names, max_name_len, "names");
} else { } else {
check_plain_strings(meta->names(), num_names, max_name_len, "names"); check_plain_strings(meta.names(), num_names, max_name_len, "names");
} }
size_t num_symlink_strings = 0; size_t num_symlink_strings = 0;
if (!meta->symlink_table().empty()) { if (!meta.symlink_table().empty()) {
num_symlink_strings = *std::max_element(meta->symlink_table().begin(), num_symlink_strings = *std::max_element(meta.symlink_table().begin(),
meta->symlink_table().end()) + meta.symlink_table().end()) +
1; 1;
} }
if (auto cs = meta->compact_symlinks()) { if (auto cs = meta.compact_symlinks()) {
check_compact_strings(*cs, num_symlink_strings, max_symlink_len, check_compact_strings(*cs, num_symlink_strings, max_symlink_len,
"symlink strings"); "symlink strings");
} else { } else {
check_plain_strings(meta->symlinks(), num_symlink_strings, max_symlink_len, check_plain_strings(meta.symlinks(), num_symlink_strings, max_symlink_len,
"symlink strings"); "symlink strings");
} }
} }
void check_chunks(global_metadata::Meta const* meta) { void check_chunks(global_metadata::Meta const& meta) {
auto block_size = meta->block_size(); auto block_size = meta.block_size();
if (block_size == 0 || (block_size & (block_size - 1))) { if (block_size == 0 || (block_size & (block_size - 1))) {
DWARFS_THROW(runtime_error, "invalid block size"); DWARFS_THROW(runtime_error, "invalid block size");
} }
if (meta->chunks().size() >= std::numeric_limits<uint32_t>::max()) { if (meta.chunks().size() >= std::numeric_limits<uint32_t>::max()) {
DWARFS_THROW(runtime_error, "invalid number of chunks"); DWARFS_THROW(runtime_error, "invalid number of chunks");
} }
for (auto c : meta->chunks()) { for (auto c : meta.chunks()) {
if (c.offset() >= block_size || c.size() > block_size) { if (c.offset() >= block_size || c.size() > block_size) {
DWARFS_THROW(runtime_error, "chunk offset/size out of range"); DWARFS_THROW(runtime_error, "chunk offset/size out of range");
} }
@ -402,15 +401,15 @@ void check_chunks(global_metadata::Meta const* meta) {
} }
} }
std::array<size_t, 6> check_partitioning(global_metadata::Meta const* meta) { std::array<size_t, 6> check_partitioning(global_metadata::Meta const& meta) {
std::array<size_t, 6> offsets; std::array<size_t, 6> offsets;
for (int r = 0; r < static_cast<int>(offsets.size()); ++r) { for (int r = 0; r < static_cast<int>(offsets.size()); ++r) {
if (auto dep = meta->dir_entries()) { if (auto dep = meta.dir_entries()) {
auto pred = [r, modes = meta->modes()](auto ino) { auto pred = [r, modes = meta.modes()](auto ino) {
return mode_rank(modes[ino.mode_index()]) < r; return mode_rank(modes[ino.mode_index()]) < r;
}; };
auto inodes = meta->inodes(); auto inodes = meta.inodes();
if (!std::is_partitioned(inodes.begin(), inodes.end(), pred)) { if (!std::is_partitioned(inodes.begin(), inodes.end(), pred)) {
DWARFS_THROW(runtime_error, "inode table inconsistency"); DWARFS_THROW(runtime_error, "inode table inconsistency");
@ -420,11 +419,10 @@ std::array<size_t, 6> check_partitioning(global_metadata::Meta const* meta) {
inodes.begin(), inodes.begin(),
std::partition_point(inodes.begin(), inodes.end(), pred)); std::partition_point(inodes.begin(), inodes.end(), pred));
} else { } else {
auto pred = [r, modes = meta->modes(), auto pred = [r, modes = meta.modes(), inodes = meta.inodes()](auto ent) {
inodes = meta->inodes()](auto ent) {
return mode_rank(modes[inodes[ent].mode_index()]) < r; return mode_rank(modes[inodes[ent].mode_index()]) < r;
}; };
auto entries = meta->entry_table_v2_2(); auto entries = meta.entry_table_v2_2();
if (!std::is_partitioned(entries.begin(), entries.end(), pred)) { if (!std::is_partitioned(entries.begin(), entries.end(), pred)) {
DWARFS_THROW(runtime_error, "entry_table_v2_2 inconsistency"); DWARFS_THROW(runtime_error, "entry_table_v2_2 inconsistency");
@ -439,8 +437,8 @@ std::array<size_t, 6> check_partitioning(global_metadata::Meta const* meta) {
return offsets; return offsets;
} }
global_metadata::Meta const* global_metadata::Meta const&
check_metadata(logger& lgr, global_metadata::Meta const* meta, bool check) { check_metadata(logger& lgr, global_metadata::Meta const& meta, bool check) {
if (check) { if (check) {
LOG_PROXY(debug_logger_policy, lgr); LOG_PROXY(debug_logger_policy, lgr);
@ -455,13 +453,13 @@ check_metadata(logger& lgr, global_metadata::Meta const* meta, bool check) {
check_chunks(meta); check_chunks(meta);
auto offsets = check_partitioning(meta); auto offsets = check_partitioning(meta);
auto num_dir = meta->directories().size() - 1; auto num_dir = meta.directories().size() - 1;
auto num_lnk = meta->symlink_table().size(); auto num_lnk = meta.symlink_table().size();
auto num_reg_unique = meta->chunk_table().size() - 1; auto num_reg_unique = meta.chunk_table().size() - 1;
size_t num_reg_shared = 0; size_t num_reg_shared = 0;
if (auto sfp = meta->shared_files_table()) { if (auto sfp = meta.shared_files_table()) {
if (meta->options()->packed_shared_files_table()) { if (meta.options()->packed_shared_files_table()) {
num_reg_shared = num_reg_shared =
std::accumulate(sfp->begin(), sfp->end(), 2 * sfp->size()); std::accumulate(sfp->begin(), sfp->end(), 2 * sfp->size());
num_reg_unique -= sfp->size(); num_reg_unique -= sfp->size();
@ -477,7 +475,7 @@ check_metadata(logger& lgr, global_metadata::Meta const* meta, bool check) {
} }
} }
size_t num_dev = meta->devices() ? meta->devices()->size() : 0; size_t num_dev = meta.devices() ? meta.devices()->size() : 0;
if (num_dir != offsets[1]) { if (num_dir != offsets[1]) {
DWARFS_THROW(runtime_error, "wrong number of directories"); DWARFS_THROW(runtime_error, "wrong number of directories");
@ -495,9 +493,9 @@ check_metadata(logger& lgr, global_metadata::Meta const* meta, bool check) {
DWARFS_THROW(runtime_error, "wrong number of devices"); DWARFS_THROW(runtime_error, "wrong number of devices");
} }
if (!meta->dir_entries()) { if (!meta.dir_entries()) {
for (auto ino : meta->inodes()) { for (auto ino : meta.inodes()) {
auto mode = meta->modes()[ino.mode_index()]; auto mode = meta.modes()[ino.mode_index()];
auto i = ino.inode_v2_2(); auto i = ino.inode_v2_2();
int base = mode_rank(mode); int base = mode_rank(mode);
@ -514,24 +512,31 @@ check_metadata(logger& lgr, global_metadata::Meta const* meta, bool check) {
} // namespace } // namespace
global_metadata::global_metadata(logger& lgr, Meta const* meta, global_metadata::global_metadata(logger& lgr, Meta const& meta)
bool check_consistency) : meta_{meta}
: meta_{check_metadata(lgr, meta, check_consistency)}
, directories_storage_{unpack_directories(lgr, meta_)} , directories_storage_{unpack_directories(lgr, meta_)}
, directories_{directories_storage_.empty() ? nullptr , directories_{directories_storage_.empty() ? nullptr
: directories_storage_.data()} : directories_storage_.data()}
, names_{meta_->compact_names() , names_{meta_.compact_names()
? string_table(lgr, "names", *meta_->compact_names()) ? string_table(lgr, "names", *meta_.compact_names())
: string_table(meta_->names())} {} : string_table(meta_.names())} {}
void global_metadata::check_consistency(logger& lgr, Meta const& meta) {
check_metadata(lgr, meta, true);
}
void global_metadata::check_consistency(logger& lgr) const {
check_consistency(lgr, meta_);
}
uint32_t global_metadata::first_dir_entry(uint32_t ino) const { uint32_t global_metadata::first_dir_entry(uint32_t ino) const {
return directories_ ? directories_[ino].first_entry().value() return directories_ ? directories_[ino].first_entry().value()
: meta_->directories()[ino].first_entry(); : meta_.directories()[ino].first_entry();
} }
uint32_t global_metadata::parent_dir_entry(uint32_t ino) const { uint32_t global_metadata::parent_dir_entry(uint32_t ino) const {
return directories_ ? directories_[ino].parent_entry().value() return directories_ ? directories_[ino].parent_entry().value()
: meta_->directories()[ino].parent_entry(); : meta_.directories()[ino].parent_entry();
} }
auto inode_view::mode() const -> mode_type { auto inode_view::mode() const -> mode_type {
@ -555,7 +560,7 @@ std::string dir_entry_view::name() const {
}, },
[this](InodeView const& iv) { [this](InodeView const& iv) {
return std::string( return std::string(
g_->meta()->names()[iv.name_index_v2_2()]); g_->meta().names()[iv.name_index_v2_2()]);
}, },
}, },
v_); v_);
@ -565,7 +570,7 @@ inode_view dir_entry_view::inode() const {
return std::visit(overloaded{ return std::visit(overloaded{
[this](DirEntryView const& dev) { [this](DirEntryView const& dev) {
return inode_view( return inode_view(
g_->meta()->inodes()[dev.inode_num()], g_->meta().inodes()[dev.inode_num()],
dev.inode_num(), g_->meta()); dev.inode_num(), g_->meta());
}, },
[this](InodeView const& iv) { [this](InodeView const& iv) {
@ -592,10 +597,10 @@ bool dir_entry_view::is_root() const {
dir_entry_view dir_entry_view
dir_entry_view::from_dir_entry_index(uint32_t self_index, uint32_t parent_index, dir_entry_view::from_dir_entry_index(uint32_t self_index, uint32_t parent_index,
global_metadata const* g) { global_metadata const& g) {
auto meta = g->meta(); auto& meta = g.meta();
if (auto de = meta->dir_entries()) { if (auto de = meta.dir_entries()) {
DWARFS_CHECK(self_index < de->size(), "self_index out of range"); DWARFS_CHECK(self_index < de->size(), "self_index out of range");
DWARFS_CHECK(parent_index < de->size(), "parent_index out of range"); DWARFS_CHECK(parent_index < de->size(), "parent_index out of range");
@ -604,36 +609,36 @@ dir_entry_view::from_dir_entry_index(uint32_t self_index, uint32_t parent_index,
return dir_entry_view(dev, self_index, parent_index, g); return dir_entry_view(dev, self_index, parent_index, g);
} }
DWARFS_CHECK(self_index < meta->inodes().size(), "self_index out of range"); DWARFS_CHECK(self_index < meta.inodes().size(), "self_index out of range");
DWARFS_CHECK(parent_index < meta->inodes().size(), "self_index out of range"); DWARFS_CHECK(parent_index < meta.inodes().size(), "self_index out of range");
auto iv = meta->inodes()[self_index]; auto iv = meta.inodes()[self_index];
return dir_entry_view(iv, self_index, parent_index, g); return dir_entry_view(iv, self_index, parent_index, g);
} }
dir_entry_view dir_entry_view::from_dir_entry_index(uint32_t self_index, dir_entry_view dir_entry_view::from_dir_entry_index(uint32_t self_index,
global_metadata const* g) { global_metadata const& g) {
auto meta = g->meta(); auto& meta = g.meta();
if (auto de = meta->dir_entries()) { if (auto de = meta.dir_entries()) {
DWARFS_CHECK(self_index < de->size(), "self_index out of range"); DWARFS_CHECK(self_index < de->size(), "self_index out of range");
auto dev = (*de)[self_index]; auto dev = (*de)[self_index];
DWARFS_CHECK(dev.inode_num() < meta->directories().size(), DWARFS_CHECK(dev.inode_num() < meta.directories().size(),
"self_index inode out of range"); "self_index inode out of range");
return dir_entry_view(dev, self_index, g->parent_dir_entry(dev.inode_num()), return dir_entry_view(dev, self_index, g.parent_dir_entry(dev.inode_num()),
g); g);
} }
DWARFS_CHECK(self_index < meta->inodes().size(), "self_index out of range"); DWARFS_CHECK(self_index < meta.inodes().size(), "self_index out of range");
auto iv = meta->inodes()[self_index]; auto iv = meta.inodes()[self_index];
DWARFS_CHECK(iv.inode_v2_2() < meta->directories().size(), DWARFS_CHECK(iv.inode_v2_2() < meta.directories().size(),
"parent_index out of range"); "parent_index out of range");
return dir_entry_view( return dir_entry_view(
iv, self_index, iv, self_index,
meta->entry_table_v2_2()[meta->directories()[iv.inode_v2_2()] meta.entry_table_v2_2()[meta.directories()[iv.inode_v2_2()]
.parent_entry()], .parent_entry()],
g); g);
} }
@ -642,32 +647,32 @@ std::optional<dir_entry_view> dir_entry_view::parent() const {
return std::nullopt; return std::nullopt;
} }
return from_dir_entry_index(parent_index_, g_); return from_dir_entry_index(parent_index_, *g_);
} }
std::string dir_entry_view::name(uint32_t index, global_metadata const* g) { std::string dir_entry_view::name(uint32_t index, global_metadata const& g) {
if (auto de = g->meta()->dir_entries()) { if (auto de = g.meta().dir_entries()) {
DWARFS_CHECK(index < de->size(), "index out of range"); DWARFS_CHECK(index < de->size(), "index out of range");
auto dev = (*de)[index]; auto dev = (*de)[index];
return g->names()[dev.name_index()]; return g.names()[dev.name_index()];
} }
DWARFS_CHECK(index < g->meta()->inodes().size(), "index out of range"); DWARFS_CHECK(index < g.meta().inodes().size(), "index out of range");
auto iv = g->meta()->inodes()[index]; auto iv = g.meta().inodes()[index];
return std::string(g->meta()->names()[iv.name_index_v2_2()]); return std::string(g.meta().names()[iv.name_index_v2_2()]);
} }
inode_view dir_entry_view::inode(uint32_t index, global_metadata const* g) { inode_view dir_entry_view::inode(uint32_t index, global_metadata const& g) {
if (auto de = g->meta()->dir_entries()) { if (auto de = g.meta().dir_entries()) {
DWARFS_CHECK(index < de->size(), "index out of range"); DWARFS_CHECK(index < de->size(), "index out of range");
auto dev = (*de)[index]; auto dev = (*de)[index];
return inode_view(g->meta()->inodes()[dev.inode_num()], dev.inode_num(), return inode_view(g.meta().inodes()[dev.inode_num()], dev.inode_num(),
g->meta()); g.meta());
} }
DWARFS_CHECK(index < g->meta()->inodes().size(), "index out of range"); DWARFS_CHECK(index < g.meta().inodes().size(), "index out of range");
auto iv = g->meta()->inodes()[index]; auto iv = g.meta().inodes()[index];
return inode_view(iv, iv.inode_v2_2(), g->meta()); return inode_view(iv, iv.inode_v2_2(), g.meta());
} }
std::string dir_entry_view::path() const { std::string dir_entry_view::path() const {
@ -728,7 +733,7 @@ uint32_t directory_view::parent_inode() const {
auto ent = parent_entry(inode_); auto ent = parent_entry(inode_);
if (auto e = g_->meta()->dir_entries()) { if (auto e = g_->meta().dir_entries()) {
ent = (*e)[ent].inode_num(); ent = (*e)[ent].inode_num();
} }

34
src/dwarfs/metadata_v2.cpp
View File

@ -153,6 +153,15 @@ check_frozen(MappedFrozen<thrift::metadata::metadata> meta) {
return meta; return meta;
} }
global_metadata::Meta const&
check_metadata_consistency(logger& lgr, global_metadata::Meta const& meta,
bool force_consistency_check) {
if (force_consistency_check) {
global_metadata::check_consistency(lgr, meta);
}
return meta;
}
void analyze_frozen(std::ostream& os, void analyze_frozen(std::ostream& os,
MappedFrozen<thrift::metadata::metadata> const& meta, MappedFrozen<thrift::metadata::metadata> const& meta,
size_t total_size, int detail) { size_t total_size, int detail) {
@ -337,9 +346,10 @@ class metadata_ final : public metadata_v2::impl {
: data_(data) : data_(data)
, meta_( , meta_(
check_frozen(map_frozen<thrift::metadata::metadata>(schema, data_))) check_frozen(map_frozen<thrift::metadata::metadata>(schema, data_)))
, global_(lgr, &meta_, , global_(lgr, check_metadata_consistency(lgr, meta_,
options.check_consistency || force_consistency_check) options.check_consistency ||
, root_(dir_entry_view::from_dir_entry_index(0, &global_)) force_consistency_check))
, root_(dir_entry_view::from_dir_entry_index(0, global_))
, LOG_PROXY_INIT(lgr) , LOG_PROXY_INIT(lgr)
, inode_offset_(inode_offset) , inode_offset_(inode_offset)
, symlink_inode_offset_(find_inode_offset(inode_rank::INO_LNK)) , symlink_inode_offset_(find_inode_offset(inode_rank::INO_LNK))
@ -483,13 +493,13 @@ class metadata_ final : public metadata_v2::impl {
// TODO: move compatibility details to metadata_types // TODO: move compatibility details to metadata_types
uint32_t index = uint32_t index =
meta_.dir_entries() ? inode : meta_.entry_table_v2_2()[inode]; meta_.dir_entries() ? inode : meta_.entry_table_v2_2()[inode];
return inode_view(meta_.inodes()[index], inode, &meta_); return inode_view(meta_.inodes()[index], inode, meta_);
} }
dir_entry_view dir_entry_view
make_dir_entry_view(uint32_t self_index, uint32_t parent_index) const { make_dir_entry_view(uint32_t self_index, uint32_t parent_index) const {
return dir_entry_view::from_dir_entry_index(self_index, parent_index, return dir_entry_view::from_dir_entry_index(self_index, parent_index,
&global_); global_);
} }
// This represents the order in which inodes are stored in inodes // This represents the order in which inodes are stored in inodes
@ -571,7 +581,7 @@ class metadata_ final : public metadata_v2::impl {
directory_view make_directory_view(inode_view iv) const { directory_view make_directory_view(inode_view iv) const {
// TODO: revisit: is this the way to do it? // TODO: revisit: is this the way to do it?
return directory_view(iv.inode_num(), &global_); return directory_view(iv.inode_num(), global_);
} }
void analyze_chunks(std::ostream& os) const; void analyze_chunks(std::ostream& os) const;
@ -626,7 +636,7 @@ class metadata_ final : public metadata_v2::impl {
inode < (static_cast<int>(meta_.chunk_table().size()) - 1)) { inode < (static_cast<int>(meta_.chunk_table().size()) - 1)) {
uint32_t begin = chunk_table_lookup(inode); uint32_t begin = chunk_table_lookup(inode);
uint32_t end = chunk_table_lookup(inode + 1); uint32_t end = chunk_table_lookup(inode + 1);
rv = chunk_range(&meta_, begin, end); rv = chunk_range(meta_, begin, end);
} }
return rv; return rv;
@ -1369,14 +1379,14 @@ metadata_<LoggerPolicy>::find(directory_view dir, std::string_view name) const {
auto it = std::lower_bound(range.begin(), range.end(), name, auto it = std::lower_bound(range.begin(), range.end(), name,
[&](auto ix, std::string_view name) { [&](auto ix, std::string_view name) {
return dir_entry_view::name(ix, &global_) < name; return dir_entry_view::name(ix, global_) < name;
}); });
std::optional<inode_view> rv; std::optional<inode_view> rv;
if (it != range.end()) { if (it != range.end()) {
if (dir_entry_view::name(*it, &global_) == name) { if (dir_entry_view::name(*it, global_) == name) {
rv = dir_entry_view::inode(*it, &global_); rv = dir_entry_view::inode(*it, global_);
} }
} }
@ -1502,8 +1512,8 @@ metadata_<LoggerPolicy>::readdir(directory_view dir, size_t offset) const {
} }
auto index = dir.first_entry() + offset; auto index = dir.first_entry() + offset;
auto inode = dir_entry_view::inode(index, &global_); auto inode = dir_entry_view::inode(index, global_);
return std::pair(inode, dir_entry_view::name(index, &global_)); return std::pair(inode, dir_entry_view::name(index, global_));
} }
return std::nullopt; return std::nullopt;