More naming cleanups

This commit is contained in:
Marcus Holland-Moritz 2021-03-14 20:47:59 +01:00
parent 93e7ba7e09
commit 3608072836
6 changed files with 47 additions and 43 deletions

View File

@ -254,7 +254,7 @@ void dir::scan(os_access&, progress&) {}
void dir::pack_entry(thrift::metadata::metadata& mv2, void dir::pack_entry(thrift::metadata::metadata& mv2,
global_entry_data const& data) const { global_entry_data const& data) const {
DWARFS_NOTHROW(mv2.entry_index.at(inode_num())) = mv2.entries.size(); DWARFS_NOTHROW(mv2.entry_table_v2_2.at(inode_num())) = mv2.entries.size();
mv2.entries.emplace_back(); mv2.entries.emplace_back();
entry::pack(mv2.entries.back(), data); entry::pack(mv2.entries.back(), data);
} }
@ -268,7 +268,8 @@ void dir::pack(thrift::metadata::metadata& mv2,
// d.entry_count = entries_.size(); // d.entry_count = entries_.size();
mv2.directories.push_back(d); mv2.directories.push_back(d);
for (entry_ptr const& e : entries_) { for (entry_ptr const& e : entries_) {
DWARFS_NOTHROW(mv2.entry_index.at(e->inode_num())) = mv2.entries.size(); DWARFS_NOTHROW(mv2.entry_table_v2_2.at(e->inode_num())) =
mv2.entries.size();
mv2.entries.emplace_back(); mv2.entries.emplace_back();
e->pack(mv2.entries.back(), data); e->pack(mv2.entries.back(), data);
} }

View File

@ -100,7 +100,8 @@ uint32_t global_entry_data::get_name_index(std::string const& name) const {
return DWARFS_NOTHROW(names_.at(name)); return DWARFS_NOTHROW(names_.at(name));
} }
uint32_t global_entry_data::get_symlink_table_entry(std::string const& link) const { uint32_t
global_entry_data::get_symlink_table_entry(std::string const& link) const {
return DWARFS_NOTHROW(symlinks_.at(link)); return DWARFS_NOTHROW(symlinks_.at(link));
} }

View File

@ -66,7 +66,7 @@ uint32_t directory_view::parent_inode() const {
} }
directory_view::directory_view(uint32_t inode, Meta const* meta) directory_view::directory_view(uint32_t inode, Meta const* meta)
: entry_(meta->entries()[meta->entry_index()[inode]]) : entry_(meta->entries()[meta->entry_table_v2_2()[inode]])
, meta_(meta) {} , meta_(meta) {}
std::string directory_view::path() const { std::string directory_view::path() const {

View File

@ -113,7 +113,7 @@ class metadata_ final : public metadata_v2::impl {
int inode_offset) int inode_offset)
: data_(data) : data_(data)
, meta_(map_frozen<thrift::metadata::metadata>(schema, data_)) , meta_(map_frozen<thrift::metadata::metadata>(schema, data_))
, root_(meta_.entries()[meta_.entry_index()[0]], &meta_) , root_(meta_.entries()[meta_.entry_table_v2_2()[0]], &meta_)
, log_(lgr) , log_(lgr)
, inode_offset_(inode_offset) , inode_offset_(inode_offset)
, symlink_table_offset_(find_index_offset(inode_rank::INO_LNK)) , symlink_table_offset_(find_index_offset(inode_rank::INO_LNK))
@ -141,17 +141,18 @@ class metadata_ final : public metadata_v2::impl {
"metadata inconsistency: number of symlinks ({}) does not match " "metadata inconsistency: number of symlinks ({}) does not match "
"chunk/symlink table delta ({} - {} = {})", "chunk/symlink table delta ({} - {} = {})",
meta_.symlink_table().size(), file_index_offset_, meta_.symlink_table().size(), file_index_offset_,
symlink_table_offset_, file_index_offset_ - symlink_table_offset_)); symlink_table_offset_,
file_index_offset_ - symlink_table_offset_));
} }
if (int(meta_.chunk_index().size() - 1) != if (int(meta_.chunk_table().size() - 1) !=
(dev_index_offset_ - file_index_offset_)) { (dev_index_offset_ - file_index_offset_)) {
DWARFS_THROW( DWARFS_THROW(
runtime_error, runtime_error,
fmt::format( fmt::format(
"metadata inconsistency: number of files ({}) does not match " "metadata inconsistency: number of files ({}) does not match "
"device/chunk index delta ({} - {} = {})", "device/chunk index delta ({} - {} = {})",
meta_.chunk_index().size() - 1, dev_index_offset_, meta_.chunk_table().size() - 1, dev_index_offset_,
file_index_offset_, dev_index_offset_ - file_index_offset_)); file_index_offset_, dev_index_offset_ - file_index_offset_));
} }
@ -239,10 +240,10 @@ class metadata_ final : public metadata_v2::impl {
} }
entry_view make_entry_view_from_inode(uint32_t inode) const { entry_view make_entry_view_from_inode(uint32_t inode) const {
return make_entry_view(meta_.entry_index()[inode]); return make_entry_view(meta_.entry_table_v2_2()[inode]);
} }
// This represents the order in which inodes are stored in entry_index // This represents the order in which inodes are stored in entry_table_v2_2
enum class inode_rank { enum class inode_rank {
INO_DIR, INO_DIR,
INO_LNK, INO_LNK,
@ -294,7 +295,7 @@ class metadata_ final : public metadata_v2::impl {
} }
size_t find_index_offset(inode_rank rank) const { size_t find_index_offset(inode_rank rank) const {
auto range = boost::irange(size_t(0), meta_.entry_index().size()); auto range = boost::irange(size_t(0), meta_.entry_table_v2_2().size());
auto it = std::lower_bound(range.begin(), range.end(), rank, auto it = std::lower_bound(range.begin(), range.end(), rank,
[&](auto inode, inode_rank r) { [&](auto inode, inode_rank r) {
@ -326,8 +327,8 @@ class metadata_ final : public metadata_v2::impl {
size_t reg_file_size(entry_view entry) const { size_t reg_file_size(entry_view entry) const {
auto inode = entry.inode() - file_index_offset_; auto inode = entry.inode() - file_index_offset_;
uint32_t cur = meta_.chunk_index()[inode]; uint32_t cur = meta_.chunk_table()[inode];
uint32_t end = meta_.chunk_index()[inode + 1]; uint32_t end = meta_.chunk_table()[inode + 1];
if (cur > end) { if (cur > end) {
DWARFS_THROW(runtime_error, DWARFS_THROW(runtime_error,
fmt::format("invalid chunk range: [{0}..{1}]", cur, end)); fmt::format("invalid chunk range: [{0}..{1}]", cur, end));
@ -371,7 +372,7 @@ class metadata_ final : public metadata_v2::impl {
template <typename T> template <typename T>
void walk_tree(T&& func) const { void walk_tree(T&& func) const {
set_type<int> seen; set_type<int> seen;
auto root = meta_.entry_index()[0]; auto root = meta_.entry_table_v2_2()[0];
walk(root, root, seen, std::forward<T>(func)); walk(root, root, seen, std::forward<T>(func));
} }
@ -388,15 +389,16 @@ class metadata_ final : public metadata_v2::impl {
std::optional<entry_view> get_entry(int inode) const { std::optional<entry_view> get_entry(int inode) const {
inode -= inode_offset_; inode -= inode_offset_;
std::optional<entry_view> rv; std::optional<entry_view> rv;
if (inode >= 0 && inode < static_cast<int>(meta_.entry_index().size())) { if (inode >= 0 &&
inode < static_cast<int>(meta_.entry_table_v2_2().size())) {
rv = make_entry_view_from_inode(inode); rv = make_entry_view_from_inode(inode);
} }
return rv; return rv;
} }
std::string_view link_value(entry_view entry) const { std::string_view link_value(entry_view entry) const {
return meta_ return meta_.symlinks()[meta_.symlink_table()[entry.inode() -
.symlinks()[meta_.symlink_table()[entry.inode() - symlink_table_offset_]]; symlink_table_offset_]];
} }
uint64_t get_device_id(int inode) const { uint64_t get_device_id(int inode) const {
@ -456,8 +458,8 @@ void metadata_<LoggerPolicy>::dump(
} }
if (S_ISREG(mode)) { if (S_ISREG(mode)) {
uint32_t beg = meta_.chunk_index()[inode - file_index_offset_]; uint32_t beg = meta_.chunk_table()[inode - file_index_offset_];
uint32_t end = meta_.chunk_index()[inode - file_index_offset_ + 1]; uint32_t end = meta_.chunk_table()[inode - file_index_offset_ + 1];
os << " [" << beg << ", " << end << "]"; os << " [" << beg << ", " << end << "]";
os << " " << file_size(entry, mode) << "\n"; os << " " << file_size(entry, mode) << "\n";
if (detail_level > 3) { if (detail_level > 3) {
@ -511,15 +513,16 @@ void metadata_<LoggerPolicy>::dump(
os << "chunks: " << meta_.chunks().size() << std::endl; os << "chunks: " << meta_.chunks().size() << std::endl;
os << "directories: " << meta_.directories().size() << std::endl; os << "directories: " << meta_.directories().size() << std::endl;
os << "entries: " << meta_.entries().size() << std::endl; os << "entries: " << meta_.entries().size() << std::endl;
os << "chunk_index: " << meta_.chunk_index().size() << std::endl; os << "chunk_table: " << meta_.chunk_table().size() << std::endl;
os << "entry_index: " << meta_.entry_index().size() << std::endl; os << "entry_table_v2_2: " << meta_.entry_table_v2_2().size() << std::endl;
os << "symlink_table: " << meta_.symlink_table().size() << std::endl; os << "symlink_table: " << meta_.symlink_table().size() << std::endl;
os << "uids: " << meta_.uids().size() << std::endl; os << "uids: " << meta_.uids().size() << std::endl;
os << "gids: " << meta_.gids().size() << std::endl; os << "gids: " << meta_.gids().size() << std::endl;
os << "modes: " << meta_.modes().size() << std::endl; os << "modes: " << meta_.modes().size() << std::endl;
os << "names: " << meta_.names().size() << std::endl; os << "names: " << meta_.names().size() << std::endl;
os << "symlinks: " << meta_.symlinks().size() << std::endl; os << "symlinks: " << meta_.symlinks().size() << std::endl;
os << "hardlinks: " << std::accumulate(nlinks_.begin(), nlinks_.end(), 0) << std::endl; os << "hardlinks: " << std::accumulate(nlinks_.begin(), nlinks_.end(), 0)
<< std::endl;
os << "devices: " << meta_.devices()->size() << std::endl; os << "devices: " << meta_.devices()->size() << std::endl;
os << "symlink_table_offset: " << symlink_table_offset_ << std::endl; os << "symlink_table_offset: " << symlink_table_offset_ << std::endl;
os << "file_index_offset: " << file_index_offset_ << std::endl; os << "file_index_offset: " << file_index_offset_ << std::endl;
@ -790,8 +793,7 @@ int metadata_<LoggerPolicy>::getattr(entry_view entry,
: resolution * (timebase + entry.atime_offset()); : resolution * (timebase + entry.atime_offset());
stbuf->st_ctime = mtime_only ? stbuf->st_mtime stbuf->st_ctime = mtime_only ? stbuf->st_mtime
: resolution * (timebase + entry.ctime_offset()); : resolution * (timebase + entry.ctime_offset());
stbuf->st_nlink = stbuf->st_nlink = options_.enable_nlink && S_ISREG(mode)
options_.enable_nlink && S_ISREG(mode)
? DWARFS_NOTHROW(nlinks_.at(inode - file_index_offset_)) ? DWARFS_NOTHROW(nlinks_.at(inode - file_index_offset_))
: 1; : 1;
@ -910,7 +912,7 @@ int metadata_<LoggerPolicy>::statvfs(struct ::statvfs* stbuf) const {
stbuf->f_bsize = meta_.block_size(); stbuf->f_bsize = meta_.block_size();
stbuf->f_frsize = 1UL; stbuf->f_frsize = 1UL;
stbuf->f_blocks = meta_.total_fs_size(); stbuf->f_blocks = meta_.total_fs_size();
stbuf->f_files = meta_.entry_index().size(); stbuf->f_files = meta_.entry_table_v2_2().size();
stbuf->f_flag = ST_RDONLY; stbuf->f_flag = ST_RDONLY;
stbuf->f_namemax = PATH_MAX; stbuf->f_namemax = PATH_MAX;
@ -923,9 +925,9 @@ metadata_<LoggerPolicy>::get_chunks(int inode) const {
std::optional<chunk_range> rv; std::optional<chunk_range> rv;
inode -= inode_offset_ + file_index_offset_; inode -= inode_offset_ + file_index_offset_;
if (inode >= 0 && if (inode >= 0 &&
inode < (static_cast<int>(meta_.chunk_index().size()) - 1)) { inode < (static_cast<int>(meta_.chunk_table().size()) - 1)) {
uint32_t begin = meta_.chunk_index()[inode]; uint32_t begin = meta_.chunk_table()[inode];
uint32_t end = meta_.chunk_index()[inode + 1]; uint32_t end = meta_.chunk_table()[inode + 1];
rv = chunk_range(&meta_, begin, end); rv = chunk_range(&meta_, begin, end);
} }
return rv; return rv;

View File

@ -532,7 +532,8 @@ void scanner_<LoggerPolicy>::scan(filesystem_writer& fsw,
root->walk([&](entry* ep) { root->walk([&](entry* ep) {
ep->update(ge_data); ep->update(ge_data);
if (auto lp = dynamic_cast<link*>(ep)) { if (auto lp = dynamic_cast<link*>(ep)) {
DWARFS_NOTHROW(mv2.symlink_table.at(ep->inode_num() - first_link_inode)) = DWARFS_NOTHROW(
mv2.symlink_table.at(ep->inode_num() - first_link_inode)) =
ge_data.get_symlink_table_entry(lp->linkname()); ge_data.get_symlink_table_entry(lp->linkname());
} }
}); });
@ -574,24 +575,24 @@ void scanner_<LoggerPolicy>::scan(filesystem_writer& fsw,
root->set_name(std::string()); root->set_name(std::string());
LOG_INFO << "saving chunks..."; LOG_INFO << "saving chunks...";
mv2.chunk_index.resize(im.count() + 1); mv2.chunk_table.resize(im.count() + 1);
// TODO: we should be able to start this once all blocks have been // TODO: we should be able to start this once all blocks have been
// submitted for compression // submitted for compression
im.for_each_inode([&](std::shared_ptr<inode> const& ino) { im.for_each_inode([&](std::shared_ptr<inode> const& ino) {
DWARFS_NOTHROW(mv2.chunk_index.at(ino->num() - first_file_inode)) = DWARFS_NOTHROW(mv2.chunk_table.at(ino->num() - first_file_inode)) =
mv2.chunks.size(); mv2.chunks.size();
ino->append_chunks_to(mv2.chunks); ino->append_chunks_to(mv2.chunks);
}); });
// insert dummy inode to help determine number of chunks per inode // insert dummy inode to help determine number of chunks per inode
DWARFS_NOTHROW(mv2.chunk_index.at(im.count())) = mv2.chunks.size(); DWARFS_NOTHROW(mv2.chunk_table.at(im.count())) = mv2.chunks.size();
LOG_DEBUG << "total number of file inodes: " << im.count(); LOG_DEBUG << "total number of file inodes: " << im.count();
LOG_DEBUG << "total number of chunks: " << mv2.chunks.size(); LOG_DEBUG << "total number of chunks: " << mv2.chunks.size();
LOG_INFO << "saving directories..."; LOG_INFO << "saving directories...";
mv2.entry_index.resize(first_pipe_inode); mv2.entry_table_v2_2.resize(first_pipe_inode);
mv2.directories.reserve(first_link_inode + 1); mv2.directories.reserve(first_link_inode + 1);
save_directories_visitor sdv(first_link_inode); save_directories_visitor sdv(first_link_inode);
root->accept(sdv); root->accept(sdv);

View File

@ -66,7 +66,7 @@ struct entry {
* - For links, (inode - link_index_offset) can be * - For links, (inode - link_index_offset) can be
* used as an index into metadata.links. * used as an index into metadata.links.
* - For files, (inode - chunk_index_offset) can be * - For files, (inode - chunk_index_offset) can be
* used as in index into metadata.chunk_index. * used as in index into metadata.chunk_table.
*/ */
3: required UInt32 inode, 3: required UInt32 inode,
@ -101,7 +101,7 @@ struct metadata {
* files share the same inode number. The range of chunks * files share the same inode number. The range of chunks
* for a regular file inode are: * for a regular file inode are:
* *
* chunks[chunk_index[inode]] .. chunks[chunk_index[inode + 1] - 1] * chunks[chunk_table[inode]] .. chunks[chunk_table[inode + 1] - 1]
*/ */
1: required list<chunk> chunks, 1: required list<chunk> chunks,
@ -114,18 +114,18 @@ struct metadata {
2: required list<directory> directories, 2: required list<directory> directories,
/** /**
* All entries, can be looked up by inode through entry_index, or by * All entries, can be looked up by inode through entry_table_v2_2, or by
* directory through `first_entry`, where the entries will be between * directory through `first_entry`, where the entries will be between
* `directories[n].first_entry` and `directories[n+1].first_entry`. * `directories[n].first_entry` and `directories[n+1].first_entry`.
*/ */
3: required list<entry> entries, 3: required list<entry> entries,
/** /**
* Chunk index, indexed by (inode - chunk_index_offset). * Chunk lookup table, indexed by (inode - chunk_index_offset).
* There's one extra dummy item at the end that points to the * There's one extra dummy item at the end that points to the
* end of `chunks`, so chunk lookups work the same for all inodes. * end of `chunks`, so chunk lookups work the same for all inodes.
*/ */
4: required list<UInt32> chunk_index, 4: required list<UInt32> chunk_table,
/** /**
* Entry index, indexed by inode * Entry index, indexed by inode
@ -138,10 +138,9 @@ struct metadata {
* - character and block devices * - character and block devices
* - named pipes and sockets * - named pipes and sockets
*/ */
5: required list<UInt32> entry_index, ///// <------------ deprecate (see above) 5: required list<UInt32> entry_table_v2_2,
5: required list<UInt32> entry_index,
// symlink lookup table, indexed by (inode - link_index_offset) // symlink lookup table, indexed by (inode - symlink_table_offset)
6: required list<UInt32> symlink_table, 6: required list<UInt32> symlink_table,
// user ids, for lookup by index in entry.owner // user ids, for lookup by index in entry.owner