TheCloud/dwarfs
1
0
Fork 0
mirror of https://github.com/mhx/dwarfs.git synced 2025-09-08 11:59:48 -04:00
Code Issues Packages Projects Releases Wiki Activity

More naming cleanups

Browse Source
This commit is contained in:
Marcus Holland-Moritz 2021-03-14 20:47:59 +01:00
parent 93e7ba7e09
commit 3608072836
6 changed files with 47 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/dwarfs/entry.cpp
View File

@ -254,7 +254,7 @@ void dir::scan(os_access&, progress&) {}
void dir::pack_entry(thrift::metadata::metadata& mv2,
global_entry_data const& data) const {
DWARFS_NOTHROW(mv2.entry_index.at(inode_num())) = mv2.entries.size();
DWARFS_NOTHROW(mv2.entry_table_v2_2.at(inode_num())) = mv2.entries.size();
mv2.entries.emplace_back();
entry::pack(mv2.entries.back(), data);
}
@ -268,7 +268,8 @@ void dir::pack(thrift::metadata::metadata& mv2,
// d.entry_count = entries_.size();
mv2.directories.push_back(d);
for (entry_ptr const& e : entries_) {
DWARFS_NOTHROW(mv2.entry_index.at(e->inode_num())) = mv2.entries.size();
DWARFS_NOTHROW(mv2.entry_table_v2_2.at(e->inode_num())) =
mv2.entries.size();
mv2.entries.emplace_back();
e->pack(mv2.entries.back(), data);
}

3
src/dwarfs/global_entry_data.cpp
View File

@ -100,7 +100,8 @@ uint32_t global_entry_data::get_name_index(std::string const& name) const {
return DWARFS_NOTHROW(names_.at(name));
}
uint32_t global_entry_data::get_symlink_table_entry(std::string const& link) const {
uint32_t
global_entry_data::get_symlink_table_entry(std::string const& link) const {
return DWARFS_NOTHROW(symlinks_.at(link));
}

2
src/dwarfs/metadata_types.cpp
View File

@ -66,7 +66,7 @@ uint32_t directory_view::parent_inode() const {
}
directory_view::directory_view(uint32_t inode, Meta const* meta)
: entry_(meta->entries()[meta->entry_index()[inode]])
: entry_(meta->entries()[meta->entry_table_v2_2()[inode]])
, meta_(meta) {}
std::string directory_view::path() const {

54
src/dwarfs/metadata_v2.cpp
View File

@ -113,7 +113,7 @@ class metadata_ final : public metadata_v2::impl {
int inode_offset)
: data_(data)
, meta_(map_frozen<thrift::metadata::metadata>(schema, data_))
, root_(meta_.entries()[meta_.entry_index()[0]], &meta_)
, root_(meta_.entries()[meta_.entry_table_v2_2()[0]], &meta_)
, log_(lgr)
, inode_offset_(inode_offset)
, symlink_table_offset_(find_index_offset(inode_rank::INO_LNK))
@ -141,17 +141,18 @@ class metadata_ final : public metadata_v2::impl {
"metadata inconsistency: number of symlinks ({}) does not match "
"chunk/symlink table delta ({} - {} = {})",
meta_.symlink_table().size(), file_index_offset_,
symlink_table_offset_, file_index_offset_ - symlink_table_offset_));
symlink_table_offset_,
file_index_offset_ - symlink_table_offset_));
}
if (int(meta_.chunk_index().size() - 1) !=
if (int(meta_.chunk_table().size() - 1) !=
(dev_index_offset_ - file_index_offset_)) {
DWARFS_THROW(
runtime_error,
fmt::format(
"metadata inconsistency: number of files ({}) does not match "
"device/chunk index delta ({} - {} = {})",
meta_.chunk_index().size() - 1, dev_index_offset_,
meta_.chunk_table().size() - 1, dev_index_offset_,
file_index_offset_, dev_index_offset_ - file_index_offset_));
}
@ -239,10 +240,10 @@ class metadata_ final : public metadata_v2::impl {
}
entry_view make_entry_view_from_inode(uint32_t inode) const {
return make_entry_view(meta_.entry_index()[inode]);
return make_entry_view(meta_.entry_table_v2_2()[inode]);
}
// This represents the order in which inodes are stored in entry_index
// This represents the order in which inodes are stored in entry_table_v2_2
enum class inode_rank {
INO_DIR,
INO_LNK,
@ -294,7 +295,7 @@ class metadata_ final : public metadata_v2::impl {
}
size_t find_index_offset(inode_rank rank) const {
auto range = boost::irange(size_t(0), meta_.entry_index().size());
auto range = boost::irange(size_t(0), meta_.entry_table_v2_2().size());
auto it = std::lower_bound(range.begin(), range.end(), rank,
[&](auto inode, inode_rank r) {
@ -326,8 +327,8 @@ class metadata_ final : public metadata_v2::impl {
size_t reg_file_size(entry_view entry) const {
auto inode = entry.inode() - file_index_offset_;
uint32_t cur = meta_.chunk_index()[inode];
uint32_t end = meta_.chunk_index()[inode + 1];
uint32_t cur = meta_.chunk_table()[inode];
uint32_t end = meta_.chunk_table()[inode + 1];
if (cur > end) {
DWARFS_THROW(runtime_error,
fmt::format("invalid chunk range: [{0}..{1}]", cur, end));
@ -371,7 +372,7 @@ class metadata_ final : public metadata_v2::impl {
template <typename T>
void walk_tree(T&& func) const {
set_type<int> seen;
auto root = meta_.entry_index()[0];
auto root = meta_.entry_table_v2_2()[0];
walk(root, root, seen, std::forward<T>(func));
}
@ -388,15 +389,16 @@ class metadata_ final : public metadata_v2::impl {
std::optional<entry_view> get_entry(int inode) const {
inode -= inode_offset_;
std::optional<entry_view> rv;
if (inode >= 0 && inode < static_cast<int>(meta_.entry_index().size())) {
if (inode >= 0 &&
inode < static_cast<int>(meta_.entry_table_v2_2().size())) {
rv = make_entry_view_from_inode(inode);
}
return rv;
}
std::string_view link_value(entry_view entry) const {
return meta_
.symlinks()[meta_.symlink_table()[entry.inode() - symlink_table_offset_]];
return meta_.symlinks()[meta_.symlink_table()[entry.inode() -
symlink_table_offset_]];
}
uint64_t get_device_id(int inode) const {
@ -456,8 +458,8 @@ void metadata_<LoggerPolicy>::dump(
}
if (S_ISREG(mode)) {
uint32_t beg = meta_.chunk_index()[inode - file_index_offset_];
uint32_t end = meta_.chunk_index()[inode - file_index_offset_ + 1];
uint32_t beg = meta_.chunk_table()[inode - file_index_offset_];
uint32_t end = meta_.chunk_table()[inode - file_index_offset_ + 1];
os << " [" << beg << ", " << end << "]";
os << " " << file_size(entry, mode) << "\n";
if (detail_level > 3) {
@ -511,15 +513,16 @@ void metadata_<LoggerPolicy>::dump(
os << "chunks: " << meta_.chunks().size() << std::endl;
os << "directories: " << meta_.directories().size() << std::endl;
os << "entries: " << meta_.entries().size() << std::endl;
os << "chunk_index: " << meta_.chunk_index().size() << std::endl;
os << "entry_index: " << meta_.entry_index().size() << std::endl;
os << "chunk_table: " << meta_.chunk_table().size() << std::endl;
os << "entry_table_v2_2: " << meta_.entry_table_v2_2().size() << std::endl;
os << "symlink_table: " << meta_.symlink_table().size() << std::endl;
os << "uids: " << meta_.uids().size() << std::endl;
os << "gids: " << meta_.gids().size() << std::endl;
os << "modes: " << meta_.modes().size() << std::endl;
os << "names: " << meta_.names().size() << std::endl;
os << "symlinks: " << meta_.symlinks().size() << std::endl;
os << "hardlinks: " << std::accumulate(nlinks_.begin(), nlinks_.end(), 0) << std::endl;
os << "hardlinks: " << std::accumulate(nlinks_.begin(), nlinks_.end(), 0)
<< std::endl;
os << "devices: " << meta_.devices()->size() << std::endl;
os << "symlink_table_offset: " << symlink_table_offset_ << std::endl;
os << "file_index_offset: " << file_index_offset_ << std::endl;
@ -790,10 +793,9 @@ int metadata_<LoggerPolicy>::getattr(entry_view entry,
: resolution * (timebase + entry.atime_offset());
stbuf->st_ctime = mtime_only ? stbuf->st_mtime
: resolution * (timebase + entry.ctime_offset());
stbuf->st_nlink =
options_.enable_nlink && S_ISREG(mode)
? DWARFS_NOTHROW(nlinks_.at(inode - file_index_offset_))
: 1;
stbuf->st_nlink = options_.enable_nlink && S_ISREG(mode)
? DWARFS_NOTHROW(nlinks_.at(inode - file_index_offset_))
: 1;
if (S_ISBLK(mode) || S_ISCHR(mode)) {
stbuf->st_rdev = get_device_id(inode);
@ -910,7 +912,7 @@ int metadata_<LoggerPolicy>::statvfs(struct ::statvfs* stbuf) const {
stbuf->f_bsize = meta_.block_size();
stbuf->f_frsize = 1UL;
stbuf->f_blocks = meta_.total_fs_size();
stbuf->f_files = meta_.entry_index().size();
stbuf->f_files = meta_.entry_table_v2_2().size();
stbuf->f_flag = ST_RDONLY;
stbuf->f_namemax = PATH_MAX;
@ -923,9 +925,9 @@ metadata_<LoggerPolicy>::get_chunks(int inode) const {
std::optional<chunk_range> rv;
inode -= inode_offset_ + file_index_offset_;
if (inode >= 0 &&
inode < (static_cast<int>(meta_.chunk_index().size()) - 1)) {
uint32_t begin = meta_.chunk_index()[inode];
uint32_t end = meta_.chunk_index()[inode + 1];
inode < (static_cast<int>(meta_.chunk_table().size()) - 1)) {
uint32_t begin = meta_.chunk_table()[inode];
uint32_t end = meta_.chunk_table()[inode + 1];
rv = chunk_range(&meta_, begin, end);
}
return rv;

11
src/dwarfs/scanner.cpp
View File

@ -532,7 +532,8 @@ void scanner_<LoggerPolicy>::scan(filesystem_writer& fsw,
root->walk([&](entry* ep) {
ep->update(ge_data);
if (auto lp = dynamic_cast<link*>(ep)) {
DWARFS_NOTHROW(mv2.symlink_table.at(ep->inode_num() - first_link_inode)) =
DWARFS_NOTHROW(
mv2.symlink_table.at(ep->inode_num() - first_link_inode)) =
ge_data.get_symlink_table_entry(lp->linkname());
}
});
@ -574,24 +575,24 @@ void scanner_<LoggerPolicy>::scan(filesystem_writer& fsw,
root->set_name(std::string());
LOG_INFO << "saving chunks...";
mv2.chunk_index.resize(im.count() + 1);
mv2.chunk_table.resize(im.count() + 1);
// TODO: we should be able to start this once all blocks have been
// submitted for compression
im.for_each_inode([&](std::shared_ptr<inode> const& ino) {
DWARFS_NOTHROW(mv2.chunk_index.at(ino->num() - first_file_inode)) =
DWARFS_NOTHROW(mv2.chunk_table.at(ino->num() - first_file_inode)) =
mv2.chunks.size();
ino->append_chunks_to(mv2.chunks);
});
// insert dummy inode to help determine number of chunks per inode
DWARFS_NOTHROW(mv2.chunk_index.at(im.count())) = mv2.chunks.size();
DWARFS_NOTHROW(mv2.chunk_table.at(im.count())) = mv2.chunks.size();
LOG_DEBUG << "total number of file inodes: " << im.count();
LOG_DEBUG << "total number of chunks: " << mv2.chunks.size();
LOG_INFO << "saving directories...";
mv2.entry_index.resize(first_pipe_inode);
mv2.entry_table_v2_2.resize(first_pipe_inode);
mv2.directories.reserve(first_link_inode + 1);
save_directories_visitor sdv(first_link_inode);
root->accept(sdv);

15
thrift/metadata.thrift
View File

@ -66,7 +66,7 @@ struct entry {
* - For links, (inode - link_index_offset) can be
* used as an index into metadata.links.
* - For files, (inode - chunk_index_offset) can be
* used as in index into metadata.chunk_index.
* used as in index into metadata.chunk_table.
*/
3: required UInt32 inode,
@ -101,7 +101,7 @@ struct metadata {
* files share the same inode number. The range of chunks
* for a regular file inode are:
*
* chunks[chunk_index[inode]] .. chunks[chunk_index[inode + 1] - 1]
* chunks[chunk_table[inode]] .. chunks[chunk_table[inode + 1] - 1]
*/
1: required list<chunk> chunks,
@ -114,18 +114,18 @@ struct metadata {
2: required list<directory> directories,
/**
* All entries, can be looked up by inode through entry_index, or by
* All entries, can be looked up by inode through entry_table_v2_2, or by
* directory through `first_entry`, where the entries will be between
* `directories[n].first_entry` and `directories[n+1].first_entry`.
*/
3: required list<entry> entries,
/**
* Chunk index, indexed by (inode - chunk_index_offset).
* Chunk lookup table, indexed by (inode - chunk_index_offset).
* There's one extra dummy item at the end that points to the
* end of `chunks`, so chunk lookups work the same for all inodes.
*/
4: required list<UInt32> chunk_index,
4: required list<UInt32> chunk_table,
/**
* Entry index, indexed by inode
@ -138,10 +138,9 @@ struct metadata {
* - character and block devices
* - named pipes and sockets
*/
5: required list<UInt32> entry_index, ///// <------------ deprecate (see above)
5: required list<UInt32> entry_index,
5: required list<UInt32> entry_table_v2_2,
// symlink lookup table, indexed by (inode - link_index_offset)
// symlink lookup table, indexed by (inode - symlink_table_offset)
6: required list<UInt32> symlink_table,
// user ids, for lookup by index in entry.owner