TheCloud/dwarfs
1
0
Fork 0
mirror of https://github.com/mhx/dwarfs.git synced 2025-09-08 03:49:44 -04:00
Code Issues Packages Projects Releases Wiki Activity

Cleanups & notes

Browse Source
This commit is contained in:
Marcus Holland-Moritz 2020-11-28 13:18:01 +01:00
parent fc4f1546b2
commit 527e08a0da
8 changed files with 59 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/dwarfs/filesystem_v2.h
View File

@ -56,7 +56,9 @@ class filesystem_v2 {
static void identify(logger& lgr, std::shared_ptr<mmif> mm, std::ostream& os);
void dump(std::ostream& os) const { impl_->dump(os); }
void dump(std::ostream& os, int detail_level) const {
impl_->dump(os, detail_level);
}
void walk(std::function<void(entry_view)> const& func) const {
impl_->walk(func);
@ -116,7 +118,7 @@ class filesystem_v2 {
public:
virtual ~impl() = default;
virtual void dump(std::ostream& os) const = 0;
virtual void dump(std::ostream& os, int detail_level) const = 0;
virtual void walk(std::function<void(entry_view)> const& func) const = 0;
virtual std::optional<entry_view> find(const char* path) const = 0;
virtual std::optional<entry_view> find(int inode) const = 0;

6
include/dwarfs/metadata_v2.h
View File

@ -53,9 +53,9 @@ class metadata_v2 {
metadata_v2& operator=(metadata_v2&&) = default;
void
dump(std::ostream& os,
dump(std::ostream& os, int detail_level,
std::function<void(const std::string&, uint32_t)> const& icb) const {
impl_->dump(os, icb);
impl_->dump(os, detail_level, icb);
}
static void get_stat_defaults(struct ::stat* defaults);
@ -125,7 +125,7 @@ class metadata_v2 {
virtual ~impl() = default;
virtual void dump(
std::ostream& os,
std::ostream& os, int detail_level,
std::function<void(const std::string&, uint32_t)> const& icb) const = 0;
virtual size_t size() const = 0;

7
src/dwarfs/block_cache.cpp
View File

@ -180,8 +180,13 @@ class block_cache_ : public block_cache::impl {
, options_(options) {}
~block_cache_() noexcept override {
log_.info() << "stopping cache workers";
log_.debug() << "stopping cache workers";
wg_.stop();
if (!blocks_created_.load()) {
return;
}
log_.debug() << "cached blocks:";
for (const auto& cb : cache_) {

13
src/dwarfs/filesystem_v2.cpp
View File

@ -157,7 +157,7 @@ class filesystem_ : public filesystem_v2::impl {
const block_cache_options& bc_options,
const struct ::stat* stat_defaults, int inode_offset);
void dump(std::ostream& os) const override;
void dump(std::ostream& os, int detail_level) const override;
void walk(std::function<void(entry_view)> const& func) const override;
std::optional<entry_view> find(const char* path) const override;
std::optional<entry_view> find(int inode) const override;
@ -224,8 +224,8 @@ filesystem_<LoggerPolicy>::filesystem_(logger& lgr, std::shared_ptr<mmif> mm,
}
template <typename LoggerPolicy>
void filesystem_<LoggerPolicy>::dump(std::ostream& os) const {
meta_.dump(os, [&](const std::string& indent, uint32_t inode) {
void filesystem_<LoggerPolicy>::dump(std::ostream& os, int detail_level) const {
meta_.dump(os, detail_level, [&](const std::string& indent, uint32_t inode) {
if (auto chunks = meta_.get_chunks(inode)) {
os << indent << chunks->size() << " chunks in inode " << inode << "\n";
ir_.dump(os, indent + " ", *chunks);
@ -417,12 +417,7 @@ void filesystem_v2::identify(logger& lgr, std::shared_ptr<mmif> mm,
auto meta = make_metadata(lgr, mm, sections, schema_raw, meta_raw);
struct ::statvfs stbuf;
meta.statvfs(&stbuf);
os << "block size: " << stbuf.f_bsize << std::endl;
os << "inode count: " << stbuf.f_files << std::endl;
os << "original filesystem size: " << stbuf.f_blocks << std::endl;
meta.dump(os, 0, [](const std::string&, uint32_t) {});
}
} // namespace dwarfs

14
src/dwarfs/inode_reader_v2.cpp
View File

@ -43,12 +43,14 @@ class inode_reader_ : public inode_reader_v2::impl {
~inode_reader_() {
std::lock_guard<std::mutex> lock(iovec_sizes_mutex_);
log_.info() << "iovec size p90: "
<< iovec_sizes_.getPercentileEstimate(0.9);
log_.info() << "iovec size p95: "
<< iovec_sizes_.getPercentileEstimate(0.95);
log_.info() << "iovec size p99: "
<< iovec_sizes_.getPercentileEstimate(0.99);
if (iovec_sizes_.computeTotalCount() > 0) {
log_.info() << "iovec size p90: "
<< iovec_sizes_.getPercentileEstimate(0.9);
log_.info() << "iovec size p95: "
<< iovec_sizes_.getPercentileEstimate(0.95);
log_.info() << "iovec size p99: "
<< iovec_sizes_.getPercentileEstimate(0.99);
}
}
ssize_t

49
src/dwarfs/metadata_v2.cpp
View File

@ -36,6 +36,8 @@ namespace dwarfs {
namespace {
using ::apache::thrift::frozen::MappedFrozen;
template <class T>
std::pair<std::vector<uint8_t>, std::vector<uint8_t>>
freeze_to_buffer(const T& x) {
@ -62,9 +64,8 @@ freeze_to_buffer(const T& x) {
return {schema_buffer, data_buffer};
}
template <class T>
::apache::thrift::frozen::MappedFrozen<T>
map_frozen(folly::ByteRange schema, folly::ByteRange data) {
template <typename T>
MappedFrozen<T> map_frozen(folly::ByteRange schema, folly::ByteRange data) {
using namespace ::apache::thrift::frozen;
auto layout = std::make_unique<Layout<T>>();
deserializeRootLayout(schema, *layout);
@ -73,11 +74,20 @@ map_frozen(folly::ByteRange schema, folly::ByteRange data) {
return ret;
}
void analyze_frozen(std::ostream& os,
MappedFrozen<thrift::metadata::metadata> const& meta) {
using namespace ::apache::thrift::frozen;
auto layout = meta.findFirstOfType<
std::unique_ptr<Layout<thrift::metadata::metadata>>>();
(*layout)->print(os, 0);
}
const uint16_t READ_ONLY_MASK = ~(S_IWUSR | S_IWGRP | S_IWOTH);
template <typename LoggerPolicy>
class metadata_ : public metadata_v2::impl {
public:
// TODO: defaults?, remove
metadata_(logger& lgr, folly::ByteRange schema, folly::ByteRange data,
const struct ::stat* /*defaults*/, int inode_offset)
: data_(data)
@ -85,19 +95,9 @@ class metadata_ : public metadata_v2::impl {
, root_(meta_.entries()[meta_.entry_index()[0]], &meta_)
, inode_offset_(inode_offset)
, chunk_index_offset_(meta_.chunk_index_offset())
, log_(lgr) {
// TODO: defaults?, remove
, log_(lgr) {}
// TODO: move to separate function (e.g. dump?)
// cannot be done easily with schema, though, as it's temporary
// log_.debug() << ::apache::thrift::debugString(meta_.thaw());
// ::apache::thrift::frozen::Layout<thrift::metadata::metadata> layout;
// ::apache::thrift::frozen::schema::Schema mem_schema;
// ::apache::thrift::CompactSerializer::deserialize(schema, mem_schema);
// log_.debug() << ::apache::thrift::debugString(mem_schema);
}
void dump(std::ostream& os,
void dump(std::ostream& os, int detail_level,
std::function<void(const std::string&, uint32_t)> const& icb)
const override;
@ -203,7 +203,7 @@ class metadata_ : public metadata_v2::impl {
}
folly::ByteRange data_;
::apache::thrift::frozen::MappedFrozen<thrift::metadata::metadata> meta_;
MappedFrozen<thrift::metadata::metadata> meta_;
entry_view root_;
const int inode_offset_;
const int chunk_index_offset_;
@ -255,9 +255,22 @@ void metadata_<LoggerPolicy>::dump(
template <typename LoggerPolicy>
void metadata_<LoggerPolicy>::dump(
std::ostream& os,
std::ostream& os, int detail_level,
std::function<void(const std::string&, uint32_t)> const& icb) const {
dump(os, "", root_, icb);
struct ::statvfs stbuf;
statvfs(&stbuf);
os << "block size: " << stbuf.f_bsize << std::endl;
os << "inode count: " << stbuf.f_files << std::endl;
os << "original filesystem size: " << stbuf.f_blocks << std::endl;
if (detail_level > 0) {
analyze_frozen(os, meta_);
}
if (detail_level > 1) {
dump(os, "", root_, icb);
}
}
template <typename LoggerPolicy>

4
src/dwarfsck.cpp
View File

@ -45,8 +45,8 @@ int main(int argc, char** argv) {
}
} else {
// TODO: add more usage options...
dwarfs::filesystem_v2::identify(lgr, mm, std::cout);
fs.dump(std::cout);
// dwarfs::filesystem_v2::identify(lgr, mm, std::cout);
fs.dump(std::cout, 1);
}
} catch (const std::exception& e) {
std::cerr << "Error: " << e.what() << std::endl;

2
thrift/metadata.thrift
View File

@ -44,7 +44,7 @@ struct chunk {
struct directory {
1: required UInt32 parent_inode,
2: required UInt32 first_entry,
3: required UInt32 entry_count,
3: required UInt32 entry_count, // TODO: we can remove this if we sort entries by directory
}
/**