Cleanups & notes

include/dwarfs/filesystem_v2.h

@@ -56,7 +56,9 @@ class filesystem_v2 {
 
   static void identify(logger& lgr, std::shared_ptr<mmif> mm, std::ostream& os);
 
-  void dump(std::ostream& os) const { impl_->dump(os); }
+  void dump(std::ostream& os, int detail_level) const {
+    impl_->dump(os, detail_level);
+  }
 
   void walk(std::function<void(entry_view)> const& func) const {
     impl_->walk(func);
@@ -116,7 +118,7 @@ class filesystem_v2 {
    public:
     virtual ~impl() = default;
 
-    virtual void dump(std::ostream& os) const = 0;
+    virtual void dump(std::ostream& os, int detail_level) const = 0;
     virtual void walk(std::function<void(entry_view)> const& func) const = 0;
     virtual std::optional<entry_view> find(const char* path) const = 0;
     virtual std::optional<entry_view> find(int inode) const = 0;

include/dwarfs/metadata_v2.h

@@ -53,9 +53,9 @@ class metadata_v2 {
   metadata_v2& operator=(metadata_v2&&) = default;
 
   void
-  dump(std::ostream& os,
+  dump(std::ostream& os, int detail_level,
        std::function<void(const std::string&, uint32_t)> const& icb) const {
-    impl_->dump(os, icb);
+    impl_->dump(os, detail_level, icb);
   }
 
   static void get_stat_defaults(struct ::stat* defaults);
@@ -125,7 +125,7 @@ class metadata_v2 {
     virtual ~impl() = default;
 
     virtual void dump(
-        std::ostream& os,
+        std::ostream& os, int detail_level,
         std::function<void(const std::string&, uint32_t)> const& icb) const = 0;
 
     virtual size_t size() const = 0;

src/dwarfs/block_cache.cpp

@@ -180,8 +180,13 @@ class block_cache_ : public block_cache::impl {
       , options_(options) {}
 
   ~block_cache_() noexcept override {
-    log_.info() << "stopping cache workers";
+    log_.debug() << "stopping cache workers";
     wg_.stop();
+
+    if (!blocks_created_.load()) {
+      return;
+    }
+
     log_.debug() << "cached blocks:";
 
     for (const auto& cb : cache_) {

src/dwarfs/filesystem_v2.cpp

@@ -157,7 +157,7 @@ class filesystem_ : public filesystem_v2::impl {
               const block_cache_options& bc_options,
               const struct ::stat* stat_defaults, int inode_offset);
 
-  void dump(std::ostream& os) const override;
+  void dump(std::ostream& os, int detail_level) const override;
   void walk(std::function<void(entry_view)> const& func) const override;
   std::optional<entry_view> find(const char* path) const override;
   std::optional<entry_view> find(int inode) const override;
@@ -224,8 +224,8 @@ filesystem_<LoggerPolicy>::filesystem_(logger& lgr, std::shared_ptr<mmif> mm,
 }
 
 template <typename LoggerPolicy>
-void filesystem_<LoggerPolicy>::dump(std::ostream& os) const {
+void filesystem_<LoggerPolicy>::dump(std::ostream& os, int detail_level) const {
-  meta_.dump(os, [&](const std::string& indent, uint32_t inode) {
+  meta_.dump(os, detail_level, [&](const std::string& indent, uint32_t inode) {
     if (auto chunks = meta_.get_chunks(inode)) {
       os << indent << chunks->size() << " chunks in inode " << inode << "\n";
       ir_.dump(os, indent + "  ", *chunks);
@@ -417,12 +417,7 @@ void filesystem_v2::identify(logger& lgr, std::shared_ptr<mmif> mm,
 
   auto meta = make_metadata(lgr, mm, sections, schema_raw, meta_raw);
 
-  struct ::statvfs stbuf;
+  meta.dump(os, 0, [](const std::string&, uint32_t) {});
-  meta.statvfs(&stbuf);
-
-  os << "block size: " << stbuf.f_bsize << std::endl;
-  os << "inode count: " << stbuf.f_files << std::endl;
-  os << "original filesystem size: " << stbuf.f_blocks << std::endl;
 }
 
 } // namespace dwarfs

src/dwarfs/inode_reader_v2.cpp

@@ -43,12 +43,14 @@ class inode_reader_ : public inode_reader_v2::impl {
 
   ~inode_reader_() {
     std::lock_guard<std::mutex> lock(iovec_sizes_mutex_);
-    log_.info() << "iovec size p90: "
+    if (iovec_sizes_.computeTotalCount() > 0) {
-                << iovec_sizes_.getPercentileEstimate(0.9);
+      log_.info() << "iovec size p90: "
-    log_.info() << "iovec size p95: "
+                  << iovec_sizes_.getPercentileEstimate(0.9);
-                << iovec_sizes_.getPercentileEstimate(0.95);
+      log_.info() << "iovec size p95: "
-    log_.info() << "iovec size p99: "
+                  << iovec_sizes_.getPercentileEstimate(0.95);
-                << iovec_sizes_.getPercentileEstimate(0.99);
+      log_.info() << "iovec size p99: "
+                  << iovec_sizes_.getPercentileEstimate(0.99);
+    }
   }
 
   ssize_t

src/dwarfs/metadata_v2.cpp

@@ -36,6 +36,8 @@ namespace dwarfs {
 
 namespace {
 
+using ::apache::thrift::frozen::MappedFrozen;
+
 template <class T>
 std::pair<std::vector<uint8_t>, std::vector<uint8_t>>
 freeze_to_buffer(const T& x) {
@@ -62,9 +64,8 @@ freeze_to_buffer(const T& x) {
   return {schema_buffer, data_buffer};
 }
 
-template <class T>
+template <typename T>
-::apache::thrift::frozen::MappedFrozen<T>
+MappedFrozen<T> map_frozen(folly::ByteRange schema, folly::ByteRange data) {
-map_frozen(folly::ByteRange schema, folly::ByteRange data) {
   using namespace ::apache::thrift::frozen;
   auto layout = std::make_unique<Layout<T>>();
   deserializeRootLayout(schema, *layout);
@@ -73,11 +74,20 @@ map_frozen(folly::ByteRange schema, folly::ByteRange data) {
   return ret;
 }
 
+void analyze_frozen(std::ostream& os,
+                    MappedFrozen<thrift::metadata::metadata> const& meta) {
+  using namespace ::apache::thrift::frozen;
+  auto layout = meta.findFirstOfType<
+      std::unique_ptr<Layout<thrift::metadata::metadata>>>();
+  (*layout)->print(os, 0);
+}
+
 const uint16_t READ_ONLY_MASK = ~(S_IWUSR | S_IWGRP | S_IWOTH);
 
 template <typename LoggerPolicy>
 class metadata_ : public metadata_v2::impl {
  public:
+  // TODO: defaults?, remove
   metadata_(logger& lgr, folly::ByteRange schema, folly::ByteRange data,
             const struct ::stat* /*defaults*/, int inode_offset)
       : data_(data)
@@ -85,19 +95,9 @@ class metadata_ : public metadata_v2::impl {
       , root_(meta_.entries()[meta_.entry_index()[0]], &meta_)
       , inode_offset_(inode_offset)
       , chunk_index_offset_(meta_.chunk_index_offset())
-      , log_(lgr) {
+      , log_(lgr) {}
-    // TODO: defaults?, remove
 
-    // TODO: move to separate function (e.g. dump?)
+  void dump(std::ostream& os, int detail_level,
-    //       cannot be done easily with schema, though, as it's temporary
-    // log_.debug() << ::apache::thrift::debugString(meta_.thaw());
-    // ::apache::thrift::frozen::Layout<thrift::metadata::metadata> layout;
-    // ::apache::thrift::frozen::schema::Schema mem_schema;
-    // ::apache::thrift::CompactSerializer::deserialize(schema, mem_schema);
-    // log_.debug() << ::apache::thrift::debugString(mem_schema);
-  }
-
-  void dump(std::ostream& os,
             std::function<void(const std::string&, uint32_t)> const& icb)
       const override;
 
@@ -203,7 +203,7 @@ class metadata_ : public metadata_v2::impl {
   }
 
   folly::ByteRange data_;
-  ::apache::thrift::frozen::MappedFrozen<thrift::metadata::metadata> meta_;
+  MappedFrozen<thrift::metadata::metadata> meta_;
   entry_view root_;
   const int inode_offset_;
   const int chunk_index_offset_;
@@ -255,9 +255,22 @@ void metadata_<LoggerPolicy>::dump(
 
 template <typename LoggerPolicy>
 void metadata_<LoggerPolicy>::dump(
-    std::ostream& os,
+    std::ostream& os, int detail_level,
     std::function<void(const std::string&, uint32_t)> const& icb) const {
-  dump(os, "", root_, icb);
+  struct ::statvfs stbuf;
+  statvfs(&stbuf);
+
+  os << "block size: " << stbuf.f_bsize << std::endl;
+  os << "inode count: " << stbuf.f_files << std::endl;
+  os << "original filesystem size: " << stbuf.f_blocks << std::endl;
+
+  if (detail_level > 0) {
+    analyze_frozen(os, meta_);
+  }
+
+  if (detail_level > 1) {
+    dump(os, "", root_, icb);
+  }
 }
 
 template <typename LoggerPolicy>

src/dwarfsck.cpp

@@ -45,8 +45,8 @@ int main(int argc, char** argv) {
         }
       } else {
         // TODO: add more usage options...
-        dwarfs::filesystem_v2::identify(lgr, mm, std::cout);
+        // dwarfs::filesystem_v2::identify(lgr, mm, std::cout);
-        fs.dump(std::cout);
+        fs.dump(std::cout, 1);
       }
     } catch (const std::exception& e) {
       std::cerr << "Error: " << e.what() << std::endl;

thrift/metadata.thrift

@@ -44,7 +44,7 @@ struct chunk {
 struct directory {
    1: required UInt32 parent_inode,
    2: required UInt32 first_entry,
-   3: required UInt32 entry_count,
+   3: required UInt32 entry_count,   // TODO: we can remove this if we sort entries by directory
 }
 
 /**