From 04f2f1b9767757fb68605ada44775e16de639e28 Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Mon, 14 Dec 2020 22:58:27 +0100 Subject: [PATCH] Add metadata export to `dwarfsck` --- CMakeLists.txt | 1 + include/dwarfs/filesystem_v2.h | 5 +++++ include/dwarfs/metadata_v2.h | 5 +++++ src/dwarfs/filesystem_v2.cpp | 7 +++++++ src/dwarfs/metadata_v2.cpp | 15 +++++++++++++++ src/dwarfsck.cpp | 20 ++++++++++++++++++-- 6 files changed, 51 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d4da5991..c1b16cf5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -332,6 +332,7 @@ add_library( ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/protocol/BinaryProtocol.cpp ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/protocol/DebugProtocol.cpp ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/protocol/JSONProtocolCommon.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/protocol/JSONProtocol.cpp ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp/protocol/TProtocolException.cpp ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp/util/VarintUtils.cpp ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/gen/module_types_cpp.cpp diff --git a/include/dwarfs/filesystem_v2.h b/include/dwarfs/filesystem_v2.h index 8b8161c2..1995a33c 100644 --- a/include/dwarfs/filesystem_v2.h +++ b/include/dwarfs/filesystem_v2.h @@ -74,6 +74,10 @@ class filesystem_v2 { return impl_->metadata_as_dynamic(); } + std::string serialize_metadata_as_json(bool simple) const { + return impl_->serialize_metadata_as_json(simple); + } + void walk(std::function const& func) const { impl_->walk(func); } @@ -134,6 +138,7 @@ class filesystem_v2 { virtual void dump(std::ostream& os, int detail_level) const = 0; virtual folly::dynamic metadata_as_dynamic() const = 0; + virtual std::string serialize_metadata_as_json(bool simple) const = 0; virtual void walk(std::function const& func) const = 0; virtual std::optional find(const char* path) const = 0; virtual std::optional find(int inode) const = 0; diff --git a/include/dwarfs/metadata_v2.h b/include/dwarfs/metadata_v2.h index dfc55ca2..e68191d6 100644 --- a/include/dwarfs/metadata_v2.h +++ b/include/dwarfs/metadata_v2.h @@ -71,6 +71,10 @@ class metadata_v2 { folly::dynamic as_dynamic() const { return impl_->as_dynamic(); } + std::string serialize_as_json(bool simple) const { + return impl_->serialize_as_json(simple); + } + static void get_stat_defaults(struct ::stat* defaults); // TODO: check if this is needed @@ -142,6 +146,7 @@ class metadata_v2 { std::function const& icb) const = 0; virtual folly::dynamic as_dynamic() const = 0; + virtual std::string serialize_as_json(bool simple) const = 0; virtual size_t size() const = 0; virtual bool empty() const = 0; diff --git a/src/dwarfs/filesystem_v2.cpp b/src/dwarfs/filesystem_v2.cpp index 35854c6d..1b8a88ed 100644 --- a/src/dwarfs/filesystem_v2.cpp +++ b/src/dwarfs/filesystem_v2.cpp @@ -195,6 +195,7 @@ class filesystem_ : public filesystem_v2::impl { void dump(std::ostream& os, int detail_level) const override; folly::dynamic metadata_as_dynamic() const override; + std::string serialize_metadata_as_json(bool simple) const override; void walk(std::function const& func) const override; std::optional find(const char* path) const override; std::optional find(int inode) const override; @@ -278,6 +279,12 @@ folly::dynamic filesystem_::metadata_as_dynamic() const { return meta_.as_dynamic(); } +template +std::string +filesystem_::serialize_metadata_as_json(bool simple) const { + return meta_.serialize_as_json(simple); +} + template void filesystem_::walk( std::function const& func) const { diff --git a/src/dwarfs/metadata_v2.cpp b/src/dwarfs/metadata_v2.cpp index 9bee6b46..657b54b9 100644 --- a/src/dwarfs/metadata_v2.cpp +++ b/src/dwarfs/metadata_v2.cpp @@ -34,6 +34,7 @@ #include #include +#include #include @@ -170,6 +171,7 @@ class metadata_ : public metadata_v2::impl { const override; folly::dynamic as_dynamic() const override; + std::string serialize_as_json(bool simple) const override; size_t size() const override { return data_.size(); } @@ -525,6 +527,19 @@ folly::dynamic metadata_::as_dynamic() const { return obj; } +template +std::string metadata_::serialize_as_json(bool simple) const { + std::string json; + if (simple) { + apache::thrift::SimpleJSONSerializer serializer; + serializer.serialize(meta_.thaw(), &json); + } else { + apache::thrift::JSONSerializer serializer; + serializer.serialize(meta_.thaw(), &json); + } + return json; +} + template std::string metadata_::modestring(uint16_t mode) const { std::ostringstream oss; diff --git a/src/dwarfsck.cpp b/src/dwarfsck.cpp index 76cb33ab..1e1123c4 100644 --- a/src/dwarfsck.cpp +++ b/src/dwarfsck.cpp @@ -24,6 +24,8 @@ #include +#include +#include #include #include @@ -38,7 +40,7 @@ namespace dwarfs { namespace po = boost::program_options; int dwarfsck(int argc, char** argv) { - std::string log_level, input; + std::string log_level, input, export_metadata; int detail; bool json = false; @@ -54,6 +56,9 @@ int dwarfsck(int argc, char** argv) { ("json", po::value(&json)->zero_tokens(), "print metadata in JSON format") + ("export-metadata", + po::value(&export_metadata), + "export raw metadata as JSON to file") ("log-level", po::value(&log_level)->default_value("info"), "log level (error, warn, info, debug, trace)") @@ -87,7 +92,15 @@ int dwarfsck(int argc, char** argv) { try { auto mm = std::make_shared(input); - if (json) { + if (!export_metadata.empty()) { + auto of = folly::File(export_metadata, O_RDWR | O_CREAT); + filesystem_v2 fs(lgr, mm); + auto json = fs.serialize_metadata_as_json(true); + if (folly::writeFull(of.fd(), json.data(), json.size()) < 0) { + LOG_ERROR << "failed to export metadata"; + } + of.close(); + } else if (json) { filesystem_v2 fs(lgr, mm); std::cout << folly::toPrettyJson(fs.metadata_as_dynamic()) << std::endl; } else { @@ -99,6 +112,9 @@ int dwarfsck(int argc, char** argv) { } catch (runtime_error const& e) { LOG_ERROR << folly::exceptionStr(e); return 1; + } catch (std::system_error const& e) { + LOG_ERROR << folly::exceptionStr(e); + return 1; } return 0;