From c2840cbe79f8e580ffafe877ce0d33f1d115b53d Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Wed, 29 Nov 2023 13:11:57 +0100 Subject: [PATCH] feat(filesystem): initial support for file system history --- CMakeLists.txt | 55 +++++++++++++- doc/dwarfs-format.md | 5 ++ include/dwarfs/block_data.h | 3 + include/dwarfs/filesystem_v2.h | 4 + include/dwarfs/filesystem_writer.h | 5 ++ include/dwarfs/fstypes.h | 3 + include/dwarfs/history.h | 52 +++++++++++++ include/dwarfs/options.h | 8 ++ scripts/extract_blocks.py | 1 + src/dwarfs/filesystem_v2.cpp | 53 ++++++++++---- src/dwarfs/filesystem_writer.cpp | 8 ++ src/dwarfs/fstypes.cpp | 3 + src/dwarfs/history.cpp | 114 +++++++++++++++++++++++++++++ src/dwarfs/scanner.cpp | 7 ++ src/mkdwarfs_main.cpp | 28 ++++++- thrift/history.thrift | 55 ++++++++++++++ 16 files changed, 388 insertions(+), 16 deletions(-) create mode 100644 include/dwarfs/history.h create mode 100644 src/dwarfs/history.cpp create mode 100644 thrift/history.thrift diff --git a/CMakeLists.txt b/CMakeLists.txt index 93e494fa..fe1097b2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -378,6 +378,7 @@ list( src/dwarfs/fstypes.cpp src/dwarfs/fs_section.cpp src/dwarfs/global_entry_data.cpp + src/dwarfs/history.cpp src/dwarfs/inode_element_view.cpp src/dwarfs/inode_fragments.cpp src/dwarfs/inode_manager.cpp @@ -466,6 +467,12 @@ target_compile_definitions( dwarfs_tool PRIVATE PRJ_BUILD_ID="${CMAKE_SYSTEM_PROCESSOR}, ${CMAKE_SYSTEM}, ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}" ) +target_compile_definitions( + dwarfs PRIVATE + PRJ_SYSTEM_ID="${CMAKE_SYSTEM} [${CMAKE_SYSTEM_PROCESSOR}]" + PRJ_COMPILER_ID="${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}" +) + target_link_libraries(dwarfs_categorizer folly) target_link_libraries(dwarfs_compression folly compression_thrift) target_link_libraries(dwarfs_tool dwarfs) @@ -720,6 +727,27 @@ list( ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/compression_visit_union.h ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/compression_visitation.h) +list( + APPEND + HISTORY_THRIFT_SRC + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_clients.h + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_constants.cpp + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_constants.h + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_data.cpp + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_data.h + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_for_each_field.h + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_handlers.h + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_metadata.cpp + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_metadata.h + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_types.cpp + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_types.h + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_types.tcc + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_types_custom_protocol.h + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_types_fwd.h + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_visit_by_thrift_field_metadata.h + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_visit_union.h + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_visitation.h) + add_custom_command( OUTPUT thrift/lib/thrift/_keep COMMAND ${CMAKE_COMMAND} -E make_directory thrift/lib/thrift @@ -780,6 +808,22 @@ add_custom_command( WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs ) +add_custom_command( + OUTPUT ${HISTORY_THRIFT_SRC} + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_SOURCE_DIR}/thrift/history.thrift + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/history.thrift + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/bin/thrift1 + -I ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift + -o ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs + --gen mstch_cpp2 + history.thrift + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/bin/thrift1 + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/_keep + ${CMAKE_CURRENT_SOURCE_DIR}/thrift/history.thrift + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs +) + list( APPEND INCLUDE_DIRS @@ -841,14 +885,22 @@ add_library( ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/compression_types.cpp ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/compression_data.cpp) +add_library( + history_thrift + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_types.cpp + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_data.cpp) + set_property(TARGET metadata_thrift PROPERTY CXX_STANDARD 20) set_property(TARGET compression_thrift PROPERTY CXX_STANDARD 20) +set_property(TARGET history_thrift PROPERTY CXX_STANDARD 20) target_include_directories(metadata_thrift PRIVATE ${INCLUDE_DIRS}) target_include_directories(compression_thrift PRIVATE ${INCLUDE_DIRS}) +target_include_directories(history_thrift PRIVATE ${INCLUDE_DIRS}) target_link_libraries(metadata_thrift thrift_light) target_link_libraries(compression_thrift thrift_light) +target_link_libraries(history_thrift thrift_light) foreach(tgt dwarfs dwarfs_compression dwarfs_categorizer dwarfs_compression_metadata dwarfs_tool @@ -889,7 +941,7 @@ foreach(tgt dwarfs dwarfs_compression dwarfs_categorizer set_property(TARGET ${tgt} PROPERTY CXX_STANDARD_REQUIRED ON) set_property(TARGET ${tgt} PROPERTY CXX_EXTENSIONS OFF) - add_dependencies(${tgt} metadata_thrift) + add_dependencies(${tgt} metadata_thrift history_thrift) if(ENABLE_ASAN) target_compile_options(${tgt} PRIVATE -fsanitize=address @@ -939,6 +991,7 @@ target_include_directories(dwarfs PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/fsst) target_link_libraries( dwarfs metadata_thrift + history_thrift thrift_light folly fsst diff --git a/doc/dwarfs-format.md b/doc/dwarfs-format.md index 41c24249..9c05f50c 100644 --- a/doc/dwarfs-format.md +++ b/doc/dwarfs-format.md @@ -114,6 +114,11 @@ There are currently 4 different section types. file, you should find a valid section header for the section index. +- `HISTORY` (10): + File system history information as defined `thrift/history.thrift`. + This is stored in "compact" thrift encoding. Zero or more history + sections are supported. + ## METADATA FORMAT Here is a high-level overview of how all the bits and pieces relate diff --git a/include/dwarfs/block_data.h b/include/dwarfs/block_data.h index 3b4bcd06..adb848c0 100644 --- a/include/dwarfs/block_data.h +++ b/include/dwarfs/block_data.h @@ -22,6 +22,7 @@ #pragma once #include +#include #include namespace dwarfs { @@ -31,6 +32,8 @@ class block_data { block_data() = default; explicit block_data(std::vector&& vec) : vec_{std::move(vec)} {} + explicit block_data(std::string_view str) + : vec_{str.begin(), str.end()} {} std::vector const& vec() const { return vec_; } std::vector& vec() { return vec_; } diff --git a/include/dwarfs/filesystem_v2.h b/include/dwarfs/filesystem_v2.h index 12b18cd9..387de852 100644 --- a/include/dwarfs/filesystem_v2.h +++ b/include/dwarfs/filesystem_v2.h @@ -50,6 +50,7 @@ struct file_stat; struct vfs_stat; class filesystem_writer; +class history; class logger; class mmif; class performance_monitor; @@ -171,6 +172,8 @@ class filesystem_v2 { bool has_symlinks() const { return impl_->has_symlinks(); } + history const& get_history() const { return impl_->get_history(); } + class impl { public: virtual ~impl() = default; @@ -210,6 +213,7 @@ class filesystem_v2 { virtual void set_cache_tidy_config(cache_tidy_config const& cfg) = 0; virtual size_t num_blocks() const = 0; virtual bool has_symlinks() const = 0; + virtual history const& get_history() const = 0; }; private: diff --git a/include/dwarfs/filesystem_writer.h b/include/dwarfs/filesystem_writer.h index fec1ccfe..84c656f1 100644 --- a/include/dwarfs/filesystem_writer.h +++ b/include/dwarfs/filesystem_writer.h @@ -102,6 +102,10 @@ class filesystem_writer { impl_->write_metadata_v2(std::move(data)); } + void write_history(std::shared_ptr&& data) { + impl_->write_history(std::move(data)); + } + void write_compressed_section(section_type type, compression_type compression, std::span data) { impl_->write_compressed_section(type, compression, data); @@ -136,6 +140,7 @@ class filesystem_writer { virtual void write_metadata_v2_schema(std::shared_ptr&& data) = 0; virtual void write_metadata_v2(std::shared_ptr&& data) = 0; + virtual void write_history(std::shared_ptr&& data) = 0; virtual void write_compressed_section(section_type type, compression_type compression, std::span data) = 0; diff --git a/include/dwarfs/fstypes.h b/include/dwarfs/fstypes.h index 67d0efef..d5bc053e 100644 --- a/include/dwarfs/fstypes.h +++ b/include/dwarfs/fstypes.h @@ -46,6 +46,9 @@ enum class section_type : uint16_t { SECTION_INDEX = 9, // Section index. + + HISTORY = 10, + // History of file system changes. }; struct file_header { diff --git a/include/dwarfs/history.h b/include/dwarfs/history.h new file mode 100644 index 00000000..f2033974 --- /dev/null +++ b/include/dwarfs/history.h @@ -0,0 +1,52 @@ +/* vim:set ts=2 sw=2 sts=2 et: */ +/** + * \author Marcus Holland-Moritz (github@mhxnet.de) + * \copyright Copyright (c) Marcus Holland-Moritz + * + * This file is part of dwarfs. + * + * dwarfs is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * dwarfs is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with dwarfs. If not, see . + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "dwarfs/options.h" + +#include "dwarfs/gen-cpp2/history_types.h" + +namespace dwarfs { + +class history { + public: + explicit history(history_config const& cfg = {}); + + void parse(std::span data); + void parse_append(std::span data); + thrift::history::history const& get() const { return history_; } + void append(std::optional> args); + std::vector serialize() const; + void dump(std::ostream& os) const; + + private: + thrift::history::history history_; + history_config const cfg_; +}; + +} // namespace dwarfs diff --git a/include/dwarfs/options.h b/include/dwarfs/options.h index f32273d3..72fa8c7d 100644 --- a/include/dwarfs/options.h +++ b/include/dwarfs/options.h @@ -27,6 +27,7 @@ #include #include #include +#include #include "dwarfs/categorized_option.h" #include "dwarfs/file_stat.h" @@ -50,6 +51,10 @@ struct block_cache_options { bool disable_block_integrity_check{false}; }; +struct history_config { + bool with_timestamps{false}; +}; + struct cache_tidy_config { cache_tidy_strategy strategy{cache_tidy_strategy::NONE}; std::chrono::milliseconds interval; @@ -123,6 +128,9 @@ struct scanner_options { bool no_create_timestamp{true}; std::optional> debug_filter_function; size_t num_segmenter_workers{1}; + bool enable_history{true}; + std::optional> command_line_arguments; + history_config history; }; struct rewrite_options { diff --git a/scripts/extract_blocks.py b/scripts/extract_blocks.py index 16b48a85..ea46aa5f 100755 --- a/scripts/extract_blocks.py +++ b/scripts/extract_blocks.py @@ -28,6 +28,7 @@ sectypes = { 7: "schema", 8: "metadata", 9: "index", + 10: "history", } compalgs = { diff --git a/src/dwarfs/filesystem_v2.cpp b/src/dwarfs/filesystem_v2.cpp index f5284b3a..ab5f2333 100644 --- a/src/dwarfs/filesystem_v2.cpp +++ b/src/dwarfs/filesystem_v2.cpp @@ -39,6 +39,7 @@ #include "dwarfs/filesystem_writer.h" #include "dwarfs/fs_section.h" #include "dwarfs/fstypes.h" +#include "dwarfs/history.h" #include "dwarfs/inode_reader_v2.h" #include "dwarfs/logger.h" #include "dwarfs/metadata_v2.h" @@ -260,7 +261,7 @@ class filesystem_parser { std::vector index_; }; -using section_map = std::unordered_map; +using section_map = std::unordered_map>; size_t get_uncompressed_section_size(std::shared_ptr mm, fs_section const& sec) { @@ -303,11 +304,19 @@ make_metadata(logger& lgr, std::shared_ptr mm, DWARFS_THROW(runtime_error, "no metadata schema found"); } + if (schema_it->second.size() > 1) { + DWARFS_THROW(runtime_error, "multiple metadata schemas found"); + } + if (meta_it == sections.end()) { DWARFS_THROW(runtime_error, "no metadata found"); } - auto& meta_section = meta_it->second; + if (meta_it->second.size() > 1) { + DWARFS_THROW(runtime_error, "multiple metadata found"); + } + + auto& meta_section = meta_it->second.front(); auto meta_section_range = get_section_data(mm, meta_section, meta_buffer, force_buffers); @@ -329,10 +338,11 @@ make_metadata(logger& lgr, std::shared_ptr mm, } } - return metadata_v2( - lgr, - get_section_data(mm, schema_it->second, schema_buffer, force_buffers), - meta_section_range, options, inode_offset, force_consistency_check); + return metadata_v2(lgr, + get_section_data(mm, schema_it->second.front(), + schema_buffer, force_buffers), + meta_section_range, options, inode_offset, + force_consistency_check); } template @@ -376,6 +386,7 @@ class filesystem_ final : public filesystem_v2::impl { } size_t num_blocks() const override { return ir_.num_blocks(); } bool has_symlinks() const override { return meta_.has_symlinks(); } + history const& get_history() const override { return history_; } private: filesystem_info const& get_info() const; @@ -390,6 +401,7 @@ class filesystem_ final : public filesystem_v2::impl { std::vector meta_buffer_; std::optional> header_; mutable std::unique_ptr fsinfo_; + history history_; PERFMON_CLS_PROXY_DECL PERFMON_CLS_TIMER_DECL(find_path) PERFMON_CLS_TIMER_DECL(find_inode) @@ -461,6 +473,7 @@ filesystem_::filesystem_( : LOG_PROXY_INIT(lgr) , mm_(std::move(mm)) , parser_(mm_, options.image_offset) + , history_({.with_timestamps = true}) // clang-format off PERFMON_CLS_PROXY_INIT(perfmon, "filesystem_v2") PERFMON_CLS_TIMER_INIT(find_path) @@ -498,9 +511,7 @@ filesystem_::filesystem_( DWARFS_THROW(runtime_error, "checksum error in section: " + s->name()); } - if (!sections.emplace(s->type(), *s).second) { - DWARFS_THROW(runtime_error, "duplicate section: " + s->name()); - } + sections[s->type()].push_back(*s); } } @@ -516,10 +527,21 @@ filesystem_::filesystem_( cache.set_block_size(meta_.block_size()); ir_ = inode_reader_v2(lgr, std::move(cache), perfmon); + + if (auto it = sections.find(section_type::HISTORY); it != sections.end()) { + for (auto& section : it->second) { + std::vector buffer; + history_.parse_append(get_section_data(mm_, section, buffer, false)); + } + } } template void filesystem_::dump(std::ostream& os, int detail_level) const { + if (detail_level > 1) { + history_.dump(os); + } + meta_.dump(os, detail_level, get_info(), [&](const std::string& indent, uint32_t inode) { if (auto chunks = meta_.get_chunks(inode)) { @@ -716,10 +738,11 @@ void filesystem_v2::rewrite(logger& lgr, progress& prog, if (s->type() == section_type::BLOCK) { ++prog.block_count; } else if (s->type() != section_type::SECTION_INDEX) { - if (!sections.emplace(s->type(), *s).second) { - DWARFS_THROW(runtime_error, "duplicate section: " + s->name()); + auto& secvec = sections[s->type()]; + if (secvec.empty()) { + section_types.push_back(s->type()); } - section_types.push_back(s->type()); + secvec.push_back(*s); } } @@ -755,8 +778,10 @@ void filesystem_v2::rewrite(logger& lgr, progress& prog, writer.write_metadata_v2(std::make_shared(std::move(meta_raw))); } else { for (auto type : section_types) { - auto& sec = DWARFS_NOTHROW(sections.at(type)); - writer.write_compressed_section(type, sec.compression(), sec.data(*mm)); + auto& secvec = DWARFS_NOTHROW(sections.at(type)); + for (auto& sec : secvec) { + writer.write_compressed_section(type, sec.compression(), sec.data(*mm)); + } } } diff --git a/src/dwarfs/filesystem_writer.cpp b/src/dwarfs/filesystem_writer.cpp index 12fbc33f..6c3452d4 100644 --- a/src/dwarfs/filesystem_writer.cpp +++ b/src/dwarfs/filesystem_writer.cpp @@ -359,6 +359,7 @@ class filesystem_writer_ final : public filesystem_writer::impl { std::optional meta) override; void write_metadata_v2_schema(std::shared_ptr&& data) override; void write_metadata_v2(std::shared_ptr&& data) override; + void write_history(std::shared_ptr&& data) override; void write_compressed_section(section_type type, compression_type compression, std::span data) override; void flush() override; @@ -737,6 +738,13 @@ void filesystem_writer_::write_metadata_v2( write_section(section_type::METADATA_V2, std::move(data), metadata_bc_); } +template +void filesystem_writer_::write_history( + std::shared_ptr&& data) { + write_section(section_type::HISTORY, std::move(data), + metadata_bc_); // TODO: history_bc_? +} + template void filesystem_writer_::flush() { { diff --git a/src/dwarfs/fstypes.cpp b/src/dwarfs/fstypes.cpp index 953d41af..835b746c 100644 --- a/src/dwarfs/fstypes.cpp +++ b/src/dwarfs/fstypes.cpp @@ -35,14 +35,17 @@ namespace dwarfs { namespace { +// clang-format off const std::map sections{ #define SECTION_TYPE_(x) {section_type::x, #x} SECTION_TYPE_(BLOCK), SECTION_TYPE_(METADATA_V2_SCHEMA), SECTION_TYPE_(METADATA_V2), SECTION_TYPE_(SECTION_INDEX), + SECTION_TYPE_(HISTORY), #undef SECTION_TYPE_ }; +// clang-format on const std::map compressions{ #define DWARFS_COMPRESSION_TYPE_(name, value) {compression_type::name, #name} diff --git a/src/dwarfs/history.cpp b/src/dwarfs/history.cpp new file mode 100644 index 00000000..386c9691 --- /dev/null +++ b/src/dwarfs/history.cpp @@ -0,0 +1,114 @@ +/* vim:set ts=2 sw=2 sts=2 et: */ +/** + * \author Marcus Holland-Moritz (github@mhxnet.de) + * \copyright Copyright (c) Marcus Holland-Moritz + * + * This file is part of dwarfs. + * + * dwarfs is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * dwarfs is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with dwarfs. If not, see . + */ + +#include + +#include +#include + +#include + +#include "dwarfs/history.h" +#include "dwarfs/version.h" + +namespace dwarfs { + +history::history(history_config const& cfg) + : cfg_{cfg} {} + +void history::parse(std::span data) { + history_.entries()->clear(); + parse_append(data); +} + +void history::parse_append(std::span data) { + folly::Range range{data.data(), data.size()}; + thrift::history::history tmp; + apache::thrift::CompactSerializer::deserialize(range, tmp); + history_.entries()->insert(history_.entries()->end(), + std::make_move_iterator(tmp.entries()->begin()), + std::make_move_iterator(tmp.entries()->end())); +} + +void history::append(std::optional> args) { + auto& histent = history_.entries()->emplace_back(); + auto& version = histent.version().value(); + version.major() = PRJ_VERSION_MAJOR; + version.minor() = PRJ_VERSION_MINOR; + version.patch() = PRJ_VERSION_PATCH; + version.is_release() = std::string_view(PRJ_GIT_DESC) == PRJ_GIT_ID; + version.git_rev() = PRJ_GIT_REV; + version.git_branch() = PRJ_GIT_BRANCH; + version.git_desc() = PRJ_GIT_DESC; + histent.system_id() = PRJ_SYSTEM_ID; + histent.compiler_id() = PRJ_COMPILER_ID; + if (args) { + histent.arguments() = std::move(*args); + } + if (cfg_.with_timestamps) { + histent.timestamp() = std::time(nullptr); + } +} + +std::vector history::serialize() const { + std::string buf; + ::apache::thrift::CompactSerializer::serialize(history_, &buf); + return std::vector(buf.begin(), buf.end()); +} + +void history::dump(std::ostream& os) const { + if (!history_.entries()->empty()) { + size_t const iwidth{std::to_string(history_.entries()->size()).size()}; + size_t i{1}; + + os << "History:\n"; + for (auto const& histent : *history_.entries()) { + os << " " << fmt::format("{:>{}}:", i++, iwidth); + + if (histent.timestamp().has_value()) { + os << " " + << fmt::format("[{:%Y-%m-%d %H:%M:%S}]", + fmt::localtime(histent.timestamp().value())); + } + + auto const& version = histent.version().value(); + + os << " libdwarfs " << version.git_desc().value(); + + if (!version.is_release().value()) { + os << " (" << version.git_branch().value() << ")"; + } + + os << " on " << histent.system_id().value() << ", " + << histent.compiler_id().value() << "\n"; + + if (histent.arguments().has_value() && !histent.arguments()->empty()) { + os << fmt::format(" {:>{}} args:", "", iwidth); + for (auto const& arg : histent.arguments().value()) { + os << ' ' << arg; + } + os << "\n"; + } + } + } +} + +} // namespace dwarfs diff --git a/src/dwarfs/scanner.cpp b/src/dwarfs/scanner.cpp index dba6003a..662db99c 100644 --- a/src/dwarfs/scanner.cpp +++ b/src/dwarfs/scanner.cpp @@ -46,6 +46,7 @@ #include "dwarfs/filesystem_writer.h" #include "dwarfs/fragment_chunkable.h" #include "dwarfs/global_entry_data.h" +#include "dwarfs/history.h" #include "dwarfs/inode.h" #include "dwarfs/inode_manager.h" #include "dwarfs/inode_ordering.h" @@ -904,6 +905,12 @@ void scanner_::scan( fsw.write_metadata_v2_schema(std::make_shared(std::move(schema))); fsw.write_metadata_v2(std::make_shared(std::move(data))); + if (options_.enable_history) { + history hist(options_.history); + hist.append(options_.command_line_arguments); + fsw.write_history(std::make_shared(hist.serialize())); + } + LOG_INFO << "waiting for compression to finish..."; fsw.flush(); diff --git a/src/mkdwarfs_main.cpp b/src/mkdwarfs_main.cpp index 6c687b3a..1a21e2f9 100644 --- a/src/mkdwarfs_main.cpp +++ b/src/mkdwarfs_main.cpp @@ -286,7 +286,8 @@ int mkdwarfs_main(int argc, sys_char** argv) { bloom_filter_size, compression; size_t num_workers, num_scanner_workers, num_segmenter_workers; bool no_progress = false, remove_header = false, no_section_index = false, - force_overwrite = false; + force_overwrite = false, no_history = false, + no_history_timestamps = false, no_history_command_line = false; unsigned level; int compress_niceness; uint16_t uid, gid; @@ -414,6 +415,15 @@ int mkdwarfs_main(int argc, sys_char** argv) { ("no-section-index", po::value(&no_section_index)->zero_tokens(), "don't add section index to file system") + ("no-history", + po::value(&no_history)->zero_tokens(), + "don't add history to file system") + ("no-history-timestamps", + po::value(&no_history_timestamps)->zero_tokens(), + "don't add timestamps to file system history") + ("no-history-command-line", + po::value(&no_history_command_line)->zero_tokens(), + "don't add command line to file system history") ; po::options_description segmenter_opts("Segmenter options"); @@ -510,6 +520,13 @@ int mkdwarfs_main(int argc, sys_char** argv) { auto& sys_err_out = SYS_CERR; + std::vector command_line; + command_line.reserve(argc); + + for (int i = 0; i < argc; ++i) { + command_line.emplace_back(sys_string_to_string(argv[i])); + } + try { auto parsed = po::parse_command_line(argc, argv, opts); @@ -954,6 +971,15 @@ int mkdwarfs_main(int argc, sys_char** argv) { os = std::make_unique(); } + options.enable_history = !no_history; + + if (options.enable_history) { + options.history.with_timestamps = !no_history_timestamps; + if (!no_history_command_line) { + options.command_line_arguments = command_line; + } + } + // TODO: the whole re-writing thing will be a bit weird in combination // with categories; we'd likely require a "category"-section to be // present (which we'll also require for bit-identical mode) diff --git a/thrift/history.thrift b/thrift/history.thrift new file mode 100644 index 00000000..230669ca --- /dev/null +++ b/thrift/history.thrift @@ -0,0 +1,55 @@ +/* vim:set ts=2 sw=2 sts=2 et: */ +/** + * \author Marcus Holland-Moritz (github@mhxnet.de) + * \copyright Copyright (c) Marcus Holland-Moritz + * + * This file is part of dwarfs. + * + * dwarfs is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * dwarfs is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with dwarfs. If not, see . + */ + +include "thrift/annotation/cpp.thrift" + +namespace cpp2 dwarfs.thrift.history + +@cpp.Type{name = "uint8_t"} +typedef byte UInt8 +@cpp.Type{name = "uint16_t"} +typedef i16 UInt16 +@cpp.Type{name = "uint32_t"} +typedef i32 UInt32 +@cpp.Type{name = "uint64_t"} +typedef i64 UInt64 + +struct dwarfs_version { + 1: UInt16 major + 2: UInt16 minor + 3: UInt16 patch + 4: bool is_release + 5: optional string git_rev + 6: optional string git_branch + 7: optional string git_desc +} + +struct history_entry { + 1: dwarfs_version version + 2: string system_id + 3: string compiler_id + 4: optional list arguments + 5: optional UInt64 timestamp +} + +struct history { + 1: list entries +}