From 9725b8fc6173f4e2427d275d321e0a67115f1920 Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Tue, 8 Dec 2020 13:04:31 +0100 Subject: [PATCH] Add `--keep-all-times` and default to only keeping `mtime` --- doc/mkdwarfs.md | 5 +++++ include/dwarfs/global_entry_data.h | 14 +++++++------ include/dwarfs/options.h | 1 + src/dwarfs/entry.cpp | 12 +++++------ src/dwarfs/global_entry_data.cpp | 32 ++++++++++++++++++++++++++++-- src/dwarfs/metadata_v2.cpp | 7 +++++-- src/dwarfs/scanner.cpp | 4 ++++ src/mkdwarfs.cpp | 3 +++ thrift/metadata.thrift | 12 +++++++++++ 9 files changed, 74 insertions(+), 16 deletions(-) diff --git a/doc/mkdwarfs.md b/doc/mkdwarfs.md index 717333e4..eb9e44e6 100644 --- a/doc/mkdwarfs.md +++ b/doc/mkdwarfs.md @@ -119,6 +119,11 @@ Most other options are concerned with compression tuning: reduce the size of the file system. You can pass either a unix time stamp or `now`. + * `--keep-all-times`: + As of release 0.3.0, by default, `mkdwarfs` will only save the contents of + the `mtime` field in order to save metadata space. If you want to save + `atime` and `ctime` as well, use this option. + * `--order=none`|`path`|`similarity`|`nilsimsa`|`script`: The order in which files will be written to the filesystem. Choosing `none`, the files will be stored in the order in which they are discovered. With diff --git a/include/dwarfs/global_entry_data.h b/include/dwarfs/global_entry_data.h index fc2e0253..e5440051 100644 --- a/include/dwarfs/global_entry_data.h +++ b/include/dwarfs/global_entry_data.h @@ -33,6 +33,8 @@ struct scanner_options; class global_entry_data { public: + enum class timestamp_type { ATIME, MTIME, CTIME }; + global_entry_data(scanner_options const& options) : options_(options) {} @@ -41,11 +43,9 @@ class global_entry_data { void add_mode(uint16_t mode) { add(mode, modes_, next_mode_index_); } - void add_time(uint64_t time) { - if (time < timestamp_base_) { - timestamp_base_ = time; - } - } + void add_mtime(uint64_t time); + void add_atime(uint64_t time); + void add_ctime(uint64_t time); void add_name(std::string const& name) { names_.emplace(name, 0); } void add_link(std::string const& link) { links_.emplace(link, 0); } @@ -67,7 +67,9 @@ class global_entry_data { return links_.at(link); } - uint64_t get_time_offset(uint64_t time) const; + uint64_t get_mtime_offset(uint64_t time) const; + uint64_t get_atime_offset(uint64_t time) const; + uint64_t get_ctime_offset(uint64_t time) const; std::vector get_uids() const; std::vector get_gids() const; diff --git a/include/dwarfs/options.h b/include/dwarfs/options.h index d4fe01fb..e8b7b00e 100644 --- a/include/dwarfs/options.h +++ b/include/dwarfs/options.h @@ -60,6 +60,7 @@ struct scanner_options { std::optional uid; std::optional gid; std::optional timestamp; + bool keep_all_times{false}; bool remove_empty_dirs{false}; inode_options inode; }; diff --git a/src/dwarfs/entry.cpp b/src/dwarfs/entry.cpp index b65f72b8..cedff45f 100644 --- a/src/dwarfs/entry.cpp +++ b/src/dwarfs/entry.cpp @@ -97,9 +97,9 @@ void entry::update(global_entry_data& data) const { data.add_uid(stat_.st_uid); data.add_gid(stat_.st_gid); data.add_mode(stat_.st_mode & 0xFFFF); - data.add_time(stat_.st_atime); - data.add_time(stat_.st_mtime); - data.add_time(stat_.st_ctime); + data.add_atime(stat_.st_atime); + data.add_mtime(stat_.st_mtime); + data.add_ctime(stat_.st_ctime); } void entry::pack(thrift::metadata::entry& entry_v2, @@ -108,9 +108,9 @@ void entry::pack(thrift::metadata::entry& entry_v2, entry_v2.mode_index = data.get_mode_index(stat_.st_mode & 0xFFFF); entry_v2.owner_index = data.get_uid_index(stat_.st_uid); entry_v2.group_index = data.get_gid_index(stat_.st_gid); - entry_v2.atime_offset = data.get_time_offset(stat_.st_atime); - entry_v2.mtime_offset = data.get_time_offset(stat_.st_mtime); - entry_v2.ctime_offset = data.get_time_offset(stat_.st_ctime); + entry_v2.atime_offset = data.get_atime_offset(stat_.st_atime); + entry_v2.mtime_offset = data.get_mtime_offset(stat_.st_mtime); + entry_v2.ctime_offset = data.get_ctime_offset(stat_.st_ctime); entry_v2.inode = inode_num(); } diff --git a/src/dwarfs/global_entry_data.cpp b/src/dwarfs/global_entry_data.cpp index 09ef8902..f4f08485 100644 --- a/src/dwarfs/global_entry_data.cpp +++ b/src/dwarfs/global_entry_data.cpp @@ -61,8 +61,18 @@ void global_entry_data::index(std::unordered_map& map) { from(map) | get<0>() | order | [&](std::string const& s) { map[s] = ix++; }; } -uint64_t global_entry_data::get_time_offset(uint64_t time) const { - return options_.timestamp ? 0 : time - timestamp_base_; +uint64_t global_entry_data::get_mtime_offset(uint64_t time) const { + return !options_.timestamp ? time - timestamp_base_ : UINT64_C(0); +} + +uint64_t global_entry_data::get_atime_offset(uint64_t time) const { + return !options_.timestamp && options_.keep_all_times ? time - timestamp_base_ + : UINT64_C(0); +} + +uint64_t global_entry_data::get_ctime_offset(uint64_t time) const { + return !options_.timestamp && options_.keep_all_times ? time - timestamp_base_ + : UINT64_C(0); } uint64_t global_entry_data::get_timestamp_base() const { @@ -89,4 +99,22 @@ void global_entry_data::add_gid(uint16_t gid) { } } +void global_entry_data::add_mtime(uint64_t time) { + if (time < timestamp_base_) { + timestamp_base_ = time; + } +} + +void global_entry_data::add_atime(uint64_t time) { + if (options_.keep_all_times) { + add_mtime(time); + } +} + +void global_entry_data::add_ctime(uint64_t time) { + if (options_.keep_all_times) { + add_mtime(time); + } +} + } // namespace dwarfs diff --git a/src/dwarfs/metadata_v2.cpp b/src/dwarfs/metadata_v2.cpp index 96cea9c8..95453884 100644 --- a/src/dwarfs/metadata_v2.cpp +++ b/src/dwarfs/metadata_v2.cpp @@ -624,6 +624,7 @@ int metadata_::getattr(entry_view entry, auto mode = entry.mode(); auto timebase = meta_.timestamp_base(); auto inode = entry.inode(); + bool mtime_only = meta_.options() && meta_.options()->mtime_only(); stbuf->st_mode = mode; @@ -633,9 +634,11 @@ int metadata_::getattr(entry_view entry, stbuf->st_blocks = (stbuf->st_size + 511) / 512; stbuf->st_uid = entry.getuid(); stbuf->st_gid = entry.getgid(); - stbuf->st_atime = timebase + entry.atime_offset(); stbuf->st_mtime = timebase + entry.mtime_offset(); - stbuf->st_ctime = timebase + entry.ctime_offset(); + stbuf->st_atime = + mtime_only ? stbuf->st_mtime : timebase + entry.atime_offset(); + stbuf->st_ctime = + mtime_only ? stbuf->st_mtime : timebase + entry.ctime_offset(); stbuf->st_nlink = options_.enable_nlink && S_ISREG(mode) ? nlinks_.at(inode - chunk_index_offset_) : 1; diff --git a/src/dwarfs/scanner.cpp b/src/dwarfs/scanner.cpp index 36d6443c..d8854807 100644 --- a/src/dwarfs/scanner.cpp +++ b/src/dwarfs/scanner.cpp @@ -560,6 +560,9 @@ void scanner_::scan(filesystem_writer& fsw, root->accept(sdv); sdv.pack(mv2, ge_data); + thrift::metadata::fs_options fsopts; + fsopts.mtime_only = !options_.keep_all_times; + mv2.uids = ge_data.get_uids(); mv2.gids = ge_data.get_gids(); mv2.modes = ge_data.get_modes(); @@ -568,6 +571,7 @@ void scanner_::scan(filesystem_writer& fsw, mv2.timestamp_base = ge_data.get_timestamp_base(); mv2.block_size = UINT32_C(1) << cfg_.block_size_bits; mv2.total_fs_size = prog.original_size; + mv2.options_ref() = fsopts; auto [schema, data] = metadata_v2::freeze(mv2); diff --git a/src/mkdwarfs.cpp b/src/mkdwarfs.cpp index 3c421076..48204762 100644 --- a/src/mkdwarfs.cpp +++ b/src/mkdwarfs.cpp @@ -337,6 +337,9 @@ int mkdwarfs(int argc, char** argv) { ("set-time", po::value(×tamp), "set timestamp for whole file system (unixtime or 'now')") + ("keep-all-times", + po::value(&options.keep_all_times)->zero_tokens(), + "save atime and ctime in addition to mtime") ("order", po::value(&options.file_order) ->default_value(file_order_mode::SIMILARITY, "similarity"), diff --git a/thrift/metadata.thrift b/thrift/metadata.thrift index 8f485a4f..fe24356e 100644 --- a/thrift/metadata.thrift +++ b/thrift/metadata.thrift @@ -86,6 +86,11 @@ struct entry { 8: required UInt64 ctime_offset, } +struct fs_options { + // file system contains only mtime time stamps + 1: required bool mtime_only, +} + struct metadata { /** * Ranges of chunks that make up regular files. Identical @@ -169,6 +174,13 @@ struct metadata { // total file system size 16: required UInt64 total_fs_size, + //=========================================================// + // fields added with dwarfs-0.3.0, file system version 2.1 // + //=========================================================// + // device ids, for lookup by (inode - device_index_offset) 17: optional list devices, + + // file system options + 18: optional fs_options options, }