feat(features): use thrift feature enum and implement feature_set

CMakeLists.txt

@@ -748,6 +748,27 @@ list(
   ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_visit_union.h
   ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_visitation.h)
 
+list(
+  APPEND
+  FEATURES_THRIFT_SRC
+  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_clients.h
+  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_constants.cpp
+  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_constants.h
+  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_data.cpp
+  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_data.h
+  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_for_each_field.h
+  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_handlers.h
+  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_metadata.cpp
+  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_metadata.h
+  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_types.cpp
+  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_types.h
+  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_types.tcc
+  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_types_custom_protocol.h
+  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_types_fwd.h
+  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_visit_by_thrift_field_metadata.h
+  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_visit_union.h
+  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_visitation.h)
+
 add_custom_command(
   OUTPUT thrift/lib/thrift/_keep
   COMMAND ${CMAKE_COMMAND} -E make_directory thrift/lib/thrift
@@ -824,6 +845,21 @@ add_custom_command(
   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs
 )
 
+add_custom_command(
+  OUTPUT ${FEATURES_THRIFT_SRC}
+  COMMAND ${CMAKE_COMMAND} -E copy
+              ${CMAKE_CURRENT_SOURCE_DIR}/thrift/features.thrift
+              ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/features.thrift
+  COMMAND ${CMAKE_CURRENT_BINARY_DIR}/bin/thrift1
+              -o ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs
+              --gen mstch_cpp2
+              features.thrift
+  DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/bin/thrift1
+          ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/_keep
+          ${CMAKE_CURRENT_SOURCE_DIR}/thrift/features.thrift
+  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs
+)
+
 list(
   APPEND
   INCLUDE_DIRS
@@ -890,17 +926,25 @@ add_library(
   ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_types.cpp
   ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_data.cpp)
 
+add_library(
+  features_thrift
+  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_types.cpp
+  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_data.cpp)
+
 set_property(TARGET metadata_thrift PROPERTY CXX_STANDARD 20)
 set_property(TARGET compression_thrift PROPERTY CXX_STANDARD 20)
 set_property(TARGET history_thrift PROPERTY CXX_STANDARD 20)
+set_property(TARGET features_thrift PROPERTY CXX_STANDARD 20)
 
 target_include_directories(metadata_thrift PRIVATE ${INCLUDE_DIRS})
 target_include_directories(compression_thrift PRIVATE ${INCLUDE_DIRS})
 target_include_directories(history_thrift PRIVATE ${INCLUDE_DIRS})
+target_include_directories(features_thrift PRIVATE ${INCLUDE_DIRS})
 
 target_link_libraries(metadata_thrift thrift_light)
 target_link_libraries(compression_thrift thrift_light)
 target_link_libraries(history_thrift thrift_light)
+target_link_libraries(features_thrift thrift_light)
 
 foreach(tgt dwarfs dwarfs_compression dwarfs_categorizer
             dwarfs_compression_metadata dwarfs_tool
@@ -992,7 +1036,7 @@ target_link_libraries(
   dwarfs
   metadata_thrift
   history_thrift
-  thrift_light
+  features_thrift
   folly
   fsst
   ${Boost_LIBRARIES})

include/dwarfs/features.h

@@ -24,8 +24,21 @@
 #include <set>
 #include <string>
 
+#include "dwarfs/gen-cpp2/features_types.h"
+
 namespace dwarfs {
 
-std::set<std::string> get_unsupported_features(std::set<std::string> features);
+class feature_set {
+ public:
+  static std::set<std::string> get_supported();
+  static std::set<std::string> get_unsupported(std::set<std::string> features);
+
+  void add(feature f);
+
+  std::set<std::string> const& get() const { return features_; }
+
+ private:
+  std::set<std::string> features_;
+};
 
 } // namespace dwarfs

src/dwarfs/features.cpp

@@ -22,38 +22,42 @@
 #include <algorithm>
 #include <iterator>
 
+#include <thrift/lib/cpp/util/EnumUtils.h>
+
 #include "dwarfs/features.h"
 
 namespace dwarfs {
 
 namespace {
 
-std::set<std::string> supported_features{
+constexpr bool is_supported_feature(feature /*f*/) { return true; }
-#ifdef DWARFS_HAVE_LIBZSTD
+
-    "zstd",
+std::string feature_name(feature f) {
-#endif
+  return apache::thrift::util::enumNameOrThrow(f);
-#ifdef DWARFS_HAVE_LIBLZ4
+}
-    "lz4",
-#endif
-#ifdef DWARFS_HAVE_LIBLZMA
-    "lzma",
-#endif
-#ifdef DWARFS_HAVE_LIBBROTLI
-    "brotli",
-#endif
-#ifdef DWARFS_HAVE_FLAC
-    "flac",
-#endif
-};
 
 } // namespace
 
-std::set<std::string> get_unsupported_features(std::set<std::string> features) {
+void feature_set::add(feature f) { features_.insert(feature_name(f)); }
+
+std::set<std::string> feature_set::get_supported() {
   std::set<std::string> rv;
-  std::set_difference(features.begin(), features.end(),
+  for (auto f : apache::thrift::TEnumTraits<feature>::values) {
-                      supported_features.begin(), supported_features.end(),
+    if (is_supported_feature(f)) {
-                      std::inserter(rv, rv.end()));
+      rv.insert(feature_name(f));
+    }
+  }
   return rv;
+};
+
+std::set<std::string>
+feature_set::get_unsupported(std::set<std::string> wanted_features) {
+  auto const supported_features = get_supported();
+  std::set<std::string> missing;
+  std::set_difference(wanted_features.begin(), wanted_features.end(),
+                      supported_features.begin(), supported_features.end(),
+                      std::inserter(missing, missing.end()));
+  return missing;
 }
 
 } // namespace dwarfs

src/dwarfs/metadata_v2.cpp

@@ -141,7 +141,7 @@ map_frozen(std::span<uint8_t const> schema, std::span<uint8_t const> data) {
 MappedFrozen<thrift::metadata::metadata>
 check_frozen(MappedFrozen<thrift::metadata::metadata> meta) {
   if (meta.features()) {
-    auto unsupported = get_unsupported_features(meta.features()->thaw());
+    auto unsupported = feature_set::get_unsupported(meta.features()->thaw());
     if (!unsupported.empty()) {
       DWARFS_THROW(runtime_error,
                    fmt::format("file system uses the following features "

thrift/features.thrift

@@ -0,0 +1,34 @@
+/* vim:set ts=2 sw=2 sts=2 et: */
+/**
+ * \author     Marcus Holland-Moritz (github@mhxnet.de)
+ * \copyright  Copyright (c) Marcus Holland-Moritz
+ *
+ * This file is part of dwarfs.
+ *
+ * dwarfs is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * dwarfs is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with dwarfs.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+namespace cpp2 dwarfs
+
+// It is actually ok to change the values of the enumerators,
+// to add new enumerators, or to remove existing enumerators.
+// However, *never* change the name of an enumerator, because
+// the stringified name is used to serialize feature sets in
+// the metadata. Also, *never* reuse an enumerator name, for
+// the same reason. Be extra careful when removing enumerators,
+// as this will break compatibility with older metadata using
+// the feature defined by the removed enumerator.
+enum feature {
+  // There are no features yet :-)
+}