diff --git a/CMakeLists.txt b/CMakeLists.txt index fe1097b2..02d62159 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -748,6 +748,27 @@ list( ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_visit_union.h ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_visitation.h) +list( + APPEND + FEATURES_THRIFT_SRC + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_clients.h + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_constants.cpp + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_constants.h + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_data.cpp + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_data.h + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_for_each_field.h + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_handlers.h + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_metadata.cpp + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_metadata.h + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_types.cpp + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_types.h + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_types.tcc + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_types_custom_protocol.h + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_types_fwd.h + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_visit_by_thrift_field_metadata.h + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_visit_union.h + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_visitation.h) + add_custom_command( OUTPUT thrift/lib/thrift/_keep COMMAND ${CMAKE_COMMAND} -E make_directory thrift/lib/thrift @@ -824,6 +845,21 @@ add_custom_command( WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs ) +add_custom_command( + OUTPUT ${FEATURES_THRIFT_SRC} + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_SOURCE_DIR}/thrift/features.thrift + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/features.thrift + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/bin/thrift1 + -o ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs + --gen mstch_cpp2 + features.thrift + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/bin/thrift1 + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/_keep + ${CMAKE_CURRENT_SOURCE_DIR}/thrift/features.thrift + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs +) + list( APPEND INCLUDE_DIRS @@ -890,17 +926,25 @@ add_library( ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_types.cpp ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_data.cpp) +add_library( + features_thrift + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_types.cpp + ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_data.cpp) + set_property(TARGET metadata_thrift PROPERTY CXX_STANDARD 20) set_property(TARGET compression_thrift PROPERTY CXX_STANDARD 20) set_property(TARGET history_thrift PROPERTY CXX_STANDARD 20) +set_property(TARGET features_thrift PROPERTY CXX_STANDARD 20) target_include_directories(metadata_thrift PRIVATE ${INCLUDE_DIRS}) target_include_directories(compression_thrift PRIVATE ${INCLUDE_DIRS}) target_include_directories(history_thrift PRIVATE ${INCLUDE_DIRS}) +target_include_directories(features_thrift PRIVATE ${INCLUDE_DIRS}) target_link_libraries(metadata_thrift thrift_light) target_link_libraries(compression_thrift thrift_light) target_link_libraries(history_thrift thrift_light) +target_link_libraries(features_thrift thrift_light) foreach(tgt dwarfs dwarfs_compression dwarfs_categorizer dwarfs_compression_metadata dwarfs_tool @@ -992,7 +1036,7 @@ target_link_libraries( dwarfs metadata_thrift history_thrift - thrift_light + features_thrift folly fsst ${Boost_LIBRARIES}) diff --git a/include/dwarfs/features.h b/include/dwarfs/features.h index 83a787ee..2e6a5399 100644 --- a/include/dwarfs/features.h +++ b/include/dwarfs/features.h @@ -24,8 +24,21 @@ #include #include +#include "dwarfs/gen-cpp2/features_types.h" + namespace dwarfs { -std::set get_unsupported_features(std::set features); +class feature_set { + public: + static std::set get_supported(); + static std::set get_unsupported(std::set features); + + void add(feature f); + + std::set const& get() const { return features_; } + + private: + std::set features_; +}; } // namespace dwarfs diff --git a/src/dwarfs/features.cpp b/src/dwarfs/features.cpp index 6a309171..9674a4e0 100644 --- a/src/dwarfs/features.cpp +++ b/src/dwarfs/features.cpp @@ -22,38 +22,42 @@ #include #include +#include + #include "dwarfs/features.h" namespace dwarfs { namespace { -std::set supported_features{ -#ifdef DWARFS_HAVE_LIBZSTD - "zstd", -#endif -#ifdef DWARFS_HAVE_LIBLZ4 - "lz4", -#endif -#ifdef DWARFS_HAVE_LIBLZMA - "lzma", -#endif -#ifdef DWARFS_HAVE_LIBBROTLI - "brotli", -#endif -#ifdef DWARFS_HAVE_FLAC - "flac", -#endif -}; +constexpr bool is_supported_feature(feature /*f*/) { return true; } + +std::string feature_name(feature f) { + return apache::thrift::util::enumNameOrThrow(f); +} } // namespace -std::set get_unsupported_features(std::set features) { +void feature_set::add(feature f) { features_.insert(feature_name(f)); } + +std::set feature_set::get_supported() { std::set rv; - std::set_difference(features.begin(), features.end(), - supported_features.begin(), supported_features.end(), - std::inserter(rv, rv.end())); + for (auto f : apache::thrift::TEnumTraits::values) { + if (is_supported_feature(f)) { + rv.insert(feature_name(f)); + } + } return rv; +}; + +std::set +feature_set::get_unsupported(std::set wanted_features) { + auto const supported_features = get_supported(); + std::set missing; + std::set_difference(wanted_features.begin(), wanted_features.end(), + supported_features.begin(), supported_features.end(), + std::inserter(missing, missing.end())); + return missing; } } // namespace dwarfs diff --git a/src/dwarfs/metadata_v2.cpp b/src/dwarfs/metadata_v2.cpp index 181d7ffa..11642e21 100644 --- a/src/dwarfs/metadata_v2.cpp +++ b/src/dwarfs/metadata_v2.cpp @@ -141,7 +141,7 @@ map_frozen(std::span schema, std::span data) { MappedFrozen check_frozen(MappedFrozen meta) { if (meta.features()) { - auto unsupported = get_unsupported_features(meta.features()->thaw()); + auto unsupported = feature_set::get_unsupported(meta.features()->thaw()); if (!unsupported.empty()) { DWARFS_THROW(runtime_error, fmt::format("file system uses the following features " diff --git a/thrift/features.thrift b/thrift/features.thrift new file mode 100644 index 00000000..b8b10911 --- /dev/null +++ b/thrift/features.thrift @@ -0,0 +1,34 @@ +/* vim:set ts=2 sw=2 sts=2 et: */ +/** + * \author Marcus Holland-Moritz (github@mhxnet.de) + * \copyright Copyright (c) Marcus Holland-Moritz + * + * This file is part of dwarfs. + * + * dwarfs is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * dwarfs is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with dwarfs. If not, see . + */ + +namespace cpp2 dwarfs + +// It is actually ok to change the values of the enumerators, +// to add new enumerators, or to remove existing enumerators. +// However, *never* change the name of an enumerator, because +// the stringified name is used to serialize feature sets in +// the metadata. Also, *never* reuse an enumerator name, for +// the same reason. Be extra careful when removing enumerators, +// as this will break compatibility with older metadata using +// the feature defined by the removed enumerator. +enum feature { + // There are no features yet :-) +}