mirror of
https://github.com/mhx/dwarfs.git
synced 2025-09-08 03:49:44 -04:00
feat(features): use thrift feature enum and implement feature_set
This commit is contained in:
Marcus Holland-Moritz
parent
88425dd38f
commit
cb022d3dbc
@ -748,6 +748,27 @@ list(
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_visit_union.h
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_visitation.h)
|
||||
|
||||
list(
|
||||
APPEND
|
||||
FEATURES_THRIFT_SRC
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_clients.h
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_constants.cpp
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_constants.h
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_data.cpp
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_data.h
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_for_each_field.h
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_handlers.h
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_metadata.cpp
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_metadata.h
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_types.cpp
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_types.h
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_types.tcc
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_types_custom_protocol.h
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_types_fwd.h
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_visit_by_thrift_field_metadata.h
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_visit_union.h
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_visitation.h)
|
||||
|
||||
add_custom_command(
|
||||
OUTPUT thrift/lib/thrift/_keep
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory thrift/lib/thrift
|
||||
@ -824,6 +845,21 @@ add_custom_command(
|
||||
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs
|
||||
)
|
||||
|
||||
add_custom_command(
|
||||
OUTPUT ${FEATURES_THRIFT_SRC}
|
||||
COMMAND ${CMAKE_COMMAND} -E copy
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/thrift/features.thrift
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/features.thrift
|
||||
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/bin/thrift1
|
||||
-o ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs
|
||||
--gen mstch_cpp2
|
||||
features.thrift
|
||||
DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/bin/thrift1
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/_keep
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/thrift/features.thrift
|
||||
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs
|
||||
)
|
||||
|
||||
list(
|
||||
APPEND
|
||||
INCLUDE_DIRS
|
||||
@ -890,17 +926,25 @@ add_library(
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_types.cpp
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/history_data.cpp)
|
||||
|
||||
add_library(
|
||||
features_thrift
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_types.cpp
|
||||
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/features_data.cpp)
|
||||
|
||||
set_property(TARGET metadata_thrift PROPERTY CXX_STANDARD 20)
|
||||
set_property(TARGET compression_thrift PROPERTY CXX_STANDARD 20)
|
||||
set_property(TARGET history_thrift PROPERTY CXX_STANDARD 20)
|
||||
set_property(TARGET features_thrift PROPERTY CXX_STANDARD 20)
|
||||
|
||||
target_include_directories(metadata_thrift PRIVATE ${INCLUDE_DIRS})
|
||||
target_include_directories(compression_thrift PRIVATE ${INCLUDE_DIRS})
|
||||
target_include_directories(history_thrift PRIVATE ${INCLUDE_DIRS})
|
||||
target_include_directories(features_thrift PRIVATE ${INCLUDE_DIRS})
|
||||
|
||||
target_link_libraries(metadata_thrift thrift_light)
|
||||
target_link_libraries(compression_thrift thrift_light)
|
||||
target_link_libraries(history_thrift thrift_light)
|
||||
target_link_libraries(features_thrift thrift_light)
|
||||
|
||||
foreach(tgt dwarfs dwarfs_compression dwarfs_categorizer
|
||||
dwarfs_compression_metadata dwarfs_tool
|
||||
@ -992,7 +1036,7 @@ target_link_libraries(
|
||||
dwarfs
|
||||
metadata_thrift
|
||||
history_thrift
|
||||
thrift_light
|
||||
features_thrift
|
||||
folly
|
||||
fsst
|
||||
${Boost_LIBRARIES})
|
||||
|
||||
@ -24,8 +24,21 @@
|
||||
#include <set>
|
||||
#include <string>
|
||||
|
||||
#include "dwarfs/gen-cpp2/features_types.h"
|
||||
|
||||
namespace dwarfs {
|
||||
|
||||
std::set<std::string> get_unsupported_features(std::set<std::string> features);
|
||||
class feature_set {
|
||||
public:
|
||||
static std::set<std::string> get_supported();
|
||||
static std::set<std::string> get_unsupported(std::set<std::string> features);
|
||||
|
||||
void add(feature f);
|
||||
|
||||
std::set<std::string> const& get() const { return features_; }
|
||||
|
||||
private:
|
||||
std::set<std::string> features_;
|
||||
};
|
||||
|
||||
} // namespace dwarfs
|
||||
|
||||
@ -22,38 +22,42 @@
|
||||
#include <algorithm>
|
||||
#include <iterator>
|
||||
|
||||
#include <thrift/lib/cpp/util/EnumUtils.h>
|
||||
|
||||
#include "dwarfs/features.h"
|
||||
|
||||
namespace dwarfs {
|
||||
|
||||
namespace {
|
||||
|
||||
std::set<std::string> supported_features{
|
||||
#ifdef DWARFS_HAVE_LIBZSTD
|
||||
"zstd",
|
||||
#endif
|
||||
#ifdef DWARFS_HAVE_LIBLZ4
|
||||
"lz4",
|
||||
#endif
|
||||
#ifdef DWARFS_HAVE_LIBLZMA
|
||||
"lzma",
|
||||
#endif
|
||||
#ifdef DWARFS_HAVE_LIBBROTLI
|
||||
"brotli",
|
||||
#endif
|
||||
#ifdef DWARFS_HAVE_FLAC
|
||||
"flac",
|
||||
#endif
|
||||
};
|
||||
constexpr bool is_supported_feature(feature /*f*/) { return true; }
|
||||
|
||||
std::string feature_name(feature f) {
|
||||
return apache::thrift::util::enumNameOrThrow(f);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::set<std::string> get_unsupported_features(std::set<std::string> features) {
|
||||
void feature_set::add(feature f) { features_.insert(feature_name(f)); }
|
||||
|
||||
std::set<std::string> feature_set::get_supported() {
|
||||
std::set<std::string> rv;
|
||||
std::set_difference(features.begin(), features.end(),
|
||||
supported_features.begin(), supported_features.end(),
|
||||
std::inserter(rv, rv.end()));
|
||||
for (auto f : apache::thrift::TEnumTraits<feature>::values) {
|
||||
if (is_supported_feature(f)) {
|
||||
rv.insert(feature_name(f));
|
||||
}
|
||||
}
|
||||
return rv;
|
||||
};
|
||||
|
||||
std::set<std::string>
|
||||
feature_set::get_unsupported(std::set<std::string> wanted_features) {
|
||||
auto const supported_features = get_supported();
|
||||
std::set<std::string> missing;
|
||||
std::set_difference(wanted_features.begin(), wanted_features.end(),
|
||||
supported_features.begin(), supported_features.end(),
|
||||
std::inserter(missing, missing.end()));
|
||||
return missing;
|
||||
}
|
||||
|
||||
} // namespace dwarfs
|
||||
|
||||
@ -141,7 +141,7 @@ map_frozen(std::span<uint8_t const> schema, std::span<uint8_t const> data) {
|
||||
MappedFrozen<thrift::metadata::metadata>
|
||||
check_frozen(MappedFrozen<thrift::metadata::metadata> meta) {
|
||||
if (meta.features()) {
|
||||
auto unsupported = get_unsupported_features(meta.features()->thaw());
|
||||
auto unsupported = feature_set::get_unsupported(meta.features()->thaw());
|
||||
if (!unsupported.empty()) {
|
||||
DWARFS_THROW(runtime_error,
|
||||
fmt::format("file system uses the following features "
|
||||
|
||||
34
thrift/features.thrift
Normal file
34
thrift/features.thrift
Normal file
@ -0,0 +1,34 @@
|
||||
/* vim:set ts=2 sw=2 sts=2 et: */
|
||||
/**
|
||||
* \author Marcus Holland-Moritz (github@mhxnet.de)
|
||||
* \copyright Copyright (c) Marcus Holland-Moritz
|
||||
*
|
||||
* This file is part of dwarfs.
|
||||
*
|
||||
* dwarfs is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* dwarfs is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with dwarfs. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
namespace cpp2 dwarfs
|
||||
|
||||
// It is actually ok to change the values of the enumerators,
|
||||
// to add new enumerators, or to remove existing enumerators.
|
||||
// However, *never* change the name of an enumerator, because
|
||||
// the stringified name is used to serialize feature sets in
|
||||
// the metadata. Also, *never* reuse an enumerator name, for
|
||||
// the same reason. Be extra careful when removing enumerators,
|
||||
// as this will break compatibility with older metadata using
|
||||
// the feature defined by the removed enumerator.
|
||||
enum feature {
|
||||
// There are no features yet :-)
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user