From c005b841c0f712b36d9f96e130713419c02bbfdb Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Thu, 14 Dec 2023 09:10:38 +0100 Subject: [PATCH] chore(metadata-requirements): support folly::dynamic requirements --- CMakeLists.txt | 5 + .../compression_metadata_requirements.h | 37 +++- .../compression_metadata_requirements.cpp | 186 +++++++++++++++++- test/metadata_requirements_test.cpp | 93 +++++++++ test/pcmaudio_categorizer_test.cpp | 2 +- 5 files changed, 315 insertions(+), 8 deletions(-) create mode 100644 test/metadata_requirements_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 18ae5a70..4a98a067 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -619,6 +619,11 @@ if(WITH_TESTS) target_link_libraries(block_merger_test gtest gtest_main) list(APPEND TEST_TARGETS block_merger_test) + add_executable(dwarfs_metadata_requirements_test test/metadata_requirements_test.cpp) + target_link_libraries(dwarfs_metadata_requirements_test test_helpers + gtest gtest_main gmock_main) + list(APPEND TEST_TARGETS dwarfs_metadata_requirements_test) + add_executable(dwarfs_pcm_sample_transformer_test test/pcm_sample_transformer_test.cpp) target_link_libraries(dwarfs_pcm_sample_transformer_test gtest gtest_main) list(APPEND TEST_TARGETS dwarfs_pcm_sample_transformer_test) diff --git a/include/dwarfs/compression_metadata_requirements.h b/include/dwarfs/compression_metadata_requirements.h index c1188f50..9892d0d5 100644 --- a/include/dwarfs/compression_metadata_requirements.h +++ b/include/dwarfs/compression_metadata_requirements.h @@ -22,6 +22,7 @@ #pragma once #include +#include #include #include #include @@ -61,7 +62,8 @@ bool parse_metadata_requirements_set(T& container, folly::dynamic& req, if (it->second[1].type() != folly::dynamic::ARRAY) { throw std::runtime_error( - fmt::format("non-array type argument for requirement '{}'", name)); + fmt::format("non-array type argument for requirement '{}', got '{}'", + name, it->second[1].typeName())); } for (auto v : it->second[1]) { @@ -138,6 +140,21 @@ class checked_metadata_requirement_base : public metadata_requirement_base { virtual void check(Meta const& m) const = 0; }; +class dynamic_metadata_requirement_base { + public: + virtual ~dynamic_metadata_requirement_base() = default; + + dynamic_metadata_requirement_base(std::string const& name) + : name_{name} {} + + virtual void check(folly::dynamic const& m) const = 0; + + std::string_view name() const { return name_; } + + private: + std::string const name_; +}; + template class typed_metadata_requirement_base : public checked_metadata_requirement_base { @@ -188,8 +205,9 @@ class metadata_requirement_set protected: void check_value(T const& value) const override { if (set_ && set_->count(value) == 0) { - throw std::range_error(fmt::format("{} '{}' does not meet requirements", - this->name(), value)); + throw std::range_error( + fmt::format("{} '{}' does not meet requirements [{}]", this->name(), + value, fmt::join(*set_, ", "))); } } @@ -288,4 +306,17 @@ class compression_metadata_requirements { } }; +template <> +class compression_metadata_requirements { + public: + compression_metadata_requirements(std::string const& req); + compression_metadata_requirements(folly::dynamic const& req); + + void check(std::string const& meta) const; + void check(folly::dynamic const& meta) const; + + private: + std::vector> req_; +}; + } // namespace dwarfs diff --git a/src/dwarfs/compression_metadata_requirements.cpp b/src/dwarfs/compression_metadata_requirements.cpp index 02b4b44a..9e4ee72b 100644 --- a/src/dwarfs/compression_metadata_requirements.cpp +++ b/src/dwarfs/compression_metadata_requirements.cpp @@ -21,16 +21,21 @@ #include +#include + #include "dwarfs/compression_metadata_requirements.h" -namespace dwarfs::detail { +namespace dwarfs { + +namespace detail { void check_dynamic_common(folly::dynamic const& dyn, std::string_view expected_type, size_t expected_size, std::string_view name) { if (dyn.type() != folly::dynamic::ARRAY) { throw std::runtime_error( - fmt::format("found non-array type for requirement '{}'", name)); + fmt::format("found non-array type for requirement '{}', got type '{}'", + name, dyn.typeName())); } if (dyn.empty()) { throw std::runtime_error( @@ -56,8 +61,181 @@ void check_unsupported_metadata_requirements(folly::dynamic& req) { } std::sort(keys.begin(), keys.end()); throw std::runtime_error(fmt::format( - "unsupported metadata requirements: {}", folly::join(", ", keys))); + "unsupported metadata requirements: {}", fmt::join(keys, ", "))); } } -} // namespace dwarfs::detail +template +class dynamic_metadata_requirement_set + : public dynamic_metadata_requirement_base { + public: + static_assert(std::is_same_v || std::is_integral_v); + + dynamic_metadata_requirement_set(std::string const& name, + folly::dynamic const& req) + : dynamic_metadata_requirement_base{name} { + auto tmp = req; + if (!parse_metadata_requirements_set(set_, tmp, name, + detail::value_parser)) { + throw std::runtime_error( + fmt::format("could not parse set requirement '{}'", name)); + } + } + + void check(folly::dynamic const& dyn) const override { + if constexpr (std::is_same_v) { + if (!dyn.isString()) { + throw std::runtime_error( + fmt::format("non-string type for requirement '{}', got type '{}'", + name(), dyn.typeName())); + } + + if (set_.find(dyn.asString()) == set_.end()) { + throw std::runtime_error( + fmt::format("{} '{}' does not meet requirements [{}]", name(), + dyn.asString(), fmt::join(ordered_set(), ", "))); + } + } else { + if (!dyn.isInt()) { + throw std::runtime_error( + fmt::format("non-integral type for requirement '{}', got type '{}'", + name(), dyn.typeName())); + } + + if (set_.find(dyn.asInt()) == set_.end()) { + throw std::runtime_error( + fmt::format("{} '{}' does not meet requirements [{}]", name(), + dyn.asInt(), fmt::join(ordered_set(), ", "))); + } + } + } + + private: + std::vector ordered_set() const { + std::vector result; + result.reserve(set_.size()); + std::copy(set_.begin(), set_.end(), std::back_inserter(result)); + std::sort(result.begin(), result.end()); + return result; + } + + std::unordered_set set_; +}; + +class dynamic_metadata_requirement_range + : public dynamic_metadata_requirement_base { + public: + dynamic_metadata_requirement_range(std::string const& name, + folly::dynamic const& req) + : dynamic_metadata_requirement_base{name} { + auto tmp = req; + if (!parse_metadata_requirements_range(min_, max_, tmp, name, + detail::value_parser)) { + throw std::runtime_error( + fmt::format("could not parse range requirement '{}'", name)); + } + } + + void check(folly::dynamic const& dyn) const override { + if (!dyn.isInt()) { + throw std::runtime_error( + fmt::format("non-integral type for requirement '{}', got type '{}'", + name(), dyn.typeName())); + } + + auto v = dyn.asInt(); + + if (v < min_ || v > max_) { + throw std::runtime_error( + fmt::format("{} '{}' does not meet requirements [{}, {}]", name(), v, + min_, max_)); + } + } + + private: + int64_t min_, max_; +}; + +} // namespace detail + +compression_metadata_requirements< + folly::dynamic>::compression_metadata_requirements(std::string const& req) + : compression_metadata_requirements(folly::parseJson(req)) {} + +compression_metadata_requirements:: + compression_metadata_requirements(folly::dynamic const& req) { + if (req.type() != folly::dynamic::OBJECT) { + throw std::runtime_error( + fmt::format("metadata requirements must be an object, got type '{}'", + req.typeName())); + } + + for (auto const& [k, v] : req.items()) { + if (v.type() != folly::dynamic::ARRAY) { + throw std::runtime_error( + fmt::format("requirement '{}' must be an array, got type '{}'", + k.asString(), v.typeName())); + } + + if (v.size() < 2) { + throw std::runtime_error( + fmt::format("requirement '{}' must be an array of at least 2 " + "elements, got only {}", + k.asString(), v.size())); + } + + if (v[0].type() != folly::dynamic::STRING) { + throw std::runtime_error(fmt::format( + "type for requirement '{}' must be a string, got type '{}'", + k.asString(), v[0].typeName())); + } + + if (v[0].asString() == "set") { + if (v[1].type() != folly::dynamic::ARRAY) { + throw std::runtime_error(fmt::format( + "set for requirement '{}' must be an array, got type '{}'", + k.asString(), v[1].typeName())); + } + if (v[1].empty()) { + throw std::runtime_error(fmt::format( + "set for requirement '{}' must not be empty", k.asString())); + } + if (v[1][0].isString()) { + req_.emplace_back( + std::make_unique< + detail::dynamic_metadata_requirement_set>( + k.asString(), req)); + } else { + req_.emplace_back( + std::make_unique>( + k.asString(), req)); + } + } else if (v[0].asString() == "range") { + req_.emplace_back( + std::make_unique( + k.asString(), req)); + } else { + throw std::runtime_error( + fmt::format("unsupported requirement type '{}'", v[0].asString())); + } + } +} + +void compression_metadata_requirements::check( + folly::dynamic const& dyn) const { + for (auto const& r : req_) { + if (auto it = dyn.find(r->name()); it != dyn.items().end()) { + r->check(it->second); + } else { + throw std::runtime_error( + fmt::format("missing requirement '{}'", r->name())); + } + } +} + +void compression_metadata_requirements::check( + std::string const& metadata) const { + check(folly::parseJson(metadata)); +} + +} // namespace dwarfs diff --git a/test/metadata_requirements_test.cpp b/test/metadata_requirements_test.cpp new file mode 100644 index 00000000..3c47d5ae --- /dev/null +++ b/test/metadata_requirements_test.cpp @@ -0,0 +1,93 @@ +/* vim:set ts=2 sw=2 sts=2 et: */ +/** + * \author Marcus Holland-Moritz (github@mhxnet.de) + * \copyright Copyright (c) Marcus Holland-Moritz + * + * This file is part of dwarfs. + * + * dwarfs is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * dwarfs is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with dwarfs. If not, see . + */ + +#include + +#include +#include + +#include "dwarfs/compression_metadata_requirements.h" + +using namespace dwarfs; + +TEST(metadata_requirements, dynamic_test) { + std::string requirements = R"({ + "compression": ["set", ["lz4", "zstd"]], + "block_size": ["range", 16, 1024] + })"; + + std::unique_ptr> req; + + ASSERT_NO_THROW( + req = std::make_unique>( + requirements)); + { + std::string metadata = R"({ + "compression": "lz4", + "block_size": 256 + })"; + + EXPECT_NO_THROW(req->check(metadata)); + } + + { + std::string metadata = R"({ + "compression": "lz4", + "foo": "bar", + "block_size": 256 + })"; + + EXPECT_NO_THROW(req->check(metadata)); + } + + { + std::string metadata = R"({ + "compression": "lzma", + "block_size": 256 + })"; + + EXPECT_THAT( + [&]() { req->check(metadata); }, + ThrowsMessage(testing::HasSubstr( + "compression 'lzma' does not meet requirements [lz4, zstd]"))); + } + + { + std::string metadata = R"({ + "block_size": 256 + })"; + + EXPECT_THAT([&]() { req->check(metadata); }, + ThrowsMessage( + testing::HasSubstr("missing requirement 'compression'"))); + } + + { + std::string metadata = R"({ + "compression": "zstd", + "block_size": 8 + })"; + + EXPECT_THAT([&]() { req->check(metadata); }, + ThrowsMessage(testing::HasSubstr( + "block_size '8' does not meet requirements [16, 1024]"))); + } +} diff --git a/test/pcmaudio_categorizer_test.cpp b/test/pcmaudio_categorizer_test.cpp index c2f1d882..b27c3a4f 100644 --- a/test/pcmaudio_categorizer_test.cpp +++ b/test/pcmaudio_categorizer_test.cpp @@ -85,7 +85,7 @@ TEST(pcmaudio_categorizer, requirements) { EXPECT_THAT( ent.output, MatchesRegex( - R"(^\[WAV\] ".*": endianness 'little' does not meet requirements$)")); + R"(^\[WAV\] ".*": endianness 'little' does not meet requirements \[\]$)")); EXPECT_TRUE(frag.empty());