diff --git a/CMakeLists.txt b/CMakeLists.txt index 4184923c..5c8cd2a8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -604,6 +604,7 @@ list(APPEND LIBDWARFS_COMMON_SRC src/dwarfs/block_compressor.cpp src/dwarfs/block_compressor_parser.cpp src/dwarfs/checksum.cpp + src/dwarfs/compression_metadata_requirements.cpp src/dwarfs/conv.cpp src/dwarfs/error.cpp src/dwarfs/file_access_generic.cpp @@ -621,6 +622,7 @@ list(APPEND LIBDWARFS_COMMON_SRC src/dwarfs/option_map.cpp src/dwarfs/options.cpp src/dwarfs/os_access_generic.cpp + src/dwarfs/pcm_sample_transformer.cpp src/dwarfs/performance_monitor.cpp src/dwarfs/terminal.cpp src/dwarfs/thread_pool.cpp @@ -761,25 +763,17 @@ endif() # list(APPEND LIBDWARFS_CATEGORIZER_SRC src/dwarfs/categorizer/libmagic_categorizer.cpp) # endif() -add_library(dwarfs_common ${LIBDWARFS_COMMON_SRC}) +add_library(dwarfs_common ${LIBDWARFS_COMMON_SRC} ${LIBDWARFS_COMPRESSION_SRC}) add_library(dwarfs_reader ${LIBDWARFS_READER_SRC}) -add_library(dwarfs_writer ${LIBDWARFS_WRITER_SRC}) +add_library(dwarfs_writer ${LIBDWARFS_WRITER_SRC} ${LIBDWARFS_CATEGORIZER_SRC}) add_library(dwarfs_extractor ${LIBDWARFS_EXTRACTOR_SRC}) -add_library(dwarfs_compression ${LIBDWARFS_COMPRESSION_SRC}) -add_library(dwarfs_categorizer ${LIBDWARFS_CATEGORIZER_SRC}) -add_library(dwarfs_compression_metadata src/dwarfs/compression_metadata_requirements.cpp) -add_library(dwarfs_pcm_sample_transformer src/dwarfs/pcm_sample_transformer.cpp) add_library(dwarfs_tool ${LIBDWARFS_TOOL_SRC}) -target_link_libraries(dwarfs_compression_metadata PUBLIC dwarfs_folly_lite) -target_link_libraries(dwarfs_pcm_sample_transformer PUBLIC dwarfs_folly_lite) -target_link_libraries(dwarfs_categorizer PUBLIC dwarfs_compression_metadata) -target_link_libraries(dwarfs_categorizer PRIVATE range-v3::range-v3) target_link_libraries(dwarfs_common PUBLIC dwarfs_folly_lite) target_link_libraries(dwarfs_common PRIVATE PkgConfig::LIBCRYPTO xxHash::xxhash phmap range-v3::range-v3) target_link_libraries(dwarfs_reader PUBLIC dwarfs_common) target_link_libraries(dwarfs_reader PRIVATE range-v3::range-v3) -target_link_libraries(dwarfs_writer PUBLIC dwarfs_common dwarfs_compression_metadata) +target_link_libraries(dwarfs_writer PUBLIC dwarfs_common) target_link_libraries(dwarfs_writer PRIVATE phmap range-v3::range-v3) target_link_libraries(dwarfs_extractor PUBLIC dwarfs_reader) target_link_libraries(dwarfs_tool PUBLIC dwarfs_common) @@ -798,10 +792,8 @@ target_compile_definitions( PRJ_COMPILER_ID="${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}" ) -target_link_libraries(dwarfs_categorizer PUBLIC dwarfs_folly_lite range-v3::range-v3) -target_link_libraries(dwarfs_compression PUBLIC dwarfs_common dwarfs_compression_thrift dwarfs_pcm_sample_transformer) if(ENABLE_RICEPP) - target_link_libraries(dwarfs_compression PRIVATE ricepp) + target_link_libraries(dwarfs_common PRIVATE ricepp) endif() target_link_libraries(dwarfs_tool PUBLIC dwarfs_common) @@ -988,7 +980,7 @@ if(WITH_TESTS) incompressible_categorizer_test pcmaudio_categorizer_test tool_main_test) - target_link_libraries(${tgt} PRIVATE "$") + target_link_libraries(${tgt} PRIVATE dwarfs_writer) endforeach() foreach(tgt ${TEST_TARGETS}) @@ -1028,7 +1020,7 @@ endif() if(WITH_FUZZ) add_executable(fuzz_categorizers test/fuzz_categorizers.cpp) - target_link_libraries(fuzz_categorizers PRIVATE "$") + target_link_libraries(fuzz_categorizers PRIVATE dwarfs_writer) list(APPEND BINARY_TARGETS fuzz_categorizers) add_executable(fuzz_mkdwarfs test/fuzz_mkdwarfs.cpp) @@ -1092,11 +1084,7 @@ add_cpp2_thrift_library(thrift/history.thrift add_cpp2_thrift_library(thrift/features.thrift TARGET dwarfs_features_thrift OUTPUT_PATH dwarfs) -target_link_libraries(dwarfs_categorizer PRIVATE dwarfs_metadata_thrift) - -foreach(tgt dwarfs_common dwarfs_reader dwarfs_writer dwarfs_extractor - dwarfs_compression dwarfs_categorizer dwarfs_tool - dwarfs_compression_metadata dwarfs_pcm_sample_transformer +foreach(tgt dwarfs_common dwarfs_reader dwarfs_writer dwarfs_extractor dwarfs_tool ${BINARY_TARGETS} ${TEST_TARGETS} ${MAIN_TARGETS}) set_target_properties(${tgt} PROPERTIES EXPORT_COMPILE_COMMANDS ON) @@ -1206,6 +1194,7 @@ target_link_libraries( Boost::boost Boost::chrono Boost::iostreams + dwarfs_compression_thrift dwarfs_metadata_thrift dwarfs_history_thrift dwarfs_features_thrift @@ -1216,28 +1205,28 @@ target_link_libraries( # target_link_libraries(dwarfs PkgConfig::LIBMAGIC) # endif() -target_link_libraries(dwarfs_categorizer PRIVATE libzstd_static) -target_link_libraries(dwarfs_compression PRIVATE libzstd_static) +target_link_libraries(dwarfs_common PRIVATE libzstd_static) +target_link_libraries(dwarfs_writer PRIVATE libzstd_static) if(LIBLZ4_FOUND) - target_link_libraries(dwarfs_compression PRIVATE PkgConfig::LIBLZ4) + target_link_libraries(dwarfs_common PRIVATE PkgConfig::LIBLZ4) endif() if(LIBLZMA_FOUND) - target_link_libraries(dwarfs_compression PRIVATE PkgConfig::LIBLZMA) + target_link_libraries(dwarfs_common PRIVATE PkgConfig::LIBLZMA) endif() if(FLAC_FOUND) - target_link_libraries(dwarfs_compression PRIVATE PkgConfig::FLAC) + target_link_libraries(dwarfs_common PRIVATE PkgConfig::FLAC) endif() if(LIBBROTLIDEC_FOUND AND LIBBROTLIENC_FOUND) - target_link_libraries(dwarfs_compression PRIVATE PkgConfig::LIBBROTLIDEC PkgConfig::LIBBROTLIENC) + target_link_libraries(dwarfs_common PRIVATE PkgConfig::LIBBROTLIDEC PkgConfig::LIBBROTLIENC) endif() if(NOT STATIC_BUILD_DO_NOT_USE) target_link_libraries(dwarfs_extractor PUBLIC PkgConfig::LIBARCHIVE) - # target_link_libraries(dwarfs_categorizer PkgConfig::LIBMAGIC) + # target_link_libraries(dwarfs_writer PkgConfig::LIBMAGIC) endif(NOT STATIC_BUILD_DO_NOT_USE) if(DWARFS_USE_EXCEPTION_TRACER) @@ -1260,14 +1249,13 @@ endforeach() foreach(tgt ${BINARY_TARGETS} ${TEST_TARGETS}) target_link_libraries(${tgt} PRIVATE dwarfs_common) - target_link_libraries(${tgt} PRIVATE "$") if(DWARFS_USE_EXCEPTION_TRACER) target_link_libraries(${tgt} PRIVATE "$") endif() endforeach() -target_link_libraries(mkdwarfs_main PRIVATE dwarfs_reader dwarfs_writer "$") +target_link_libraries(mkdwarfs_main PRIVATE dwarfs_reader dwarfs_writer) target_link_libraries(dwarfsck_main PRIVATE dwarfs_reader) target_link_libraries(dwarfsextract_main PRIVATE dwarfs_extractor PkgConfig::LIBARCHIVE) target_link_libraries(dwarfsbench_main PRIVATE dwarfs_extractor) @@ -1278,7 +1266,7 @@ target_link_libraries(dwarfsextract PRIVATE dwarfsextract_main) target_link_libraries(dwarfsbench PRIVATE dwarfsbench_main) if(TARGET dwarfsuniversal) - target_link_libraries(dwarfsuniversal PRIVATE dwarfs_writer dwarfs_extractor "$") + target_link_libraries(dwarfsuniversal PRIVATE dwarfs_writer dwarfs_extractor) endif() if(TARGET dwarfs_main) target_link_libraries(dwarfs_main PRIVATE dwarfs_reader) @@ -1344,9 +1332,6 @@ if(STATIC_BUILD_DO_NOT_USE) set_target_properties(static_libmagic PROPERTIES INTERFACE_LINK_LIBRARIES static_libz) - target_link_libraries(dwarfs_categorizer PRIVATE static_libmagic) - target_link_libraries(dwarfs_compression PRIVATE static_libflac) - foreach(tgt ${BINARY_TARGETS} ${TEST_TARGETS}) if(PREFER_SYSTEM_LIBFMT) target_link_libraries( diff --git a/include/dwarfs/block_compressor.h b/include/dwarfs/block_compressor.h index ec727256..6edcb8e8 100644 --- a/include/dwarfs/block_compressor.h +++ b/include/dwarfs/block_compressor.h @@ -176,19 +176,13 @@ class compression_factory : public compression_info { namespace detail { -template -class compression_factory_registrar { - public: - compression_factory_registrar(compression_type type); -}; +template +struct compression_factory_registrar; } // namespace detail class compression_registry { public: - template - friend class detail::compression_factory_registrar; - static compression_registry& instance(); std::unique_ptr @@ -201,13 +195,13 @@ class compression_registry { std::function const& fn) const; + void register_factory(compression_type type, + std::unique_ptr&& factory); + private: compression_registry(); ~compression_registry(); - void register_factory(compression_type type, - std::unique_ptr&& factory); - std::unordered_map> factories_; @@ -216,19 +210,23 @@ class compression_registry { namespace detail { -template -compression_factory_registrar::compression_factory_registrar( - compression_type type) { - ::dwarfs::compression_registry::instance().register_factory( - type, std::make_unique()); -} +#define DWARFS_COMPRESSION_TYPE_ENUMERATION_(name, value) \ + template <> \ + struct compression_factory_registrar { \ + static void reg(compression_registry&); \ + }; +#define DWARFS_NO_SEPARATOR_ +DWARFS_COMPRESSION_TYPE_LIST(DWARFS_COMPRESSION_TYPE_ENUMERATION_, + DWARFS_NO_SEPARATOR_) +#undef DWARFS_COMPRESSION_TYPE_ENUMERATION_ +#undef DWARFS_NO_SEPARATOR_ } // namespace detail -#define REGISTER_COMPRESSION_FACTORY(type, factory) \ - namespace { \ - ::dwarfs::detail::compression_factory_registrar \ - the_##factory##_registrar(type); \ +#define REGISTER_COMPRESSION_FACTORY(factory) \ + void ::dwarfs::detail::compression_factory_registrar::reg( \ + ::dwarfs::compression_registry& cr) { \ + cr.register_factory(factory::type, std::make_unique()); \ } } // namespace dwarfs diff --git a/include/dwarfs/categorizer.h b/include/dwarfs/categorizer.h index 1ac5916a..6551119a 100644 --- a/include/dwarfs/categorizer.h +++ b/include/dwarfs/categorizer.h @@ -210,21 +210,8 @@ class categorizer_factory : public categorizer_info { boost::program_options::variables_map const& vm) const = 0; }; -namespace detail { - -template -class categorizer_factory_registrar { - public: - categorizer_factory_registrar(); -}; - -} // namespace detail - class categorizer_registry { public: - template - friend class detail::categorizer_factory_registrar; - static categorizer_registry& instance(); std::unique_ptr @@ -235,29 +222,29 @@ class categorizer_registry { std::vector categorizer_names() const; + void register_factory(std::unique_ptr&& factory); + private: categorizer_registry(); ~categorizer_registry(); - void register_factory(std::unique_ptr&& factory); - std::map> factories_; }; namespace detail { -template -categorizer_factory_registrar::categorizer_factory_registrar() { - ::dwarfs::categorizer_registry::instance().register_factory( - std::make_unique()); -} +void binary_categorizer_factory_registrar(categorizer_registry&); +void fits_categorizer_factory_registrar(categorizer_registry&); +void incompressible_categorizer_factory_registrar(categorizer_registry&); +void libmagic_categorizer_factory_registrar(categorizer_registry&); +void pcmaudio_categorizer_factory_registrar(categorizer_registry&); } // namespace detail #define REGISTER_CATEGORIZER_FACTORY(factory) \ - namespace { \ - ::dwarfs::detail::categorizer_factory_registrar \ - the_##factory##_registrar; \ + void ::dwarfs::detail::factory##_registrar( \ + ::dwarfs::categorizer_registry& cr) { \ + cr.register_factory(std::make_unique()); \ } } // namespace dwarfs diff --git a/src/dwarfs/block_compressor.cpp b/src/dwarfs/block_compressor.cpp index 5468feff..417f7273 100644 --- a/src/dwarfs/block_compressor.cpp +++ b/src/dwarfs/block_compressor.cpp @@ -115,7 +115,32 @@ void compression_registry::for_each_algorithm( } } -compression_registry::compression_registry() = default; +compression_registry::compression_registry() { + using namespace ::dwarfs::detail; + using enum compression_type; + + compression_factory_registrar::reg(*this); +#ifdef DWARFS_HAVE_LIBBROTLI + compression_factory_registrar::reg(*this); +#endif +#ifdef DWARFS_HAVE_FLAC + compression_factory_registrar::reg(*this); +#endif +#ifdef DWARFS_HAVE_LIBLZ4 + compression_factory_registrar::reg(*this); + compression_factory_registrar::reg(*this); +#endif +#ifdef DWARFS_HAVE_LIBLZMA + compression_factory_registrar::reg(*this); +#endif +#ifdef DWARFS_HAVE_RICEPP + compression_factory_registrar::reg(*this); +#endif +#ifdef DWARFS_HAVE_LIBZSTD + compression_factory_registrar::reg(*this); +#endif +} + compression_registry::~compression_registry() = default; } // namespace dwarfs diff --git a/src/dwarfs/categorizer.cpp b/src/dwarfs/categorizer.cpp index d2eb01c4..7632c739 100644 --- a/src/dwarfs/categorizer.cpp +++ b/src/dwarfs/categorizer.cpp @@ -406,7 +406,16 @@ std::vector categorizer_registry::categorizer_names() const { return rv; } -categorizer_registry::categorizer_registry() = default; +categorizer_registry::categorizer_registry() { + using namespace ::dwarfs::detail; + + // binary_categorizer_factory_registrar(*this); + fits_categorizer_factory_registrar(*this); + incompressible_categorizer_factory_registrar(*this); + // libmagic_categorizer_factory_registrar(*this); + pcmaudio_categorizer_factory_registrar(*this); +} + categorizer_registry::~categorizer_registry() = default; } // namespace dwarfs diff --git a/src/dwarfs/compression/brotli.cpp b/src/dwarfs/compression/brotli.cpp index 59fd7b8a..d06563f0 100644 --- a/src/dwarfs/compression/brotli.cpp +++ b/src/dwarfs/compression/brotli.cpp @@ -174,6 +174,8 @@ class brotli_block_decompressor final : public block_decompressor::impl { class brotli_compression_factory : public compression_factory { public: + static constexpr compression_type type{compression_type::BROTLI}; + brotli_compression_factory() : options_{ fmt::format("quality=[{}..{}]", BROTLI_MIN_QUALITY, @@ -224,7 +226,6 @@ class brotli_compression_factory : public compression_factory { } // namespace -REGISTER_COMPRESSION_FACTORY(compression_type::BROTLI, - brotli_compression_factory) +REGISTER_COMPRESSION_FACTORY(brotli_compression_factory) } // namespace dwarfs diff --git a/src/dwarfs/compression/flac.cpp b/src/dwarfs/compression/flac.cpp index e9758bea..fc01381b 100644 --- a/src/dwarfs/compression/flac.cpp +++ b/src/dwarfs/compression/flac.cpp @@ -485,6 +485,8 @@ class flac_block_decompressor final : public block_decompressor::impl { class flac_compression_factory : public compression_factory { public: + static constexpr compression_type type{compression_type::FLAC}; + flac_compression_factory() : options_{ fmt::format("level=[0..8]"), @@ -524,6 +526,6 @@ class flac_compression_factory : public compression_factory { } // namespace -REGISTER_COMPRESSION_FACTORY(compression_type::FLAC, flac_compression_factory) +REGISTER_COMPRESSION_FACTORY(flac_compression_factory) } // namespace dwarfs diff --git a/src/dwarfs/compression/lz4.cpp b/src/dwarfs/compression/lz4.cpp index dd324f3a..cc9214fa 100644 --- a/src/dwarfs/compression/lz4.cpp +++ b/src/dwarfs/compression/lz4.cpp @@ -168,6 +168,8 @@ class lz4_block_decompressor final : public block_decompressor::impl { class lz4_compression_factory : public compression_factory { public: + static constexpr compression_type type{compression_type::LZ4}; + std::string_view name() const override { return "lz4"; } std::string_view description() const override { @@ -200,6 +202,8 @@ class lz4_compression_factory : public compression_factory { class lz4hc_compression_factory : public compression_factory { public: + static constexpr compression_type type{compression_type::LZ4HC}; + lz4hc_compression_factory() : options_{fmt::format("level=[{}..{}]", 0, LZ4HC_CLEVEL_MAX)} {} @@ -236,7 +240,7 @@ class lz4hc_compression_factory : public compression_factory { } // namespace -REGISTER_COMPRESSION_FACTORY(compression_type::LZ4, lz4_compression_factory) -REGISTER_COMPRESSION_FACTORY(compression_type::LZ4HC, lz4hc_compression_factory) +REGISTER_COMPRESSION_FACTORY(lz4_compression_factory) +REGISTER_COMPRESSION_FACTORY(lz4hc_compression_factory) } // namespace dwarfs diff --git a/src/dwarfs/compression/lzma.cpp b/src/dwarfs/compression/lzma.cpp index 11679825..ff311568 100644 --- a/src/dwarfs/compression/lzma.cpp +++ b/src/dwarfs/compression/lzma.cpp @@ -347,6 +347,8 @@ size_t lzma_block_decompressor::get_uncompressed_size(const uint8_t* data, class lzma_compression_factory : public compression_factory { public: + static constexpr compression_type type{compression_type::LZMA}; + std::string_view name() const override { return "lzma"; } std::string_view description() const override { @@ -386,6 +388,6 @@ class lzma_compression_factory : public compression_factory { } // namespace -REGISTER_COMPRESSION_FACTORY(compression_type::LZMA, lzma_compression_factory) +REGISTER_COMPRESSION_FACTORY(lzma_compression_factory) } // namespace dwarfs diff --git a/src/dwarfs/compression/null.cpp b/src/dwarfs/compression/null.cpp index 35a24723..0f31c36e 100644 --- a/src/dwarfs/compression/null.cpp +++ b/src/dwarfs/compression/null.cpp @@ -113,6 +113,8 @@ class null_block_decompressor final : public block_decompressor::impl { class null_compression_factory : public compression_factory { public: + static constexpr compression_type type{compression_type::NONE}; + std::string_view name() const override { return "null"; } std::string_view description() const override { @@ -141,6 +143,6 @@ class null_compression_factory : public compression_factory { } // namespace -REGISTER_COMPRESSION_FACTORY(compression_type::NONE, null_compression_factory) +REGISTER_COMPRESSION_FACTORY(null_compression_factory) } // namespace dwarfs diff --git a/src/dwarfs/compression/ricepp.cpp b/src/dwarfs/compression/ricepp.cpp index 7d5dcbdb..4563bb89 100644 --- a/src/dwarfs/compression/ricepp.cpp +++ b/src/dwarfs/compression/ricepp.cpp @@ -262,6 +262,8 @@ class ricepp_block_decompressor final : public block_decompressor::impl { class ricepp_compression_factory : public compression_factory { public: + static constexpr compression_type type{compression_type::RICEPP}; + ricepp_compression_factory() : options_{ fmt::format("block_size=[{}..{}]", 16, 512), @@ -297,7 +299,6 @@ class ricepp_compression_factory : public compression_factory { } // namespace -REGISTER_COMPRESSION_FACTORY(compression_type::RICEPP, - ricepp_compression_factory) +REGISTER_COMPRESSION_FACTORY(ricepp_compression_factory) } // namespace dwarfs diff --git a/src/dwarfs/compression/zstd.cpp b/src/dwarfs/compression/zstd.cpp index e6be21ca..cd6552ae 100644 --- a/src/dwarfs/compression/zstd.cpp +++ b/src/dwarfs/compression/zstd.cpp @@ -177,6 +177,8 @@ class zstd_block_decompressor final : public block_decompressor::impl { class zstd_compression_factory : public compression_factory { public: + static constexpr compression_type type{compression_type::ZSTD}; + zstd_compression_factory() : options_{ fmt::format("level=[{}..{}]", ZSTD_MIN_LEVEL, ZSTD_maxCLevel())} {} @@ -214,6 +216,6 @@ class zstd_compression_factory : public compression_factory { } // namespace -REGISTER_COMPRESSION_FACTORY(compression_type::ZSTD, zstd_compression_factory) +REGISTER_COMPRESSION_FACTORY(zstd_compression_factory) } // namespace dwarfs