TheCloud/dwarfs
1
0
Fork 0
mirror of https://github.com/mhx/dwarfs.git synced 2025-09-16 07:46:27 -04:00
Code Issues Packages Projects Releases Wiki Activity

refactor(compression,categorizer): drop WHOLE_ARCHIVE dependency

Browse Source
This commit is contained in:
Marcus Holland-Moritz 2024-08-01 13:30:58 +02:00
parent 4f95d66d57
commit 6d2059f8b4
12 changed files with 108 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
CMakeLists.txt
View File

@ -604,6 +604,7 @@ list(APPEND LIBDWARFS_COMMON_SRC
src/dwarfs/block_compressor.cpp
src/dwarfs/block_compressor_parser.cpp
src/dwarfs/checksum.cpp
src/dwarfs/compression_metadata_requirements.cpp
src/dwarfs/conv.cpp
src/dwarfs/error.cpp
src/dwarfs/file_access_generic.cpp
@ -621,6 +622,7 @@ list(APPEND LIBDWARFS_COMMON_SRC
src/dwarfs/option_map.cpp
src/dwarfs/options.cpp
src/dwarfs/os_access_generic.cpp
src/dwarfs/pcm_sample_transformer.cpp
src/dwarfs/performance_monitor.cpp
src/dwarfs/terminal.cpp
src/dwarfs/thread_pool.cpp
@ -761,25 +763,17 @@ endif()
# list(APPEND LIBDWARFS_CATEGORIZER_SRC src/dwarfs/categorizer/libmagic_categorizer.cpp)
# endif()
add_library(dwarfs_common ${LIBDWARFS_COMMON_SRC})
add_library(dwarfs_common ${LIBDWARFS_COMMON_SRC} ${LIBDWARFS_COMPRESSION_SRC})
add_library(dwarfs_reader ${LIBDWARFS_READER_SRC})
add_library(dwarfs_writer ${LIBDWARFS_WRITER_SRC})
add_library(dwarfs_writer ${LIBDWARFS_WRITER_SRC} ${LIBDWARFS_CATEGORIZER_SRC})
add_library(dwarfs_extractor ${LIBDWARFS_EXTRACTOR_SRC})
add_library(dwarfs_compression ${LIBDWARFS_COMPRESSION_SRC})
add_library(dwarfs_categorizer ${LIBDWARFS_CATEGORIZER_SRC})
add_library(dwarfs_compression_metadata src/dwarfs/compression_metadata_requirements.cpp)
add_library(dwarfs_pcm_sample_transformer src/dwarfs/pcm_sample_transformer.cpp)
add_library(dwarfs_tool ${LIBDWARFS_TOOL_SRC})
target_link_libraries(dwarfs_compression_metadata PUBLIC dwarfs_folly_lite)
target_link_libraries(dwarfs_pcm_sample_transformer PUBLIC dwarfs_folly_lite)
target_link_libraries(dwarfs_categorizer PUBLIC dwarfs_compression_metadata)
target_link_libraries(dwarfs_categorizer PRIVATE range-v3::range-v3)
target_link_libraries(dwarfs_common PUBLIC dwarfs_folly_lite)
target_link_libraries(dwarfs_common PRIVATE PkgConfig::LIBCRYPTO xxHash::xxhash phmap range-v3::range-v3)
target_link_libraries(dwarfs_reader PUBLIC dwarfs_common)
target_link_libraries(dwarfs_reader PRIVATE range-v3::range-v3)
target_link_libraries(dwarfs_writer PUBLIC dwarfs_common dwarfs_compression_metadata)
target_link_libraries(dwarfs_writer PUBLIC dwarfs_common)
target_link_libraries(dwarfs_writer PRIVATE phmap range-v3::range-v3)
target_link_libraries(dwarfs_extractor PUBLIC dwarfs_reader)
target_link_libraries(dwarfs_tool PUBLIC dwarfs_common)
@ -798,10 +792,8 @@ target_compile_definitions(
PRJ_COMPILER_ID="${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}"
)
target_link_libraries(dwarfs_categorizer PUBLIC dwarfs_folly_lite range-v3::range-v3)
target_link_libraries(dwarfs_compression PUBLIC dwarfs_common dwarfs_compression_thrift dwarfs_pcm_sample_transformer)
if(ENABLE_RICEPP)
target_link_libraries(dwarfs_compression PRIVATE ricepp)
target_link_libraries(dwarfs_common PRIVATE ricepp)
endif()
target_link_libraries(dwarfs_tool PUBLIC dwarfs_common)
@ -988,7 +980,7 @@ if(WITH_TESTS)
incompressible_categorizer_test
pcmaudio_categorizer_test
tool_main_test)
target_link_libraries(${tgt} PRIVATE "$<LINK_LIBRARY:WHOLE_ARCHIVE,dwarfs_categorizer>")
target_link_libraries(${tgt} PRIVATE dwarfs_writer)
endforeach()
foreach(tgt ${TEST_TARGETS})
@ -1028,7 +1020,7 @@ endif()
if(WITH_FUZZ)
add_executable(fuzz_categorizers test/fuzz_categorizers.cpp)
target_link_libraries(fuzz_categorizers PRIVATE "$<LINK_LIBRARY:WHOLE_ARCHIVE,dwarfs_categorizer>")
target_link_libraries(fuzz_categorizers PRIVATE dwarfs_writer)
list(APPEND BINARY_TARGETS fuzz_categorizers)
add_executable(fuzz_mkdwarfs test/fuzz_mkdwarfs.cpp)
@ -1092,11 +1084,7 @@ add_cpp2_thrift_library(thrift/history.thrift
add_cpp2_thrift_library(thrift/features.thrift
TARGET dwarfs_features_thrift OUTPUT_PATH dwarfs)
target_link_libraries(dwarfs_categorizer PRIVATE dwarfs_metadata_thrift)
foreach(tgt dwarfs_common dwarfs_reader dwarfs_writer dwarfs_extractor
dwarfs_compression dwarfs_categorizer dwarfs_tool
dwarfs_compression_metadata dwarfs_pcm_sample_transformer
foreach(tgt dwarfs_common dwarfs_reader dwarfs_writer dwarfs_extractor dwarfs_tool
${BINARY_TARGETS} ${TEST_TARGETS} ${MAIN_TARGETS})
set_target_properties(${tgt} PROPERTIES EXPORT_COMPILE_COMMANDS ON)
@ -1206,6 +1194,7 @@ target_link_libraries(
Boost::boost
Boost::chrono
Boost::iostreams
dwarfs_compression_thrift
dwarfs_metadata_thrift
dwarfs_history_thrift
dwarfs_features_thrift
@ -1216,28 +1205,28 @@ target_link_libraries(
# target_link_libraries(dwarfs PkgConfig::LIBMAGIC)
# endif()
target_link_libraries(dwarfs_categorizer PRIVATE libzstd_static)
target_link_libraries(dwarfs_compression PRIVATE libzstd_static)
target_link_libraries(dwarfs_common PRIVATE libzstd_static)
target_link_libraries(dwarfs_writer PRIVATE libzstd_static)
if(LIBLZ4_FOUND)
target_link_libraries(dwarfs_compression PRIVATE PkgConfig::LIBLZ4)
target_link_libraries(dwarfs_common PRIVATE PkgConfig::LIBLZ4)
endif()
if(LIBLZMA_FOUND)
target_link_libraries(dwarfs_compression PRIVATE PkgConfig::LIBLZMA)
target_link_libraries(dwarfs_common PRIVATE PkgConfig::LIBLZMA)
endif()
if(FLAC_FOUND)
target_link_libraries(dwarfs_compression PRIVATE PkgConfig::FLAC)
target_link_libraries(dwarfs_common PRIVATE PkgConfig::FLAC)
endif()
if(LIBBROTLIDEC_FOUND AND LIBBROTLIENC_FOUND)
target_link_libraries(dwarfs_compression PRIVATE PkgConfig::LIBBROTLIDEC PkgConfig::LIBBROTLIENC)
target_link_libraries(dwarfs_common PRIVATE PkgConfig::LIBBROTLIDEC PkgConfig::LIBBROTLIENC)
endif()
if(NOT STATIC_BUILD_DO_NOT_USE)
target_link_libraries(dwarfs_extractor PUBLIC PkgConfig::LIBARCHIVE)
# target_link_libraries(dwarfs_categorizer PkgConfig::LIBMAGIC)
# target_link_libraries(dwarfs_writer PkgConfig::LIBMAGIC)
endif(NOT STATIC_BUILD_DO_NOT_USE)
if(DWARFS_USE_EXCEPTION_TRACER)
@ -1260,14 +1249,13 @@ endforeach()
foreach(tgt ${BINARY_TARGETS} ${TEST_TARGETS})
target_link_libraries(${tgt} PRIVATE dwarfs_common)
target_link_libraries(${tgt} PRIVATE "$<LINK_LIBRARY:WHOLE_ARCHIVE,dwarfs_compression>")
if(DWARFS_USE_EXCEPTION_TRACER)
target_link_libraries(${tgt} PRIVATE
"$<LINK_LIBRARY:WHOLE_ARCHIVE,dwarfs_folly_exception_tracer_base_lite>")
endif()
endforeach()
target_link_libraries(mkdwarfs_main PRIVATE dwarfs_reader dwarfs_writer "$<LINK_LIBRARY:WHOLE_ARCHIVE,dwarfs_categorizer>")
target_link_libraries(mkdwarfs_main PRIVATE dwarfs_reader dwarfs_writer)
target_link_libraries(dwarfsck_main PRIVATE dwarfs_reader)
target_link_libraries(dwarfsextract_main PRIVATE dwarfs_extractor PkgConfig::LIBARCHIVE)
target_link_libraries(dwarfsbench_main PRIVATE dwarfs_extractor)
@ -1278,7 +1266,7 @@ target_link_libraries(dwarfsextract PRIVATE dwarfsextract_main)
target_link_libraries(dwarfsbench PRIVATE dwarfsbench_main)
if(TARGET dwarfsuniversal)
target_link_libraries(dwarfsuniversal PRIVATE dwarfs_writer dwarfs_extractor "$<LINK_LIBRARY:WHOLE_ARCHIVE,dwarfs_categorizer>")
target_link_libraries(dwarfsuniversal PRIVATE dwarfs_writer dwarfs_extractor)
endif()
if(TARGET dwarfs_main)
target_link_libraries(dwarfs_main PRIVATE dwarfs_reader)
@ -1344,9 +1332,6 @@ if(STATIC_BUILD_DO_NOT_USE)
set_target_properties(static_libmagic PROPERTIES INTERFACE_LINK_LIBRARIES
static_libz)
target_link_libraries(dwarfs_categorizer PRIVATE static_libmagic)
target_link_libraries(dwarfs_compression PRIVATE static_libflac)
foreach(tgt ${BINARY_TARGETS} ${TEST_TARGETS})
if(PREFER_SYSTEM_LIBFMT)
target_link_libraries(

40
include/dwarfs/block_compressor.h
View File

@ -176,19 +176,13 @@ class compression_factory : public compression_info {
namespace detail {
template <typename T>
class compression_factory_registrar {
public:
compression_factory_registrar(compression_type type);
};
template <compression_type T>
struct compression_factory_registrar;
} // namespace detail
class compression_registry {
public:
template <typename T>
friend class detail::compression_factory_registrar;
static compression_registry& instance();
std::unique_ptr<block_compressor::impl>
@ -201,13 +195,13 @@ class compression_registry {
std::function<void(compression_type, compression_info const&)> const& fn)
const;
void register_factory(compression_type type,
std::unique_ptr<compression_factory const>&& factory);
private:
compression_registry();
~compression_registry();
void register_factory(compression_type type,
std::unique_ptr<compression_factory const>&& factory);
std::unordered_map<compression_type,
std::unique_ptr<compression_factory const>>
factories_;
@ -216,19 +210,23 @@ class compression_registry {
namespace detail {
template <typename T>
compression_factory_registrar<T>::compression_factory_registrar(
compression_type type) {
::dwarfs::compression_registry::instance().register_factory(
type, std::make_unique<T>());
}
#define DWARFS_COMPRESSION_TYPE_ENUMERATION_(name, value) \
template <> \
struct compression_factory_registrar<compression_type::name> { \
static void reg(compression_registry&); \
};
#define DWARFS_NO_SEPARATOR_
DWARFS_COMPRESSION_TYPE_LIST(DWARFS_COMPRESSION_TYPE_ENUMERATION_,
DWARFS_NO_SEPARATOR_)
#undef DWARFS_COMPRESSION_TYPE_ENUMERATION_
#undef DWARFS_NO_SEPARATOR_
} // namespace detail
#define REGISTER_COMPRESSION_FACTORY(type, factory) \
namespace { \
::dwarfs::detail::compression_factory_registrar<factory> \
the_##factory##_registrar(type); \
#define REGISTER_COMPRESSION_FACTORY(factory) \
void ::dwarfs::detail::compression_factory_registrar<factory::type>::reg( \
::dwarfs::compression_registry& cr) { \
cr.register_factory(factory::type, std::make_unique<factory>()); \
}
} // namespace dwarfs

33
include/dwarfs/categorizer.h
View File

@ -210,21 +210,8 @@ class categorizer_factory : public categorizer_info {
boost::program_options::variables_map const& vm) const = 0;
};
namespace detail {
template <typename T>
class categorizer_factory_registrar {
public:
categorizer_factory_registrar();
};
} // namespace detail
class categorizer_registry {
public:
template <typename T>
friend class detail::categorizer_factory_registrar;
static categorizer_registry& instance();
std::unique_ptr<categorizer>
@ -235,29 +222,29 @@ class categorizer_registry {
std::vector<std::string> categorizer_names() const;
void register_factory(std::unique_ptr<categorizer_factory const>&& factory);
private:
categorizer_registry();
~categorizer_registry();
void register_factory(std::unique_ptr<categorizer_factory const>&& factory);
std::map<std::string, std::unique_ptr<categorizer_factory const>> factories_;
};
namespace detail {
template <typename T>
categorizer_factory_registrar<T>::categorizer_factory_registrar() {
::dwarfs::categorizer_registry::instance().register_factory(
std::make_unique<T>());
}
void binary_categorizer_factory_registrar(categorizer_registry&);
void fits_categorizer_factory_registrar(categorizer_registry&);
void incompressible_categorizer_factory_registrar(categorizer_registry&);
void libmagic_categorizer_factory_registrar(categorizer_registry&);
void pcmaudio_categorizer_factory_registrar(categorizer_registry&);
} // namespace detail
#define REGISTER_CATEGORIZER_FACTORY(factory) \
namespace { \
::dwarfs::detail::categorizer_factory_registrar<factory> \
the_##factory##_registrar; \
void ::dwarfs::detail::factory##_registrar( \
::dwarfs::categorizer_registry& cr) { \
cr.register_factory(std::make_unique<factory>()); \
}
} // namespace dwarfs

27
src/dwarfs/block_compressor.cpp
View File

@ -115,7 +115,32 @@ void compression_registry::for_each_algorithm(
}
}
compression_registry::compression_registry() = default;
compression_registry::compression_registry() {
using namespace ::dwarfs::detail;
using enum compression_type;
compression_factory_registrar<NONE>::reg(*this);
#ifdef DWARFS_HAVE_LIBBROTLI
compression_factory_registrar<BROTLI>::reg(*this);
#endif
#ifdef DWARFS_HAVE_FLAC
compression_factory_registrar<FLAC>::reg(*this);
#endif
#ifdef DWARFS_HAVE_LIBLZ4
compression_factory_registrar<LZ4>::reg(*this);
compression_factory_registrar<LZ4HC>::reg(*this);
#endif
#ifdef DWARFS_HAVE_LIBLZMA
compression_factory_registrar<LZMA>::reg(*this);
#endif
#ifdef DWARFS_HAVE_RICEPP
compression_factory_registrar<RICEPP>::reg(*this);
#endif
#ifdef DWARFS_HAVE_LIBZSTD
compression_factory_registrar<ZSTD>::reg(*this);
#endif
}
compression_registry::~compression_registry() = default;
} // namespace dwarfs

11
src/dwarfs/categorizer.cpp
View File

@ -406,7 +406,16 @@ std::vector<std::string> categorizer_registry::categorizer_names() const {
return rv;
}
categorizer_registry::categorizer_registry() = default;
categorizer_registry::categorizer_registry() {
using namespace ::dwarfs::detail;
// binary_categorizer_factory_registrar(*this);
fits_categorizer_factory_registrar(*this);
incompressible_categorizer_factory_registrar(*this);
// libmagic_categorizer_factory_registrar(*this);
pcmaudio_categorizer_factory_registrar(*this);
}
categorizer_registry::~categorizer_registry() = default;
} // namespace dwarfs

5
src/dwarfs/compression/brotli.cpp
View File

@ -174,6 +174,8 @@ class brotli_block_decompressor final : public block_decompressor::impl {
class brotli_compression_factory : public compression_factory {
public:
static constexpr compression_type type{compression_type::BROTLI};
brotli_compression_factory()
: options_{
fmt::format("quality=[{}..{}]", BROTLI_MIN_QUALITY,
@ -224,7 +226,6 @@ class brotli_compression_factory : public compression_factory {
} // namespace
REGISTER_COMPRESSION_FACTORY(compression_type::BROTLI,
brotli_compression_factory)
REGISTER_COMPRESSION_FACTORY(brotli_compression_factory)
} // namespace dwarfs

4
src/dwarfs/compression/flac.cpp
View File

@ -485,6 +485,8 @@ class flac_block_decompressor final : public block_decompressor::impl {
class flac_compression_factory : public compression_factory {
public:
static constexpr compression_type type{compression_type::FLAC};
flac_compression_factory()
: options_{
fmt::format("level=[0..8]"),
@ -524,6 +526,6 @@ class flac_compression_factory : public compression_factory {
} // namespace
REGISTER_COMPRESSION_FACTORY(compression_type::FLAC, flac_compression_factory)
REGISTER_COMPRESSION_FACTORY(flac_compression_factory)
} // namespace dwarfs

8
src/dwarfs/compression/lz4.cpp
View File

@ -168,6 +168,8 @@ class lz4_block_decompressor final : public block_decompressor::impl {
class lz4_compression_factory : public compression_factory {
public:
static constexpr compression_type type{compression_type::LZ4};
std::string_view name() const override { return "lz4"; }
std::string_view description() const override {
@ -200,6 +202,8 @@ class lz4_compression_factory : public compression_factory {
class lz4hc_compression_factory : public compression_factory {
public:
static constexpr compression_type type{compression_type::LZ4HC};
lz4hc_compression_factory()
: options_{fmt::format("level=[{}..{}]", 0, LZ4HC_CLEVEL_MAX)} {}
@ -236,7 +240,7 @@ class lz4hc_compression_factory : public compression_factory {
} // namespace
REGISTER_COMPRESSION_FACTORY(compression_type::LZ4, lz4_compression_factory)
REGISTER_COMPRESSION_FACTORY(compression_type::LZ4HC, lz4hc_compression_factory)
REGISTER_COMPRESSION_FACTORY(lz4_compression_factory)
REGISTER_COMPRESSION_FACTORY(lz4hc_compression_factory)
} // namespace dwarfs

4
src/dwarfs/compression/lzma.cpp
View File

@ -347,6 +347,8 @@ size_t lzma_block_decompressor::get_uncompressed_size(const uint8_t* data,
class lzma_compression_factory : public compression_factory {
public:
static constexpr compression_type type{compression_type::LZMA};
std::string_view name() const override { return "lzma"; }
std::string_view description() const override {
@ -386,6 +388,6 @@ class lzma_compression_factory : public compression_factory {
} // namespace
REGISTER_COMPRESSION_FACTORY(compression_type::LZMA, lzma_compression_factory)
REGISTER_COMPRESSION_FACTORY(lzma_compression_factory)
} // namespace dwarfs

4
src/dwarfs/compression/null.cpp
View File

@ -113,6 +113,8 @@ class null_block_decompressor final : public block_decompressor::impl {
class null_compression_factory : public compression_factory {
public:
static constexpr compression_type type{compression_type::NONE};
std::string_view name() const override { return "null"; }
std::string_view description() const override {
@ -141,6 +143,6 @@ class null_compression_factory : public compression_factory {
} // namespace
REGISTER_COMPRESSION_FACTORY(compression_type::NONE, null_compression_factory)
REGISTER_COMPRESSION_FACTORY(null_compression_factory)
} // namespace dwarfs

5
src/dwarfs/compression/ricepp.cpp
View File

@ -262,6 +262,8 @@ class ricepp_block_decompressor final : public block_decompressor::impl {
class ricepp_compression_factory : public compression_factory {
public:
static constexpr compression_type type{compression_type::RICEPP};
ricepp_compression_factory()
: options_{
fmt::format("block_size=[{}..{}]", 16, 512),
@ -297,7 +299,6 @@ class ricepp_compression_factory : public compression_factory {
} // namespace
REGISTER_COMPRESSION_FACTORY(compression_type::RICEPP,
ricepp_compression_factory)
REGISTER_COMPRESSION_FACTORY(ricepp_compression_factory)
} // namespace dwarfs

4
src/dwarfs/compression/zstd.cpp
View File

@ -177,6 +177,8 @@ class zstd_block_decompressor final : public block_decompressor::impl {
class zstd_compression_factory : public compression_factory {
public:
static constexpr compression_type type{compression_type::ZSTD};
zstd_compression_factory()
: options_{
fmt::format("level=[{}..{}]", ZSTD_MIN_LEVEL, ZSTD_maxCLevel())} {}
@ -214,6 +216,6 @@ class zstd_compression_factory : public compression_factory {
} // namespace
REGISTER_COMPRESSION_FACTORY(compression_type::ZSTD, zstd_compression_factory)
REGISTER_COMPRESSION_FACTORY(zstd_compression_factory)
} // namespace dwarfs