From 084176d5fdfff85e0f4430c1da42a29161c4997d Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Tue, 8 Apr 2025 22:36:59 +0200 Subject: [PATCH] feat: try supporting mimalloc --- .docker/build-linux.sh | 8 ++++++++ .docker/check_lib_versions.py | 1 + .docker/install-static-libs.sh | 16 +++++++++++++++- .github/workflows/build.yml | 4 ++-- CMakeLists.txt | 10 ++++++++++ cmake/libdwarfs_tool.cmake | 5 +++++ ricepp/CMakeLists.txt | 7 +++++++ tools/src/tool/tool.cpp | 18 ++++++++++++++++++ 8 files changed, 66 insertions(+), 3 deletions(-) diff --git a/.docker/build-linux.sh b/.docker/build-linux.sh index c9eaca06..05b3b9bf 100755 --- a/.docker/build-linux.sh +++ b/.docker/build-linux.sh @@ -190,6 +190,10 @@ if [[ "-$BUILD_TYPE-" == *-nojemalloc-* ]]; then CMAKE_ARGS="${CMAKE_ARGS} -DUSE_JEMALLOC=0" fi +if [[ "-$BUILD_TYPE-" == *-mimalloc-* ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DUSE_JEMALLOC=0 -DUSE_MIMALLOC=1" +fi + if [[ "-$BUILD_TYPE-" == *-noperfmon-* ]]; then CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_PERFMON=0 -DWITH_MAN_OPTION=0" fi @@ -223,6 +227,10 @@ if [[ "$BUILD_DIST" == "alpine" ]]; then SUFFIX="${SUFFIX}-minimal" fi + if [[ "-$BUILD_TYPE-" == *-mimalloc-* ]]; then + SUFFIX="${SUFFIX}-mimalloc" + fi + if [[ "-$BUILD_TYPE-" == *-lto-* ]]; then SUFFIX="${SUFFIX}-lto" fi diff --git a/.docker/check_lib_versions.py b/.docker/check_lib_versions.py index 12f6b824..8a723ee0 100755 --- a/.docker/check_lib_versions.py +++ b/.docker/check_lib_versions.py @@ -17,6 +17,7 @@ repositories = [ "google/brotli", "facebook/zstd", "libfuse/libfuse", + "microsoft/mimalloc", ] # Function to fetch the latest release information for a repository diff --git a/.docker/install-static-libs.sh b/.docker/install-static-libs.sh index 99fa01b6..988743ee 100644 --- a/.docker/install-static-libs.sh +++ b/.docker/install-static-libs.sh @@ -29,6 +29,7 @@ LZ4_VERSION=1.10.0 # 2024-07-22 BROTLI_VERSION=1.1.0 # 2023-08-31 ZSTD_VERSION=1.5.7 # 2025-02-19 LIBFUSE_VERSION=3.17.1 # 2025-03-24 +MIMALLOC_VERSION=2.1.7 # 2024-05-21 echo "Using $GCC and $CLANG" @@ -36,7 +37,7 @@ if [[ "$PKGS" == ":ubuntu" ]]; then PKGS="file,bzip2,libarchive,flac,libunwind,benchmark,openssl,cpptrace" COMPILERS="clang gcc" elif [[ "$PKGS" == ":alpine" ]]; then - PKGS="benchmark,brotli,cpptrace,double-conversion,flac,fmt,fuse,glog,libarchive,lz4,openssl,xxhash,zstd" + PKGS="benchmark,brotli,cpptrace,double-conversion,flac,fmt,fuse,glog,libarchive,lz4,mimalloc,openssl,xxhash,zstd" export COMMON_CFLAGS="-ffunction-sections -fdata-sections -fmerge-all-constants" export COMMON_CXXFLAGS="$COMMON_CFLAGS" COMPILERS="clang clang-lto clang-minsize-lto gcc" @@ -61,6 +62,7 @@ LZ4_TARBALL="lz4-${LZ4_VERSION}.tar.gz" BROTLI_TARBALL="brotli-${BROTLI_VERSION}.tar.gz" ZSTD_TARBALL="zstd-${ZSTD_VERSION}.tar.gz" LIBFUSE_TARBALL="fuse-${LIBFUSE_VERSION}.tar.gz" +MIMALLOC_TARBALL="mimalloc-${MIMALLOC_VERSION}.tar.gz" use_lib() { local lib="$1" @@ -111,6 +113,7 @@ fetch_lib lz4 https://github.com/lz4/lz4/releases/download/v${LZ4_VERSION}/${LZ4 fetch_lib brotli https://github.com/google/brotli/archive/refs/tags/v${BROTLI_VERSION}.tar.gz ${BROTLI_TARBALL} fetch_lib zstd https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/${ZSTD_TARBALL} fetch_lib fuse https://github.com/libfuse/libfuse/releases/download/fuse-${LIBFUSE_VERSION}/${LIBFUSE_TARBALL} +fetch_lib mimalloc https://github.com/microsoft/mimalloc/archive/refs/tags/v${MIMALLOC_VERSION}.tar.gz ${MIMALLOC_TARBALL} for COMPILER in $COMPILERS; do export CFLAGS="$COMMON_CFLAGS" @@ -178,6 +181,17 @@ for COMPILER in $COMPILERS; do INSTALL_DIR=/opt/static-libs/$COMPILER + if use_lib mimalloc; then + cd "$HOME/pkgs/$COMPILER" + tar xf ../${MIMALLOC_TARBALL} + cd mimalloc-${MIMALLOC_VERSION} + mkdir build + cd build + cmake .. -DMI_LIBC_MUSL=ON -DMI_BUILD_SHARED=OFF -DMI_BUILD_OBJECT=OFF -DMI_BUILD_TESTS=OFF -DMI_OPT_ARCH=OFF -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" + make -j$(nproc) + make install + fi + if use_lib fuse; then cd "$HOME/pkgs/$COMPILER" tar xf ../${LIBFUSE_TARBALL} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 49194ef9..e912bc0c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -364,10 +364,10 @@ jobs: build_type: clang-relsize-minimal-lto-ninja-static - arch: amd64 dist: alpine - build_type: clang-relsize-minimal-nojemalloc-lto-ninja-static + build_type: clang-relsize-minimal-mimalloc-lto-ninja-static - arch: arm64v8 dist: alpine - build_type: clang-relsize-minimal-nojemalloc-lto-ninja-static + build_type: clang-relsize-minimal-mimalloc-lto-ninja-static - arch: amd64 dist: alpine build_type: clang-reldbg-stacktrace-ninja-static diff --git a/CMakeLists.txt b/CMakeLists.txt index 6fa0ea1a..d847dd4f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,6 +55,7 @@ if(NOT WIN32) option(ENABLE_UBSAN "enable undefined behaviour sanitizer" OFF) option(ENABLE_COVERAGE "enable code coverage" OFF) option(USE_JEMALLOC "build with jemalloc if available" ON) + option(USE_MIMALLOC "build with mimalloc if available" OFF) option(PREFER_SYSTEM_FAST_FLOAT "use system fast_float if available" OFF) option(PREFER_SYSTEM_GTEST "use system gtest if available" OFF) option(DISABLE_CCACHE "disable ccache" OFF) @@ -66,6 +67,10 @@ if(STATIC_BUILD_DO_NOT_USE AND NOT(WITH_LIBDWARFS AND WITH_TOOLS AND WITH_FUSE_D message(FATAL_ERROR "STATIC_BUILD_DO_NOT_USE requires WITH_LIBDWARFS, WITH_TOOLS and WITH_FUSE_DRIVER") endif() +if(USE_JEMALLOC AND USE_MIMALLOC) + message(FATAL_ERROR "USE_JEMALLOC and USE_MIMALLOC are mutually exclusive") +endif() + # Libraries that we can fetch on demand if necessary # # All of these libraries are header-only and not strictly required once @@ -98,6 +103,7 @@ set(ZSTD_REQUIRED_VERSION 1.4.8) set(XXHASH_REQUIRED_VERSION 0.8.1) set(FLAC_REQUIRED_VERSION 1.4.2) set(JEMALLOC_REQUIRED_VERSION 5.2.1) +set(MIMALLOC_REQUIRED_VERSION 2.0.0) if(DEFINED ENV{DWARFS_LOCAL_REPO_PATH}) set(LIBFMT_GIT_REPO $ENV{DWARFS_LOCAL_REPO_PATH}/fmt) @@ -228,6 +234,10 @@ if(WITH_LIBDWARFS) pkg_check_modules(JEMALLOC IMPORTED_TARGET jemalloc>=${JEMALLOC_REQUIRED_VERSION}) endif() + if(USE_MIMALLOC) + find_package(mimalloc ${MIMALLOC_REQUIRED_VERSION} REQUIRED CONFIG) + endif() + include(${CMAKE_SOURCE_DIR}/cmake/thrift_library.cmake) include(${CMAKE_SOURCE_DIR}/cmake/folly.cmake) diff --git a/cmake/libdwarfs_tool.cmake b/cmake/libdwarfs_tool.cmake index cf6a2755..a4362166 100644 --- a/cmake/libdwarfs_tool.cmake +++ b/cmake/libdwarfs_tool.cmake @@ -43,6 +43,11 @@ if(USE_JEMALLOC AND JEMALLOC_FOUND) target_compile_definitions(dwarfs_tool PRIVATE DWARFS_USE_JEMALLOC) endif() +if(USE_MIMALLOC AND mimalloc_FOUND) + target_link_libraries(dwarfs_tool PRIVATE mimalloc-static) + target_compile_definitions(dwarfs_tool PRIVATE DWARFS_USE_MIMALLOC) +endif() + target_compile_definitions( dwarfs_tool PRIVATE DWARFS_BUILD_ID="${CMAKE_SYSTEM_PROCESSOR}, ${CMAKE_SYSTEM}, ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}" ) diff --git a/ricepp/CMakeLists.txt b/ricepp/CMakeLists.txt index 559480fc..c1f46587 100644 --- a/ricepp/CMakeLists.txt +++ b/ricepp/CMakeLists.txt @@ -171,6 +171,10 @@ if(WITH_BENCHMARKS) target_link_libraries(ricepp_benchmark PRIVATE PkgConfig::JEMALLOC) target_link_libraries(ricepp_benchmark_fits PRIVATE PkgConfig::JEMALLOC) endif() + if(mimalloc_FOUND) + target_link_libraries(ricepp_benchmark PRIVATE mimalloc-static) + target_link_libraries(ricepp_benchmark_fits PRIVATE mimalloc-static) + endif() endif() endif() @@ -200,6 +204,9 @@ if(WITH_TESTS) if (JEMALLOC_FOUND) target_link_libraries(ricepp_test PRIVATE PkgConfig::JEMALLOC) endif() + if (mimalloc_FOUND) + target_link_libraries(ricepp_test PRIVATE mimalloc-static) + endif() if(ENABLE_COVERAGE) if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") diff --git a/tools/src/tool/tool.cpp b/tools/src/tool/tool.cpp index 4b315b2d..18ed7129 100644 --- a/tools/src/tool/tool.cpp +++ b/tools/src/tool/tool.cpp @@ -47,6 +47,10 @@ #include #endif +#ifdef DWARFS_USE_MIMALLOC +#include +#endif + namespace po = boost::program_options; namespace boost { @@ -82,6 +86,16 @@ std::string get_jemalloc_version() { } #endif +#ifdef DWARFS_USE_MIMALLOC +std::string get_mimalloc_version() { + auto v = mi_version(); + auto major = v / 100; + auto minor = (v % 100) / 10; + auto patch = v % 10; + return fmt::format("{}.{}.{}", major, minor, patch); +} +#endif + std::string tool_header_impl(std::string_view tool_name, std::string_view extra_info = {}) { std::string date; @@ -112,6 +126,10 @@ std::string tool_header(std::string_view tool_name, std::string_view extra_info, deps.add_library("libjemalloc", get_jemalloc_version()); #endif +#ifdef DWARFS_USE_MIMALLOC + deps.add_library("libmimalloc", get_mimalloc_version()); +#endif + if (extra_deps) { extra_deps(deps); }