TheCloud/dwarfs
1
0
Fork 0
mirror of https://github.com/mhx/dwarfs.git synced 2025-08-03 09:47:01 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat: try supporting mimalloc

Browse Source
This commit is contained in:
Marcus Holland-Moritz 2025-04-08 22:36:59 +02:00
parent f26211b72a
commit 084176d5fd
8 changed files with 66 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.docker/build-linux.sh
View File

@ -190,6 +190,10 @@ if [[ "-$BUILD_TYPE-" == *-nojemalloc-* ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DUSE_JEMALLOC=0"
fi
if [[ "-$BUILD_TYPE-" == *-mimalloc-* ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DUSE_JEMALLOC=0 -DUSE_MIMALLOC=1"
fi
if [[ "-$BUILD_TYPE-" == *-noperfmon-* ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_PERFMON=0 -DWITH_MAN_OPTION=0"
fi
@ -223,6 +227,10 @@ if [[ "$BUILD_DIST" == "alpine" ]]; then
SUFFIX="${SUFFIX}-minimal"
fi
if [[ "-$BUILD_TYPE-" == *-mimalloc-* ]]; then
SUFFIX="${SUFFIX}-mimalloc"
fi
if [[ "-$BUILD_TYPE-" == *-lto-* ]]; then
SUFFIX="${SUFFIX}-lto"
fi

1
.docker/check_lib_versions.py
View File

@ -17,6 +17,7 @@ repositories = [
"google/brotli",
"facebook/zstd",
"libfuse/libfuse",
"microsoft/mimalloc",
]
# Function to fetch the latest release information for a repository

16
.docker/install-static-libs.sh
View File

@ -29,6 +29,7 @@ LZ4_VERSION=1.10.0 # 2024-07-22
BROTLI_VERSION=1.1.0 # 2023-08-31
ZSTD_VERSION=1.5.7 # 2025-02-19
LIBFUSE_VERSION=3.17.1 # 2025-03-24
MIMALLOC_VERSION=2.1.7 # 2024-05-21
echo "Using $GCC and $CLANG"
@ -36,7 +37,7 @@ if [[ "$PKGS" == ":ubuntu" ]]; then
PKGS="file,bzip2,libarchive,flac,libunwind,benchmark,openssl,cpptrace"
COMPILERS="clang gcc"
elif [[ "$PKGS" == ":alpine" ]]; then
PKGS="benchmark,brotli,cpptrace,double-conversion,flac,fmt,fuse,glog,libarchive,lz4,openssl,xxhash,zstd"
PKGS="benchmark,brotli,cpptrace,double-conversion,flac,fmt,fuse,glog,libarchive,lz4,mimalloc,openssl,xxhash,zstd"
export COMMON_CFLAGS="-ffunction-sections -fdata-sections -fmerge-all-constants"
export COMMON_CXXFLAGS="$COMMON_CFLAGS"
COMPILERS="clang clang-lto clang-minsize-lto gcc"
@ -61,6 +62,7 @@ LZ4_TARBALL="lz4-${LZ4_VERSION}.tar.gz"
BROTLI_TARBALL="brotli-${BROTLI_VERSION}.tar.gz"
ZSTD_TARBALL="zstd-${ZSTD_VERSION}.tar.gz"
LIBFUSE_TARBALL="fuse-${LIBFUSE_VERSION}.tar.gz"
MIMALLOC_TARBALL="mimalloc-${MIMALLOC_VERSION}.tar.gz"
use_lib() {
local lib="$1"
@ -111,6 +113,7 @@ fetch_lib lz4 https://github.com/lz4/lz4/releases/download/v${LZ4_VERSION}/${LZ4
fetch_lib brotli https://github.com/google/brotli/archive/refs/tags/v${BROTLI_VERSION}.tar.gz ${BROTLI_TARBALL}
fetch_lib zstd https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/${ZSTD_TARBALL}
fetch_lib fuse https://github.com/libfuse/libfuse/releases/download/fuse-${LIBFUSE_VERSION}/${LIBFUSE_TARBALL}
fetch_lib mimalloc https://github.com/microsoft/mimalloc/archive/refs/tags/v${MIMALLOC_VERSION}.tar.gz ${MIMALLOC_TARBALL}
for COMPILER in $COMPILERS; do
export CFLAGS="$COMMON_CFLAGS"
@ -178,6 +181,17 @@ for COMPILER in $COMPILERS; do
INSTALL_DIR=/opt/static-libs/$COMPILER
if use_lib mimalloc; then
cd "$HOME/pkgs/$COMPILER"
tar xf ../${MIMALLOC_TARBALL}
cd mimalloc-${MIMALLOC_VERSION}
mkdir build
cd build
cmake .. -DMI_LIBC_MUSL=ON -DMI_BUILD_SHARED=OFF -DMI_BUILD_OBJECT=OFF -DMI_BUILD_TESTS=OFF -DMI_OPT_ARCH=OFF -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR"
make -j$(nproc)
make install
fi
if use_lib fuse; then
cd "$HOME/pkgs/$COMPILER"
tar xf ../${LIBFUSE_TARBALL}

4
.github/workflows/build.yml vendored
View File

@ -364,10 +364,10 @@ jobs:
build_type: clang-relsize-minimal-lto-ninja-static
- arch: amd64
dist: alpine
build_type: clang-relsize-minimal-nojemalloc-lto-ninja-static
build_type: clang-relsize-minimal-mimalloc-lto-ninja-static
- arch: arm64v8
dist: alpine
build_type: clang-relsize-minimal-nojemalloc-lto-ninja-static
build_type: clang-relsize-minimal-mimalloc-lto-ninja-static
- arch: amd64
dist: alpine
build_type: clang-reldbg-stacktrace-ninja-static

10
CMakeLists.txt
View File

@ -55,6 +55,7 @@ if(NOT WIN32)
option(ENABLE_UBSAN "enable undefined behaviour sanitizer" OFF)
option(ENABLE_COVERAGE "enable code coverage" OFF)
option(USE_JEMALLOC "build with jemalloc if available" ON)
option(USE_MIMALLOC "build with mimalloc if available" OFF)
option(PREFER_SYSTEM_FAST_FLOAT "use system fast_float if available" OFF)
option(PREFER_SYSTEM_GTEST "use system gtest if available" OFF)
option(DISABLE_CCACHE "disable ccache" OFF)
@ -66,6 +67,10 @@ if(STATIC_BUILD_DO_NOT_USE AND NOT(WITH_LIBDWARFS AND WITH_TOOLS AND WITH_FUSE_D
message(FATAL_ERROR "STATIC_BUILD_DO_NOT_USE requires WITH_LIBDWARFS, WITH_TOOLS and WITH_FUSE_DRIVER")
endif()
if(USE_JEMALLOC AND USE_MIMALLOC)
message(FATAL_ERROR "USE_JEMALLOC and USE_MIMALLOC are mutually exclusive")
endif()
# Libraries that we can fetch on demand if necessary
#
# All of these libraries are header-only and not strictly required once
@ -98,6 +103,7 @@ set(ZSTD_REQUIRED_VERSION 1.4.8)
set(XXHASH_REQUIRED_VERSION 0.8.1)
set(FLAC_REQUIRED_VERSION 1.4.2)
set(JEMALLOC_REQUIRED_VERSION 5.2.1)
set(MIMALLOC_REQUIRED_VERSION 2.0.0)
if(DEFINED ENV{DWARFS_LOCAL_REPO_PATH})
set(LIBFMT_GIT_REPO $ENV{DWARFS_LOCAL_REPO_PATH}/fmt)
@ -228,6 +234,10 @@ if(WITH_LIBDWARFS)
pkg_check_modules(JEMALLOC IMPORTED_TARGET jemalloc>=${JEMALLOC_REQUIRED_VERSION})
endif()
if(USE_MIMALLOC)
find_package(mimalloc ${MIMALLOC_REQUIRED_VERSION} REQUIRED CONFIG)
endif()
include(${CMAKE_SOURCE_DIR}/cmake/thrift_library.cmake)
include(${CMAKE_SOURCE_DIR}/cmake/folly.cmake)

5
cmake/libdwarfs_tool.cmake
View File

@ -43,6 +43,11 @@ if(USE_JEMALLOC AND JEMALLOC_FOUND)
target_compile_definitions(dwarfs_tool PRIVATE DWARFS_USE_JEMALLOC)
endif()
if(USE_MIMALLOC AND mimalloc_FOUND)
target_link_libraries(dwarfs_tool PRIVATE mimalloc-static)
target_compile_definitions(dwarfs_tool PRIVATE DWARFS_USE_MIMALLOC)
endif()
target_compile_definitions(
dwarfs_tool PRIVATE DWARFS_BUILD_ID="${CMAKE_SYSTEM_PROCESSOR}, ${CMAKE_SYSTEM}, ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}"
)

7
ricepp/CMakeLists.txt
View File

@ -171,6 +171,10 @@ if(WITH_BENCHMARKS)
target_link_libraries(ricepp_benchmark PRIVATE PkgConfig::JEMALLOC)
target_link_libraries(ricepp_benchmark_fits PRIVATE PkgConfig::JEMALLOC)
endif()
if(mimalloc_FOUND)
target_link_libraries(ricepp_benchmark PRIVATE mimalloc-static)
target_link_libraries(ricepp_benchmark_fits PRIVATE mimalloc-static)
endif()
endif()
endif()
@ -200,6 +204,9 @@ if(WITH_TESTS)
if (JEMALLOC_FOUND)
target_link_libraries(ricepp_test PRIVATE PkgConfig::JEMALLOC)
endif()
if (mimalloc_FOUND)
target_link_libraries(ricepp_test PRIVATE mimalloc-static)
endif()
if(ENABLE_COVERAGE)
if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")

18
tools/src/tool/tool.cpp
View File

@ -47,6 +47,10 @@
#include <jemalloc/jemalloc.h>
#endif
#ifdef DWARFS_USE_MIMALLOC
#include <mimalloc.h>
#endif
namespace po = boost::program_options;
namespace boost {
@ -82,6 +86,16 @@ std::string get_jemalloc_version() {
}
#endif
#ifdef DWARFS_USE_MIMALLOC
std::string get_mimalloc_version() {
auto v = mi_version();
auto major = v / 100;
auto minor = (v % 100) / 10;
auto patch = v % 10;
return fmt::format("{}.{}.{}", major, minor, patch);
}
#endif
std::string
tool_header_impl(std::string_view tool_name, std::string_view extra_info = {}) {
std::string date;
@ -112,6 +126,10 @@ std::string tool_header(std::string_view tool_name, std::string_view extra_info,
deps.add_library("libjemalloc", get_jemalloc_version());
#endif
#ifdef DWARFS_USE_MIMALLOC
deps.add_library("libmimalloc", get_mimalloc_version());
#endif
if (extra_deps) {
extra_deps(deps);
}