dwarfs/.docker/build-linux.sh
2025-07-30 14:11:51 +02:00

474 lines
13 KiB
Bash
Executable File

#!/bin/bash
set -ex
export CCACHE_DIR=/ccache
LOCAL_REPO_PATH=/local/repos
mkdir -p "$LOCAL_REPO_PATH"
LAST_UPDATE_FILE="$LOCAL_REPO_PATH/last-update"
WORKFLOW_LOG_DIR="/artifacts/workflow-logs/${GITHUB_RUN_ID}"
NINJA_LOG_FILE="${WORKFLOW_LOG_DIR}/ninja-${BUILD_ARCH},${CROSS_ARCH},${BUILD_DIST},${BUILD_TYPE}.log"
BUILD_LOG_FILE="${WORKFLOW_LOG_DIR}/build-${BUILD_ARCH},${CROSS_ARCH},${BUILD_DIST},${BUILD_TYPE}.log"
mkdir -p "$WORKFLOW_LOG_DIR"
log() {
local event="$1"
# log timestamp (with microsecond resolution) + tab + event
echo "$(python3 - <<<'from datetime import datetime; print(datetime.now().isoformat())') $event" >> "$BUILD_LOG_FILE"
}
if [ -f "$LAST_UPDATE_FILE" ] && [ $(find "$LAST_UPDATE_FILE" -mmin -180) ]; then
echo "Skipping git repo update because it already ran in the last three hours."
else
echo "Running git repo update."
log "begin:repo-update"
for repo in "fmtlib/fmt" \
"google/googletest" \
"ericniebler/range-v3" \
"greg7mdp/parallel-hashmap"; do
reponame=$(basename "$repo")
cd "$LOCAL_REPO_PATH"
if [ -d "$reponame" ]; then
cd "$reponame"
time git fetch
else
time git clone "https://github.com/$repo.git"
fi
done
log "end:repo-update"
touch "$LAST_UPDATE_FILE"
fi
if [[ "$BUILD_TYPE" != "clang-release-ninja-static" ]]; then
export DWARFS_LOCAL_REPO_PATH="$LOCAL_REPO_PATH"
fi
if [[ "-$BUILD_TYPE-" == *-debug-* ]] && [[ "-$BUILD_TYPE-" != *-coverage-* ]] &&
[[ "-$BUILD_TYPE-" != *-[at]san-* ]] && [[ "-$BUILD_TYPE-" != *-ubsan-* ]]; then
export DWARFS_SKIP_SLOW_TESTS=1
fi
ARCH="$(uname -m)"
cd "$HOME"
rm -rf dwarfs dwarfs-*
if [[ "$BUILD_FROM_TARBALL" == "1" ]]; then
log "begin:tarball-extract"
tar xf "/artifacts/cache/dwarfs-source-${GITHUB_RUN_NUMBER}.tar.zst"
ln -s dwarfs-* dwarfs
log "end:tarball-extract"
else
git config --global --add safe.directory /workspace
ln -s /workspace dwarfs
fi
rm -rf build
mkdir build
cd build
# Stick to clang-18, clang-19 has a regression for nilsimsa performance
if [[ "$BUILD_DIST" == "alpine" ]]; then
GCC_VERSION=
CLANG_VERSION=-20
elif [[ "$BUILD_DIST" == "ubuntu-2204" ]]; then
GCC_VERSION=-12
CLANG_VERSION=-15
else
GCC_VERSION=-14
CLANG_VERSION=-18
fi
case "-$BUILD_TYPE-" in
*-ninja-*)
if [[ "$BUILD_DIST" == "alpine" ]]; then
BUILD_TOOL=/usr/lib/ninja-build/bin/ninja
else
BUILD_TOOL=ninja
fi
CMAKE_TOOL_ARGS="-GNinja"
SAVE_BUILD_LOG="cp -a .ninja_log $NINJA_LOG_FILE"
;;
*-make-*)
BUILD_TOOL="make -j$(nproc)"
CMAKE_TOOL_ARGS=
SAVE_BUILD_LOG=
;;
*)
echo "missing build tool in: $BUILD_TYPE"
exit 1
esac
CMAKE_ARGS="${CMAKE_TOOL_ARGS}"
case "-$BUILD_TYPE-" in
*-gcc-*)
case "-$BUILD_DIST-" in
*-ubuntu-*|*-debian-*)
export CC=gcc$GCC_VERSION CXX=g++$GCC_VERSION
;;
*)
export CC=gcc CXX=g++
;;
esac
export COMPILER=gcc
;;
*-oldgcc-*)
export CC=gcc-11 CXX=g++-11
;;
*-clang-*)
case "-$BUILD_DIST-" in
*-ubuntu-*|*-debian-*|*-alpine-*)
export CC=clang$CLANG_VERSION CXX=clang++$CLANG_VERSION
;;
*)
export CC=clang CXX=clang++
;;
esac
export COMPILER=clang
;;
*-oldclang-*)
export CC=clang-16 CXX=clang++-16
;;
*)
echo "missing compiler in: $BUILD_TYPE"
exit 1
esac
case "-$BUILD_TYPE-" in
*-debug-*)
CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=Debug"
;;
*-release-*)
CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=Release"
if [[ "-$BUILD_TYPE-" != *-minimal-* ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DWITH_BENCHMARKS=1"
if [[ "-$BUILD_TYPE-" != *-lto-* ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DWITH_ALL_BENCHMARKS=1"
fi
fi
if [[ "-$BUILD_TYPE-" == *-static-* ]]; then
export CFLAGS="-ffunction-sections -fdata-sections -fvisibility=hidden -fmerge-all-constants"
export CXXFLAGS="${CFLAGS}"
export LDFLAGS="-Wl,--gc-sections"
fi
;;
*-relsize-*)
CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=MinSizeRel"
if [[ "-$BUILD_TYPE-" != *-minimal-* ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DWITH_BENCHMARKS=1"
if [[ "-$BUILD_TYPE-" != *-lto-* ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DWITH_ALL_BENCHMARKS=1"
fi
fi
if [[ "-$BUILD_TYPE-" == *-static-* ]]; then
export CFLAGS="-ffunction-sections -fdata-sections -fvisibility=hidden -fmerge-all-constants"
export CXXFLAGS="${CFLAGS}"
export LDFLAGS="-Wl,--gc-sections"
fi
export COMPILER="${COMPILER}-minsize"
;;
*-reldbg-*)
CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo"
;;
*)
echo "missing build type: $BUILD_TYPE"
exit 1
esac
case "-$BUILD_TYPE-" in
*-static-*)
case "$CROSS_ARCH" in
ppc64le)
# https://github.com/rui314/mold/issues/1490
CMAKE_ARGS="${CMAKE_ARGS} -DDISABLE_MOLD=1"
export LDFLAGS="${LDFLAGS} -fuse-ld=lld"
;;
*)
export LDFLAGS="${LDFLAGS} -fuse-ld=mold"
;;
esac
;;
esac
case "-$BUILD_TYPE-" in
*-lto-*)
export CFLAGS="${CFLAGS} -flto=auto"
export CXXFLAGS="${CXXFLAGS} -flto=auto"
# The -L option is needed so that boost_iostreams finds the right libzstd...
export LDFLAGS="${LDFLAGS} -flto=auto -Wl,--icf=all"
export COMPILER="${COMPILER}-lto"
;;
esac
case "-$BUILD_TYPE-" in
*-minimal-*)
CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_PERFMON=0 -DWITH_MAN_OPTION=0 -DENABLE_RICEPP=0"
CMAKE_ARGS="${CMAKE_ARGS} -DTRY_ENABLE_BROTLI=0 -DTRY_ENABLE_LZ4=0 -DTRY_ENABLE_FLAC=0"
CMAKE_ARGS="${CMAKE_ARGS} -DDISABLE_FILESYSTEM_EXTRACTOR_FORMAT=1"
;;
esac
case "-$BUILD_TYPE-" in
*-asan-*)
CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_ASAN=1"
;;
*-tsan-*)
CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_TSAN=1"
export TSAN_OPTIONS="suppressions=/workspace/tsan.supp"
;;
*-ubsan-*)
CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_UBSAN=1"
;;
*-coverage-*)
CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_COVERAGE=1 -DWITH_UNIVERSAL_BINARY=1"
;;
esac
if [[ "-$BUILD_TYPE-" == *-O2-* ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DDWARFS_OPTIMIZE=2"
fi
if [[ "-$BUILD_TYPE-" == *-nojemalloc-* ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DUSE_JEMALLOC=0"
fi
if [[ "-$BUILD_TYPE-" == *-mimalloc-* ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DUSE_JEMALLOC=0 -DUSE_MIMALLOC=1"
fi
if [[ "-$BUILD_TYPE-" == *-noperfmon-* ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_PERFMON=0 -DWITH_MAN_OPTION=0"
fi
if [[ "-$BUILD_TYPE-" == *-stacktrace-* ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_STACKTRACE=ON"
fi
if [[ "-$BUILD_TYPE-" == *-source-* ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DDWARFS_OPTIMIZE=0"
else
CMAKE_ARGS="${CMAKE_ARGS} -DWITH_LEGACY_FUSE=1"
fi
if [[ "-$BUILD_TYPE-" == *-notest-* ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DWITH_TESTS=0"
RUN_TESTS="echo 'skipping tests'"
else
CMAKE_ARGS="${CMAKE_ARGS} -DWITH_TESTS=1"
RUN_TESTS="ctest --output-on-failure -j$(nproc)"
fi
CMAKE_ARGS="${CMAKE_ARGS} -DDWARFS_ARTIFACTS_DIR=/artifacts"
SUFFIX=""
if [[ "$BUILD_DIST" == "alpine" ]]; then
SUFFIX="-musl"
if [[ "-$BUILD_TYPE-" == *-minimal-* ]]; then
SUFFIX="${SUFFIX}-minimal"
fi
if [[ "-$BUILD_TYPE-" == *-mimalloc-* ]]; then
SUFFIX="${SUFFIX}-mimalloc"
fi
if [[ "-$BUILD_TYPE-" == *-libressl-* ]]; then
SUFFIX="${SUFFIX}-libressl"
fi
if [[ "-$BUILD_TYPE-" == *-lto-* ]]; then
SUFFIX="${SUFFIX}-lto"
fi
fi
if [[ -n "$SUFFIX" ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DDWARFS_ARTIFACT_SUFFIX=$SUFFIX"
fi
if [[ "-$BUILD_TYPE-" == *-shared-* ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DBUILD_SHARED_LIBS=1 -DCMAKE_POSITION_INDEPENDENT_CODE=1"
fi
if [[ "-$BUILD_TYPE-" == *-static-* ]]; then
# A size-optimized libgcc/libstdc++/musl will only save about 10k for the universal
# and fuse-extract binaries, so we'll just stick to using the -O2 toolchain.
_SYSROOT="/opt/cross/O2"
_MARCH="${CROSS_ARCH}"
if [[ -z "$CROSS_ARCH" ]]; then
_MARCH="${ARCH}"
fi
if [[ "$_MARCH" == "arm" ]]; then
_TARGET="${_MARCH}-linux-musleabihf"
else
_TARGET="${_MARCH}-alpine-linux-musl"
fi
export CC="${_TARGET}-${CC}"
export CXX="${_TARGET}-${CXX}"
export STRIP_TOOL="${_TARGET}-strip"
export PATH="$_SYSROOT/usr/bin:$PATH"
_staticprefix="/opt/static-libs/$COMPILER/$_TARGET"
if [[ "$BUILD_TYPE" == *-minimal-* ]]; then
_jemallocprefix="/opt/static-libs/$COMPILER-jemalloc-minimal/$_TARGET"
else
CMAKE_ARGS="${CMAKE_ARGS} -DWITH_PXATTR=1"
_jemallocprefix="/opt/static-libs/$COMPILER-jemalloc-full/$_TARGET"
fi
if [[ "$BUILD_TYPE" == *-libressl-* ]]; then
_sslprefix="/opt/static-libs/$COMPILER-libressl/$_TARGET"
else
_sslprefix="/opt/static-libs/$COMPILER-openssl/$_TARGET"
fi
export LDFLAGS="${LDFLAGS} -static-libgcc -L$_staticprefix/lib -L$_sslprefix/lib"
export CFLAGS="${CFLAGS} -isystem $_staticprefix/include"
export CXXFLAGS="${CXXFLAGS} -isystem $_staticprefix/include"
if [[ "$_MARCH" == "i386" ]]; then
export LDFLAGS="${LDFLAGS} -lucontext -latomic"
fi
CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_SYSROOT=$_SYSROOT -DCMAKE_FIND_ROOT_PATH=$_staticprefix;$_sslprefix;$_jemallocprefix -DCMAKE_PREFIX_PATH=$_staticprefix;$_sslprefix;$_jemallocprefix -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY -DSTATIC_BUILD_DO_NOT_USE=1 -DWITH_UNIVERSAL_BINARY=1 -DWITH_FUSE_EXTRACT_BINARY=1"
if [[ -n "$CROSS_ARCH" ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_SYSTEM_NAME=Linux -DCMAKE_SYSTEM_PROCESSOR=$_MARCH -DCMAKE_CROSSCOMPILING_EMULATOR=/usr/bin/qemu-$_MARCH -DFOLLY_HAVE_UNALIGNED_ACCESS=OFF -DFOLLY_HAVE_WEAK_SYMBOLS=ON -DFOLLY_HAVE_LINUX_VDSO=OFF -DFOLLY_HAVE_WCHAR_SUPPORT=OFF -DHAVE_VSNPRINTF_ERRORS=OFF"
fi
fi
INSTALLDIR="$HOME/install"
PREFIXPATH="$INSTALLDIR/usr/local"
rm -rf "$INSTALLDIR"
if [[ "-$BUILD_TYPE-" == *-shared-* ]]; then
LDLIBPATH="$(readlink -m "$PREFIXPATH/lib/$(gcc -print-multi-os-directory)")"
if [[ ":$LD_LIBRARY_PATH:" != *":$LDLIBPATH:"* ]]; then
export "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}$LDLIBPATH"
fi
fi
case "-$BUILD_TYPE-" in
*-full-*)
log "begin:cmake"
cmake ../dwarfs/ $CMAKE_ARGS -DWITH_EXAMPLE=1
log "end:cmake"
log "begin:build"
time $BUILD_TOOL
log "end:build"
$SAVE_BUILD_LOG
log "begin:test"
$RUN_TESTS
log "end:test"
;;
*-split-*)
# ==== libdwarfs ====
cmake ../dwarfs/ $CMAKE_ARGS -DWITH_LIBDWARFS=ON -DWITH_TOOLS=OFF -DWITH_FUSE_DRIVER=OFF
time $BUILD_TOOL
$RUN_TESTS
DESTDIR="$INSTALLDIR" $BUILD_TOOL install
$BUILD_TOOL distclean
rm -rf *
# ==== example binary ====
cmake ../dwarfs/example $CMAKE_TOOL_ARGS -DCMAKE_PREFIX_PATH="$PREFIXPATH"
time $BUILD_TOOL
rm -rf *
# ==== dwarfs tools ====
cmake ../dwarfs/ $CMAKE_ARGS -DWITH_LIBDWARFS=OFF -DWITH_TOOLS=ON -DWITH_FUSE_DRIVER=OFF -DCMAKE_PREFIX_PATH="$PREFIXPATH"
time $BUILD_TOOL
$RUN_TESTS
DESTDIR="$INSTALLDIR" $BUILD_TOOL install
$BUILD_TOOL distclean
rm -rf *
# ==== dwarfs fuse driver ====
cmake ../dwarfs/ $CMAKE_ARGS -DWITH_LIBDWARFS=OFF -DWITH_TOOLS=OFF -DWITH_FUSE_DRIVER=ON -DCMAKE_PREFIX_PATH="$PREFIXPATH"
time $BUILD_TOOL
$RUN_TESTS
DESTDIR="$INSTALLDIR" $BUILD_TOOL install
$BUILD_TOOL distclean
# That's it for split builds, we are done
exit 0
;;
*)
log "begin:cmake"
if [[ "-$BUILD_TYPE-" == *-static-* ]]; then
cmake ../dwarfs/ $CMAKE_ARGS
else
cmake ../dwarfs/ $CMAKE_ARGS -DWITH_EXAMPLE=1
fi
log "end:cmake"
log "begin:build"
time $BUILD_TOOL
log "end:build"
$SAVE_BUILD_LOG
log "begin:test"
$RUN_TESTS
log "end:test"
;;
esac
if [[ "-$BUILD_TYPE-" == *-coverage-* ]]; then
rm -f /tmp-runner/dwarfs-coverage.txt
llvm-profdata$CLANG_VERSION merge -sparse profile/* -o dwarfs.profdata
llvm-cov$CLANG_VERSION show -instr-profile=dwarfs.profdata \
$(for i in mkdwarfs dwarfs dwarfsck dwarfsextract *_test *_tests ricepp/ricepp_test; do echo $i; done | sed -e's/^/-object=/') \
>/tmp-runner/dwarfs-coverage.txt
fi
if [[ "-$BUILD_TYPE-" == *-static-* ]]; then
# for release and relsize builds, strip the binaries
if [[ "-$BUILD_TYPE-" =~ -(release|relsize)- ]]; then
$BUILD_TOOL strip
fi
log "begin:package"
$BUILD_TOOL package
log "end:package"
log "begin:upx"
$BUILD_TOOL universal_upx
log "end:upx"
log "begin:copy-artifacts"
$BUILD_TOOL copy_artifacts
log "end:copy-artifacts"
rm -rf /tmp-runner/artifacts
mkdir -p /tmp-runner/artifacts
cp artifacts.env /tmp-runner
cp dwarfs-universal-* /tmp-runner/artifacts
cp dwarfs-fuse-extract-* /tmp-runner/artifacts
cp dwarfs-*-Linux*.tar.zst /tmp-runner/artifacts
elif [[ "-$BUILD_TYPE-" == *-source-* ]]; then
log "begin:package-source"
$BUILD_TOOL package_source
log "end:package-source"
log "begin:copy-artifacts"
$BUILD_TOOL copy_source_artifacts
log "end:copy-artifacts"
fi
if [[ "-$BUILD_TYPE-" != *-[at]san-* ]] && \
[[ "-$BUILD_TYPE-" != *-ubsan-* ]] && \
[[ "-$BUILD_TYPE-" != *-source-* ]] && \
[[ "-$BUILD_TYPE-" != *-static-* ]]; then
DESTDIR="$INSTALLDIR" $BUILD_TOOL install
$BUILD_TOOL distclean
cmake ../dwarfs/example $CMAKE_TOOL_ARGS -DCMAKE_PREFIX_PATH="$INSTALLDIR/usr/local"
$BUILD_TOOL
$BUILD_TOOL clean
fi