From f144da5c5fa51008ee957a6ee95e6b0ff9c9778c Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Fri, 25 Jul 2025 13:29:28 +0200 Subject: [PATCH] wip: more cleanups --- .docker/Dockerfile.alpine | 104 +++++++++++++++++-------------- .docker/Dockerfile.alpine-xx | 108 --------------------------------- .docker/Makefile | 16 +---- .docker/install-static-libs.sh | 23 +++---- .docker/install-toolchain.sh | 24 +++++--- 5 files changed, 88 insertions(+), 187 deletions(-) delete mode 100644 .docker/Dockerfile.alpine-xx diff --git a/.docker/Dockerfile.alpine b/.docker/Dockerfile.alpine index 6062a150..cc04a098 100644 --- a/.docker/Dockerfile.alpine +++ b/.docker/Dockerfile.alpine @@ -1,9 +1,18 @@ # syntax=docker/dockerfile:1 ARG ARCH=amd64 FROM $ARCH/alpine:latest + +ARG ARCH +ARG TARGET_ARCHS +ARG TOOLCHAIN_OPTIMIZE=s + RUN apk update RUN apk upgrade -RUN apk add --no-cache \ +RUN qemu_pkgs=""; \ + for arch in $(echo $TARGET_ARCHS | tr ',' ' '); do \ + qemu_pkgs="$qemu_pkgs qemu-$arch"; \ + done; \ + apk add --no-cache \ bash-completion \ build-base \ wget \ @@ -11,81 +20,88 @@ RUN apk add --no-cache \ vim \ tar \ ripgrep \ - py3-pip \ less \ gcc \ g++ \ clang18 \ clang19 \ + clang20 \ git \ + xz \ + gzip \ + bzip2 \ + zstd \ ccache \ - samurai \ + ninja-build \ cmake \ make \ bison \ flex \ ronn \ fuse \ - fuse-dev \ - fuse-static \ fuse3 \ - fuse3-dev \ - fuse3-static \ pkgconf \ - binutils-dev \ - libevent-dev \ - libevent-static \ - linux-headers \ - date-dev \ - range-v3-dev \ - zlib-static \ - libucontext-dev \ - libdwarf-dev \ - elfutils-dev \ - utfcpp \ - nlohmann-json \ meson \ autoconf \ strace \ - gdb + gdb \ + gmp-dev \ + zlib-static \ + zstd-static \ + zlib-dev \ + zstd-dev \ + mpfr-dev \ + mpc1-dev \ + isl-dev \ + texinfo \ + linux-headers \ + rsync \ + $qemu_pkgs -# Install UPX -ARG ARCH -# UPX isn't supported on RISC-V yet, so skip this -RUN if [ "$ARCH" != "riscv64" ]; then \ - wget -O - https://github.com/upx/upx/releases/download/v5.0.0/upx-5.0.0-$(bash -c "echo \${0//v8/}" $ARCH)_linux.tar.xz | tar -xJf - -C /usr/local/bin --strip-components=1 --wildcards "*/upx"; \ - else \ - echo "Skipping UPX installation for RISC-V architecture while it is not supported."; \ - echo "See https://github.com/upx/upx/discussions/793 for details."; \ - fi - -# Install mistletoe -RUN pip3 install --break-system-packages --root-user-action ignore mistletoe - -# Install size-optimized and static-link-optimized libstdc++ -RUN apk add --no-cache gmp-dev mpfr-dev mpc1-dev isl-dev -COPY install-libstdcxx.sh /usr/local/bin/install-libstdcxx.sh -RUN bash /usr/local/bin/install-libstdcxx.sh +COPY fetch.sh /usr/local/bin/fetch.sh # Install mold COPY install-mold.sh /usr/local/bin/install-mold.sh -RUN bash /usr/local/bin/install-mold.sh +RUN --mount=type=cache,id=ccache,target=/root/.ccache --mount=type=cache,id=pkgcache,target=/root/.pkgcache \ + bash /usr/local/bin/install-mold.sh # Install bloaty COPY install-bloaty.sh /usr/local/bin/install-bloaty.sh -RUN if [ "$ARCH" != "riscv64" ]; then \ +RUN --mount=type=cache,id=ccache,target=/root/.ccache --mount=type=cache,id=pkgcache,target=/root/.pkgcache \ + if [ "$ARCH" == "amd64" ]; then \ bash /usr/local/bin/install-bloaty.sh; \ else \ - echo "Skipping bloaty installation for RISC-V."; \ + echo "Skipping bloaty installation for $ARCH."; \ fi +RUN apk del linux-headers + +# Install size-optimized and static-link-optimized libstdc++ +COPY install-toolchain.sh /usr/local/bin/install-toolchain.sh +RUN --mount=type=cache,id=ccache,target=/root/.ccache --mount=type=cache,id=pkgcache,target=/root/.pkgcache \ + bash /usr/local/bin/install-toolchain.sh $TOOLCHAIN_OPTIMIZE $TARGET_ARCHS + +RUN apk del zlib-dev zstd-dev + +# Clear the config cache after a toolchain update +RUN --mount=type=cache,id=cfgcache,target=/root/.cfgcache \ + rm -rf /root/.cfgcache/* + # Install all static libraries COPY install-static-libs.sh /usr/local/bin/install-static-libs.sh -RUN bash /usr/local/bin/install-static-libs.sh gcc clang-19 :alpine -# Install the *real* ninja... -# TODO: move this up if we change anything -RUN apk add --no-cache ninja-build +RUN --mount=type=cache,id=ccache,target=/root/.ccache --mount=type=cache,id=pkgcache,target=/root/.pkgcache \ + --mount=type=cache,id=cfgcache,target=/root/.cfgcache \ + bash /usr/local/bin/install-static-libs.sh :all $TARGET_ARCHS + +# Install UPX +# UPX isn't supported on RISC-V/S390 (yet), so skip this +RUN if ! [ "$ARCH" == "amd64" ] || [ "$ARCH" == "arm64v8" ]; then \ + wget -O - https://github.com/upx/upx/releases/download/v5.0.2/upx-5.0.2-$(bash -c "echo \${0//v8/}" $ARCH)_linux.tar.xz | tar -xJf - -C /usr/local/bin --strip-components=1 --wildcards "*/upx"; \ + else \ + echo "Skipping UPX installation for $ARCH architecture while it is not supported."; \ + echo "See https://github.com/upx/upx/discussions/793 for details."; \ + fi # Set up git & user RUN git config --global --add safe.directory /workspace diff --git a/.docker/Dockerfile.alpine-xx b/.docker/Dockerfile.alpine-xx deleted file mode 100644 index 5570fe6d..00000000 --- a/.docker/Dockerfile.alpine-xx +++ /dev/null @@ -1,108 +0,0 @@ -# syntax=docker/dockerfile:1 -ARG ARCH=amd64 -FROM $ARCH/alpine:latest - -ARG ARCH -ARG TARGET_ARCHS=$ARCH -ARG TOOLCHAIN_OPTIMIZE=s - -RUN apk update -RUN apk upgrade -RUN qemu_pkgs=""; \ - for arch in $(echo $TARGET_ARCHS | tr ',' ' '); do \ - qemu_pkgs="$qemu_pkgs qemu-$arch"; \ - done; \ - apk add --no-cache \ - bash-completion \ - build-base \ - wget \ - curl \ - vim \ - tar \ - ripgrep \ - less \ - gcc \ - g++ \ - clang19 \ - git \ - xz \ - gzip \ - bzip2 \ - zstd \ - ccache \ - ninja-build \ - cmake \ - make \ - bison \ - flex \ - ronn \ - fuse \ - fuse3 \ - pkgconf \ - meson \ - autoconf \ - strace \ - gdb \ - gmp-dev \ - zlib-static \ - zstd-static \ - zlib-dev \ - zstd-dev \ - mpfr-dev \ - mpc1-dev \ - isl-dev \ - texinfo \ - linux-headers \ - rsync \ - $qemu_pkgs - -COPY fetch.sh /usr/local/bin/fetch.sh - -# Install mold -COPY install-mold.sh /usr/local/bin/install-mold.sh -RUN --mount=type=cache,id=ccache,target=/root/.ccache --mount=type=cache,id=pkgcache,target=/root/.pkgcache \ - bash /usr/local/bin/install-mold.sh - -# Install bloaty -COPY install-bloaty.sh /usr/local/bin/install-bloaty.sh -RUN --mount=type=cache,id=ccache,target=/root/.ccache --mount=type=cache,id=pkgcache,target=/root/.pkgcache \ - if [ "$ARCH" == "amd64" ]; then \ - bash /usr/local/bin/install-bloaty.sh; \ - else \ - echo "Skipping bloaty installation for $ARCH."; \ - fi - -RUN apk del linux-headers - -# Install size-optimized and static-link-optimized libstdc++ -COPY install-toolchain.sh /usr/local/bin/install-toolchain.sh -RUN --mount=type=cache,id=ccache,target=/root/.ccache --mount=type=cache,id=pkgcache,target=/root/.pkgcache \ - bash /usr/local/bin/install-toolchain.sh $TARGET_ARCHS $TOOLCHAIN_OPTIMIZE - -RUN apk del zlib-dev zstd-dev - -# Clear the config cache after a toolchain update -RUN --mount=type=cache,id=cfgcache,target=/root/.cfgcache \ - rm -rf /root/.cfgcache/* - -# Install all static libraries -COPY install-static-libs.sh /usr/local/bin/install-static-libs.sh - -RUN --mount=type=cache,id=ccache,target=/root/.ccache --mount=type=cache,id=pkgcache,target=/root/.pkgcache \ - --mount=type=cache,id=cfgcache,target=/root/.cfgcache \ - bash /usr/local/bin/install-static-libs.sh :all $TARGET_ARCHS - -# Install UPX -# UPX isn't supported on RISC-V/S390 (yet), so skip this -RUN if ! [ "$ARCH" == "amd64" ] || [ "$ARCH" == "arm64v8" ]; then \ - wget -O - https://github.com/upx/upx/releases/download/v5.0.2/upx-5.0.2-$(bash -c "echo \${0//v8/}" $ARCH)_linux.tar.xz | tar -xJf - -C /usr/local/bin --strip-components=1 --wildcards "*/upx"; \ - else \ - echo "Skipping UPX installation for $ARCH architecture while it is not supported."; \ - echo "See https://github.com/upx/upx/discussions/793 for details."; \ - fi - -# Set up git & user -RUN git config --global --add safe.directory /workspace -RUN adduser -G users -s bash -u 1000 -D mhx -USER mhx -ENTRYPOINT /workspace/.docker/build-linux.sh diff --git a/.docker/Makefile b/.docker/Makefile index c67d55ba..a823c7d9 100644 --- a/.docker/Makefile +++ b/.docker/Makefile @@ -1,8 +1,6 @@ IMAGE_UBUNTU="dwarfs-buildenv" IMAGE_DEBIAN="dwarfs-buildenv-debian" IMAGE_ALPINE="dwarfs-buildenv-alpine" -IMAGE_ALPINE_RISCV="dwarfs-buildenv-alpine-riscv64" -IMAGE_ALPINE_XX="dwarfs-buildenv-alpine-xx" IMAGE_UBUNTU2204="dwarfs-buildenv-ubuntu2204" IMAGE_FEDORA="dwarfs-buildenv-fedora" IMAGE_ARCH="dwarfs-buildenv-arch" @@ -35,23 +33,11 @@ run_debian: @docker run $(COMMON_RUN_OPTS) $(IMAGE_DEBIAN) build_alpine: - docker build -f Dockerfile.alpine -t $(IMAGE_ALPINE) . + docker buildx build -f Dockerfile.alpine --build-arg TARGET_ARCHS=x86_64,aarch64,riscv64,i386 -t $(IMAGE_ALPINE) . run_alpine: @docker run $(COMMON_RUN_OPTS) $(IMAGE_ALPINE) -build_alpine_xx: - docker buildx build -f Dockerfile.alpine-xx --build-arg TARGET_ARCHS=x86_64,riscv64,i386 -t $(IMAGE_ALPINE_XX) . - -run_alpine_xx: - @docker run $(COMMON_RUN_OPTS) $(IMAGE_ALPINE_XX) - -build_alpine_riscv: - docker build -f Dockerfile.alpine -t $(IMAGE_ALPINE_RISCV) --build-arg ARCH=riscv64 . - -run_alpine_riscv: - @docker run $(COMMON_RUN_OPTS) $(IMAGE_ALPINE_RISCV) - build_ubuntu2204: docker build -f Dockerfile.ubuntu-2204 -t $(IMAGE_UBUNTU2204) . diff --git a/.docker/install-static-libs.sh b/.docker/install-static-libs.sh index 6ce74f82..a1619ee3 100644 --- a/.docker/install-static-libs.sh +++ b/.docker/install-static-libs.sh @@ -9,7 +9,7 @@ cd pkgs ARCH="$(uname -m)" GCC="gcc" -CLANG="clang" +CLANG="clang-20" PKGS="$1" TARGET_ARCH_STR="${2:-${ARCH}}" @@ -398,16 +398,17 @@ EOF fetch.sh https://gitlab.alpinelinux.org/alpine/aports/-/raw/abc0b4170e42e2a7d835e4490ecbae49e6f3d137/main/jemalloc/musl-exception-specification-errors.patch - | patch -p1 fetch.sh https://gitlab.alpinelinux.org/alpine/aports/-/raw/abc0b4170e42e2a7d835e4490ecbae49e6f3d137/main/jemalloc/pkgconf.patch - | patch -p1 ./autogen.sh ${TRIPLETS} - # mkdir build-minimal - # cd build-minimal - # ../configure ${TRIPLETS} --prefix="$INSTALL_DIR-jemalloc-minimal" --localstatedir=/var --sysconfdir=/etc --with-lg-hugepage=21 --disable-stats --disable-prof --enable-static --disable-shared --disable-log --disable-debug - # make -j$(nproc) - # make install - # cd .. - # mkdir build-full - mkdir build - cd build - ../configure ${TRIPLETS} --prefix="$INSTALL_DIR" --localstatedir=/var --sysconfdir=/etc --with-lg-hugepage=21 \ + mkdir build-minimal + cd build-minimal + ../configure ${TRIPLETS} --prefix="$INSTALL_DIR-jemalloc-minimal" --localstatedir=/var --sysconfdir=/etc --with-lg-hugepage=21 \ + --disable-stats --disable-prof --enable-static --disable-shared --disable-log --disable-debug \ + --cache-file=$HOME/.cfgcache/jemalloc-minimal-${JEMALLOC_VERSION}-${CARCH}-${COMPILER}.cache + make -j$(nproc) + make install + cd .. + mkdir build-full + cd build-full + ../configure ${TRIPLETS} --prefix="$INSTALL_DIR-jemalloc-full" --localstatedir=/var --sysconfdir=/etc --with-lg-hugepage=21 \ --enable-stats --enable-prof --enable-static --disable-shared --disable-log --disable-debug \ --cache-file=$HOME/.cfgcache/jemalloc-${JEMALLOC_VERSION}-${CARCH}-${COMPILER}.cache make -j$(nproc) diff --git a/.docker/install-toolchain.sh b/.docker/install-toolchain.sh index 22883d5e..1bc67d7b 100644 --- a/.docker/install-toolchain.sh +++ b/.docker/install-toolchain.sh @@ -3,8 +3,8 @@ set -ex ARCH="$(uname -m)" -TARGET_ARCH_STR="${1:-$ARCH}" -OPTIMIZE_STR="${2:-s}" +OPTIMIZE_STR="${1:-s}" +TARGET_ARCH_STR="${2:-$ARCH}" BINUTILS_VERSION=2.44 GCC_VERSION=14.2.0 @@ -173,14 +173,20 @@ for target_arch in ${TARGET_ARCH_STR//,/ }; do make -j"$(nproc)" ${GCC_NODOCS} make install - # Symbolic links for clang - ln -s /usr/bin/clang $PREFIX/bin/$TARGET-clang - ln -s /usr/bin/clang++ $PREFIX/bin/$TARGET-clang++ - - # Also provide ccache symlinks + # Directory for ccache symlinks mkdir -p $PREFIX/lib/ccache/bin - for binary in gcc g++ clang clang++; do - ln -s /usr/bin/ccache $PREFIX/lib/ccache/bin/$TARGET-$binary + + # Symbolic links for clang + for clang_binary in /usr/bin/clang{++,}{-[1-9]*,}; do + name=$(basename $clang_binary) + ln -s $clang_binary $PREFIX/bin/$TARGET-$name + ln -s /usr/bin/ccache $PREFIX/lib/ccache/bin/$TARGET-$name + done + + # Also provide ccache symlinks for gcc + for gcc_binary in $PREFIX/bin/$TARGET-{c++,g++,gcc,gcc-[1-9]*}; do + name=$(basename $gcc_binary) + ln -s /usr/bin/ccache $PREFIX/lib/ccache/bin/$name done done done