diff --git a/.docker/Dockerfile b/.docker/Dockerfile deleted file mode 100644 index 146fb68a..00000000 --- a/.docker/Dockerfile +++ /dev/null @@ -1,79 +0,0 @@ -# syntax=docker/dockerfile:1 -ARG ARCH=amd64 -FROM $ARCH/ubuntu:24.04 -RUN apt update -RUN apt upgrade -y -RUN apt install -y gnupg wget software-properties-common apt-utils -RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections -RUN apt install -y \ - gcc-11 \ - g++-11 \ - gcc-12 \ - g++-12 \ - gcc-13 \ - g++-13 \ - gcc-14 \ - g++-14 \ - clang-15 \ - clang-16 \ - clang-17 \ - clang-18 \ - git \ - vim \ - zstd \ - ccache \ - ninja-build \ - cmake \ - make \ - bison \ - flex \ - ronn \ - fuse3 \ - pkg-config \ - binutils-dev \ - libacl1-dev \ - libarchive-dev \ - libbenchmark-dev \ - libboost-chrono-dev \ - libboost-context-dev \ - libboost-filesystem-dev \ - libboost-iostreams-dev \ - libboost-program-options-dev \ - libboost-regex-dev \ - libboost-system-dev \ - libboost-thread-dev \ - libbrotli-dev \ - libevent-dev \ - libhowardhinnant-date-dev \ - libjemalloc-dev \ - libdouble-conversion-dev \ - libiberty-dev \ - liblz4-dev \ - liblzma-dev \ - libmagic-dev \ - librange-v3-dev \ - libssl-dev \ - libunwind-dev \ - libdwarf-dev \ - libelf-dev \ - libfmt-dev \ - libfuse-dev \ - libfuse3-dev \ - libgoogle-glog-dev \ - libutfcpp-dev \ - libflac++-dev \ - python3-mistletoe \ - bash-completion -COPY install-static-libs.sh /usr/local/bin/install-static-libs.sh -RUN bash /usr/local/bin/install-static-libs.sh -ARG ARCH -# upx-4.2.2 is broken for arm64 (https://github.com/upx/upx/issues/758) -RUN wget -O - https://github.com/upx/upx/releases/download/v4.2.1/upx-4.2.1-$(bash -c "echo \${0//v8/}" $ARCH)_linux.tar.xz | tar -xJf - -C /usr/local/bin --strip-components=1 --wildcards "*/upx" -# current ronn version is horribly broken -RUN gem install ronn-ng -# RUN useradd -g users -u 1000 -m mhx -ARG SCRIPT=build-linux.sh -COPY $SCRIPT /usr/local/bin/run.sh -RUN chmod +x /usr/local/bin/run.sh -USER ubuntu -ENTRYPOINT /usr/local/bin/run.sh diff --git a/.docker/Dockerfile b/.docker/Dockerfile new file mode 120000 index 00000000..fd45f0f7 --- /dev/null +++ b/.docker/Dockerfile @@ -0,0 +1 @@ +Dockerfile.ubuntu \ No newline at end of file diff --git a/.docker/Dockerfile.arch b/.docker/Dockerfile.arch index d4e38d06..f125aa60 100644 --- a/.docker/Dockerfile.arch +++ b/.docker/Dockerfile.arch @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 -FROM archlinux/archlinux:latest +FROM archlinux:latest RUN pacman -Syu --noconfirm -RUN pacman -S --noconfirm \ +RUN pacman -S --noconfirm --needed \ bash-completion \ vim \ devtools \ @@ -42,6 +42,9 @@ RUN pacman -S --noconfirm --needed \ google-glog \ flac \ utf8cpp +ARG SCRIPT=build-linux.sh +COPY $SCRIPT /usr/local/bin/run.sh +RUN chmod +x /usr/local/bin/run.sh RUN useradd -g users -u 1000 -m mhx USER mhx -ENTRYPOINT /bin/bash +ENTRYPOINT /usr/local/bin/run.sh diff --git a/.docker/Dockerfile.arch_head b/.docker/Dockerfile.arch_head new file mode 100644 index 00000000..d4e38d06 --- /dev/null +++ b/.docker/Dockerfile.arch_head @@ -0,0 +1,47 @@ +# syntax=docker/dockerfile:1 +FROM archlinux/archlinux:latest +RUN pacman -Syu --noconfirm +RUN pacman -S --noconfirm \ + bash-completion \ + vim \ + devtools \ + less +RUN pacman -S --noconfirm --needed \ + gcc \ + clang \ + man \ + git \ + zstd \ + ccache \ + ninja \ + cmake \ + make \ + bison \ + flex \ + ruby-ronn \ + python-mistletoe \ + fuse \ + fuse3 \ + pkg-config \ + binutils \ + libarchive \ + benchmark \ + boost \ + brotli \ + libevent \ + openssl \ + fmt \ + chrono-date \ + range-v3 \ + jemalloc \ + double-conversion \ + lz4 \ + xz \ + libunwind \ + libdwarf \ + google-glog \ + flac \ + utf8cpp +RUN useradd -g users -u 1000 -m mhx +USER mhx +ENTRYPOINT /bin/bash diff --git a/.docker/Dockerfile.fedora b/.docker/Dockerfile.fedora index d70b8202..8a661812 100644 --- a/.docker/Dockerfile.fedora +++ b/.docker/Dockerfile.fedora @@ -2,6 +2,10 @@ ARG ARCH=amd64 FROM $ARCH/fedora:rawhide RUN dnf update -y +RUN dnf install -y \ + bash-completion \ + vim \ + less RUN dnf install -y \ gcc \ g++ \ @@ -9,7 +13,6 @@ RUN dnf install -y \ man \ pip \ git \ - vim \ zstd \ ccache \ ninja-build \ @@ -41,8 +44,11 @@ RUN dnf install -y \ glog-devel \ flac-devel \ utf8cpp-devel \ - bash-completion + range-v3-devel RUN pip install mistletoe +ARG SCRIPT=build-linux.sh +COPY $SCRIPT /usr/local/bin/run.sh +RUN chmod +x /usr/local/bin/run.sh RUN useradd -g users -u 1000 -m mhx USER mhx -ENTRYPOINT /bin/bash +ENTRYPOINT /usr/local/bin/run.sh diff --git a/.docker/Dockerfile.ubuntu b/.docker/Dockerfile.ubuntu new file mode 100644 index 00000000..eb1de88d --- /dev/null +++ b/.docker/Dockerfile.ubuntu @@ -0,0 +1,79 @@ +# syntax=docker/dockerfile:1 +ARG ARCH=amd64 +FROM $ARCH/ubuntu:23.10 +RUN apt update +RUN apt upgrade -y +RUN apt install -y gnupg wget software-properties-common +RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null +RUN apt-add-repository 'deb https://apt.kitware.com/ubuntu/ jammy main' +RUN apt install -y \ + bash-completion \ + vim \ + less +RUN apt install -y \ + gcc-11 \ + g++-11 \ + gcc-12 \ + g++-12 \ + gcc-13 \ + g++-13 \ + clang-15 \ + clang-16 \ + clang-17 \ + git \ + zstd \ + ccache \ + ninja-build \ + cmake \ + make \ + bison \ + flex \ + ronn \ + fuse3 \ + pkg-config \ + binutils-dev \ + libacl1-dev \ + libarchive-dev \ + libbenchmark-dev \ + libboost-chrono-dev \ + libboost-context-dev \ + libboost-filesystem-dev \ + libboost-iostreams-dev \ + libboost-program-options-dev \ + libboost-regex-dev \ + libboost-system-dev \ + libboost-thread-dev \ + libbrotli-dev \ + libevent-dev \ + libhowardhinnant-date-dev \ + libjemalloc-dev \ + libdouble-conversion-dev \ + libiberty-dev \ + liblz4-dev \ + liblzma-dev \ + libmagic-dev \ + librange-v3-dev \ + libssl-dev \ + libunwind-dev \ + libdwarf-dev \ + libelf-dev \ + libfmt-dev \ + libfuse-dev \ + libfuse3-dev \ + libgoogle-glog-dev \ + libutfcpp-dev \ + libflac++-dev \ + python3-mistletoe +COPY install-static-libs.sh /usr/local/bin/install-static-libs.sh +RUN bash /usr/local/bin/install-static-libs.sh +ARG ARCH +# upx-4.2.2 is broken for arm64 (https://github.com/upx/upx/issues/758) +RUN wget -O - https://github.com/upx/upx/releases/download/v4.2.1/upx-4.2.1-$(bash -c "echo \${0//v8/}" $ARCH)_linux.tar.xz | tar -xJf - -C /usr/local/bin --strip-components=1 --wildcards "*/upx" +# current ronn version is horribly broken +RUN gem install ronn-ng +# RUN useradd -g users -u 1000 -m mhx +ARG SCRIPT=build-linux.sh +COPY $SCRIPT /usr/local/bin/run.sh +RUN chmod +x /usr/local/bin/run.sh +USER ubuntu +ENTRYPOINT /usr/local/bin/run.sh diff --git a/.docker/Makefile b/.docker/Makefile index e53d58ab..4ee989c7 100644 --- a/.docker/Makefile +++ b/.docker/Makefile @@ -1,4 +1,4 @@ -IMAGE="dwarfs-buildenv" +IMAGE_UBUNTU="dwarfs-buildenv" IMAGE_FEDORA="dwarfs-buildenv-fedora" IMAGE_ARCH="dwarfs-buildenv-arch" @@ -6,10 +6,10 @@ ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) NPROC:=$(shell nproc) build: - docker build -t $(IMAGE) . + docker build -t $(IMAGE_UBUNTU) . build_arm: - docker build -t $(IMAGE) --build-arg ARCH=arm64v8 . + docker build -f Dockerfile.ubuntu -t $(IMAGE_UBUNTU) --build-arg ARCH=arm64v8 . run: @docker run -it --rm \ @@ -19,7 +19,7 @@ run: --mount type=bind,source="$(ROOT_DIR)"/../@docker-home,target=/home/ubuntu \ --mount type=bind,source="$(ROOT_DIR)"/../@docker-build,target=/home/ubuntu/build \ --env CCACHE_DIR=/ccache --env CTEST_PARALLEL_LEVEL=$(NPROC) \ - --entrypoint /bin/bash $(IMAGE) + --entrypoint /bin/bash $(IMAGE_UBUNTU) build_fedora: docker build -f Dockerfile.fedora -t $(IMAGE_FEDORA) . diff --git a/.docker/build-linux.sh b/.docker/build-linux.sh index cde0bd45..8bdea531 100644 --- a/.docker/build-linux.sh +++ b/.docker/build-linux.sh @@ -15,6 +15,9 @@ rm -rf build mkdir build cd build +GCC_VERSION=13 +CLANG_VERSION=17 + case "-$BUILD_TYPE-" in *-ninja-*) BUILD_TOOL=ninja @@ -31,18 +34,29 @@ esac case "-$BUILD_TYPE-" in *-gcc-*) - export CC=gcc-14 CXX=g++-14 + case "-$BUILD_DIST-" in + *-ubuntu-*) + export CC=gcc-$GCC_VERSION CXX=g++-$GCC_VERSION + ;; + esac export COMPILER=gcc ;; *-oldgcc-*) export CC=gcc-11 CXX=g++-11 ;; *-clang-*) - export CC=clang-18 CXX=clang++-18 + case "-$BUILD_DIST-" in + *-ubuntu-*) + export CC=clang-$CLANG_VERSION CXX=clang++-$CLANG_VERSION + ;; + *) + export CC=clang CXX=clang++ + ;; + esac export COMPILER=clang ;; *-oldclang-*) - export CC=clang-15 CXX=clang++-16 + export CC=clang-15 CXX=clang++-15 ;; *) echo "missing compiler in: $BUILD_TYPE" @@ -114,9 +128,9 @@ if [[ "-$BUILD_TYPE-" == *-coverage-* ]]; then unset LLVM_PROFILE_FILE rm -rf /tmp-runner/coverage mkdir -p /tmp-runner/coverage - llvm-profdata-18 merge -sparse profile/* -o dwarfs.profdata + llvm-profdata-$CLANG_VERSION merge -sparse profile/* -o dwarfs.profdata for binary in mkdwarfs dwarfs dwarfsck dwarfsextract *_test ricepp/ricepp_test; do - llvm-cov-18 show -instr-profile=dwarfs.profdata $binary >/tmp-runner/coverage/$(basename $binary).txt + llvm-cov-$CLANG_VERSION show -instr-profile=dwarfs.profdata $binary >/tmp-runner/coverage/$(basename $binary).txt done fi diff --git a/.docker/install-static-libs.sh b/.docker/install-static-libs.sh index fe179553..56131c52 100644 --- a/.docker/install-static-libs.sh +++ b/.docker/install-static-libs.sh @@ -6,6 +6,9 @@ cd "$HOME" mkdir pkgs cd pkgs +GCC_VERSION=13 +CLANG_VERSION=17 + FILE_VERSION=5.45 FILE_SHA512=12611a59ff766c22a55db4b4a9f80f95a0a2e916a1d8593612c6ead32c247102a8fdc23693c6bf81bda9b604d951a62c0051e91580b1b79e190a3504c0efc20a BZIP2_VERSION=1.0.8 @@ -36,11 +39,11 @@ wget https://github.com/google/benchmark/archive/refs/tags/v${BENCHMARK_VERSION} for COMPILER in clang gcc; do if [[ "$COMPILER" == "clang" ]]; then - export CC=clang-18 - export CXX=clang++-18 + export CC=clang-$CLANG_VERSION + export CXX=clang++-$CLANG_VERSION elif [[ "$COMPILER" == "gcc" ]]; then - export CC=gcc-13 - export CXX=g++-13 + export CC=gcc-$GCC_VERSION + export CXX=g++-$GCC_VERSION else echo "Unknown compiler: $COMPILER" exit 1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 02cb0a77..7c720abb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -200,31 +200,82 @@ jobs: arch: - amd64 - arm64v8 + dist: + - ubuntu + - fedora + - arch build_type: - gcc-debug-ninja - gcc-release-ninja - clang-release-ninja - - gcc-release-ninja-static - - gcc-release-ninja-O2-static - - clang-release-ninja-static - - clang-release-ninja-O2-static - - clang-reldbg-ninja-static - - clang-debug-ninja-static + exclude: + - arch: arm64v8 + dist: arch + build_type: gcc-debug-ninja + - arch: arm64v8 + dist: arch + build_type: gcc-release-ninja + - arch: arm64v8 + dist: arch + build_type: clang-release-ninja include: - arch: amd64 + dist: ubuntu build_type: clang-asan-ninja - arch: amd64 + dist: ubuntu build_type: clang-tsan-ninja - arch: amd64 + dist: ubuntu build_type: clang-ubsan-ninja - arch: amd64 + dist: ubuntu build_type: clang-coverage-ninja - arch: amd64 + dist: ubuntu build_type: oldgcc-debug-make - arch: arm64v8 + dist: ubuntu build_type: gcc-debug-noperfmon-ninja - arch: arm64v8 + dist: ubuntu build_type: oldclang-debug-make + - arch: amd64 + dist: ubuntu + build_type: gcc-release-ninja-static + - arch: arm64v8 + dist: ubuntu + build_type: gcc-release-ninja-static + - arch: amd64 + dist: ubuntu + build_type: gcc-release-ninja-O2-static + - arch: arm64v8 + dist: ubuntu + build_type: gcc-release-ninja-O2-static + - arch: amd64 + dist: ubuntu + build_type: clang-release-ninja-static + - arch: arm64v8 + dist: ubuntu + build_type: clang-release-ninja-static + - arch: amd64 + dist: ubuntu + build_type: clang-release-ninja-O2-static + - arch: arm64v8 + dist: ubuntu + build_type: clang-release-ninja-O2-static + - arch: amd64 + dist: ubuntu + build_type: clang-reldbg-ninja-static + - arch: arm64v8 + dist: ubuntu + build_type: clang-reldbg-ninja-static + - arch: amd64 + dist: ubuntu + build_type: clang-debug-ninja-static + - arch: arm64v8 + dist: ubuntu + build_type: clang-debug-ninja-static runs-on: - self-hosted @@ -242,7 +293,8 @@ jobs: - name: Build Docker Image run: | docker build \ - -t dwarfs-linux-build \ + -f .docker/Dockerfile.${{ matrix.dist }} \ + -t dwarfs-linux-build-${{ matrix.dist }} \ --build-arg ARCH=${{ matrix.arch }} \ --build-arg SCRIPT=build-linux.sh \ .docker @@ -257,12 +309,13 @@ jobs: --mount type=bind,source=/mnt/opensource/artifacts/dwarfs,target=/artifacts \ --env BUILD_TYPE=${{ matrix.build_type }} \ --env BUILD_ARCH=${{ matrix.arch }} \ + --env BUILD_DIST=${{ matrix.dist }} \ --env GITHUB_REF_NAME \ --env GITHUB_REF_TYPE \ --env GITHUB_RUN_ID \ --env GITHUB_RUN_NUMBER \ --env GITHUB_RUN_ATTEMPT \ - dwarfs-linux-build + dwarfs-linux-build-${{ matrix.dist }} - name: List Coverage Files id: coverage-files