diff --git a/.docker/build-linux.sh b/.docker/build-linux.sh index 54456192..2d687f9c 100755 --- a/.docker/build-linux.sh +++ b/.docker/build-linux.sh @@ -297,114 +297,108 @@ if [[ "-$BUILD_TYPE-" == *-static-* ]]; then CMAKE_ARGS="${CMAKE_ARGS} -DSTATIC_BUILD_EXTRA_PREFIX=/opt/static-libs/$COMPILER;$_sslprefix" fi -if [[ "$BUILD_FROM_TARBALL" == "1" ]]; then - INSTALLDIR="$HOME/install" - PREFIXPATH="$INSTALLDIR/usr/local" - rm -rf "$INSTALLDIR" +INSTALLDIR="$HOME/install" +PREFIXPATH="$INSTALLDIR/usr/local" +rm -rf "$INSTALLDIR" - if [[ "-$BUILD_TYPE-" == *-shared-* ]]; then - LDLIBPATH="$(readlink -m "$PREFIXPATH/lib/$(gcc -print-multi-os-directory)")" - if [[ ":$LD_LIBRARY_PATH:" != *":$LDLIBPATH:"* ]]; then - export "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}$LDLIBPATH" - fi - fi - - case "-$BUILD_TYPE-" in - *-full-*) - cmake ../dwarfs/ $CMAKE_ARGS - time $BUILD_TOOL - $RUN_TESTS - DESTDIR="$INSTALLDIR" $BUILD_TOOL install - $BUILD_TOOL distclean - ;; - - *-split-*) - # ==== libdwarfs ==== - cmake ../dwarfs/ $CMAKE_ARGS -DWITH_LIBDWARFS=ON -DWITH_TOOLS=OFF -DWITH_FUSE_DRIVER=OFF - time $BUILD_TOOL - $RUN_TESTS - DESTDIR="$INSTALLDIR" $BUILD_TOOL install - $BUILD_TOOL distclean - rm -rf * - - # ==== example binary ==== - cmake ../dwarfs/example $CMAKE_TOOL_ARGS -DCMAKE_PREFIX_PATH="$PREFIXPATH" - time $BUILD_TOOL - rm -rf * - - # ==== dwarfs tools ==== - cmake ../dwarfs/ $CMAKE_ARGS -DWITH_LIBDWARFS=OFF -DWITH_TOOLS=ON -DWITH_FUSE_DRIVER=OFF -DCMAKE_PREFIX_PATH="$PREFIXPATH" - time $BUILD_TOOL - $RUN_TESTS - DESTDIR="$INSTALLDIR" $BUILD_TOOL install - $BUILD_TOOL distclean - rm -rf * - - # ==== dwarfs fuse driver ==== - cmake ../dwarfs/ $CMAKE_ARGS -DWITH_LIBDWARFS=OFF -DWITH_TOOLS=OFF -DWITH_FUSE_DRIVER=ON -DCMAKE_PREFIX_PATH="$PREFIXPATH" - time $BUILD_TOOL - $RUN_TESTS - DESTDIR="$INSTALLDIR" $BUILD_TOOL install - $BUILD_TOOL distclean - ;; - - *) - echo "builds from source tarball must be 'full' or 'split'" - exit 1 - ;; - esac -else - # shellcheck disable=SC2086 - cmake ../dwarfs/ $CMAKE_ARGS -DWITH_EXAMPLE=1 - - time $BUILD_TOOL - - $RUN_TESTS - - if [[ "-$BUILD_TYPE-" == *-coverage-* ]]; then - rm -f /tmp-runner/dwarfs-coverage.txt - llvm-profdata$CLANG_VERSION merge -sparse profile/* -o dwarfs.profdata - llvm-cov$CLANG_VERSION show -instr-profile=dwarfs.profdata \ - $(for i in mkdwarfs dwarfs dwarfsck dwarfsextract *_test ricepp/ricepp_test; do echo $i; done | sed -e's/^/-object=/') \ - >/tmp-runner/dwarfs-coverage.txt - fi - - if [[ "-$BUILD_TYPE-" == *-static-* ]]; then - # for release and relsize builds, strip the binaries - if [[ "-$BUILD_TYPE-" =~ -(release|relsize)- ]]; then - $BUILD_TOOL strip - fi - - $BUILD_TOOL package - $BUILD_TOOL universal_upx - - $BUILD_TOOL copy_artifacts - - rm -rf /tmp-runner/artifacts - mkdir -p /tmp-runner/artifacts - cp artifacts.env /tmp-runner - cp dwarfs-universal-* /tmp-runner/artifacts - cp dwarfs-fuse-extract-* /tmp-runner/artifacts - cp dwarfs-*-Linux*.tar.zst /tmp-runner/artifacts - elif [[ "-$BUILD_TYPE-" == *-source-* ]]; then - $BUILD_TOOL package_source - $BUILD_TOOL copy_source_artifacts - fi - - if [[ "-$BUILD_TYPE-" != *-[at]san-* ]] && \ - [[ "-$BUILD_TYPE-" != *-ubsan-* ]] && \ - [[ "-$BUILD_TYPE-" != *-source-* ]] && \ - [[ "-$BUILD_TYPE-" != *-static-* ]]; then - INSTALLDIR="$HOME/install" - rm -rf "$INSTALLDIR" - DESTDIR="$INSTALLDIR" $BUILD_TOOL install - - $BUILD_TOOL distclean - - cmake ../dwarfs/example $CMAKE_TOOL_ARGS -DCMAKE_PREFIX_PATH="$INSTALLDIR/usr/local" - $BUILD_TOOL - $BUILD_TOOL clean - else - $BUILD_TOOL distclean +if [[ "-$BUILD_TYPE-" == *-shared-* ]]; then + LDLIBPATH="$(readlink -m "$PREFIXPATH/lib/$(gcc -print-multi-os-directory)")" + if [[ ":$LD_LIBRARY_PATH:" != *":$LDLIBPATH:"* ]]; then + export "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}$LDLIBPATH" fi fi + +case "-$BUILD_TYPE-" in + *-full-*) + cmake ../dwarfs/ $CMAKE_ARGS -DWITH_EXAMPLE=1 + time $BUILD_TOOL + $RUN_TESTS + ;; + + *-split-*) + # ==== libdwarfs ==== + cmake ../dwarfs/ $CMAKE_ARGS -DWITH_LIBDWARFS=ON -DWITH_TOOLS=OFF -DWITH_FUSE_DRIVER=OFF + time $BUILD_TOOL + $RUN_TESTS + DESTDIR="$INSTALLDIR" $BUILD_TOOL install + $BUILD_TOOL distclean + rm -rf * + + # ==== example binary ==== + cmake ../dwarfs/example $CMAKE_TOOL_ARGS -DCMAKE_PREFIX_PATH="$PREFIXPATH" + time $BUILD_TOOL + rm -rf * + + # ==== dwarfs tools ==== + cmake ../dwarfs/ $CMAKE_ARGS -DWITH_LIBDWARFS=OFF -DWITH_TOOLS=ON -DWITH_FUSE_DRIVER=OFF -DCMAKE_PREFIX_PATH="$PREFIXPATH" + time $BUILD_TOOL + $RUN_TESTS + DESTDIR="$INSTALLDIR" $BUILD_TOOL install + $BUILD_TOOL distclean + rm -rf * + + # ==== dwarfs fuse driver ==== + cmake ../dwarfs/ $CMAKE_ARGS -DWITH_LIBDWARFS=OFF -DWITH_TOOLS=OFF -DWITH_FUSE_DRIVER=ON -DCMAKE_PREFIX_PATH="$PREFIXPATH" + time $BUILD_TOOL + $RUN_TESTS + DESTDIR="$INSTALLDIR" $BUILD_TOOL install + $BUILD_TOOL distclean + + # That's it for split builds, we are done + exit 0 + ;; + + *) + if [[ "-$BUILD_TYPE-" == *-static-* ]]; then + cmake ../dwarfs/ $CMAKE_ARGS + else + cmake ../dwarfs/ $CMAKE_ARGS -DWITH_EXAMPLE=1 + fi + time $BUILD_TOOL + $RUN_TESTS + ;; +esac + +if [[ "-$BUILD_TYPE-" == *-coverage-* ]]; then + rm -f /tmp-runner/dwarfs-coverage.txt + llvm-profdata$CLANG_VERSION merge -sparse profile/* -o dwarfs.profdata + llvm-cov$CLANG_VERSION show -instr-profile=dwarfs.profdata \ + $(for i in mkdwarfs dwarfs dwarfsck dwarfsextract *_test ricepp/ricepp_test; do echo $i; done | sed -e's/^/-object=/') \ + >/tmp-runner/dwarfs-coverage.txt +fi + +if [[ "-$BUILD_TYPE-" == *-static-* ]]; then + # for release and relsize builds, strip the binaries + if [[ "-$BUILD_TYPE-" =~ -(release|relsize)- ]]; then + $BUILD_TOOL strip + fi + + $BUILD_TOOL package + $BUILD_TOOL universal_upx + + $BUILD_TOOL copy_artifacts + + rm -rf /tmp-runner/artifacts + mkdir -p /tmp-runner/artifacts + cp artifacts.env /tmp-runner + cp dwarfs-universal-* /tmp-runner/artifacts + cp dwarfs-fuse-extract-* /tmp-runner/artifacts + cp dwarfs-*-Linux*.tar.zst /tmp-runner/artifacts +elif [[ "-$BUILD_TYPE-" == *-source-* ]]; then + $BUILD_TOOL package_source + $BUILD_TOOL copy_source_artifacts +fi + +if [[ "-$BUILD_TYPE-" != *-[at]san-* ]] && \ + [[ "-$BUILD_TYPE-" != *-ubsan-* ]] && \ + [[ "-$BUILD_TYPE-" != *-source-* ]] && \ + [[ "-$BUILD_TYPE-" != *-static-* ]]; then + DESTDIR="$INSTALLDIR" $BUILD_TOOL install + $BUILD_TOOL distclean + + cmake ../dwarfs/example $CMAKE_TOOL_ARGS -DCMAKE_PREFIX_PATH="$INSTALLDIR/usr/local" + $BUILD_TOOL + $BUILD_TOOL clean +else + $BUILD_TOOL distclean +fi diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 51bd797a..1f590e07 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -92,48 +92,18 @@ jobs: # env: # ACTIONS_ARTIFACT_UPLOAD_CONCURRENCY: 5 +################################################################################ + package-source: - runs-on: - - self-hosted - - Linux - - amd64 + uses: ./.github/workflows/docker-run-build.yml + with: + build_type: clang-release-ninja-source-notest + build_arch: amd64 + build_dist: arch - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: 'true' - fetch-depth: '0' - ref: ${{ github.ref }} +################################################################################ - - name: Build Docker Image - run: | - docker build \ - -f .docker/Dockerfile.arch \ - -t dwarfs-linux-build-arch \ - --build-arg ARCH=amd64 \ - .docker - - - name: Run Build - run: | - docker run --rm \ - --cap-add SYS_ADMIN --device /dev/fuse --privileged \ - --mount type=bind,source=${GITHUB_WORKSPACE},target=/workspace,readonly \ - --mount type=bind,source=${{ runner.temp }},target=/tmp-runner \ - --mount type=bind,source=${HOME}/github-ccache,target=/ccache \ - --mount type=bind,source=${HOME}/github-local,target=/local \ - --mount type=bind,source=/mnt/opensource/artifacts/dwarfs,target=/artifacts \ - --env BUILD_TYPE=clang-release-ninja-source-notest \ - --env BUILD_ARCH=amd64 \ - --env BUILD_DIST=arch \ - --env GITHUB_REF_NAME \ - --env GITHUB_REF_TYPE \ - --env GITHUB_RUN_ID \ - --env GITHUB_RUN_NUMBER \ - --env GITHUB_RUN_ATTEMPT \ - dwarfs-linux-build-arch - - linux-src: + linux: needs: package-source strategy: @@ -163,7 +133,28 @@ jobs: - arch: arm64v8 dist: arch build_type: clang-release-shared-ninja-full + - arch: arm64v8 + dist: ubuntu + build_type: gcc-release-ninja-split + - arch: arm64v8 + dist: ubuntu + build_type: gcc-release-shared-ninja-split + - arch: arm64v8 + dist: fedora + build_type: gcc-release-ninja-split + - arch: arm64v8 + dist: fedora + build_type: gcc-release-shared-ninja-split include: + - arch: amd64 + dist: ubuntu + build_type: clang-reldbg-shared-asan-ninja + - arch: amd64 + dist: ubuntu + build_type: clang-debug-shared-tsan-ninja + - arch: amd64 + dist: ubuntu + build_type: clang-reldbg-shared-ubsan-ninja - arch: amd64 dist: ubuntu build_type: oldgcc-debug-shared-make-split @@ -182,82 +173,6 @@ jobs: - arch: amd64 dist: ubuntu-2204 build_type: clang-release-ninja-split - - runs-on: - - self-hosted - - Linux - - ${{ matrix.arch }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: 'true' - fetch-depth: '0' - ref: ${{ github.ref }} - - - name: Build Docker Image - run: | - docker build \ - -f .docker/Dockerfile.${{ matrix.dist }} \ - -t dwarfs-linux-build-${{ matrix.dist }} \ - --build-arg ARCH=${{ matrix.arch }} \ - .docker - - - name: Run Build (${{ matrix.build_type }}) - run: | - docker run --rm \ - --cap-add SYS_ADMIN --device /dev/fuse --privileged \ - --mount type=bind,source=${GITHUB_WORKSPACE},target=/workspace,readonly \ - --mount type=bind,source=${{ runner.temp }},target=/tmp-runner \ - --mount type=bind,source=${HOME}/github-ccache,target=/ccache \ - --mount type=bind,source=${HOME}/github-local,target=/local \ - --mount type=bind,source=/mnt/opensource/artifacts/dwarfs,target=/artifacts \ - --env BUILD_FROM_TARBALL=1 \ - --env BUILD_TYPE=${{ matrix.build_type }} \ - --env BUILD_ARCH=${{ matrix.arch }} \ - --env BUILD_DIST=${{ matrix.dist }} \ - --env GITHUB_REF_NAME \ - --env GITHUB_REF_TYPE \ - --env GITHUB_RUN_ID \ - --env GITHUB_RUN_NUMBER \ - --env GITHUB_RUN_ATTEMPT \ - dwarfs-linux-build-${{ matrix.dist }} - - linux-git: - needs: package-source - - strategy: - matrix: - arch: - - amd64 - - arm64v8 - dist: - - ubuntu - - fedora - - arch - build_type: - - gcc-release-shared-ninja - exclude: - - arch: arm64v8 - dist: ubuntu - build_type: gcc-release-shared-ninja - - arch: amd64 - dist: fedora - build_type: gcc-release-shared-ninja - - arch: arm64v8 - dist: arch - build_type: gcc-release-shared-ninja - include: - - arch: amd64 - dist: ubuntu - build_type: clang-reldbg-shared-asan-ninja - - arch: amd64 - dist: ubuntu - build_type: clang-debug-shared-tsan-ninja - - arch: amd64 - dist: ubuntu - build_type: clang-reldbg-shared-ubsan-ninja - arch: amd64 dist: ubuntu build_type: clang-debug-coverage-ninja @@ -316,90 +231,16 @@ jobs: dist: alpine build_type: clang-reldbg-stacktrace-ninja-static - runs-on: - - self-hosted - - Linux - - ${{ matrix.arch }} + uses: ./.github/workflows/docker-run-build.yml + with: + build_type: ${{ matrix.build_type }} + build_arch: ${{ matrix.arch }} + build_dist: ${{ matrix.dist }} + build_from_tarball: true + upload_coverage: ${{ matrix.build_type == 'clang-debug-coverage-ninja' }} + upload_artifacts: ${{ endsWith(matrix.build_type, '-static') }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: 'true' - fetch-depth: '0' - ref: ${{ github.ref }} - - - name: Build Docker Image - run: | - docker build \ - -f .docker/Dockerfile.${{ matrix.dist }} \ - -t dwarfs-linux-build-${{ matrix.dist }} \ - --build-arg ARCH=${{ matrix.arch }} \ - .docker - - - name: Run Build (${{ matrix.build_type }}) - run: | - docker run --rm \ - --cap-add SYS_ADMIN --device /dev/fuse --privileged \ - --mount type=bind,source=${GITHUB_WORKSPACE},target=/workspace,readonly \ - --mount type=bind,source=${{ runner.temp }},target=/tmp-runner \ - --mount type=bind,source=${HOME}/github-ccache,target=/ccache \ - --mount type=bind,source=${HOME}/github-local,target=/local \ - --mount type=bind,source=/mnt/opensource/artifacts/dwarfs,target=/artifacts \ - --env BUILD_TYPE=${{ matrix.build_type }} \ - --env BUILD_ARCH=${{ matrix.arch }} \ - --env BUILD_DIST=${{ matrix.dist }} \ - --env GITHUB_REF_NAME \ - --env GITHUB_REF_TYPE \ - --env GITHUB_RUN_ID \ - --env GITHUB_RUN_NUMBER \ - --env GITHUB_RUN_ATTEMPT \ - dwarfs-linux-build-${{ matrix.dist }} - - - name: Upload Coverage Data - if: matrix.build_type == 'clang-debug-coverage-ninja' - uses: codecov/codecov-action@v4 - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: ${{ runner.temp }}/dwarfs-coverage.txt - fail_ci_if_error: false - - - name: Prepare Artifact Upload - run: cat ${{ runner.temp }}/artifacts.env >> $GITHUB_ENV - if: ${{ endsWith(matrix.build_type, '-static') }} - - - name: Upload Binary Tarball - uses: actions/upload-artifact@v4 - with: - name: ${{ env.binary_tarball }} - path: ${{ runner.temp }}/artifacts/${{ env.binary_tarball }} - if-no-files-found: error - compression-level: 0 - env: - ACTIONS_ARTIFACT_UPLOAD_CONCURRENCY: 5 - if: ${{ endsWith(matrix.build_type, '-static') }} - - - name: Upload Universal Binary - uses: actions/upload-artifact@v4 - with: - name: ${{ env.dwarfsuniversal_binary }} - path: ${{ runner.temp }}/artifacts/${{ env.dwarfsuniversal_binary }} - if-no-files-found: error - compression-level: 0 - env: - ACTIONS_ARTIFACT_UPLOAD_CONCURRENCY: 5 - if: ${{ endsWith(matrix.build_type, '-static') }} - - - name: Upload Fuse-Extract Binary - uses: actions/upload-artifact@v4 - with: - name: ${{ env.dwarfsfuseextract_binary }} - path: ${{ runner.temp }}/artifacts/${{ env.dwarfsfuseextract_binary }} - if-no-files-found: error - compression-level: 0 - env: - ACTIONS_ARTIFACT_UPLOAD_CONCURRENCY: 5 - if: ${{ endsWith(matrix.build_type, '-static') }} +################################################################################ macos: needs: package-source diff --git a/.github/workflows/docker-run-build.yml b/.github/workflows/docker-run-build.yml new file mode 100644 index 00000000..8dcaf1be --- /dev/null +++ b/.github/workflows/docker-run-build.yml @@ -0,0 +1,118 @@ +--- +name: Run Docker Build + +on: + workflow_call: + inputs: + build_type: + required: true + type: string + build_arch: + required: true + type: string + build_dist: + required: true + type: string + build_from_tarball: + required: false + type: boolean + default: false + upload_artifacts: + required: false + type: boolean + default: false + upload_coverage: + required: false + type: boolean + default: false + +permissions: + contents: read + +jobs: + docker-build: + runs-on: + - self-hosted + - Linux + - ${{ inputs.build_arch }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: 'true' + fetch-depth: '0' + ref: ${{ github.ref }} + + - name: Build Docker Image + run: | + docker build \ + -f .docker/Dockerfile.${{ inputs.build_dist }} \ + -t dwarfs-linux-build-${{ inputs.build_dist }} \ + --build-arg ARCH=${{ inputs.build_arch }} \ + .docker + + - name: Run Build + run: | + docker run --rm \ + --cap-add SYS_ADMIN --device /dev/fuse --privileged \ + --mount type=bind,source=${GITHUB_WORKSPACE},target=/workspace,readonly \ + --mount type=bind,source=${{ runner.temp }},target=/tmp-runner \ + --mount type=bind,source=${HOME}/github-ccache,target=/ccache \ + --mount type=bind,source=${HOME}/github-local,target=/local \ + --mount type=bind,source=/mnt/opensource/artifacts/dwarfs,target=/artifacts \ + ${{ inputs.build_from_tarball && '--env BUILD_FROM_TARBALL=1' || '' }} \ + --env BUILD_TYPE=${{ inputs.build_type }} \ + --env BUILD_ARCH=${{ inputs.build_arch }} \ + --env BUILD_DIST=${{ inputs.build_dist }} \ + --env GITHUB_REF_NAME \ + --env GITHUB_REF_TYPE \ + --env GITHUB_RUN_ID \ + --env GITHUB_RUN_NUMBER \ + --env GITHUB_RUN_ATTEMPT \ + dwarfs-linux-build-${{ inputs.build_dist }} + + - name: Upload Coverage Data + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: ${{ runner.temp }}/dwarfs-coverage.txt + fail_ci_if_error: false + if: ${{ inputs.upload_coverage }} + + - name: Prepare Artifact Upload + run: cat ${{ runner.temp }}/artifacts.env >> $GITHUB_ENV + if: ${{ inputs.upload_artifacts }} + + - name: Upload Binary Tarball + uses: actions/upload-artifact@v4 + with: + name: ${{ env.binary_tarball }} + path: ${{ runner.temp }}/artifacts/${{ env.binary_tarball }} + if-no-files-found: error + compression-level: 0 + env: + ACTIONS_ARTIFACT_UPLOAD_CONCURRENCY: 5 + if: ${{ inputs.upload_artifacts }} + + - name: Upload Universal Binary + uses: actions/upload-artifact@v4 + with: + name: ${{ env.dwarfsuniversal_binary }} + path: ${{ runner.temp }}/artifacts/${{ env.dwarfsuniversal_binary }} + if-no-files-found: error + compression-level: 0 + env: + ACTIONS_ARTIFACT_UPLOAD_CONCURRENCY: 5 + if: ${{ inputs.upload_artifacts }} + + - name: Upload Fuse-Extract Binary + uses: actions/upload-artifact@v4 + with: + name: ${{ env.dwarfsfuseextract_binary }} + path: ${{ runner.temp }}/artifacts/${{ env.dwarfsfuseextract_binary }} + if-no-files-found: error + compression-level: 0 + env: + ACTIONS_ARTIFACT_UPLOAD_CONCURRENCY: 5 + if: ${{ inputs.upload_artifacts }}