build: clean up github workflow

This commit is contained in:
Marcus Holland-Moritz 2025-04-20 12:06:51 +02:00
parent 6a8048f2b0
commit 5cf77cc8d9
3 changed files with 259 additions and 306 deletions

View File

@ -297,114 +297,108 @@ if [[ "-$BUILD_TYPE-" == *-static-* ]]; then
CMAKE_ARGS="${CMAKE_ARGS} -DSTATIC_BUILD_EXTRA_PREFIX=/opt/static-libs/$COMPILER;$_sslprefix"
fi
if [[ "$BUILD_FROM_TARBALL" == "1" ]]; then
INSTALLDIR="$HOME/install"
PREFIXPATH="$INSTALLDIR/usr/local"
rm -rf "$INSTALLDIR"
INSTALLDIR="$HOME/install"
PREFIXPATH="$INSTALLDIR/usr/local"
rm -rf "$INSTALLDIR"
if [[ "-$BUILD_TYPE-" == *-shared-* ]]; then
LDLIBPATH="$(readlink -m "$PREFIXPATH/lib/$(gcc -print-multi-os-directory)")"
if [[ ":$LD_LIBRARY_PATH:" != *":$LDLIBPATH:"* ]]; then
export "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}$LDLIBPATH"
fi
fi
case "-$BUILD_TYPE-" in
*-full-*)
cmake ../dwarfs/ $CMAKE_ARGS
time $BUILD_TOOL
$RUN_TESTS
DESTDIR="$INSTALLDIR" $BUILD_TOOL install
$BUILD_TOOL distclean
;;
*-split-*)
# ==== libdwarfs ====
cmake ../dwarfs/ $CMAKE_ARGS -DWITH_LIBDWARFS=ON -DWITH_TOOLS=OFF -DWITH_FUSE_DRIVER=OFF
time $BUILD_TOOL
$RUN_TESTS
DESTDIR="$INSTALLDIR" $BUILD_TOOL install
$BUILD_TOOL distclean
rm -rf *
# ==== example binary ====
cmake ../dwarfs/example $CMAKE_TOOL_ARGS -DCMAKE_PREFIX_PATH="$PREFIXPATH"
time $BUILD_TOOL
rm -rf *
# ==== dwarfs tools ====
cmake ../dwarfs/ $CMAKE_ARGS -DWITH_LIBDWARFS=OFF -DWITH_TOOLS=ON -DWITH_FUSE_DRIVER=OFF -DCMAKE_PREFIX_PATH="$PREFIXPATH"
time $BUILD_TOOL
$RUN_TESTS
DESTDIR="$INSTALLDIR" $BUILD_TOOL install
$BUILD_TOOL distclean
rm -rf *
# ==== dwarfs fuse driver ====
cmake ../dwarfs/ $CMAKE_ARGS -DWITH_LIBDWARFS=OFF -DWITH_TOOLS=OFF -DWITH_FUSE_DRIVER=ON -DCMAKE_PREFIX_PATH="$PREFIXPATH"
time $BUILD_TOOL
$RUN_TESTS
DESTDIR="$INSTALLDIR" $BUILD_TOOL install
$BUILD_TOOL distclean
;;
*)
echo "builds from source tarball must be 'full' or 'split'"
exit 1
;;
esac
else
# shellcheck disable=SC2086
cmake ../dwarfs/ $CMAKE_ARGS -DWITH_EXAMPLE=1
time $BUILD_TOOL
$RUN_TESTS
if [[ "-$BUILD_TYPE-" == *-coverage-* ]]; then
rm -f /tmp-runner/dwarfs-coverage.txt
llvm-profdata$CLANG_VERSION merge -sparse profile/* -o dwarfs.profdata
llvm-cov$CLANG_VERSION show -instr-profile=dwarfs.profdata \
$(for i in mkdwarfs dwarfs dwarfsck dwarfsextract *_test ricepp/ricepp_test; do echo $i; done | sed -e's/^/-object=/') \
>/tmp-runner/dwarfs-coverage.txt
fi
if [[ "-$BUILD_TYPE-" == *-static-* ]]; then
# for release and relsize builds, strip the binaries
if [[ "-$BUILD_TYPE-" =~ -(release|relsize)- ]]; then
$BUILD_TOOL strip
fi
$BUILD_TOOL package
$BUILD_TOOL universal_upx
$BUILD_TOOL copy_artifacts
rm -rf /tmp-runner/artifacts
mkdir -p /tmp-runner/artifacts
cp artifacts.env /tmp-runner
cp dwarfs-universal-* /tmp-runner/artifacts
cp dwarfs-fuse-extract-* /tmp-runner/artifacts
cp dwarfs-*-Linux*.tar.zst /tmp-runner/artifacts
elif [[ "-$BUILD_TYPE-" == *-source-* ]]; then
$BUILD_TOOL package_source
$BUILD_TOOL copy_source_artifacts
fi
if [[ "-$BUILD_TYPE-" != *-[at]san-* ]] && \
[[ "-$BUILD_TYPE-" != *-ubsan-* ]] && \
[[ "-$BUILD_TYPE-" != *-source-* ]] && \
[[ "-$BUILD_TYPE-" != *-static-* ]]; then
INSTALLDIR="$HOME/install"
rm -rf "$INSTALLDIR"
DESTDIR="$INSTALLDIR" $BUILD_TOOL install
$BUILD_TOOL distclean
cmake ../dwarfs/example $CMAKE_TOOL_ARGS -DCMAKE_PREFIX_PATH="$INSTALLDIR/usr/local"
$BUILD_TOOL
$BUILD_TOOL clean
else
$BUILD_TOOL distclean
if [[ "-$BUILD_TYPE-" == *-shared-* ]]; then
LDLIBPATH="$(readlink -m "$PREFIXPATH/lib/$(gcc -print-multi-os-directory)")"
if [[ ":$LD_LIBRARY_PATH:" != *":$LDLIBPATH:"* ]]; then
export "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}$LDLIBPATH"
fi
fi
case "-$BUILD_TYPE-" in
*-full-*)
cmake ../dwarfs/ $CMAKE_ARGS -DWITH_EXAMPLE=1
time $BUILD_TOOL
$RUN_TESTS
;;
*-split-*)
# ==== libdwarfs ====
cmake ../dwarfs/ $CMAKE_ARGS -DWITH_LIBDWARFS=ON -DWITH_TOOLS=OFF -DWITH_FUSE_DRIVER=OFF
time $BUILD_TOOL
$RUN_TESTS
DESTDIR="$INSTALLDIR" $BUILD_TOOL install
$BUILD_TOOL distclean
rm -rf *
# ==== example binary ====
cmake ../dwarfs/example $CMAKE_TOOL_ARGS -DCMAKE_PREFIX_PATH="$PREFIXPATH"
time $BUILD_TOOL
rm -rf *
# ==== dwarfs tools ====
cmake ../dwarfs/ $CMAKE_ARGS -DWITH_LIBDWARFS=OFF -DWITH_TOOLS=ON -DWITH_FUSE_DRIVER=OFF -DCMAKE_PREFIX_PATH="$PREFIXPATH"
time $BUILD_TOOL
$RUN_TESTS
DESTDIR="$INSTALLDIR" $BUILD_TOOL install
$BUILD_TOOL distclean
rm -rf *
# ==== dwarfs fuse driver ====
cmake ../dwarfs/ $CMAKE_ARGS -DWITH_LIBDWARFS=OFF -DWITH_TOOLS=OFF -DWITH_FUSE_DRIVER=ON -DCMAKE_PREFIX_PATH="$PREFIXPATH"
time $BUILD_TOOL
$RUN_TESTS
DESTDIR="$INSTALLDIR" $BUILD_TOOL install
$BUILD_TOOL distclean
# That's it for split builds, we are done
exit 0
;;
*)
if [[ "-$BUILD_TYPE-" == *-static-* ]]; then
cmake ../dwarfs/ $CMAKE_ARGS
else
cmake ../dwarfs/ $CMAKE_ARGS -DWITH_EXAMPLE=1
fi
time $BUILD_TOOL
$RUN_TESTS
;;
esac
if [[ "-$BUILD_TYPE-" == *-coverage-* ]]; then
rm -f /tmp-runner/dwarfs-coverage.txt
llvm-profdata$CLANG_VERSION merge -sparse profile/* -o dwarfs.profdata
llvm-cov$CLANG_VERSION show -instr-profile=dwarfs.profdata \
$(for i in mkdwarfs dwarfs dwarfsck dwarfsextract *_test ricepp/ricepp_test; do echo $i; done | sed -e's/^/-object=/') \
>/tmp-runner/dwarfs-coverage.txt
fi
if [[ "-$BUILD_TYPE-" == *-static-* ]]; then
# for release and relsize builds, strip the binaries
if [[ "-$BUILD_TYPE-" =~ -(release|relsize)- ]]; then
$BUILD_TOOL strip
fi
$BUILD_TOOL package
$BUILD_TOOL universal_upx
$BUILD_TOOL copy_artifacts
rm -rf /tmp-runner/artifacts
mkdir -p /tmp-runner/artifacts
cp artifacts.env /tmp-runner
cp dwarfs-universal-* /tmp-runner/artifacts
cp dwarfs-fuse-extract-* /tmp-runner/artifacts
cp dwarfs-*-Linux*.tar.zst /tmp-runner/artifacts
elif [[ "-$BUILD_TYPE-" == *-source-* ]]; then
$BUILD_TOOL package_source
$BUILD_TOOL copy_source_artifacts
fi
if [[ "-$BUILD_TYPE-" != *-[at]san-* ]] && \
[[ "-$BUILD_TYPE-" != *-ubsan-* ]] && \
[[ "-$BUILD_TYPE-" != *-source-* ]] && \
[[ "-$BUILD_TYPE-" != *-static-* ]]; then
DESTDIR="$INSTALLDIR" $BUILD_TOOL install
$BUILD_TOOL distclean
cmake ../dwarfs/example $CMAKE_TOOL_ARGS -DCMAKE_PREFIX_PATH="$INSTALLDIR/usr/local"
$BUILD_TOOL
$BUILD_TOOL clean
else
$BUILD_TOOL distclean
fi

View File

@ -92,48 +92,18 @@ jobs:
# env:
# ACTIONS_ARTIFACT_UPLOAD_CONCURRENCY: 5
################################################################################
package-source:
runs-on:
- self-hosted
- Linux
- amd64
uses: ./.github/workflows/docker-run-build.yml
with:
build_type: clang-release-ninja-source-notest
build_arch: amd64
build_dist: arch
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: 'true'
fetch-depth: '0'
ref: ${{ github.ref }}
################################################################################
- name: Build Docker Image
run: |
docker build \
-f .docker/Dockerfile.arch \
-t dwarfs-linux-build-arch \
--build-arg ARCH=amd64 \
.docker
- name: Run Build
run: |
docker run --rm \
--cap-add SYS_ADMIN --device /dev/fuse --privileged \
--mount type=bind,source=${GITHUB_WORKSPACE},target=/workspace,readonly \
--mount type=bind,source=${{ runner.temp }},target=/tmp-runner \
--mount type=bind,source=${HOME}/github-ccache,target=/ccache \
--mount type=bind,source=${HOME}/github-local,target=/local \
--mount type=bind,source=/mnt/opensource/artifacts/dwarfs,target=/artifacts \
--env BUILD_TYPE=clang-release-ninja-source-notest \
--env BUILD_ARCH=amd64 \
--env BUILD_DIST=arch \
--env GITHUB_REF_NAME \
--env GITHUB_REF_TYPE \
--env GITHUB_RUN_ID \
--env GITHUB_RUN_NUMBER \
--env GITHUB_RUN_ATTEMPT \
dwarfs-linux-build-arch
linux-src:
linux:
needs: package-source
strategy:
@ -163,7 +133,28 @@ jobs:
- arch: arm64v8
dist: arch
build_type: clang-release-shared-ninja-full
- arch: arm64v8
dist: ubuntu
build_type: gcc-release-ninja-split
- arch: arm64v8
dist: ubuntu
build_type: gcc-release-shared-ninja-split
- arch: arm64v8
dist: fedora
build_type: gcc-release-ninja-split
- arch: arm64v8
dist: fedora
build_type: gcc-release-shared-ninja-split
include:
- arch: amd64
dist: ubuntu
build_type: clang-reldbg-shared-asan-ninja
- arch: amd64
dist: ubuntu
build_type: clang-debug-shared-tsan-ninja
- arch: amd64
dist: ubuntu
build_type: clang-reldbg-shared-ubsan-ninja
- arch: amd64
dist: ubuntu
build_type: oldgcc-debug-shared-make-split
@ -182,82 +173,6 @@ jobs:
- arch: amd64
dist: ubuntu-2204
build_type: clang-release-ninja-split
runs-on:
- self-hosted
- Linux
- ${{ matrix.arch }}
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: 'true'
fetch-depth: '0'
ref: ${{ github.ref }}
- name: Build Docker Image
run: |
docker build \
-f .docker/Dockerfile.${{ matrix.dist }} \
-t dwarfs-linux-build-${{ matrix.dist }} \
--build-arg ARCH=${{ matrix.arch }} \
.docker
- name: Run Build (${{ matrix.build_type }})
run: |
docker run --rm \
--cap-add SYS_ADMIN --device /dev/fuse --privileged \
--mount type=bind,source=${GITHUB_WORKSPACE},target=/workspace,readonly \
--mount type=bind,source=${{ runner.temp }},target=/tmp-runner \
--mount type=bind,source=${HOME}/github-ccache,target=/ccache \
--mount type=bind,source=${HOME}/github-local,target=/local \
--mount type=bind,source=/mnt/opensource/artifacts/dwarfs,target=/artifacts \
--env BUILD_FROM_TARBALL=1 \
--env BUILD_TYPE=${{ matrix.build_type }} \
--env BUILD_ARCH=${{ matrix.arch }} \
--env BUILD_DIST=${{ matrix.dist }} \
--env GITHUB_REF_NAME \
--env GITHUB_REF_TYPE \
--env GITHUB_RUN_ID \
--env GITHUB_RUN_NUMBER \
--env GITHUB_RUN_ATTEMPT \
dwarfs-linux-build-${{ matrix.dist }}
linux-git:
needs: package-source
strategy:
matrix:
arch:
- amd64
- arm64v8
dist:
- ubuntu
- fedora
- arch
build_type:
- gcc-release-shared-ninja
exclude:
- arch: arm64v8
dist: ubuntu
build_type: gcc-release-shared-ninja
- arch: amd64
dist: fedora
build_type: gcc-release-shared-ninja
- arch: arm64v8
dist: arch
build_type: gcc-release-shared-ninja
include:
- arch: amd64
dist: ubuntu
build_type: clang-reldbg-shared-asan-ninja
- arch: amd64
dist: ubuntu
build_type: clang-debug-shared-tsan-ninja
- arch: amd64
dist: ubuntu
build_type: clang-reldbg-shared-ubsan-ninja
- arch: amd64
dist: ubuntu
build_type: clang-debug-coverage-ninja
@ -316,90 +231,16 @@ jobs:
dist: alpine
build_type: clang-reldbg-stacktrace-ninja-static
runs-on:
- self-hosted
- Linux
- ${{ matrix.arch }}
uses: ./.github/workflows/docker-run-build.yml
with:
build_type: ${{ matrix.build_type }}
build_arch: ${{ matrix.arch }}
build_dist: ${{ matrix.dist }}
build_from_tarball: true
upload_coverage: ${{ matrix.build_type == 'clang-debug-coverage-ninja' }}
upload_artifacts: ${{ endsWith(matrix.build_type, '-static') }}
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: 'true'
fetch-depth: '0'
ref: ${{ github.ref }}
- name: Build Docker Image
run: |
docker build \
-f .docker/Dockerfile.${{ matrix.dist }} \
-t dwarfs-linux-build-${{ matrix.dist }} \
--build-arg ARCH=${{ matrix.arch }} \
.docker
- name: Run Build (${{ matrix.build_type }})
run: |
docker run --rm \
--cap-add SYS_ADMIN --device /dev/fuse --privileged \
--mount type=bind,source=${GITHUB_WORKSPACE},target=/workspace,readonly \
--mount type=bind,source=${{ runner.temp }},target=/tmp-runner \
--mount type=bind,source=${HOME}/github-ccache,target=/ccache \
--mount type=bind,source=${HOME}/github-local,target=/local \
--mount type=bind,source=/mnt/opensource/artifacts/dwarfs,target=/artifacts \
--env BUILD_TYPE=${{ matrix.build_type }} \
--env BUILD_ARCH=${{ matrix.arch }} \
--env BUILD_DIST=${{ matrix.dist }} \
--env GITHUB_REF_NAME \
--env GITHUB_REF_TYPE \
--env GITHUB_RUN_ID \
--env GITHUB_RUN_NUMBER \
--env GITHUB_RUN_ATTEMPT \
dwarfs-linux-build-${{ matrix.dist }}
- name: Upload Coverage Data
if: matrix.build_type == 'clang-debug-coverage-ninja'
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
file: ${{ runner.temp }}/dwarfs-coverage.txt
fail_ci_if_error: false
- name: Prepare Artifact Upload
run: cat ${{ runner.temp }}/artifacts.env >> $GITHUB_ENV
if: ${{ endsWith(matrix.build_type, '-static') }}
- name: Upload Binary Tarball
uses: actions/upload-artifact@v4
with:
name: ${{ env.binary_tarball }}
path: ${{ runner.temp }}/artifacts/${{ env.binary_tarball }}
if-no-files-found: error
compression-level: 0
env:
ACTIONS_ARTIFACT_UPLOAD_CONCURRENCY: 5
if: ${{ endsWith(matrix.build_type, '-static') }}
- name: Upload Universal Binary
uses: actions/upload-artifact@v4
with:
name: ${{ env.dwarfsuniversal_binary }}
path: ${{ runner.temp }}/artifacts/${{ env.dwarfsuniversal_binary }}
if-no-files-found: error
compression-level: 0
env:
ACTIONS_ARTIFACT_UPLOAD_CONCURRENCY: 5
if: ${{ endsWith(matrix.build_type, '-static') }}
- name: Upload Fuse-Extract Binary
uses: actions/upload-artifact@v4
with:
name: ${{ env.dwarfsfuseextract_binary }}
path: ${{ runner.temp }}/artifacts/${{ env.dwarfsfuseextract_binary }}
if-no-files-found: error
compression-level: 0
env:
ACTIONS_ARTIFACT_UPLOAD_CONCURRENCY: 5
if: ${{ endsWith(matrix.build_type, '-static') }}
################################################################################
macos:
needs: package-source

118
.github/workflows/docker-run-build.yml vendored Normal file
View File

@ -0,0 +1,118 @@
---
name: Run Docker Build
on:
workflow_call:
inputs:
build_type:
required: true
type: string
build_arch:
required: true
type: string
build_dist:
required: true
type: string
build_from_tarball:
required: false
type: boolean
default: false
upload_artifacts:
required: false
type: boolean
default: false
upload_coverage:
required: false
type: boolean
default: false
permissions:
contents: read
jobs:
docker-build:
runs-on:
- self-hosted
- Linux
- ${{ inputs.build_arch }}
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: 'true'
fetch-depth: '0'
ref: ${{ github.ref }}
- name: Build Docker Image
run: |
docker build \
-f .docker/Dockerfile.${{ inputs.build_dist }} \
-t dwarfs-linux-build-${{ inputs.build_dist }} \
--build-arg ARCH=${{ inputs.build_arch }} \
.docker
- name: Run Build
run: |
docker run --rm \
--cap-add SYS_ADMIN --device /dev/fuse --privileged \
--mount type=bind,source=${GITHUB_WORKSPACE},target=/workspace,readonly \
--mount type=bind,source=${{ runner.temp }},target=/tmp-runner \
--mount type=bind,source=${HOME}/github-ccache,target=/ccache \
--mount type=bind,source=${HOME}/github-local,target=/local \
--mount type=bind,source=/mnt/opensource/artifacts/dwarfs,target=/artifacts \
${{ inputs.build_from_tarball && '--env BUILD_FROM_TARBALL=1' || '' }} \
--env BUILD_TYPE=${{ inputs.build_type }} \
--env BUILD_ARCH=${{ inputs.build_arch }} \
--env BUILD_DIST=${{ inputs.build_dist }} \
--env GITHUB_REF_NAME \
--env GITHUB_REF_TYPE \
--env GITHUB_RUN_ID \
--env GITHUB_RUN_NUMBER \
--env GITHUB_RUN_ATTEMPT \
dwarfs-linux-build-${{ inputs.build_dist }}
- name: Upload Coverage Data
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
file: ${{ runner.temp }}/dwarfs-coverage.txt
fail_ci_if_error: false
if: ${{ inputs.upload_coverage }}
- name: Prepare Artifact Upload
run: cat ${{ runner.temp }}/artifacts.env >> $GITHUB_ENV
if: ${{ inputs.upload_artifacts }}
- name: Upload Binary Tarball
uses: actions/upload-artifact@v4
with:
name: ${{ env.binary_tarball }}
path: ${{ runner.temp }}/artifacts/${{ env.binary_tarball }}
if-no-files-found: error
compression-level: 0
env:
ACTIONS_ARTIFACT_UPLOAD_CONCURRENCY: 5
if: ${{ inputs.upload_artifacts }}
- name: Upload Universal Binary
uses: actions/upload-artifact@v4
with:
name: ${{ env.dwarfsuniversal_binary }}
path: ${{ runner.temp }}/artifacts/${{ env.dwarfsuniversal_binary }}
if-no-files-found: error
compression-level: 0
env:
ACTIONS_ARTIFACT_UPLOAD_CONCURRENCY: 5
if: ${{ inputs.upload_artifacts }}
- name: Upload Fuse-Extract Binary
uses: actions/upload-artifact@v4
with:
name: ${{ env.dwarfsfuseextract_binary }}
path: ${{ runner.temp }}/artifacts/${{ env.dwarfsfuseextract_binary }}
if-no-files-found: error
compression-level: 0
env:
ACTIONS_ARTIFACT_UPLOAD_CONCURRENCY: 5
if: ${{ inputs.upload_artifacts }}