From 8cbd992d410b53d6afb900c986bf264e9cff7cff Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Sun, 9 Jul 2023 11:00:18 +0200 Subject: [PATCH] Add CI pipeline --- .docker/Dockerfile | 62 +++++++++++++++++ .docker/build-linux.sh | 134 ++++++++++++++++++++++++++++++++++++ .github/workflows/build.yml | 123 +++++++++++++++++++++++++++++++++ CMakeLists.txt | 61 ++++++++++++++-- 4 files changed, 376 insertions(+), 4 deletions(-) create mode 100644 .docker/Dockerfile create mode 100644 .docker/build-linux.sh create mode 100644 .github/workflows/build.yml diff --git a/.docker/Dockerfile b/.docker/Dockerfile new file mode 100644 index 00000000..4f7b5663 --- /dev/null +++ b/.docker/Dockerfile @@ -0,0 +1,62 @@ +# syntax=docker/dockerfile:1 +ARG ARCH=amd64 +FROM $ARCH/ubuntu:22.04 +RUN apt update && apt install -y gnupg wget software-properties-common +RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null +RUN apt-add-repository 'deb https://apt.kitware.com/ubuntu/ jammy main' +RUN apt install -y \ + gcc \ + g++ \ + gcc-12 \ + g++-12 \ + clang \ + clang-15 \ + git \ + vim \ + ccache \ + ninja-build \ + cmake \ + make \ + bison \ + flex \ + ronn \ + fuse3 \ + pkg-config \ + binutils-dev \ + libacl1-dev \ + libarchive-dev \ + libbenchmark-dev \ + libboost-chrono-dev \ + libboost-context-dev \ + libboost-filesystem-dev \ + libboost-iostreams-dev \ + libboost-program-options-dev \ + libboost-python-dev \ + libboost-regex-dev \ + libboost-system-dev \ + libboost-thread-dev \ + libbrotli-dev \ + libevent-dev \ + libjemalloc-dev \ + libdouble-conversion-dev \ + libiberty-dev \ + liblz4-dev \ + liblzma-dev \ + libmagic-dev \ + libssl-dev \ + libunwind-dev \ + libdwarf-dev \ + libelf-dev \ + libfmt-dev \ + libfuse-dev \ + libfuse3-dev \ + libgoogle-glog-dev \ + libutfcpp-dev \ + bash-completion \ + upx +RUN useradd -g users -u 1000 -m mhx +ARG SCRIPT=build-linux.sh +COPY $SCRIPT /usr/local/bin/run.sh +RUN chmod +x /usr/local/bin/run.sh +USER mhx +ENTRYPOINT /usr/local/bin/run.sh diff --git a/.docker/build-linux.sh b/.docker/build-linux.sh new file mode 100644 index 00000000..e94f79f4 --- /dev/null +++ b/.docker/build-linux.sh @@ -0,0 +1,134 @@ +#!/bin/bash + +set -ex + +export CTEST_PARALLEL_LEVEL=$(nproc) +export CCACHE_DIR=/ccache + +cd $HOME + +rm -f dwarfs +ln -s /workspace dwarfs + +rm -rf build +mkdir build +cd build + +case "-$BUILD_TYPE-" in + *-ninja-*) + BUILD_TOOL=ninja + CMAKE_ARGS="-GNinja" + ;; + *-make-*) + BUILD_TOOL="make -j$(nproc)" + CMAKE_ARGS= + ;; + *) + echo "missing build tool in: $BUILD_TYPE" + exit 1 +esac + +case "-$BUILD_TYPE-" in + *-gcc-*) + export CC=gcc CXX=g++ + ;; + *-clang-*) + export CC=clang CXX=clang++ + ;; + *-gcc12-*) + export CC=gcc-12 CXX=g++-12 + ;; + *-clang15-*) + export CC=clang-15 CXX=clang++-15 + ;; + *) + echo "missing compiler in: $BUILD_TYPE" + exit 1 +esac + +case "-$BUILD_TYPE-" in + *-debug-*) + CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=Debug" + ;; + *-release-*) + CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=Release" + ;; + *-asan-*) + CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=Debug -DENABLE_ASAN=1" + ;; + *-tsan-*) + CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_TSAN=1" + ;; + *-ubsan-*) + CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_UBSAN=1" + ;; + *) + echo "missing build type: $BUILD_TYPE" + exit 1 +esac + +if [[ "-$BUILD_TYPE-" == *-nojemalloc-* ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DUSE_JEMALLOC=0" +fi + +if [[ "-$BUILD_TYPE-" == *-noperfmon-* ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_PERFMON=0" +fi + +if [[ "-$BUILD_TYPE-" == *-static-* ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DSTATIC_BUILD_DO_NOT_USE=1" +else + CMAKE_ARGS="${CMAKE_ARGS} -DWITH_BENCHMARKS=1" +fi + +CMAKE_ARGS="${CMAKE_ARGS} -DWITH_TESTS=1 -DWITH_LEGACY_FUSE=1" +CMAKE_ARGS="${CMAKE_ARGS} -DDWARFS_ARTIFACTS_DIR=/artifacts" + +cmake ../dwarfs/ $CMAKE_ARGS + +$BUILD_TOOL + +$BUILD_TOOL test + +if [[ "-$BUILD_TYPE-" == *-static-* ]]; then + $BUILD_TOOL package_source + if [[ "$BUILD_ARCH" == "amd64" ]]; then + $BUILD_TOOL copy_source_artifacts + fi +fi + +$BUILD_TOOL realclean + +if [[ "-$BUILD_TYPE-" == *-static-* ]]; then + cd $HOME + + VERSION=$(git -C /workspace describe --tags --match "v*" --dirty) + VERSION=${VERSION:1} + + rm -rf dwarfs-* + rm -f dwarfs + + mv build/dwarfs-${VERSION}.tar.xz . + rm -rf build + + tar xvf dwarfs-${VERSION}.tar.xz + ln -s dwarfs-${VERSION} dwarfs + + mkdir build + cd build + + CMAKE_ARGS="${CMAKE_ARGS} -DWITH_TESTS=1 -DWITH_LEGACY_FUSE=1" + + cmake ../dwarfs/ $CMAKE_ARGS + + $BUILD_TOOL + + $BUILD_TOOL test + $BUILD_TOOL strip + $BUILD_TOOL package + $BUILD_TOOL universal_upx + + $BUILD_TOOL copy_artifacts + + $BUILD_TOOL realclean +fi diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..38f8f718 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,123 @@ +name: 'DwarFS CI Build' + +on: + workflow_dispatch: + push: + branches: + - main + - 'mhx/**' + tags: + - v* + +permissions: + contents: read + +jobs: + windows: + runs-on: + - self-hosted + - Windows + + strategy: + matrix: + arch: + - X64 + build_mode: + - Release + - Debug + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: 'true' + fetch-depth: '0' + + - name: Set MSVC Dev Environment + uses: ilammy/msvc-dev-cmd@v1 + + - name: Configure Build + shell: cmd + run: | + cmake -B${{ runner.temp }}\build -S${{ runner.workspace }}\dwarfs -GNinja -DCMAKE_MAKE_PROGRAM=C:\bin\ninja.exe -DCMAKE_BUILD_TYPE=${{ matrix.build_mode }} -DWITH_TESTS=ON -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DVCPKG_INSTALLED_DIR=${{ runner.workspace }}\vcpkg-install-dwarfs -DDWARFS_ARTIFACTS_DIR=Z:\artifacts\dwarfs + + - name: Run Build + shell: cmd + run: | + cmake --build ${{ runner.temp }}\build + + - name: Run Test + shell: cmd + run: | + copy "C:\Program Files (x86)\WinFsp\bin\winfsp-x64.dll" ${{ runner.temp }}\build + ctest --test-dir ${{ runner.temp }}\build --output-on-failure -j8 + + - name: Build Package + shell: cmd + run: | + cmake --build ${{ runner.temp }}\build --target package + + - name: Compress Universal Binary + shell: cmd + run: | + cmake --build ${{ runner.temp }}\build --target universal_upx + + - name: Copy Artifacts + shell: cmd + run: | + cmake --build ${{ runner.temp }}\build --target copy_artifacts + if: matrix.build_mode == 'Release' + + linux: + strategy: + matrix: + arch: + - amd64 + - arm64v8 + build_type: + - gcc-debug-make + - gcc12-debug-ninja + - gcc12-release-ninja + - clang15-release-ninja + - clang15-release-ninja-static + include: + - arch: amd64 + build_type: clang15-asan-ninja + - arch: amd64 + build_type: clang15-tsan-ninja + - arch: amd64 + build_type: clang15-ubsan-ninja + - arch: amd64 + build_type: gcc12-debug-noperfmon-ninja + + runs-on: + - self-hosted + - Linux + - ${{ matrix.arch }} + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: 'true' + fetch-depth: '0' + + - name: Build Docker Image + run: | + docker build \ + -t dwarfs-linux-build \ + --build-arg ARCH=${{ matrix.arch }} \ + --build-arg SCRIPT=build-linux.sh \ + .docker + + - name: Run Build (${{ matrix.build_type }}) + run: | + docker run --rm \ + --cap-add SYS_ADMIN --device /dev/fuse --privileged \ + --mount type=bind,source=${GITHUB_WORKSPACE},target=/workspace,readonly \ + --mount type=bind,source=/home/mhx/github-ccache,target=/ccache \ + --mount type=bind,source=/mnt/opensource/artifacts/dwarfs,target=/artifacts \ + --env BUILD_TYPE=${{ matrix.build_type }} \ + --env BUILD_ARCH=${{ matrix.arch }} \ + --env GITHUB_REF_NAME \ + dwarfs-linux-build diff --git a/CMakeLists.txt b/CMakeLists.txt index 690005ec..898b6991 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1093,11 +1093,63 @@ if(STATIC_BUILD_DO_NOT_USE AND NOT WIN32) add_custom_target(strip COMMAND strip -s ${FILES_TO_STRIP}) endif() -if(NOT $PRJ_VERSION_FULL} STREQUAL "") - if(WIN32) - set(CPACK_GENERATOR "NSIS;ZIP;7Z") +if(STATIC_BUILD_DO_NOT_USE OR WIN32) + find_program(UPX_EXE upx upx.exe PATHS "c:/bin" DOC "ultimate packer for executables" REQUIRED) + + set(UNIVERSAL_PACKED + "dwarfs-universal-${PRJ_VERSION_FULL}-${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}${CMAKE_EXECUTABLE_SUFFIX}") + + if(PRJ_GIT_RELEASE_TAG) + set(UPX_ARGS "--ultra-brute --best") else() - set(CPACK_GENERATOR "TGZ") + set(UPX_ARGS "") + endif() + + add_custom_command( + OUTPUT ${UNIVERSAL_PACKED} + COMMAND ${UPX_EXE} ${UPX_ARGS} -o ${UNIVERSAL_PACKED} $ + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + + add_custom_target(universal_upx DEPENDS ${UNIVERSAL_PACKED}) + + if(PRJ_GIT_RELEASE_TAG) + set(ARTIFACTS_SUBDIR "releases") + else() + set(ARTIFACTS_SUBDIR "builds") + endif() + + set(ARTIFACTS_FULL_PATH "${DWARFS_ARTIFACTS_DIR}/${ARTIFACTS_SUBDIR}/${PRJ_VERSION_FULL}") + + if(WIN32) + set(PACKAGE_EXT ".7z") + else() + set(PACKAGE_EXT ".tar.xz") + endif() + + add_custom_target(copy_artifacts + COMMAND ${CMAKE_COMMAND} -E make_directory ${ARTIFACTS_FULL_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_PROJECT_NAME}-${PRJ_VERSION_FULL}-${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}${PACKAGE_EXT} ${ARTIFACTS_FULL_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/${UNIVERSAL_PACKED} ${ARTIFACTS_FULL_PATH} + ) + + add_custom_target(copy_source_artifacts + COMMAND ${CMAKE_COMMAND} -E make_directory ${ARTIFACTS_FULL_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_PROJECT_NAME}-${PRJ_VERSION_FULL}${PACKAGE_EXT} ${ARTIFACTS_FULL_PATH} + ) +endif() + +if(NOT "${PRJ_VERSION_FULL}" STREQUAL "") + if(DWARFS_GIT_BUILD) + configure_file("${PROJECT_SOURCE_DIR}/cmake/dwarfs_install.cmake.in" dwarfs_install.cmake @ONLY) + set(CPACK_INSTALL_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/dwarfs_install.cmake") + endif() + + if(WIN32) + # set(CPACK_GENERATOR "NSIS;ZIP;7Z") + set(CPACK_GENERATOR "7Z") + else() + set(CPACK_GENERATOR "TXZ") endif() set(CPACK_SOURCE_GENERATOR "${CPACK_GENERATOR}") set(CPACK_PACKAGE_VERSION_MAJOR "${PRJ_VERSION_MAJOR}") @@ -1116,6 +1168,7 @@ if(NOT $PRJ_VERSION_FULL} STREQUAL "") "${CMAKE_SOURCE_DIR}/@" "/\\." ".*~$") set(CPACK_VERBATIM_VARIABLES YES) set(CPACK_STRIP_FILES YES) + set(CPACK_ARCHIVE_THREADS 0) include(CPack) endif()