Add CI pipeline

2025-09-09 12:28:13 -04:00 · 2023-07-09 11:00:18 +02:00 · 2023-07-09 11:00:18 +02:00 · 8cbd992d41
commit 8cbd992d41
parent 75cb03c7e4
4 changed files with 376 additions and 4 deletions
--- a/.docker/Dockerfile
+++ b/.docker/Dockerfile
@ -0,0 +1,62 @@
+# syntax=docker/dockerfile:1
+ARG ARCH=amd64
+FROM $ARCH/ubuntu:22.04
+RUN apt update && apt install -y gnupg wget software-properties-common
+RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null
+RUN apt-add-repository 'deb https://apt.kitware.com/ubuntu/ jammy main'
+RUN apt install -y \
+        gcc \
+        g++ \
+        gcc-12 \
+        g++-12 \
+        clang \
+        clang-15 \
+        git \
+        vim \
+        ccache \
+        ninja-build \
+        cmake \
+        make \
+        bison \
+        flex \
+        ronn \
+        fuse3 \
+        pkg-config \
+        binutils-dev \
+        libacl1-dev \
+        libarchive-dev \
+        libbenchmark-dev \
+        libboost-chrono-dev \
+        libboost-context-dev \
+        libboost-filesystem-dev \
+        libboost-iostreams-dev \
+        libboost-program-options-dev \
+        libboost-python-dev \
+        libboost-regex-dev \
+        libboost-system-dev \
+        libboost-thread-dev \
+        libbrotli-dev \
+        libevent-dev \
+        libjemalloc-dev \
+        libdouble-conversion-dev \
+        libiberty-dev \
+        liblz4-dev \
+        liblzma-dev \
+        libmagic-dev \
+        libssl-dev \
+        libunwind-dev \
+        libdwarf-dev \
+        libelf-dev \
+        libfmt-dev \
+        libfuse-dev \
+        libfuse3-dev \
+        libgoogle-glog-dev \
+        libutfcpp-dev \
+        bash-completion \
+        upx
+RUN useradd -g users -u 1000 -m mhx
+ARG SCRIPT=build-linux.sh
+COPY $SCRIPT /usr/local/bin/run.sh
+RUN chmod +x /usr/local/bin/run.sh
+USER mhx
+ENTRYPOINT /usr/local/bin/run.sh
--- a/.docker/build-linux.sh
+++ b/.docker/build-linux.sh
@ -0,0 +1,134 @@
+#!/bin/bash
+
+set -ex
+
+export CTEST_PARALLEL_LEVEL=$(nproc)
+export CCACHE_DIR=/ccache
+
+cd $HOME
+
+rm -f dwarfs
+ln -s /workspace dwarfs
+
+rm -rf build
+mkdir build
+cd build
+
+case "-$BUILD_TYPE-" in
+  *-ninja-*)
+    BUILD_TOOL=ninja
+    CMAKE_ARGS="-GNinja"
+    ;;
+  *-make-*)
+    BUILD_TOOL="make -j$(nproc)"
+    CMAKE_ARGS=
+    ;;
+  *)
+    echo "missing build tool in: $BUILD_TYPE"
+    exit 1
+esac
+
+case "-$BUILD_TYPE-" in
+  *-gcc-*)
+    export CC=gcc CXX=g++
+    ;;
+  *-clang-*)
+    export CC=clang CXX=clang++
+    ;;
+  *-gcc12-*)
+    export CC=gcc-12 CXX=g++-12
+    ;;
+  *-clang15-*)
+    export CC=clang-15 CXX=clang++-15
+    ;;
+  *)
+    echo "missing compiler in: $BUILD_TYPE"
+    exit 1
+esac
+
+case "-$BUILD_TYPE-" in
+  *-debug-*)
+    CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=Debug"
+    ;;
+  *-release-*)
+    CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=Release"
+    ;;
+  *-asan-*)
+    CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=Debug -DENABLE_ASAN=1"
+    ;;
+  *-tsan-*)
+    CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_TSAN=1"
+    ;;
+  *-ubsan-*)
+    CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_UBSAN=1"
+    ;;
+  *)
+    echo "missing build type: $BUILD_TYPE"
+    exit 1
+esac
+
+if [[ "-$BUILD_TYPE-" == *-nojemalloc-* ]]; then
+  CMAKE_ARGS="${CMAKE_ARGS} -DUSE_JEMALLOC=0"
+fi
+
+if [[ "-$BUILD_TYPE-" == *-noperfmon-* ]]; then
+  CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_PERFMON=0"
+fi
+
+if [[ "-$BUILD_TYPE-" == *-static-* ]]; then
+  CMAKE_ARGS="${CMAKE_ARGS} -DSTATIC_BUILD_DO_NOT_USE=1"
+else
+  CMAKE_ARGS="${CMAKE_ARGS} -DWITH_BENCHMARKS=1"
+fi
+
+CMAKE_ARGS="${CMAKE_ARGS} -DWITH_TESTS=1 -DWITH_LEGACY_FUSE=1"
+CMAKE_ARGS="${CMAKE_ARGS} -DDWARFS_ARTIFACTS_DIR=/artifacts"
+
+cmake ../dwarfs/ $CMAKE_ARGS
+
+$BUILD_TOOL
+
+$BUILD_TOOL test
+
+if [[ "-$BUILD_TYPE-" == *-static-* ]]; then
+  $BUILD_TOOL package_source
+  if [[ "$BUILD_ARCH" == "amd64" ]]; then
+    $BUILD_TOOL copy_source_artifacts
+  fi
+fi
+
+$BUILD_TOOL realclean
+
+if [[ "-$BUILD_TYPE-" == *-static-* ]]; then
+  cd $HOME
+
+  VERSION=$(git -C /workspace describe --tags --match "v*" --dirty)
+  VERSION=${VERSION:1}
+
+  rm -rf dwarfs-*
+  rm -f dwarfs
+
+  mv build/dwarfs-${VERSION}.tar.xz .
+  rm -rf build
+
+  tar xvf dwarfs-${VERSION}.tar.xz
+  ln -s dwarfs-${VERSION} dwarfs
+
+  mkdir build
+  cd build
+
+  CMAKE_ARGS="${CMAKE_ARGS} -DWITH_TESTS=1 -DWITH_LEGACY_FUSE=1"
+
+  cmake ../dwarfs/ $CMAKE_ARGS
+
+  $BUILD_TOOL
+
+  $BUILD_TOOL test
+  $BUILD_TOOL strip
+  $BUILD_TOOL package
+  $BUILD_TOOL universal_upx
+
+  $BUILD_TOOL copy_artifacts
+
+  $BUILD_TOOL realclean
+fi
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@ -0,0 +1,123 @@
+name: 'DwarFS CI Build'
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - 'mhx/**'
+    tags:
+      - v*
+
+permissions:
+  contents: read
+
+jobs:
+  windows:
+    runs-on:
+      - self-hosted
+      - Windows
+
+    strategy:
+      matrix:
+        arch:
+          - X64
+        build_mode:
+          - Release
+          - Debug
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: 'true'
+          fetch-depth: '0'
+
+      - name: Set MSVC Dev Environment
+        uses: ilammy/msvc-dev-cmd@v1
+
+      - name: Configure Build
+        shell: cmd
+        run: |
+          cmake -B${{ runner.temp }}\build -S${{ runner.workspace }}\dwarfs -GNinja -DCMAKE_MAKE_PROGRAM=C:\bin\ninja.exe -DCMAKE_BUILD_TYPE=${{ matrix.build_mode }} -DWITH_TESTS=ON -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DVCPKG_INSTALLED_DIR=${{ runner.workspace }}\vcpkg-install-dwarfs -DDWARFS_ARTIFACTS_DIR=Z:\artifacts\dwarfs
+
+      - name: Run Build
+        shell: cmd
+        run: |
+          cmake --build ${{ runner.temp }}\build
+
+      - name: Run Test
+        shell: cmd
+        run: |
+          copy "C:\Program Files (x86)\WinFsp\bin\winfsp-x64.dll" ${{ runner.temp }}\build
+          ctest --test-dir ${{ runner.temp }}\build --output-on-failure -j8
+
+      - name: Build Package
+        shell: cmd
+        run: |
+          cmake --build ${{ runner.temp }}\build --target package
+
+      - name: Compress Universal Binary
+        shell: cmd
+        run: |
+          cmake --build ${{ runner.temp }}\build --target universal_upx
+
+      - name: Copy Artifacts
+        shell: cmd
+        run: |
+          cmake --build ${{ runner.temp }}\build --target copy_artifacts
+        if: matrix.build_mode == 'Release'
+
+  linux:
+    strategy:
+      matrix:
+        arch:
+          - amd64
+          - arm64v8
+        build_type:
+          - gcc-debug-make
+          - gcc12-debug-ninja
+          - gcc12-release-ninja
+          - clang15-release-ninja
+          - clang15-release-ninja-static
+        include:
+          - arch: amd64
+            build_type: clang15-asan-ninja
+          - arch: amd64
+            build_type: clang15-tsan-ninja
+          - arch: amd64
+            build_type: clang15-ubsan-ninja
+          - arch: amd64
+            build_type: gcc12-debug-noperfmon-ninja
+
+    runs-on:
+      - self-hosted
+      - Linux
+      - ${{ matrix.arch }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: 'true'
+          fetch-depth: '0'
+
+      - name: Build Docker Image
+        run: |
+          docker build \
+            -t dwarfs-linux-build \
+            --build-arg ARCH=${{ matrix.arch }} \
+            --build-arg SCRIPT=build-linux.sh \
+            .docker
+
+      - name: Run Build (${{ matrix.build_type }})
+        run: |
+          docker run --rm \
+            --cap-add SYS_ADMIN --device /dev/fuse --privileged \
+            --mount type=bind,source=${GITHUB_WORKSPACE},target=/workspace,readonly \
+            --mount type=bind,source=/home/mhx/github-ccache,target=/ccache \
+            --mount type=bind,source=/mnt/opensource/artifacts/dwarfs,target=/artifacts \
+            --env BUILD_TYPE=${{ matrix.build_type }} \
+            --env BUILD_ARCH=${{ matrix.arch }} \
+            --env GITHUB_REF_NAME \
+            dwarfs-linux-build
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -1093,11 +1093,63 @@ if(STATIC_BUILD_DO_NOT_USE AND NOT WIN32)
  add_custom_target(strip COMMAND strip -s ${FILES_TO_STRIP})
 endif()

-if(NOT $PRJ_VERSION_FULL} STREQUAL "")
-  if(WIN32)
-    set(CPACK_GENERATOR "NSIS;ZIP;7Z")
+if(STATIC_BUILD_DO_NOT_USE OR WIN32)
+  find_program(UPX_EXE upx upx.exe PATHS "c:/bin" DOC "ultimate packer for executables" REQUIRED)
+
+  set(UNIVERSAL_PACKED
+          "dwarfs-universal-${PRJ_VERSION_FULL}-${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}${CMAKE_EXECUTABLE_SUFFIX}")
+
+  if(PRJ_GIT_RELEASE_TAG)
+    set(UPX_ARGS "--ultra-brute --best")
  else()
-    set(CPACK_GENERATOR "TGZ")
+    set(UPX_ARGS "")
+  endif()
+
+  add_custom_command(
+    OUTPUT ${UNIVERSAL_PACKED}
+    COMMAND ${UPX_EXE} ${UPX_ARGS} -o ${UNIVERSAL_PACKED} $<TARGET_FILE:dwarfsuniversal>
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+  )
+
+  add_custom_target(universal_upx DEPENDS ${UNIVERSAL_PACKED})
+
+  if(PRJ_GIT_RELEASE_TAG)
+    set(ARTIFACTS_SUBDIR "releases")
+  else()
+    set(ARTIFACTS_SUBDIR "builds")
+  endif()
+
+  set(ARTIFACTS_FULL_PATH "${DWARFS_ARTIFACTS_DIR}/${ARTIFACTS_SUBDIR}/${PRJ_VERSION_FULL}")
+
+  if(WIN32)
+    set(PACKAGE_EXT ".7z")
+  else()
+    set(PACKAGE_EXT ".tar.xz")
+  endif()
+
+  add_custom_target(copy_artifacts
+    COMMAND ${CMAKE_COMMAND} -E make_directory ${ARTIFACTS_FULL_PATH}
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_PROJECT_NAME}-${PRJ_VERSION_FULL}-${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}${PACKAGE_EXT} ${ARTIFACTS_FULL_PATH}
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/${UNIVERSAL_PACKED} ${ARTIFACTS_FULL_PATH}
+  )
+
+  add_custom_target(copy_source_artifacts
+    COMMAND ${CMAKE_COMMAND} -E make_directory ${ARTIFACTS_FULL_PATH}
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_PROJECT_NAME}-${PRJ_VERSION_FULL}${PACKAGE_EXT} ${ARTIFACTS_FULL_PATH}
+  )
+endif()
+
+if(NOT "${PRJ_VERSION_FULL}" STREQUAL "")
+  if(DWARFS_GIT_BUILD)
+    configure_file("${PROJECT_SOURCE_DIR}/cmake/dwarfs_install.cmake.in" dwarfs_install.cmake @ONLY)
+    set(CPACK_INSTALL_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/dwarfs_install.cmake")
+  endif()
+
+  if(WIN32)
+    # set(CPACK_GENERATOR "NSIS;ZIP;7Z")
+    set(CPACK_GENERATOR "7Z")
+  else()
+    set(CPACK_GENERATOR "TXZ")
  endif()
  set(CPACK_SOURCE_GENERATOR "${CPACK_GENERATOR}")
  set(CPACK_PACKAGE_VERSION_MAJOR "${PRJ_VERSION_MAJOR}")
@ -1116,6 +1168,7 @@ if(NOT $PRJ_VERSION_FULL} STREQUAL "")
                                "${CMAKE_SOURCE_DIR}/@" "/\\." ".*~$")
  set(CPACK_VERBATIM_VARIABLES YES)
  set(CPACK_STRIP_FILES YES)
+  set(CPACK_ARCHIVE_THREADS 0)

  include(CPack)
 endif()