diff --git a/.github/scripts/build_definition.py b/.github/scripts/build_definition.py index 6395dfa..2a1cc78 100644 --- a/.github/scripts/build_definition.py +++ b/.github/scripts/build_definition.py @@ -17,10 +17,54 @@ import csv, io, re BUILD_DEF = """ | OS_NAME | COMPILE_CONFIG | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name | dependency_name | ============================================================================================================================================== +# manylinux is a special case as we need to compile libzim on old arch for python + | manylinux | native_mixed | BP | | | | | linux-x86_64-manylinux | | + | manylinux | aarch64_mixed | BP | | | | | linux-aarch64-manylinux | | + ---------------------------------------------------------------------------------------------------------------------------------------------- +# On Windows, we build only libzim for now. And only native_mixed as xapian doesn't compile as dll + | windows | native_static | Bd | d | BPd | BPd | | win-x86_64 | win-x86_64-static | + | windows | native_dyn | Bd | | | | | win-x86_64 | win-x86_64-dyn | + | windows | native_mixed | BPd | d | | | BPd | win-x86_64 | win-x86_64-mixed | + ---------------------------------------------------------------------------------------------------------------------------------------------- # Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries - | macos | native_dyn | B | | | | | | macos-x86_64-dyn | - | macos | native_static | B | | | | | macos-x86_64 | | - | macos | native_mixed | B | | | | | macos-x86_64 | | + | macos | native_dyn | d | d | dB | B | | | macos-x86_64-dyn | + | macos | native_static | | | BP | BP | | macos-x86_64 | | + | macos | native_mixed | BP | BP | | | | macos-x86_64 | | + | macos | iOS_arm64 | dB | dB | | | | | ios-arm64-dyn | + | macos | iOSSimulator_x86_64| dB | dB | | | | | ios-x86_64-dyn | + | macos | iOSSimulator_arm64 | B | B | | | | | | + | macos | macOS_arm64_static | | | BP | BP | | macos-arm64 | | + | macos | macOS_arm64_mixed | dBP | dBP | d | | | macos-arm64 | macos-aarch64-dyn | + | macos | macOS_x86_64 | B | B | | | | | | + | macos | apple_all_static | | BP | | | | xcframework | | + ---------------------------------------------------------------------------------------------------------------------------------------------- + | focal | flatpak | | | | | BP | | | + | focal | native_static | d | d | dBPSD | dBPSD | | linux-x86_64 | linux-x86_64-static | + | focal | native_mixed | BPS | BPS | | | | linux-x86_64 | | + | focal | native_dyn | d | d | dB | dB | | | linux-x86_64-dyn | + | jammy | native_dyn | | | | | dBPS | | linux-x86_64-dyn | + # libzim CI is building alpine_dyn but not us + | focal | android_arm | dBP | dBP | | | | android-arm | android-arm | + | focal | android_arm64 | dBP | dBP | | | | android-arm64 | android-arm64 | + | focal | android_x86 | BP | BP | | | | android-x86 | | + | focal | android_x86_64 | BP | BP | | | | android-x86_64 | | + | focal | armv6_static | | | BP | BP | | linux-armv6 | | + | focal | armv6_mixed | BP | | | | | linux-armv6 | | + | focal | armv6_dyn | | | B | B | | | | + | focal | armv8_static | | | BP | BP | | linux-armv8 | | + | focal | armv8_mixed | BP | | | | | linux-armv8 | | + | focal | armv8_dyn | | | B | B | | | | + | focal | aarch64_static | | | BP | BP | | linux-aarch64 | | + | focal | aarch64_mixed | BP | | | | | linux-aarch64 | | + | focal | aarch64_dyn | d | | B | B | | | linux-aarch64-dyn | + | focal | aarch64_musl_static| | | BP | BP | | linux-aarch64-musl | | + | focal | aarch64_musl_mixed | BP | | | | | linux-aarch64-musl | | + | focal | aarch64_musl_dyn | d | | B | B | | | linux-aarch64-musl-dyn | + | focal | x86-64_musl_static | | | BP | BP | | linux-x86_64-musl | | + | focal | x86-64_musl_mixed | BP | | | | | linux-x86_64-musl | | + | focal | i586_static | | | BP | BP | | linux-i586 | | + | focal | i586_dyn | | | B | B | | | | + | focal | wasm | dBP | | | | | wasm-emscripten | wasm | """ diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index b86ea27..2ec4a69 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -8,6 +8,220 @@ on: - cron: '0 3 * * *' jobs: + Windows: + strategy: + fail-fast: false + matrix: + config: + - native_mixed + - native_dyn + - native_static + runs-on: windows-2022 + env: + OS_NAME: windows + COMPILE_CONFIG: ${{matrix.config}} + HOME: 'C:\\Users\\runneradmin' + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Setup python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install packages + run: | + choco.exe install pkgconfiglite ninja + - name: Install python modules + shell: bash + run: | + pip3 install meson pytest requests distro paramiko + pip3 install --no-deps $GITHUB_WORKSPACE + - name: Install QT + if: ${{ matrix.config == 'native_mixed' }} + uses: jurplel/install-qt-action@v4 + with: + version: 5.15.2 + modules: "qtwebengine" + setup-python: false + env: + AQT_CONFIG: ${{ github.workspace }}/.github/configs/aqt.ini + - name: Setup MSVC compiler + uses: bus1/cabuild/action/msdevshell@v1 + with: + architecture: x64 + - name: secret + shell: bash + run: | + echo "${{secrets.ssh_key}}" > $SSH_KEY + env: + SSH_KEY: ${{ runner.temp }}/id_rsa + - name: Install and configure eSigner CKA and Windows SDK + if: github.event_name == 'push' + env: + ESIGNER_URL: https://github.com/SSLcom/eSignerCKA/releases/download/v1.0.7/SSL.COM-eSigner-CKA_1.0.7.zip + run: | + Set-StrictMode -Version 'Latest' + + # Download and Unzip eSignerCKA Setup + Invoke-WebRequest -OutFile eSigner_CKA_Setup.zip "$env:ESIGNER_URL" + Expand-Archive -Force eSigner_CKA_Setup.zip + Remove-Item eSigner_CKA_Setup.zip + Move-Item -Destination “eSigner_CKA_Installer.exe” -Path “eSigner_CKA_*\*.exe” + + # Install eSignerCKA + New-Item -ItemType Directory -Force -Path "C:\esigner" + ./eSigner_CKA_Installer.exe /CURRENTUSER /VERYSILENT /SUPPRESSMSGBOXES /DIR=”C:\esigner” /TYPE=automatic | Out-Null + Remove-Item "eSigner_CKA_Installer.exe" + + # Configure the CKA with SSL.com credentials + C:\esigner\eSignerCKATool.exe config -mode product -user "${{ secrets.ESIGNER_USERNAME }}" -pass "${{ secrets.ESIGNER_PASSWORD }}" -totp "${{ secrets.ESIGNER_TOTP_SECRET }}" -key "C:\esigner\master.key" -r + C:\esigner\eSignerCKATool.exe unload + C:\esigner\eSignerCKATool.exe load + + # Find certificate + $CodeSigningCert = Get-ChildItem Cert:\CurrentUser\My -CodeSigningCert | Select-Object -First 1 + echo Certificate: $CodeSigningCert + + # Extract thumbprint and subject name + $Thumbprint = $CodeSigningCert.Thumbprint + echo "SIGNTOOL_THUMBPRINT=$Thumbprint" >> $env:GITHUB_ENV + + - name: Ensure base deps + run: | + python .github\\scripts\\ensure_base_deps.py + env: + SSH_KEY: ${{ runner.temp }}/id_rsa + - name: Build Release + run: | + python .github\\scripts\\build_release_nightly.py + env: + SIGNTOOL_PATH: "C:/Program Files (x86)/Windows Kits/10/bin/10.0.22621.0/x86/signtool.exe" + SSH_KEY: ${{ runner.temp }}/id_rsa + - name: Upload failure logs + if: failure() + run: | + python .github\\scripts\\upload_failure_logs.py + env: + SSH_KEY: ${{ runner.temp }}/id_rsa + + Linux: + strategy: + fail-fast: false + matrix: + config: + - native_static + - native_mixed + - native_dyn + - wasm + - armv6_static + - armv6_mixed + - armv8_static + - armv8_mixed + - aarch64_static + - aarch64_mixed + - aarch64_musl_static + - aarch64_musl_mixed + - x86-64_musl_static + - x86-64_musl_mixed + - i586_static + - android_arm + - android_arm64 + - android_x86 + - android_x86_64 + image_variant: ['focal'] + include: + - config: native_mixed + image_variant: manylinux + - config: aarch64_mixed + image_variant: manylinux + - config: native_dyn + image_variant: jammy + env: + HOME: /home/runner + SSH_KEY: /tmp/id_rsa + runs-on: ubuntu-22.04 + container: + image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:2024-11-30" + options: "--device /dev/fuse --privileged" + steps: + - name: Checkout code + shell: bash + run: | + cd $HOME + git clone https://github.com/${REP} + cd ./${REP##*/} + git checkout --force ${GITHUB_SHA} + pip3 install --user --no-deps . + env: + REP: ${{github.repository}} + - name: secret + shell: bash + run: | + echo "${{secrets.ssh_key}}" > $SSH_KEY + chmod 600 $SSH_KEY + - name: Ensure base deps + shell: bash + run: | + cd $HOME + kiwix-build/.github/scripts/ensure_base_deps.py + env: + COMPILE_CONFIG: ${{matrix.config}} + - name: Build release + shell: bash + run: | + cd $HOME + kiwix-build/.github/scripts/build_release_nightly.py + env: + COMPILE_CONFIG: ${{matrix.config}} + - name: Upload failure logs + if: failure() + run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py + env: + COMPILE_CONFIG: ${{matrix.config}} + + Flatpak: + strategy: + fail-fast: false + env: + HOME: /home/runner + SSH_KEY: /tmp/id_rsa + COMPILE_CONFIG: flatpak + OS_NAME: focal + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + shell: bash + run: | + cd $HOME + git clone https://github.com/${REP} + cd ./${REP##*/} + git checkout --force ${GITHUB_SHA} + pip3 install --user --no-deps . + env: + REP: ${{github.repository}} + - name: Install flatpak tools + run: | + sudo apt-get update + sudo apt-get install flatpak-builder ninja-build meson + - name: secret + shell: bash + run: | + echo "${{secrets.ssh_key}}" > $SSH_KEY + chmod 600 $SSH_KEY + - name: Ensure base deps + shell: bash + run: | + cd $HOME + kiwix-build/.github/scripts/ensure_base_deps.py + - name: Build release + shell: bash + run: | + cd $HOME + kiwix-build/.github/scripts/build_release_nightly.py + - name: Upload failure logs + if: failure() + run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py + Macos: strategy: fail-fast: false @@ -16,6 +230,9 @@ jobs: - native_dyn - native_static - native_mixed + - macOS_arm64_static + - macOS_arm64_mixed + - apple_all_static runs-on: macos-13 env: SSH_KEY: /tmp/id_rsa @@ -85,3 +302,25 @@ jobs: run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.py env: COMPILE_CONFIG: ${{matrix.config}} + + + Trigger_Docker: + needs: [Linux] + runs-on: ubuntu-22.04 + env: + COMPILE_CONFIG: native_static + OS_NAME: focal + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Install python modules + shell: bash + run: | + pip3 install --user --no-deps $GITHUB_WORKSPACE + - name: Trigger docker workflow + shell: bash + run: | + cd $HOME + $GITHUB_WORKSPACE/.github/scripts/trigger_docker_workflow.py + env: + GITHUB_PAT: ${{secrets.DOCKER_TRIGGER_GITHUB_PAT}} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0bb8f95..56391c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,211 @@ on: - cron: '0 1 * * *' jobs: + Windows: + strategy: + fail-fast: false + matrix: + config: + - native_mixed + - native_dyn + - native_static + runs-on: windows-2022 + env: + OS_NAME: windows + COMPILE_CONFIG: ${{matrix.config}} + HOME: 'C:\\Users\\runneradmin' + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Setup python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install packages + run: | + choco.exe install pkgconfiglite ninja + - name: Install python modules + shell: bash + run: | + pip3 install meson pytest requests distro paramiko + pip3 install --no-deps $GITHUB_WORKSPACE + - name: Install QT + if: ${{ matrix.config == 'native_mixed' }} + uses: jurplel/install-qt-action@v4 + with: + version: 6.4.3 + modules: "qtwebengine qtwebchannel qtpositioning" + setup-python: false + env: + AQT_CONFIG: ${{ github.workspace }}/.github/configs/aqt.ini + - name: Setup MSVC compiler + uses: bus1/cabuild/action/msdevshell@v1 + with: + architecture: x64 + - name: secret + shell: bash + run: | + echo "${{secrets.ssh_key}}" > $SSH_KEY + env: + SSH_KEY: ${{ runner.temp }}/id_rsa + - name: Ensure base deps + run: | + python .github\\scripts\\ensure_base_deps.py + env: + SSH_KEY: ${{ runner.temp }}/id_rsa + - name: Compile all deps + run: | + python .github\\scripts\\compile_all_deps.py + env: + SSH_KEY: ${{ runner.temp }}/id_rsa + - name: Build projects + run: | + python .github\\scripts\\build_projects.py + env: + SSH_KEY: ${{ runner.temp }}/id_rsa + - name: Upload failure logs + if: failure() + run: | + python .github\\scripts\\upload_failure_logs.py + env: + SSH_KEY: ${{ runner.temp }}/id_rsa + + Linux: + strategy: + fail-fast: false + matrix: + config: + - native_static + - native_dyn + - native_mixed + - wasm + - armv6_static + - armv6_dyn + - armv6_mixed + - armv8_static + - armv8_dyn + - armv8_mixed + - aarch64_static + - aarch64_dyn + - aarch64_mixed + - aarch64_musl_static + - aarch64_musl_dyn + - aarch64_musl_mixed + - x86-64_musl_static + - x86-64_musl_mixed + - i586_static + - i586_dyn + - android_arm + - android_arm64 + - android_x86 + - android_x86_64 + image_variant: ['focal'] + include: + - config: native_mixed + image_variant: manylinux + - config: aarch64_mixed + image_variant: manylinux + - config: native_dyn + image_variant: jammy + env: + HOME: /home/runner + SSH_KEY: /tmp/id_rsa + OS_NAME: ${{matrix.image_variant}} + runs-on: ubuntu-22.04 + container: + image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:2024-11-30" + options: "--device /dev/fuse --privileged" + steps: + - name: Checkout code + shell: bash + run: | + cd $HOME + git clone https://github.com/${REP} + cd ./${REP##*/} + git checkout --force ${GITHUB_SHA} + pip3 install --user --no-deps . + env: + REP: ${{github.repository}} + - name: Install paramiko + if: ${{matrix.image_variant != 'bionic' }} + shell: bash + run: pip3 install --user paramiko + - name: secret + shell: bash + run: | + echo "${{secrets.ssh_key}}" > $SSH_KEY + chmod 600 $SSH_KEY + - name: Ensure base deps + shell: bash + run: | + cd $HOME + kiwix-build/.github/scripts/ensure_base_deps.py + env: + COMPILE_CONFIG: ${{matrix.config}} + - name: Compile all deps + shell: bash + run: | + cd $HOME + kiwix-build/.github/scripts/compile_all_deps.py + env: + COMPILE_CONFIG: ${{matrix.config}} + - name: Build projects + shell: bash + run: | + cd $HOME + kiwix-build/.github/scripts/build_projects.py + env: + COMPILE_CONFIG: ${{matrix.config}} + - name: Upload failure logs + if: failure() + run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py + env: + COMPILE_CONFIG: ${{matrix.config}} + + Flatpak: + strategy: + fail-fast: false + env: + HOME: /home/runner + SSH_KEY: /tmp/id_rsa + COMPILE_CONFIG: flatpak + OS_NAME: focal + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + shell: bash + run: | + cd $HOME + git clone https://github.com/${REP} + cd ./${REP##*/} + git checkout --force ${GITHUB_SHA} + pip3 install --user paramiko + pip3 install --user --no-deps . + env: + REP: ${{github.repository}} + - name: Install flatpak tools + run: | + sudo apt-get update + sudo apt-get install flatpak-builder ninja-build meson + - name: secret + shell: bash + run: | + echo "${{secrets.ssh_key}}" > $SSH_KEY + chmod 600 $SSH_KEY + - name: Ensure base deps + shell: bash + run: | + cd $HOME + kiwix-build/.github/scripts/ensure_base_deps.py + - name: Build projects + shell: bash + run: | + cd $HOME + kiwix-build/.github/scripts/build_projects.py + - name: Upload failure logs + if: failure() + run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py + Macos: strategy: fail-fast: false @@ -14,6 +219,13 @@ jobs: - native_dyn - native_static - native_mixed + - iOS_arm64 + - iOSSimulator_x86_64 + - iOSSimulator_arm64 + - macOS_arm64_static + - macOS_arm64_mixed + - macOS_x86_64 + - apple_all_static runs-on: macos-13 env: SSH_KEY: /tmp/id_rsa