Restore all CI/CD jobs

This commit is contained in:
Veloman Yunkan 2025-05-06 15:48:53 +04:00
parent b1fc821b1f
commit d96ee42e0d
3 changed files with 498 additions and 3 deletions

View File

@ -17,10 +17,54 @@ import csv, io, re
BUILD_DEF = """
| OS_NAME | COMPILE_CONFIG | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name | dependency_name |
==============================================================================================================================================
# manylinux is a special case as we need to compile libzim on old arch for python
| manylinux | native_mixed | BP | | | | | linux-x86_64-manylinux | |
| manylinux | aarch64_mixed | BP | | | | | linux-aarch64-manylinux | |
----------------------------------------------------------------------------------------------------------------------------------------------
# On Windows, we build only libzim for now. And only native_mixed as xapian doesn't compile as dll
| windows | native_static | Bd | d | BPd | BPd | | win-x86_64 | win-x86_64-static |
| windows | native_dyn | Bd | | | | | win-x86_64 | win-x86_64-dyn |
| windows | native_mixed | BPd | d | | | BPd | win-x86_64 | win-x86_64-mixed |
----------------------------------------------------------------------------------------------------------------------------------------------
# Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries
| macos | native_dyn | B | | | | | | macos-x86_64-dyn |
| macos | native_static | B | | | | | macos-x86_64 | |
| macos | native_mixed | B | | | | | macos-x86_64 | |
| macos | native_dyn | d | d | dB | B | | | macos-x86_64-dyn |
| macos | native_static | | | BP | BP | | macos-x86_64 | |
| macos | native_mixed | BP | BP | | | | macos-x86_64 | |
| macos | iOS_arm64 | dB | dB | | | | | ios-arm64-dyn |
| macos | iOSSimulator_x86_64| dB | dB | | | | | ios-x86_64-dyn |
| macos | iOSSimulator_arm64 | B | B | | | | | |
| macos | macOS_arm64_static | | | BP | BP | | macos-arm64 | |
| macos | macOS_arm64_mixed | dBP | dBP | d | | | macos-arm64 | macos-aarch64-dyn |
| macos | macOS_x86_64 | B | B | | | | | |
| macos | apple_all_static | | BP | | | | xcframework | |
----------------------------------------------------------------------------------------------------------------------------------------------
| focal | flatpak | | | | | BP | | |
| focal | native_static | d | d | dBPSD | dBPSD | | linux-x86_64 | linux-x86_64-static |
| focal | native_mixed | BPS | BPS | | | | linux-x86_64 | |
| focal | native_dyn | d | d | dB | dB | | | linux-x86_64-dyn |
| jammy | native_dyn | | | | | dBPS | | linux-x86_64-dyn |
# libzim CI is building alpine_dyn but not us
| focal | android_arm | dBP | dBP | | | | android-arm | android-arm |
| focal | android_arm64 | dBP | dBP | | | | android-arm64 | android-arm64 |
| focal | android_x86 | BP | BP | | | | android-x86 | |
| focal | android_x86_64 | BP | BP | | | | android-x86_64 | |
| focal | armv6_static | | | BP | BP | | linux-armv6 | |
| focal | armv6_mixed | BP | | | | | linux-armv6 | |
| focal | armv6_dyn | | | B | B | | | |
| focal | armv8_static | | | BP | BP | | linux-armv8 | |
| focal | armv8_mixed | BP | | | | | linux-armv8 | |
| focal | armv8_dyn | | | B | B | | | |
| focal | aarch64_static | | | BP | BP | | linux-aarch64 | |
| focal | aarch64_mixed | BP | | | | | linux-aarch64 | |
| focal | aarch64_dyn | d | | B | B | | | linux-aarch64-dyn |
| focal | aarch64_musl_static| | | BP | BP | | linux-aarch64-musl | |
| focal | aarch64_musl_mixed | BP | | | | | linux-aarch64-musl | |
| focal | aarch64_musl_dyn | d | | B | B | | | linux-aarch64-musl-dyn |
| focal | x86-64_musl_static | | | BP | BP | | linux-x86_64-musl | |
| focal | x86-64_musl_mixed | BP | | | | | linux-x86_64-musl | |
| focal | i586_static | | | BP | BP | | linux-i586 | |
| focal | i586_dyn | | | B | B | | | |
| focal | wasm | dBP | | | | | wasm-emscripten | wasm |
"""

View File

@ -8,6 +8,220 @@ on:
- cron: '0 3 * * *'
jobs:
Windows:
strategy:
fail-fast: false
matrix:
config:
- native_mixed
- native_dyn
- native_static
runs-on: windows-2022
env:
OS_NAME: windows
COMPILE_CONFIG: ${{matrix.config}}
HOME: 'C:\\Users\\runneradmin'
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup python 3.12
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install packages
run: |
choco.exe install pkgconfiglite ninja
- name: Install python modules
shell: bash
run: |
pip3 install meson pytest requests distro paramiko
pip3 install --no-deps $GITHUB_WORKSPACE
- name: Install QT
if: ${{ matrix.config == 'native_mixed' }}
uses: jurplel/install-qt-action@v4
with:
version: 5.15.2
modules: "qtwebengine"
setup-python: false
env:
AQT_CONFIG: ${{ github.workspace }}/.github/configs/aqt.ini
- name: Setup MSVC compiler
uses: bus1/cabuild/action/msdevshell@v1
with:
architecture: x64
- name: secret
shell: bash
run: |
echo "${{secrets.ssh_key}}" > $SSH_KEY
env:
SSH_KEY: ${{ runner.temp }}/id_rsa
- name: Install and configure eSigner CKA and Windows SDK
if: github.event_name == 'push'
env:
ESIGNER_URL: https://github.com/SSLcom/eSignerCKA/releases/download/v1.0.7/SSL.COM-eSigner-CKA_1.0.7.zip
run: |
Set-StrictMode -Version 'Latest'
# Download and Unzip eSignerCKA Setup
Invoke-WebRequest -OutFile eSigner_CKA_Setup.zip "$env:ESIGNER_URL"
Expand-Archive -Force eSigner_CKA_Setup.zip
Remove-Item eSigner_CKA_Setup.zip
Move-Item -Destination “eSigner_CKA_Installer.exe” -Path “eSigner_CKA_*\*.exe”
# Install eSignerCKA
New-Item -ItemType Directory -Force -Path "C:\esigner"
./eSigner_CKA_Installer.exe /CURRENTUSER /VERYSILENT /SUPPRESSMSGBOXES /DIR=”C:\esigner” /TYPE=automatic | Out-Null
Remove-Item "eSigner_CKA_Installer.exe"
# Configure the CKA with SSL.com credentials
C:\esigner\eSignerCKATool.exe config -mode product -user "${{ secrets.ESIGNER_USERNAME }}" -pass "${{ secrets.ESIGNER_PASSWORD }}" -totp "${{ secrets.ESIGNER_TOTP_SECRET }}" -key "C:\esigner\master.key" -r
C:\esigner\eSignerCKATool.exe unload
C:\esigner\eSignerCKATool.exe load
# Find certificate
$CodeSigningCert = Get-ChildItem Cert:\CurrentUser\My -CodeSigningCert | Select-Object -First 1
echo Certificate: $CodeSigningCert
# Extract thumbprint and subject name
$Thumbprint = $CodeSigningCert.Thumbprint
echo "SIGNTOOL_THUMBPRINT=$Thumbprint" >> $env:GITHUB_ENV
- name: Ensure base deps
run: |
python .github\\scripts\\ensure_base_deps.py
env:
SSH_KEY: ${{ runner.temp }}/id_rsa
- name: Build Release
run: |
python .github\\scripts\\build_release_nightly.py
env:
SIGNTOOL_PATH: "C:/Program Files (x86)/Windows Kits/10/bin/10.0.22621.0/x86/signtool.exe"
SSH_KEY: ${{ runner.temp }}/id_rsa
- name: Upload failure logs
if: failure()
run: |
python .github\\scripts\\upload_failure_logs.py
env:
SSH_KEY: ${{ runner.temp }}/id_rsa
Linux:
strategy:
fail-fast: false
matrix:
config:
- native_static
- native_mixed
- native_dyn
- wasm
- armv6_static
- armv6_mixed
- armv8_static
- armv8_mixed
- aarch64_static
- aarch64_mixed
- aarch64_musl_static
- aarch64_musl_mixed
- x86-64_musl_static
- x86-64_musl_mixed
- i586_static
- android_arm
- android_arm64
- android_x86
- android_x86_64
image_variant: ['focal']
include:
- config: native_mixed
image_variant: manylinux
- config: aarch64_mixed
image_variant: manylinux
- config: native_dyn
image_variant: jammy
env:
HOME: /home/runner
SSH_KEY: /tmp/id_rsa
runs-on: ubuntu-22.04
container:
image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:2024-11-30"
options: "--device /dev/fuse --privileged"
steps:
- name: Checkout code
shell: bash
run: |
cd $HOME
git clone https://github.com/${REP}
cd ./${REP##*/}
git checkout --force ${GITHUB_SHA}
pip3 install --user --no-deps .
env:
REP: ${{github.repository}}
- name: secret
shell: bash
run: |
echo "${{secrets.ssh_key}}" > $SSH_KEY
chmod 600 $SSH_KEY
- name: Ensure base deps
shell: bash
run: |
cd $HOME
kiwix-build/.github/scripts/ensure_base_deps.py
env:
COMPILE_CONFIG: ${{matrix.config}}
- name: Build release
shell: bash
run: |
cd $HOME
kiwix-build/.github/scripts/build_release_nightly.py
env:
COMPILE_CONFIG: ${{matrix.config}}
- name: Upload failure logs
if: failure()
run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
env:
COMPILE_CONFIG: ${{matrix.config}}
Flatpak:
strategy:
fail-fast: false
env:
HOME: /home/runner
SSH_KEY: /tmp/id_rsa
COMPILE_CONFIG: flatpak
OS_NAME: focal
runs-on: ubuntu-22.04
steps:
- name: Checkout code
shell: bash
run: |
cd $HOME
git clone https://github.com/${REP}
cd ./${REP##*/}
git checkout --force ${GITHUB_SHA}
pip3 install --user --no-deps .
env:
REP: ${{github.repository}}
- name: Install flatpak tools
run: |
sudo apt-get update
sudo apt-get install flatpak-builder ninja-build meson
- name: secret
shell: bash
run: |
echo "${{secrets.ssh_key}}" > $SSH_KEY
chmod 600 $SSH_KEY
- name: Ensure base deps
shell: bash
run: |
cd $HOME
kiwix-build/.github/scripts/ensure_base_deps.py
- name: Build release
shell: bash
run: |
cd $HOME
kiwix-build/.github/scripts/build_release_nightly.py
- name: Upload failure logs
if: failure()
run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
Macos:
strategy:
fail-fast: false
@ -16,6 +230,9 @@ jobs:
- native_dyn
- native_static
- native_mixed
- macOS_arm64_static
- macOS_arm64_mixed
- apple_all_static
runs-on: macos-13
env:
SSH_KEY: /tmp/id_rsa
@ -85,3 +302,25 @@ jobs:
run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.py
env:
COMPILE_CONFIG: ${{matrix.config}}
Trigger_Docker:
needs: [Linux]
runs-on: ubuntu-22.04
env:
COMPILE_CONFIG: native_static
OS_NAME: focal
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install python modules
shell: bash
run: |
pip3 install --user --no-deps $GITHUB_WORKSPACE
- name: Trigger docker workflow
shell: bash
run: |
cd $HOME
$GITHUB_WORKSPACE/.github/scripts/trigger_docker_workflow.py
env:
GITHUB_PAT: ${{secrets.DOCKER_TRIGGER_GITHUB_PAT}}

View File

@ -6,6 +6,211 @@ on:
- cron: '0 1 * * *'
jobs:
Windows:
strategy:
fail-fast: false
matrix:
config:
- native_mixed
- native_dyn
- native_static
runs-on: windows-2022
env:
OS_NAME: windows
COMPILE_CONFIG: ${{matrix.config}}
HOME: 'C:\\Users\\runneradmin'
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup python 3.12
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install packages
run: |
choco.exe install pkgconfiglite ninja
- name: Install python modules
shell: bash
run: |
pip3 install meson pytest requests distro paramiko
pip3 install --no-deps $GITHUB_WORKSPACE
- name: Install QT
if: ${{ matrix.config == 'native_mixed' }}
uses: jurplel/install-qt-action@v4
with:
version: 6.4.3
modules: "qtwebengine qtwebchannel qtpositioning"
setup-python: false
env:
AQT_CONFIG: ${{ github.workspace }}/.github/configs/aqt.ini
- name: Setup MSVC compiler
uses: bus1/cabuild/action/msdevshell@v1
with:
architecture: x64
- name: secret
shell: bash
run: |
echo "${{secrets.ssh_key}}" > $SSH_KEY
env:
SSH_KEY: ${{ runner.temp }}/id_rsa
- name: Ensure base deps
run: |
python .github\\scripts\\ensure_base_deps.py
env:
SSH_KEY: ${{ runner.temp }}/id_rsa
- name: Compile all deps
run: |
python .github\\scripts\\compile_all_deps.py
env:
SSH_KEY: ${{ runner.temp }}/id_rsa
- name: Build projects
run: |
python .github\\scripts\\build_projects.py
env:
SSH_KEY: ${{ runner.temp }}/id_rsa
- name: Upload failure logs
if: failure()
run: |
python .github\\scripts\\upload_failure_logs.py
env:
SSH_KEY: ${{ runner.temp }}/id_rsa
Linux:
strategy:
fail-fast: false
matrix:
config:
- native_static
- native_dyn
- native_mixed
- wasm
- armv6_static
- armv6_dyn
- armv6_mixed
- armv8_static
- armv8_dyn
- armv8_mixed
- aarch64_static
- aarch64_dyn
- aarch64_mixed
- aarch64_musl_static
- aarch64_musl_dyn
- aarch64_musl_mixed
- x86-64_musl_static
- x86-64_musl_mixed
- i586_static
- i586_dyn
- android_arm
- android_arm64
- android_x86
- android_x86_64
image_variant: ['focal']
include:
- config: native_mixed
image_variant: manylinux
- config: aarch64_mixed
image_variant: manylinux
- config: native_dyn
image_variant: jammy
env:
HOME: /home/runner
SSH_KEY: /tmp/id_rsa
OS_NAME: ${{matrix.image_variant}}
runs-on: ubuntu-22.04
container:
image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:2024-11-30"
options: "--device /dev/fuse --privileged"
steps:
- name: Checkout code
shell: bash
run: |
cd $HOME
git clone https://github.com/${REP}
cd ./${REP##*/}
git checkout --force ${GITHUB_SHA}
pip3 install --user --no-deps .
env:
REP: ${{github.repository}}
- name: Install paramiko
if: ${{matrix.image_variant != 'bionic' }}
shell: bash
run: pip3 install --user paramiko
- name: secret
shell: bash
run: |
echo "${{secrets.ssh_key}}" > $SSH_KEY
chmod 600 $SSH_KEY
- name: Ensure base deps
shell: bash
run: |
cd $HOME
kiwix-build/.github/scripts/ensure_base_deps.py
env:
COMPILE_CONFIG: ${{matrix.config}}
- name: Compile all deps
shell: bash
run: |
cd $HOME
kiwix-build/.github/scripts/compile_all_deps.py
env:
COMPILE_CONFIG: ${{matrix.config}}
- name: Build projects
shell: bash
run: |
cd $HOME
kiwix-build/.github/scripts/build_projects.py
env:
COMPILE_CONFIG: ${{matrix.config}}
- name: Upload failure logs
if: failure()
run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
env:
COMPILE_CONFIG: ${{matrix.config}}
Flatpak:
strategy:
fail-fast: false
env:
HOME: /home/runner
SSH_KEY: /tmp/id_rsa
COMPILE_CONFIG: flatpak
OS_NAME: focal
runs-on: ubuntu-22.04
steps:
- name: Checkout code
shell: bash
run: |
cd $HOME
git clone https://github.com/${REP}
cd ./${REP##*/}
git checkout --force ${GITHUB_SHA}
pip3 install --user paramiko
pip3 install --user --no-deps .
env:
REP: ${{github.repository}}
- name: Install flatpak tools
run: |
sudo apt-get update
sudo apt-get install flatpak-builder ninja-build meson
- name: secret
shell: bash
run: |
echo "${{secrets.ssh_key}}" > $SSH_KEY
chmod 600 $SSH_KEY
- name: Ensure base deps
shell: bash
run: |
cd $HOME
kiwix-build/.github/scripts/ensure_base_deps.py
- name: Build projects
shell: bash
run: |
cd $HOME
kiwix-build/.github/scripts/build_projects.py
- name: Upload failure logs
if: failure()
run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
Macos:
strategy:
fail-fast: false
@ -14,6 +219,13 @@ jobs:
- native_dyn
- native_static
- native_mixed
- iOS_arm64
- iOSSimulator_x86_64
- iOSSimulator_arm64
- macOS_arm64_static
- macOS_arm64_mixed
- macOS_x86_64
- apple_all_static
runs-on: macos-13
env:
SSH_KEY: /tmp/id_rsa