From dd1a276d7a0b6b2f6a7c28e054446de0b9c480c6 Mon Sep 17 00:00:00 2001 From: Emmanuel Engelhart Date: Thu, 9 Feb 2023 17:55:23 +0100 Subject: [PATCH 1/9] Move CI container building to a dedicated CI workflow --- .github/workflows/container.yml | 40 ++++++++++++++++++++++++++++ .github/workflows/releaseNigthly.yml | 23 ---------------- 2 files changed, 40 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/container.yml diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml new file mode 100644 index 0000000..d8ef0f6 --- /dev/null +++ b/.github/workflows/container.yml @@ -0,0 +1,40 @@ +name: CI Containers + +on: + push: + paths: + - '.github/ci_images/**' + release: + types: [published] + +env: + IMAGE_PREFIX: kiwix/kiwix-build_ci_ + +jobs: + Container: + strategy: + fail-fast: false + matrix: + variant: [bionic, f35, focal, alpine] + runs-on: ubuntu-22.04 + + steps: + - name: Retrieve the code + uses: actions/checkout@v3 + + - name: Build & upload dev container image + env: + IMAGE_NAME: ${IMAGE_PREFIX}${{matrix.variant}}:dev + run: | + docker build -t ${IMAGE_NAME} - < ${GITHUB_WORKSPACE}/.github/ci_images/${{matrix.variant}}_builder.dockerfile + echo "${{ secrets.GHCR_TOKEN }}" | docker login -u "${{ secrets.GHCR_USERNAME }}" --password-stdin + docker push ${IMAGE_NAME} + + - name: Build & upload release container image + if: github.event.action == 'published' + env: + IMAGE_NAME: ${IMAGE_PREFIX}${{matrix.variant}}:${GITHUB_REF_NAME} + run: | + docker build -t ${IMAGE_NAME} - < ${GITHUB_WORKSPACE}/.github/ci_images/${{matrix.variant}}_builder.dockerfile + echo "${{ secrets.GHCR_TOKEN }}" | docker login -u "${{ secrets.GHCR_USERNAME }}" --password-stdin + docker push --all-tags ${IMAGE_NAME} diff --git a/.github/workflows/releaseNigthly.yml b/.github/workflows/releaseNigthly.yml index 00234ca..dd3ebe5 100644 --- a/.github/workflows/releaseNigthly.yml +++ b/.github/workflows/releaseNigthly.yml @@ -7,30 +7,7 @@ on: schedule: - cron: '0 1 * * *' -env: - DOCKER_VERSION: 32 - jobs: - Docker: - strategy: - fail-fast: false - matrix: - variant: [bionic, f35, focal, alpine] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Build docker image - shell: bash - run: | - TAGNAME=${{matrix.variant}}-${DOCKER_VERSION} - if [ ! $(curl -sflL https://hub.docker.com/v2/repositories/kiwix/kiwix-build_ci/tags/${TAGNAME}) ] - then - echo "${{secrets.docker_password}}" | docker login -u "${{secrets.docker_username}}" --password-stdin - FULLTAGNAME=kiwix/kiwix-build_ci:${TAGNAME} - docker build -t ${FULLTAGNAME} - < ${GITHUB_WORKSPACE}/.github/ci_images/${{matrix.variant}}_builder.dockerfile - docker push ${FULLTAGNAME} - fi - Linux: strategy: fail-fast: false From b16cedc6be0507d5a011f5b6e2480a0778ae9103 Mon Sep 17 00:00:00 2001 From: Emmanuel Engelhart Date: Thu, 9 Feb 2023 18:33:22 +0100 Subject: [PATCH 2/9] Move ci_images --- .github/workflows/container.yml | 10 +++++----- .../ci_images => ci_images}/alpine_builder.dockerfile | 0 .../ci_images => ci_images}/bionic_builder.dockerfile | 0 .../ci_images => ci_images}/f35_builder.dockerfile | 0 .../ci_images => ci_images}/focal_builder.dockerfile | 0 5 files changed, 5 insertions(+), 5 deletions(-) rename {.github/ci_images => ci_images}/alpine_builder.dockerfile (100%) rename {.github/ci_images => ci_images}/bionic_builder.dockerfile (100%) rename {.github/ci_images => ci_images}/f35_builder.dockerfile (100%) rename {.github/ci_images => ci_images}/focal_builder.dockerfile (100%) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index d8ef0f6..4711e9c 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -3,7 +3,7 @@ name: CI Containers on: push: paths: - - '.github/ci_images/**' + - 'ci_images/**' release: types: [published] @@ -24,17 +24,17 @@ jobs: - name: Build & upload dev container image env: - IMAGE_NAME: ${IMAGE_PREFIX}${{matrix.variant}}:dev + IMAGE_NAME: ${{ env.IMAGE_PREFIX }}${{ matrix.variant }}:dev run: | - docker build -t ${IMAGE_NAME} - < ${GITHUB_WORKSPACE}/.github/ci_images/${{matrix.variant}}_builder.dockerfile + docker build -t ${IMAGE_NAME} - < ${GITHUB_WORKSPACE}/ci_images/${{matrix.variant}}_builder.dockerfile echo "${{ secrets.GHCR_TOKEN }}" | docker login -u "${{ secrets.GHCR_USERNAME }}" --password-stdin docker push ${IMAGE_NAME} - name: Build & upload release container image if: github.event.action == 'published' env: - IMAGE_NAME: ${IMAGE_PREFIX}${{matrix.variant}}:${GITHUB_REF_NAME} + IMAGE_NAME: ${{ env.IMAGE_PREFIX }}${{ matrix.variant }}:${{ env.GITHUB_REF_NAME }} run: | - docker build -t ${IMAGE_NAME} - < ${GITHUB_WORKSPACE}/.github/ci_images/${{matrix.variant}}_builder.dockerfile + docker build -t ${IMAGE_NAME} - < ${GITHUB_WORKSPACE}/ci_images/${{matrix.variant}}_builder.dockerfile echo "${{ secrets.GHCR_TOKEN }}" | docker login -u "${{ secrets.GHCR_USERNAME }}" --password-stdin docker push --all-tags ${IMAGE_NAME} diff --git a/.github/ci_images/alpine_builder.dockerfile b/ci_images/alpine_builder.dockerfile similarity index 100% rename from .github/ci_images/alpine_builder.dockerfile rename to ci_images/alpine_builder.dockerfile diff --git a/.github/ci_images/bionic_builder.dockerfile b/ci_images/bionic_builder.dockerfile similarity index 100% rename from .github/ci_images/bionic_builder.dockerfile rename to ci_images/bionic_builder.dockerfile diff --git a/.github/ci_images/f35_builder.dockerfile b/ci_images/f35_builder.dockerfile similarity index 100% rename from .github/ci_images/f35_builder.dockerfile rename to ci_images/f35_builder.dockerfile diff --git a/.github/ci_images/focal_builder.dockerfile b/ci_images/focal_builder.dockerfile similarity index 100% rename from .github/ci_images/focal_builder.dockerfile rename to ci_images/focal_builder.dockerfile From b977392446da0c08755ff19935635e002efa3557 Mon Sep 17 00:00:00 2001 From: Emmanuel Engelhart Date: Thu, 9 Feb 2023 20:23:37 +0100 Subject: [PATCH 3/9] Remove CI container builds --- .github/workflows/ci.yml | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5df2753..8d94ef3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,30 +5,7 @@ on: schedule: - cron: '0 1 * * *' -env: - DOCKER_VERSION: 32 - jobs: - Docker: - strategy: - fail-fast: false - matrix: - variant: [bionic, f35, focal, alpine] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Build docker image - shell: bash - run: | - TAGNAME=${{matrix.variant}}-${DOCKER_VERSION} - if [ ! $(curl -sflL https://hub.docker.com/v2/repositories/kiwix/kiwix-build_ci/tags/${TAGNAME}) ] - then - echo "${{secrets.docker_password}}" | docker login -u "${{secrets.docker_username}}" --password-stdin - FULLTAGNAME=kiwix/kiwix-build_ci:${TAGNAME} - docker build -t ${FULLTAGNAME} - < ${GITHUB_WORKSPACE}/.github/ci_images/${{matrix.variant}}_builder.dockerfile - docker push ${FULLTAGNAME} - fi - Linux: strategy: fail-fast: false @@ -60,7 +37,6 @@ jobs: HOME: /home/runner SSH_KEY: /tmp/id_rsa runs-on: ubuntu-latest - needs: Docker container: image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-32" options: "--device /dev/fuse --privileged" From c1b8ff5d8649e0d9a0a0f5e77eab249e449529f4 Mon Sep 17 00:00:00 2001 From: Emmanuel Engelhart Date: Fri, 10 Feb 2023 10:15:38 +0100 Subject: [PATCH 4/9] Better container image creation/upload workflow --- .github/workflows/container.yml | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 4711e9c..1fbf25d 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -8,7 +8,7 @@ on: types: [published] env: - IMAGE_PREFIX: kiwix/kiwix-build_ci_ + IMAGE_PREFIX: ghcr.io/kiwix/kiwix-build_ci_ jobs: Container: @@ -22,19 +22,21 @@ jobs: - name: Retrieve the code uses: actions/checkout@v3 - - name: Build & upload dev container image - env: - IMAGE_NAME: ${{ env.IMAGE_PREFIX }}${{ matrix.variant }}:dev + - name: Setup container image name + id: env run: | - docker build -t ${IMAGE_NAME} - < ${GITHUB_WORKSPACE}/ci_images/${{matrix.variant}}_builder.dockerfile - echo "${{ secrets.GHCR_TOKEN }}" | docker login -u "${{ secrets.GHCR_USERNAME }}" --password-stdin - docker push ${IMAGE_NAME} + echo "IMAGE_NAME=${{ env.IMAGE_PREFIX }}${{ matrix.variant }}" >> $GITHUB_OUTPUT - - name: Build & upload release container image - if: github.event.action == 'published' - env: - IMAGE_NAME: ${{ env.IMAGE_PREFIX }}${{ matrix.variant }}:${{ env.GITHUB_REF_NAME }} + - name: Build container image run: | - docker build -t ${IMAGE_NAME} - < ${GITHUB_WORKSPACE}/ci_images/${{matrix.variant}}_builder.dockerfile - echo "${{ secrets.GHCR_TOKEN }}" | docker login -u "${{ secrets.GHCR_USERNAME }}" --password-stdin - docker push --all-tags ${IMAGE_NAME} + docker build -t ${{ steps.env.outputs.IMAGE_NAME }}:dev - < ci_images/${{ matrix.variant }}_builder.dockerfile + + - name: Tag release container image + if: github.event.action == 'published' + run: | + docker tag ${{ steps.env.outputs.IMAGE_NAME }} ${{ env.GITHUB_REF_NAME }} + + - name: Upload container image + run: | + echo "${{ secrets.GHCR_TOKEN }}" | docker login ghcr.io -u "${{ secrets.GHCR_USERNAME }}" --password-stdin + docker push --all-tags ${{ steps.env.outputs.IMAGE_NAME }} From fed574d0c93ee848693b19a846a0ca203d268385 Mon Sep 17 00:00:00 2001 From: Emmanuel Engelhart Date: Fri, 10 Feb 2023 11:16:42 +0100 Subject: [PATCH 5/9] Update CI container image doc --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 572c4c3..3f582ff 100644 --- a/README.md +++ b/README.md @@ -179,8 +179,8 @@ Libmicrohttps, ...) but it contains dependencies installed with package manager. These container images are necessary and intended to be used by Kiwix itself in many of its CI. But they are free to download et can be reused, although they can break anytime, therefore at your won risk. -You can find them on [Docker -Hub](https://hub.docker.com/r/kiwix/kiwix-build_ci). +You can find them on +[here](https://github.com/orgs/kiwix/packages?repo_name=kiwix-build). Troubleshooting --------------- From f16f55fd07cbac93b0ce7ca247be76a0b50677ae Mon Sep 17 00:00:00 2001 From: Emmanuel Engelhart Date: Fri, 10 Feb 2023 11:47:56 +0100 Subject: [PATCH 6/9] Add container 'latest' tag --- .github/workflows/container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 1fbf25d..0649c3c 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -35,6 +35,7 @@ jobs: if: github.event.action == 'published' run: | docker tag ${{ steps.env.outputs.IMAGE_NAME }} ${{ env.GITHUB_REF_NAME }} + docker tag ${{ steps.env.outputs.IMAGE_NAME }} latest - name: Upload container image run: | From 672c6fdcb5dc609635a9ad1232d6f5730248d7d2 Mon Sep 17 00:00:00 2001 From: Emmanuel Engelhart Date: Fri, 10 Feb 2023 11:25:00 +0100 Subject: [PATCH 7/9] Update container user config --- ci_images/alpine_builder.dockerfile | 6 +++--- ci_images/bionic_builder.dockerfile | 12 ++++++------ ci_images/f35_builder.dockerfile | 6 +++--- ci_images/focal_builder.dockerfile | 6 +++--- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/ci_images/alpine_builder.dockerfile b/ci_images/alpine_builder.dockerfile index 8672a42..19c2070 100644 --- a/ci_images/alpine_builder.dockerfile +++ b/ci_images/alpine_builder.dockerfile @@ -15,9 +15,9 @@ RUN apk update -q \ gtest-dev # Create user -RUN adduser -h /home/runner -D runner +RUN addgroup --gid 121 runner +RUN adduser -u 1001 -G runner -h /home/runner -D runner USER runner -WORKDIR /home/runner ENV PATH /home/runner/.local/bin:$PATH RUN pip3 install meson ninja ; \ - ln -s /usr/bin/python3 .local/bin/python + ln -s /usr/bin/python3 /home/runner/.local/bin/python diff --git a/ci_images/bionic_builder.dockerfile b/ci_images/bionic_builder.dockerfile index cefdf30..69cf8c9 100644 --- a/ci_images/bionic_builder.dockerfile +++ b/ci_images/bionic_builder.dockerfile @@ -32,14 +32,14 @@ RUN apt update -q \ && rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* \ && pip3 install meson pytest gcovr requests distro -# Create user -RUN useradd --create-home runner -USER runner -WORKDIR /home/runner -ENV PATH /home/runner/.local/bin:$PATH - # Set qt515 environment (the equivalent of "source /opt/qt515/bin/qt515-env.sh") # RUN echo "source /opt/qt515/bin/qt515-env.sh" >> /home/runner/.bashrc ENV PATH=/opt/qt515/bin:$PATH \ LD_LIBRARY_PATH=/opt/qt515/lib/x86_64-linux-gnu:/opt/qt515/lib:$LD_LIBRARY_PATH \ PKG_CONFIG_PATH=/opt/qt515/lib/pkgconfig:$PKG_CONFIG_PATH + +# Create user +RUN groupadd --gid 121 runner +RUN useradd --uid 1001 --gid 121 --create-home runner +USER runner +ENV PATH /home/runner/.local/bin:$PATH \ No newline at end of file diff --git a/ci_images/f35_builder.dockerfile b/ci_images/f35_builder.dockerfile index bb3fc95..5dcf261 100644 --- a/ci_images/f35_builder.dockerfile +++ b/ci_images/f35_builder.dockerfile @@ -22,7 +22,7 @@ RUN dnf install -y --nodocs \ && pip3 install meson pytest requests distro # Create user -RUN useradd --create-home runner +RUN groupadd --gid 121 runner +RUN useradd --uid 1001 --gid 121 --create-home runner USER runner -WORKDIR /home/runner -ENV PATH /home/runner/.local/bin:$PATH +ENV PATH /home/runner/.local/bin:$PATH \ No newline at end of file diff --git a/ci_images/focal_builder.dockerfile b/ci_images/focal_builder.dockerfile index 3998210..fc74bd1 100644 --- a/ci_images/focal_builder.dockerfile +++ b/ci_images/focal_builder.dockerfile @@ -28,7 +28,7 @@ RUN apt update -q \ && pip3 install meson pytest gcovr requests distro # Create user -RUN useradd --create-home runner +RUN groupadd --gid 121 runner +RUN useradd --uid 1001 --gid 121 --create-home runner USER runner -WORKDIR /home/runner -ENV PATH /home/runner/.local/bin:$PATH +ENV PATH /home/runner/.local/bin:$PATH \ No newline at end of file From fb9ca9ec894e9e764f182440c77c756ac293cc5e Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 1 Mar 2023 15:39:04 +0100 Subject: [PATCH 8/9] Add missing packages in bionic container. --- ci_images/bionic_builder.dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci_images/bionic_builder.dockerfile b/ci_images/bionic_builder.dockerfile index 69cf8c9..4f09f2e 100644 --- a/ci_images/bionic_builder.dockerfile +++ b/ci_images/bionic_builder.dockerfile @@ -18,6 +18,8 @@ RUN apt update -q \ # Packaged dependencies libbz2-dev libmagic-dev uuid-dev zlib1g-dev \ libmicrohttpd-dev aria2 libgtest-dev libgl-dev \ +# Devel package to compile python modules + libxml2-dev libxslt-dev python3-dev \ # Qt packages qt515base qt515webengine qt515svg qt515imageformats qt515wayland \ # To create the appimage of kiwix-desktop @@ -42,4 +44,4 @@ ENV PATH=/opt/qt515/bin:$PATH \ RUN groupadd --gid 121 runner RUN useradd --uid 1001 --gid 121 --create-home runner USER runner -ENV PATH /home/runner/.local/bin:$PATH \ No newline at end of file +ENV PATH /home/runner/.local/bin:$PATH From 534dee9c94f43acf74971cb36d38a3ebec9f268a Mon Sep 17 00:00:00 2001 From: Emmanuel Engelhart Date: Sat, 11 Mar 2023 16:13:00 +0100 Subject: [PATCH 9/9] Use new Docker images for kiwix-build CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d94ef3..70d5688 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: SSH_KEY: /tmp/id_rsa runs-on: ubuntu-latest container: - image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-32" + image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:dev" options: "--device /dev/fuse --privileged" steps: - name: Checkout code