test: introduce SSH based CI for non-native test hosts (#644)

* feat: ssh based CI

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test: implement SSH ci with caches and github actions

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): fix known hosts secret

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): clone the repo, that's important

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: spelling

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): speed up ci by prebaking the SSH CI image

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): set -euo

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): enable pull_request_target so things work

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: spelling

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): oh goody it's broken

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): add cronjob to rebuild ci runner image

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): also run yeet

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): force git version for yeet

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): run set -x in the container

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): fix yeet?

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): remove yeet for now

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): disable for PRs for now

Signed-off-by: Xe Iaso <me@xeiaso.net>

---------

Signed-off-by: Xe Iaso <me@xeiaso.net>

.github/actions/spelling/expect.txt

@@ -6,6 +6,7 @@ amazonbot
 anthro
 anubis
 anubistest
+apk
 Applebot
 archlinux
 badregexes
@@ -68,6 +69,7 @@ duckduckbot
 eerror
 ellenjoe
 enbyware
+euo
 everyones
 evilbot
 evilsite
@@ -117,6 +119,7 @@ imgproxy
 inp
 iss
 isset
+itv
 ivh
 Jenomis
 JGit
@@ -246,6 +249,7 @@ traefik
 uberspace
 unixhttpd
 unmarshal
+uuidgen
 uvx
 UXP
 Varis

.github/workflows/docker.yml

@@ -3,8 +3,8 @@ name: Docker image builds
 on:
   workflow_dispatch:
   push:
-    branches: [ "main" ]
+    branches: ["main"]
-    tags: [ "v*" ]
+    tags: ["v*"]
 
 env:
   DOCKER_METADATA_SET_OUTPUT_ENV: "true"
@@ -55,7 +55,7 @@ jobs:
         run: |
           brew bundle
 
-      - name: Log into registry 
+      - name: Log into registry
         uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           registry: ghcr.io
@@ -77,7 +77,6 @@ jobs:
           DOCKER_REPO: ${{ env.IMAGE }}
           SLOG_LEVEL: debug
 
-      
       - name: Generate artifact attestation
         uses: actions/attest-build-provenance@db473fddc028af60658334401dc6fa3ffd8669fd # v2.3.0
         with:

.github/workflows/ssh-ci-runner-cron.yml

@@ -0,0 +1,36 @@
+name: Regenerate ssh ci runner image
+
+on:
+  # pull_request:
+  #   branches: ["main"]
+  schedule:
+    - cron: "0 0 1,8,15,22 * *"
+  workflow_dispatch:
+
+permissions:
+  pull-requests: write
+  contents: write
+  packages: write
+
+jobs:
+  ssh-ci-rebuild:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-tags: true
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Log into registry
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
+      - name: Build and push
+        run: |
+          cd ./test/ssh-ci
+          docker buildx bake --push

.github/workflows/ssh-ci.yml

@@ -0,0 +1,36 @@
+name: SSH CI
+
+on:
+  push:
+    branches: ["main"]
+  # pull_request:
+  #   branches: ["main"]
+
+permissions:
+  contents: read
+
+jobs:
+  ssh:
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        host:
+          - ubuntu@riscv64.techaro.lol
+          - ci@ppc64le.techaro.lol
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-tags: true
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Install CI target SSH key
+        uses: shimataro/ssh-key-action@d4fffb50872869abe2d9a9098a6d9c5aa7d16be4 # v2.7.0
+        with:
+          key: ${{ secrets.CI_SSH_KEY }}
+          name: id_rsa
+          known_hosts: ${{ secrets.CI_SSH_KNOWN_HOSTS }}
+      - name: Run CI
+        run: bash test/ssh-ci/rigging.sh ${{ matrix.host }}
+        env:
+          GITHUB_RUN_ID: ${{ github.run_id }}

test/ssh-ci/Dockerfile

@@ -0,0 +1,5 @@
+ARG ALPINE_VERSION=3.22
+
+FROM alpine:${ALPINE_VERSION}
+RUN apk add -U go nodejs git build-base git npm bash zstd brotli gzip
+LABEL org.opencontainers.image.source="https://github.com/TecharoHQ/anubis"

test/ssh-ci/docker-bake.hcl

@@ -0,0 +1,26 @@
+variable "ALPINE_VERSION" { default = "3.22" }
+
+group "default" {
+  targets = [
+    "ci-runner",
+  ]
+}
+
+target "ci-runner" {
+  args = {
+    ALPINE_VERSION = "3.22"
+  }
+  context = "."
+  dockerfile = "./Dockerfile"
+  platforms = [
+    "linux/amd64",
+    "linux/arm64",
+    "linux/arm/v7",
+    "linux/ppc64le",
+    "linux/riscv64",
+  ]
+  pull = true
+  tags = [
+    "ghcr.io/techarohq/anubis/ci-runner:latest"
+  ]
+}

test/ssh-ci/in-container.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env sh
+
+set -euo pipefail
+set -x
+
+npm ci
+npm run build
+SKIP_INTEGRATION=1 go test ./...

test/ssh-ci/rigging.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+[ ! -z "${DEBUG:-}" ] && set -x
+
+if [ "$#" -ne 1 ]; then
+    echo "Usage: rigging.sh <user@host>"
+fi
+
+CIRunnerImage="ghcr.io/techarohq/anubis/ci-runner:latest"
+RunID=${GITHUB_RUN_ID:-$(uuidgen)}
+RunFolder="anubis/runs/${RunID}"
+Target="${1}"
+
+ssh "${Target}" uname -av
+ssh "${Target}" mkdir -p "${RunFolder}"
+git archive HEAD | ssh "${Target}" tar xC "${RunFolder}"
+
+ssh "${Target}" << EOF
+  set -euo pipefail
+  set -x
+  mkdir -p "anubis/cache/{go,go-build,node}"
+  podman pull ${CIRunnerImage}
+  podman run --rm -it \
+    -v "\$HOME/${RunFolder}:/app/anubis" \
+    -v "\$HOME/anubis/cache/go:/root/go" \
+    -v "\$HOME/anubis/cache/go-build:/root/.cache/go-build" \
+    -v "\$HOME/anubis/cache/node:/root/.npm" \
+    -w /app/anubis \
+    ${CIRunnerImage} \
+    sh /app/anubis/test/ssh-ci/in-container.sh
+  ssh "${Target}" rm -rf "${RunFolder}"
+EOF