From 7ca45f72e3a843ce107b699408b5af32781127da Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Fri, 18 Jul 2025 22:02:34 -0400 Subject: [PATCH 1/9] test: adapt caddy testing environment into a smoke test Signed-off-by: Xe Iaso --- test/anubis_configs/less_paranoid.yaml | 10 +++++ test/caddy/docker-compose.yaml | 8 ++-- test/caddy/start.sh | 22 ----------- test/caddy/test.mjs | 27 +++++++++++++ test/caddy/test.sh | 13 +++++++ test/lib/lib.sh | 54 ++++++++++++++++++++++++++ 6 files changed, 108 insertions(+), 26 deletions(-) create mode 100644 test/anubis_configs/less_paranoid.yaml delete mode 100644 test/caddy/start.sh create mode 100644 test/caddy/test.mjs create mode 100755 test/caddy/test.sh create mode 100644 test/lib/lib.sh diff --git a/test/anubis_configs/less_paranoid.yaml b/test/anubis_configs/less_paranoid.yaml new file mode 100644 index 0000000..f3679f2 --- /dev/null +++ b/test/anubis_configs/less_paranoid.yaml @@ -0,0 +1,10 @@ +bots: + - name: challenge + user_agent_regex: Mozilla + action: WEIGH + weight: + adjust: 10 + +status_codes: + CHALLENGE: 401 + DENY: 403 diff --git a/test/caddy/docker-compose.yaml b/test/caddy/docker-compose.yaml index 73ccce5..a7456ff 100644 --- a/test/caddy/docker-compose.yaml +++ b/test/caddy/docker-compose.yaml @@ -9,14 +9,14 @@ services: - "../pki/caddy.local.cetacean.club:/etc/techaro/pki/caddy.local.cetacean.club/" anubis: - image: ghcr.io/techarohq/anubis:main + image: ko.local/anubis environment: BIND: ":3000" TARGET: http://httpdebug:3000 - POLICY_FNAME: /etc/techaro/anubis/less_paranoid.yaml + POLICY_FNAME: /cfg/less_paranoid.yaml + SLOG_LEVEL: DEBUG volumes: - - ../anubis_configs:/etc/techaro/anubis + - ../anubis_configs:/cfg httpdebug: image: ghcr.io/xe/x/httpdebug - pull_policy: always diff --git a/test/caddy/start.sh b/test/caddy/start.sh deleted file mode 100644 index 7e617e2..0000000 --- a/test/caddy/start.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -# If the transient local TLS certificate doesn't exist, mint a new one -if [ ! -f ../pki/caddy.local.cetacean.club/cert.pem ]; then - # Subshell to contain the directory change - ( - cd ../pki \ - && mkdir -p caddy.local.cetacean.club \ - && \ - # Try using https://github.com/FiloSottile/mkcert for better DevEx, - # but fall back to using https://github.com/jsha/minica in case - # you don't have that installed. - ( - mkcert \ - --cert-file ./caddy.local.cetacean.club/cert.pem \ - --key-file ./caddy.local.cetacean.club/key.pem caddy.local.cetacean.club \ - || go tool minica -domains caddy.local.cetacean.club - ) - ) -fi - -docker compose up --build \ No newline at end of file diff --git a/test/caddy/test.mjs b/test/caddy/test.mjs new file mode 100644 index 0000000..1e53a23 --- /dev/null +++ b/test/caddy/test.mjs @@ -0,0 +1,27 @@ +async function testWithUserAgent(userAgent) { + const statusCode = + await fetch("https://relayd.local.cetacean.club:8443/reqmeta", { + headers: { + "User-Agent": userAgent, + } + }) + .then(resp => resp.status); + return statusCode; +} + +const codes = { + Mozilla: await testWithUserAgent("Mozilla"), + curl: await testWithUserAgent("curl"), +} + +const expected = { + Mozilla: 401, + curl: 200, +}; + +console.log("Mozilla:", codes.Mozilla); +console.log("curl: ", codes.curl); + +if (JSON.stringify(codes) !== JSON.stringify(expected)) { + throw new Error(`wanted ${JSON.stringify(expected)}, got: ${JSON.stringify(codes)}`); +} \ No newline at end of file diff --git a/test/caddy/test.sh b/test/caddy/test.sh new file mode 100755 index 0000000..489418c --- /dev/null +++ b/test/caddy/test.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source ../lib/lib.sh + +build_anubis_ko + +docker compose up -d --build + +export NODE_TLS_REJECT_UNAUTHORIZED=0 + +backoff-retry node test.mjs diff --git a/test/lib/lib.sh b/test/lib/lib.sh new file mode 100644 index 0000000..37dcc89 --- /dev/null +++ b/test/lib/lib.sh @@ -0,0 +1,54 @@ +REPO_ROOT=$(git rev-parse --show-toplevel) +(cd $REPO_ROOT && go install ./utils/cmd/...) + +function cleanup() { + pkill -P $$ + + if [ -f "docker-compose.yaml" ]; then + docker compose down + fi +} + +trap cleanup EXIT SIGINT + +function build_anubis_ko() { + ( + cd ../.. && + VERSION=devel ko build \ + --platform=all \ + --base-import-paths \ + --tags="latest" \ + --image-user=1000 \ + --image-annotation="" \ + --image-label="" \ + ./cmd/anubis \ + -L + ) +} + +function mint_cert() { + if [ "$#" -ne 1 ]; then + echo "Usage: mint_cert " + fi + + domainName="$1" + + # If the transient local TLS certificate doesn't exist, mint a new one + if [ ! -f "../pki/${domainName}/cert.pem" ]; then + # Subshell to contain the directory change + ( + cd ../pki && + mkdir -p "${domainName}" && + # Try using https://github.com/FiloSottile/mkcert for better DevEx, + # but fall back to using https://github.com/jsha/minica in case + # you don't have that installed. + ( + mkcert \ + --cert-file ./"${domainName}"/cert.pem \ + --key-file ./"${domainName}"/key.pem \ + "${domainName}" || + go tool minica -domains "${domainName}" + ) + ) + fi +} From eacc001d69c7417bf2dc34fa1fd5a3e861a8e722 Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Fri, 18 Jul 2025 22:03:07 -0400 Subject: [PATCH 2/9] fix(cmd/anubis): ignore errors for unix socket chmod Signed-off-by: Xe Iaso --- cmd/anubis/main.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/anubis/main.go b/cmd/anubis/main.go index 9eb2f54..ab550ad 100644 --- a/cmd/anubis/main.go +++ b/cmd/anubis/main.go @@ -166,19 +166,19 @@ func setupListener(network string, address string) (net.Listener, string) { // additional permission handling for unix sockets if network == "unix" { + slog.Debug("parsing socket mode", "mode_string", *socketMode, "address", address) + mode, err := strconv.ParseUint(*socketMode, 8, 0) if err != nil { listener.Close() - log.Fatal(fmt.Errorf("could not parse socket mode %s: %w", *socketMode, err)) + slog.Error("could not parse socket mode", "mode", *socketMode, "err", err) + os.Exit(1) } - err = os.Chmod(address, os.FileMode(mode)) - if err != nil { - err := listener.Close() - if err != nil { - log.Printf("failed to close listener: %v", err) - } - log.Fatal(fmt.Errorf("could not change socket mode: %w", err)) + if err := os.Chmod(address, os.FileMode(mode)); err != nil { + // Ignore chmod errors on Unix domain sockets - this is expected behavior + // on many systems/containers where socket permissions cannot be changed + slog.Debug("chmod failed on socket (ignoring)", "path", address, "err", err) } } From 7cfefcd882b252fd67698952aec34c5e6ca204d2 Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Fri, 18 Jul 2025 22:06:00 -0400 Subject: [PATCH 3/9] test: make unix-socket-xff into a smoke test Signed-off-by: Xe Iaso --- test/unix-socket-xff/start.sh | 55 ----------------------------------- test/unix-socket-xff/test.sh | 33 +++++++++++++++++++++ 2 files changed, 33 insertions(+), 55 deletions(-) delete mode 100755 test/unix-socket-xff/start.sh create mode 100755 test/unix-socket-xff/test.sh diff --git a/test/unix-socket-xff/start.sh b/test/unix-socket-xff/start.sh deleted file mode 100755 index 840d768..0000000 --- a/test/unix-socket-xff/start.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -# Remove lingering .sock files, relayd and unixhttpd will do that too but -# measure twice, cut once. -rm *.sock ||: - -# If the transient local TLS certificate doesn't exist, mint a new one -if [ ! -f ../pki/relayd.local.cetacean.club/cert.pem ]; then - # Subshell to contain the directory change - ( - cd ../pki \ - && mkdir -p relayd.local.cetacean.club \ - && \ - # Try using https://github.com/FiloSottile/mkcert for better DevEx, - # but fall back to using https://github.com/jsha/minica in case - # you don't have that installed. - ( - mkcert \ - --cert-file ./relayd.local.cetacean.club/cert.pem \ - --key-file ./relayd.local.cetacean.club/key.pem relayd.local.cetacean.club \ - || go tool minica -domains relayd.local.cetacean.club - ) - ) -fi - -# Build static assets -(cd ../.. && npm ci && npm run assets) - -# Spawn three jobs: - -# HTTP daemon that listens over a unix socket (implicitly ./unixhttpd.sock) -go run ../cmd/unixhttpd & - -# A copy of Anubis, specifically for the current Git checkout -go tool anubis \ - --bind=./anubis.sock \ - --bind-network=unix \ - --policy-fname=../anubis_configs/aggressive_403.yaml \ - --target=unix://$(pwd)/unixhttpd.sock & - -# A simple TLS terminator that forwards to Anubis, which will forward to -# unixhttpd -go run ../cmd/relayd \ - --proxy-to=unix://./anubis.sock \ - --cert-dir=../pki/relayd.local.cetacean.club & - -# When you press control c, kill all the child processes to clean things up -trap 'echo signal received!; kill $(jobs -p); wait' SIGINT SIGTERM - -echo "open https://relayd.local.cetacean.club:3004/reqmeta" - -# Wait for all child processes to exit -wait diff --git a/test/unix-socket-xff/test.sh b/test/unix-socket-xff/test.sh new file mode 100755 index 0000000..9d1e7a5 --- /dev/null +++ b/test/unix-socket-xff/test.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source ../lib/lib.sh + +mint_cert "relayd.local.cetacean.club" + +# Build static assets +(cd ../.. && npm ci && npm run assets) + +# Spawn three jobs: + +# HTTP daemon that listens over a unix socket (implicitly ./unixhttpd.sock) +go run ../cmd/unixhttpd & + +# A copy of Anubis, specifically for the current Git checkout +go tool anubis \ + --bind=./anubis.sock \ + --bind-network=unix \ + --socket-mode=0700 \ + --policy-fname=../anubis_configs/aggressive_403.yaml \ + --target=unix://$(pwd)/unixhttpd.sock & + +# A simple TLS terminator that forwards to Anubis, which will forward to +# unixhttpd +go run ../cmd/relayd \ + --proxy-to=unix://./anubis.sock \ + --cert-dir=../pki/relayd.local.cetacean.club & + +export NODE_TLS_REJECT_UNAUTHORIZED=0 + +backoff-retry node test.mjs From 950ec24cf111af9a68ccc5dc6be7b8b05879eda3 Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Fri, 18 Jul 2025 22:07:06 -0400 Subject: [PATCH 4/9] ci: enable caddy and unix-socket-xff tests Signed-off-by: Xe Iaso --- .github/workflows/smoke-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/smoke-tests.yml b/.github/workflows/smoke-tests.yml index c598a2a..01cfe9c 100644 --- a/.github/workflows/smoke-tests.yml +++ b/.github/workflows/smoke-tests.yml @@ -14,10 +14,12 @@ jobs: strategy: matrix: test: + - caddy - git-clone - git-push - healthcheck - i18n + - unix-socket-xff runs-on: ubuntu-24.04 steps: - name: Checkout code From 06122fcd899aed77533693f55286cd4109e076c0 Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Fri, 18 Jul 2025 22:07:22 -0400 Subject: [PATCH 5/9] chore(devcontainer): put node_modules in a volume Signed-off-by: Xe Iaso --- .devcontainer/docker-compose.yaml | 4 ++++ .devcontainer/poststart.sh | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml index d2fbc3b..5b7272d 100644 --- a/.devcontainer/docker-compose.yaml +++ b/.devcontainer/docker-compose.yaml @@ -20,7 +20,11 @@ services: dockerfile: .devcontainer/Dockerfile volumes: - ../:/workspace/anubis:cached + - node_modules:/workspace/anubis/node_modules environment: VALKEY_URL: redis://valkey:6379/0 #entrypoint: ["/usr/bin/sleep", "infinity"] user: vscode + +volumes: + node_modules: diff --git a/.devcontainer/poststart.sh b/.devcontainer/poststart.sh index 93ed3a9..f3af4cf 100644 --- a/.devcontainer/poststart.sh +++ b/.devcontainer/poststart.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash -pwd +sudo chown -R vscode:vscode ./node_modules npm ci & go mod download & go install ./utils/cmd/... & +go install mvdan.cc/sh/v3/cmd/shfmt@latest & wait From a47e781d91da3fd0bb587a80d36c712e4cf570d2 Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Sat, 19 Jul 2025 04:44:34 +0000 Subject: [PATCH 6/9] test(caddy): fix automated testing Signed-off-by: Xe Iaso --- .devcontainer/devcontainer.json | 3 ++- .vscode/extensions.json | 3 ++- test/caddy/Caddyfile | 11 ++--------- test/caddy/docker-compose.yaml | 2 -- test/caddy/test.mjs | 2 +- test/caddy/test.sh | 2 ++ test/lib/lib.sh | 2 +- 7 files changed, 10 insertions(+), 15 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b11a762..983ae6e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -21,7 +21,8 @@ "golang.go", "unifiedjs.vscode-mdx", "a-h.templ", - "redhat.vscode-yaml" + "redhat.vscode-yaml", + "matthewpi.caddyfile-support" ] } } diff --git a/.vscode/extensions.json b/.vscode/extensions.json index c85abc8..4d213bb 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -5,6 +5,7 @@ "golang.go", "unifiedjs.vscode-mdx", "a-h.templ", - "redhat.vscode-yaml" + "redhat.vscode-yaml", + "matthewpi.caddyfile-support" ] } \ No newline at end of file diff --git a/test/caddy/Caddyfile b/test/caddy/Caddyfile index a80a93a..c99999e 100644 --- a/test/caddy/Caddyfile +++ b/test/caddy/Caddyfile @@ -1,12 +1,5 @@ -:80 { - reverse_proxy http://anubis:3000 { - header_up X-Real-Ip {remote_host} - header_up X-Http-Version {http.request.proto} - } -} - -:443 { - tls /etc/techaro/pki/caddy.local.cetacean.club/cert.pem /etc/techaro/pki/caddy.local.cetacean.club/key.pem +caddy.local.cetacean.club { + tls internal reverse_proxy http://anubis:3000 { header_up X-Real-Ip {remote_host} diff --git a/test/caddy/docker-compose.yaml b/test/caddy/docker-compose.yaml index a7456ff..65fbe4c 100644 --- a/test/caddy/docker-compose.yaml +++ b/test/caddy/docker-compose.yaml @@ -5,8 +5,6 @@ services: ports: - 8080:80 - 8443:443 - volumes: - - "../pki/caddy.local.cetacean.club:/etc/techaro/pki/caddy.local.cetacean.club/" anubis: image: ko.local/anubis diff --git a/test/caddy/test.mjs b/test/caddy/test.mjs index 1e53a23..c1ec537 100644 --- a/test/caddy/test.mjs +++ b/test/caddy/test.mjs @@ -1,6 +1,6 @@ async function testWithUserAgent(userAgent) { const statusCode = - await fetch("https://relayd.local.cetacean.club:8443/reqmeta", { + await fetch("https://caddy.local.cetacean.club:8443/reqmeta", { headers: { "User-Agent": userAgent, } diff --git a/test/caddy/test.sh b/test/caddy/test.sh index 489418c..c6d8602 100755 --- a/test/caddy/test.sh +++ b/test/caddy/test.sh @@ -10,4 +10,6 @@ docker compose up -d --build export NODE_TLS_REJECT_UNAUTHORIZED=0 +sleep 2 + backoff-retry node test.mjs diff --git a/test/lib/lib.sh b/test/lib/lib.sh index 37dcc89..4838d9d 100644 --- a/test/lib/lib.sh +++ b/test/lib/lib.sh @@ -14,7 +14,7 @@ trap cleanup EXIT SIGINT function build_anubis_ko() { ( cd ../.. && - VERSION=devel ko build \ + KO_DOCKER_REPO=ko.local/anubis VERSION=devel ko build \ --platform=all \ --base-import-paths \ --tags="latest" \ From cf510eeace7aa26a6caa1c5b2bb45a2aa4c4c1eb Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Sat, 19 Jul 2025 00:46:23 -0400 Subject: [PATCH 7/9] chore: spelling Signed-off-by: Xe Iaso --- .github/actions/spelling/expect.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index c67e2d9..ad4a77f 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -32,7 +32,7 @@ byteslice Bytespider cachebuster cachediptoasn -Caddyfile +caddyfile caninetools Cardyb celchecker @@ -181,6 +181,7 @@ lol lominsa maintainership malware +matthewpi mcr memes metarefresh From b320af79ee5d302716bac6ccb31ff08e00975557 Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Sat, 19 Jul 2025 04:48:43 +0000 Subject: [PATCH 8/9] test(caddy): fix automated testing? Signed-off-by: Xe Iaso --- test/caddy/docker-compose.yaml | 2 +- test/lib/lib.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/caddy/docker-compose.yaml b/test/caddy/docker-compose.yaml index 65fbe4c..4fa758d 100644 --- a/test/caddy/docker-compose.yaml +++ b/test/caddy/docker-compose.yaml @@ -7,7 +7,7 @@ services: - 8443:443 anubis: - image: ko.local/anubis + image: ghcr.io/techarohq/anubis environment: BIND: ":3000" TARGET: http://httpdebug:3000 diff --git a/test/lib/lib.sh b/test/lib/lib.sh index 4838d9d..24ef968 100644 --- a/test/lib/lib.sh +++ b/test/lib/lib.sh @@ -14,7 +14,7 @@ trap cleanup EXIT SIGINT function build_anubis_ko() { ( cd ../.. && - KO_DOCKER_REPO=ko.local/anubis VERSION=devel ko build \ + VERSION=devel ko build \ --platform=all \ --base-import-paths \ --tags="latest" \ @@ -22,7 +22,7 @@ function build_anubis_ko() { --image-annotation="" \ --image-label="" \ ./cmd/anubis \ - -L + --local ) } From b12c5945b3e702d25491ce1cc6597724e634b67a Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Sat, 19 Jul 2025 04:51:12 +0000 Subject: [PATCH 9/9] test(caddy): why are you failing? Signed-off-by: Xe Iaso --- test/caddy/test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/caddy/test.sh b/test/caddy/test.sh index c6d8602..354160d 100755 --- a/test/caddy/test.sh +++ b/test/caddy/test.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -x + set -euo pipefail source ../lib/lib.sh