all: do not commit generated JS/CSS to source control (#148)

Closes #125
Closes #40

Among other things, this moves all of the asset generation to run within
the context of an npm script. Developer documentation stubs have been
added so that people can get started more easily.

The top-level Dockerfile (which is no longer used in production) has
been removed as its presence has been causing confusion. This changeset
will break it anyways.

These changes will make for less "repo churn" as the static assets are
built and rebuilt, at the cost of making the build step more complicated
for downstream packagers. If this becomes a burden, we can explore
making a "release tarball" that contains pre-massaged outputs.

.github/workflows/docker-pr.yml

@@ -20,11 +20,29 @@ jobs:
           fetch-tags: true
           fetch-depth: 0
 
-      - uses: actions/setup-go@v5
+      - name: Set up Homebrew
-        with:
+        uses: Homebrew/actions/setup-homebrew@master
-          go-version: '1.24.x'
 
-      - uses: ko-build/setup-ko@v0.8
+      - name: Setup Homebrew cellar cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            /home/linuxbrew/.linuxbrew/Cellar
+            /home/linuxbrew/.linuxbrew/bin
+            /home/linuxbrew/.linuxbrew/etc
+            /home/linuxbrew/.linuxbrew/include
+            /home/linuxbrew/.linuxbrew/lib
+            /home/linuxbrew/.linuxbrew/opt
+            /home/linuxbrew/.linuxbrew/sbin
+            /home/linuxbrew/.linuxbrew/share
+            /home/linuxbrew/.linuxbrew/var
+          key: ${{ runner.os }}-go-homebrew-cellar-${{ hashFiles('go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-homebrew-cellar-
+
+      - name: Install Brew dependencies
+        run: |
+          brew bundle
 
       - name: Docker meta
         id: meta
@@ -35,9 +53,12 @@ jobs:
       - name: Build and push
         id: build
         run: |
-          go run ./cmd/containerbuild --docker-repo ghcr.io/techarohq/anubis --slog-level debug
+          npm ci
+          npm run container
         env:
           PULL_REQUEST_ID: ${{ github.event.number }}
+          DOCKER_REPO: ghcr.io/techarohq/anubis
+          SLOG_LEVEL: debug
 
       - run: |
           echo "Test this with:"

.github/workflows/docker.yml

@@ -26,11 +26,29 @@ jobs:
           fetch-tags: true
           fetch-depth: 0
 
-      - uses: actions/setup-go@v5
+      - name: Set up Homebrew
-        with:
+        uses: Homebrew/actions/setup-homebrew@master
-          go-version: '1.24.x'
 
-      - uses: ko-build/setup-ko@v0.8
+      - name: Setup Homebrew cellar cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            /home/linuxbrew/.linuxbrew/Cellar
+            /home/linuxbrew/.linuxbrew/bin
+            /home/linuxbrew/.linuxbrew/etc
+            /home/linuxbrew/.linuxbrew/include
+            /home/linuxbrew/.linuxbrew/lib
+            /home/linuxbrew/.linuxbrew/opt
+            /home/linuxbrew/.linuxbrew/sbin
+            /home/linuxbrew/.linuxbrew/share
+            /home/linuxbrew/.linuxbrew/var
+          key: ${{ runner.os }}-go-homebrew-cellar-${{ hashFiles('go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-homebrew-cellar-
+
+      - name: Install Brew dependencies
+        run: |
+          brew bundle
 
       - name: Log into registry 
         uses: docker/login-action@v3
@@ -48,11 +66,14 @@ jobs:
       - name: Build and push
         id: build
         run: |
-          go run ./cmd/containerbuild --docker-repo ghcr.io/techarohq/anubis --slog-level debug
+          npm ci
+          npm run container
+        env:
+          DOCKER_REPO: ghcr.io/techarohq/anubis
+          SLOG_LEVEL: debug
       
       - name: Generate artifact attestation
         uses: actions/attest-build-provenance@v2
-        if: ${{github.event_name == 'pull_request'}}
         with:
           subject-name: ghcr.io/techarohq/anubis
           subject-digest: ${{ steps.build.outputs.digest }}

.github/workflows/go.yml

@@ -69,8 +69,13 @@ jobs:
         npx --yes playwright@1.50.1 install --with-deps
         npx --yes playwright@1.50.1 run-server --port 3000 &
 
+    - name: install node deps
+      run: |
+        npm ci
+        npm run assets
+
     - name: Build
       run: go build ./...
 
     - name: Test
-      run: go test -v ./...
+      run: npm run test

.gitignore

@@ -5,3 +5,5 @@
 # Go binaries and test artifacts
 main
 *.test
+
+node_modules

Brewfile

@@ -1,4 +1,7 @@
 # programming languages
 brew "go@1.24"
 brew "node"
-brew "ko"
+brew "ko"
+brew "esbuild"
+brew "zstd"
+brew "brotli"

Dockerfile

@@ -1,23 +0,0 @@
-FROM docker.io/library/golang:1.24 AS build
-ARG BUILDKIT_SBOM_SCAN_CONTEXT=true BUILDKIT_SBOM_SCAN_STAGE=true
-
-WORKDIR /app
-COPY go.mod go.sum /app/
-RUN go mod download
-
-COPY . .
-RUN --mount=type=cache,target=/root/.cache \
-  VERSION=$(git describe --tags --always --dirty) \
-  && go build -o /app/bin/anubis -ldflags="-X github.com/TecharoHQ/anubis.Version=${VERSION}" ./cmd/anubis
-
-FROM docker.io/library/debian:bookworm AS runtime
-ARG BUILDKIT_SBOM_SCAN_STAGE=true
-RUN apt-get update \
-  && apt-get -y install ca-certificates
-
-COPY --from=build /app/bin/anubis /app/bin/anubis
-
-HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 CMD ["/app/bin/anubis", "--healthcheck"]
-CMD ["/app/bin/anubis"]
-
-LABEL org.opencontainers.image.source="https://github.com/TecharoHQ/anubis"

PULL_REQUEST_TEMPLATE.md

@@ -1,6 +1,11 @@
-<!-- delete me and describe your change here -->
+<!--
+delete me and describe your change here, give enough context for a maintainer to understand what and why
+
+See https://anubis.techaro.lol/docs/developer/code-quality for more information
+-->
 
 Checklist:
 
 - [ ] Added a description of the changes to the `[Unreleased]` section of docs/docs/CHANGELOG.md
-- [ ] Tested this at least manually
+- [ ] Added test cases to [the relevant parts of the codebase](https://anubis.techaro.lol/docs/developer/code-quality)
+- [ ] Ran integration tests `npm run test:integration`

docs/docs/CHANGELOG.md

@@ -14,6 +14,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Hide the directory listings for Anubis' internal static content
 - Changed `--debug-x-real-ip-default` to `--use-remote-address`, getting the IP address from the request's socket address instead.
 - DroneBL lookups have been disabled by default
+- Static asset builds are now done on demand instead of the results being committed to source control
+- The Dockerfile has been removed as it is no longer in use
+- Developer documentation has been added to the docs site
 
 ## v1.15.0
 

docs/docs/developer/_category_.json

@@ -0,0 +1,8 @@
+{
+  "label": "Developer guides",
+  "position": 50,
+  "link": {
+    "type": "generated-index",
+    "description": "Guides and suggestions to make Anubis development go smoothly for everyone."
+  }
+}

docs/docs/developer/code-quality.md

@@ -0,0 +1,31 @@
+---
+title: Code quality guidelines
+---
+
+When submitting code to Anubis, please take the time to consider the fact that this project is security software. If things go bad, bots can pummel sites into oblivion. This is not ideal for uptime.
+
+As such, code reviews will be a bit more strict than you have seen in other projects. This is not people trying to be mean, this is a side effect of taking the problem seriously.
+
+When making code changes, try to do the following:
+
+- If you're submitting a bugfix, add a test case for it
+- If you're changing the JavaScript, make sure the integration tests pass (`npm run test:integration`)
+
+## Commit messages
+
+Anubis follows the Go project's conventions for commit messages. In general, an ideal commit message should read like this:
+
+```text
+path/to/folder: brief description of the change
+
+If the change is subtle, has implementation consequences, or is otherwise
+not entirely self-describing: take the time to spell out why. If things
+are very subtle, please also amend the documentation accordingly
+```
+
+The subject of a commit message should be the second half of the sentence "This commit changes the Anubis project to:". Here's a few examples:
+
+- `disable DroneBL by default`
+- `port the challenge to WebAssembly`
+
+The extended commit message is also your place to give rationale for a new feature. When maintainers are reviewing your code, they will use this to figure out if the burden from feature maintainership is worth the merge.

docs/docs/developer/local-dev.md

@@ -0,0 +1,57 @@
+---
+title: Local development
+---
+
+:::note
+
+TL;DR: `npm ci && npm run dev`
+
+:::
+
+Anubis requires the following tools to be installed to do local development:
+
+- [Go](https://go.dev) - the programming language that Anubis is written in
+- [esbuild](https://esbuild.github.io/) - the JavaScript bundler Anubis uses for its production JS assets
+- [Node.JS & NPM](https://nodejs.org/en) - manages some build dependencies
+- `gzip` - compresses production JS (part of coreutils)
+- `zstd` - compresses production JS
+- `brotli` - compresses production JS
+
+If you have [Homebrew](https://brew.sh) installed, you can install all the dependencies with one command:
+
+```text
+brew bundle
+```
+
+If you don't, you may need to figure out equivalents to the packages in Homebrew.
+
+## Running Anubis locally
+
+```text
+npm run dev
+```
+
+Or to do it manually:
+
+- Run `npm run assets` every time you change the CSS/JavaScript
+- `go run ./cmd/anubis` with any CLI flags you want
+
+## Building JS/CSS assets
+
+```text
+npm run assets
+```
+
+If you change the build process, make sure to update `build.sh` accordingly.
+
+## Production-ready builds
+
+```text
+npm run container
+```
+
+This builds a prod-ready container image with [ko](https://ko.build). If you want to change where the container image is pushed, you need to use environment variables:
+
+```text
+DOCKER_REPO=registry.host/org/repo DOCKER_METADATA_OUTPUT_TAGS=registry.host/org/repo:latest npm run container
+```

docs/docs/developer/signed-commits.md

@@ -0,0 +1,7 @@
+---
+title: Signed commits
+---
+
+Anubis requires developers to sign their commits. This is done so that we can have a better chain of custody from contribution to owner. For more information about commit signing, [read here](https://www.freecodecamp.org/news/what-is-commit-signing-in-git/).
+
+We do not require GPG. SSH signed commits are fine. For an overview on how to set up commit signing with your SSH key, [read here](https://dev.to/ccoveille/git-the-complete-guide-to-sign-your-commits-with-an-ssh-key-35bg).

package-lock.json

@@ -2408,4 +2408,4 @@
       }
     }
   }
-}
+}

package.json

@@ -0,0 +1,23 @@
+{
+  "name": "@techaro/anubis",
+  "version": "1.0.0-see-VERSION-file",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "npm run assets && go test ./...",
+    "test:integration": "npm run assets && go test ./internal/test",
+    "assets": "./web/build.sh && ./xess/build.sh",
+    "dev": "npm run assets && go run ./cmd/anubis",
+    "container": "npm run assets && go run ./cmd/containerbuild"
+  },
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "cssnano": "^7.0.6",
+    "cssnano-preset-advanced": "^7.0.6",
+    "postcss-cli": "^11.0.0",
+    "postcss-import": "^16.1.0",
+    "postcss-import-url": "^7.2.0",
+    "postcss-url": "^10.1.3"
+  }
+}

web/build.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+cd "$(dirname "$0")"
+
+esbuild js/main.mjs --sourcemap --bundle --minify --outfile=static/js/main.mjs
+gzip -f -k static/js/main.mjs
+zstd -f -k --ultra -22 static/js/main.mjs
+brotli -fZk static/js/main.mjs

web/embed.go

@@ -3,10 +3,6 @@ package web
 import "embed"
 
 //go:generate go tool github.com/a-h/templ/cmd/templ generate
-//go:generate esbuild js/main.mjs --sourcemap --bundle --minify --outfile=static/js/main.mjs
-//go:generate gzip -f -k static/js/main.mjs
-//go:generate zstd -f -k --ultra -22 static/js/main.mjs
-//go:generate brotli -fZk static/js/main.mjs
 
 var (
 	//go:embed static

web/static/js/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

web/static/js/main.mjs

@@ -1,2 +0,0 @@
-(()=>{function p(r,n=5,t=navigator.hardwareConcurrency||1){return console.debug("fast algo"),new Promise((e,o)=>{let s=URL.createObjectURL(new Blob(["(",y(),")()"],{type:"application/javascript"})),a=[];for(let i=0;i<t;i++){let c=new Worker(s);c.onmessage=d=>{a.forEach(u=>u.terminate()),c.terminate(),e(d.data)},c.onerror=d=>{c.terminate(),o()},c.postMessage({data:r,difficulty:n,nonce:i,threads:t}),a.push(c)}URL.revokeObjectURL(s)})}function y(){return function(){let r=t=>{let e=new TextEncoder().encode(t);return crypto.subtle.digest("SHA-256",e.buffer)};function n(t){return Array.from(t).map(e=>e.toString(16).padStart(2,"0")).join("")}addEventListener("message",async t=>{let e=t.data.data,o=t.data.difficulty,s,a=t.data.nonce,i=t.data.threads;for(;;){let c=await r(e+a),d=new Uint8Array(c),u=!0;for(let m=0;m<o;m++){let l=Math.floor(m/2),g=m%2;if((d[l]>>(g===0?4:0)&15)!==0){u=!1;break}}if(u){s=n(d),console.log(s);break}a+=i}postMessage({hash:s,data:e,difficulty:o,nonce:a})})}.toString()}function f(r,n=5,t=1){return console.debug("slow algo"),new Promise((e,o)=>{let s=URL.createObjectURL(new Blob(["(",b(),")()"],{type:"application/javascript"})),a=new Worker(s);a.onmessage=i=>{a.terminate(),e(i.data)},a.onerror=i=>{a.terminate(),o()},a.postMessage({data:r,difficulty:n}),URL.revokeObjectURL(s)})}function b(){return function(){let r=n=>{let t=new TextEncoder().encode(n);return crypto.subtle.digest("SHA-256",t.buffer).then(e=>Array.from(new Uint8Array(e)).map(o=>o.toString(16).padStart(2,"0")).join(""))};addEventListener("message",async n=>{let t=n.data.data,e=n.data.difficulty,o,s=0;do o=await r(t+s++);while(o.substring(0,e)!==Array(e+1).join("0"));s-=1,postMessage({hash:o,data:t,difficulty:e,nonce:s})})}.toString()}var L={fast:p,slow:f},w=(r="",n={})=>{let t=new URL(r,window.location.href);return Object.entries(n).forEach(e=>{let[o,s]=e;t.searchParams.set(o,s)}),t.toString()},h=(r,n)=>w(`/.within.website/x/cmd/anubis/static/img/${r}.webp`,{cacheBuster:n});(async()=>{let r=document.getElementById("status"),n=document.getElementById("image"),t=document.getElementById("title"),e=document.getElementById("spinner"),o=JSON.parse(document.getElementById("anubis_version").textContent);r.innerHTML="Calculating...";let{challenge:s,rules:a}=await fetch("/.within.website/x/cmd/anubis/api/make-challenge",{method:"POST"}).then(l=>{if(!l.ok)throw new Error("Failed to fetch config");return l.json()}).catch(l=>{throw t.innerHTML="Oh no!",r.innerHTML=`Failed to fetch config: ${l.message}`,n.src=h("sad",o),e.innerHTML="",e.style.display="none",l}),i=L[a.algorithm];if(!i){t.innerHTML="Oh no!",r.innerHTML="Failed to resolve check algorithm. You may want to reload the page.",n.src=h("sad",o),e.innerHTML="",e.style.display="none";return}r.innerHTML=`Calculating...<br/>Difficulty: ${a.report_as}`;let c=Date.now(),{hash:d,nonce:u}=await i(s,a.difficulty),m=Date.now();console.log({hash:d,nonce:u}),t.innerHTML="Success!",r.innerHTML=`Done! Took ${m-c}ms, ${u} iterations`,n.src=h("happy",o),e.innerHTML="",e.style.display="none",setTimeout(()=>{let l=window.location.href;window.location.href=w("/.within.website/x/cmd/anubis/api/pass-challenge",{response:d,nonce:u,redir:l,elapsedTime:m-c})},250)})();})();
-//# sourceMappingURL=main.mjs.map

xess/.gitignore

@@ -1 +1 @@
-node_modules
+xess.min.css

xess/build.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+cd "$(dirname "$0")"
+postcss ./xess.css -o xess.min.css

xess/package.json

@@ -1,20 +0,0 @@
-{
-  "name": "@xeserv/xess",
-  "version": "1.0.0",
-  "description": "Xe's CSS",
-  "main": "index.js",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
-    "build": "postcss xess.css -o xess.min.css"
-  },
-  "author": "",
-  "license": "ISC",
-  "devDependencies": {
-    "cssnano": "^7.0.6",
-    "cssnano-preset-advanced": "^7.0.6",
-    "postcss-cli": "^11.0.0",
-    "postcss-import": "^16.1.0",
-    "postcss-import-url": "^7.2.0",
-    "postcss-url": "^10.1.3"
-  }
-}

xess/xess.go

@@ -13,11 +13,10 @@ import (
 )
 
 //go:generate go run github.com/a-h/templ/cmd/templ@latest generate
-//go:generate npm ci
 //go:generate npm run build
 
 var (
-	//go:embed xess.min.css xess.css static
+	//go:embed *.css static
 	Static embed.FS
 
 	URL = "/.within.website/x/xess/xess.css"

yeetfile.js

@@ -4,7 +4,7 @@ go.install();
     [deb, rpm].forEach(method => method.build({
         name: "anubis",
         description: "Anubis weighs the souls of incoming HTTP requests and uses a sha256 proof-of-work challenge in order to protect upstream resources from scraper bots.",
-        homepage: "https://xeiaso.net/blog/2025/anubis",
+        homepage: "https://anubis.techaro.lol",
         license: "MIT",
         goarch,