implement packaging proof of concept with yeet (#194)

* implement packaging proof of concept with yeet Signed-off-by: Xe Iaso <me@xeiaso.net> * docs/developer: add local dev docs for yeet Signed-off-by: Xe Iaso <me@xeiaso.net> * apply review feedback Signed-off-by: Xe Iaso <me@xeiaso.net> * build package artifacts in CI Signed-off-by: Xe Iaso <me@xeiaso.net> * tell CI to fetch all git metadata Signed-off-by: Xe Iaso <me@xeiaso.net> * rename package builds job Signed-off-by: Xe Iaso <me@xeiaso.net> * upload each package individually Signed-off-by: Xe Iaso <me@xeiaso.net> * split package build CI jobs Signed-off-by: Xe Iaso <me@xeiaso.net> * fix code injection? Signed-off-by: Xe Iaso <me@xeiaso.net> * fix ci? Signed-off-by: Xe Iaso <me@xeiaso.net> * fix security alert Signed-off-by: Xe Iaso <me@xeiaso.net> * docs/local-dev: point people to yeet v1.13.3 Signed-off-by: Xe Iaso <me@xeiaso.net> --------- Signed-off-by: Xe Iaso <me@xeiaso.net>
2025-08-03 01:38:14 -04:00 · 2025-04-04 08:15:04 -04:00 · 2025-04-04 08:15:04 -04:00 · 878b37178d
commit 878b37178d
parent a230a58a1d
10 changed files with 342 additions and 17 deletions
--- a/.github/workflows/package-builds-stable.yml
+++ b/.github/workflows/package-builds-stable.yml
@ -0,0 +1,81 @@
+name: Package builds (stable)
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+  actions: write
+
+jobs:
+  package_builds:
+    #runs-on: alrest-techarohq
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
+        fetch-tags: true
+        fetch-depth: 0
+
+    - name: build essential
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y build-essential
+
+    - name: Set up Homebrew
+      uses: Homebrew/actions/setup-homebrew@master
+
+    - name: Setup Homebrew cellar cache
+      uses: actions/cache@v4
+      with:
+        path: |
+          /home/linuxbrew/.linuxbrew/Cellar
+          /home/linuxbrew/.linuxbrew/bin
+          /home/linuxbrew/.linuxbrew/etc
+          /home/linuxbrew/.linuxbrew/include
+          /home/linuxbrew/.linuxbrew/lib
+          /home/linuxbrew/.linuxbrew/opt
+          /home/linuxbrew/.linuxbrew/sbin
+          /home/linuxbrew/.linuxbrew/share
+          /home/linuxbrew/.linuxbrew/var
+        key: ${{ runner.os }}-go-homebrew-cellar-${{ hashFiles('go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-homebrew-cellar-
+
+    - name: Install Brew dependencies
+      run: |
+        brew bundle
+
+    - name: Setup Golang caches
+      uses: actions/cache@v4
+      with:
+        path: |
+          ~/.cache/go-build
+          ~/go/pkg/mod
+        key: ${{ runner.os }}-golang-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-golang-
+
+    - name: install node deps
+      run: |
+        npm ci
+
+    - name: Build Packages
+      run: |
+        wget https://github.com/Xe/x/releases/download/v1.13.3/yeet_1.13.3_amd64.deb -O var/yeet.deb
+        sudo apt -y install -f ./var/yeet.deb
+        yeet
+
+    - name: Upload released artifacts
+      env:
+        GITHUB_TOKEN: ${{ github.TOKEN }}
+        RELEASE_VERSION: ${{github.event.release.tag_name}}
+      shell: bash
+      run: |
+        RELEASE="${RELEASE_VERSION}"
+        cd var
+        for file in *; do
+          gh release upload $RELEASE $file
+        done
--- a/.github/workflows/package-builds-unstable.yml
+++ b/.github/workflows/package-builds-unstable.yml
@ -0,0 +1,76 @@
+name: Package builds (unstable)
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+permissions:
+  contents: read
+  actions: write
+
+jobs:
+  package_builds:
+    #runs-on: alrest-techarohq
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
+        fetch-tags: true
+        fetch-depth: 0
+
+    - name: build essential
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y build-essential
+
+    - name: Set up Homebrew
+      uses: Homebrew/actions/setup-homebrew@master
+
+    - name: Setup Homebrew cellar cache
+      uses: actions/cache@v4
+      with:
+        path: |
+          /home/linuxbrew/.linuxbrew/Cellar
+          /home/linuxbrew/.linuxbrew/bin
+          /home/linuxbrew/.linuxbrew/etc
+          /home/linuxbrew/.linuxbrew/include
+          /home/linuxbrew/.linuxbrew/lib
+          /home/linuxbrew/.linuxbrew/opt
+          /home/linuxbrew/.linuxbrew/sbin
+          /home/linuxbrew/.linuxbrew/share
+          /home/linuxbrew/.linuxbrew/var
+        key: ${{ runner.os }}-go-homebrew-cellar-${{ hashFiles('go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-homebrew-cellar-
+
+    - name: Install Brew dependencies
+      run: |
+        brew bundle
+
+    - name: Setup Golang caches
+      uses: actions/cache@v4
+      with:
+        path: |
+          ~/.cache/go-build
+          ~/go/pkg/mod
+        key: ${{ runner.os }}-golang-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-golang-
+
+    - name: install node deps
+      run: |
+        npm ci
+
+    - name: Build Packages
+      run: |
+        wget https://github.com/Xe/x/releases/download/v1.13.3/yeet_1.13.3_amd64.deb -O var/yeet.deb
+        sudo apt -y install -f ./var/yeet.deb
+        yeet
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: packages
+        path: var/*
--- a/docs/docs/CHANGELOG.md
+++ b/docs/docs/CHANGELOG.md
@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

 ## [Unreleased]

+- Added support for native Debian, Red Hat, and tarball packaging strategies including installation and use directions.
 - The placeholder Anubis mascot has been replaced with a design by [CELPHASE](https://bsky.app/profile/celphase.bsky.social).
 - Allow iMessage's link preview fetcher through Anubis by default.
 - Added a periodic cleanup routine for the decaymap that removes expired entries, ensuring stale data is properly pruned.
--- a/docs/docs/admin/installation.mdx
+++ b/docs/docs/admin/installation.mdx
@ -24,6 +24,8 @@ TLS terminator)

 </center>

+## Docker image conventions
+
 Anubis is shipped in the Docker repo [`ghcr.io/techarohq/anubis`](https://github.com/TecharoHQ/anubis/pkgs/container/anubis). The following tags exist for your convenience:

 | Tag                 | Meaning                                                                                                                            |
@ -31,14 +33,13 @@ Anubis is shipped in the Docker repo [`ghcr.io/techarohq/anubis`](https://github
 | `latest`            | The latest [tagged release](https://github.com/TecharoHQ/anubis/releases), if you are in doubt, start here.                        |
 | `v<version number>` | The Anubis image for [any given tagged release](https://github.com/TecharoHQ/anubis/tags)                                          |
 | `main`              | The current build on the `main` branch. Only use this if you need the latest and greatest features as they are merged into `main`. |
-| `pr-<number>`       | The build associated with PR `#<number>`. Only use this for debugging issues fixed by a PR.                                        |
-
-Other methods to install Anubis may exist, but the Docker image is currently the only supported method.

 The Docker image runs Anubis as user ID 1000 and group ID 1000. If you are mounting external volumes into Anubis' container, please be sure they are owned by or writable to this user/group.

 Anubis has very minimal system requirements. I suspect that 128Mi of ram may be sufficient for a large number of concurrent clients. Anubis may be a poor fit for apps that use WebSockets and maintain open connections, but I don't have enough real-world experience to know one way or another.

+## Environment variables
+
 Anubis uses these environment variables for configuration:

 | Environment Variable           | Default value           | Explanation                                                                                                                                                                                                                                                                              |
--- a/docs/docs/admin/native-install.mdx
+++ b/docs/docs/admin/native-install.mdx
@ -0,0 +1,131 @@
+---
+title: Installing Anubis with a native package
+---
+
+import Tabs from "@theme/Tabs";
+import TabItem from "@theme/TabItem";
+
+Install the Anubis package using your package manager of choice:
+
+<Tabs>
+  <TabItem value="deb" label="Debian-based (apt)" default>
+  
+Install Anubis with `apt`:
+
+```text
+sudo apt install ./anubis-$VERSION-$ARCH.deb
+```
+
+  </TabItem>
+  <TabItem value="tarball" label="Tarball">
+  
+Extract the tarball to a folder:
+
+```text
+tar zxf ./anubis-$VERSION-$OS-$ARCH.tar.gz
+cd anubis-$VERSION-$OS-$ARCH
+```
+
+Install the binary to your system:
+
+```text
+sudo install -D ./bin/anubis /usr/local/bin
+```
+
+Edit the systemd unit to point to `/usr/local/bin/anubis` instead of `/usr/bin/anubis`:
+
+```text
+perl -pi -e 's$/usr/bin/anubis$/usr/local/bin/anubis$g' ./run/anubis@.service
+```
+
+Install the systemd unit to your system:
+
+```text
+sudo install -D ./run/anubis@.service /etc/systemd/system
+```
+
+Install the default configuration file to your system:
+
+```text
+sudo install -D ./run/default.env /etc/anubis
+```
+
+  </TabItem>
+  <TabItem value="rpm" label="Red Hat-based (rpm)">
+  
+Install Anubis with `dnf`:
+
+```text
+sudo dnf -y install ./anubis-$VERSION.$ARCH.rpm
+```
+
+OR
+
+Install Anubis with `yum`:
+
+```text
+sudo yum -y install ./anubis-$VERSION.$ARCH.rpm
+```
+
+OR
+
+Install Anubis with `rpm`:
+
+```
+sudo rpm -ivh ./anubis-$VERSION.$ARCH.rpm
+```
+
+  </TabItem>
+</Tabs>
+
+Once it's installed, make a copy of the default configuration file `/etc/anubis/default.env` based on which service you want to protect. For example, to protect a `gitea` server:
+
+```text
+sudo cp /etc/anubis/default.env /etc/anubis/gitea.env
+```
+
+Copy the default bot policies file to `/etc/anubis/gitea.botPolicies.json`:
+
+<Tabs>
+<TabItem value="debrpm" label="Debian or Red Hat" default>
+
+```text
+sudo cp /usr/share/doc/anubis/botPolicies.json /etc/anubis/gitea.botPolicies.json
+```
+
+</TabItem>
+<TabItem value="tarball" label="Tarball">
+
+```text
+sudo cp ./doc/botPolicies.json /etc/anubis/gitea.botPolicies.json
+```
+
+</TabItem>
+
+</Tabs>
+
+Then open `gitea.env` in your favorite text editor and customize [the environment variables](./installation.mdx#environment-variables) as needed. Here's an example configuration for a Gitea server:
+
+```sh
+BIND=[::1]:8239
+BIND_NETWORK=tcp
+DIFFICULTY=4
+METRICS_BIND=[::1]:8240
+METRICS_BIND_NETWORK=tcp
+POLICY_FNAME=/etc/anubis/gitea.botPolicies.json
+TARGET=http://localhost:3000
+```
+
+Then start Anubis with `systemctl enable --now`:
+
+```text
+sudo systemctl enable --now anubis@gitea.service
+```
+
+Test to make sure it's running with `curl`:
+
+```text
+curl http://localhost:8240/metrics
+```
+
+Then set up your reverse proxy (Nginx, Caddy, etc.) to point to the Anubis port. Anubis will then reverse proxy all requests that meet the policies in `/etc/anubis/gitea.botPolicies.json` to the target service.
--- a/docs/docs/developer/local-dev.md
+++ b/docs/docs/developer/local-dev.md
@ -55,3 +55,32 @@ This builds a prod-ready container image with [ko](https://ko.build). If you wan
 ```text
 DOCKER_REPO=registry.host/org/repo DOCKER_METADATA_OUTPUT_TAGS=registry.host/org/repo:latest npm run container
 ```
+
+## Building packages
+
+For more information, see [Building native packages is complicated](https://xeiaso.net/blog/2025/anubis-packaging/) and [#156: Debian, RPM, and binary tarball packages](https://github.com/TecharoHQ/anubis/issues/156).
+
+Install `yeet`:
+
+:::note
+
+`yeet` will soon be moved to a dedicated TecharoHQ repository. This is currently done in a hacky way in order to get this ready for user feedback.
+
+:::
+
+```text
+go install within.website/x/cmd/yeet@v1.13.3
+```
+
+Install the dependencies for Anubis:
+
+```text
+npm ci
+go mod download
+```
+
+Build the packages into `./var`:
+
+```text
+yeet
+```
--- a/package.json
+++ b/package.json
@ -9,7 +9,8 @@
    "assets": "go generate ./... && ./web/build.sh && ./xess/build.sh",
    "build": "npm run assets && go build -o ./var/anubis ../cmd/anubis",
    "dev": "npm run assets && go run ./cmd/anubis --use-remote-address",
-    "container": "npm run assets && go run ./cmd/containerbuild"
+    "container": "npm run assets && go run ./cmd/containerbuild",
+    "package": "yeet"
  },
  "author": "",
  "license": "ISC",
--- a/run/anubis@.service
+++ b/run/anubis@.service
@ -5,8 +5,14 @@ Description="Anubis HTTP defense proxy (instance %i)"
 ExecStart=/usr/bin/anubis
 Restart=always
 RestartSec=30s
-EnvironmentFile=/etc/anubis/anubis-%i.env
+EnvironmentFile=/etc/anubis/%i.env
 LimitNOFILE=infinity
+DynamicUser=yes
+CacheDirectory=anubis/%i
+CacheDirectoryMode=0755
+StateDirectory=anubis/%i
+StateDirectoryMode=0755
+ReadWritePaths=/run

 [Install]
 WantedBy=multi-user.target
--- a/run/anubis.env.default
+++ b/run/anubis.env.default
@ -1,5 +1,5 @@
 BIND=:8923
-DIFFICULTY=3
+DIFFICULTY=4
 METRICS_BIND=:9090
 SERVE_ROBOTS_TXT=0
 TARGET=http://localhost:3000
--- a/yeetfile.js
+++ b/yeetfile.js
@ -1,5 +1,7 @@
+$`npm run assets`;
+
 ["amd64", "arm64", "riscv64"].forEach(goarch => {
-    [deb, rpm].forEach(method => method.build({
+    [deb, rpm, tarball].forEach(method => method.build({
        name: "anubis",
        description: "Anubis weighs the souls of incoming HTTP requests and uses a sha256 proof-of-work challenge in order to protect upstream resources from scraper bots.",
        homepage: "https://anubis.techaro.lol",
@ -10,19 +12,16 @@
            "./README.md": "README.md",
            "./LICENSE": "LICENSE",
            "./docs/docs/CHANGELOG.md": "CHANGELOG.md",
+            "./docs/docs/admin/policies.md": "policies.md",
+            "./docs/docs/admin/native-install.mdx": "native-install.mdx",
+            "./data/botPolicies.json": "botPolicies.json",
        },

-        build: (out) => {
-            // install Anubis binary
-            go.build("-o", `${out}/usr/bin/anubis`, "./cmd/anubis");
+        build: ({ bin, etc, systemd, out }) => {
+            $`go build -o ${bin}/anubis -ldflags '-s -w -extldflags "-static" -X "github.com/TecharoHQ/anubis.Version=${git.tag()}"' ./cmd/anubis`;

-            // install systemd unit
-            yeet.run("mkdir", "-p", `${out}/usr/lib/systemd/system`);
-            yeet.run("cp", "run/anubis@.service", `${out}/usr/lib/systemd/system/anubis@.service`);
-
-            // install default config
-            yeet.run("mkdir", "-p", `${out}/etc/anubis`);
-            yeet.run("cp", "run/anubis.env.default", `${out}/etc/anubis/anubis-default.env`);
+            file.install("./run/anubis@.service", `${systemd}/anubis@.service`);
+            file.install("./run/default.env", `${etc}/default.env`);
        },
    }));
 });