diff --git a/.github/actions/spelling/excludes.txt b/.github/actions/spelling/excludes.txt index c96c86a..388c1ea 100644 --- a/.github/actions/spelling/excludes.txt +++ b/.github/actions/spelling/excludes.txt @@ -83,6 +83,7 @@ ^\Q.github/FUNDING.yml\E$ ^\Q.github/workflows/spelling.yml\E$ ^data/crawlers/ +^docs/manifest/.*$ ^docs/static/\.nojekyll$ ignore$ -robots.txt +robots.txt \ No newline at end of file diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 92d584c..00e5fd8 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -9,10 +9,13 @@ anubistest apk Applebot archlinux +asnc +asnchecker +asns +aspirational badregexes bdba berr -betteralign bingbot bitcoin blogging @@ -25,6 +28,7 @@ Brightbot broked Bytespider cachebuster +cachediptoasn Caddyfile caninetools Cardyb @@ -89,9 +93,14 @@ Fordola forgejo fsys fullchain +gaissmai Galvus +geoip +geoipchecker gha +gipc gitea +godotenv goland gomod goodbot @@ -101,6 +110,7 @@ goyaml GPG GPT gptbot +grpcprom grw Hashcash hashrate @@ -113,6 +123,7 @@ hostable htmlc htmx httpdebug +Huawei hypertext iaskspider iat @@ -120,11 +131,14 @@ ifm Imagesift imgproxy inp +IPTo +iptoasn iss isset ivh Jenomis JGit +joho journalctl jshelter JWTs @@ -164,7 +178,6 @@ mojeekbot mozilla nbf netsurf -NFlag nginx nobots NONINFRINGEMENT @@ -241,11 +254,14 @@ subrequest SVCNAME tagline tarballs +tarrif techaro techarohq templ templruntime testarea +thoth +thothmock Tik Timpibot torproject @@ -270,6 +286,7 @@ websecure websites Webzio wildbase +withthothmock wordpress Workaround workdir diff --git a/.vscode/settings.json b/.vscode/settings.json index 7a4f095..bb035f9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -11,5 +11,24 @@ "zig": false, "javascript": false, "properties": false + }, + "[markdown]": { + "editor.wordWrap": "wordWrapColumn", + "editor.wordWrapColumn": 80, + "editor.wordBasedSuggestions": "off" + }, + "[mdx]": { + "editor.wordWrap": "wordWrapColumn", + "editor.wordWrapColumn": 80, + "editor.wordBasedSuggestions": "off" + }, + "[nunjucks]": { + "editor.wordWrap": "wordWrapColumn", + "editor.wordWrapColumn": 80, + "editor.wordBasedSuggestions": "off" + }, + "cSpell.enabledFileTypes": { + "mdx": true, + "md": true } } diff --git a/cmd/anubis/main.go b/cmd/anubis/main.go index d67f414..bc21473 100644 --- a/cmd/anubis/main.go +++ b/cmd/anubis/main.go @@ -30,11 +30,13 @@ import ( "github.com/TecharoHQ/anubis" "github.com/TecharoHQ/anubis/data" "github.com/TecharoHQ/anubis/internal" + "github.com/TecharoHQ/anubis/internal/thoth" libanubis "github.com/TecharoHQ/anubis/lib" botPolicy "github.com/TecharoHQ/anubis/lib/policy" "github.com/TecharoHQ/anubis/lib/policy/config" "github.com/TecharoHQ/anubis/web" "github.com/facebookgo/flagenv" + _ "github.com/joho/godotenv/autoload" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -70,6 +72,10 @@ var ( webmasterEmail = flag.String("webmaster-email", "", "if set, displays webmaster's email on the reject page for appeals") versionFlag = flag.Bool("version", false, "print Anubis version") xffStripPrivate = flag.Bool("xff-strip-private", true, "if set, strip private addresses from X-Forwarded-For") + + thothInsecure = flag.Bool("thoth-insecure", false, "if set, connect to Thoth over plain HTTP/2, don't enable this unless support told you to") + thothURL = flag.String("thoth-url", "", "if set, URL for Thoth, the IP reputation database for Anubis") + thothToken = flag.String("thoth-token", "", "if set, API token for Thoth, the IP reputation database for Anubis") ) func keyFromHex(value string) (ed25519.PrivateKey, error) { @@ -233,7 +239,25 @@ func main() { } } - policy, err := libanubis.LoadPoliciesOrDefault(*policyFname, *challengeDifficulty) + ctx := context.Background() + + // Thoth configuration + switch { + case *thothURL != "" && *thothToken == "": + slog.Warn("THOTH_URL is set but no THOTH_TOKEN is set") + case *thothURL == "" && *thothToken != "": + slog.Warn("THOTH_TOKEN is set but no THOTH_URL is set") + case *thothURL != "" && *thothToken != "": + slog.Debug("connecting to Thoth") + thothClient, err := thoth.New(ctx, *thothURL, *thothToken, *thothInsecure) + if err != nil { + log.Fatalf("can't dial thoth at %s: %v", *thothURL, err) + } + + ctx = thoth.With(ctx, thothClient) + } + + policy, err := libanubis.LoadPoliciesOrDefault(ctx, *policyFname, *challengeDifficulty) if err != nil { log.Fatalf("can't parse policy file: %v", err) } diff --git a/data/botPolicies.yaml b/data/botPolicies.yaml index 78fb087..8c24209 100644 --- a/data/botPolicies.yaml +++ b/data/botPolicies.yaml @@ -51,6 +51,29 @@ bots: # report_as: 4 # lie to the operator # algorithm: slow # intentionally waste CPU cycles and time + # Requires a subscription to Thoth to use, see + # https://anubis.techaro.lol/docs/admin/thoth#geoip-based-filtering + - name: countries-with-aggressive-scrapers + action: WEIGH + geoip: + counties: + - BR + - CN + weight: + adjust: 10 + + # Requires a subscription to Thoth to use, see + # https://anubis.techaro.lol/docs/admin/thoth#asn-based-filtering + - name: aggressive-asns-without-functional-abuse-contact + action: WEIGH + asns: + match: + - 13335 # Cloudflare + - 136907 # Huawei Cloud + - 45102 # Alibaba Cloud + weight: + adjust: 10 + # Generic catchall rule - name: generic-browser user_agent_regex: >- diff --git a/docs/docs/CHANGELOG.md b/docs/docs/CHANGELOG.md index b029cb9..7662ee4 100644 --- a/docs/docs/CHANGELOG.md +++ b/docs/docs/CHANGELOG.md @@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Optimized the OGTags subsystem with reduced allocations and runtime per request by up to 66% - Add `--strip-base-prefix` flag/envvar to strip the base prefix from request paths when forwarding to target servers - Add `robots2policy` CLI utility to convert robots.txt files to Anubis challenge policies using CEL expressions ([#409](https://github.com/TecharoHQ/anubis/issues/409)) +- Implement GeoIP and ASN based checks via [Thoth](https://anubis.techaro.lol/docs/admin/thoth) ([#206](https://github.com/TecharoHQ/anubis/issues/206)) ## v1.19.1: Jenomis cen Lexentale - Echo 1 diff --git a/docs/docs/admin/thoth.mdx b/docs/docs/admin/thoth.mdx new file mode 100644 index 0000000..ca8742d --- /dev/null +++ b/docs/docs/admin/thoth.mdx @@ -0,0 +1,81 @@ +# Thoth-based advanced checks + +Status: Beta + +Anubis instances are normally isolated. Each Anubis instance has its own configuration and exists in roughly its own world without any long term memory between requests. As threats, workarounds, and AI scraper toolchains evolve, administrators will need a way to get more up to date information faster than Anubis' release cycle. + +Thus, Thoth is being created. Thoth is the reputation database for Anubis. Thoth feeds information to Anubis so that it can make better decisions about which traffic is innocuous and which traffic is suspicious. + +:::note + +Thoth is hosted by [Techaro](https://techaro.lol). Thoth is a paid service. Thoth is opt-in and requires manual intervention (including payment) to use. The code that powers Thoth is currently closed source. + +To get access to Thoth, please subscribe [on GitHub Sponsors](https://github.com/sponsors/Xe) and [email Xe](mailto:xe@techaro.lol). This will be self-service soon. + +::: + +## Implementation + +Thoth is a web service that listens over [gRPC](https://grpc.io/). Thoth's API is documented in protocol buffer definitions in the GitHub repo [TecharoHQ/thoth-proto](https://github.com/TecharoHQ/thoth-proto). + +Thoth is designed to be _informative_, not _authoritative_. Thoth cannot and will not arbitrarily block requests, origins, or other traffic. Thoth is there to inform Anubis and influence the weight of requests so that upstream resources can be protected. Additionally, Anubis aggressively caches data from Thoth such that over time Anubis will not need to request data very often. This makes the fast path for repeat visitors even faster and reduces the amount of data that Thoth is exposed to. + +## Thoth features + +Thoth is currently in active development. Currently, Thoth provides the following features to Anubis: + +- BGP Autonomous System (ASN) based filtering +- GeoIP location based filtering + +### ASN-based filtering + +When companies link their backbone infrastructure to the Internet, they do so via a [BGP Autonomous System](), denoted by a number (the Autonomous System Number or ASN). Every IP address on the Internet is owned by an ASN with a 1:1 lookup that does not change very frequently. + +Anubis uses Thoth to match IP addresses to BGP Autonomous Systems so that you can either issue arbitrary challenges to individual internet service providers (such as Cloudflare or Huawei Cloud) or, at the administrator's explicit instruction, block them altogether. For example, here's how you add 10 weight points to requests from Cloudflare, Huawei Cloud, and Alibaba Cloud: + +```yaml +- name: aggressive-asns-without-functional-abuse-contact + action: WEIGH + asns: + match: + - 13335 # Cloudflare + - 136907 # Huawei Cloud + - 45102 # Alibaba Cloud + weight: + adjust: 10 +``` + +You can look up details for [AS13335](https://bgp.tools/as/13335) or any of these other top offenders on [bgp.tools](https://bgp.tools). + +### GeoIP-based filtering + +In extreme cases, an administrator may have to take action against an entire country. This is not an ideal circumstance, but sometimes reality forces their hands and the administrators just want to sleep at night. + +Anubis uses Thoth to look up the geographic location registered to an IP address. This lookup is not the best and will get better with time, but you ship what you can so you can make it better for next time. + +For example, to add 10 weight points to requests from Brazil and China: + +```yaml +- name: countries-with-aggressive-scrapers + action: WEIGH + geoip: + counties: + - BR + - CN + weight: + adjust: 10 +``` + +Use this with care. + +## Work-in-progress features + +This section is a bit aspirational and is where Thoth will end up rather than things you can use today. + +In general, a lot of Thoth features are focused on taking the same Anubis you know and love and making it better, smarter, and less paranoid. These include: + +- Private rulesets for advanced patterns, current known exploits, and other recognition tactics that need to be kept cloak and dagger for operational security reasons +- Private challenge implementations via WebAssembly, including advanced browser detection logic +- Reputation querying so that Thoth can arbitrarily influence the weight of requests based on the net aggregate pass rate so that the most common browsers can get through with no challenge issued at all +- APIs for trusted administrators to report abusive request fingerprints so that Anubis can react to threats as they evolve +- A way for Anubis to periodically report the pass rate per ASN and other fingerprints so that methodology can be improved diff --git a/docs/manifest/1password.yaml b/docs/manifest/1password.yaml new file mode 100644 index 0000000..f6e3098 --- /dev/null +++ b/docs/manifest/1password.yaml @@ -0,0 +1,6 @@ +apiVersion: onepassword.com/v1 +kind: OnePasswordItem +metadata: + name: anubis-docs-thoth +spec: + itemPath: "vaults/lc5zo4zjz3if3mkeuhufjmgmui/items/pwguumqcmtxvqbeb7y4gj7l36i" diff --git a/docs/manifest/deployment.yaml b/docs/manifest/deployment.yaml index f39b7fd..ff4d65a 100644 --- a/docs/manifest/deployment.yaml +++ b/docs/manifest/deployment.yaml @@ -68,3 +68,6 @@ spec: - ALL seccompProfile: type: RuntimeDefault + envFrom: + - secretRef: + name: anubis-docs-thoth diff --git a/docs/manifest/kustomization.yaml b/docs/manifest/kustomization.yaml index a06219a..c53a1af 100644 --- a/docs/manifest/kustomization.yaml +++ b/docs/manifest/kustomization.yaml @@ -1,7 +1,9 @@ resources: + - 1password.yaml - deployment.yaml - ingress.yaml - onionservice.yaml + - poddisruptionbudget.yaml - service.yaml configMapGenerator: diff --git a/docs/manifest/poddisruptionbudget.yaml b/docs/manifest/poddisruptionbudget.yaml new file mode 100644 index 0000000..018cfe5 --- /dev/null +++ b/docs/manifest/poddisruptionbudget.yaml @@ -0,0 +1,9 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: anubis-docs +spec: + minAvailable: 1 + selector: + matchLabels: + app: anubis-docs diff --git a/go.mod b/go.mod index 4ec185f..9c60fda 100644 --- a/go.mod +++ b/go.mod @@ -3,22 +3,28 @@ module github.com/TecharoHQ/anubis go 1.24.2 require ( + github.com/TecharoHQ/thoth-proto v0.4.0 github.com/a-h/templ v0.3.898 github.com/facebookgo/flagenv v0.0.0-20160425205200-fcd59fca7456 + github.com/gaissmai/bart v0.20.4 github.com/golang-jwt/jwt/v5 v5.2.2 github.com/google/cel-go v0.25.0 + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 + github.com/joho/godotenv v1.5.1 github.com/playwright-community/playwright-go v0.5200.0 github.com/prometheus/client_golang v1.22.0 github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a github.com/yl2chen/cidranger v1.0.2 golang.org/x/net v0.41.0 gopkg.in/yaml.v3 v3.0.1 + google.golang.org/grpc v1.72.2 k8s.io/apimachinery v0.33.1 sigs.k8s.io/yaml v1.4.0 ) require ( al.essio.dev/pkg/shellescape v1.6.0 // indirect + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250425153114-8976f5be98c1.1 // indirect cel.dev/expr v0.23.1 // indirect dario.cat/mergo v1.0.2 // indirect github.com/AlekSi/pointer v1.2.0 // indirect @@ -66,6 +72,7 @@ require ( github.com/goreleaser/chglog v0.7.0 // indirect github.com/goreleaser/fileglob v1.3.0 // indirect github.com/goreleaser/nfpm/v2 v2.42.1 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/huandu/xstrings v1.5.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect @@ -86,7 +93,7 @@ require ( github.com/shopspring/decimal v1.4.0 // indirect github.com/skeema/knownhosts v1.3.1 // indirect github.com/spf13/cast v1.7.1 // indirect - github.com/stoewer/go-strcase v1.2.0 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect github.com/ulikunitz/xz v0.5.12 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect gitlab.com/digitalxero/go-conventional-commit v1.0.7 // indirect @@ -102,9 +109,9 @@ require ( golang.org/x/tools v0.33.0 // indirect golang.org/x/vuln v1.1.4 // indirect golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 // indirect - google.golang.org/protobuf v1.36.5 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect + google.golang.org/protobuf v1.36.6 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect honnef.co/go/tools v0.6.1 // indirect mvdan.cc/sh/v3 v3.11.0 // indirect diff --git a/go.sum b/go.sum index b695e71..3ea7fe3 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ al.essio.dev/pkg/shellescape v1.6.0 h1:NxFcEqzFSEVCGN2yq7Huv/9hyCEGVa/TncnOOBBeXHA= al.essio.dev/pkg/shellescape v1.6.0/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250425153114-8976f5be98c1.1 h1:YhMSc48s25kr7kv31Z8vf7sPUIq5YJva9z1mn/hAt0M= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250425153114-8976f5be98c1.1/go.mod h1:avRlCjnFzl98VPaeCtJ24RrV/wwHFzB8sWXhj26+n/U= cel.dev/expr v0.23.1 h1:K4KOtPCJQjVggkARsjG9RWXP6O4R73aHeJMa/dmCQQg= cel.dev/expr v0.23.1/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= @@ -28,6 +30,8 @@ github.com/ProtonMail/gopenpgp/v2 v2.7.1 h1:Awsg7MPc2gD3I7IFac2qE3Gdls0lZW8SzrFZ github.com/ProtonMail/gopenpgp/v2 v2.7.1/go.mod h1:/BU5gfAVwqyd8EfC3Eu7zmuhwYQpKs+cGD8M//iiaxs= github.com/Songmu/gitconfig v0.2.0 h1:pX2++u4KUq+K2k/ZCzGXLtkD3ceCqIdi0tDyb+IbSyo= github.com/Songmu/gitconfig v0.2.0/go.mod h1:cB5bYJer+pl7W8g6RHFwL/0X6aJROVrYuHlvc7PT+hE= +github.com/TecharoHQ/thoth-proto v0.4.0 h1:UbkvfgCku0Dm1R6O4ug3HOsJNnE6F3wB8x+Dpw2lzFI= +github.com/TecharoHQ/thoth-proto v0.4.0/go.mod h1:IcGnZt3iYUZQVEa0Lwk5l4ix0hCeXlWUV1TJMZvbWx0= github.com/TecharoHQ/yeet v0.6.0 h1:RCBAjr7wIlllsgy0tpvWpLX7jsZgu2tiuBY3RrprcR0= github.com/TecharoHQ/yeet v0.6.0/go.mod h1:bj2V4Fg8qKQXoiuPZa3HuawrE8g+LsOQv/9q2WyGSsA= github.com/a-h/parse v0.0.0-20250122154542-74294addb73e h1:HjVbSQHy+dnlS6C3XajZ69NYAb5jbGNfHanvm1+iYlo= @@ -99,6 +103,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/gaissmai/bart v0.20.4 h1:Ik47r1fy3jRVU+1eYzKSW3ho2UgBVTVnUS8O993584U= +github.com/gaissmai/bart v0.20.4/go.mod h1:cEed+ge8dalcbpi8wtS9x9m2hn/fNJH5suhdGQOHnYk= github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= @@ -111,6 +117,10 @@ github.com/go-git/go-git/v5 v5.14.0 h1:/MD3lCrGjCen5WfEAzKg00MJJffKhC8gzS80ycmCi github.com/go-git/go-git/v5 v5.14.0/go.mod h1:Z5Xhoia5PcWA3NF8vRLURn9E5FRhSl7dGj9ItW3Wk5k= github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY= github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= @@ -134,6 +144,8 @@ github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeD github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/cel-go v0.25.0 h1:jsFw9Fhn+3y2kBbltZR4VEz5xKkcIFRPDnuEzAGv5GY= github.com/google/cel-go v0.25.0/go.mod h1:hjEb6r5SuOSlhCHmFoLzu8HGCERvIsDAbxDAyNU/MmI= github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786 h1:rcv+Ippz6RAtvaGgKxc+8FQIpxHgsF+HBzPyYL2cyVU= @@ -159,12 +171,18 @@ github.com/goreleaser/fileglob v1.3.0 h1:/X6J7U8lbDpQtBvGcwwPS6OpzkNVlVEsFUVRx9+ github.com/goreleaser/fileglob v1.3.0/go.mod h1:Jx6BoXv3mbYkEzwm9THo7xbr5egkAraxkGorbJb4RxU= github.com/goreleaser/nfpm/v2 v2.42.1 h1:xu2pLRgQuz2ab+YZFoeIzwU/M5jjjCKDGwv1lRbVGvk= github.com/goreleaser/nfpm/v2 v2.42.1/go.mod h1:dY53KWYKebkOocxgkmpM7SRX0Nv5hU+jEu2kIaM4/LI= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/henvic/httpretty v0.0.6/go.mod h1:X38wLjWXHkXT7r2+uK8LjCMne9rsuNaBLJ+5cU2/Pmo= github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= @@ -248,13 +266,17 @@ github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sS github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/thlib/go-timezone-local v0.0.0-20210907160436-ef149e42d28e/go.mod h1:/Tnicc6m/lsJE0irFMA0LfIwTBo4QP7A8IfyIv4zZKI= @@ -269,6 +291,18 @@ github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= gitlab.com/digitalxero/go-conventional-commit v1.0.7 h1:8/dO6WWG+98PMhlZowt/YjuiKhqhGlOCwlIV8SqqGh8= gitlab.com/digitalxero/go-conventional-commit v1.0.7/go.mod h1:05Xc2BFsSyC5tKhK0y+P3bs0AwUtNuTp+mTpbCU/DZ0= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -353,12 +387,14 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 h1:LLhsEBxRTBLuKlQxFBYUOU8xyFgXv6cOTp2HASDlsDk= golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 h1:YcyjlL1PRr2Q17/I0dPk2JmYS5CDXfcdb2Z3YRioEbw= -google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 h1:2035KHhUv+EpyB+hWgJnaWKJOdX1E95w2S8Rr4uWKTs= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a h1:nwKuGPlUAt+aR+pcrkfFRrTU1BVrSmYyYMxYbUIVHr0= +google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a/go.mod h1:3kWAYMk1I75K4vykHtKt2ycnOgpA6974V7bREqbsenU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a h1:51aaUVRocpvUOSQKM6Q7VuoaktNIaMCLuhZB6DKksq4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a/go.mod h1:uRxBH1mhmO8PGhU89cMcHaXKZqO+OfakD8QQO0oYwlQ= +google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8= +google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/test/playwright_test.go b/internal/test/playwright_test.go index 2ac5d94..0cb72e9 100644 --- a/internal/test/playwright_test.go +++ b/internal/test/playwright_test.go @@ -595,7 +595,7 @@ func spawnAnubisWithOptions(t *testing.T, basePrefix string) string { fmt.Fprintf(w, "%d", time.Now().Unix()) }) - policy, err := libanubis.LoadPoliciesOrDefault("", anubis.DefaultDifficulty) + policy, err := libanubis.LoadPoliciesOrDefault(t.Context(), "", anubis.DefaultDifficulty) if err != nil { t.Fatal(err) } diff --git a/internal/thoth/asnchecker.go b/internal/thoth/asnchecker.go new file mode 100644 index 0000000..b2697c8 --- /dev/null +++ b/internal/thoth/asnchecker.go @@ -0,0 +1,69 @@ +package thoth + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "strings" + "time" + + "github.com/TecharoHQ/anubis/internal" + "github.com/TecharoHQ/anubis/lib/policy/checker" + iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1" +) + +func (c *Client) ASNCheckerFor(asns []uint32) checker.Impl { + asnMap := map[uint32]struct{}{} + var sb strings.Builder + fmt.Fprintln(&sb, "ASNChecker") + for _, asn := range asns { + asnMap[asn] = struct{}{} + fmt.Fprintln(&sb, "AS", asn) + } + + return &ASNChecker{ + iptoasn: c.IPToASN, + asns: asnMap, + hash: internal.SHA256sum(sb.String()), + } +} + +type ASNChecker struct { + iptoasn iptoasnv1.IpToASNServiceClient + asns map[uint32]struct{} + hash string +} + +func (asnc *ASNChecker) Check(r *http.Request) (bool, error) { + ctx, cancel := context.WithTimeout(r.Context(), 500*time.Millisecond) + defer cancel() + + ipInfo, err := asnc.iptoasn.Lookup(ctx, &iptoasnv1.LookupRequest{ + IpAddress: r.Header.Get("X-Real-Ip"), + }) + if err != nil { + switch { + case errors.Is(err, context.DeadlineExceeded): + slog.Debug("error contacting thoth", "err", err, "actionable", false) + return false, nil + default: + slog.Error("error contacting thoth, please contact support", "err", err, "actionable", true) + return false, nil + } + } + + // If IP is not publicly announced, return false + if !ipInfo.GetAnnounced() { + return false, nil + } + + _, ok := asnc.asns[uint32(ipInfo.GetAsNumber())] + + return ok, nil +} + +func (asnc *ASNChecker) Hash() string { + return asnc.hash +} diff --git a/internal/thoth/asnchecker_test.go b/internal/thoth/asnchecker_test.go new file mode 100644 index 0000000..a80e815 --- /dev/null +++ b/internal/thoth/asnchecker_test.go @@ -0,0 +1,81 @@ +package thoth_test + +import ( + "fmt" + "net/http/httptest" + "testing" + + "github.com/TecharoHQ/anubis/internal/thoth" + "github.com/TecharoHQ/anubis/lib/policy/checker" + iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1" +) + +var _ checker.Impl = &thoth.ASNChecker{} + +func TestASNChecker(t *testing.T) { + cli := loadSecrets(t) + + asnc := cli.ASNCheckerFor([]uint32{13335}) + + for _, cs := range []struct { + ipAddress string + wantMatch bool + wantError bool + }{ + { + ipAddress: "1.1.1.1", + wantMatch: true, + wantError: false, + }, + { + ipAddress: "2.2.2.2", + wantMatch: false, + wantError: false, + }, + { + ipAddress: "taco", + wantMatch: false, + wantError: false, + }, + { + ipAddress: "127.0.0.1", + wantMatch: false, + wantError: false, + }, + } { + t.Run(fmt.Sprintf("%v", cs), func(t *testing.T) { + req := httptest.NewRequest("GET", "/", nil) + req.Header.Set("X-Real-Ip", cs.ipAddress) + + match, err := asnc.Check(req) + + if match != cs.wantMatch { + t.Errorf("Wanted match: %v, got: %v", cs.wantMatch, match) + } + + switch { + case err != nil && !cs.wantError: + t.Errorf("Did not want error but got: %v", err) + case err == nil && cs.wantError: + t.Error("Wanted error but got none") + } + }) + } +} + +func BenchmarkWithCache(b *testing.B) { + cli := loadSecrets(b) + req := &iptoasnv1.LookupRequest{IpAddress: "1.1.1.1"} + + _, err := cli.IPToASN.Lookup(b.Context(), req) + if err != nil { + b.Error(err) + } + + for b.Loop() { + _, err := cli.IPToASN.Lookup(b.Context(), req) + if err != nil { + b.Error(err) + } + } +} diff --git a/internal/thoth/auth.go b/internal/thoth/auth.go new file mode 100644 index 0000000..f4d52c6 --- /dev/null +++ b/internal/thoth/auth.go @@ -0,0 +1,39 @@ +package thoth + +import ( + "context" + + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" +) + +func authUnaryClientInterceptor(token string) grpc.UnaryClientInterceptor { + return func( + ctx context.Context, + method string, + req interface{}, + reply interface{}, + cc *grpc.ClientConn, + invoker grpc.UnaryInvoker, + opts ...grpc.CallOption, + ) error { + md := metadata.Pairs("authorization", "Bearer "+token) + ctx = metadata.NewOutgoingContext(ctx, md) + return invoker(ctx, method, req, reply, cc, opts...) + } +} + +func authStreamClientInterceptor(token string) grpc.StreamClientInterceptor { + return func( + ctx context.Context, + desc *grpc.StreamDesc, + cc *grpc.ClientConn, + method string, + streamer grpc.Streamer, + opts ...grpc.CallOption, + ) (grpc.ClientStream, error) { + md := metadata.Pairs("authorization", "Bearer "+token) + ctx = metadata.NewOutgoingContext(ctx, md) + return streamer(ctx, desc, cc, method, opts...) + } +} diff --git a/internal/thoth/cachediptoasn.go b/internal/thoth/cachediptoasn.go new file mode 100644 index 0000000..c10fbae --- /dev/null +++ b/internal/thoth/cachediptoasn.go @@ -0,0 +1,84 @@ +package thoth + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/netip" + + iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1" + "github.com/gaissmai/bart" + "google.golang.org/grpc" +) + +type IPToASNWithCache struct { + next iptoasnv1.IpToASNServiceClient + table *bart.Table[*iptoasnv1.LookupResponse] +} + +func NewIpToASNWithCache(next iptoasnv1.IpToASNServiceClient) *IPToASNWithCache { + result := &IPToASNWithCache{ + next: next, + table: &bart.Table[*iptoasnv1.LookupResponse]{}, + } + + for _, pfx := range []netip.Prefix{ + netip.MustParsePrefix("10.0.0.0/8"), // RFC 1918 + netip.MustParsePrefix("172.16.0.0/12"), // RFC 1918 + netip.MustParsePrefix("192.168.0.0/16"), // RFC 1918 + netip.MustParsePrefix("127.0.0.0/8"), // Loopback + netip.MustParsePrefix("169.254.0.0/16"), // Link-local + netip.MustParsePrefix("100.64.0.0/10"), // CGNAT + netip.MustParsePrefix("192.0.0.0/24"), // Protocol assignments + netip.MustParsePrefix("192.0.2.0/24"), // TEST-NET-1 + netip.MustParsePrefix("198.18.0.0/15"), // Benchmarking + netip.MustParsePrefix("198.51.100.0/24"), // TEST-NET-2 + netip.MustParsePrefix("203.0.113.0/24"), // TEST-NET-3 + netip.MustParsePrefix("240.0.0.0/4"), // Reserved + netip.MustParsePrefix("255.255.255.255/32"), // Broadcast + netip.MustParsePrefix("fc00::/7"), // Unique local address + netip.MustParsePrefix("fe80::/10"), // Link-local + netip.MustParsePrefix("::1/128"), // Loopback + netip.MustParsePrefix("::/128"), // Unspecified + netip.MustParsePrefix("100::/64"), // Discard-only + netip.MustParsePrefix("2001:db8::/32"), // Documentation + } { + result.table.Insert(pfx, &iptoasnv1.LookupResponse{Announced: false}) + } + + return result +} + +func (ip2asn *IPToASNWithCache) Lookup(ctx context.Context, lr *iptoasnv1.LookupRequest, opts ...grpc.CallOption) (*iptoasnv1.LookupResponse, error) { + addr, err := netip.ParseAddr(lr.GetIpAddress()) + if err != nil { + return nil, fmt.Errorf("input is not an IP address: %w", err) + } + + cachedResponse, ok := ip2asn.table.Lookup(addr) + if ok { + return cachedResponse, nil + } + + resp, err := ip2asn.next.Lookup(ctx, lr, opts...) + if err != nil { + return nil, err + } + + var errs []error + for _, cidr := range resp.GetCidr() { + pfx, err := netip.ParsePrefix(cidr) + if err != nil { + errs = append(errs, err) + continue + } + ip2asn.table.Insert(pfx, resp) + } + + if len(errs) != 0 { + slog.Error("errors parsing IP prefixes", "err", errors.Join(errs...)) + } + + return resp, nil +} diff --git a/internal/thoth/context.go b/internal/thoth/context.go new file mode 100644 index 0000000..f58d550 --- /dev/null +++ b/internal/thoth/context.go @@ -0,0 +1,14 @@ +package thoth + +import "context" + +type ctxKey struct{} + +func With(ctx context.Context, cli *Client) context.Context { + return context.WithValue(ctx, ctxKey{}, cli) +} + +func FromContext(ctx context.Context) (*Client, bool) { + cli, ok := ctx.Value(ctxKey{}).(*Client) + return cli, ok +} diff --git a/internal/thoth/geoipchecker.go b/internal/thoth/geoipchecker.go new file mode 100644 index 0000000..ef6dcb8 --- /dev/null +++ b/internal/thoth/geoipchecker.go @@ -0,0 +1,68 @@ +package thoth + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "strings" + "time" + + "github.com/TecharoHQ/anubis/lib/policy/checker" + iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1" +) + +func (c *Client) GeoIPCheckerFor(countries []string) checker.Impl { + countryMap := map[string]struct{}{} + var sb strings.Builder + fmt.Fprintln(&sb, "GeoIPChecker") + for _, cc := range countries { + countryMap[cc] = struct{}{} + fmt.Fprintln(&sb, cc) + } + + return &GeoIPChecker{ + IPToASN: c.IPToASN, + Countries: countryMap, + hash: sb.String(), + } +} + +type GeoIPChecker struct { + IPToASN iptoasnv1.IpToASNServiceClient + Countries map[string]struct{} + hash string +} + +func (gipc *GeoIPChecker) Check(r *http.Request) (bool, error) { + ctx, cancel := context.WithTimeout(r.Context(), 500*time.Millisecond) + defer cancel() + + ipInfo, err := gipc.IPToASN.Lookup(ctx, &iptoasnv1.LookupRequest{ + IpAddress: r.Header.Get("X-Real-Ip"), + }) + if err != nil { + switch { + case errors.Is(err, context.DeadlineExceeded): + slog.Debug("error contacting thoth", "err", err, "actionable", false) + return false, nil + default: + slog.Error("error contacting thoth, please contact support", "err", err, "actionable", true) + return false, nil + } + } + + // If IP is not publicly announced, return false + if !ipInfo.GetAnnounced() { + return false, nil + } + + _, ok := gipc.Countries[strings.ToLower(ipInfo.GetCountryCode())] + + return ok, nil +} + +func (gipc *GeoIPChecker) Hash() string { + return gipc.hash +} diff --git a/internal/thoth/geoipchecker_test.go b/internal/thoth/geoipchecker_test.go new file mode 100644 index 0000000..25b37b9 --- /dev/null +++ b/internal/thoth/geoipchecker_test.go @@ -0,0 +1,63 @@ +package thoth_test + +import ( + "fmt" + "net/http/httptest" + "testing" + + "github.com/TecharoHQ/anubis/internal/thoth" + "github.com/TecharoHQ/anubis/lib/policy/checker" +) + +var _ checker.Impl = &thoth.GeoIPChecker{} + +func TestGeoIPChecker(t *testing.T) { + cli := loadSecrets(t) + + asnc := cli.GeoIPCheckerFor([]string{"us"}) + + for _, cs := range []struct { + ipAddress string + wantMatch bool + wantError bool + }{ + { + ipAddress: "1.1.1.1", + wantMatch: true, + wantError: false, + }, + { + ipAddress: "2.2.2.2", + wantMatch: false, + wantError: false, + }, + { + ipAddress: "taco", + wantMatch: false, + wantError: false, + }, + { + ipAddress: "127.0.0.1", + wantMatch: false, + wantError: false, + }, + } { + t.Run(fmt.Sprintf("%v", cs), func(t *testing.T) { + req := httptest.NewRequest("GET", "/", nil) + req.Header.Set("X-Real-Ip", cs.ipAddress) + + match, err := asnc.Check(req) + + if match != cs.wantMatch { + t.Errorf("Wanted match: %v, got: %v", cs.wantMatch, match) + } + + switch { + case err != nil && !cs.wantError: + t.Errorf("Did not want error but got: %v", err) + case err == nil && cs.wantError: + t.Error("Wanted error but got none") + } + }) + } +} diff --git a/internal/thoth/thoth.go b/internal/thoth/thoth.go new file mode 100644 index 0000000..893f2d7 --- /dev/null +++ b/internal/thoth/thoth.go @@ -0,0 +1,88 @@ +package thoth + +import ( + "context" + "crypto/tls" + "fmt" + "time" + + "github.com/TecharoHQ/anubis" + iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1" + grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" + "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/timeout" + "github.com/prometheus/client_golang/prometheus" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + healthv1 "google.golang.org/grpc/health/grpc_health_v1" +) + +type Client struct { + conn *grpc.ClientConn + health healthv1.HealthClient + IPToASN iptoasnv1.IpToASNServiceClient +} + +func New(ctx context.Context, thothURL, apiToken string, plaintext bool) (*Client, error) { + clMetrics := grpcprom.NewClientMetrics( + grpcprom.WithClientHandlingTimeHistogram( + grpcprom.WithHistogramBuckets([]float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}), + ), + ) + prometheus.DefaultRegisterer.Register(clMetrics) + + do := []grpc.DialOption{ + grpc.WithChainUnaryInterceptor( + timeout.UnaryClientInterceptor(500*time.Millisecond), + clMetrics.UnaryClientInterceptor(), + authUnaryClientInterceptor(apiToken), + ), + grpc.WithChainStreamInterceptor( + clMetrics.StreamClientInterceptor(), + authStreamClientInterceptor(apiToken), + ), + grpc.WithUserAgent(fmt.Sprint("Techaro/anubis:", anubis.Version)), + } + + if plaintext { + do = append(do, grpc.WithTransportCredentials(insecure.NewCredentials())) + } else { + do = append(do, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{}))) + } + + conn, err := grpc.NewClient( + thothURL, + do..., + ) + if err != nil { + return nil, fmt.Errorf("can't dial thoth at %s: %w", thothURL, err) + } + + hc := healthv1.NewHealthClient(conn) + + resp, err := hc.Check(ctx, &healthv1.HealthCheckRequest{}) + if err != nil { + return nil, fmt.Errorf("can't verify thoth health at %s: %w", thothURL, err) + } + + if resp.Status != healthv1.HealthCheckResponse_SERVING { + return nil, fmt.Errorf("thoth is not healthy, wanted %s but got %s", healthv1.HealthCheckResponse_SERVING, resp.Status) + } + + return &Client{ + conn: conn, + health: hc, + IPToASN: NewIpToASNWithCache(iptoasnv1.NewIpToASNServiceClient(conn)), + }, nil +} + +func (c *Client) Close() error { + if c.conn != nil { + return c.conn.Close() + } + return nil +} + +func (c *Client) WithIPToASNService(impl iptoasnv1.IpToASNServiceClient) { + c.IPToASN = impl +} diff --git a/internal/thoth/thoth_test.go b/internal/thoth/thoth_test.go new file mode 100644 index 0000000..437f984 --- /dev/null +++ b/internal/thoth/thoth_test.go @@ -0,0 +1,36 @@ +package thoth_test + +import ( + "os" + "testing" + + "github.com/TecharoHQ/anubis/internal/thoth" + "github.com/TecharoHQ/anubis/internal/thoth/thothmock" + "github.com/joho/godotenv" +) + +func loadSecrets(t testing.TB) *thoth.Client { + t.Helper() + + if err := godotenv.Load(); err != nil { + t.Log("using mock thoth") + result := &thoth.Client{} + result.WithIPToASNService(thothmock.MockIpToASNService()) + return result + } + + cli, err := thoth.New(t.Context(), os.Getenv("THOTH_URL"), os.Getenv("THOTH_API_KEY"), false) + if err != nil { + t.Fatal(err) + } + + return cli +} + +func TestNew(t *testing.T) { + cli := loadSecrets(t) + + if err := cli.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/internal/thoth/thothmock/iptoasn.go b/internal/thoth/thothmock/iptoasn.go new file mode 100644 index 0000000..fceba64 --- /dev/null +++ b/internal/thoth/thothmock/iptoasn.go @@ -0,0 +1,59 @@ +package thothmock + +import ( + "context" + "net/netip" + + iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func MockIpToASNService() *IpToASNService { + responses := map[string]*iptoasnv1.LookupResponse{ + "127.0.0.1": {Announced: false}, + "::1": {Announced: false}, + "10.10.10.10": { + Announced: true, + AsNumber: 13335, + Cidr: []string{"1.1.1.0/24"}, + CountryCode: "US", + Description: "Cloudflare", + }, + "2.2.2.2": { + Announced: true, + AsNumber: 420, + Cidr: []string{"2.2.2.0/24"}, + CountryCode: "CA", + Description: "test canada", + }, + "1.1.1.1": { + Announced: true, + AsNumber: 13335, + Cidr: []string{"1.1.1.0/24"}, + CountryCode: "US", + Description: "Cloudflare", + }, + } + + return &IpToASNService{Responses: responses} +} + +type IpToASNService struct { + iptoasnv1.UnimplementedIpToASNServiceServer + Responses map[string]*iptoasnv1.LookupResponse +} + +func (ip2asn *IpToASNService) Lookup(ctx context.Context, lr *iptoasnv1.LookupRequest, opts ...grpc.CallOption) (*iptoasnv1.LookupResponse, error) { + if _, err := netip.ParseAddr(lr.GetIpAddress()); err != nil { + return nil, err + } + + resp, ok := ip2asn.Responses[lr.GetIpAddress()] + if !ok { + return nil, status.Error(codes.NotFound, "IP address not found in mock") + } + + return resp, nil +} diff --git a/internal/thoth/thothmock/withthothmock.go b/internal/thoth/thothmock/withthothmock.go new file mode 100644 index 0000000..9565007 --- /dev/null +++ b/internal/thoth/thothmock/withthothmock.go @@ -0,0 +1,17 @@ +package thothmock + +import ( + "context" + "testing" + + "github.com/TecharoHQ/anubis/internal/thoth" +) + +func WithMockThoth(t *testing.T) context.Context { + t.Helper() + + thothCli := &thoth.Client{} + thothCli.WithIPToASNService(MockIpToASNService()) + ctx := thoth.With(t.Context(), thothCli) + return ctx +} diff --git a/lib/anubis.go b/lib/anubis.go index bd4038f..f700281 100644 --- a/lib/anubis.go +++ b/lib/anubis.go @@ -26,6 +26,7 @@ import ( "github.com/TecharoHQ/anubis/internal/ogtags" "github.com/TecharoHQ/anubis/lib/challenge" "github.com/TecharoHQ/anubis/lib/policy" + "github.com/TecharoHQ/anubis/lib/policy/checker" "github.com/TecharoHQ/anubis/lib/policy/config" // challenge implementations @@ -483,7 +484,7 @@ func (s *Server) check(r *http.Request) (policy.CheckResult, *policy.Bot, error) ReportAs: s.policy.DefaultDifficulty, Algorithm: config.DefaultAlgorithm, }, - Rules: &policy.CheckerList{}, + Rules: &checker.List{}, }, nil } diff --git a/lib/anubis_test.go b/lib/anubis_test.go index 3b9a1b1..615a845 100644 --- a/lib/anubis_test.go +++ b/lib/anubis_test.go @@ -15,6 +15,7 @@ import ( "github.com/TecharoHQ/anubis" "github.com/TecharoHQ/anubis/data" "github.com/TecharoHQ/anubis/internal" + "github.com/TecharoHQ/anubis/internal/thoth/thothmock" "github.com/TecharoHQ/anubis/lib/policy" "github.com/TecharoHQ/anubis/lib/policy/config" ) @@ -26,7 +27,9 @@ func init() { func loadPolicies(t *testing.T, fname string) *policy.ParsedConfig { t.Helper() - anubisPolicy, err := LoadPoliciesOrDefault(fname, anubis.DefaultDifficulty) + ctx := thothmock.WithMockThoth(t) + + anubisPolicy, err := LoadPoliciesOrDefault(ctx, fname, anubis.DefaultDifficulty) if err != nil { t.Fatal(err) } @@ -164,7 +167,7 @@ func TestLoadPolicies(t *testing.T) { } defer fin.Close() - if _, err := policy.ParseConfig(fin, fname, 4); err != nil { + if _, err := policy.ParseConfig(t.Context(), fin, fname, 4); err != nil { t.Fatal(err) } }) @@ -313,7 +316,7 @@ func TestCheckDefaultDifficultyMatchesPolicy(t *testing.T) { for i := 1; i < 10; i++ { t.Run(fmt.Sprint(i), func(t *testing.T) { - anubisPolicy, err := LoadPoliciesOrDefault("", i) + anubisPolicy, err := LoadPoliciesOrDefault(t.Context(), "", i) if err != nil { t.Fatal(err) } diff --git a/lib/config.go b/lib/config.go index 47a1b6e..a893a43 100644 --- a/lib/config.go +++ b/lib/config.go @@ -1,6 +1,7 @@ package lib import ( + "context" "crypto/ed25519" "crypto/rand" "errors" @@ -43,7 +44,7 @@ type Options struct { ServeRobotsTXT bool } -func LoadPoliciesOrDefault(fname string, defaultDifficulty int) (*policy.ParsedConfig, error) { +func LoadPoliciesOrDefault(ctx context.Context, fname string, defaultDifficulty int) (*policy.ParsedConfig, error) { var fin io.ReadCloser var err error @@ -67,7 +68,7 @@ func LoadPoliciesOrDefault(fname string, defaultDifficulty int) (*policy.ParsedC } }(fin) - anubisPolicy, err := policy.ParseConfig(fin, fname, defaultDifficulty) + anubisPolicy, err := policy.ParseConfig(ctx, fin, fname, defaultDifficulty) if err != nil { return nil, fmt.Errorf("can't parse policy file %s: %w", fname, err) } diff --git a/lib/config_test.go b/lib/config_test.go index 2927051..6ce8bc9 100644 --- a/lib/config_test.go +++ b/lib/config_test.go @@ -7,11 +7,12 @@ import ( "testing" "github.com/TecharoHQ/anubis" + "github.com/TecharoHQ/anubis/internal/thoth/thothmock" "github.com/TecharoHQ/anubis/lib/policy" ) func TestInvalidChallengeMethod(t *testing.T) { - if _, err := LoadPoliciesOrDefault("testdata/invalid-challenge-method.yaml", 4); !errors.Is(err, policy.ErrChallengeRuleHasWrongAlgorithm) { + if _, err := LoadPoliciesOrDefault(t.Context(), "testdata/invalid-challenge-method.yaml", 4); !errors.Is(err, policy.ErrChallengeRuleHasWrongAlgorithm) { t.Fatalf("wanted error %v but got %v", policy.ErrChallengeRuleHasWrongAlgorithm, err) } } @@ -25,7 +26,7 @@ func TestBadConfigs(t *testing.T) { for _, st := range finfos { st := st t.Run(st.Name(), func(t *testing.T) { - if _, err := LoadPoliciesOrDefault(filepath.Join("policy", "config", "testdata", "good", st.Name()), anubis.DefaultDifficulty); err == nil { + if _, err := LoadPoliciesOrDefault(t.Context(), filepath.Join("policy", "config", "testdata", "good", st.Name()), anubis.DefaultDifficulty); err == nil { t.Fatal(err) } else { t.Log(err) @@ -43,9 +44,18 @@ func TestGoodConfigs(t *testing.T) { for _, st := range finfos { st := st t.Run(st.Name(), func(t *testing.T) { - if _, err := LoadPoliciesOrDefault(filepath.Join("policy", "config", "testdata", "good", st.Name()), anubis.DefaultDifficulty); err != nil { - t.Fatal(err) - } + t.Run("with-thoth", func(t *testing.T) { + ctx := thothmock.WithMockThoth(t) + if _, err := LoadPoliciesOrDefault(ctx, filepath.Join("policy", "config", "testdata", "good", st.Name()), anubis.DefaultDifficulty); err != nil { + t.Fatal(err) + } + }) + + t.Run("without-thoth", func(t *testing.T) { + if _, err := LoadPoliciesOrDefault(t.Context(), filepath.Join("policy", "config", "testdata", "good", st.Name()), anubis.DefaultDifficulty); err != nil { + t.Fatal(err) + } + }) }) } } diff --git a/lib/policy/bot.go b/lib/policy/bot.go index 831c737..ba884d6 100644 --- a/lib/policy/bot.go +++ b/lib/policy/bot.go @@ -4,11 +4,12 @@ import ( "fmt" "github.com/TecharoHQ/anubis/internal" + "github.com/TecharoHQ/anubis/lib/policy/checker" "github.com/TecharoHQ/anubis/lib/policy/config" ) type Bot struct { - Rules Checker + Rules checker.Impl Challenge *config.ChallengeRules Weight *config.Weight Name string diff --git a/lib/policy/checker.go b/lib/policy/checker.go index 447a7ad..0096578 100644 --- a/lib/policy/checker.go +++ b/lib/policy/checker.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/TecharoHQ/anubis/internal" + "github.com/TecharoHQ/anubis/lib/policy/checker" "github.com/yl2chen/cidranger" ) @@ -16,37 +17,6 @@ var ( ErrMisconfiguration = errors.New("[unexpected] policy: administrator misconfiguration") ) -type Checker interface { - Check(*http.Request) (bool, error) - Hash() string -} - -type CheckerList []Checker - -func (cl CheckerList) Check(r *http.Request) (bool, error) { - for _, c := range cl { - ok, err := c.Check(r) - if err != nil { - return ok, err - } - if ok { - return ok, nil - } - } - - return false, nil -} - -func (cl CheckerList) Hash() string { - var sb strings.Builder - - for _, c := range cl { - fmt.Fprintln(&sb, c.Hash()) - } - - return internal.SHA256sum(sb.String()) -} - type staticHashChecker struct { hash string } @@ -57,7 +27,7 @@ func (staticHashChecker) Check(r *http.Request) (bool, error) { func (s staticHashChecker) Hash() string { return s.hash } -func NewStaticHashChecker(hashable string) Checker { +func NewStaticHashChecker(hashable string) checker.Impl { return staticHashChecker{hash: internal.SHA256sum(hashable)} } @@ -66,7 +36,7 @@ type RemoteAddrChecker struct { hash string } -func NewRemoteAddrChecker(cidrs []string) (Checker, error) { +func NewRemoteAddrChecker(cidrs []string) (checker.Impl, error) { ranger := cidranger.NewPCTrieRanger() var sb strings.Builder @@ -122,11 +92,11 @@ type HeaderMatchesChecker struct { hash string } -func NewUserAgentChecker(rexStr string) (Checker, error) { +func NewUserAgentChecker(rexStr string) (checker.Impl, error) { return NewHeaderMatchesChecker("User-Agent", rexStr) } -func NewHeaderMatchesChecker(header, rexStr string) (Checker, error) { +func NewHeaderMatchesChecker(header, rexStr string) (checker.Impl, error) { rex, err := regexp.Compile(strings.TrimSpace(rexStr)) if err != nil { return nil, fmt.Errorf("%w: regex %s failed parse: %w", ErrMisconfiguration, rexStr, err) @@ -151,7 +121,7 @@ type PathChecker struct { hash string } -func NewPathChecker(rexStr string) (Checker, error) { +func NewPathChecker(rexStr string) (checker.Impl, error) { rex, err := regexp.Compile(strings.TrimSpace(rexStr)) if err != nil { return nil, fmt.Errorf("%w: regex %s failed parse: %w", ErrMisconfiguration, rexStr, err) @@ -171,7 +141,7 @@ func (pc *PathChecker) Hash() string { return pc.hash } -func NewHeaderExistsChecker(key string) Checker { +func NewHeaderExistsChecker(key string) checker.Impl { return headerExistsChecker{strings.TrimSpace(key)} } @@ -191,8 +161,8 @@ func (hec headerExistsChecker) Hash() string { return internal.SHA256sum(hec.header) } -func NewHeadersChecker(headermap map[string]string) (Checker, error) { - var result CheckerList +func NewHeadersChecker(headermap map[string]string) (checker.Impl, error) { + var result checker.List var errs []error for key, rexStr := range headermap { diff --git a/lib/policy/checker/checker.go b/lib/policy/checker/checker.go new file mode 100644 index 0000000..4d7b5c7 --- /dev/null +++ b/lib/policy/checker/checker.go @@ -0,0 +1,41 @@ +// Package checker defines the Checker interface and a helper utility to avoid import cycles. +package checker + +import ( + "fmt" + "net/http" + "strings" + + "github.com/TecharoHQ/anubis/internal" +) + +type Impl interface { + Check(*http.Request) (bool, error) + Hash() string +} + +type List []Impl + +func (l List) Check(r *http.Request) (bool, error) { + for _, c := range l { + ok, err := c.Check(r) + if err != nil { + return ok, err + } + if ok { + return ok, nil + } + } + + return false, nil +} + +func (l List) Hash() string { + var sb strings.Builder + + for _, c := range l { + fmt.Fprintln(&sb, c.Hash()) + } + + return internal.SHA256sum(sb.String()) +} diff --git a/lib/policy/config/asn.go b/lib/policy/config/asn.go new file mode 100644 index 0000000..2b92ff3 --- /dev/null +++ b/lib/policy/config/asn.go @@ -0,0 +1,44 @@ +package config + +import ( + "errors" + "fmt" +) + +var ( + ErrPrivateASN = errors.New("bot.ASNs: you have specified a private use ASN") +) + +type ASNs struct { + Match []uint32 `json:"match"` +} + +func (a *ASNs) Valid() error { + var errs []error + + for _, asn := range a.Match { + if isPrivateASN(asn) { + errs = append(errs, fmt.Errorf("%w: %d is private (see RFC 6996)", ErrPrivateASN, asn)) + } + } + + if len(errs) != 0 { + return fmt.Errorf("bot.ASNs: invalid ASN settings: %w", errors.Join(errs...)) + } + + return nil +} + +// isPrivateASN checks if an ASN is in the private use area. +// +// Based on RFC 6996 and IANA allocations. +func isPrivateASN(asn uint32) bool { + switch { + case asn >= 64512 && asn <= 65534: + return true + case asn >= 4200000000 && asn <= 4294967294: + return true + default: + return false + } +} diff --git a/lib/policy/config/config.go b/lib/policy/config/config.go index 5e2a96b..78cbb97 100644 --- a/lib/policy/config/config.go +++ b/lib/policy/config/config.go @@ -55,6 +55,10 @@ type BotConfig struct { Name string `json:"name" yaml:"name"` Action Rule `json:"action" yaml:"action"` RemoteAddr []string `json:"remote_addresses,omitempty" yaml:"remote_addresses,omitempty"` + + // Thoth features + GeoIP *GeoIP `json:"geoip,omitempty"` + ASNs *ASNs `json:"asns,omitempty"` } func (b BotConfig) Zero() bool { @@ -66,6 +70,8 @@ func (b BotConfig) Zero() bool { b.Action != "", len(b.RemoteAddr) != 0, b.Challenge != nil, + b.GeoIP != nil, + b.ASNs != nil, } { if cond { return false @@ -85,7 +91,9 @@ func (b *BotConfig) Valid() error { allFieldsEmpty := b.UserAgentRegex == nil && b.PathRegex == nil && len(b.RemoteAddr) == 0 && - len(b.HeadersRegex) == 0 + len(b.HeadersRegex) == 0 && + b.ASNs == nil && + b.GeoIP == nil if allFieldsEmpty && b.Expression == nil { errs = append(errs, ErrBotMustHaveUserAgentOrPath) diff --git a/lib/policy/config/geoip.go b/lib/policy/config/geoip.go new file mode 100644 index 0000000..d62d027 --- /dev/null +++ b/lib/policy/config/geoip.go @@ -0,0 +1,36 @@ +package config + +import ( + "errors" + "fmt" + "regexp" + "strings" +) + +var ( + countryCodeRegexp = regexp.MustCompile(`^\w{2}$`) + + ErrNotCountryCode = errors.New("config.Bot: invalid country code") +) + +type GeoIP struct { + Countries []string `json:"countries"` +} + +func (g *GeoIP) Valid() error { + var errs []error + + for i, cc := range g.Countries { + if !countryCodeRegexp.MatchString(cc) { + errs = append(errs, fmt.Errorf("%w: %s", ErrNotCountryCode, cc)) + } + + g.Countries[i] = strings.ToLower(cc) + } + + if len(errs) != 0 { + return fmt.Errorf("bot.GeoIP: invalid GeoIP settings: %w", errors.Join(errs...)) + } + + return nil +} diff --git a/lib/policy/config/testdata/good/challenge_cloudflare.yaml b/lib/policy/config/testdata/good/challenge_cloudflare.yaml new file mode 100644 index 0000000..1c728cb --- /dev/null +++ b/lib/policy/config/testdata/good/challenge_cloudflare.yaml @@ -0,0 +1,6 @@ +bots: + - name: challenge-cloudflare + action: CHALLENGE + asns: + match: + - 13335 # Cloudflare diff --git a/lib/policy/config/testdata/good/geoip_us.yaml b/lib/policy/config/testdata/good/geoip_us.yaml new file mode 100644 index 0000000..b5e4280 --- /dev/null +++ b/lib/policy/config/testdata/good/geoip_us.yaml @@ -0,0 +1,6 @@ +bots: + - name: compute-tarrif-us + action: CHALLENGE + geoip: + countries: + - US diff --git a/lib/policy/policy.go b/lib/policy/policy.go index d67ca1c..aed30d1 100644 --- a/lib/policy/policy.go +++ b/lib/policy/policy.go @@ -1,10 +1,14 @@ package policy import ( + "context" "errors" "fmt" "io" + "log/slog" + "github.com/TecharoHQ/anubis/internal/thoth" + "github.com/TecharoHQ/anubis/lib/policy/checker" "github.com/TecharoHQ/anubis/lib/policy/config" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -35,7 +39,7 @@ func NewParsedConfig(orig *config.Config) *ParsedConfig { } } -func ParseConfig(fin io.Reader, fname string, defaultDifficulty int) (*ParsedConfig, error) { +func ParseConfig(ctx context.Context, fin io.Reader, fname string, defaultDifficulty int) (*ParsedConfig, error) { c, err := config.Load(fin, fname) if err != nil { return nil, err @@ -43,6 +47,8 @@ func ParseConfig(fin io.Reader, fname string, defaultDifficulty int) (*ParsedCon var validationErrs []error + tc, hasThothClient := thoth.FromContext(ctx) + result := NewParsedConfig(c) result.DefaultDifficulty = defaultDifficulty @@ -57,7 +63,7 @@ func ParseConfig(fin io.Reader, fname string, defaultDifficulty int) (*ParsedCon Action: b.Action, } - cl := CheckerList{} + cl := checker.List{} if len(b.RemoteAddr) > 0 { c, err := NewRemoteAddrChecker(b.RemoteAddr) @@ -104,6 +110,24 @@ func ParseConfig(fin io.Reader, fname string, defaultDifficulty int) (*ParsedCon } } + if b.ASNs != nil { + if !hasThothClient { + slog.Warn("You have specified a Thoth specific check but you have no Thoth client configured. Please read https://anubis.techaro.lol/docs/admin/thoth for more information", "check", "asn", "settings", b.ASNs) + continue + } + + cl = append(cl, tc.ASNCheckerFor(b.ASNs.Match)) + } + + if b.GeoIP != nil { + if !hasThothClient { + slog.Warn("You have specified a Thoth specific check but you have no Thoth client configured. Please read https://anubis.techaro.lol/docs/admin/thoth for more information", "check", "geoip", "settings", b.GeoIP) + continue + } + + cl = append(cl, tc.GeoIPCheckerFor(b.GeoIP.Countries)) + } + if b.Challenge == nil { parsedBot.Challenge = &config.ChallengeRules{ Difficulty: defaultDifficulty, diff --git a/lib/policy/policy_test.go b/lib/policy/policy_test.go index 16ca9c7..9ada1c9 100644 --- a/lib/policy/policy_test.go +++ b/lib/policy/policy_test.go @@ -7,21 +7,25 @@ import ( "github.com/TecharoHQ/anubis" "github.com/TecharoHQ/anubis/data" + "github.com/TecharoHQ/anubis/internal/thoth/thothmock" ) func TestDefaultPolicyMustParse(t *testing.T) { + ctx := thothmock.WithMockThoth(t) + fin, err := data.BotPolicies.Open("botPolicies.json") if err != nil { t.Fatal(err) } defer fin.Close() - if _, err := ParseConfig(fin, "botPolicies.json", anubis.DefaultDifficulty); err != nil { + if _, err := ParseConfig(ctx, fin, "botPolicies.json", anubis.DefaultDifficulty); err != nil { t.Fatalf("can't parse config: %v", err) } } func TestGoodConfigs(t *testing.T) { + finfos, err := os.ReadDir("config/testdata/good") if err != nil { t.Fatal(err) @@ -30,20 +34,37 @@ func TestGoodConfigs(t *testing.T) { for _, st := range finfos { st := st t.Run(st.Name(), func(t *testing.T) { - fin, err := os.Open(filepath.Join("config", "testdata", "good", st.Name())) - if err != nil { - t.Fatal(err) - } - defer fin.Close() + t.Run("with-thoth", func(t *testing.T) { + fin, err := os.Open(filepath.Join("config", "testdata", "good", st.Name())) + if err != nil { + t.Fatal(err) + } + defer fin.Close() - if _, err := ParseConfig(fin, fin.Name(), anubis.DefaultDifficulty); err != nil { - t.Fatal(err) - } + ctx := thothmock.WithMockThoth(t) + if _, err := ParseConfig(ctx, fin, fin.Name(), anubis.DefaultDifficulty); err != nil { + t.Fatal(err) + } + }) + + t.Run("without-thoth", func(t *testing.T) { + fin, err := os.Open(filepath.Join("config", "testdata", "good", st.Name())) + if err != nil { + t.Fatal(err) + } + defer fin.Close() + + if _, err := ParseConfig(t.Context(), fin, fin.Name(), anubis.DefaultDifficulty); err != nil { + t.Fatal(err) + } + }) }) } } func TestBadConfigs(t *testing.T) { + ctx := thothmock.WithMockThoth(t) + finfos, err := os.ReadDir("config/testdata/bad") if err != nil { t.Fatal(err) @@ -58,7 +79,7 @@ func TestBadConfigs(t *testing.T) { } defer fin.Close() - if _, err := ParseConfig(fin, fin.Name(), anubis.DefaultDifficulty); err == nil { + if _, err := ParseConfig(ctx, fin, fin.Name(), anubis.DefaultDifficulty); err == nil { t.Fatal(err) } else { t.Log(err)