TheCloud/anubis
1
0
Fork 0
mirror of https://github.com/TecharoHQ/anubis.git synced 2025-08-03 09:48:08 -04:00
Code Issues Packages Projects Releases Wiki Activity

Add headers bot rule (#300)

Browse Source 
* Closes #291: add headers support to bot policy rules

* Fix config validator
This commit is contained in:
Neur0toxine 2025-04-21 01:18:21 +03:00 committed by GitHub
parent 1add24b907
commit 7dc545cfa9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 125 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/docs/CHANGELOG.md
View File

@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Added example nginx configuration to documentation
- Added example Apache configuration to the documentation [#277](https://github.com/TecharoHQ/anubis/issues/277)
- Move per-environment configuration details into their own pages
- Added headers support to bot policy rules
## v1.16.0

27
lib/anubis.go
View File

@ -548,6 +548,12 @@ func (s *Server) check(r *http.Request) (CheckResult, *policy.Bot, error) {
return cr("bot/"+b.Name, b.Action), &b, nil
}
}
if len(b.Headers) > 0 {
if s.checkHeaders(b, r.Header) {
return cr("bot/"+b.Name, b.Action), &b, nil
}
}
}
return cr("default/allow", config.RuleAllow), &policy.Bot{
@ -572,6 +578,27 @@ func (s *Server) checkRemoteAddress(b policy.Bot, addr net.IP) bool {
return ok
}
func (s *Server) checkHeaders(b policy.Bot, header http.Header) bool {
if len(b.Headers) == 0 {
return true
}
for name, expr := range b.Headers {
values := header.Values(name)
if values == nil {
return false
}
for _, value := range values {
if !expr.MatchString(value) {
return false
}
}
}
return true
}
func (s *Server) CleanupDecayMap() {
s.DNSBLCache.Cleanup()
s.OGTags.Cleanup()

16
lib/policy/bot.go
View File

@ -3,6 +3,7 @@ package policy
import (
"fmt"
"regexp"
"strings"
"github.com/TecharoHQ/anubis/internal"
"github.com/TecharoHQ/anubis/lib/policy/config"
@ -13,6 +14,7 @@ type Bot struct {
Name string
UserAgent *regexp.Regexp
Path *regexp.Regexp
Headers map[string]*regexp.Regexp
Action config.Rule `json:"action"`
Challenge *config.ChallengeRules
Ranger cidranger.Ranger
@ -27,6 +29,18 @@ func (b Bot) Hash() (string, error) {
if b.UserAgent != nil {
userAgentRex = b.UserAgent.String()
}
var headersRex string
if len(b.Headers) > 0 {
var sb strings.Builder
sb.Grow(len(b.Headers) * 64)
return internal.SHA256sum(fmt.Sprintf("%s::%s::%s", b.Name, pathRex, userAgentRex)), nil
for name, expr := range b.Headers {
sb.WriteString(name)
sb.WriteString(expr.String())
}
headersRex = sb.String()
}
return internal.SHA256sum(fmt.Sprintf("%s::%s::%s::%s", b.Name, pathRex, userAgentRex, headersRex)), nil
}

30
lib/policy/config/config.go
View File

@ -10,11 +10,12 @@ import (
var (
ErrNoBotRulesDefined = errors.New("config: must define at least one (1) bot rule")
ErrBotMustHaveName = errors.New("config.Bot: must set name")
ErrBotMustHaveUserAgentOrPath = errors.New("config.Bot: must set either user_agent_regex, path_regex, or remote_addresses")
ErrBotMustHaveUserAgentOrPath = errors.New("config.Bot: must set either user_agent_regex, path_regex, headers_regex, or remote_addresses")
ErrBotMustHaveUserAgentOrPathNotBoth = errors.New("config.Bot: must set either user_agent_regex, path_regex, and not both")
ErrUnknownAction = errors.New("config.Bot: unknown action")
ErrInvalidUserAgentRegex = errors.New("config.Bot: invalid user agent regex")
ErrInvalidPathRegex = errors.New("config.Bot: invalid path regex")
ErrInvalidHeadersRegex = errors.New("config.Bot: invalid headers regex")
ErrInvalidCIDR = errors.New("config.Bot: invalid CIDR")
)
@ -37,12 +38,13 @@ const (
)
type BotConfig struct {
Name string `json:"name"`
UserAgentRegex *string `json:"user_agent_regex"`
PathRegex *string `json:"path_regex"`
Action Rule `json:"action"`
RemoteAddr []string `json:"remote_addresses"`
Challenge *ChallengeRules `json:"challenge,omitempty"`
Name string `json:"name"`
UserAgentRegex *string `json:"user_agent_regex"`
PathRegex *string `json:"path_regex"`
HeadersRegex map[string]string `json:"headers_regex"`
Action Rule `json:"action"`
RemoteAddr []string `json:"remote_addresses"`
Challenge *ChallengeRules `json:"challenge,omitempty"`
}
func (b BotConfig) Valid() error {
@ -52,7 +54,7 @@ func (b BotConfig) Valid() error {
errs = append(errs, ErrBotMustHaveName)
}
if b.UserAgentRegex == nil && b.PathRegex == nil && len(b.RemoteAddr) == 0 {
if b.UserAgentRegex == nil && b.PathRegex == nil && len(b.RemoteAddr) == 0 && len(b.HeadersRegex) == 0 {
errs = append(errs, ErrBotMustHaveUserAgentOrPath)
}
@ -72,6 +74,18 @@ func (b BotConfig) Valid() error {
}
}
if len(b.HeadersRegex) > 0 {
for name, expr := range b.HeadersRegex {
if name == "" {
continue
}
if _, err := regexp.Compile(expr); err != nil {
errs = append(errs, ErrInvalidHeadersRegex, err)
}
}
}
if len(b.RemoteAddr) > 0 {
for _, cidr := range b.RemoteAddr {
if _, _, err := net.ParseCIDR(cidr); err != nil {

12
lib/policy/config/config_test.go
View File

@ -87,6 +87,18 @@ func TestBotValid(t *testing.T) {
},
err: ErrInvalidPathRegex,
},
{
name: "invalid headers regex",
bot: BotConfig{
Name: "mozilla-ua",
Action: RuleChallenge,
HeadersRegex: map[string]string{
"Content-Type": "a(b",
},
PathRegex: p("a(b"),
},
err: ErrInvalidHeadersRegex,
},
{
name: "challenge difficulty too low",
bot: BotConfig{

7
lib/policy/config/testdata/bad/badregexes.json vendored
View File

@ -9,6 +9,13 @@
"name": "user-agent-bad",
"user_agent_regex": "a(b",
"action": "DENY"
},
{
"name": "headers-bad",
"headers": {
"Accept-Encoding": "a(b"
},
"action": "DENY"
}
]
}

12
lib/policy/config/testdata/good/block_cf_workers.json vendored Normal file
View File

@ -0,0 +1,12 @@
{
"bots": [
{
"name": "Cloudflare Workers",
"headers_regex": {
"CF-Worker": ".*"
},
"action": "DENY"
}
],
"dnsbl": false
}

21
lib/policy/policy.go
View File

@ -58,8 +58,9 @@ func ParseConfig(fin io.Reader, fname string, defaultDifficulty int) (*ParsedCon
}
parsedBot := Bot{
Name: b.Name,
Action: b.Action,
Name: b.Name,
Action: b.Action,
Headers: map[string]*regexp.Regexp{},
}
if len(b.RemoteAddr) > 0 {
@ -95,6 +96,22 @@ func ParseConfig(fin io.Reader, fname string, defaultDifficulty int) (*ParsedCon
}
}
if len(b.HeadersRegex) > 0 {
for name, expr := range b.HeadersRegex {
if name == "" {
continue
}
header, err := regexp.Compile(expr)
if err != nil {
validationErrs = append(validationErrs, fmt.Errorf("while compiling header regexp: %w", err))
continue
} else {
parsedBot.Headers[name] = header
}
}
}
if b.Challenge == nil {
parsedBot.Challenge = &config.ChallengeRules{
Difficulty: defaultDifficulty,

20
web/index_templ.go generated
View File

@ -96,7 +96,7 @@ func base(title string, body templ.Component, challenge any, ogTags map[string]s
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "<style>\n body,\n html {\n height: 100%;\n display: flex;\n justify-content: center;\n align-items: center;\n margin-left: auto;\n margin-right: auto;\n }\n\n .centered-div {\n text-align: center;\n }\n\n #status {\n font-variant-numeric: tabular-nums;\n }\n\n #progress {\n display: none;\n width: min(20rem, 90%);\n height: 2rem;\n border-radius: 1rem;\n overflow: hidden;\n margin: 1rem 0 2rem;\n outline-color: #b16286;\n outline-offset: 2px;\n outline-style: solid;\n outline-width: 4px;\n }\n\n .bar-inner {\n background-color: #b16286;\n height: 100%;\n width: 0;\n transition: width 0.25s ease-in;\n }\n </style>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "<style>\n body,\n html {\n height: 100%;\n display: flex;\n justify-content: center;\n align-items: center;\n margin-left: auto;\n margin-right: auto;\n }\n\n .centered-div {\n text-align: center;\n }\n\n #status {\n font-variant-numeric: tabular-nums;\n }\n\n #progress {\n display: none;\n width: min(20rem, 90%);\n height: 2rem;\n border-radius: 1rem;\n overflow: hidden;\n margin: 1rem 0 2rem;\n outline-offset: 2px;\n outline: #b16286 solid 4px;\n }\n\n .bar-inner {\n background-color: #b16286;\n height: 100%;\n width: 0;\n transition: width 0.25s ease-in;\n }\n </style>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@ -117,7 +117,7 @@ func base(title string, body templ.Component, challenge any, ogTags map[string]s
var templ_7745c5c3_Var6 string
templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(title)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 67, Col: 52}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 65, Col: 52}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
if templ_7745c5c3_Err != nil {
@ -168,7 +168,7 @@ func index() templ.Component {
templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs("/.within.website/x/cmd/anubis/static/img/pensive.webp?cacheBuster=" +
anubis.Version)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 91, Col: 18}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 89, Col: 18}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
if templ_7745c5c3_Err != nil {
@ -182,7 +182,7 @@ func index() templ.Component {
templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs("/.within.website/x/cmd/anubis/static/img/happy.webp?cacheBuster=" +
anubis.Version)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 97, Col: 18}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 95, Col: 18}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
if templ_7745c5c3_Err != nil {
@ -196,7 +196,7 @@ func index() templ.Component {
templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(
"/.within.website/x/cmd/anubis/static/js/main.mjs?cacheBuster=" + anubis.Version)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 101, Col: 84}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 99, Col: 84}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
if templ_7745c5c3_Err != nil {
@ -238,7 +238,7 @@ func errorPage(message string, mail string) templ.Component {
var templ_7745c5c3_Var12 string
templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs("/.within.website/x/cmd/anubis/static/img/reject.webp?cacheBuster=" + anubis.Version)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 141, Col: 102}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 139, Col: 102}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12))
if templ_7745c5c3_Err != nil {
@ -251,7 +251,7 @@ func errorPage(message string, mail string) templ.Component {
var templ_7745c5c3_Var13 string
templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(message)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 143, Col: 16}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 141, Col: 16}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
if templ_7745c5c3_Err != nil {
@ -278,7 +278,7 @@ func errorPage(message string, mail string) templ.Component {
var templ_7745c5c3_Var15 string
templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(mail)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 147, Col: 9}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 145, Col: 9}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15))
if templ_7745c5c3_Err != nil {
@ -331,7 +331,7 @@ func bench() templ.Component {
templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs("/.within.website/x/cmd/anubis/static/img/pensive.webp?cacheBuster=" +
anubis.Version)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 178, Col: 22}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 176, Col: 22}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
if templ_7745c5c3_Err != nil {
@ -345,7 +345,7 @@ func bench() templ.Component {
templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(
"/.within.website/x/cmd/anubis/static/js/bench.mjs?cacheBuster=" + anubis.Version)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 182, Col: 89}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 180, Col: 89}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
if templ_7745c5c3_Err != nil {