Add headers bot rule (#300)

* Closes #291: add headers support to bot policy rules

* Fix config validator

docs/docs/CHANGELOG.md

@@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added example nginx configuration to documentation
 - Added example Apache configuration to the documentation [#277](https://github.com/TecharoHQ/anubis/issues/277)
 - Move per-environment configuration details into their own pages
+- Added headers support to bot policy rules
 
 ## v1.16.0
 

lib/anubis.go

@@ -548,6 +548,12 @@ func (s *Server) check(r *http.Request) (CheckResult, *policy.Bot, error) {
 				return cr("bot/"+b.Name, b.Action), &b, nil
 			}
 		}
+
+		if len(b.Headers) > 0 {
+			if s.checkHeaders(b, r.Header) {
+				return cr("bot/"+b.Name, b.Action), &b, nil
+			}
+		}
 	}
 
 	return cr("default/allow", config.RuleAllow), &policy.Bot{
@@ -572,6 +578,27 @@ func (s *Server) checkRemoteAddress(b policy.Bot, addr net.IP) bool {
 	return ok
 }
 
+func (s *Server) checkHeaders(b policy.Bot, header http.Header) bool {
+	if len(b.Headers) == 0 {
+		return true
+	}
+
+	for name, expr := range b.Headers {
+		values := header.Values(name)
+		if values == nil {
+			return false
+		}
+
+		for _, value := range values {
+			if !expr.MatchString(value) {
+				return false
+			}
+		}
+	}
+
+	return true
+}
+
 func (s *Server) CleanupDecayMap() {
 	s.DNSBLCache.Cleanup()
 	s.OGTags.Cleanup()

lib/policy/bot.go

@@ -3,6 +3,7 @@ package policy
 import (
 	"fmt"
 	"regexp"
+	"strings"
 
 	"github.com/TecharoHQ/anubis/internal"
 	"github.com/TecharoHQ/anubis/lib/policy/config"
@@ -13,6 +14,7 @@ type Bot struct {
 	Name      string
 	UserAgent *regexp.Regexp
 	Path      *regexp.Regexp
+	Headers   map[string]*regexp.Regexp
 	Action    config.Rule `json:"action"`
 	Challenge *config.ChallengeRules
 	Ranger    cidranger.Ranger
@@ -27,6 +29,18 @@ func (b Bot) Hash() (string, error) {
 	if b.UserAgent != nil {
 		userAgentRex = b.UserAgent.String()
 	}
+	var headersRex string
+	if len(b.Headers) > 0 {
+		var sb strings.Builder
+		sb.Grow(len(b.Headers) * 64)
 
-	return internal.SHA256sum(fmt.Sprintf("%s::%s::%s", b.Name, pathRex, userAgentRex)), nil
+		for name, expr := range b.Headers {
+			sb.WriteString(name)
+			sb.WriteString(expr.String())
+		}
+
+		headersRex = sb.String()
+	}
+
+	return internal.SHA256sum(fmt.Sprintf("%s::%s::%s::%s", b.Name, pathRex, userAgentRex, headersRex)), nil
 }

lib/policy/config/config.go

@@ -10,11 +10,12 @@ import (
 var (
 	ErrNoBotRulesDefined                 = errors.New("config: must define at least one (1) bot rule")
 	ErrBotMustHaveName                   = errors.New("config.Bot: must set name")
-	ErrBotMustHaveUserAgentOrPath        = errors.New("config.Bot: must set either user_agent_regex, path_regex, or remote_addresses")
+	ErrBotMustHaveUserAgentOrPath        = errors.New("config.Bot: must set either user_agent_regex, path_regex, headers_regex, or remote_addresses")
 	ErrBotMustHaveUserAgentOrPathNotBoth = errors.New("config.Bot: must set either user_agent_regex, path_regex, and not both")
 	ErrUnknownAction                     = errors.New("config.Bot: unknown action")
 	ErrInvalidUserAgentRegex             = errors.New("config.Bot: invalid user agent regex")
 	ErrInvalidPathRegex                  = errors.New("config.Bot: invalid path regex")
+	ErrInvalidHeadersRegex               = errors.New("config.Bot: invalid headers regex")
 	ErrInvalidCIDR                       = errors.New("config.Bot: invalid CIDR")
 )
 
@@ -37,12 +38,13 @@ const (
 )
 
 type BotConfig struct {
-	Name           string          `json:"name"`
+	Name           string            `json:"name"`
-	UserAgentRegex *string         `json:"user_agent_regex"`
+	UserAgentRegex *string           `json:"user_agent_regex"`
-	PathRegex      *string         `json:"path_regex"`
+	PathRegex      *string           `json:"path_regex"`
-	Action         Rule            `json:"action"`
+	HeadersRegex   map[string]string `json:"headers_regex"`
-	RemoteAddr     []string        `json:"remote_addresses"`
+	Action         Rule              `json:"action"`
-	Challenge      *ChallengeRules `json:"challenge,omitempty"`
+	RemoteAddr     []string          `json:"remote_addresses"`
+	Challenge      *ChallengeRules   `json:"challenge,omitempty"`
 }
 
 func (b BotConfig) Valid() error {
@@ -52,7 +54,7 @@ func (b BotConfig) Valid() error {
 		errs = append(errs, ErrBotMustHaveName)
 	}
 
-	if b.UserAgentRegex == nil && b.PathRegex == nil && len(b.RemoteAddr) == 0 {
+	if b.UserAgentRegex == nil && b.PathRegex == nil && len(b.RemoteAddr) == 0 && len(b.HeadersRegex) == 0 {
 		errs = append(errs, ErrBotMustHaveUserAgentOrPath)
 	}
 
@@ -72,6 +74,18 @@ func (b BotConfig) Valid() error {
 		}
 	}
 
+	if len(b.HeadersRegex) > 0 {
+		for name, expr := range b.HeadersRegex {
+			if name == "" {
+				continue
+			}
+
+			if _, err := regexp.Compile(expr); err != nil {
+				errs = append(errs, ErrInvalidHeadersRegex, err)
+			}
+		}
+	}
+
 	if len(b.RemoteAddr) > 0 {
 		for _, cidr := range b.RemoteAddr {
 			if _, _, err := net.ParseCIDR(cidr); err != nil {

lib/policy/config/config_test.go

@@ -87,6 +87,18 @@ func TestBotValid(t *testing.T) {
 			},
 			err: ErrInvalidPathRegex,
 		},
+		{
+			name: "invalid headers regex",
+			bot: BotConfig{
+				Name:   "mozilla-ua",
+				Action: RuleChallenge,
+				HeadersRegex: map[string]string{
+					"Content-Type": "a(b",
+				},
+				PathRegex: p("a(b"),
+			},
+			err: ErrInvalidHeadersRegex,
+		},
 		{
 			name: "challenge difficulty too low",
 			bot: BotConfig{

lib/policy/config/testdata/bad/badregexes.json

@@ -9,6 +9,13 @@
       "name": "user-agent-bad",
       "user_agent_regex": "a(b",
       "action": "DENY"
+    },
+    {
+      "name": "headers-bad",
+      "headers": {
+        "Accept-Encoding": "a(b"
+      },
+      "action": "DENY"
     }
   ]
 }

lib/policy/config/testdata/good/block_cf_workers.json

@@ -0,0 +1,12 @@
+{
+  "bots": [
+    {
+      "name": "Cloudflare Workers",
+      "headers_regex": {
+        "CF-Worker": ".*"
+      },
+      "action": "DENY"
+    }
+  ],
+  "dnsbl": false
+}

lib/policy/policy.go

@@ -58,8 +58,9 @@ func ParseConfig(fin io.Reader, fname string, defaultDifficulty int) (*ParsedCon
 		}
 
 		parsedBot := Bot{
-			Name:   b.Name,
+			Name:    b.Name,
-			Action: b.Action,
+			Action:  b.Action,
+			Headers: map[string]*regexp.Regexp{},
 		}
 
 		if len(b.RemoteAddr) > 0 {
@@ -95,6 +96,22 @@ func ParseConfig(fin io.Reader, fname string, defaultDifficulty int) (*ParsedCon
 			}
 		}
 
+		if len(b.HeadersRegex) > 0 {
+			for name, expr := range b.HeadersRegex {
+				if name == "" {
+					continue
+				}
+
+				header, err := regexp.Compile(expr)
+				if err != nil {
+					validationErrs = append(validationErrs, fmt.Errorf("while compiling header regexp: %w", err))
+					continue
+				} else {
+					parsedBot.Headers[name] = header
+				}
+			}
+		}
+
 		if b.Challenge == nil {
 			parsedBot.Challenge = &config.ChallengeRules{
 				Difficulty: defaultDifficulty,

web/index_templ.go

@@ -96,7 +96,7 @@ func base(title string, body templ.Component, challenge any, ogTags map[string]s
 				return templ_7745c5c3_Err
 			}
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "<style>\n        body,\n        html {\n            height: 100%;\n            display: flex;\n            justify-content: center;\n            align-items: center;\n            margin-left: auto;\n            margin-right: auto;\n        }\n\n        .centered-div {\n            text-align: center;\n        }\n\n        #status {\n            font-variant-numeric: tabular-nums;\n        }\n\n        #progress {\n            display: none;\n            width: min(20rem, 90%);\n            height: 2rem;\n            border-radius: 1rem;\n            overflow: hidden;\n            margin: 1rem 0 2rem;\n            outline-color: #b16286;\n            outline-offset: 2px;\n            outline-style: solid;\n            outline-width: 4px;\n        }\n\n        .bar-inner {\n            background-color: #b16286;\n            height: 100%;\n            width: 0;\n            transition: width 0.25s ease-in;\n        }\n    </style>")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "<style>\n        body,\n        html {\n            height: 100%;\n            display: flex;\n            justify-content: center;\n            align-items: center;\n            margin-left: auto;\n            margin-right: auto;\n        }\n\n        .centered-div {\n            text-align: center;\n        }\n\n        #status {\n            font-variant-numeric: tabular-nums;\n        }\n\n        #progress {\n            display: none;\n            width: min(20rem, 90%);\n            height: 2rem;\n            border-radius: 1rem;\n            overflow: hidden;\n            margin: 1rem 0 2rem;\n            outline-offset: 2px;\n            outline: #b16286 solid 4px;\n        }\n\n        .bar-inner {\n            background-color: #b16286;\n            height: 100%;\n            width: 0;\n            transition: width 0.25s ease-in;\n        }\n    </style>")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
@@ -117,7 +117,7 @@ func base(title string, body templ.Component, challenge any, ogTags map[string]s
 		var templ_7745c5c3_Var6 string
 		templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(title)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 67, Col: 52}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 65, Col: 52}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
 		if templ_7745c5c3_Err != nil {
@@ -168,7 +168,7 @@ func index() templ.Component {
 		templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs("/.within.website/x/cmd/anubis/static/img/pensive.webp?cacheBuster=" +
 			anubis.Version)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 91, Col: 18}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 89, Col: 18}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
 		if templ_7745c5c3_Err != nil {
@@ -182,7 +182,7 @@ func index() templ.Component {
 		templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs("/.within.website/x/cmd/anubis/static/img/happy.webp?cacheBuster=" +
 			anubis.Version)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 97, Col: 18}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 95, Col: 18}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
 		if templ_7745c5c3_Err != nil {
@@ -196,7 +196,7 @@ func index() templ.Component {
 		templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(
 			"/.within.website/x/cmd/anubis/static/js/main.mjs?cacheBuster=" + anubis.Version)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 101, Col: 84}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 99, Col: 84}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
 		if templ_7745c5c3_Err != nil {
@@ -238,7 +238,7 @@ func errorPage(message string, mail string) templ.Component {
 		var templ_7745c5c3_Var12 string
 		templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs("/.within.website/x/cmd/anubis/static/img/reject.webp?cacheBuster=" + anubis.Version)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 141, Col: 102}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 139, Col: 102}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12))
 		if templ_7745c5c3_Err != nil {
@@ -251,7 +251,7 @@ func errorPage(message string, mail string) templ.Component {
 		var templ_7745c5c3_Var13 string
 		templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(message)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 143, Col: 16}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 141, Col: 16}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
 		if templ_7745c5c3_Err != nil {
@@ -278,7 +278,7 @@ func errorPage(message string, mail string) templ.Component {
 			var templ_7745c5c3_Var15 string
 			templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(mail)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 147, Col: 9}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 145, Col: 9}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15))
 			if templ_7745c5c3_Err != nil {
@@ -331,7 +331,7 @@ func bench() templ.Component {
 		templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs("/.within.website/x/cmd/anubis/static/img/pensive.webp?cacheBuster=" +
 			anubis.Version)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 178, Col: 22}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 176, Col: 22}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
 		if templ_7745c5c3_Err != nil {
@@ -345,7 +345,7 @@ func bench() templ.Component {
 		templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(
 			"/.within.website/x/cmd/anubis/static/js/bench.mjs?cacheBuster=" + anubis.Version)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 182, Col: 89}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 180, Col: 89}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
 		if templ_7745c5c3_Err != nil {