From 3f1ce2d7ac4e3d5e438ac0d83cd6bf29f3904ff5 Mon Sep 17 00:00:00 2001
From: Xe Iaso <me@xeiaso.net>
Date: Tue, 22 Apr 2025 08:11:45 -0400
Subject: [PATCH] data: disable generic-bot-catchall by default (#322)

Signed-off-by: Xe Iaso <me@xeiaso.net>
---
 data/botPolicies.json        | 10 ----------
 data/botPolicies.yaml        | 16 ++++++++--------
 docs/docs/CHANGELOG.md       |  1 +
 docs/docs/admin/policies.mdx |  4 ++++
 4 files changed, 13 insertions(+), 18 deletions(-)

diff --git a/data/botPolicies.json b/data/botPolicies.json
index 1993d22..72d38dc 100644
--- a/data/botPolicies.json
+++ b/data/botPolicies.json
@@ -670,16 +670,6 @@
       "user_agent_regex": "HeadlessChromium",
       "action": "DENY"
     },
-    {
-      "name": "generic-bot-catchall",
-      "user_agent_regex": "(?i:bot|crawler)",
-      "action": "CHALLENGE",
-      "challenge": {
-        "difficulty": 16,
-        "report_as": 4,
-        "algorithm": "slow"
-      }
-    },
     {
       "name": "generic-browser",
       "user_agent_regex": "Mozilla|Opera",
diff --git a/data/botPolicies.yaml b/data/botPolicies.yaml
index e8448ac..cb4715a 100644
--- a/data/botPolicies.yaml
+++ b/data/botPolicies.yaml
@@ -645,14 +645,14 @@ bots:
   path_regex: ^/robots.txt$
   action: ALLOW
 
-# Punish any bot with "bot" in the user-agent string
-- name: generic-bot-catchall
-  user_agent_regex: (?i:bot|crawler)
-  action: CHALLENGE
-  challenge:
-    difficulty: 16  # impossible
-    report_as: 4    # lie to the operator
-    algorithm: slow # intentionally waste CPU cycles and time
+# # Punish any bot with "bot" in the user-agent string
+# - name: generic-bot-catchall
+#   user_agent_regex: (?i:bot|crawler)
+#   action: CHALLENGE
+#   challenge:
+#     difficulty: 16  # impossible
+#     report_as: 4    # lie to the operator
+#     algorithm: slow # intentionally waste CPU cycles and time
 
 - name: generic-browser
   user_agent_regex: >
diff --git a/docs/docs/CHANGELOG.md b/docs/docs/CHANGELOG.md
index 6a90c81..8a3e410 100644
--- a/docs/docs/CHANGELOG.md
+++ b/docs/docs/CHANGELOG.md
@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added headers support to bot policy rules
 - Moved configuration file from JSON to YAML by default
 - Added documentation on how to use Anubis with Traefik in Docker
+- Disable `generic-bot-catchall` rule because of its high false positive rate in real-world scenarios
 
 ## v1.16.0
 
diff --git a/docs/docs/admin/policies.mdx b/docs/docs/admin/policies.mdx
index a5f6f1e..11af725 100644
--- a/docs/docs/admin/policies.mdx
+++ b/docs/docs/admin/policies.mdx
@@ -132,6 +132,8 @@ Rules can also have their own challenge settings. These are customized using the
 <Tabs>
 <TabItem value="json" label="JSON" default>
 
+This rule has been known to have a high false positive rate in testing. Please use this with care.
+
 ```json
 {
   "name": "generic-bot-catchall",
@@ -148,6 +150,8 @@ Rules can also have their own challenge settings. These are customized using the
 </TabItem>
 <TabItem value="yaml" label="YAML">
 
+This rule has been known to have a high false positive rate in testing. Please use this with care.
+
 ```yaml
 # Punish any bot with "bot" in the user-agent string
 - name: generic-bot-catchall