From 2f1e78cc6c6d645a9f1f5a50ad8b4b8ce24f1e4d Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Fri, 4 Jul 2025 00:22:43 +0000 Subject: [PATCH] chore(docs/manifest): allow common crawl to test with the team Signed-off-by: Xe Iaso --- docs/manifest/cfg/anubis/botPolicies.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/manifest/cfg/anubis/botPolicies.yaml b/docs/manifest/cfg/anubis/botPolicies.yaml index 0abf0dc..3808992 100644 --- a/docs/manifest/cfg/anubis/botPolicies.yaml +++ b/docs/manifest/cfg/anubis/botPolicies.yaml @@ -11,6 +11,7 @@ ## /usr/share/docs/anubis/data or in the tarball you extracted Anubis from. bots: + - import: (data)/crawlers/commoncrawl.yaml # Pathological bots to deny - # This correlates to data/bots/deny-pathological.yaml in the source tree # https://github.com/TecharoHQ/anubis/blob/main/data/bots/deny-pathological.yaml