mirror of
https://github.com/TecharoHQ/anubis.git
synced 2025-08-03 01:38:14 -04:00
10 lines
316 B
YAML
10 lines
316 B
YAML
# Screaming Frog is a smaller actor in the SEO space and their crawler
|
|
# occasionally attempts to access content despite being explicitly excluded via
|
|
# robots.txt directives.
|
|
#
|
|
# Company website: https://www.screamingfrog.co.uk
|
|
|
|
- name: screaming-frog
|
|
user_agent_regex: ^Screaming Frog SEO Spider/
|
|
action: DENY
|