TheCloud/anubis
1
0
Fork 0
mirror of https://github.com/TecharoHQ/anubis.git synced 2025-08-03 01:38:14 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix: remove unused blocks

Browse Source
This commit is contained in:
Jason Cameron 2025-07-26 20:49:54 -04:00
parent 61db9a618d
commit 2e7f37215c
No known key found for this signature in database
GPG Key ID: 0AA00EA6258337A5
1 changed files with 1 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
cmd/robots2policy/main.go
View File

@ -29,7 +29,7 @@ var (
)
type RobotsRule struct {
UserAgents []string
UserAgents []string
Disallows []string
Allows []string
CrawlDelay int
@ -333,15 +333,6 @@ func convertToAnubisRules(robotsRules []RobotsRule) []AnubisRule {
} else if len(userAgents) == 1 {
conditions = append(conditions, fmt.Sprintf("userAgent.contains(%q)", userAgents[0]))
} else {
// Multiple user agents - use any block for user agents
var uaExpressions []string
for _, ua := range userAgents {
if ua == "*" {
uaExpressions = append(uaExpressions, "true")
} else {
uaExpressions = append(uaExpressions, fmt.Sprintf("userAgent.contains(%q)", ua))
}
}
// For multiple user agents, we need to use a more complex expression
// This is a limitation - we can't easily combine any for user agents with all for path
// So we'll create separate rules for each user agent