From 81307bcb5c2100afac3b035d9550f78d67c54863 Mon Sep 17 00:00:00 2001 From: Maher <114317937+thatonecodes@users.noreply.github.com> Date: Fri, 11 Apr 2025 22:13:38 -0400 Subject: [PATCH] feat: update botPolicies for DuckDuckGo web crawler (#250) - updates botPolicies with ips from the website - adds the updated information to the `CHANGELOG.md` file Signed-off-by: Xe Iaso Co-authored-by: Xe Iaso --- data/botPolicies.json | 278 ++++++++++++++++++++++++++++++++++++++++- docs/docs/CHANGELOG.md | 1 + 2 files changed, 278 insertions(+), 1 deletion(-) diff --git a/data/botPolicies.json b/data/botPolicies.json index d0e27a2..7622e47 100644 --- a/data/botPolicies.json +++ b/data/botPolicies.json @@ -304,6 +304,282 @@ "40.77.178.0/23" ] }, + { + "name": "duckduckbot", + "user_agent_regex": "\\+http\\://duckduckgo\\.com/duckduckbot\\.html", + "action": "ALLOW", + "remote_addresses": [ + "57.152.72.128/32", + "51.8.253.152/32", + "40.80.242.63/32", + "20.12.141.99/32", + "20.49.136.28/32", + "51.116.131.221/32", + "51.107.40.209/32", + "20.40.133.240/32", + "20.50.168.91/32", + "51.120.48.122/32", + "20.193.45.113/32", + "40.76.173.151/32", + "40.76.163.7/32", + "20.185.79.47/32", + "52.142.26.175/32", + "20.185.79.15/32", + "52.142.24.149/32", + "40.76.162.208/32", + "40.76.163.23/32", + "40.76.162.191/32", + "40.76.162.247/32", + "40.88.21.235/32", + "20.191.45.212/32", + "52.146.59.12/32", + "52.146.59.156/32", + "52.146.59.154/32", + "52.146.58.236/32", + "20.62.224.44/32", + "51.104.180.53/32", + "51.104.180.47/32", + "51.104.180.26/32", + "51.104.146.225/32", + "51.104.146.235/32", + "20.73.202.147/32", + "20.73.132.240/32", + "20.71.12.143/32", + "20.56.197.58/32", + "20.56.197.63/32", + "20.43.150.93/32", + "20.43.150.85/32", + "20.44.222.1/32", + "40.89.243.175/32", + "13.89.106.77/32", + "52.143.242.6/32", + "52.143.241.111/32", + "52.154.60.82/32", + "20.197.209.11/32", + "20.197.209.27/32", + "20.226.133.105/32", + "191.234.216.4/32", + "191.234.216.178/32", + "20.53.92.211/32", + "20.53.91.2/32", + "20.207.99.197/32", + "20.207.97.190/32", + "40.81.250.205/32", + "40.64.106.11/32", + "40.64.105.247/32", + "20.72.242.93/32", + "20.99.255.235/32", + "20.113.3.121/32", + "52.224.16.221/32", + "52.224.21.53/32", + "52.224.20.204/32", + "52.224.21.19/32", + "52.224.20.249/32", + "52.224.20.203/32", + "52.224.20.190/32", + "52.224.16.229/32", + "52.224.21.20/32", + "52.146.63.80/32", + "52.224.20.227/32", + "52.224.20.193/32", + "52.190.37.160/32", + "52.224.21.23/32", + "52.224.20.223/32", + "52.224.20.181/32", + "52.224.21.49/32", + "52.224.21.55/32", + "52.224.21.61/32", + "52.224.19.152/32", + "52.224.20.186/32", + "52.224.21.27/32", + "52.224.21.51/32", + "52.224.20.174/32", + "52.224.21.4/32", + "51.104.164.109/32", + "51.104.167.71/32", + "51.104.160.177/32", + "51.104.162.149/32", + "51.104.167.95/32", + "51.104.167.54/32", + "51.104.166.111/32", + "51.104.167.88/32", + "51.104.161.32/32", + "51.104.163.250/32", + "51.104.164.189/32", + "51.104.167.19/32", + "51.104.160.167/32", + "51.104.167.110/32", + "20.191.44.119/32", + "51.104.167.104/32", + "20.191.44.234/32", + "51.104.164.215/32", + "51.104.167.52/32", + "20.191.44.22/32", + "51.104.167.87/32", + "51.104.167.96/32", + "20.191.44.16/32", + "51.104.167.61/32", + "51.104.164.147/32", + "20.50.48.159/32", + "40.114.182.172/32", + "20.50.50.130/32", + "20.50.50.163/32", + "20.50.50.46/32", + "40.114.182.153/32", + "20.50.50.118/32", + "20.50.49.55/32", + "20.50.49.25/32", + "40.114.183.251/32", + "20.50.50.123/32", + "20.50.49.237/32", + "20.50.48.192/32", + "20.50.50.134/32", + "51.138.90.233/32", + "40.114.183.196/32", + "20.50.50.146/32", + "40.114.183.88/32", + "20.50.50.145/32", + "20.50.50.121/32", + "20.50.49.40/32", + "51.138.90.206/32", + "40.114.182.45/32", + "51.138.90.161/32", + "20.50.49.0/32", + "40.119.232.215/32", + "104.43.55.167/32", + "40.119.232.251/32", + "40.119.232.50/32", + "40.119.232.146/32", + "40.119.232.218/32", + "104.43.54.127/32", + "104.43.55.117/32", + "104.43.55.116/32", + "104.43.55.166/32", + "52.154.169.50/32", + "52.154.171.70/32", + "52.154.170.229/32", + "52.154.170.113/32", + "52.154.171.44/32", + "52.154.172.2/32", + "52.143.244.81/32", + "52.154.171.87/32", + "52.154.171.250/32", + "52.154.170.28/32", + "52.154.170.122/32", + "52.143.243.117/32", + "52.143.247.235/32", + "52.154.171.235/32", + "52.154.171.196/32", + "52.154.171.0/32", + "52.154.170.243/32", + "52.154.170.26/32", + "52.154.169.200/32", + "52.154.170.96/32", + "52.154.170.88/32", + "52.154.171.150/32", + "52.154.171.205/32", + "52.154.170.117/32", + "52.154.170.209/32", + "191.235.202.48/32", + "191.233.3.202/32", + "191.235.201.214/32", + "191.233.3.197/32", + "191.235.202.38/32", + "20.53.78.144/32", + "20.193.24.10/32", + "20.53.78.236/32", + "20.53.78.138/32", + "20.53.78.123/32", + "20.53.78.106/32", + "20.193.27.215/32", + "20.193.25.197/32", + "20.193.12.126/32", + "20.193.24.251/32", + "20.204.242.101/32", + "20.207.72.113/32", + "20.204.242.19/32", + "20.219.45.67/32", + "20.207.72.11/32", + "20.219.45.190/32", + "20.204.243.55/32", + "20.204.241.148/32", + "20.207.72.110/32", + "20.204.240.172/32", + "20.207.72.21/32", + "20.204.246.81/32", + "20.207.107.181/32", + "20.204.246.254/32", + "20.219.43.246/32", + "52.149.25.43/32", + "52.149.61.51/32", + "52.149.58.139/32", + "52.149.60.38/32", + "52.148.165.38/32", + "52.143.95.162/32", + "52.149.56.151/32", + "52.149.30.45/32", + "52.149.58.173/32", + "52.143.95.204/32", + "52.149.28.83/32", + "52.149.58.69/32", + "52.148.161.87/32", + "52.149.58.27/32", + "52.149.28.18/32", + "20.79.226.26/32", + "20.79.239.66/32", + "20.79.238.198/32", + "20.113.14.159/32", + "20.75.144.152/32", + "20.43.172.120/32", + "20.53.134.160/32", + "20.201.15.208/32", + "20.93.28.24/32", + "20.61.34.40/32", + "52.242.224.168/32", + "20.80.129.80/32", + "20.195.108.47/32", + "4.195.133.120/32", + "4.228.76.163/32", + "4.182.131.108/32", + "4.209.224.56/32", + "108.141.83.74/32", + "4.213.46.14/32", + "172.169.17.165/32", + "51.8.71.117/32", + "20.3.1.178/32", + "52.149.56.151/32", + "52.149.30.45/32", + "52.149.58.173/32", + "52.143.95.204/32", + "52.149.28.83/32", + "52.149.58.69/32", + "52.148.161.87/32", + "52.149.58.27/32", + "52.149.28.18/32", + "20.79.226.26/32", + "20.79.239.66/32", + "20.79.238.198/32", + "20.113.14.159/32", + "20.75.144.152/32", + "20.43.172.120/32", + "20.53.134.160/32", + "20.201.15.208/32", + "20.93.28.24/32", + "20.61.34.40/32", + "52.242.224.168/32", + "20.80.129.80/32", + "20.195.108.47/32", + "4.195.133.120/32", + "4.228.76.163/32", + "4.182.131.108/32", + "4.209.224.56/32", + "108.141.83.74/32", + "4.213.46.14/32", + "172.169.17.165/32", + "51.8.71.117/32", + "20.3.1.178/32" + ] + }, { "name": "qwantbot", "user_agent_regex": "\\+https\\://help\\.qwant\\.com/bot/", @@ -395,4 +671,4 @@ } ], "dnsbl": false -} \ No newline at end of file +} diff --git a/docs/docs/CHANGELOG.md b/docs/docs/CHANGELOG.md index e5613bb..08afe7d 100644 --- a/docs/docs/CHANGELOG.md +++ b/docs/docs/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- Whitelisted [DuckDuckBot](https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/) in botPolicies - Improvements to build scripts to make them less independent of the build host ## v1.16.0