5 	 "url": "http://www.bing.com/bingbot.htm"  string
11 	 "url": "http://www.google.com/bot.html"  string
17 	 "url": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers"  string
23 	 "url": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers"  string
29 	 "url": "https://support.google.com/webmasters/answer/9012289"  string
35 …"url": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers"  string
41 	 "url": "http://www.apple.com/go/applebot"  string
47 	 "url": "http://www.baidu.com/search/spider.html"  string
53 	 "url": "http://www.reddit.com/feedback"  string
59 	 "url": "http://help.yahoo.com/help/us/ysearch/slurp"  string
65 	 "url": "https://help.yahoo.com/kb/SLN23615.html"  string
71 	 "url": "https://duckduckgo.com/duckduckbot.html"  string
77 	 "url": "https://marginalia-search.com/"  string
83 	 "url": "https://platform.openai.com/docs/bots/"  string
89 	 "url": "https://platform.openai.com/docs/bots/"  string
95 …"url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-…  string
101 …"url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-…  string
107 	 "url": "https://perplexity.ai/perplexitybot"  string
113 	 "url": "https://perplexity.ai/perplexitybot"  string
119 	 "url": "https://developers.facebook.com/docs/sharing/webmasters/crawler"  string
125 	 "url": "https://developers.facebook.com/docs/sharing/webmasters/crawler"  string
131 	 "url": "https://help.qwant.com/bot/"  string
137 	 "url": "http://yandex.com/bots"  string
143 	 "url": "https://o-seznam.cz/napoveda/vyhledavani/en/seznambot-crawler/"  string
149 	 "url": "https://ahrefs.com/robot/"  string
155 	 "url": "https://commoncrawl.org/bot.html"  string
161 	 "url": "http://www.majestic12.co.uk/bot.php"  string
167 	 "url": "https://webmaster.petalsearch.com/site/petalbot"  string
173 	 "url": "https://babbar.tech/crawler"  string
179 	 "url": "http://www.semrush.com/bot.html"  string
185 	 "url": "https://darkvisitors.com/agents/bytespider"  string
191 	 "url": "https://dataforseo.com/dataforseo-bot"  string
197 	 "url": "https://velen.io"  string
203 	 "url": "https://serpstatbot.com/"  string
209 	 "url": "http://www.website-datenbank.de/"  string
215 	 "url": "https://darkvisitors.com/agents/imagesiftbot"  string
221 	 "url": "https://moz.com/help/moz-procedures/crawlers/dotbot"  string
227 	 "url": "https://darkvisitors.com/agents/mauibot"  string
233 	 "url": "https://www.plagaware.com/bot"  string
239 	 "url": "https://datadome.co/bots/discordbot/"  string
245 	 "url": "https://datadome.co/bots/twitterbot/"  string
251 	 "url": "https://darkvisitors.com/agents/whatsapp/"  string
257 	 "url": "https://darkvisitors.com/agents/tiktokspider"  string
263 	 "url": "https://help.seranking.com/en/blex-crawler"  string
269 	 "url": "http://www.sogou.com/docs/help/webmasters.htm#07"  string
275 	 "url": "https://developer.amazon.com/amazonbot"  string
281 	 "url": "https://www.sitesoverpages.com/bot"  string
287 	 "url": "https://support.start.me/en/articles/9182933-what-is-startmebot"  string
293 	 "url": "https://datadome.co/bots/bitsightbot/"  string
299 	 "url": "https://github.com/CeramicTeam/CeramicTerracotta"  string
305 	 "url": "https://www.haloscan.com/"  string
311 	 "url": "https://archive.org/details/archive.org_bot"  string
317 	 "url": "https://www.aqtronix.com/useragents/?Action=ShowAgentDetails&Name=AliyunSecBot"  string
323 	 "url": "https://ibou.io/iboubot.html"  string
329 	 "url": "http://crawl.bnl.lu/"  string
335 	 "url": "https://www.turnitin.com/robot/crawlerinfo.html"  string