[ {"id": "bingbot", "n": "BingBot", "r": ["bingbot"], "rx": ["\\sbingbot\\/(\\d+\\.\\d+);"], "url": "http://www.bing.com/bingbot.htm" }, {"id": "googlebot", "n": "GoogleBot", "r": ["Googlebot"], "rx": ["Googlebot\\/(\\d+\\.\\d+)", "Googlebot-Image\\/(\\d+\\.\\d+)"], "url": "http://www.google.com/bot.html" }, {"id": "googleads", "n": "Google Ads", "r": ["AdsBot-Google", "AdsBot-Google-Mobile", "Mediapartners-Google"], "rx": ["AdsBot-Google;","AdsBot-Google-Mobile;", "Mediapartners-Google\\/(\\d+\\.\\d+);"], "url": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers" }, {"id": "googleapi", "n": "Google API Crawler", "r": ["APIs-Google"], "rx": ["APIs-Google"], "url": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers" }, {"id": "applebot", "n": "Applebot", "r": ["Applebot"], "rx": ["Applebot\\/(\\d+\\.\\d+);"], "url": "http://www.apple.com/go/applebot" }, {"id": "slurp", "n": "Slurp (Yahoo!)", "r": ["Slurp"], "rx": ["Slurp[\\s;\\)]"], "url": "http://help.yahoo.com/help/us/ysearch/slurp" }, {"id": "ddg", "n": "DuckDuckGo", "r": ["DuckDuckBot","DuckAssistBot","DuckDuckGo-Favicons-Bot"], "rx": ["DuckDuckBot\\/(\\d+\\.\\d+);", "DuckAssistBot\\/(\\d+\\.\\d+);", "DuckDuckGo-Favicons-Bot\\/(\\d+\\.\\d+);"], "url": "https://duckduckgo.com/duckduckbot.html" }, {"id": "openai", "n": "OpenAI/ChatGPT", "r": ["OAI-SearchBot", "ChatGPT-User", "GPTBot"], "rx": ["OAI-SearchBot\\/(\\d+\\.\\d+);", "ChatGPT-User\\/(\\d+\\.\\d+);", "GPTBot\\/(\\d+\\.\\d+);"], "url": "https://platform.openai.com/docs/bots/" }, {"id": "claude", "n": "Anthropic Claude", "r": ["ClaudeBot", "Claude-User", "Claude-SearchBot"], "rx": ["ClaudeBot\\/(\\d+\\.\\d+);"], "url": "https://darkvisitors.com/agents/claudebot" }, {"id": "perplexity", "n": "Perplexity", "r": ["PerplexityBot", "Perplexity‑User"], "rx": ["PerplexityBot\\/(\\d+\\.\\d+);", "Perplexity‑User\\/(\\d+\\.\\d+);"], "url": "https://perplexity.ai/perplexitybot" }, {"id": "metabots", "n": "Meta/Facebook", "r": ["facebookexternalhit", "facebookcatalog","meta-webindexer","meta-externalads","meta-externalagent","meta-externalfetcher"], "rx": ["facebook\\w+\\/(\\d+\\.\\d+)", "meta-\\w+\\/(\\d+\\.\\d+)"], "url": "https://developers.facebook.com/docs/sharing/webmasters/crawler" }, {"id": "qwant", "n": "Qwant Web crawler", "r": ["Qwantbot", "Qwantbot-news"], "rx": ["Qwantbot\\/(\\d+\\.\\d+)[_;\\s]"], "url": "https://help.qwant.com/bot/" }, {"id": "yandex", "n": "Yandex", "geo": "CN", "r": ["YandexBot", "YandexAdNet", "YandexBlogs", "YandexImages", "YandexImageResizer", "YandexMarket", "YandexMedia", "YandexOntoDB", "YandexSitelinks","YandexSpravBot", "YandexVertis", "YandexVerticals", "YandexVideo", "YandexWebmaster", "YandexComBot"], "rx": ["Yandex\\w+\\/(\\d+\\.\\d+);"], "url": "http://yandex.com/bots" }, {"id": "seznambot", "n": "SeznamBot", "geo": "CZ", "r": ["SeznamBot"], "rx": ["SeznamBot\\/(\\d+\\.\\d+);"], "url": "https://o-seznam.cz/napoveda/vyhledavani/en/seznambot-crawler/" }, {"id": "ahrefs", "n": "Ahrefs", "r": ["AhrefsBot", "AhrefsSiteAudit"], "rx": ["AhrefsBot\\/(\\d+\\.\\d+);", "AhrefsSiteAudit\\/(\\d+\\.\\d+);"], "url": "https://ahrefs.com/robot/" }, {"id": "ccbot", "n": "CommonCrawl Bot", "r": ["CCBot"], "rx": ["CCBot\\/(\\d+\\.\\d+)[\\s\\.;]*"], "url": "https://commoncrawl.org/bot.html" }, {"id": "mjbot", "n": "Majestic Crawler (UK)", "geo": "GB", "r": ["MJ12bot"], "rx": ["MJ12bot\\/v?(\\d+\\.\\d+)[\\s\\.;]"], "url": "http://www.majestic12.co.uk/bot.php" }, {"id": "petal", "n": "PetalSearch Bot (CN)", "geo": "CN", "r": ["PetalBot", "AspiegelBot"], "rx": ["[\\s;]PetalBot[\\s\\/;]", "AspiegelBot[\\)$]"], "url": "https://webmaster.petalsearch.com/site/petalbot" }, {"id": "barkrowler", "n": "Barkrowler (Babbar)", "r": ["Barkrowler"], "rx": ["[\\s;^]Barkrowler\\/(\\d+\\.\\d+)?"], "url": "https://babbar.tech/crawler" }, {"id": "semrush", "n": "SemrushBot", "r": ["SemrushBot"], "rx": ["SemrushBot-?[A-Z]*\\/(\\d+\\.?\\d*)[;~\\s]"], "url": "http://www.semrush.com/bot.html" }, {"id": "bytespider", "n": "Bytespider (ByteDance, TikTok)", "geo": "CN", "r": ["Bytespider"], "rx": ["Bytespider[;$]"], "url": "https://darkvisitors.com/agents/bytespider" }, {"id": "dfseo", "n": "DataForSEO Bot", "r": ["DataForSeoBot"], "rx": ["DataForSeoBot[;\\/](\\d\\.\\d)?"], "url": "https://dataforseo.com/dataforseo-bot" }, {"id": "hunter", "n": "Hunter (Velen) Crawler", "r": ["VelenPublicWebCrawler"], "rx": ["VelenPublicWebCrawler\\/(\\d\\.\\d)"], "url": "https://velen.io" }, {"id": "serpstat", "n": "Serpstat Bot", "r": ["serpstatbot"], "rx": ["serpstatbot\\/(\\d\\.\\d)\\s"], "url": "https://serpstatbot.com/" }, {"id": "netestate", "n": "netEstate NE Crawler", "geo": "DE", "r": ["netEstate NE Crawler"], "rx": ["netEstate NE Crawler\\s"], "url": "http://www.website-datenbank.de/" }, {"id": "imagesift", "n": "Hive Imagesift Bot (AI Scraper)", "r": ["ImagesiftBot"], "rx": ["ImagesiftBot[;\\s\\/]"], "url": "https://darkvisitors.com/agents/imagesiftbot" }, {"id": "moz", "n": "Moz.com Bots", "r": ["dotbot", "rogerbot"], "rx": ["DotBot\\/(\\d\\.\\d)[\\.;\\s]", "RogerBot\\/(\\d\\.\\d)[\\.;\\s]"], "url": "https://moz.com/help/moz-procedures/crawlers/dotbot" }, {"id": "maui", "n": "MauiBot (unknown developer)", "r": ["MauiBot"], "rx": ["MauiBot\\s"], "url": "https://darkvisitors.com/agents/mauibot" }, {"id": "plagaware", "n": "PlagAwareBot (DE)", "geo": "DE", "r": ["PlagAwareBot"], "rx": ["PlagAwareBot\\/(\\d\\.\\d)"], "url": "https://www.plagaware.com/bot" }, {"id": "blexbot", "n": "BLEXBot", "r": ["BLEXBot"], "rx": ["BLEXBot\\/(\\d\\.\\d);"], "url": "https://help.seranking.com/en/blex-crawler" }, {"id": "sogou", "n": "Sogou web spider", "r": ["Sogou web spider"], "rx": ["Sogou web spider\\/(\\d+\\.\\d+)[;\\s\\(\\.]"], "url": "http://www.sogou.com/docs/help/webmasters.htm#07" }, {"id": "amazon", "n": "Amazonbot", "r": ["Amazonbot"], "rx": ["Amazonbot\\/(\\d+\\.\\d+)[;\\s\\(\\.]"], "url": "https://developer.amazon.com/amazonbot" }, {"id": "sop", "n": "SitesOverPagesBot", "r": ["SitesOverPagesBot"], "rx": ["SitesOverPagesBot\\/(\\d+\\.\\d+)[;\\s\\(\\.]"], "url": "https://www.sitesoverpages.com/bot" }, {"id": "startme", "n": "StartmeBot", "r": ["StartmeBot"], "rx": ["StartmeBot\\/(\\d+\\.\\d+)[;\\s\\(\\.]"], "url": "https://support.start.me/en/articles/9182933-what-is-startmebot" }, {"id": "bitsight", "n": "BitSightBot", "r": ["BitSightBot"], "rx": ["BitSightBot\\/(\\d+\\.\\d+)"], "url": "https://darkvisitors.com/agents/bitsightbot" }, {"id": "terracotta", "n": "TerraCotta Bot", "r": ["TerraCotta"], "rx": ["TerraCotta"], "url": "https://github.com/CeramicTeam/CeramicTerracotta" }, {"id": "tiktok", "n": "TikTok Spider", "geo": "CN", "r": ["TikTokSpider"], "rx": ["TikTokSpider"], "url": "https://darkvisitors.com/agents/tiktokspider" } ]