Lines Matching full:com
19 url: 'https://www.plesk.com'
24 url: 'https://developers.cloudflare.com/health-checks/'
27 url: 'https://www.cloudflare.com/'
32 url: 'https://www.so.com/help/help_3_2.html'
40 url: 'http://www.aboundex.com/crawler/'
42 name: 'Aboundex.com'
43 url: 'http://www.aboundex.com'
53 - regex: 'AddThis\.com'
54 name: 'AddThis.com'
59 url: 'http://www.clearspring.com'
64 url: 'https://ahrefs.com/robot'
67 url: 'https://ahrefs.com/robot'
72 url: 'https://ahrefs.com/robot/site-audit'
75 url: 'https://ahrefs.com/'
80 url: 'https://support.alexa.com/hc/en-us/sections/200100794-Crawlers'
83 url: 'https://www.alexa.com'
88 url: 'https://support.alexa.com/hc/en-us/articles/200450194'
91 url: 'https://www.alexa.com'
96 url: 'https://developer.amazon.com/support/amazonbot'
98 name: 'Amazon.com, Inc.'
99 url: 'https://www.amazon.com/'
104 url: 'https://adbot.amazon.com/'
106 name: 'Amazon.com, Inc.'
107 url: 'https://www.amazon.com/'
114 url: 'https://aws.amazon.com/'
119 url: 'http://amorank.com/webcrawler.html'
122 url: 'http://www.amorank.com'
135 url: 'https://support.apple.com/en-us/119829'
138 url: 'https://www.apple.com/'
143 url: 'https://support.apple.com/en-us/119829'
146 url: 'https://www.apple.com/'
151 url: 'https://docs.appsignal.com/uptime-monitoring/'
154 url: 'https://appsignal.com/'
159 url: 'https://www.arachni-scanner.com/'
162 url: 'https://www.sarosys.com/'
167 url: 'https://aspiegel.com/'
170 url: 'https://www.huawei.com/'
183 url: 'http://www.analyticsseo.com/crawler'
186 url: 'http://www.analyticsseo.com'
202 url: 'http://www.ask.com'
215 url: 'http://www.backlinktest.com/crawler.html'
218 url: 'http://www.backlinktest.com'
223 url: 'http://www.baidu.com/search/spider.htm'
226 url: 'http://www.baidu.com'
230 url: 'https://bazqux.com/fetcher'
239 url: 'https://betteruptime.com/faq'
242 url: 'https://betteruptime.com/'
247 url: 'http://search.msn.com/msnbot.htmn'
250 url: 'http://www.microsoft.com'
255 url: 'https://github.com/prometheus/blackbox_exporter'
263 url: 'http://blekko.com/about/blekkobot'
266 url: 'http://blekko.com'
271 url: 'http://webmeup-crawler.com'
274 url: 'http://webmeup.com'
278 url: 'http://www.bloglovin.com'
290 url: 'https://blogtrottr.com/'
297 url: 'https://boardreader.com/'
302 url: 'http://bountii.com/contact.php'
305 url: 'http://bountii.com'
326 url: 'http://labs.topsy.com/butterfly'
329 url: 'http://labs.topsy.com'
345 url: 'http://www.reddit.com'
350 url: 'http://cliqz.com/company/cliqzbot'
353 url: 'http://cliqz.com'
358 url: 'https://amp.cloudflare.com/doc/fetcher.html'
361 url: 'http://www.cloudflare.com'
366 url: 'https://www.cloudflare.com/'
369 url: 'https://www.cloudflare.com/'
374 url: 'https://www.cloudflare.com/always-online'
377 url: 'https://www.cloudflare.com/'
382 url: 'https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/'
385 url: 'https://www.cloudflare.com/'
390 url: 'https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/'
393 url: 'https://www.cloudflare.com/'
398 url: 'https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/'
401 url: 'https://www.cloudflare.com/'
406 url: 'https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/'
409 url: 'https://www.cloudflare.com/'
414 url: 'https://developers.cloudflare.com/speed/speed-test/run-speed-test'
417 url: 'https://www.cloudflare.com/'
419 - regex: 'https://developers\.cloudflare\.com/security-center/'
422 url: 'https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/'
425 url: 'https://www.cloudflare.com/'
427 - regex: 'coccoc\.com'
429 url: 'https://help.coccoc.com/en/search-engine/coccoc-robots'
433 url: 'https://coccoc.com/'
445 url: 'http://www.commafeed.com'
454 url: 'http://www.css-security.com/certificatespider/'
457 url: 'https://www.css-security.com/company/about-us/'
461 url: 'https://github.com/DataDog/dd-agent'
465 url: 'https://www.datadoghq.com/'
473 url: 'https://www.datanyze.com'
481 url: 'https://www.dataprovider.com/'
489 url: 'http://www.kakaocorp.com/main'
502 url: 'http://discoveryengine.com/discobot.html'
505 url: 'http://discoveryengine.com'
507 - regex: 'Domain Re-Animator Bot|support@domainreanimator\.com'
513 url: 'http://domainreanimator.com'
521 url: 'http://moz.com/'
526 url: 'https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/'
529 url: 'https://duckduckgo.com/'
534 url: 'https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/'
537 url: 'https://duckduckgo.com/'
542 url: 'http://www.easou.com/search/spider.html'
545 url: 'http://www.easou.com'
552 url: 'https://ecairn.com'
573 url: 'http://www.exabot.com/go/robot'
576 url: 'http://www.3ds.com'
581 url: 'http://www.exactseek.com'
584 url: 'http://www.jaydeonlineinc.com'
592 url: 'http://moz.com/'
597 url: 'https://developers.facebook.com/docs/sharing/webmasters/web-crawlers'
600 url: 'https://www.meta.com/'
605 url: 'https://developers.facebook.com/docs/sharing/webmasters/web-crawlers'
608 url: 'https://www.meta.com/'
613 url: 'https://developers.facebook.com/docs/sharing/webmasters/web-crawlers'
616 url: 'https://www.meta.com/'
621 url: 'https://developers.facebook.com/docs/sharing/bot'
624 url: 'https://www.meta.com/'
628 url: 'http://feedbin.com/'
636 url: 'http://www.feedburner.com'
652 url: 'http://www.feedly.com'
660 url: 'http://www.feedspot.com'
668 url: 'http://feedafever.com/'
676 url: 'http://flipboard.com/browserproxy'
680 url: 'http://flipboard.com/'
685 url: 'http://www.findxbot.com'
695 url: 'http://www.genieo.com/webfilter.html'
698 url: 'http://www.genieo.com'
703 url: 'https://github.com/gigablast/open-source-search-engine'
706 url: 'http://www.gigablast.com/faq.html'
711 url: 'http://glutenfreepleasure.com/'
718 url: 'https://github.com/OJ/gobuster'
739 url: 'https://search.google.com/search-console/about'
742 url: 'https://www.google.com/'
747 url: 'http://developers.google.com/speed/pagespeed/insights/'
750 url: 'https://www.google.com/'
758 url: 'https://www.google.com/'
763 url: 'https://cloud.google.com/scheduler'
766 url: 'https://www.google.com'
771 url: 'https://search.google.com/structured-data/testing-tool'
774 url: 'https://www.google.com/'
779 url: 'https://cloud.google.com/monitoring'
782 url: 'https://www.google.com'
787 url: 'https://transparencyreport.google.com/'
790 url: 'https://www.google.com/'
795 …url: 'https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers#google-cl…
798 url: 'https://www.google.com/'
800 - regex: 'via ggpht\.com GoogleImageProxy'
806 url: 'https://www.google.com/'
811 url: 'https://support.google.com/drive/answer/176692?hl=en'
814 url: 'https://www.google.com/'
819 url: 'https://workspace.google.com/products/sheets/'
822 url: 'https://www.google.com/'
827 url: 'https://workspace.google.com/products/slides/'
830 url: 'https://www.google.com/'
835 url: 'https://docs.google.com/'
838 url: 'https://www.google.com/'
872 - regex: 'deepcrawl\.com'
875 url: 'https://deepcrawl.com/bot'
883 url: 'https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers'
886 url: 'https://www.google.com/'
891 url: 'https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers'
894 url: 'https://www.google.com/'
899 url: 'https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers'
902 url: 'https://www.google.com/'
907 url: 'https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers'
910 url: 'https://www.google.com/'
915 url: 'https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers'
918 url: 'https://www.google.com/'
923 url: 'https://area120.google.com/'
926 url: 'https://www.google.com/'
933 url: 'https://www.hubspot.com'
938 url: 'https://vuhuv.com/bot.html'
943 url: 'http://www.httpmon.com'
951 url: 'http://www.nict.go.jp/en/univ-com/plan/crawl.html'
956 - regex: 'inoreader\.com'
959 url: 'https://www.inoreader.com'
967 url: 'http://www.microsoft.com'
974 url: 'http://www.verisign.com/'
976 - regex: 'IP-Guide\.com'
982 url: 'https://ip-guide.com'
990 url: 'http://kouio.com/'
1007 url: 'https://developers.google.com/web/tools/lighthouse'
1010 url: 'https://developers.google.com/web/tools/lighthouse'
1012 - regex: 'last-modified\.com'
1015 url: 'https://last-modified.com/en/about'
1018 url: 'https://last-modified.com/en'
1020 - regex: 'linkdexbot|linkdex\.com'
1023 url: 'http://www.linkdex.com/bots'
1026 url: 'http://www.mojeek.com'
1031 url: 'http://www.linkedin.com'
1034 url: 'http://www.linkedin.com'
1039 url: 'https://ltx71.com/'
1055 url: 'http://www.brandwatch.com/magpie-crawler/'
1058 url: 'http://www.brandwatch.com'
1070 url: 'https://github.com/bi-zone/masscan-ng'
1078 url: 'https://github.com/robertdavidgraham/masscan'
1082 url: 'https://github.com/robertdavidgraham'
1091 url: 'http://www.meanpath.com/meanpathbot.html'
1094 url: 'http://www.meanpath.com'
1107 url: 'https://github.com/jaimeiniesta/metainspector'
1112 url: 'http://mixrank.com'
1130 name: 'Lavtech.Com Corp.'
1135 url: 'http://www.mojeek.com/bot.html'
1138 url: 'http://www.mojeek.com'
1164 - regex: 'nbertaupete95\(at\)gmail\.com'
1174 url: 'http://www.netcraft.com'
1186 url: 'http://www.netvibes.com/'
1194 url: 'http://www.newsblur.com'
1202 url: 'http://www.newsgator.com'
1214 url: 'http://northernlight.com'
1229 url: 'https://www.nuzzel.com/'
1234 url: 'https://nodeping.com'
1237 url: 'https://nodeping.com'
1245 url: 'https://onlineornot.com/website-monitoring'
1248 url: 'https://onlineornot.com'
1253 url: 'http://www.omgili.com/Crawler.html'
1256 url: 'http://www.omgili.com'
1272 url: 'http://www.axandra.com'
1301 url: 'https://github.com/macbre/phantomas'
1306 url: 'https://github.com/phpservermon/phpservermon'
1314 url: 'https://getpocket.com/pocketparser_ua'
1317 url: 'https://getpocket.com/'
1322 url: 'https://github.com/astro/prittorrent'
1330 url: 'https://www.paessler.com/prtg'
1333 url: 'https://www.paessler.com'
1338 url: 'http://www.picsearch.com/bot.html'
1341 url: 'http://www.picsearch.com'
1343 - regex: 'Pingdom(?:\.com|TMS)'
1349 url: 'https://www.pingdom.com'
1357 url: 'http://www.quora.com'
1365 url: 'https://www.quora.com/'
1378 url: 'http://queryseeker.com/bot.html'
1381 url: 'http://queryeye.com'
1386 url: 'https://help.qwant.com/bot/'
1389 url: 'https://www.qwant.com/'
1399 url: 'http://www.reddit.com/feedback'
1402 url: 'http://www.reddit.com'
1410 url: 'https://www.f-secure.com'
1415 url: 'http://moz.com/help/pro/what-is-rogerbot-'
1418 url: 'http://moz.com/'
1426 url: 'http://roihunter.com/'
1431 url: 'https://www.safedns.com/searchbot'
1434 url: 'https://www.safedns.com/'
1452 url: 'http://www.screenerbot.com'
1460 url: 'https://www.semrush.com/bot/'
1463 url: 'https://www.semrush.com/'
1468 url: 'https://www.semrush.com/bot/'
1471 url: 'https://www.semrush.com/'
1476 url: 'https://www.semrush.com/bot/'
1479 url: 'https://www.semrush.com/'
1484 url: 'https://www.semrush.com/bot/'
1487 url: 'https://www.semrush.com/'
1492 url: 'https://www.semrush.com/bot/'
1495 url: 'https://www.semrush.com/'
1503 url: 'http://sensika.com'
1508 url: 'http://www.seoengine.com/seoengbot.htm'
1511 url: 'http://www.seoengine.com'
1524 url: 'https://www.skype.com'
1537 url: 'https://www.shopify.com/partners'
1540 url: 'https://www.shopify.com/'
1545 url: 'http://www.shopwiki.com/wiki/Help:Bot'
1548 url: 'http://www.shopwiki.com'
1552 url: 'http://silverreader.com'
1577 url: 'https://optimizer.sistrix.com'
1585 url: 'http://ricks-apps.com/osx/sitesucker/'
1598 url: 'https://api.slack.com/robots'
1601 url: 'http://slack.com'
1606 url: 'http://www.sogou.com/docs/help/webmasters.htm'
1609 url: 'http://www.sogou.com'
1614 url: 'http://help.soso.com/webspider.htm'
1617 url: 'http://www.soso.com'
1625 url: 'https://www.sprinklr.com/'
1630 url: 'https://www.ssllabs.com/about/assessment.html'
1633 url: 'https://www.ssllabs.com/about/assessment.html'
1638 url: 'https://www.statuscake.com'
1641 url: 'https://www.statuscake.com'
1649 url: 'https://superfeedr.com/'
1654 url: 'https://github.com/USCDataScience/sparkler'
1659 url: 'http://spinn3r.com/robot'
1662 url: 'http://spinn3r.com'
1682 url: 'http://www.domaintools.com/webmasters/surveybot.php'
1685 url: 'http://www.domaintools.com'
1689 url: 'http://www.tarmot.com/gezgin/'
1698 url: 'https://scan.trustnet.venafi.com/'
1702 url: 'https://www.venafi.com'
1707 url: 'http://www.tineye.com/crawler.html'
1710 url: 'http://ideeinc.com'
1720 - regex: 'theoldreader\.com'
1723 url: 'https://theoldreader.com'
1728 url: 'https://help.chartable.com/article/34-what-is-the-trackable-analytics-prefix'
1731 url: 'https://chartable.com'
1739 url: 'http://www.talkwalker.com'
1744 url: 'http://www.turnitin.com/robot/crawlerinfo.html'
1747 url: 'http://www.turnitin.com'
1752 url: 'https://tweetedtimes.com/'
1755 url: 'https://tweetedtimes.com/'
1760 url: 'http://tweetmeme.com/'
1770 url: 'https://www.twingly.com'
1775 url: 'https://dev.twitter.com/docs/cards/getting-started'
1778 url: 'http://www.twitter.com'
1783 url: 'https://github.com/kurtmckee/feedparser'
1786 url: 'https://github.com/kurtmckee'
1788 - regex: 'via secureurl\.fwdcdn\.com'
1799 url: 'https://uptime.com/uptime-bot'
1802 url: 'https://uptime.com/'
1807 url: 'https://uptimerobot.com/'
1810 url: 'https://uptimerobot.com/'
1831 url: 'https://dev.vk.com/en/widgets/share'
1834 url: 'https://vk.com/'
1839 url: 'https://dev.vk.com/en/'
1842 url: 'https://vk.com/'
1847 url: 'http://www.visualsitemapper.com/crawler'
1850 url: 'http://www.alentum.com'
1910 url: 'https://github.com/AliasIO/Wappalyzer'
1913 url: 'https://github.com/AliasIO'
1923 url: 'http://www.wesee.com/bot'
1926 url: 'http://www.wesee.com'
1931 url: 'http://badcheese.com/crawler.html'
1934 url: 'http://badcheese.com'
1939 url: 'http://badcheese.com/crawler.html'
1942 url: 'http://www.websitepulse.com/'
1944 - regex: 'WordPress.+isitwp\.com'
1947 url: 'https://www.isitwp.com/'
1950 url: 'https://www.wpbeginner.com/'
1955 url: 'https://wordpress.com/crawler/'
1960 - regex: 'WordPress\.com mShots'
1961 name: 'WordPress.com mShots'
1968 - regex: 'wp\.com feedbot'
1969 name: 'wp.com feedbot'
1971 url: 'https://wordpress.com/'
1974 url: 'https://automattic.com/'
1987 url: 'http://www.wotbox.com/bot/'
1990 url: 'http://www.wotbox.com'
1995 url: 'https://xenforo.com/'
1998 url: 'https://xenforo.com/'
2011 url: 'http://help.yahoo.com/ysearch/slurp'
2014 url: 'http://www.yahoo.com'
2019 url: 'https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html'
2022 url: 'http://www.yahoo.com'
2027 url: 'https://help.yahoo.com/kb/yahoo-mail-proxy-SLN28749.html'
2030 url: 'http://www.yahoo.com'
2038 url: 'http://www.yahoo.com'
2075 url: 'https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html'
2078 url: 'https://yandex.com/company/'
2083 url: 'http://help.naver.com/robots/'
2086 url: 'http://www.naver.com'
2091 url: 'http://www.youdao.com/help/webmaster/spider'
2094 url: 'http://corp.163.com'
2104 url: 'http://www.yunyun.com/SiteInfo.php?r=about'
2107 url: 'http://www.yunyun.com'
2112 url: 'https://github.com/zmap/zgrab'
2117 url: 'http://zookabot.com'
2125 url: 'http://help.zum.com/inquiry'
2128 url: 'http://www.zuminternet.com/'
2133 url: 'http://www.yottaa.com/products/site-monitor'
2136 url: 'http://www.yottaa.com/'
2141 url: 'https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html'
2144 url: 'http://www.yahoo.com'
2152 url: 'http://www.outbrain.com/'
2157 url: 'https://hubpages.com/help/crawlingpolicy'
2160 url: 'https://discover.hubpages.com/'
2162 - regex: 'Pinterest(?:bot)?/.*www\.pinterest\.com'
2164 url: 'https://help.pinterest.com/en/business/article/pinterest-crawler'
2168 url: 'https://www.pinterest.com/'
2173 url: 'https://www.site24x7.com/site24x7-faq.html'
2176 url: 'https://www.site24x7.com'
2181 …url: 'https://support.site24x7.com/portal/en/kb/articles/default-user-agent-used-in-website-deface…
2184 url: 'https://www.site24x7.com/'
2189 url: 'https://www.snapchat.com'
2192 url: 'https://www.snapchat.com'
2197 url: 'https://developers.snap.com/robots'
2200 url: 'https://www.snapchat.com/'
2205 url: 'https://businesshelp.snapchat.com/s/article/adsbot-crawler?language=en_US'
2208 url: 'https://www.snapchat.com/'
2221 url: 'https://www.grapeshot.com/crawler'
2224 url: 'https://www.grapeshot.com'
2237 url: 'https://www.catchpoint.com/'
2240 url: 'https://www.catchpoint.com/'
2245 url: 'https://bitly.com'
2248 url: 'https://bitly.com'
2287 url: 'https://pagepeeker.com/robots/'
2290 url: 'https://pagepeeker.com/'
2304 - regex: 'AdMantX.*admantx\.com'
2323 url: 'https://www.spotify.com'
2335 url: 'https://medium.com/'
2340 url: 'https://www.brandverity.com/why-is-brandverity-visiting-me'
2343 url: 'https://www.brandverity.com/'
2348 url: 'https://www.kaspersky.com/'
2351 url: 'https://www.kaspersky.com/'
2364 url: 'https://www.woorank.com/'
2367 url: 'https://www.woorank.com/'
2372 url: 'https://siteimprove.com/'
2375 url: 'https://siteimprove.com/'
2388 url: 'https://hypefactors.com/'
2391 url: 'https://hypefactors.com/'
2396 url: 'https://www.tracemyfile.com/'
2399 url: 'http://ideeinc.com/'
2412 url: 'https://weborama.com/'
2415 url: 'https://weborama.com/'
2420 url: 'https://boardreader.com/'
2423 url: 'https://boardreader.com/'
2436 url: 'https://bytedance.com/'
2439 url: 'https://bytedance.com/'
2444 url: 'https://www.wikido.com/'
2447 url: 'https://www.wikido.com/'
2452 url: 'https://awario.com/bots.html'
2455 url: 'https://www.techfusion.com.cy/'
2460 url: 'https://awario.com/bots.html'
2463 url: 'https://www.techfusion.com.cy/'
2468 url: 'https://www.xforce-security.com/crawler/'
2471 url: 'https://exchange.xforce.ibmcloud.com/'
2476 url: 'https://www.similartech.com/smtbot'
2479 url: 'https://www.similartech.com/'
2508 url: 'https://gtmetrix.com/'
2511 url: 'https://www.carbon60.com/'
2537 url: 'https://vercel.com'
2542 url: 'https://www.grammarly.com'
2556 url: 'https://aspiegel.com/petalbot'
2561 url: 'https://serendeputy.com/about/serendeputy-bot'
2563 - regex: 'ias-(?:va|sg).*admantx.*service-fetcher|admantx\.com.*service-fetcher'
2566 url: 'https://www.admantx.com/service-fetcher.html'
2581 url: 'http://www.exensa.com/crawl'
2586 url: 'https://bigweb.bigdatacorp.com.br/pages/faq.aspx'
2589 url: 'https://bigdatacorp.com.br/'
2594 url: 'https://www.adbeat.com/operation_policy'
2597 url: 'https://www.adbeat.com/'
2602 url: 'https://builtwith.com/biup'
2605 url: 'https://builtwith.com/'
2607 - regex: 'https://whatis\.contentkingapp\.com'
2610 url: 'https://whatis.contentkingapp.com/'
2613 url: 'https://www.contentkingapp.com/'
2631 url: 'http://notifyninja.com'
2636 url: 'https://webdatastats.com/policy.html'
2639 url: 'https://webdatastats.com/'
2652 url: 'http://cloudsystemnetworks.com'
2657 url: 'http://capture.heartrails.com'
2662 url: 'https://project-resonance.com/'
2665 url: 'https://redhuntlabs.com/'
2670 url: 'https://advertising.roku.com/dataxu'
2673 url: 'https://roku.com'
2678 url: 'https://cocolyze.com/en/cocolyzebot'
2681 url: 'https://vsi-innovation.com/'
2686 url: 'https://veryhip.com/'
2689 url: 'https://veryhip.com/'
2702 url: 'http://www.webtop.com/'
2704 - regex: 'PageThing\.com'
2707 url: 'https://www.pagething.com/'
2710 url: 'https://www.specialnoise.com/'
2722 url: 'https://www.choosito.com/'
2726 url: 'https://www.choosito.com/'
2730 url: 'https://www.datagnion.com/bot.html'
2734 url: 'https://www.datagnion.com/'
2746 url: 'https://github.com/projectdiscovery/httpx'
2755 url: 'https://github.com/projectdiscovery/interactsh'
2760 - regex: 'scaninfo@(?:expanseinc|paloaltonetworks)\.com'
2771 url: 'https://isecurity.huawei.com'
2774 url: 'https://huawei.com'
2804 url: 'https://mediatoolkit.com'
2809 url: 'https://www.zoominfo.com'
2827 url: 'http://www.botje.com/plukkie.htm'
2832 url: 'https://www.comscore.com/Web-Crawler'
2842 url: 'http:/www.gowikibot.com'
2847 url: 'https://sabsim.com'
2852 url: 'https://umtel.com'
2857 url: 'http://www.pipl.com/bot'
2862 url: 'https://www.woorank.com/bot'
2867 url: 'https://support.cookiebot.com/hc/en-us/articles/360014264140-Scanner-User-Agent'
2870 url: 'https://www.cybot.com/'
2875 url: 'https://www.netsystemsresearch.com/'
2878 url: 'https://www.netsystemsresearch.com/'
2888 - regex: 'gdnplus\.com'
2891 url: 'https://gdnplus.com/'
2894 url: 'https://gdnplus.com/'
2917 url: 'http://serpstatbot.com/'
2925 url: 'https://github.com/gocolly/colly/'
2930 url: 'https://github.com/LeakIX/l9tcpid'
2935 url: 'https://github.com/LeakIX/l9explore'
2948 url: 'https://megaindex.com/crawler'
2953 url: 'https://bot.seekport.com/'
2961 url: 'https://seolyt.com/'
2966 url: 'https://www.linkfluence.com/'
2969 url: 'https://www.linkfluence.com/'
2974 url: 'http://www.komodia.com/newwiki/index.php/URL_server_crawler'
2977 url: 'https://www.komodia.com/'
2985 url: 'https://www.komodia.com/'
2990 url: 'https://neeva.com/neevabot'
2993 url: 'https://neeva.com/'
2998 url: 'https://go.chatwork.com/en/'
3001 url: 'https://www.kubell.com/en/'
3011 url: 'https://junglekey.com/'
3024 url: 'https://github.com/VIPnytt/SitemapParser/'
3029 url: 'https://turnitin.com/robot/crawlerinfo.html'
3034 url: 'https://www.dotcom-monitor.com'
3043 url: 'https://dataforseo.com/dataforseo-bot'
3048 url: 'https://discordapp.com'
3079 url: 'https://gist.github.com/jayhardee9/2f2a2c4dba26564ee040ae32e0dd0972'
3097 url: 'https://www.comcepta.com/'
3102 url: 'https://help.crowdtangle.com/en/articles/3009319-crowdtangle-bot'
3105 url: 'https://www.crowdtangle.com/'
3118 url: 'https://www.airslate.com/bot/explore/onalytica-bot'
3121 url: 'https://www.airslate.com/'
3126 url: 'https://deepnoc.com/bot'
3129 url: 'https://deepnoc.com/'
3142 url: 'https://www.ubermetrics-technologies.com/'
3145 url: 'https://www.ubermetrics-technologies.com/'
3150 url: 'http://abonti.com/'
3152 - regex: 'collection@infegy\.com'
3155 url: 'https://infegy.com/'
3158 url: 'https://infegy.com/'
3171 url: 'https://headline.com/legal/crawler'
3174 url: 'https://headline.com/'
3179 url: 'https://webpros.com/'
3182 url: 'https://webpros.com/'
3187 url: 'https://aws.amazon.com/elasticloadbalancing/'
3189 name: 'Amazon.com, Inc.'
3190 url: 'https://www.amazon.com/'
3192 - regex: 'Wheregoes\.com Redirect Checker'
3195 url: 'https://wheregoes.com/'
3205 url: 'https://internet-measurement.com/'
3218 url: 'https://www.freewebmonitoring.com/bot.html'
3221 url: 'http://www.greenwaveonline.com/'
3226 url: 'https://www.pagemodified.com/'
3229 url: 'https://www.pagemodified.com/'
3231 - regex: 'adstxtlab\.com'
3232 name: 'adstxtlab.com'
3234 url: 'https://adstxtlab.com/validator.php'
3237 url: 'https://adstxtlab.com/'
3242 url: 'https://iframely.com/'
3245 url: 'https://iframely.com/'
3250 url: 'https://domainstats.com/pages/our-bot'
3253 url: 'https://domainstats.com/'
3258 url: 'https://www.aihitdata.com/about'
3263 url: 'https://domaincrawler.com/about-us/'
3276 url: 'https://www.adauth.com'
3278 - regex: 'faveeo\.com'
3281 url: 'http://www.faveeo.com'
3291 url: 'https://www.criteo.com/criteo-crawler/'
3296 url: 'https://developer.paypal.com/api/nvp-soap/ipn/IPNIntro/'
3299 url: 'https://www.paypal.com/'
3328 url: 'https://infotiger.com/bot'
3331 url: 'https://infotiger.com/'
3339 url: 'https://www.swoppen.com/de'
3344 url: 'https://www.scamadviser.com/'
3347 url: 'https://www.scamadviser.com/'
3352 url: 'https://www.zaldamo.com/search.html'
3355 url: 'https://www.zaldamo.com/'
3360 url: 'https://allloadin.com/'
3365 url: 'https://www.phishlabs.com/'
3368 url: 'https://www.phishlabs.com/'
3373 url: 'https://github.com/renovatebot/renovate'
3381 url: 'https://www.inetdex.com/'
3386 url: 'https://www.netzzappen.com/'
3389 url: 'https://www.netzzappen.com/'
3391 - regex: 'panscient\.com'
3394 url: 'https://www.panscient.com/faq.htm'
3397 url: 'https://www.panscient.com/'
3410 url: 'https://www.nicecrawler.com/'
3413 url: 'https://www.intelium.com/'
3418 url: 'https://www.t3versions.com/bot'
3421 url: 'https://www.t3versions.com/'
3426 url: 'https://www.crawlson.com/about'
3429 url: 'https://www.crawlson.com/'
3437 url: 'https://blackkite.com/'
3442 url: 'https://www.xing.com/jobs'
3445 url: 'https://www.xing.com/'
3465 url: 'https://web.archive.org/web/20050930235914/http://www.everyfeed.com/'
3474 url: 'https://github.com/GossiTheDog/scanning'
3477 url: 'https://doublepulsar.com/'
3482 url: 'https://web.archive.org/web/20220626191617/https://sublinq.com/'
3498 url: 'https://www.comodo.com/'
3501 url: 'https://www.comodo.com/'
3503 - regex: 'Sectigo DCV|acme\.sectigo\.com'
3506 url: 'https://sectigo.com/'
3509 url: 'https://sectigo.com/'
3514 url: 'https://docs.klarna.com/klarna-bot/'
3517 url: 'https://www.klarna.com/'
3522 url: 'https://help.taboola.com/hc/en-us/articles/115002347594-The-Taboola-Crawler'
3525 url: 'https://www.taboola.com/'
3530 url: 'https://asana.com/'
3533 url: 'https://asana.com/'
3538 url: 'https://developer.chrome.com/blog/private-prefetch-proxy/'
3541 url: 'https://www.google.com/'
3546 url: 'https://www.urlinspector.com/bot/'
3549 url: 'https://www.linkresearchtools.com/'
3554 url: 'https://entfer.com/'
3557 url: 'https://entfer.com/'
3562 url: 'https://taginspector.com/'
3565 url: 'https://infotrust.com/'
3570 url: 'https://pageburstls.elsevier.com/'
3573 url: 'https://www.elsevier.com/'
3578 url: 'https://docs.diffbot.com/docs/getting-started-with-crawl'
3581 url: 'https://www.diffbot.com/'
3586 url: 'https://help.disqus.com/en/articles/1765357-ads-txt-implementation-guide'
3589 url: 'https://disqus.com/'
3607 url: 'https://reqbin.com/curl'
3625 url: 'https://gumgum.com/verity'
3630 url: 'https://github.com/snarfed/hackermention'
3635 url: 'https://www.bitsight.com/'
3638 url: 'https://www.bitsight.com/'
3643 url: 'https://ezgif.com/about'
3656 url: 'http://femtosearch.com/'
3659 url: 'https://www.grierforensics.com/'
3664 url: 'https://github.com/InteractiveAdvertisingBureau/adstxtcrawler'
3667 url: 'https://iabtechlab.com/'
3680 url: 'https://github.com/louislam/uptime-kuma'
3685 url: 'https://platform.openai.com/docs/bots'
3688 url: 'https://openai.com/'
3693 url: 'https://platform.openai.com/docs/bots'
3696 url: 'https://openai.com/'
3701 url: 'https://platform.openai.com/docs/bots'
3704 url: 'https://openai.com/'
3709 url: 'https://www.brightedge.com/'
3712 url: 'https://www.brightedge.com/'
3716 url: 'https://github.com/diem-project/sfFeed2Plugin'
3738 url: 'https://scrapingrobot.com/'
3741 url: 'https://sprious.com/'
3743 - regex: 'Ant(?:\.com beta|Bot)'
3746 url: 'https://www.ant.com/'
3748 name: 'Ant.com Ltd.'
3749 url: 'https://www.ant.com/'
3754 url: 'https://www.webwiki.com/'
3757 url: 'https://www.webwiki.com/'
3767 url: 'https://github.com/matomo-org/matomo'
3775 url: 'https://github.com/prometheus/prometheus'
3798 url: 'https://www.meltwater.com/'
3814 url: 'https://www.bbc.com/'
3819 url: 'https://www.bbc.com/'
3822 url: 'https://www.bbc.com/'
3827 url: 'https://github.com/ClaudeBot/ClaudeBot'
3832 url: 'https://imagesift.com/'
3850 url: 'https://www.builtbybright.com/'
3863 url: 'https://stract.com/webmasters'
3866 url: 'https://github.com/StractOrg/stract/'
3871 url: 'https://geedo.com/bot/'
3876 url: 'https://geedo.com/product-search/'
3881 url: 'https://go.backupland.com/'
3884 url: 'https://go.backupland.com/'
3910 url: 'https://www.example3.com/'
3936 - regex: 'getodin\.com'
3939 url: 'https://docs.getodin.com/'
3942 url: 'https://cyble.com/'
3947 url: 'https://about.you.com/youbot/'
3950 url: 'https://you.com/'
3960 url: 'https://www.seoptimer.com/monitor-backlinks/'
3963 url: 'https://www.seoptimer.com/'
3968 url: 'https://github.com/williamdes/mariadb-mysql-kbs'
3976 url: 'https://github.com/aaamoon/copilot-gpt4-service'
3986 url: 'https://poduptime.com'
3991 url: 'https://www.anthropic.com/'
3994 url: 'https://www.anthropic.com/'
3999 url: 'https://netpeaksoftware.com/checker'
4002 url: 'https://netpeaksoftware.com/'
4007 url: 'https://www.sandoba.com/en/crawler/'
4010 url: 'https://www.sandoba.com/'
4018 url: 'https://www.sirdata.com/'
4023 url: 'https://www.checkmarknetwork.com/spider.html/'
4026 url: 'https://www.checkmarknetwork.com/'
4031 url: 'https://cohere.com/'
4034 url: 'https://cohere.com/'
4047 url: 'https://www.thetradedesk.com/us/ttd-content'
4050 url: 'https://www.thetradedesk.com/'
4055 url: 'https://www.montastic.com/'
4058 url: 'https://www.metadot.com/'
4068 url: 'https://www.mixnode.com/'
4071 url: 'https://www.mixnode.com/'
4080 url: 'https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0'
4083 url: 'https://www.microsoft.com/'
4088 url: 'https://www.virustotal.com/'
4096 url: 'https://tineye.com/'
4099 url: 'https://tineye.com/'
4112 url: 'https://www.botify.com/'
4115 url: 'https://www.botify.com/'
4120 url: 'https://www.alleyesonscreens.com/'
4123 url: 'https://www.alleyesonscreens.com/'
4128 url: 'https://www.webceo.com/'
4131 url: 'https://www.webceo.com/'
4141 url: 'https://www.htmlyse.com/'
4144 url: 'https://www.htmlyse.com/'
4149 url: 'https://www.trendsmap.com/'
4152 url: 'https://www.trendsmap.com/'
4157 url: 'https://www.shareaholic.com/steve'
4160 url: 'https://www.shareaholic.com/'
4165 url: 'https://tools.keycdn.com/geo'
4170 url: 'https://tools.keycdn.com/'
4173 url: 'https://www.keycdn.com/'
4191 url: 'https://www.senuto.com/'
4194 url: 'https://www.senuto.com/'
4199 url: 'https://gozle.com.tm/en/blog/post/1'
4202 url: 'https://gozle.com.tm/'
4207 url: 'https://www.quantcast.com/bot/'
4210 url: 'https://www.quantcast.com/'
4215 url: 'https://www.fontradar.com/'
4218 url: 'https://www.fontradar.com/'
4223 url: 'https://www.viber.com/'
4226 url: 'https://www.viber.com/'
4231 url: 'https://github.com/internetarchive/Zeno'
4239 url: 'https://sentinel.barracudanetworks.com/'
4242 url: 'https://www.barracudanetworks.com/'
4247 url: 'https://community.dynatrace.com/t5/Troubleshooting/Basic-Commands-for-Synthetic/ta-p/198164'
4250 url: 'https://www.dynatrace.com/'
4255 url: 'https://community.dynatrace.com/t5/Troubleshooting/Basic-Commands-for-Synthetic/ta-p/198164'
4258 url: 'https://www.dynatrace.com/'
4263 url: 'https://sitebulb.com/'
4266 url: 'https://sitebulb.com/'
4271 url: 'https://monsido.com/bot-html'
4274 url: 'https://monsido.com/'
4279 url: 'https://www.accompany.com/'
4282 url: 'https://www.accompany.com/'
4287 …url: 'https://docs.ghostinspector.com/faq/#how-do-i-detect-ghost-inspector-test-runner-traffic-on-…
4290 url: 'https://www.ghostinspector.com/'
4295 url: 'https://www.google.com/script/start/'
4308 …url: 'https://support.detectify.com/support/solutions/articles/48001049001-how-to-allow-detectify-…
4311 url: 'https://detectify.com/'
4316 url: 'https://www.domcop.com/bot'
4319 url: 'https://axemantech.com/'
4337 url: 'https://github.com/matrix-org/synapse'
4358 url: 'https://raventools.com/site-auditor/'
4361 url: 'https://www.tapclicks.com/'
4374 url: 'https://help.dubbot.com/en/articles/6746594-example-custom-user-agent'
4377 url: 'https://dubbot.com/'
4382 url: 'https://swiftype.com/swiftbot'
4390 url: 'https://eyemonit.com/'
4393 url: 'https://eyemonit.com/'
4398 url: 'https://www.thousandeyes.com/'
4401 url: 'https://www.cisco.com/'
4426 url: 'https://www.clickagy.com/'
4429 url: 'https://www.clickagy.com/'
4471 url: 'https://support.plesk.com/hc/en-us/articles/13302778306199-What-is-Plesk-Screenshot-Service'
4474 url: 'https://www.plesk.com/'
4484 url: 'https://probely.com/sos/'
4487 url: 'https://probely.com/'
4492 url: 'https://www.uptimia.com/'
4495 url: 'https://www.uptimia.com/'
4500 url: 'https://2gdpr.com/tos'
4503 url: 'https://2gdpr.com/'
4532 url: 'https://www.insytful.com/'
4535 url: 'https://www.zengenti.com/'
4537 - regex: 'statista\.com'
4540 url: 'https://www.statista.com/'
4543 url: 'https://www.statista.com/'
4548 url: 'https://substack.com/'
4551 url: 'https://substack.com/'
4556 …url: 'https://www.copyright.com/blog/ccc-expands-corporate-solutions-offering-with-new-technology/'
4559 url: 'https://www.copyright.com/'
4561 - regex: 'LiveJournal\.com'
4563 url: 'https://www.livejournal.com/'
4567 url: 'https://www.livejournal.com/'
4572 url: 'https://bitdiscovery.com/'
4575 url: 'https://www.tenable.com/'
4585 url: 'https://github.com/elastic/synthetics'
4593 url: 'http://www.htmlhelp.com/tools/validator/'
4608 url: 'https://web.archive.org/web/20130821043949/http://www.branddimensions.com/'
4613 url: 'https://github.com/openeasm/punkmap'
4618 url: 'https://www.nokia.com/networks/ip-networks/deepfield/genome/'
4621 url: 'https://www.nokia.com/'
4631 url: 'https://web.archive.org/web/20041020050801/http://www.alltheweb.com/help/webmaster/crawler'
4649 url: 'https://muv.com/'
4654 url: 'https://www.redeken.com/en/help/bot.html'
4657 url: 'https://www.redeken.com/'
4683 url: 'https://www.validbot.com/'
4686 url: 'https://www.validbot.com/'
4691 url: 'https://www.cscglobal.com/cscglobal/home/'
4694 url: 'https://www.cscglobal.com/'
4696 - regex: 'domaincodex\.com'
4699 url: 'https://www.domaincodex.com/'
4702 url: 'https://www.eriedatasys.com/'
4707 url: 'https://swisscows.com/'
4710 url: 'https://swisscows.com/'
4720 url: 'https://workona.com/'
4723 url: 'https://workona.com/'
4728 url: 'https://web.archive.org/web/20140309033202/http://www.bloglines.com/'
4731 url: 'https://www.reply.com/'
4736 url: 'https://webarchive.jira.com/wiki/display/Heritrix/Heritrix'
4760 url: 'https://www.functionize.com/'
4763 url: 'https://www.functionize.com/'
4787 url: 'https://www.lcxventures.com/'
4800 url: 'https://github.com/wikimedia/mediawiki-services-zotero'
4808 url: 'https://muckrack.com/'
4811 url: 'https://muckrack.com/'
4829 url: 'https://search.brave.com/help/brave-search-crawler'
4832 url: 'https://brave.com/'
4837 url: 'https://www.1001firms.com/1001firmsbot.php'
4842 url: 'https://help.steampowered.com/en/faqs/view/595C-42F4-3B66-E02F'
4845 url: 'https://www.valvesoftware.com/'
4858 url: 'https://www.inspici.com/'
4861 url: 'https://www.inspici.com/'
4866 url: 'https://www.peer39.com/crawler-notice'
4869 url: 'https://www.peer39.com/'
4874 url: 'https://www.domainsbot.com/business-intelligence/'
4877 url: 'https://www.domainsbot.com/'
4882 url: 'https://web.archive.org/web/20151228225429/https://cloudservermarket.com/spider.html'
4887 url: 'https://visual-seo.com/Pigafetta-Bot'
4890 url: 'http://www.astonishstudio.com/'
4903 url: 'https://github.com/nettrom/suggestbot'
4921 url: 'https://www.sitesell.com/sbider.html'
4924 url: 'https://www.sitesell.com/'
4929 url: 'https://www.lightspeedsystems.com/'
4932 url: 'https://www.lightspeedsystems.com/'
4953 …url: 'https://wpmudev.com/docs/wpmu-dev-plugins/broken-link-checker/#broken-link-checker-user-agen…
4956 url: 'https://incsub.com/'
4979 url: 'https://web.archive.org/web/20121230203310/http://www.crystalsemantics.com/user-agent/'
4982 url: 'https://web.archive.org/web/20121029062239/http://www.crystalsemantics.com/'
4997 url: 'https://github.com/babycoff/xtate'
5002 url: 'https://fedilist.com/'
5007 url: 'https://github.com/grafana/grafana'
5010 url: 'https://grafana.com/'
5015 url: 'https://github.com/atmos/camo'
5018 …url: 'https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/about-anonymi…
5037 …ownload Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr\.com|teoma|oegp|http%20c…