|
498 | 498 | "Mozilla/5.0 (Linux; Android 5.1; CUBOT_NOTE_S Build/LMY47I) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36",
|
499 | 499 | "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.91 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
|
500 | 500 | "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.5845.140 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
|
| 501 | + "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.6422.175 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", |
501 | 502 | "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
|
502 | 503 | "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.99 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
|
503 | 504 | "Mozilla/5.0 (compatible; heritrix/3.1.2-SNAPSHOT-20131029-0036 +http://archive.org/details/archive.orgbot)",
|
504 | 505 | "Mozilla/5.0 (compatible; Yeti/1.1; +https://naver.me/spd)",
|
505 | 506 | "Mozilla/5.0 (iPhone; CPU iPhone OS 83 like Mac OS X) AppleWebKit/537.36 (KHTML like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
|
506 | 507 | "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
|
507 |
| - "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.6422.175 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", |
508 | 508 | "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/97.0.4692.71 Safari/537.36",
|
509 | 509 | "SocialRankIOBot; http://socialrank.io/about",
|
510 | 510 | "GoogleProducer; (+http://goo.gl/7y4SX)",
|
|
640 | 640 | "admantx-euastn/2.5 (+http://www.admantx.com/service-fetcher.html)",
|
641 | 641 | "annotategoogle; http://ponderer.org/download/annotategoogle.user.js",
|
642 | 642 | "Barkrowler/0.5.1 (experimenting / debugging - sorry for your logs ) http://www.exensa.com/crawl - [email protected] -- based on BuBiNG",
|
| 643 | + "colly - https://github.com/gocolly/colly/v2", |
643 | 644 | "Mozilla/5.0 (compatible; Baiduspider-cpro; +http://www.baidu.com/search/spider.html)",
|
644 | 645 | "Mozilla/5.0 (compatible; JikeSpider; +http://shoulu.jike.com/spider.html)",
|
645 | 646 | "Mozilla/5.0 (compatible; Yeti/1.1; +http://help.naver.com/support/robots.html)",
|
|
658 | 659 | "AdsBot-Google-Mobile (+http://www.google.com/mobile/adsbot.html) Mozilla (iPhone; U; CPU iPhone OS 3 0 like Mac OS X) AppleWebKit (KHTML like Gecko) Mobile Safari",
|
659 | 660 | "Barkrowler/0.9 (+http://www.exensa.com/crawl)",
|
660 | 661 | "CCBot/1.0 (+http://www.commoncrawl.org/bot.html)",
|
661 |
| - "colly - https://github.com/gocolly/colly/v2", |
662 | 662 | "CRAZYWEBCRAWLER 0.9.10 http://www.crazywebcrawler.com",
|
663 | 663 | "CRAZYWEBCRAWLER 0.9.6 http://www.crazywebcrawler.com",
|
664 | 664 | "DomainCrawler/3.0 ( [email protected]; http://www.domaincrawler.com/whatmyip.co)",
|
|
999 | 999 | "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/117.0.5938.88 Safari/537.36",
|
1000 | 1000 | "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/117.0.5938.92 Safari/537.36",
|
1001 | 1001 | "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/89.0.4389.127 Safari/537.36",
|
| 1002 | + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)", |
1002 | 1003 | "My Nutch Spider/Nutch-1.6",
|
1003 | 1004 | "Re-re Studio (+http://re-re.ru/)",
|
1004 | 1005 | "SemrushBot",
|
|
1054 | 1055 | "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/81.0.4044.108 Safari/537.36",
|
1055 | 1056 | "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/89.0.4389.84 Safari/537.36",
|
1056 | 1057 | "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/96.0.4664.93 Safari/537.36",
|
1057 |
| - "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)", |
1058 | 1058 | "MSNbot",
|
1059 | 1059 | "niki-bot",
|
1060 | 1060 | "omgili/0.5 +http://omgili.com",
|
|
1359 | 1359 | "Mozilla/5.0 (compatible; PaperLiBot/2.1; https://support.paper.li/entries/20023257-what-is-paper-li)",
|
1360 | 1360 | "Mozilla/5.0 (compatible; Pinterestbot/1.0; +http://www.pinterest.com/bot.html)",
|
1361 | 1361 | "Mozilla/5.0 (compatible; Scopia Crawler 1.2; +http://www.scopia.co)",
|
| 1362 | + "Mozilla/5.0 (compatible; SemrushBot-SWA/0.1; +http://www.semrush.com/bot.html)", |
1362 | 1363 | "Mozilla/5.0 (compatible; SemrushBot/0.97bl; +http://www.semrush.com/bot.html)",
|
1363 | 1364 | "Mozilla/5.0 (compatible; SemrushBot/0.99bl; +http://www.semrush.com/bot.html)",
|
1364 | 1365 | "Mozilla/5.0 (compatible; SEOkicks-Robot +http://www.seokicks.de/robot.html)",
|
|
1535 | 1536 | "Mozilla/5.0 (compatible; Qwantify/vR; +https://www.qwant.com/)",
|
1536 | 1537 | "Mozilla/5.0 (compatible; rogerBot/1.0; UrlCrawler; http://www.seomoz.org/dp/rogerbot)",
|
1537 | 1538 | "Mozilla/5.0 (compatible; SemrushBot-SA/0.97; +http://www.semrush.com/bot.html)",
|
1538 |
| - "Mozilla/5.0 (compatible; SemrushBot-SWA/0.1; +http://www.semrush.com/bot.html)", |
1539 | 1539 | "Mozilla/5.0 (compatible; SemrushBot/1.2~bl; http://www.semrush.com/bot.html)",
|
1540 | 1540 | "Mozilla/5.0 (compatible; SemrushBot; +http",
|
1541 | 1541 | "Mozilla/5.0 (compatible; SeznamBot/3.2-test1; +http://fulltext.sblog.cz/)",
|
|
2201 | 2201 | "IABot: Checking if link from Wikipedia is broken and needs removal - See https://meta.wikimedia.org/wiki/InternetArchiveBot/FAQ_for_sysadmins",
|
2202 | 2202 | "IAS crawler (page scorer; http://integralads.com/site-indexing-policy/)",
|
2203 | 2203 | "ias-jp/3.1 (+https://www.admantx.com/service-fetcher.html)",
|
| 2204 | + "ias-or/3.3 (former https://www.admantx.com + https://integralads.com/about-ias/)", |
2204 | 2205 | "ICC-Crawler/2.0 (Mozilla-compatible; ; http://kc.nict.go.jp/project1/crawl.html)",
|
2205 | 2206 | "ICC-Crawler/2.0 (Mozilla-compatible; ; http://www.nict.go.jp/en/univ-com/plan/crawl.html)",
|
2206 | 2207 | "Iframely/0.8.6 (+http://iframely.com/;)",
|
|
2776 | 2777 | "HTTP_Request2/2.3.0 (http://pear.php.net/package/http_request2) PHP/7.0.32-0ubuntu0.16.04.1",
|
2777 | 2778 | "HybridBot (hybrid.ru/about. If our bot caused problems please contact us. Contact email: [email protected])",
|
2778 | 2779 | "ias-au/3.1 (+https://www.admantx.com/service-fetcher.html)",
|
2779 |
| - "ias-or/3.3 (former https://www.admantx.com + https://integralads.com/about-ias/)", |
2780 | 2780 | "ICC-Crawler/2.0 (Mozilla-compatible; ; http://ucri.nict.go.jp/en/icccrawler.html)",
|
2781 | 2781 | "Iframely/1.3.1 (+https://iframely.com/docs/about) Notion",
|
2782 | 2782 | "Is is up? (+https://isitup.org)",
|
|
2828 | 2828 | "Mozilla/5.0 (compatible; KZ BRAIN Mobile; http://servermobile.net) @124.146.175.4",
|
2829 | 2829 | "Mozilla/5.0 (compatible; LinkpadBot/2.3; +http://linkpad.org/robot/)",
|
2830 | 2830 | "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Img/2.0; +https://help.mail.ru/webmaster/indexing/robots)",
|
| 2831 | + "Mozilla/5.0 (compatible; Linux; AGAKIDSBOT/3.1 +agakids.ru/project) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", |
2831 | 2832 | "Mozilla/5.0 (compatible; MJ12bot/v1.2.1; http://www.majestic12.co.uk/bot.php?+)",
|
2832 | 2833 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.0; http://www.majestic12.co.uk/bot.php?+)",
|
2833 | 2834 | "Mozilla/5.0 (compatible; MojeekBot/0.11; +https://www.mojeek.com/bot.html)",
|
|
3541 | 3542 | "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1718785656-0",
|
3542 | 3543 | "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1718790668-0",
|
3543 | 3544 | "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1718834075-0",
|
| 3545 | + "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1719230533-0", |
| 3546 | + "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1719236756-0", |
| 3547 | + "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1719240581-0", |
| 3548 | + "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1719259503-0", |
| 3549 | + "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1719318807-0", |
3544 | 3550 | "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) tands-prod-eng.hlfs-prod---sieve.hlfs-desktop/1603028876-0",
|
3545 | 3551 | "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) tands-prod-eng.hlfs-prod---sieve.hlfs-desktop/1603256091-0",
|
3546 | 3552 | "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) tands-prod-eng.hlfs-prod---sieve.hlfs-desktop/1603374815-0",
|
|
0 commit comments