[extensions] Update some more bots #780: Better Uptime Bot, SemrushBot, Yahoo! Slurp

Author: faisalman

src/extensions/ua-parser-extensions.js

@@ -78,7 +78,7 @@ const Crawlers = Object.freeze({
             /(ia_archiver|archive\.org_bot)\/?([\w\.]*)/i,
 
             // SemrushBot - http://www.semrush.com/bot.html
-            /((?:semrush|splitsignal)bot[-abcfimostw]*)\/([\w\.-]+)/i,
+            /((?:semrush|splitsignal)bot[-abcfimostw]*)\/?([\w\.-]*)/i,
 
             // Sogou Spider
             /(sogou (?:pic|head|web|orion|news) spider)\/([\w\.]+)/i, 
@@ -110,7 +110,7 @@ const Crawlers = Object.freeze({
             // Qihoo 360Spider
             // TurnitinBot - https://www.turnitin.com/robot/crawlerinfo.html
             // Yahoo! Slurp - http://help.yahoo.com/help/us/ysearch/slurp
-            /\b(360spider-?(?:image|video)?|bytespider|(?:ai2|aspiegel|dataforseo|imagesift|petal|turnitin)bot|teoma|(?=yahoo! )slurp)/i
+            /\b(360spider-?(?:image|video)?|bytespider|(?:ai2|aspiegel|dataforseo|imagesift|petal|turnitin)bot|teoma|yahoo! slurp)/i
         ], 
         [NAME, [TYPE, CRAWLER]]
     ]
@@ -219,7 +219,7 @@ const Fetchers = Object.freeze({
             // AhrefsSiteAudit - https://ahrefs.com/robot/site-audit
             // ChatGPT-User - https://platform.openai.com/docs/plugins/bot
             // DuckAssistBot - https://duckduckgo.com/duckassistbot/
-            // BingPreview / Mastodon / Pinterestbot / Redditbot / Rogerbot / SiteAuditBot / Telegrambot / Twitterbot / UptimeRobot
+            // Better Uptime / BingPreview / Mastodon / Pinterestbot / Redditbot / Rogerbot / SiteAuditBot / Telegrambot / Twitterbot / UptimeRobot
             // Google Site Verifier / Meta / Yahoo! Japan
             // Yandex Bots - https://yandex.com/bots
             /(ahrefssiteaudit|bingpreview|chatgpt-user|mastodon|(?:discord|duckassist|linkedin|pinterest|reddit|roger|siteaudit|twitter|uptimero)bot|google-site-verification|meta-externalfetcher|y!?j-dlc|yandex(?:calendar|direct(?:dyn)?|searchshop)|yadirectfetcher)\/([\w\.]+)/i,
@@ -240,7 +240,7 @@ const Fetchers = Object.freeze({
 
         [
             // Google Bots / Cohere / Snapchat / Vercelbot / Yandex Bots
-            /(cohere-ai|vercelbot|feedfetcher-google|google(?:imageproxy|-read-aloud|-pagerenderer|producer)|snap url preview|telegrambot|yandex(?:sitelinks|userproxy))/i
+            /((?:better uptime |telegram|vercel)bot|cohere-ai|feedfetcher-google|google(?:imageproxy|-read-aloud|-pagerenderer|producer)|snap url preview|yandex(?:sitelinks|userproxy))/i
         ], 
         [NAME, [TYPE, FETCHER]],
     ]

test/data/ua/extension/crawler.json

@@ -480,6 +480,26 @@
             "type"    : "crawler"
         }
     },
+    {
+        "desc"    : "SemrushBot for Backlink Audit tool",
+        "ua"      : "Mozilla/5.0 (compatible; SemrushBot-BA; +http://www.semrush.com/bot.html)",
+        "expect"  :
+        {
+            "name"    : "SemrushBot-BA",
+            "version" : "undefined",
+            "type"    : "crawler"
+        }
+    },
+    {
+        "desc"    : "SemrushBot for On Page SEO Checker tool",
+        "ua"      : "Mozilla/5.0 (compatible; SemrushBot-SI/0.97; +http://www.semrush.com/bot.html)",
+        "expect"  :
+        {
+            "name"    : "SemrushBot-SI",
+            "version" : "0.97",
+            "type"    : "crawler"
+        }
+    },
     {
         "desc"    : "SemrushBot for ContentShake AI tool",
         "ua"      : "Mozilla/5.0 (compatible; SemrushBot-OCOB/1; +https://www.semrush.com/bot/)",
@@ -550,6 +570,16 @@
             "type"    : "crawler"
         }
     },
+    {
+        "desc"    : "Yahoo! Slurp",
+        "ua"      : "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)  ",
+        "expect"  :
+        {
+            "name"    : "Yahoo! Slurp",
+            "version" : "undefined",
+            "type"    : "crawler"
+        }
+    },
     {
         "desc"    : "YandexBot",
         "ua"      : "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)",

test/data/ua/extension/fetcher.json

@@ -9,6 +9,16 @@
             "type"    : "fetcher"
         }
     },
+    {
+        "desc"    : "Better Uptime Bot",
+        "ua"      : "Better Uptime Bot Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36",
+        "expect"  :
+        {
+            "name"    : "Better Uptime Bot",
+            "version" : "undefined",
+            "type"    : "fetcher"
+        }
+    },
     {
         "desc"    : "BingPreview",
         "ua"      : "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b",