[extensions][helpers] Update Semrush bot variants

Author: faisalman

package-lock.json

@@ -52,7 +52,6 @@ const Crawlers = Object.freeze({
             // MojeekBot - https://www.mojeek.com/bot.html
             // OpenAI's SearchGPT - https://platform.openai.com/docs/bots
             // PerplexityBot - https://perplexity.ai/perplexitybot
-            // SemrushBot - http://www.semrush.com/bot.html
             // SeznamBot - http://napoveda.seznam.cz/seznambot-intro
             /((?:ahrefs|amazon|bing|cc|dot|duckduck|exa|facebook|gpt|mj12|mojeek|oai-search|perplexity|semrush|seznam)bot)\/([\w\.-]+)/i,
 
@@ -78,6 +77,9 @@ const Crawlers = Object.freeze({
             // Internet Archive (archive.org)
             /(ia_archiver|archive\.org_bot)\/?([\w\.]*)/i,
 
+            // SemrushBot - http://www.semrush.com/bot.html
+            /((?:semrush|splitsignal)bot[-abcfimostw]*)\/([\w\.-]+)/i,
+
             // Sogou Spider
             /(sogou (?:pic|head|web|orion|news) spider)\/([\w\.]+)/i, 
 
@@ -217,10 +219,10 @@ const Fetchers = Object.freeze({
             // AhrefsSiteAudit - https://ahrefs.com/robot/site-audit
             // ChatGPT-User - https://platform.openai.com/docs/plugins/bot
             // DuckAssistBot - https://duckduckgo.com/duckassistbot/
-            // BingPreview / Mastodon / Pinterestbot / Redditbot / Rogerbot / Telegrambot / Twitterbot / UptimeRobot
+            // BingPreview / Mastodon / Pinterestbot / Redditbot / Rogerbot / SiteAuditBot / Telegrambot / Twitterbot / UptimeRobot
             // Google Site Verifier / Meta / Yahoo! Japan
             // Yandex Bots - https://yandex.com/bots
-            /(ahrefssiteaudit|bingpreview|chatgpt-user|mastodon|(?:discord|duckassist|linkedin|pinterest|reddit|roger|telegram|twitter|uptimero)bot|google-site-verification|meta-externalfetcher|y!?j-dlc|yandex(?:calendar|direct(?:dyn)?|searchshop)|yadirectfetcher)\/([\w\.]+)/i,
+            /(ahrefssiteaudit|bingpreview|chatgpt-user|mastodon|(?:discord|duckassist|linkedin|pinterest|reddit|roger|siteaudit|telegram|twitter|uptimero)bot|google-site-verification|meta-externalfetcher|y!?j-dlc|yandex(?:calendar|direct(?:dyn)?|searchshop)|yadirectfetcher)\/([\w\.]+)/i,
 
             // Bluesky
             /(bluesky) cardyb\/([\w\.]+)/i,

src/extensions/ua-parser-extensions.js

@@ -93,6 +93,9 @@ const isAIBot = (resultOrUA) => [
     // Perplexity
     'perplexitybot',
 
+    // Semrush
+    'semrushbot-ocob',
+
     // Timpi
     'timpibot',
 

src/helpers/ua-parser-helpers.js

@@ -480,6 +480,16 @@
             "type"    : "crawler"
         }
     },
+    {
+        "desc"    : "SemrushBot for ContentShake AI tool",
+        "ua"      : "Mozilla/5.0 (compatible; SemrushBot-OCOB/1; +https://www.semrush.com/bot/)",
+        "expect"  :
+        {
+            "name"    : "SemrushBot-OCOB",
+            "version" : "1",
+            "type"    : "crawler"
+        }
+    },
     {
         "desc"    : "SeznamBot",
         "ua"      : "Mozilla/5.0 (compatible; SeznamBot/4.0-RC1; +http://napoveda.seznam.cz/seznambot-intro/)",

test/data/ua/extension/crawler.json

@@ -119,6 +119,16 @@
             "type"    : "fetcher"
         }
     },
+    {
+        "desc"    : "SiteAuditBot",
+        "ua"      : "Mozilla/5.0 (compatible; SiteAuditBot/0.97; +http://www.semrush.com/bot.html)",
+        "expect"  :
+        {
+            "name"    : "SiteAuditBot",
+            "version" : "0.97",
+            "type"    : "fetcher"
+        }
+    },
     {
         "desc"    : "UptimeRobot",
         "ua"      : "Mozilla/5.0 (compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)",

test/data/ua/extension/fetcher.json

@@ -40,11 +40,13 @@ describe('isAIBot', () => {
         const claudeBot = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)';
         const firefox = 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/111.0';
         const searchGPT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot';
+        const semrushAI = 'Mozilla/5.0 (compatible; SemrushBot-OCOB/1; +https://www.semrush.com/bot/)';
 
         assert.equal(isAIBot(UAParser(claudeBot, Bots)), true);
         assert.equal(isAIBot(claudeBot), true);
         assert.equal(isAIBot(firefox), false);
         assert.equal(isAIBot(searchGPT), true);
+        assert.equal(isAIBot(semrushAI), true);
     });
 });