feat(ai): Normalize model names for a better cost calculation (#103508)

After the changes in getsentry/relay#5374, we
can now start generating globs only for prefix matching. We no longer
need to match suffixes, because the model name normalization is now done
in relay.

Closes [TET-1268: Normalize models name in relay to make it easier to
calculate
costs](https://linear.app/getsentry/issue/TET-1268/normalize-models-name-in-relay-to-make-it-easier-to-calculate-costs)

Author: vgrozdanic

src/sentry/tasks/ai_agent_monitoring.py

@@ -26,55 +26,28 @@
 MODELS_DEV_API_URL = "https://models.dev/api.json"
 
 
-def _create_suffix_glob_model_name(model_id: str) -> str:
+def _normalize_model_id(model_id: str) -> str:
     """
-    Create a suffix glob version of a model name by stripping dates and versions.
-
-    Examples:
-    - "claude-4-sonnet-20250522" -> "claude-4-sonnet-*"
-    - "o3-pro-2025-06-10" -> "o3-pro-*"
-    - "claude-3-5-haiku@20241022" -> "claude-3-5-haiku@*"
-    - "claude-opus-4-1-20250805-v1:0" -> "claude-opus-4-1-*"
+    Normalize a model id by removing dates and versions.
+    Example:
+    - "gpt-4" -> "gpt-4"
+    - "gpt-4-20241022" -> "gpt-4"
+    - "gpt-4-v1.0" -> "gpt-4"
+    - "gpt-4-20241022-v1.0" -> "gpt-4"
+    - "gpt-4-20241022-v1.0-beta" -> "gpt-4"
+    - "gpt-4-20241022-v1.0-beta-1" -> "gpt-4"
+    - "gpt-4-20241022-v1.0-beta-1" -> "gpt-4"
 
     Args:
-        model_id: The original model ID
+        model_id: The model id to normalize
 
     Returns:
-        The glob version of the model name
+        The normalized model id
     """
-    # Pattern to match various date and version formats
-    # Matches:
-    # - YYYYMMDD (e.g., 20250522)
-    # - YYYY-MM-DD (e.g., 2025-06-10)
-    # - YYYY/MM/DD (e.g., 2025/06/10)
-    # - YYYY.MM.DD (e.g., 2025.06.10)
-    # - v followed by version numbers (e.g., v1:0, v2.1, v3)
-    # - @ followed by dates (e.g., @20241022)
-    # - -v followed by version numbers (e.g., -v1.0)
-    # - _v followed by version numbers (e.g., _v1.0)
-
-    # Use a single comprehensive regex that handles all patterns
-    # This regex matches:
-    # 1. Date patterns: -YYYYMMDD, -YYYY-MM-DD, -YYYY/MM/DD, -YYYY.MM.DD
-    # 2. Version patterns: -v1.0, -v1:0, _v1.0, _v1:0
-    # 3. @date patterns: @YYYYMMDD, @YYYY-MM-DD, @YYYY/MM/DD, @YYYY.MM.DD
-    # 4. Combined patterns: -YYYYMMDD-v1:0, @YYYYMMDD-v1:0
-
-    # First, handle @date patterns (they have special handling)
-    glob_name = re.sub(
-        r"@(?:19|20)\d{2}(?:[-_/.]?\d{2}){2}(?:[-_]v\d+(?:[.:]\d+)*)?", "@*", model_id
-    )
-
-    # Then handle regular date and version patterns
-    glob_name = re.sub(
-        r"([-_])(?:19|20)\d{2}(?:[-_/.]?\d{2}){2}(?:[-_]v\d+(?:[.:]\d+)*)?", r"\1*", glob_name
+    return re.sub(
+        r"(([-_@])(\d{4}[-/.]\d{2}[-/.]\d{2}|\d{8}))?([-_]v\d+[:.]?\d*([-:].*)?)?$", "", model_id
     )
 
-    # Handle standalone version patterns (without dates)
-    glob_name = re.sub(r"([-_])v\d+(?:[.:]\d+)*", r"\1*", glob_name)
-
-    return glob_name
-
 
 def _create_prefix_glob_model_name(model_id: str) -> str:
     """
@@ -87,8 +60,6 @@ def _create_prefix_glob_model_name(model_id: str) -> str:
     - "gpt-4" -> "*gpt-4"
     - "claude-3-5-sonnet" -> "*claude-3-5-sonnet"
     - "o3-pro" -> "*o3-pro"
-    - "gpt-4o-mini-*" -> "*gpt-4o-mini-*"
-    - "model@*" -> "*model@*"
 
     Args:
         model_id: The original model ID or a suffix-globbed model name
@@ -104,13 +75,10 @@ def _add_glob_model_names(models_dict: dict[ModelId, AIModelCostV2]) -> None:
     """
     Add glob versions of model names to the models dictionary.
 
-    For each model, creates three types of glob versions:
-    1. Suffix pattern: stripping dates and versions (e.g., "model-20241022" -> "model-*")
-    2. Prefix pattern: adding wildcard prefix (e.g., "gpt-4" -> "*gpt-4")
-    3. Prefix-suffix pattern: both wildcards for suffix-globbed names (e.g., "model-*" -> "*model-*")
+    For each model, it creates a normalized model name, and a prefix glob version of
+    the model name.
+
 
-    We DO NOT want to have prefix-suffix glob patterns for models that don't have a suffix glob pattern
-    because that would result in too fuzzy matching, e.g. "gpt-4" -> "*gpt-4*" would match "gpt-4o-mini".
 
     Args:
         models_dict: The dictionary of models to add glob versions to
@@ -120,20 +88,13 @@ def _add_glob_model_names(models_dict: dict[ModelId, AIModelCostV2]) -> None:
     model_ids = list(models_dict.keys())
 
     for model_id in model_ids:
-        # Add suffix glob pattern (strip dates/versions)
-        suffix_glob_name = _create_suffix_glob_model_name(model_id)
-        if suffix_glob_name != model_id and suffix_glob_name not in models_dict:
-            models_dict[suffix_glob_name] = models_dict[model_id]
-
-            # Add prefix-suffix glob pattern (both wildcards) only for models that have a suffix glob
-            prefix_suffix_glob_name = _create_prefix_glob_model_name(suffix_glob_name)
-            if prefix_suffix_glob_name not in models_dict:
-                models_dict[prefix_suffix_glob_name] = models_dict[model_id]
-
-        # Add prefix glob pattern (wildcard prefix)
-        prefix_glob_name = _create_prefix_glob_model_name(model_id)
+        normalized_model_id = _normalize_model_id(model_id)
+        if normalized_model_id != model_id and normalized_model_id not in models_dict:
+            models_dict[normalized_model_id] = models_dict[model_id]
+
+        prefix_glob_name = _create_prefix_glob_model_name(normalized_model_id)
         if prefix_glob_name not in models_dict:
-            models_dict[prefix_glob_name] = models_dict[model_id]
+            models_dict[prefix_glob_name] = models_dict[normalized_model_id]
 
 
 @instrumented_task(

tests/sentry/tasks/test_ai_agent_monitoring.py

@@ -482,9 +482,9 @@ def test_fetch_ai_model_costs_custom_model_mapping(self) -> None:
         assert "nonexistent-mapping" not in models
 
     @responses.activate
-    def test_fetch_ai_model_costs_with_glob_model_names(self) -> None:
-        """Test that glob versions of model names are added correctly"""
-        # Mock responses with models that should generate glob patterns
+    def test_fetch_ai_model_costs_with_normalized_and_prefix_glob_names(self) -> None:
+        """Test that normalized and prefix glob versions of model names are added correctly"""
+        # Mock responses with models that have dates/versions that should be normalized
         mock_openrouter_response = {
             "data": [
                 {
@@ -502,7 +502,7 @@ def test_fetch_ai_model_costs_with_glob_model_names(self) -> None:
                     },
                 },
                 {
-                    "id": "openai/gpt-4",  # No date/version, should not generate glob
+                    "id": "openai/gpt-4",  # No date/version, normalized version same as original
                     "pricing": {
                         "prompt": "0.0000003",
                         "completion": "0.00000165",
@@ -548,104 +548,100 @@ def test_fetch_ai_model_costs_with_glob_model_names(self) -> None:
         assert "claude-3-5-haiku@20241022" in models
         assert "o3-pro-2025-06-10" in models
 
-        # Check suffix glob versions were added
-        assert "gpt-4o-mini-*" in models
-        assert "claude-3-5-sonnet-*" in models
-        assert "claude-3-5-haiku@*" in models
-        assert "o3-pro-*" in models
+        # Check normalized versions were added (dates/versions removed)
+        assert "gpt-4o-mini" in models
+        assert "claude-3-5-sonnet" in models
+        assert "claude-3-5-haiku" in models  # @ is not part of the date pattern
+        assert "o3-pro" in models
 
-        # Check prefix glob versions were added
-        assert "*gpt-4o-mini-20250522" in models
-        assert "*claude-3-5-sonnet-20241022" in models
+        # Check prefix glob versions of normalized models were added
+        assert "*gpt-4o-mini" in models
+        assert "*claude-3-5-sonnet" in models
         assert "*gpt-4" in models
-        assert "*claude-3-5-haiku@20241022" in models
-        assert "*o3-pro-2025-06-10" in models
+        assert "*claude-3-5-haiku" in models
+        assert "*o3-pro" in models
 
-        # Check prefix-suffix glob versions were added (only for models with suffix globs)
-        assert "*gpt-4o-mini-*" in models
-        assert "*claude-3-5-sonnet-*" in models
-        assert "*claude-3-5-haiku@*" in models
-        assert "*o3-pro-*" in models
-
-        # Verify glob versions have same pricing as original models
+        # Verify normalized versions have same pricing as original models
         gpt4o_mini_original = models["gpt-4o-mini-20250522"]
-        gpt4o_mini_glob = models["gpt-4o-mini-*"]
-        assert gpt4o_mini_original.get("inputPerToken") == gpt4o_mini_glob.get("inputPerToken")
-        assert gpt4o_mini_original.get("outputPerToken") == gpt4o_mini_glob.get("outputPerToken")
+        gpt4o_mini_normalized = models["gpt-4o-mini"]
+        assert gpt4o_mini_original.get("inputPerToken") == gpt4o_mini_normalized.get(
+            "inputPerToken"
+        )
+        assert gpt4o_mini_original.get("outputPerToken") == gpt4o_mini_normalized.get(
+            "outputPerToken"
+        )
 
         claude_sonnet_original = models["claude-3-5-sonnet-20241022"]
-        claude_sonnet_glob = models["claude-3-5-sonnet-*"]
-        assert claude_sonnet_original.get("inputPerToken") == claude_sonnet_glob.get(
+        claude_sonnet_normalized = models["claude-3-5-sonnet"]
+        assert claude_sonnet_original.get("inputPerToken") == claude_sonnet_normalized.get(
             "inputPerToken"
         )
-        assert claude_sonnet_original.get("outputPerToken") == claude_sonnet_glob.get(
+        assert claude_sonnet_original.get("outputPerToken") == claude_sonnet_normalized.get(
             "outputPerToken"
         )
 
         claude_haiku_original = models["claude-3-5-haiku@20241022"]
-        claude_haiku_glob = models["claude-3-5-haiku@*"]
-        assert claude_haiku_original.get("inputPerToken") == claude_haiku_glob.get("inputPerToken")
-        assert claude_haiku_original.get("outputPerToken") == claude_haiku_glob.get(
+        claude_haiku_normalized = models["claude-3-5-haiku"]
+        assert claude_haiku_original.get("inputPerToken") == claude_haiku_normalized.get(
+            "inputPerToken"
+        )
+        assert claude_haiku_original.get("outputPerToken") == claude_haiku_normalized.get(
             "outputPerToken"
         )
 
         o3_pro_original = models["o3-pro-2025-06-10"]
-        o3_pro_glob = models["o3-pro-*"]
-        assert o3_pro_original.get("inputPerToken") == o3_pro_glob.get("inputPerToken")
-        assert o3_pro_original.get("outputPerToken") == o3_pro_glob.get("outputPerToken")
+        o3_pro_normalized = models["o3-pro"]
+        assert o3_pro_original.get("inputPerToken") == o3_pro_normalized.get("inputPerToken")
+        assert o3_pro_original.get("outputPerToken") == o3_pro_normalized.get("outputPerToken")
 
-        # Verify gpt-4 (no date/version) doesn't have a suffix glob version
-        assert "gpt-4*" not in models
-
-        # Verify prefix glob versions have same pricing as original models
-        gpt4_original = models["gpt-4"]
+        # Verify prefix glob versions have same pricing as normalized models
+        gpt4_normalized = models["gpt-4"]
         gpt4_prefix_glob = models["*gpt-4"]
-        assert gpt4_original.get("inputPerToken") == gpt4_prefix_glob.get("inputPerToken")
-        assert gpt4_original.get("outputPerToken") == gpt4_prefix_glob.get("outputPerToken")
+        assert gpt4_normalized.get("inputPerToken") == gpt4_prefix_glob.get("inputPerToken")
+        assert gpt4_normalized.get("outputPerToken") == gpt4_prefix_glob.get("outputPerToken")
 
-        # Verify prefix-suffix glob versions have same pricing as original models
-        gpt4o_mini_prefix_suffix_glob = models["*gpt-4o-mini-*"]
-        assert gpt4o_mini_original.get("inputPerToken") == gpt4o_mini_prefix_suffix_glob.get(
+        gpt4o_mini_prefix_glob = models["*gpt-4o-mini"]
+        assert gpt4o_mini_normalized.get("inputPerToken") == gpt4o_mini_prefix_glob.get(
             "inputPerToken"
         )
-        assert gpt4o_mini_original.get("outputPerToken") == gpt4o_mini_prefix_suffix_glob.get(
+        assert gpt4o_mini_normalized.get("outputPerToken") == gpt4o_mini_prefix_glob.get(
             "outputPerToken"
         )
 
-    @responses.activate
-    def test_create_suffix_glob_model_name_various_formats(self) -> None:
-        """Test suffix glob generation with various date and version formats"""
-        from sentry.tasks.ai_agent_monitoring import _create_suffix_glob_model_name
+    def test_normalize_model_id(self) -> None:
+        """Test model ID normalization with various date and version formats"""
+        from sentry.tasks.ai_agent_monitoring import _normalize_model_id
 
         # Test cases with expected outputs
         test_cases = [
-            ("model-20250522", "model-*"),  # YYYYMMDD -> *
-            ("model-2025-06-10", "model-*"),  # YYYY-MM-DD -> *
-            ("model-2025/06/10", "model-*"),  # YYYY/MM/DD -> *
-            ("model-2025.06.10", "model-*"),  # YYYY.MM.DD -> *
-            ("model-v1.0", "model-*"),  # v1.0 -> *
-            ("model-v2.1.0", "model-*"),  # v2.1.0 -> *
-            ("model@20241022", "model@*"),  # @YYYYMMDD -> @*
-            ("model-v1:0", "model-*"),  # v1:0 -> *
-            ("model-20250610-v1:0", "model-*"),  # YYYYMMDD-v1:0 -> *
-            ("model@20250610-v1:0", "model@*"),  # @YYYYMMDD-v1:0 -> @*
+            ("model-20250522", "model"),  # YYYYMMDD removed
+            ("model-2025-06-10", "model"),  # YYYY-MM-DD removed
+            ("model-2025/06/10", "model"),  # YYYY/MM/DD removed
+            ("model-2025.06.10", "model"),  # YYYY.MM.DD removed
+            ("model-v1.0", "model"),  # v1.0 removed
+            ("model@20241022", "model"),  # @YYYYMMDD removed
+            ("model-v1:0", "model"),  # v1:0 removed
+            ("model-20250610-v1:0", "model"),  # YYYYMMDD-v1:0 removed
+            ("model@20250610-v1:0", "model"),  # @YYYYMMDD-v1:0 removed
+            ("gpt-4", "gpt-4"),  # No date/version, unchanged
+            ("claude-3-5-sonnet", "claude-3-5-sonnet"),  # Numbers are part of model name, unchanged
         ]
 
-        for model_id, expected_glob in test_cases:
-            actual_glob = _create_suffix_glob_model_name(model_id)
+        for model_id, expected_normalized in test_cases:
+            actual_normalized = _normalize_model_id(model_id)
             assert (
-                actual_glob == expected_glob
-            ), f"Expected {expected_glob} for {model_id}, got {actual_glob}"
+                actual_normalized == expected_normalized
+            ), f"Expected {expected_normalized} for {model_id}, got {actual_normalized}"
 
-    @responses.activate
     def test_create_prefix_glob_model_name(self) -> None:
         """Test prefix glob generation for model names"""
         from sentry.tasks.ai_agent_monitoring import _create_prefix_glob_model_name
 
         # Test cases with expected outputs
         test_cases = [
             ("gpt-4", "*gpt-4"),
-            ("gpt-4o-mini-*", "*gpt-4o-mini-*"),
+            ("gpt-4o-mini", "*gpt-4o-mini"),
+            ("claude-3-5-sonnet", "*claude-3-5-sonnet"),
             ("", "*"),
         ]