ci: add [pyX.Y] suffix to test names (#15191)

## Description

Workaround for our Flaky Test Management product, since a test that is
flaky for a single Python version would be quarantined for all Python
versions at the moment (as it is quarantined by name).

The workaround consists in adding the python version as a suffix, so the
same test for other Python versions would not be quarantined.

## Testing

Manually tested that all the tests are sent with the suffix.

## Risks

It effectively changes the test names, so it will impact reporting, etc.

It also complicates some bits, as we want the original test name for
snapshots and similar tests.

## Additional Notes

<!-- Any other information that would be helpful for reviewers -->

Author: gnufede

riotfile.py

@@ -544,6 +544,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
             env={
                 "DD_INSTRUMENTATION_TELEMETRY_ENABLED": "0",
                 "DD_CIVISIBILITY_ITR_ENABLED": "0",
+                "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED": "0",  # DEV: Temporary, remove once merged
             },
             command="pytest -v {cmdargs} tests/internal/",
             pkgs={

tests/appsec/integrations/conftest.py

@@ -2,6 +2,7 @@
 
 from tests.appsec.integrations.utils_testagent import clear_session
 from tests.appsec.integrations.utils_testagent import start_trace
+from tests.conftest import get_original_test_name
 
 
 @pytest.fixture
@@ -13,7 +14,7 @@ def iast_test_token(request):
     afterwards. This centralizes the repeated start/clear logic used by
     testagent-based IAST tests.
     """
-    token = request.node.name
+    token = get_original_test_name(request)
     _ = start_trace(token)
     try:
         yield token

tests/conftest.py

@@ -24,6 +24,7 @@
 from typing import Generator  # noqa:F401
 from typing import List
 from typing import Tuple  # noqa:F401
+from unittest import TestCase
 from unittest import mock
 from urllib import parse
 import warnings
@@ -47,11 +48,53 @@
 from tests.utils import snapshot_context as _snapshot_context
 
 
+try:
+    from pytest import StashKey
+except ImportError:
+    StashKey = None
+
+
 code_to_pyc = getattr(importlib._bootstrap_external, "_code_to_timestamp_pyc")
 
 
 DEFAULT_DDTRACE_SUBPROCESS_TEST_SERVICE_NAME = "ddtrace_subprocess_dir"
 
+# Stash keys for storing original test name and nodeid before Python version suffix is added
+# For pytest >= 7.1.0, use StashKey; for older versions, use attribute names
+if StashKey:
+    original_test_name_key = StashKey[str]()
+    original_test_nodeid_key = StashKey[str]()
+else:
+    # Fallback attribute names for pytest < 7.1.0
+    original_test_name_key = "_ddtrace_original_name"
+    original_test_nodeid_key = "_ddtrace_original_nodeid"
+
+
+def get_original_test_name(request_or_item):
+    """Get the original test name (before Python version suffix was added).
+
+    Works with both pytest >= 7.1.0 (using stash) and older versions (using attributes).
+
+    Args:
+        request_or_item: Either a pytest.FixtureRequest or pytest.Item
+
+    Returns:
+        The original test name string, or the current name if not found
+    """
+    if hasattr(request_or_item, "node"):
+        # It's a FixtureRequest
+        item = request_or_item.node
+    else:
+        # It's an Item
+        item = request_or_item
+
+    if StashKey:
+        # pytest >= 7.1.0: use stash
+        return item.stash.get(original_test_name_key, item.name)
+    else:
+        # pytest < 7.1.0: use attribute
+        return getattr(item, original_test_name_key, item.name)
+
 
 # Hack to try and capture more logging data from pytest failing on `internal` jobs on
 # pytest shutdown. This is a temporary workaround until we can figure out... why....
@@ -404,7 +447,13 @@ def _subprocess_wrapper():
 
 @pytest.hookimpl(tryfirst=True)
 def pytest_collection_modifyitems(session, config, items):
-    """Don't let ITR skip tests that use the subprocess marker because coverage collection in subprocesses is broken"""
+    """
+    Don't let ITR skip tests that use the subprocess marker
+    because coverage collection in subprocesses is broken.
+
+    Also: add py39 - py314 suffix as parametrization in test names
+    """
+    py_tag = f"py{sys.version_info.major}.{sys.version_info.minor}"
     for item in items:
         if item.get_closest_marker("subprocess"):
             if item.get_closest_marker("skipif"):
@@ -413,6 +462,26 @@ def pytest_collection_modifyitems(session, config, items):
             unskippable = pytest.mark.skipif(False, reason="datadog_itr_unskippable")
             item.add_marker(unskippable)
 
+        # Store original name and nodeid before modification
+        if StashKey:
+            # pytest >= 7.1.0: use stash
+            item.stash[original_test_name_key] = item.name
+            item.stash[original_test_nodeid_key] = item.nodeid
+        else:
+            # pytest < 7.1.0: use attributes
+            setattr(item, original_test_name_key, item.name)
+            setattr(item, original_test_nodeid_key, item.nodeid)
+
+        name_base = item.name
+        nodeid_base = item.nodeid
+
+        item._nodeid = f"{nodeid_base}[{py_tag}]"
+
+        cls = getattr(item, "cls", None)
+        is_unittest = isinstance(cls, type) and issubclass(cls, TestCase)
+        if not is_unittest:
+            item.name = f"{name_base}[{py_tag}]"
+
 
 def pytest_generate_tests(metafunc):
     marker = metafunc.definition.get_closest_marker("subprocess")

tests/contrib/kafka/test_kafka.py

@@ -26,6 +26,7 @@
 from ddtrace.internal.utils.retry import fibonacci_backoff_with_jitter
 from ddtrace.trace import TraceFilter
 from ddtrace.trace import tracer as ddtracer
+from tests.conftest import get_original_test_name
 from tests.contrib.config import KAFKA_CONFIG
 from tests.datastreams.test_public_api import MockedTracer
 from tests.utils import DummyTracer
@@ -58,7 +59,7 @@ def process_trace(self, trace):
 @pytest.fixture()
 def kafka_topic(request):
     # todo: add a UUID, but it makes snapshot tests fail.
-    topic_name = request.node.name.replace("[", "_").replace("]", "")
+    topic_name = get_original_test_name(request).replace("[", "_").replace("]", "")
 
     client = kafka_admin.AdminClient({"bootstrap.servers": BOOTSTRAP_SERVERS})
     for _, future in client.create_topics([kafka_admin.NewTopic(topic_name, 1, 1)]).items():
@@ -74,7 +75,7 @@ def empty_kafka_topic(request):
     """
     Deletes a kafka topic to clear message if it exists.
     """
-    topic_name = request.node.name.replace("[", "_").replace("]", "")
+    topic_name = get_original_test_name(request).replace("[", "_").replace("]", "")
     client = kafka_admin.AdminClient({"bootstrap.servers": BOOTSTRAP_SERVERS})
     for _, future in client.delete_topics([topic_name]).items():
         try:

tests/llmobs/test_experiments.py

@@ -69,7 +69,9 @@ def test_dataset_records() -> List[DatasetRecord]:
 
 @pytest.fixture
 def test_dataset_name(request) -> str:
-    return f"test-dataset-{request.node.name}"
+    from tests.conftest import get_original_test_name
+
+    return f"test-dataset-{get_original_test_name(request)}"
 
 
 @pytest.fixture

tests/profiling/collector/test_stack.py

@@ -15,6 +15,7 @@
 from ddtrace.internal.datadog.profiling import ddup
 from ddtrace.profiling import _threading
 from ddtrace.profiling.collector import stack
+from tests.conftest import get_original_test_name
 from tests.profiling.collector import pprof_utils
 
 from . import test_collector
@@ -656,7 +657,7 @@ def test_exception_collection_trace(tmp_path, tracer):
 # if you don't need to check the output profile, you can use this fixture
 @pytest.fixture
 def tracer_and_collector(tracer, request, tmp_path):
-    test_name = request.node.name
+    test_name = get_original_test_name(request)
     pprof_prefix = str(tmp_path / test_name)
 
     assert ddup.is_available
@@ -785,7 +786,7 @@ def test_collect_span_id(tracer, tmp_path):
 
 
 def test_collect_span_resource_after_finish(tracer, tmp_path, request):
-    test_name = request.node.name
+    test_name = get_original_test_name(request)
     pprof_prefix = str(tmp_path / test_name)
     output_filename = pprof_prefix + "." + str(os.getpid())
 
@@ -861,7 +862,7 @@ def test_resource_not_collected(tmp_path, tracer):
 
 
 def test_collect_nested_span_id(tmp_path, tracer, request):
-    test_name = request.node.name
+    test_name = get_original_test_name(request)
     pprof_prefix = str(tmp_path / test_name)
     output_filename = pprof_prefix + "." + str(os.getpid())
 

tests/utils.py

@@ -1439,10 +1439,12 @@ def call_program(*args, **kwargs):
 
 def request_token(request):
     # type: (pytest.FixtureRequest) -> str
+    from tests.conftest import get_original_test_name
+
     token = ""
     token += request.module.__name__
     token += ".%s" % request.cls.__name__ if request.cls else ""
-    token += ".%s" % request.node.name
+    token += ".%s" % get_original_test_name(request)
     return token