chore(profiling): enable memalloc to use sample

.github/workflows/build_python_3.yml

@@ -94,9 +94,6 @@ jobs:
       CIBW_BEFORE_ALL_MACOS: rustup target add aarch64-apple-darwin
       CIBW_BEFORE_ALL_LINUX: |
         if [[ "$(uname -m)-$(uname -i)-$(uname -o | tr '[:upper:]' '[:lower:]')-$(ldd --version 2>&1 | head -n 1 | awk '{print $1}')" != "i686-unknown-linux-musl" ]]; then
-          if command -v yum &> /dev/null; then
-            yum install -y libatomic.i686
-          fi
           curl -sSf https://sh.rustup.rs | sh -s -- -y;
         fi
       CIBW_ENVIRONMENT_LINUX: PATH=$HOME/.cargo/bin:$PATH CMAKE_BUILD_PARALLEL_LEVEL=24 CMAKE_ARGS="-DNATIVE_TESTING=OFF" SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }}

ddtrace/internal/datadog/profiling/cmake/AnalysisFunc.cmake

@@ -1,9 +1,9 @@
 include(CheckIPOSupported)
 
 function(add_ddup_config target)
-    # Profiling native extensions are built with C++17, even though underlying repo adheres to the manylinux 2014
+    # Profiling native extensions are built with C++20, even though underlying repo adheres to the manylinux 2014
     # standard. This isn't currently a problem, but if it becomes one, we may have to structure the library differently.
-    target_compile_features(${target} PUBLIC cxx_std_17)
+    target_compile_features(${target} PUBLIC cxx_std_20)
 
     # Common compile options
     target_compile_options(

ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt

@@ -1,4 +1,12 @@
 cmake_minimum_required(VERSION 3.19)
+
+# Disable testing by default (can be overridden with -DBUILD_TESTING=ON) Don't use FORCE to allow command-line overrides
+if(NOT DEFINED BUILD_TESTING)
+    set(BUILD_TESTING
+        OFF
+        CACHE BOOL "Build the testing tree")
+endif()
+
 project(
     dd_wrapper
     VERSION 0.1.1

ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt

@@ -40,14 +40,23 @@ function(dd_wrapper_add_test name)
     add_ddup_config(${name})
     target_link_options(${name} PRIVATE -Wl,--no-as-needed)
 
+    # Test executable needs to find libdd_wrapper in parent directory Append to INSTALL_RPATH instead of replacing to
+    # preserve sanitizer library paths set by add_ddup_config
+    get_target_property(EXISTING_INSTALL_RPATH ${name} INSTALL_RPATH)
+    if(EXISTING_INSTALL_RPATH)
+        set_target_properties(${name} PROPERTIES INSTALL_RPATH "${EXISTING_INSTALL_RPATH};$ORIGIN/.."
+                                                 BUILD_WITH_INSTALL_RPATH TRUE)
+    else()
+        set_target_properties(${name} PROPERTIES INSTALL_RPATH "$ORIGIN/.." BUILD_WITH_INSTALL_RPATH TRUE)
+    endif()
+
     gtest_discover_tests(
-        ${name}
+        ${name} DISCOVERY_MODE PRE_TEST # Delay test discovery until test execution to avoid running sanitizer-built
+                                        # executables during build
         PROPERTIES # We start new threads after fork(), and we want to continue running the tests after that instead of
                    # dying.
                    ENVIRONMENT "TSAN_OPTIONS=die_after_fork=0:suppressions=${CMAKE_CURRENT_SOURCE_DIR}/TSan.supp")
 
-    set_target_properties(${name} PROPERTIES INSTALL_RPATH "$ORIGIN/..")
-
     if(LIB_INSTALL_DIR)
         install(TARGETS ${name} RUNTIME DESTINATION ${LIB_INSTALL_DIR}/../test)
     endif()

ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt

@@ -68,11 +68,23 @@ set_target_properties(${EXTENSION_NAME} PROPERTIES PREFIX "")
 set_target_properties(${EXTENSION_NAME} PROPERTIES SUFFIX "")
 
 # RPATH is needed for sofile discovery at runtime, since Python packages are not installed in the system path. This is
-# typical.
+# typical. Use BUILD_WITH_INSTALL_RPATH to avoid rpath manipulation during install phase which can cause conflicts For
+# standalone/test builds, also include LIB_INSTALL_DIR so tests can find libdd_wrapper
 if(APPLE)
-    set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/..")
+    if(BUILD_TESTING AND LIB_INSTALL_DIR)
+        set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/..;${LIB_INSTALL_DIR}"
+                                                           BUILD_WITH_INSTALL_RPATH TRUE)
+    else()
+        set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/.." BUILD_WITH_INSTALL_RPATH
+                                                                                           TRUE)
+    endif()
 elseif(UNIX)
-    set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/..")
+    if(BUILD_TESTING AND LIB_INSTALL_DIR)
+        set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/..;${LIB_INSTALL_DIR}"
+                                                           BUILD_WITH_INSTALL_RPATH TRUE)
+    else()
+        set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/.." BUILD_WITH_INSTALL_RPATH TRUE)
+    endif()
 endif()
 target_include_directories(
     ${EXTENSION_NAME}

ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt

@@ -1,5 +1,12 @@
 cmake_minimum_required(VERSION 3.19)
 
+# Disable testing by default (can be overridden with -DBUILD_TESTING=ON) Don't use FORCE to allow command-line overrides
+if(NOT DEFINED BUILD_TESTING)
+    set(BUILD_TESTING
+        OFF
+        CACHE BOOL "Build the testing tree")
+endif()
+
 # The exact name of this extension determines aspects of the installation and build paths, which need to be kept in sync
 # with setup.py. Accordingly, take the name passed in by the caller, defaulting to "stack_v2" if needed.
 set(EXTENSION_NAME
@@ -109,11 +116,23 @@ set_target_properties(${EXTENSION_NAME} PROPERTIES PREFIX "")
 set_target_properties(${EXTENSION_NAME} PROPERTIES SUFFIX "")
 
 # RPATH is needed for sofile discovery at runtime, since Python packages are not installed in the system path. This is
-# typical.
+# typical. Use BUILD_WITH_INSTALL_RPATH to avoid rpath manipulation during install phase which can cause conflicts For
+# standalone/test builds, also include LIB_INSTALL_DIR so tests can find libdd_wrapper
 if(APPLE)
-    set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/..")
+    if(BUILD_TESTING AND LIB_INSTALL_DIR)
+        set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/..;${LIB_INSTALL_DIR}"
+                                                           BUILD_WITH_INSTALL_RPATH TRUE)
+    else()
+        set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/.." BUILD_WITH_INSTALL_RPATH
+                                                                                           TRUE)
+    endif()
 elseif(UNIX)
-    set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/..")
+    if(BUILD_TESTING AND LIB_INSTALL_DIR)
+        set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/..;${LIB_INSTALL_DIR}"
+                                                           BUILD_WITH_INSTALL_RPATH TRUE)
+    else()
+        set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/.." BUILD_WITH_INSTALL_RPATH TRUE)
+    endif()
 endif()
 
 target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper Threads::Threads)

ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt

@@ -40,11 +40,20 @@ function(dd_wrapper_add_test name)
         target_link_libraries(${name} PRIVATE ${Python3_LIBRARIES})
     endif()
 
-    set_target_properties(${name} PROPERTIES INSTALL_RPATH "$ORIGIN/../stack_v2")
-
     add_ddup_config(${name})
 
-    gtest_discover_tests(${name})
+    # Test executable needs to find _stack_v2 in parent directory Append to INSTALL_RPATH instead of replacing to
+    # preserve sanitizer library paths set by add_ddup_config
+    get_target_property(EXISTING_INSTALL_RPATH ${name} INSTALL_RPATH)
+    if(EXISTING_INSTALL_RPATH)
+        set_target_properties(${name} PROPERTIES INSTALL_RPATH "${EXISTING_INSTALL_RPATH};$ORIGIN/.."
+                                                 BUILD_WITH_INSTALL_RPATH TRUE)
+    else()
+        set_target_properties(${name} PROPERTIES INSTALL_RPATH "$ORIGIN/.." BUILD_WITH_INSTALL_RPATH TRUE)
+    endif()
+
+    gtest_discover_tests(${name} DISCOVERY_MODE PRE_TEST) # Delay test discovery until test execution to avoid running
+                                                          # sanitizer-built executables during build
 
     # This is supplemental artifact so make sure to install it in the right place
     if(INPLACE_LIB_INSTALL_DIR)

ddtrace/profiling/collector/_memalloc_heap.c → ddtrace/profiling/collector/_memalloc_heap.cpp

@@ -156,7 +156,7 @@ heap_tracker_thaw_no_cpython(heap_tracker_t* heap_tracker, size_t* n_to_free)
     *n_to_free = heap_tracker->freezer.frees.count;
     if (*n_to_free > 0) {
         /* TODO: can we put traceback_t* directly in freezer.frees so we don't need new storage? */
-        to_free = malloc(*n_to_free * sizeof(traceback_t*));
+        to_free = static_cast<traceback_t**>(malloc(*n_to_free * sizeof(traceback_t*)));
         for (size_t i = 0; i < *n_to_free; i++) {
             traceback_t* tb = memalloc_heap_map_remove(heap_tracker->allocs_m, heap_tracker->freezer.frees.tab[i]);
             to_free[i] = tb;

ddtrace/profiling/collector/_memalloc_heap_map.c → ddtrace/profiling/collector/_memalloc_heap_map.cpp

@@ -90,7 +90,7 @@ typedef struct memalloc_heap_map_iter_t
 memalloc_heap_map_t*
 memalloc_heap_map_new()
 {
-    memalloc_heap_map_t* m = calloc(sizeof(memalloc_heap_map_t), 1);
+    memalloc_heap_map_t* m = static_cast<memalloc_heap_map_t*>(calloc(sizeof(memalloc_heap_map_t), 1));
     m->map = HeapSamples_new(0);
     return m;
 }
@@ -104,7 +104,7 @@ memalloc_heap_map_size(memalloc_heap_map_t* m)
 traceback_t*
 memalloc_heap_map_insert(memalloc_heap_map_t* m, void* key, traceback_t* value)
 {
-    HeapSamples_Entry k = { key = key, value = value };
+    HeapSamples_Entry k = { .key = key, .val = value };
     HeapSamples_Insert res = HeapSamples_insert(&m->map, &k);
     traceback_t* prev = NULL;
     if (!res.inserted) {
@@ -187,7 +187,7 @@ memalloc_heap_map_delete(memalloc_heap_map_t* m)
 memalloc_heap_map_iter_t*
 memalloc_heap_map_iter_new(memalloc_heap_map_t* m)
 {
-    memalloc_heap_map_iter_t* it = malloc(sizeof(memalloc_heap_map_iter_t));
+    memalloc_heap_map_iter_t* it = static_cast<memalloc_heap_map_iter_t*>(malloc(sizeof(memalloc_heap_map_iter_t)));
     if (it) {
         it->iter = HeapSamples_citer(&m->map);
     }

ddtrace/profiling/collector/_memalloc_reentrant.h

@@ -7,7 +7,11 @@
 #define _POSIX_C_SOURCE 200809L
 #include <errno.h>
 #include <pthread.h>
+#ifdef __cplusplus
+#include <atomic>
+#else
 #include <stdatomic.h>
+#endif
 #include <time.h>
 #include <unistd.h>
 #endif

ddtrace/profiling/collector/_memalloc_tb.c → ddtrace/profiling/collector/_memalloc_tb.cpp

@@ -63,7 +63,7 @@ memalloc_tb_buffer_pool_get(memalloc_tb_buffer_pool* pool, uint16_t max_nframe)
         pool->pool[pool->count - 1] = NULL;
         pool->count--;
     } else {
-        t = malloc(TRACEBACK_SIZE(max_nframe));
+        t = static_cast<traceback_t*>(malloc(TRACEBACK_SIZE(max_nframe)));
     }
     return t;
 }
@@ -238,7 +238,7 @@ memalloc_frame_to_traceback(PyFrameObject* pyframe, uint16_t max_nframe)
     }
 
     size_t traceback_size = TRACEBACK_SIZE(traceback_buffer->nframe);
-    traceback_t* traceback = PyMem_RawMalloc(traceback_size);
+    traceback_t* traceback = static_cast<traceback_t*>(PyMem_RawMalloc(traceback_size));
 
     if (traceback)
         memcpy(traceback, traceback_buffer, traceback_size);

ddtrace/profiling/collector/_utils.h

@@ -14,14 +14,27 @@ random_range(uint64_t max)
 
 #define DO_NOTHING(...)
 
+#ifdef __cplusplus
+#define p_new(type, count) static_cast<type*>(PyMem_RawMalloc(sizeof(type) * (count)))
+#else
 #define p_new(type, count) PyMem_RawMalloc(sizeof(type) * (count))
+#endif
+
 #define p_delete(mem_p) PyMem_RawFree(mem_p);
 // Allocate at least 16 and 50% more than requested to avoid allocating items one by one.
 #define p_alloc_nr(x) (((x) + 16) * 3 / 2)
+
+#ifdef __cplusplus
+#define p_realloc(p, count)                                                                                            \
+    do {                                                                                                               \
+        (p) = static_cast<decltype(p)>(PyMem_RawRealloc((p), sizeof(*p) * (count)));                                   \
+    } while (0)
+#else
 #define p_realloc(p, count)                                                                                            \
     do {                                                                                                               \
         (p) = PyMem_RawRealloc((p), sizeof(*p) * (count));                                                             \
     } while (0)
+#endif
 
 #define p_grow(p, goalnb, allocnb)                                                                                     \
     do {                                                                                                               \

setup.py

@@ -700,6 +700,31 @@ def build_extension(self, ext):
                     return
                 raise
         else:
+            # For the memalloc extension, dynamically add libdd_wrapper to extra_objects
+            # if it's not already there (needed for non-editable builds)
+            if ext.name == "ddtrace.profiling.collector._memalloc" and CURRENT_OS in ("Linux", "Darwin"):
+                dd_wrapper_suffix = sysconfig.get_config_var("EXT_SUFFIX")
+
+                # For non-editable builds, remove any source directory paths and add build directory path
+                if not (IS_EDITABLE or getattr(self, "inplace", False)):
+                    # Remove any source directory libdd_wrapper paths
+                    source_wrapper_dir = Path(__file__).parent / "ddtrace" / "internal" / "datadog" / "profiling"
+                    ext.extra_objects = [
+                        obj for obj in ext.extra_objects if not obj.startswith(str(source_wrapper_dir))
+                    ]
+
+                    # Add build directory path
+                    wrapper_dir = (
+                        Path(__file__).parent / Path(self.build_lib) / "ddtrace" / "internal" / "datadog" / "profiling"
+                    )
+                    wrapper_path = wrapper_dir / f"libdd_wrapper{dd_wrapper_suffix}"
+
+                    if wrapper_path.exists():
+                        wrapper_path_str = str(wrapper_path)
+                        if wrapper_path_str not in ext.extra_objects:
+                            ext.extra_objects.append(wrapper_path_str)
+                            print(f"Added libdd_wrapper to link: {wrapper_path_str}")
+
             super().build_extension(ext)
 
         if COMPILE_MODE.lower() in ("release", "minsizerel"):
@@ -792,6 +817,8 @@ def build_extension_cmake(self, ext: "CMakeExtension") -> None:
 
         if BUILD_PROFILING_NATIVE_TESTS:
             cmake_args += ["-DBUILD_TESTING=ON"]
+        else:
+            cmake_args += ["-DBUILD_TESTING=OFF"]
 
         # If this is an inplace build, propagate this fact to CMake in case it's helpful
         # In particular, this is needed for build products which are not otherwise managed
@@ -1038,39 +1065,66 @@ def get_exts_for(name):
 
 
 if not IS_PYSTON:
+    # Determine the libdd_wrapper filename with the Python extension suffix
+    # For editable builds, the library will be in the source directory
+    # For regular builds, it will be built during build_ext and found via the build system
+    _dd_wrapper_suffix = sysconfig.get_config_var("EXT_SUFFIX")
+    _dd_wrapper_source_path = (
+        HERE / "ddtrace" / "internal" / "datadog" / "profiling" / f"libdd_wrapper{_dd_wrapper_suffix}"
+    )
+
+    # Only include extra_objects if the library already exists (editable builds)
+    # For non-editable builds, build_libdd_wrapper() will handle it and the linker will find it
+    _dd_wrapper_extra_objects = []
+    if CURRENT_OS in ("Linux", "Darwin") and _dd_wrapper_source_path.exists():
+        _dd_wrapper_extra_objects = [str(_dd_wrapper_source_path)]
+
     ext_modules: t.List[t.Union[Extension, Cython.Distutils.Extension, RustExtension]] = [
-        Extension(
-            "ddtrace.profiling.collector._memalloc",
-            sources=[
-                "ddtrace/profiling/collector/_memalloc.c",
-                "ddtrace/profiling/collector/_memalloc_tb.c",
-                "ddtrace/profiling/collector/_memalloc_heap.c",
-                "ddtrace/profiling/collector/_memalloc_reentrant.c",
-                "ddtrace/profiling/collector/_memalloc_heap_map.c",
-            ],
-            extra_compile_args=(
-                debug_compile_args
-                # If NDEBUG is set, assert statements are compiled out. Make
-                # sure we explicitly set this for normal builds, and explicitly
-                # _unset_ it for debug builds in case the CFLAGS from sysconfig
-                # include -DNDEBUG
-                + (["-DNDEBUG"] if not debug_compile_args else ["-UNDEBUG"])
-                + ["-D_POSIX_C_SOURCE=200809L", "-std=c11"]
-                + fast_build_args
-                if CURRENT_OS != "Windows"
-                else ["/std:c11", "/experimental:c11atomics"]
-            ),
-        ),
         Extension(
             "ddtrace.internal._threads",
             sources=["ddtrace/internal/_threads.cpp"],
             extra_compile_args=(
-                ["-std=c++17", "-Wall", "-Wextra"] + fast_build_args
+                ["-std=c++20", "-Wall", "-Wextra"] + fast_build_args
                 if CURRENT_OS != "Windows"
                 else ["/std:c++20", "/MT"]
             ),
         ),
     ]
+
+    # _memalloc uses cwisstable which is not supported on Windows
+    # Profiler extensions are only needed on Linux and macOS
+    if CURRENT_OS != "Windows":
+        ext_modules.insert(
+            0,
+            Extension(
+                "ddtrace.profiling.collector._memalloc",
+                sources=[
+                    "ddtrace/profiling/collector/_memalloc.cpp",
+                    "ddtrace/profiling/collector/_memalloc_tb.cpp",
+                    "ddtrace/profiling/collector/_memalloc_heap.cpp",
+                    "ddtrace/profiling/collector/_memalloc_reentrant.cpp",
+                    "ddtrace/profiling/collector/_memalloc_heap_map.cpp",
+                ],
+                include_dirs=[
+                    "ddtrace/internal/datadog/profiling/dd_wrapper/include",
+                ],
+                extra_objects=_dd_wrapper_extra_objects,
+                extra_link_args=(
+                    ["-Wl,-rpath,$ORIGIN/../../internal/datadog/profiling", "-latomic"]
+                    if CURRENT_OS == "Linux"
+                    else ["-Wl,-rpath,@loader_path/../../internal/datadog/profiling"]
+                    if CURRENT_OS == "Darwin"
+                    else []
+                ),
+                language="c++",
+                extra_compile_args=(
+                    debug_compile_args
+                    + (["-DNDEBUG"] if not debug_compile_args else ["-UNDEBUG"])
+                    + ["-D_POSIX_C_SOURCE=200809L", "-std=c++20"]
+                    + fast_build_args
+                ),
+            ),
+        )
     if platform.system() not in ("Windows", ""):
         ext_modules.append(
             Extension(
@@ -1134,7 +1188,8 @@ def get_exts_for(name):
         "ddtrace.appsec._ddwaf": ["libddwaf/*/lib/libddwaf.*"],
         "ddtrace.appsec._iast._taint_tracking": ["CMakeLists.txt"],
         "ddtrace.internal.datadog.profiling": (
-            ["libdd_wrapper*.*"] + ["ddtrace/internal/datadog/profiling/test/*"] if BUILD_PROFILING_NATIVE_TESTS else []
+            ["libdd_wrapper*.*"]
+            + (["ddtrace/internal/datadog/profiling/test/*"] if BUILD_PROFILING_NATIVE_TESTS else [])
         ),
     },
     zip_safe=False,