From dffb41ca9ad15366f7156ef9ac3781de8d313ef2 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Fri, 7 Nov 2025 16:51:59 -0500 Subject: [PATCH 01/27] chore(profiling) enable memalloc to use sample --- setup.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 3dfb7dbbcf8..0ef3850eb2e 100644 --- a/setup.py +++ b/setup.py @@ -1038,6 +1038,11 @@ def get_exts_for(name): if not IS_PYSTON: + # Determine the libdd_wrapper filename with the Python extension suffix + _dd_wrapper_suffix = sysconfig.get_config_var("EXT_SUFFIX") + _dd_wrapper_path = str( + HERE / "ddtrace" / "internal" / "datadog" / "profiling" / f"libdd_wrapper{_dd_wrapper_suffix}" + ) ext_modules: t.List[t.Union[Extension, Cython.Distutils.Extension, RustExtension]] = [ Extension( "ddtrace.profiling.collector._memalloc", @@ -1048,12 +1053,19 @@ def get_exts_for(name): "ddtrace/profiling/collector/_memalloc_reentrant.c", "ddtrace/profiling/collector/_memalloc_heap_map.c", ], + include_dirs=[ + "ddtrace/internal/datadog/profiling/dd_wrapper/include", + ], + extra_objects=[_dd_wrapper_path] if CURRENT_OS in ("Linux", "Darwin") else [], + extra_link_args=( + ["-Wl,-rpath,$ORIGIN/../../../internal/datadog/profiling"] + if CURRENT_OS == "Linux" + else ["-Wl,-rpath,@loader_path/../../../internal/datadog/profiling"] + if CURRENT_OS == "Darwin" + else [] + ), extra_compile_args=( debug_compile_args - # If NDEBUG is set, assert statements are compiled out. Make - # sure we explicitly set this for normal builds, and explicitly - # _unset_ it for debug builds in case the CFLAGS from sysconfig - # include -DNDEBUG + (["-DNDEBUG"] if not debug_compile_args else ["-UNDEBUG"]) + ["-D_POSIX_C_SOURCE=200809L", "-std=c11"] + fast_build_args From 7880b1273ecf1de5deceff5ff6cef67dd86d36ad Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Fri, 7 Nov 2025 17:08:44 -0500 Subject: [PATCH 02/27] cursor turned C into C++ --- .../collector/{_memalloc.c => _memalloc.cpp} | 0 .../{_memalloc_heap.c => _memalloc_heap.cpp} | 2 +- ...memalloc_heap_map.c => _memalloc_heap_map.cpp} | 8 +++++--- ...malloc_reentrant.c => _memalloc_reentrant.cpp} | 0 .../{_memalloc_tb.c => _memalloc_tb.cpp} | 4 ++-- ddtrace/profiling/collector/_utils.h | 13 +++++++++++++ setup.py | 15 ++++++++------- 7 files changed, 29 insertions(+), 13 deletions(-) rename ddtrace/profiling/collector/{_memalloc.c => _memalloc.cpp} (100%) rename ddtrace/profiling/collector/{_memalloc_heap.c => _memalloc_heap.cpp} (99%) rename ddtrace/profiling/collector/{_memalloc_heap_map.c => _memalloc_heap_map.cpp} (95%) rename ddtrace/profiling/collector/{_memalloc_reentrant.c => _memalloc_reentrant.cpp} (100%) rename ddtrace/profiling/collector/{_memalloc_tb.c => _memalloc_tb.cpp} (98%) diff --git a/ddtrace/profiling/collector/_memalloc.c b/ddtrace/profiling/collector/_memalloc.cpp similarity index 100% rename from ddtrace/profiling/collector/_memalloc.c rename to ddtrace/profiling/collector/_memalloc.cpp diff --git a/ddtrace/profiling/collector/_memalloc_heap.c b/ddtrace/profiling/collector/_memalloc_heap.cpp similarity index 99% rename from ddtrace/profiling/collector/_memalloc_heap.c rename to ddtrace/profiling/collector/_memalloc_heap.cpp index 5616fe0c0ab..67b71d5003e 100644 --- a/ddtrace/profiling/collector/_memalloc_heap.c +++ b/ddtrace/profiling/collector/_memalloc_heap.cpp @@ -156,7 +156,7 @@ heap_tracker_thaw_no_cpython(heap_tracker_t* heap_tracker, size_t* n_to_free) *n_to_free = heap_tracker->freezer.frees.count; if (*n_to_free > 0) { /* TODO: can we put traceback_t* directly in freezer.frees so we don't need new storage? */ - to_free = malloc(*n_to_free * sizeof(traceback_t*)); + to_free = static_cast(malloc(*n_to_free * sizeof(traceback_t*))); for (size_t i = 0; i < *n_to_free; i++) { traceback_t* tb = memalloc_heap_map_remove(heap_tracker->allocs_m, heap_tracker->freezer.frees.tab[i]); to_free[i] = tb; diff --git a/ddtrace/profiling/collector/_memalloc_heap_map.c b/ddtrace/profiling/collector/_memalloc_heap_map.cpp similarity index 95% rename from ddtrace/profiling/collector/_memalloc_heap_map.c rename to ddtrace/profiling/collector/_memalloc_heap_map.cpp index 18cb0d19beb..2f430d55122 100644 --- a/ddtrace/profiling/collector/_memalloc_heap_map.c +++ b/ddtrace/profiling/collector/_memalloc_heap_map.cpp @@ -90,7 +90,7 @@ typedef struct memalloc_heap_map_iter_t memalloc_heap_map_t* memalloc_heap_map_new() { - memalloc_heap_map_t* m = calloc(sizeof(memalloc_heap_map_t), 1); + memalloc_heap_map_t* m = static_cast(calloc(sizeof(memalloc_heap_map_t), 1)); m->map = HeapSamples_new(0); return m; } @@ -104,7 +104,9 @@ memalloc_heap_map_size(memalloc_heap_map_t* m) traceback_t* memalloc_heap_map_insert(memalloc_heap_map_t* m, void* key, traceback_t* value) { - HeapSamples_Entry k = { key = key, value = value }; + HeapSamples_Entry k; + k.key = key; + k.val = value; HeapSamples_Insert res = HeapSamples_insert(&m->map, &k); traceback_t* prev = NULL; if (!res.inserted) { @@ -187,7 +189,7 @@ memalloc_heap_map_delete(memalloc_heap_map_t* m) memalloc_heap_map_iter_t* memalloc_heap_map_iter_new(memalloc_heap_map_t* m) { - memalloc_heap_map_iter_t* it = malloc(sizeof(memalloc_heap_map_iter_t)); + memalloc_heap_map_iter_t* it = static_cast(malloc(sizeof(memalloc_heap_map_iter_t))); if (it) { it->iter = HeapSamples_citer(&m->map); } diff --git a/ddtrace/profiling/collector/_memalloc_reentrant.c b/ddtrace/profiling/collector/_memalloc_reentrant.cpp similarity index 100% rename from ddtrace/profiling/collector/_memalloc_reentrant.c rename to ddtrace/profiling/collector/_memalloc_reentrant.cpp diff --git a/ddtrace/profiling/collector/_memalloc_tb.c b/ddtrace/profiling/collector/_memalloc_tb.cpp similarity index 98% rename from ddtrace/profiling/collector/_memalloc_tb.c rename to ddtrace/profiling/collector/_memalloc_tb.cpp index a4518d2d632..b4990c3e9d0 100644 --- a/ddtrace/profiling/collector/_memalloc_tb.c +++ b/ddtrace/profiling/collector/_memalloc_tb.cpp @@ -63,7 +63,7 @@ memalloc_tb_buffer_pool_get(memalloc_tb_buffer_pool* pool, uint16_t max_nframe) pool->pool[pool->count - 1] = NULL; pool->count--; } else { - t = malloc(TRACEBACK_SIZE(max_nframe)); + t = static_cast(malloc(TRACEBACK_SIZE(max_nframe))); } return t; } @@ -238,7 +238,7 @@ memalloc_frame_to_traceback(PyFrameObject* pyframe, uint16_t max_nframe) } size_t traceback_size = TRACEBACK_SIZE(traceback_buffer->nframe); - traceback_t* traceback = PyMem_RawMalloc(traceback_size); + traceback_t* traceback = static_cast(PyMem_RawMalloc(traceback_size)); if (traceback) memcpy(traceback, traceback_buffer, traceback_size); diff --git a/ddtrace/profiling/collector/_utils.h b/ddtrace/profiling/collector/_utils.h index 86d17837fb5..076cbe86741 100644 --- a/ddtrace/profiling/collector/_utils.h +++ b/ddtrace/profiling/collector/_utils.h @@ -14,14 +14,27 @@ random_range(uint64_t max) #define DO_NOTHING(...) +#ifdef __cplusplus +#define p_new(type, count) static_cast(PyMem_RawMalloc(sizeof(type) * (count))) +#else #define p_new(type, count) PyMem_RawMalloc(sizeof(type) * (count)) +#endif + #define p_delete(mem_p) PyMem_RawFree(mem_p); // Allocate at least 16 and 50% more than requested to avoid allocating items one by one. #define p_alloc_nr(x) (((x) + 16) * 3 / 2) + +#ifdef __cplusplus +#define p_realloc(p, count) \ + do { \ + (p) = static_cast(PyMem_RawRealloc((p), sizeof(*p) * (count))); \ + } while (0) +#else #define p_realloc(p, count) \ do { \ (p) = PyMem_RawRealloc((p), sizeof(*p) * (count)); \ } while (0) +#endif #define p_grow(p, goalnb, allocnb) \ do { \ diff --git a/setup.py b/setup.py index 0ef3850eb2e..918db9aad9a 100644 --- a/setup.py +++ b/setup.py @@ -1047,11 +1047,11 @@ def get_exts_for(name): Extension( "ddtrace.profiling.collector._memalloc", sources=[ - "ddtrace/profiling/collector/_memalloc.c", - "ddtrace/profiling/collector/_memalloc_tb.c", - "ddtrace/profiling/collector/_memalloc_heap.c", - "ddtrace/profiling/collector/_memalloc_reentrant.c", - "ddtrace/profiling/collector/_memalloc_heap_map.c", + "ddtrace/profiling/collector/_memalloc.cpp", + "ddtrace/profiling/collector/_memalloc_tb.cpp", + "ddtrace/profiling/collector/_memalloc_heap.cpp", + "ddtrace/profiling/collector/_memalloc_reentrant.cpp", + "ddtrace/profiling/collector/_memalloc_heap_map.cpp", ], include_dirs=[ "ddtrace/internal/datadog/profiling/dd_wrapper/include", @@ -1064,13 +1064,14 @@ def get_exts_for(name): if CURRENT_OS == "Darwin" else [] ), + language="c++", extra_compile_args=( debug_compile_args + (["-DNDEBUG"] if not debug_compile_args else ["-UNDEBUG"]) - + ["-D_POSIX_C_SOURCE=200809L", "-std=c11"] + + ["-D_POSIX_C_SOURCE=200809L", "-std=c++17"] + fast_build_args if CURRENT_OS != "Windows" - else ["/std:c11", "/experimental:c11atomics"] + else ["/std:c++17", "/MT"] ), ), Extension( From 550c89f153046e364420124a1d61529179f4b2d9 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Fri, 7 Nov 2025 17:28:41 -0500 Subject: [PATCH 03/27] non editable builds too --- setup.py | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 918db9aad9a..9b94cd2cc94 100644 --- a/setup.py +++ b/setup.py @@ -700,6 +700,31 @@ def build_extension(self, ext): return raise else: + # For the memalloc extension, dynamically add libdd_wrapper to extra_objects + # if it's not already there (needed for non-editable builds) + if ext.name == "ddtrace.profiling.collector._memalloc" and CURRENT_OS in ("Linux", "Darwin"): + dd_wrapper_suffix = sysconfig.get_config_var("EXT_SUFFIX") + + # For non-editable builds, remove any source directory paths and add build directory path + if not (IS_EDITABLE or getattr(self, "inplace", False)): + # Remove any source directory libdd_wrapper paths + source_wrapper_dir = Path(__file__).parent / "ddtrace" / "internal" / "datadog" / "profiling" + ext.extra_objects = [ + obj for obj in ext.extra_objects if not obj.startswith(str(source_wrapper_dir)) + ] + + # Add build directory path + wrapper_dir = ( + Path(__file__).parent / Path(self.build_lib) / "ddtrace" / "internal" / "datadog" / "profiling" + ) + wrapper_path = wrapper_dir / f"libdd_wrapper{dd_wrapper_suffix}" + + if wrapper_path.exists(): + wrapper_path_str = str(wrapper_path) + if wrapper_path_str not in ext.extra_objects: + ext.extra_objects.append(wrapper_path_str) + print(f"Added libdd_wrapper to link: {wrapper_path_str}") + super().build_extension(ext) if COMPILE_MODE.lower() in ("release", "minsizerel"): @@ -1039,10 +1064,19 @@ def get_exts_for(name): if not IS_PYSTON: # Determine the libdd_wrapper filename with the Python extension suffix + # For editable builds, the library will be in the source directory + # For regular builds, it will be built during build_ext and found via the build system _dd_wrapper_suffix = sysconfig.get_config_var("EXT_SUFFIX") - _dd_wrapper_path = str( + _dd_wrapper_source_path = ( HERE / "ddtrace" / "internal" / "datadog" / "profiling" / f"libdd_wrapper{_dd_wrapper_suffix}" ) + + # Only include extra_objects if the library already exists (editable builds) + # For non-editable builds, build_libdd_wrapper() will handle it and the linker will find it + _dd_wrapper_extra_objects = [] + if CURRENT_OS in ("Linux", "Darwin") and _dd_wrapper_source_path.exists(): + _dd_wrapper_extra_objects = [str(_dd_wrapper_source_path)] + ext_modules: t.List[t.Union[Extension, Cython.Distutils.Extension, RustExtension]] = [ Extension( "ddtrace.profiling.collector._memalloc", @@ -1056,7 +1090,7 @@ def get_exts_for(name): include_dirs=[ "ddtrace/internal/datadog/profiling/dd_wrapper/include", ], - extra_objects=[_dd_wrapper_path] if CURRENT_OS in ("Linux", "Darwin") else [], + extra_objects=_dd_wrapper_extra_objects, extra_link_args=( ["-Wl,-rpath,$ORIGIN/../../../internal/datadog/profiling"] if CURRENT_OS == "Linux" From 00b74c00f31007e468708a7c73ab72d8e63736c6 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Mon, 10 Nov 2025 10:38:26 -0500 Subject: [PATCH 04/27] try to fix ci --- ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt | 2 ++ ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt | 2 ++ 2 files changed, 4 insertions(+) diff --git a/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt b/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt index dbe4395cc16..32d8c249395 100644 --- a/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt @@ -69,6 +69,8 @@ set_target_properties(${EXTENSION_NAME} PROPERTIES SUFFIX "") # RPATH is needed for sofile discovery at runtime, since Python packages are not installed in the system path. This is # typical. +# Use BUILD_WITH_INSTALL_RPATH to avoid rpath manipulation during install phase which can cause conflicts +set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) if(APPLE) set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/..") elseif(UNIX) diff --git a/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt index 89d900134bc..9d6d6f84911 100644 --- a/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt @@ -110,6 +110,8 @@ set_target_properties(${EXTENSION_NAME} PROPERTIES SUFFIX "") # RPATH is needed for sofile discovery at runtime, since Python packages are not installed in the system path. This is # typical. +# Use BUILD_WITH_INSTALL_RPATH to avoid rpath manipulation during install phase which can cause conflicts +set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) if(APPLE) set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/..") elseif(UNIX) From c56659a0f8d79a939dfda6e5811932fbb1a494bb Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Mon, 10 Nov 2025 10:48:50 -0500 Subject: [PATCH 05/27] fix atomic --- ddtrace/profiling/collector/_memalloc_reentrant.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ddtrace/profiling/collector/_memalloc_reentrant.h b/ddtrace/profiling/collector/_memalloc_reentrant.h index d7e67823df9..2e803486754 100644 --- a/ddtrace/profiling/collector/_memalloc_reentrant.h +++ b/ddtrace/profiling/collector/_memalloc_reentrant.h @@ -7,7 +7,11 @@ #define _POSIX_C_SOURCE 200809L #include #include +#ifdef __cplusplus +#include +#else #include +#endif #include #include #endif From 7d4c808ada9289f8f972243dbde2f0e62515cc9f Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Mon, 10 Nov 2025 11:31:10 -0500 Subject: [PATCH 06/27] fix ci again --- .github/workflows/build_python_3.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_python_3.yml b/.github/workflows/build_python_3.yml index 18c6bd60b45..c70b7c8156e 100644 --- a/.github/workflows/build_python_3.yml +++ b/.github/workflows/build_python_3.yml @@ -95,7 +95,7 @@ jobs: CIBW_BEFORE_ALL_LINUX: | if [[ "$(uname -m)-$(uname -i)-$(uname -o | tr '[:upper:]' '[:lower:]')-$(ldd --version 2>&1 | head -n 1 | awk '{print $1}')" != "i686-unknown-linux-musl" ]]; then if command -v yum &> /dev/null; then - yum install -y libatomic.i686 + yum install -y libatomic.i686 || echo "libatomic.i686 not available, assuming it's already installed" fi curl -sSf https://sh.rustup.rs | sh -s -- -y; fi diff --git a/setup.py b/setup.py index 6afd8c4f683..55cbaa42531 100644 --- a/setup.py +++ b/setup.py @@ -1092,7 +1092,7 @@ def get_exts_for(name): ], extra_objects=_dd_wrapper_extra_objects, extra_link_args=( - ["-Wl,-rpath,$ORIGIN/../../../internal/datadog/profiling"] + ["-Wl,-rpath,$ORIGIN/../../../internal/datadog/profiling", "-latomic"] if CURRENT_OS == "Linux" else ["-Wl,-rpath,@loader_path/../../../internal/datadog/profiling"] if CURRENT_OS == "Darwin" From 0a0dc5b8f9a7b29bad720250d6bb66be88dedec6 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Mon, 10 Nov 2025 11:41:18 -0500 Subject: [PATCH 07/27] format --- ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt | 3 +-- ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt b/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt index 32d8c249395..f0949d9017b 100644 --- a/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt @@ -68,8 +68,7 @@ set_target_properties(${EXTENSION_NAME} PROPERTIES PREFIX "") set_target_properties(${EXTENSION_NAME} PROPERTIES SUFFIX "") # RPATH is needed for sofile discovery at runtime, since Python packages are not installed in the system path. This is -# typical. -# Use BUILD_WITH_INSTALL_RPATH to avoid rpath manipulation during install phase which can cause conflicts +# typical. Use BUILD_WITH_INSTALL_RPATH to avoid rpath manipulation during install phase which can cause conflicts set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) if(APPLE) set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/..") diff --git a/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt index 9d6d6f84911..bf1ff938838 100644 --- a/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt @@ -109,8 +109,7 @@ set_target_properties(${EXTENSION_NAME} PROPERTIES PREFIX "") set_target_properties(${EXTENSION_NAME} PROPERTIES SUFFIX "") # RPATH is needed for sofile discovery at runtime, since Python packages are not installed in the system path. This is -# typical. -# Use BUILD_WITH_INSTALL_RPATH to avoid rpath manipulation during install phase which can cause conflicts +# typical. Use BUILD_WITH_INSTALL_RPATH to avoid rpath manipulation during install phase which can cause conflicts set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) if(APPLE) set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/..") From 7d9a2b12d4b0af2c3528738da21a2e1fc2c381b5 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Mon, 10 Nov 2025 11:55:11 -0500 Subject: [PATCH 08/27] more ci fixes --- ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt | 5 ++--- ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt b/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt index f0949d9017b..9d0ce672f37 100644 --- a/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt @@ -69,11 +69,10 @@ set_target_properties(${EXTENSION_NAME} PROPERTIES SUFFIX "") # RPATH is needed for sofile discovery at runtime, since Python packages are not installed in the system path. This is # typical. Use BUILD_WITH_INSTALL_RPATH to avoid rpath manipulation during install phase which can cause conflicts -set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) if(APPLE) - set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/..") + set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/.." BUILD_WITH_INSTALL_RPATH TRUE) elseif(UNIX) - set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/..") + set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/.." BUILD_WITH_INSTALL_RPATH TRUE) endif() target_include_directories( ${EXTENSION_NAME} diff --git a/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt index bf1ff938838..0472d2bd653 100644 --- a/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt @@ -110,11 +110,10 @@ set_target_properties(${EXTENSION_NAME} PROPERTIES SUFFIX "") # RPATH is needed for sofile discovery at runtime, since Python packages are not installed in the system path. This is # typical. Use BUILD_WITH_INSTALL_RPATH to avoid rpath manipulation during install phase which can cause conflicts -set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) if(APPLE) - set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/..") + set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/.." BUILD_WITH_INSTALL_RPATH TRUE) elseif(UNIX) - set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/..") + set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/.." BUILD_WITH_INSTALL_RPATH TRUE) endif() target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper Threads::Threads) From 33734da1bb6525ec25af8331fdaff91e3695a2ba Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Mon, 10 Nov 2025 12:48:49 -0500 Subject: [PATCH 09/27] more ci fix --- setup.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 55cbaa42531..b8f839554a0 100644 --- a/setup.py +++ b/setup.py @@ -1092,9 +1092,9 @@ def get_exts_for(name): ], extra_objects=_dd_wrapper_extra_objects, extra_link_args=( - ["-Wl,-rpath,$ORIGIN/../../../internal/datadog/profiling", "-latomic"] + ["-Wl,-rpath,$ORIGIN/../../internal/datadog/profiling", "-latomic"] if CURRENT_OS == "Linux" - else ["-Wl,-rpath,@loader_path/../../../internal/datadog/profiling"] + else ["-Wl,-rpath,@loader_path/../../internal/datadog/profiling"] if CURRENT_OS == "Darwin" else [] ), @@ -1181,7 +1181,8 @@ def get_exts_for(name): "ddtrace.appsec._ddwaf": ["libddwaf/*/lib/libddwaf.*"], "ddtrace.appsec._iast._taint_tracking": ["CMakeLists.txt"], "ddtrace.internal.datadog.profiling": ( - ["libdd_wrapper*.*"] + ["ddtrace/internal/datadog/profiling/test/*"] if BUILD_PROFILING_NATIVE_TESTS else [] + ["libdd_wrapper*.*"] + + (["ddtrace/internal/datadog/profiling/test/*"] if BUILD_PROFILING_NATIVE_TESTS else []) ), }, zip_safe=False, From c30948ccd9e9a5238f9ac17259e362bf7f2d5ca2 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Mon, 10 Nov 2025 12:56:01 -0500 Subject: [PATCH 10/27] more ci fixes --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index b8f839554a0..e8bb75f106f 100644 --- a/setup.py +++ b/setup.py @@ -817,6 +817,8 @@ def build_extension_cmake(self, ext: "CMakeExtension") -> None: if BUILD_PROFILING_NATIVE_TESTS: cmake_args += ["-DBUILD_TESTING=ON"] + else: + cmake_args += ["-DBUILD_TESTING=OFF"] # If this is an inplace build, propagate this fact to CMake in case it's helpful # In particular, this is needed for build products which are not otherwise managed From 21065dd064fabbc1fc6b18e536ac01fc8ae00d25 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Mon, 10 Nov 2025 13:19:26 -0500 Subject: [PATCH 11/27] even more ci fixes --- ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt | 4 ++++ ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt | 3 +++ 2 files changed, 7 insertions(+) diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt b/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt index d48435dfa51..e2fd2073f7e 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt @@ -1,4 +1,8 @@ cmake_minimum_required(VERSION 3.19) + +# Disable testing by default (can be overridden with -DBUILD_TESTING=ON) +option(BUILD_TESTING "Build the testing tree" OFF) + project( dd_wrapper VERSION 0.1.1 diff --git a/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt index 0472d2bd653..97a2aafaf66 100644 --- a/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt @@ -1,5 +1,8 @@ cmake_minimum_required(VERSION 3.19) +# Disable testing by default (can be overridden with -DBUILD_TESTING=ON) +option(BUILD_TESTING "Build the testing tree" OFF) + # The exact name of this extension determines aspects of the installation and build paths, which need to be kept in sync # with setup.py. Accordingly, take the name passed in by the caller, defaulting to "stack_v2" if needed. set(EXTENSION_NAME From 104710d613d3136113aa598e8f905b5a843a45dd Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Mon, 10 Nov 2025 13:55:31 -0500 Subject: [PATCH 12/27] more cmake changes --- ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt | 5 ++++- ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt b/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt index e2fd2073f7e..c5f2d9303be 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt @@ -1,7 +1,10 @@ cmake_minimum_required(VERSION 3.19) # Disable testing by default (can be overridden with -DBUILD_TESTING=ON) -option(BUILD_TESTING "Build the testing tree" OFF) +# Use FORCE to override any cached values +if(NOT DEFINED BUILD_TESTING) + set(BUILD_TESTING OFF CACHE BOOL "Build the testing tree" FORCE) +endif() project( dd_wrapper diff --git a/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt index 97a2aafaf66..0c3a96f48ab 100644 --- a/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt @@ -1,7 +1,10 @@ cmake_minimum_required(VERSION 3.19) # Disable testing by default (can be overridden with -DBUILD_TESTING=ON) -option(BUILD_TESTING "Build the testing tree" OFF) +# Use FORCE to override any cached values +if(NOT DEFINED BUILD_TESTING) + set(BUILD_TESTING OFF CACHE BOOL "Build the testing tree" FORCE) +endif() # The exact name of this extension determines aspects of the installation and build paths, which need to be kept in sync # with setup.py. Accordingly, take the name passed in by the caller, defaulting to "stack_v2" if needed. From a6b3c4327fb36f042242ef2ad28bba54e546e947 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Mon, 10 Nov 2025 13:59:54 -0500 Subject: [PATCH 13/27] format cmake --- .../internal/datadog/profiling/dd_wrapper/CMakeLists.txt | 7 ++++--- ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt b/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt index c5f2d9303be..345c824b8d7 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt @@ -1,9 +1,10 @@ cmake_minimum_required(VERSION 3.19) -# Disable testing by default (can be overridden with -DBUILD_TESTING=ON) -# Use FORCE to override any cached values +# Disable testing by default (can be overridden with -DBUILD_TESTING=ON) Use FORCE to override any cached values if(NOT DEFINED BUILD_TESTING) - set(BUILD_TESTING OFF CACHE BOOL "Build the testing tree" FORCE) + set(BUILD_TESTING + OFF + CACHE BOOL "Build the testing tree" FORCE) endif() project( diff --git a/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt index 0c3a96f48ab..1671b388090 100644 --- a/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt @@ -1,9 +1,10 @@ cmake_minimum_required(VERSION 3.19) -# Disable testing by default (can be overridden with -DBUILD_TESTING=ON) -# Use FORCE to override any cached values +# Disable testing by default (can be overridden with -DBUILD_TESTING=ON) Use FORCE to override any cached values if(NOT DEFINED BUILD_TESTING) - set(BUILD_TESTING OFF CACHE BOOL "Build the testing tree" FORCE) + set(BUILD_TESTING + OFF + CACHE BOOL "Build the testing tree" FORCE) endif() # The exact name of this extension determines aspects of the installation and build paths, which need to be kept in sync From badaf86d6096768ef6b56ade9a81a300cc4b25bf Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Mon, 10 Nov 2025 14:10:53 -0500 Subject: [PATCH 14/27] AI trying again to fix ci --- ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt | 4 ++-- ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt b/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt index 345c824b8d7..a2ad965873b 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt @@ -1,10 +1,10 @@ cmake_minimum_required(VERSION 3.19) -# Disable testing by default (can be overridden with -DBUILD_TESTING=ON) Use FORCE to override any cached values +# Disable testing by default (can be overridden with -DBUILD_TESTING=ON) Don't use FORCE to allow command-line overrides if(NOT DEFINED BUILD_TESTING) set(BUILD_TESTING OFF - CACHE BOOL "Build the testing tree" FORCE) + CACHE BOOL "Build the testing tree") endif() project( diff --git a/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt index 1671b388090..36225b5c73f 100644 --- a/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt @@ -1,10 +1,10 @@ cmake_minimum_required(VERSION 3.19) -# Disable testing by default (can be overridden with -DBUILD_TESTING=ON) Use FORCE to override any cached values +# Disable testing by default (can be overridden with -DBUILD_TESTING=ON) Don't use FORCE to allow command-line overrides if(NOT DEFINED BUILD_TESTING) set(BUILD_TESTING OFF - CACHE BOOL "Build the testing tree" FORCE) + CACHE BOOL "Build the testing tree") endif() # The exact name of this extension determines aspects of the installation and build paths, which need to be kept in sync From c4cd5c809f2d6411d23bf6013217353b20056729 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Mon, 10 Nov 2025 14:13:48 -0500 Subject: [PATCH 15/27] only build memalloc profiler for windows --- setup.py | 65 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/setup.py b/setup.py index e8bb75f106f..034b9e53071 100644 --- a/setup.py +++ b/setup.py @@ -1080,36 +1080,6 @@ def get_exts_for(name): _dd_wrapper_extra_objects = [str(_dd_wrapper_source_path)] ext_modules: t.List[t.Union[Extension, Cython.Distutils.Extension, RustExtension]] = [ - Extension( - "ddtrace.profiling.collector._memalloc", - sources=[ - "ddtrace/profiling/collector/_memalloc.cpp", - "ddtrace/profiling/collector/_memalloc_tb.cpp", - "ddtrace/profiling/collector/_memalloc_heap.cpp", - "ddtrace/profiling/collector/_memalloc_reentrant.cpp", - "ddtrace/profiling/collector/_memalloc_heap_map.cpp", - ], - include_dirs=[ - "ddtrace/internal/datadog/profiling/dd_wrapper/include", - ], - extra_objects=_dd_wrapper_extra_objects, - extra_link_args=( - ["-Wl,-rpath,$ORIGIN/../../internal/datadog/profiling", "-latomic"] - if CURRENT_OS == "Linux" - else ["-Wl,-rpath,@loader_path/../../internal/datadog/profiling"] - if CURRENT_OS == "Darwin" - else [] - ), - language="c++", - extra_compile_args=( - debug_compile_args - + (["-DNDEBUG"] if not debug_compile_args else ["-UNDEBUG"]) - + ["-D_POSIX_C_SOURCE=200809L", "-std=c++17"] - + fast_build_args - if CURRENT_OS != "Windows" - else ["/std:c++17", "/MT"] - ), - ), Extension( "ddtrace.internal._threads", sources=["ddtrace/internal/_threads.cpp"], @@ -1120,6 +1090,41 @@ def get_exts_for(name): ), ), ] + + # _memalloc uses cwisstable which is not supported on Windows + # Profiler extensions are only needed on Linux and macOS + if CURRENT_OS != "Windows": + ext_modules.insert( + 0, + Extension( + "ddtrace.profiling.collector._memalloc", + sources=[ + "ddtrace/profiling/collector/_memalloc.cpp", + "ddtrace/profiling/collector/_memalloc_tb.cpp", + "ddtrace/profiling/collector/_memalloc_heap.cpp", + "ddtrace/profiling/collector/_memalloc_reentrant.cpp", + "ddtrace/profiling/collector/_memalloc_heap_map.cpp", + ], + include_dirs=[ + "ddtrace/internal/datadog/profiling/dd_wrapper/include", + ], + extra_objects=_dd_wrapper_extra_objects, + extra_link_args=( + ["-Wl,-rpath,$ORIGIN/../../internal/datadog/profiling", "-latomic"] + if CURRENT_OS == "Linux" + else ["-Wl,-rpath,@loader_path/../../internal/datadog/profiling"] + if CURRENT_OS == "Darwin" + else [] + ), + language="c++", + extra_compile_args=( + debug_compile_args + + (["-DNDEBUG"] if not debug_compile_args else ["-UNDEBUG"]) + + ["-D_POSIX_C_SOURCE=200809L", "-std=c++17"] + + fast_build_args + ), + ), + ) if platform.system() not in ("Windows", ""): ext_modules.append( Extension( From ed8aa12907d038cbda94531249e93bc01ff76bf8 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Mon, 10 Nov 2025 14:56:48 -0500 Subject: [PATCH 16/27] try to fix sanitizers --- .../profiling/dd_wrapper/test/CMakeLists.txt | 3 ++- .../datadog/profiling/ddup/CMakeLists.txt | 18 +++++++++++++++--- .../datadog/profiling/stack_v2/CMakeLists.txt | 18 +++++++++++++++--- .../profiling/stack_v2/test/CMakeLists.txt | 3 ++- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt b/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt index e5fcaef782d..b26f2647681 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt @@ -46,7 +46,8 @@ function(dd_wrapper_add_test name) # dying. ENVIRONMENT "TSAN_OPTIONS=die_after_fork=0:suppressions=${CMAKE_CURRENT_SOURCE_DIR}/TSan.supp") - set_target_properties(${name} PROPERTIES INSTALL_RPATH "$ORIGIN/..") + # Test executable needs to find libdd_wrapper in parent directory + set_target_properties(${name} PROPERTIES INSTALL_RPATH "$ORIGIN/.." BUILD_WITH_INSTALL_RPATH TRUE) if(LIB_INSTALL_DIR) install(TARGETS ${name} RUNTIME DESTINATION ${LIB_INSTALL_DIR}/../test) diff --git a/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt b/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt index 9d0ce672f37..c6780223053 100644 --- a/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt @@ -68,11 +68,23 @@ set_target_properties(${EXTENSION_NAME} PROPERTIES PREFIX "") set_target_properties(${EXTENSION_NAME} PROPERTIES SUFFIX "") # RPATH is needed for sofile discovery at runtime, since Python packages are not installed in the system path. This is -# typical. Use BUILD_WITH_INSTALL_RPATH to avoid rpath manipulation during install phase which can cause conflicts +# typical. Use BUILD_WITH_INSTALL_RPATH to avoid rpath manipulation during install phase which can cause conflicts For +# standalone/test builds, also include LIB_INSTALL_DIR so tests can find libdd_wrapper if(APPLE) - set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/.." BUILD_WITH_INSTALL_RPATH TRUE) + if(BUILD_TESTING AND LIB_INSTALL_DIR) + set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/..;${LIB_INSTALL_DIR}" + BUILD_WITH_INSTALL_RPATH TRUE) + else() + set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/.." BUILD_WITH_INSTALL_RPATH + TRUE) + endif() elseif(UNIX) - set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/.." BUILD_WITH_INSTALL_RPATH TRUE) + if(BUILD_TESTING AND LIB_INSTALL_DIR) + set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/..;${LIB_INSTALL_DIR}" + BUILD_WITH_INSTALL_RPATH TRUE) + else() + set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/.." BUILD_WITH_INSTALL_RPATH TRUE) + endif() endif() target_include_directories( ${EXTENSION_NAME} diff --git a/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt index 36225b5c73f..7f8f7e4f837 100644 --- a/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt @@ -116,11 +116,23 @@ set_target_properties(${EXTENSION_NAME} PROPERTIES PREFIX "") set_target_properties(${EXTENSION_NAME} PROPERTIES SUFFIX "") # RPATH is needed for sofile discovery at runtime, since Python packages are not installed in the system path. This is -# typical. Use BUILD_WITH_INSTALL_RPATH to avoid rpath manipulation during install phase which can cause conflicts +# typical. Use BUILD_WITH_INSTALL_RPATH to avoid rpath manipulation during install phase which can cause conflicts For +# standalone/test builds, also include LIB_INSTALL_DIR so tests can find libdd_wrapper if(APPLE) - set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/.." BUILD_WITH_INSTALL_RPATH TRUE) + if(BUILD_TESTING AND LIB_INSTALL_DIR) + set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/..;${LIB_INSTALL_DIR}" + BUILD_WITH_INSTALL_RPATH TRUE) + else() + set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "@loader_path/.." BUILD_WITH_INSTALL_RPATH + TRUE) + endif() elseif(UNIX) - set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/.." BUILD_WITH_INSTALL_RPATH TRUE) + if(BUILD_TESTING AND LIB_INSTALL_DIR) + set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/..;${LIB_INSTALL_DIR}" + BUILD_WITH_INSTALL_RPATH TRUE) + else() + set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/.." BUILD_WITH_INSTALL_RPATH TRUE) + endif() endif() target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper Threads::Threads) diff --git a/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt index 6cf4b2fe7a4..f1dd1e01c0f 100644 --- a/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt @@ -40,7 +40,8 @@ function(dd_wrapper_add_test name) target_link_libraries(${name} PRIVATE ${Python3_LIBRARIES}) endif() - set_target_properties(${name} PROPERTIES INSTALL_RPATH "$ORIGIN/../stack_v2") + # Test executable needs to find _stack_v2 in parent directory + set_target_properties(${name} PROPERTIES INSTALL_RPATH "$ORIGIN/.." BUILD_WITH_INSTALL_RPATH TRUE) add_ddup_config(${name}) From e55c5e02c7c2b01ede4a3370ae55c0416d5802e0 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Mon, 10 Nov 2025 15:08:13 -0500 Subject: [PATCH 17/27] sanitizer discovery mode --- .../internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt | 3 ++- .../internal/datadog/profiling/stack_v2/test/CMakeLists.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt b/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt index b26f2647681..f6c50d1563c 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt @@ -41,7 +41,8 @@ function(dd_wrapper_add_test name) target_link_options(${name} PRIVATE -Wl,--no-as-needed) gtest_discover_tests( - ${name} + ${name} DISCOVERY_MODE PRE_TEST # Delay test discovery until test execution to avoid running sanitizer-built + # executables during build PROPERTIES # We start new threads after fork(), and we want to continue running the tests after that instead of # dying. ENVIRONMENT "TSAN_OPTIONS=die_after_fork=0:suppressions=${CMAKE_CURRENT_SOURCE_DIR}/TSan.supp") diff --git a/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt index f1dd1e01c0f..95d79b802ae 100644 --- a/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt @@ -45,7 +45,8 @@ function(dd_wrapper_add_test name) add_ddup_config(${name}) - gtest_discover_tests(${name}) + gtest_discover_tests(${name} DISCOVERY_MODE PRE_TEST) # Delay test discovery until test execution to avoid running + # sanitizer-built executables during build # This is supplemental artifact so make sure to install it in the right place if(INPLACE_LIB_INSTALL_DIR) From d7a05b2c1e2bbad030dbe1f9cb3822d6688d410c Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Mon, 10 Nov 2025 15:16:28 -0500 Subject: [PATCH 18/27] yet another try at asan --- .../profiling/dd_wrapper/test/CMakeLists.txt | 13 ++++++++++--- .../datadog/profiling/stack_v2/test/CMakeLists.txt | 13 ++++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt b/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt index f6c50d1563c..ada2662c4f5 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt @@ -40,6 +40,16 @@ function(dd_wrapper_add_test name) add_ddup_config(${name}) target_link_options(${name} PRIVATE -Wl,--no-as-needed) + # Test executable needs to find libdd_wrapper in parent directory + # Append to INSTALL_RPATH instead of replacing to preserve sanitizer library paths set by add_ddup_config + get_target_property(EXISTING_INSTALL_RPATH ${name} INSTALL_RPATH) + if(EXISTING_INSTALL_RPATH) + set_target_properties(${name} PROPERTIES INSTALL_RPATH "${EXISTING_INSTALL_RPATH};$ORIGIN/.." + BUILD_WITH_INSTALL_RPATH TRUE) + else() + set_target_properties(${name} PROPERTIES INSTALL_RPATH "$ORIGIN/.." BUILD_WITH_INSTALL_RPATH TRUE) + endif() + gtest_discover_tests( ${name} DISCOVERY_MODE PRE_TEST # Delay test discovery until test execution to avoid running sanitizer-built # executables during build @@ -47,9 +57,6 @@ function(dd_wrapper_add_test name) # dying. ENVIRONMENT "TSAN_OPTIONS=die_after_fork=0:suppressions=${CMAKE_CURRENT_SOURCE_DIR}/TSan.supp") - # Test executable needs to find libdd_wrapper in parent directory - set_target_properties(${name} PROPERTIES INSTALL_RPATH "$ORIGIN/.." BUILD_WITH_INSTALL_RPATH TRUE) - if(LIB_INSTALL_DIR) install(TARGETS ${name} RUNTIME DESTINATION ${LIB_INSTALL_DIR}/../test) endif() diff --git a/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt index 95d79b802ae..68b0382528c 100644 --- a/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt @@ -40,11 +40,18 @@ function(dd_wrapper_add_test name) target_link_libraries(${name} PRIVATE ${Python3_LIBRARIES}) endif() - # Test executable needs to find _stack_v2 in parent directory - set_target_properties(${name} PROPERTIES INSTALL_RPATH "$ORIGIN/.." BUILD_WITH_INSTALL_RPATH TRUE) - add_ddup_config(${name}) + # Test executable needs to find _stack_v2 in parent directory + # Append to INSTALL_RPATH instead of replacing to preserve sanitizer library paths set by add_ddup_config + get_target_property(EXISTING_INSTALL_RPATH ${name} INSTALL_RPATH) + if(EXISTING_INSTALL_RPATH) + set_target_properties(${name} PROPERTIES INSTALL_RPATH "${EXISTING_INSTALL_RPATH};$ORIGIN/.." + BUILD_WITH_INSTALL_RPATH TRUE) + else() + set_target_properties(${name} PROPERTIES INSTALL_RPATH "$ORIGIN/.." BUILD_WITH_INSTALL_RPATH TRUE) + endif() + gtest_discover_tests(${name} DISCOVERY_MODE PRE_TEST) # Delay test discovery until test execution to avoid running # sanitizer-built executables during build From 5ae6cfd11d9cc8731524d7af68eacbd473869a59 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Mon, 10 Nov 2025 15:27:10 -0500 Subject: [PATCH 19/27] format --- .../internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt | 4 ++-- .../internal/datadog/profiling/stack_v2/test/CMakeLists.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt b/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt index ada2662c4f5..ade703ca4ad 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt @@ -40,8 +40,8 @@ function(dd_wrapper_add_test name) add_ddup_config(${name}) target_link_options(${name} PRIVATE -Wl,--no-as-needed) - # Test executable needs to find libdd_wrapper in parent directory - # Append to INSTALL_RPATH instead of replacing to preserve sanitizer library paths set by add_ddup_config + # Test executable needs to find libdd_wrapper in parent directory Append to INSTALL_RPATH instead of replacing to + # preserve sanitizer library paths set by add_ddup_config get_target_property(EXISTING_INSTALL_RPATH ${name} INSTALL_RPATH) if(EXISTING_INSTALL_RPATH) set_target_properties(${name} PROPERTIES INSTALL_RPATH "${EXISTING_INSTALL_RPATH};$ORIGIN/.." diff --git a/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt index 68b0382528c..ea2bc39ef9d 100644 --- a/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt @@ -42,8 +42,8 @@ function(dd_wrapper_add_test name) add_ddup_config(${name}) - # Test executable needs to find _stack_v2 in parent directory - # Append to INSTALL_RPATH instead of replacing to preserve sanitizer library paths set by add_ddup_config + # Test executable needs to find _stack_v2 in parent directory Append to INSTALL_RPATH instead of replacing to + # preserve sanitizer library paths set by add_ddup_config get_target_property(EXISTING_INSTALL_RPATH ${name} INSTALL_RPATH) if(EXISTING_INSTALL_RPATH) set_target_properties(${name} PROPERTIES INSTALL_RPATH "${EXISTING_INSTALL_RPATH};$ORIGIN/.." From 55827a56c25b53077e04bd66e834eb7cfd9c9b12 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Mon, 10 Nov 2025 23:12:39 -0500 Subject: [PATCH 20/27] test C++ 20 --- ddtrace/internal/datadog/profiling/cmake/AnalysisFunc.cmake | 4 ++-- ddtrace/profiling/collector/_memalloc_heap_map.cpp | 4 +--- setup.py | 4 ++-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/cmake/AnalysisFunc.cmake b/ddtrace/internal/datadog/profiling/cmake/AnalysisFunc.cmake index 1a7d66493c9..90e1fbd3704 100644 --- a/ddtrace/internal/datadog/profiling/cmake/AnalysisFunc.cmake +++ b/ddtrace/internal/datadog/profiling/cmake/AnalysisFunc.cmake @@ -1,9 +1,9 @@ include(CheckIPOSupported) function(add_ddup_config target) - # Profiling native extensions are built with C++17, even though underlying repo adheres to the manylinux 2014 + # Profiling native extensions are built with C++20, even though underlying repo adheres to the manylinux 2014 # standard. This isn't currently a problem, but if it becomes one, we may have to structure the library differently. - target_compile_features(${target} PUBLIC cxx_std_17) + target_compile_features(${target} PUBLIC cxx_std_20) # Common compile options target_compile_options( diff --git a/ddtrace/profiling/collector/_memalloc_heap_map.cpp b/ddtrace/profiling/collector/_memalloc_heap_map.cpp index 2f430d55122..6f4c3443ed9 100644 --- a/ddtrace/profiling/collector/_memalloc_heap_map.cpp +++ b/ddtrace/profiling/collector/_memalloc_heap_map.cpp @@ -104,9 +104,7 @@ memalloc_heap_map_size(memalloc_heap_map_t* m) traceback_t* memalloc_heap_map_insert(memalloc_heap_map_t* m, void* key, traceback_t* value) { - HeapSamples_Entry k; - k.key = key; - k.val = value; + HeapSamples_Entry k = { .key = key, .val = value }; HeapSamples_Insert res = HeapSamples_insert(&m->map, &k); traceback_t* prev = NULL; if (!res.inserted) { diff --git a/setup.py b/setup.py index 034b9e53071..343bc3d2583 100644 --- a/setup.py +++ b/setup.py @@ -1084,7 +1084,7 @@ def get_exts_for(name): "ddtrace.internal._threads", sources=["ddtrace/internal/_threads.cpp"], extra_compile_args=( - ["-std=c++17", "-Wall", "-Wextra"] + fast_build_args + ["-std=c++20", "-Wall", "-Wextra"] + fast_build_args if CURRENT_OS != "Windows" else ["/std:c++20", "/MT"] ), @@ -1120,7 +1120,7 @@ def get_exts_for(name): extra_compile_args=( debug_compile_args + (["-DNDEBUG"] if not debug_compile_args else ["-UNDEBUG"]) - + ["-D_POSIX_C_SOURCE=200809L", "-std=c++17"] + + ["-D_POSIX_C_SOURCE=200809L", "-std=c++20"] + fast_build_args ), ), From 3620d53700c9cb2290ab8f2426ce7d3d6e75d0d0 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Tue, 11 Nov 2025 10:33:50 -0500 Subject: [PATCH 21/27] test not installing libatomic at all --- .github/workflows/build_python_3.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build_python_3.yml b/.github/workflows/build_python_3.yml index c70b7c8156e..6b718bb4502 100644 --- a/.github/workflows/build_python_3.yml +++ b/.github/workflows/build_python_3.yml @@ -94,9 +94,6 @@ jobs: CIBW_BEFORE_ALL_MACOS: rustup target add aarch64-apple-darwin CIBW_BEFORE_ALL_LINUX: | if [[ "$(uname -m)-$(uname -i)-$(uname -o | tr '[:upper:]' '[:lower:]')-$(ldd --version 2>&1 | head -n 1 | awk '{print $1}')" != "i686-unknown-linux-musl" ]]; then - if command -v yum &> /dev/null; then - yum install -y libatomic.i686 || echo "libatomic.i686 not available, assuming it's already installed" - fi curl -sSf https://sh.rustup.rs | sh -s -- -y; fi CIBW_ENVIRONMENT_LINUX: PATH=$HOME/.cargo/bin:$PATH CMAKE_BUILD_PARALLEL_LEVEL=24 CMAKE_ARGS="-DNATIVE_TESTING=OFF" SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }} From eacd51d1f0346dc293fb3da87d1b5278e548daed Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Tue, 11 Nov 2025 11:23:21 -0500 Subject: [PATCH 22/27] Update ddtrace/profiling/collector/_utils.h Co-authored-by: Vlad Scherbich --- ddtrace/profiling/collector/_utils.h | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/ddtrace/profiling/collector/_utils.h b/ddtrace/profiling/collector/_utils.h index 076cbe86741..eaf7aca2914 100644 --- a/ddtrace/profiling/collector/_utils.h +++ b/ddtrace/profiling/collector/_utils.h @@ -25,17 +25,16 @@ random_range(uint64_t max) #define p_alloc_nr(x) (((x) + 16) * 3 / 2) #ifdef __cplusplus -#define p_realloc(p, count) \ - do { \ - (p) = static_cast(PyMem_RawRealloc((p), sizeof(*p) * (count))); \ - } while (0) +#define P_REALLOC_CAST(p, expr) static_cast(expr) #else -#define p_realloc(p, count) \ - do { \ - (p) = PyMem_RawRealloc((p), sizeof(*p) * (count)); \ - } while (0) +#define P_REALLOC_CAST(p, expr) (expr) #endif +#define p_realloc(p, count) \ + do { \ + (p) = P_REALLOC_CAST(p, PyMem_RawRealloc((p), sizeof(*p) * (count))); \ + } while (0) + #define p_grow(p, goalnb, allocnb) \ do { \ if ((goalnb) > *(allocnb)) { \ From dccc22e88a2445fa1f975207aa381ea308b98e3f Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Tue, 11 Nov 2025 11:38:31 -0500 Subject: [PATCH 23/27] cformat --- ddtrace/profiling/collector/_utils.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ddtrace/profiling/collector/_utils.h b/ddtrace/profiling/collector/_utils.h index eaf7aca2914..26b26806e87 100644 --- a/ddtrace/profiling/collector/_utils.h +++ b/ddtrace/profiling/collector/_utils.h @@ -30,9 +30,9 @@ random_range(uint64_t max) #define P_REALLOC_CAST(p, expr) (expr) #endif -#define p_realloc(p, count) \ - do { \ - (p) = P_REALLOC_CAST(p, PyMem_RawRealloc((p), sizeof(*p) * (count))); \ +#define p_realloc(p, count) \ + do { \ + (p) = P_REALLOC_CAST(p, PyMem_RawRealloc((p), sizeof(*p) * (count))); \ } while (0) #define p_grow(p, goalnb, allocnb) \ From 64e0523bc97d0c0bd0eb88b9a6c7f14955a2cf7f Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Tue, 11 Nov 2025 11:49:03 -0500 Subject: [PATCH 24/27] types in setup.py --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 343bc3d2583..e4e2c17d99e 100644 --- a/setup.py +++ b/setup.py @@ -1068,14 +1068,14 @@ def get_exts_for(name): # Determine the libdd_wrapper filename with the Python extension suffix # For editable builds, the library will be in the source directory # For regular builds, it will be built during build_ext and found via the build system - _dd_wrapper_suffix = sysconfig.get_config_var("EXT_SUFFIX") - _dd_wrapper_source_path = ( + _dd_wrapper_suffix: t.Optional[str] = sysconfig.get_config_var("EXT_SUFFIX") + _dd_wrapper_source_path: Path = ( HERE / "ddtrace" / "internal" / "datadog" / "profiling" / f"libdd_wrapper{_dd_wrapper_suffix}" ) # Only include extra_objects if the library already exists (editable builds) # For non-editable builds, build_libdd_wrapper() will handle it and the linker will find it - _dd_wrapper_extra_objects = [] + _dd_wrapper_extra_objects: t.List[str] = [] if CURRENT_OS in ("Linux", "Darwin") and _dd_wrapper_source_path.exists(): _dd_wrapper_extra_objects = [str(_dd_wrapper_source_path)] From 1c4f663d3685c8339caa3f2441f02f25c907720d Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Tue, 11 Nov 2025 12:01:54 -0500 Subject: [PATCH 25/27] setup.py uses append --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e4e2c17d99e..b0dd8b8d5ee 100644 --- a/setup.py +++ b/setup.py @@ -1094,8 +1094,7 @@ def get_exts_for(name): # _memalloc uses cwisstable which is not supported on Windows # Profiler extensions are only needed on Linux and macOS if CURRENT_OS != "Windows": - ext_modules.insert( - 0, + ext_modules.append( Extension( "ddtrace.profiling.collector._memalloc", sources=[ From 7244664332c981d678c53fcbfba4e9a156f68c46 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Tue, 11 Nov 2025 12:13:02 -0500 Subject: [PATCH 26/27] cleaner setup.py --- setup.py | 41 ++++++++++------------------------------- 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/setup.py b/setup.py index b0dd8b8d5ee..b272acc0e13 100644 --- a/setup.py +++ b/setup.py @@ -701,29 +701,23 @@ def build_extension(self, ext): raise else: # For the memalloc extension, dynamically add libdd_wrapper to extra_objects - # if it's not already there (needed for non-editable builds) if ext.name == "ddtrace.profiling.collector._memalloc" and CURRENT_OS in ("Linux", "Darwin"): dd_wrapper_suffix = sysconfig.get_config_var("EXT_SUFFIX") - # For non-editable builds, remove any source directory paths and add build directory path - if not (IS_EDITABLE or getattr(self, "inplace", False)): - # Remove any source directory libdd_wrapper paths - source_wrapper_dir = Path(__file__).parent / "ddtrace" / "internal" / "datadog" / "profiling" - ext.extra_objects = [ - obj for obj in ext.extra_objects if not obj.startswith(str(source_wrapper_dir)) - ] - - # Add build directory path + if IS_EDITABLE or getattr(self, "inplace", False): + # Editable build: use source directory + wrapper_dir = Path(__file__).parent / "ddtrace" / "internal" / "datadog" / "profiling" + else: + # Non-editable build: use build directory wrapper_dir = ( Path(__file__).parent / Path(self.build_lib) / "ddtrace" / "internal" / "datadog" / "profiling" ) - wrapper_path = wrapper_dir / f"libdd_wrapper{dd_wrapper_suffix}" - if wrapper_path.exists(): - wrapper_path_str = str(wrapper_path) - if wrapper_path_str not in ext.extra_objects: - ext.extra_objects.append(wrapper_path_str) - print(f"Added libdd_wrapper to link: {wrapper_path_str}") + wrapper_path = wrapper_dir / f"libdd_wrapper{dd_wrapper_suffix}" + if wrapper_path.exists(): + wrapper_path_str = str(wrapper_path) + if wrapper_path_str not in ext.extra_objects: + ext.extra_objects.append(wrapper_path_str) super().build_extension(ext) @@ -1065,20 +1059,6 @@ def get_exts_for(name): if not IS_PYSTON: - # Determine the libdd_wrapper filename with the Python extension suffix - # For editable builds, the library will be in the source directory - # For regular builds, it will be built during build_ext and found via the build system - _dd_wrapper_suffix: t.Optional[str] = sysconfig.get_config_var("EXT_SUFFIX") - _dd_wrapper_source_path: Path = ( - HERE / "ddtrace" / "internal" / "datadog" / "profiling" / f"libdd_wrapper{_dd_wrapper_suffix}" - ) - - # Only include extra_objects if the library already exists (editable builds) - # For non-editable builds, build_libdd_wrapper() will handle it and the linker will find it - _dd_wrapper_extra_objects: t.List[str] = [] - if CURRENT_OS in ("Linux", "Darwin") and _dd_wrapper_source_path.exists(): - _dd_wrapper_extra_objects = [str(_dd_wrapper_source_path)] - ext_modules: t.List[t.Union[Extension, Cython.Distutils.Extension, RustExtension]] = [ Extension( "ddtrace.internal._threads", @@ -1107,7 +1087,6 @@ def get_exts_for(name): include_dirs=[ "ddtrace/internal/datadog/profiling/dd_wrapper/include", ], - extra_objects=_dd_wrapper_extra_objects, extra_link_args=( ["-Wl,-rpath,$ORIGIN/../../internal/datadog/profiling", "-latomic"] if CURRENT_OS == "Linux" From fd3cca6e55b1d580fda41f9bed05656c0a599cb5 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Tue, 11 Nov 2025 12:18:34 -0500 Subject: [PATCH 27/27] setup.py type annotations --- setup.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index b272acc0e13..88b679aaa03 100644 --- a/setup.py +++ b/setup.py @@ -702,8 +702,9 @@ def build_extension(self, ext): else: # For the memalloc extension, dynamically add libdd_wrapper to extra_objects if ext.name == "ddtrace.profiling.collector._memalloc" and CURRENT_OS in ("Linux", "Darwin"): - dd_wrapper_suffix = sysconfig.get_config_var("EXT_SUFFIX") + dd_wrapper_suffix: t.Optional[str] = sysconfig.get_config_var("EXT_SUFFIX") + wrapper_dir: Path if IS_EDITABLE or getattr(self, "inplace", False): # Editable build: use source directory wrapper_dir = Path(__file__).parent / "ddtrace" / "internal" / "datadog" / "profiling" @@ -713,9 +714,9 @@ def build_extension(self, ext): Path(__file__).parent / Path(self.build_lib) / "ddtrace" / "internal" / "datadog" / "profiling" ) - wrapper_path = wrapper_dir / f"libdd_wrapper{dd_wrapper_suffix}" + wrapper_path: Path = wrapper_dir / f"libdd_wrapper{dd_wrapper_suffix}" if wrapper_path.exists(): - wrapper_path_str = str(wrapper_path) + wrapper_path_str: str = str(wrapper_path) if wrapper_path_str not in ext.extra_objects: ext.extra_objects.append(wrapper_path_str)