From 0e4003a42c2219a431530b3947d9a1653fd7921c Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Wed, 1 Jul 2026 16:02:07 -0400 Subject: [PATCH 01/28] fix(profiling): harden stack sampler against foreign SIGSEGV handlers The native stack sampler's safe_memcpy relies on owning the SIGSEGV handler for its fault recovery. Components such as PyTorch/CUDA and abseil (pulled in by vLLM/gRPC) install their own SIGSEGV handler during the import-heavy startup, after which a fault in safe_memcpy no longer recovers and instead crashes the process (PROF-14568). Start the sampler on the safe syscall-based copy for a warmup window, then upgrade to safe_memcpy only if we still own the handler, and keep checking ownership every cycle so we fall back permanently if a handler is installed later (e.g. lazy CUDA init on first GPU use). The warmup window is configurable via DD_PROFILING_STACK_FAST_COPY_WARMUP_S. --- .../profiling/stack/echion/echion/danger.h | 8 ++ .../profiling/stack/src/echion/danger.cc | 10 ++ .../datadog/profiling/stack/src/sampler.cpp | 81 ++++++++++++++ ...ign-handler-fallback-3d8f1b6a0e25c947.yaml | 12 +++ .../repro_prof_14568_foreign_handler.py | 101 ++++++++++++++++++ tests/profiling/test_main.py | 65 +++++++++++ 6 files changed, 277 insertions(+) create mode 100644 releasenotes/notes/profiling-stack-sampler-foreign-handler-fallback-3d8f1b6a0e25c947.yaml create mode 100644 scripts/profiling/repro_prof_14568_foreign_handler.py diff --git a/ddtrace/internal/datadog/profiling/stack/echion/echion/danger.h b/ddtrace/internal/datadog/profiling/stack/echion/echion/danger.h index 71de5f12e1b..eed1298df07 100644 --- a/ddtrace/internal/datadog/profiling/stack/echion/echion/danger.h +++ b/ddtrace/internal/datadog/profiling/stack/echion/echion/danger.h @@ -27,6 +27,14 @@ init_segv_catcher(); void uninstall_segv_handler(); +// Returns true when our SIGSEGV handler (segv_handler) is the currently +// installed disposition. Used by the sampler to detect when another component +// (e.g. PyTorch/CUDA) has taken over the handler, in which case safe_memcpy's +// fault recovery would no longer work and we must fall back to a syscall-based +// copy. Returns false on any sigaction error. +bool +segv_handler_installed(); + #if defined PL_LINUX ssize_t safe_memcpy_wrapper(pid_t, diff --git a/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc b/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc index 4fe7e1a57b3..13a96112ba8 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc +++ b/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc @@ -140,6 +140,16 @@ init_segv_catcher() return 0; } +bool +segv_handler_installed() +{ + struct sigaction current; + if (sigaction(SIGSEGV, nullptr, ¤t) != 0) { + return false; + } + return current.sa_sigaction == segv_handler; +} + void uninstall_segv_handler() { diff --git a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp index 90cbd5abcb2..a9f80874ab8 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp @@ -25,6 +25,25 @@ using namespace Datadog; +// Number of seconds the stack sampler runs on the safe syscall-based memory copy +// before upgrading to the faster safe_memcpy path. Running on the safe path during +// the import-heavy startup window means a fault there can never crash, even if +// another component (PyTorch/CUDA, or abseil pulled in by vLLM/gRPC) installs its +// own SIGSEGV handler. Overridable via DD_PROFILING_STACK_FAST_COPY_WARMUP_S +// (PROF-14568). +static long +fast_copy_warmup_seconds() +{ + if (const char* val = getenv("DD_PROFILING_STACK_FAST_COPY_WARMUP_S")) { + char* end = nullptr; + long parsed = strtol(val, &end, 10); + if (end != val && parsed >= 0) { + return parsed; + } + } + return 15; +} + // Helper class for spawning a std::thread with control over its default stack size #ifdef __linux__ #include @@ -355,6 +374,34 @@ Sampler::sampling_thread(const uint64_t seq_num) auto sample_time_prev = steady_clock::now(); auto interval_adjust_time_prev = sample_time_prev; + // Foreign SIGSEGV handler handling for safe_memcpy (PROF-14568). + // + // safe_memcpy's fault recovery only works while we own the SIGSEGV handler. + // Other components (PyTorch/CUDA, or abseil pulled in by vLLM/gRPC) install + // their own handler, most often during the import-heavy startup. We therefore: + // 1. run on the safe syscall-based copy for a warmup window, so a fault during + // startup can never crash regardless of who owns the handler, then upgrade + // to safe_memcpy only if we still own the handler; and + // 2. keep checking ownership every cycle afterwards, permanently falling back + // to the syscall copy if a handler is installed later (e.g. lazy CUDA init + // on first GPU use). + const bool fast_copy_desired = fast_copy_active; +#if defined PL_LINUX + const bool syscall_copy_available = process_vm_readv_available; +#else + const bool syscall_copy_available = true; // mach_vm_read_overwrite is always available +#endif + // Only warm up when fast copy is wanted and a safe path exists to run on in the + // meantime; otherwise leave the active copy method untouched. + const bool fast_copy_warmup = fast_copy_desired && syscall_copy_available; + bool fast_copy_upgraded = !fast_copy_warmup; + bool handler_fallback_done = false; + const auto fast_copy_warmup_deadline = sample_time_prev + seconds(fast_copy_warmup_seconds()); + if (fast_copy_warmup) { + // Drop to the safe syscall copy for the startup window. + set_fast_copy_enabled(false); + } + while (seq_num == thread_seq_num.load()) { // Check if a pause has been requested (e.g., for signal handler swapping). // Block until resumed or the thread is asked to stop. @@ -379,6 +426,40 @@ Sampler::sampling_thread(const uint64_t seq_num) auto wall_time_us = duration_cast(sample_time_now - sample_time_prev).count(); sample_time_prev = sample_time_now; + // Foreign SIGSEGV handler handling (see the notes before the loop). The + // ownership check is one cheap sigaction read; faulthandler's transient + // swaps are not a concern because the sampler is paused around them. + if (fast_copy_desired) { + if (!fast_copy_upgraded) { + // Warmup window: still on the safe syscall copy. Once it elapses, + // upgrade to safe_memcpy only if we still own the handler. + if (sample_time_now >= fast_copy_warmup_deadline) { + fast_copy_upgraded = true; // decide once + if (segv_handler_installed()) { + set_fast_copy_enabled(true); + } else { + // Another component already owns the handler; stay on the + // safe syscall copy for the life of the process. + handler_fallback_done = true; + std::cerr << "ddtrace stack profiler: another component owns the SIGSEGV " + "handler; keeping the syscall-based memory copy to avoid crashing." + << std::endl; + } + } + } else if (fast_copy_active && !handler_fallback_done && !segv_handler_installed()) { + // A component took over the handler after we upgraded to safe_memcpy; + // fall back permanently. A false positive only costs the slower copy + // path, so we do not debounce. Attempted/logged once: if the syscall + // copy is also unavailable, set_fast_copy_enabled reports it and there + // is nothing more to do. + handler_fallback_done = true; + std::cerr << "ddtrace stack profiler: SIGSEGV handler was taken over by another " + "component; falling back to syscall-based memory copy to avoid crashing." + << std::endl; + set_fast_copy_enabled(false); + } + } + // Reset per-cycle asyncio task accumulator before iterating sampled threads echion->reset_asyncio_task_count(); diff --git a/releasenotes/notes/profiling-stack-sampler-foreign-handler-fallback-3d8f1b6a0e25c947.yaml b/releasenotes/notes/profiling-stack-sampler-foreign-handler-fallback-3d8f1b6a0e25c947.yaml new file mode 100644 index 00000000000..e71de08a731 --- /dev/null +++ b/releasenotes/notes/profiling-stack-sampler-foreign-handler-fallback-3d8f1b6a0e25c947.yaml @@ -0,0 +1,12 @@ +--- +fixes: + - | + profiling: The stack profiler no longer crashes in environments where another + component (for example PyTorch/CUDA, or abseil pulled in by vLLM/gRPC) installs + its own ``SIGSEGV`` handler. The sampler now runs on a syscall-based memory copy + during a short startup warmup window, upgrades to the faster fault-recovery copy + only if it still owns the ``SIGSEGV`` handler afterwards, and permanently falls + back to the syscall-based copy if a foreign handler is installed later. This + removes the need for the ``_DD_PROFILING_STACK_FAST_COPY=0`` workaround in those + environments. The warmup duration can be tuned with the + ``DD_PROFILING_STACK_FAST_COPY_WARMUP_S`` environment variable (default 15). diff --git a/scripts/profiling/repro_prof_14568_foreign_handler.py b/scripts/profiling/repro_prof_14568_foreign_handler.py new file mode 100644 index 00000000000..fbb341507a8 --- /dev/null +++ b/scripts/profiling/repro_prof_14568_foreign_handler.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Repro for the stack-profiler SIGSEGV under a foreign SIGSEGV handler (PROF-14568, Layer 1). + +``safe_memcpy``'s fault recovery only works while the profiler owns the +``SIGSEGV`` disposition. Libraries such as PyTorch/CUDA install their own +``SIGSEGV`` handler after the profiler has started. Once that happens, a fault on +a stale read (which the profiler would normally recover from via ``siglongjmp``) +is delivered to the foreign handler instead, and the process crashes. + +This script reproduces the condition: + + 1. Start the profiler with fast copy enabled. + 2. Overwrite the ``SIGSEGV`` disposition with a foreign handler (here, the + default disposition, standing in for torch/CUDA installing their own). + 3. Churn deep Python stacks while the sampler runs. + +On ``main`` the sampler keeps using ``safe_memcpy`` even though it no longer owns +the handler, so a fault terminates the process. With the fix, the sampler detects +within ~2s that it lost handler ownership and falls back to ``process_vm_readv`` +(Linux) / ``mach_vm_read_overwrite`` (macOS), so the process survives. + +Usage: + DD_PROFILING_STACK_FAST_COPY=1 python scripts/profiling/repro_prof_14568_foreign_handler.py + +Optionally import torch to install a *real* foreign handler instead of the +synthetic one: + DD_PROFILING_STACK_FAST_COPY=1 python scripts/profiling/repro_prof_14568_foreign_handler.py --torch + +Exit code 0 means the process survived (fix working). A crash (negative exit / +SIGSEGV from the parent's perspective) reproduces the bug. +""" + +from __future__ import annotations + +import argparse +import os +import signal +import sys +import threading +import time + + +def _busy_worker(stop_evt: threading.Event) -> None: + def recurse(depth: int) -> int: + if depth <= 0: + return sum(len(str(i)) for i in range(16)) + return recurse(depth - 1) + + while not stop_evt.is_set(): + recurse(28) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--threads", type=int, default=8) + parser.add_argument("--run", type=float, default=6.0, help="seconds to sample under the foreign handler") + parser.add_argument("--torch", action="store_true", help="import torch to install a real foreign handler") + args = parser.parse_args() + + os.environ.setdefault("DD_PROFILING_STACK_V2_ENABLED", "true") + os.environ.setdefault("DD_PROFILING_STACK_FAST_COPY", "1") + + from ddtrace.profiling.profiler import Profiler + + prof = Profiler() + prof.start() + + if args.torch: + try: + import torch # noqa: F401 + + print("torch imported; its SIGSEGV handler (if any) is now installed") + except Exception as exc: # pragma: no cover - torch is optional + print(f"could not import torch ({exc}); installing synthetic foreign handler instead") + signal.signal(signal.SIGSEGV, signal.SIG_DFL) + else: + # Stand in for a library that installs its own SIGSEGV handler after start. + signal.signal(signal.SIGSEGV, signal.SIG_DFL) + print("installed synthetic foreign SIGSEGV handler (SIG_DFL)") + + stop_evt = threading.Event() + workers = [threading.Thread(target=_busy_worker, args=(stop_evt,), daemon=True) for _ in range(args.threads)] + for w in workers: + w.start() + + # Sample for longer than the handler-ownership check cadence (~2s) so the + # fixed sampler observes the foreign handler and falls back. + print(f"sampling for {args.run}s under a foreign SIGSEGV handler...") + time.sleep(args.run) + + stop_evt.set() + for w in workers: + w.join(timeout=1.0) + prof.stop(flush=True) + + print("OK: process survived under a foreign SIGSEGV handler") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/profiling/test_main.py b/tests/profiling/test_main.py index bf1adea360c..7d9337da7f2 100644 --- a/tests/profiling/test_main.py +++ b/tests/profiling/test_main.py @@ -247,3 +247,68 @@ def test_profiler_start_up_with_module_clean_up_in_protobuf_app() -> None: from google.protobuf import empty_pb2 # noqa:F401 print("OK") + + +@pytest.mark.skipif(sys.platform == "win32", reason="stack v2 profiler is not available on Windows") +@pytest.mark.subprocess( + env=dict( + DD_PROFILING_STACK_V2_ENABLED="true", + DD_PROFILING_STACK_FAST_COPY="1", + # Skip the warmup window so the sampler decides on safe_memcpy immediately; + # otherwise it would run on the safe syscall copy for the default warmup and + # the degrade path would not engage within the test's short sampling window. + DD_PROFILING_STACK_FAST_COPY_WARMUP_S="0", + ), + out="OK\n", + # The sampler logs a one-time warning to stderr when it sees a foreign handler + # and stays on / drops to the syscall-based copy. Either "falling back ..." (it + # upgraded then lost the handler) or "keeping ..." (the foreign handler was + # already present at the upgrade check) proves the degrade path fired; both + # contain this substring. We assert it rather than requiring empty stderr. + err=lambda e: "syscall-based memory copy" in e, +) +def test_stack_profiler_foreign_segv_handler_no_crash() -> None: + # Regression test for PROF-14568 (Layer 1): if another component installs its + # own SIGSEGV handler after the profiler starts (as PyTorch/CUDA do), the + # native sampler must detect it does not own the handler and degrade to a + # syscall-based copy instead of crashing. Here we overwrite the SIGSEGV + # disposition from Python and keep sampling; the process must not crash and + # must use the syscall-based copy. + import signal + import threading + import time + + from ddtrace.profiling.profiler import Profiler + + prof = Profiler() + prof.start() + + # Overwrite the profiler's SIGSEGV handler with a foreign one. + signal.signal(signal.SIGSEGV, signal.SIG_DFL) + + stop_evt = threading.Event() + + def churn() -> None: + def recurse(depth): + if depth <= 0: + return sum(len(str(i)) for i in range(8)) + return recurse(depth - 1) + + while not stop_evt.is_set(): + recurse(20) + + workers = [threading.Thread(target=churn, daemon=True) for _ in range(4)] + for w in workers: + w.start() + + # With the warmup disabled the sampler checks handler ownership every cycle and + # degrades on the first observation, so detection happens within a few sample + # intervals. Sample for a comfortable margin so the degrade reliably fires. + time.sleep(2.0) + + stop_evt.set() + for w in workers: + w.join(timeout=1.0) + prof.stop(flush=True) + + print("OK") From 76013f35b65355b3f8f99163db9262d59e7e04dc Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Wed, 1 Jul 2026 21:09:08 -0400 Subject: [PATCH 02/28] test(profiling): drop warmup env override; add handler-ownership introspection The stack fast-copy warmup window was overridable via a raw DD_PROFILING_STACK_FAST_COPY_WARMUP_S getenv that existed only so the regression test could skip the warmup. It was never a registered config and users never need to tune it, so hardcode the 15s warmup as a constant. To keep the foreign-handler behavior testable without that knob, expose segv_handler_installed() on the stack module (mirroring is_safe_copy_failed and reinstall_segv_handler) and assert the ownership predicate directly: our handler is installed at import, flips to not-owned when a foreign handler is installed, and is reclaimed by reinstall_segv_handler(). This also removes the flaky stderr-warning assertion from the previous test. --- .../datadog/profiling/stack/__init__.pyi | 8 +++ .../datadog/profiling/stack/src/sampler.cpp | 18 +---- .../datadog/profiling/stack/src/stack.cpp | 16 +++++ ...ign-handler-fallback-3d8f1b6a0e25c947.yaml | 3 +- tests/profiling/test_main.py | 72 ++++++------------- 5 files changed, 49 insertions(+), 68 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/stack/__init__.pyi b/ddtrace/internal/datadog/profiling/stack/__init__.pyi index dea8a30197b..d8a6f148f4a 100644 --- a/ddtrace/internal/datadog/profiling/stack/__init__.pyi +++ b/ddtrace/internal/datadog/profiling/stack/__init__.pyi @@ -52,6 +52,14 @@ def reinstall_segv_handler() -> None: """ ... +def segv_handler_installed() -> bool: + """Return True if our SIGSEGV handler is the currently installed disposition. + + Used to detect when another component (e.g. PyTorch/CUDA) has taken over the + handler, in which case safe_memcpy's fault recovery would no longer work. + """ + ... + # Pause/resume sampling def pause_sampling() -> bool | None: """Pause the sampling thread and wait for any in-flight sample to complete. diff --git a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp index a9f80874ab8..239bfc7757d 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp @@ -29,20 +29,8 @@ using namespace Datadog; // before upgrading to the faster safe_memcpy path. Running on the safe path during // the import-heavy startup window means a fault there can never crash, even if // another component (PyTorch/CUDA, or abseil pulled in by vLLM/gRPC) installs its -// own SIGSEGV handler. Overridable via DD_PROFILING_STACK_FAST_COPY_WARMUP_S -// (PROF-14568). -static long -fast_copy_warmup_seconds() -{ - if (const char* val = getenv("DD_PROFILING_STACK_FAST_COPY_WARMUP_S")) { - char* end = nullptr; - long parsed = strtol(val, &end, 10); - if (end != val && parsed >= 0) { - return parsed; - } - } - return 15; -} +// own SIGSEGV handler during that window (PROF-14568). +static constexpr long kStackFastCopyWarmupSeconds = 15; // Helper class for spawning a std::thread with control over its default stack size #ifdef __linux__ @@ -396,7 +384,7 @@ Sampler::sampling_thread(const uint64_t seq_num) const bool fast_copy_warmup = fast_copy_desired && syscall_copy_available; bool fast_copy_upgraded = !fast_copy_warmup; bool handler_fallback_done = false; - const auto fast_copy_warmup_deadline = sample_time_prev + seconds(fast_copy_warmup_seconds()); + const auto fast_copy_warmup_deadline = sample_time_prev + seconds(kStackFastCopyWarmupSeconds); if (fast_copy_warmup) { // Drop to the safe syscall copy for the startup window. set_fast_copy_enabled(false); diff --git a/ddtrace/internal/datadog/profiling/stack/src/stack.cpp b/ddtrace/internal/datadog/profiling/stack/src/stack.cpp index e87f2522463..a3aae48528f 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/stack.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/stack.cpp @@ -882,6 +882,18 @@ stack_reinstall_segv_handler(PyObject* Py_UNUSED(self), PyObject* Py_UNUSED(args Py_RETURN_NONE; } +static PyObject* +stack_segv_handler_installed(PyObject* Py_UNUSED(self), PyObject* Py_UNUSED(args)) +{ + // Introspection used by tests: reports whether our SIGSEGV handler is the + // currently installed disposition. This is the same primitive the sampler + // uses to decide whether safe_memcpy's fault recovery is still trustworthy. + if (segv_handler_installed()) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyObject* stack_is_safe_copy_failed(PyObject* Py_UNUSED(self), PyObject* Py_UNUSED(args)) { @@ -964,6 +976,10 @@ static PyMethodDef stack_methods[] = { stack_reinstall_segv_handler, METH_NOARGS, "Reinstall SIGSEGV handler after another component overwrites it" }, + { "segv_handler_installed", + stack_segv_handler_installed, + METH_NOARGS, + "Return True if our SIGSEGV handler is the currently installed disposition" }, // Sampling pause/resume for safe signal handler swapping { "pause_sampling", stack_pause_sampling, diff --git a/releasenotes/notes/profiling-stack-sampler-foreign-handler-fallback-3d8f1b6a0e25c947.yaml b/releasenotes/notes/profiling-stack-sampler-foreign-handler-fallback-3d8f1b6a0e25c947.yaml index e71de08a731..021d4cf957b 100644 --- a/releasenotes/notes/profiling-stack-sampler-foreign-handler-fallback-3d8f1b6a0e25c947.yaml +++ b/releasenotes/notes/profiling-stack-sampler-foreign-handler-fallback-3d8f1b6a0e25c947.yaml @@ -8,5 +8,4 @@ fixes: only if it still owns the ``SIGSEGV`` handler afterwards, and permanently falls back to the syscall-based copy if a foreign handler is installed later. This removes the need for the ``_DD_PROFILING_STACK_FAST_COPY=0`` workaround in those - environments. The warmup duration can be tuned with the - ``DD_PROFILING_STACK_FAST_COPY_WARMUP_S`` environment variable (default 15). + environments. diff --git a/tests/profiling/test_main.py b/tests/profiling/test_main.py index 7d9337da7f2..1ab48d11a7c 100644 --- a/tests/profiling/test_main.py +++ b/tests/profiling/test_main.py @@ -251,64 +251,34 @@ def test_profiler_start_up_with_module_clean_up_in_protobuf_app() -> None: @pytest.mark.skipif(sys.platform == "win32", reason="stack v2 profiler is not available on Windows") @pytest.mark.subprocess( - env=dict( - DD_PROFILING_STACK_V2_ENABLED="true", - DD_PROFILING_STACK_FAST_COPY="1", - # Skip the warmup window so the sampler decides on safe_memcpy immediately; - # otherwise it would run on the safe syscall copy for the default warmup and - # the degrade path would not engage within the test's short sampling window. - DD_PROFILING_STACK_FAST_COPY_WARMUP_S="0", - ), + env=dict(_DD_PROFILING_STACK_FAST_COPY="1"), out="OK\n", - # The sampler logs a one-time warning to stderr when it sees a foreign handler - # and stays on / drops to the syscall-based copy. Either "falling back ..." (it - # upgraded then lost the handler) or "keeping ..." (the foreign handler was - # already present at the upgrade check) proves the degrade path fired; both - # contain this substring. We assert it rather than requiring empty stderr. - err=lambda e: "syscall-based memory copy" in e, + err=None, ) -def test_stack_profiler_foreign_segv_handler_no_crash() -> None: - # Regression test for PROF-14568 (Layer 1): if another component installs its - # own SIGSEGV handler after the profiler starts (as PyTorch/CUDA do), the - # native sampler must detect it does not own the handler and degrade to a - # syscall-based copy instead of crashing. Here we overwrite the SIGSEGV - # disposition from Python and keep sampling; the process must not crash and - # must use the syscall-based copy. +def test_stack_profiler_foreign_segv_handler_detection() -> None: + # Regression test for PROF-14568: safe_memcpy's fault recovery only works while + # we own the SIGSEGV handler. If another component (PyTorch/CUDA, or abseil + # pulled in by vLLM/gRPC) installs its own handler, the native sampler must + # detect that it no longer owns the handler so it can stay on / fall back to the + # syscall-based memory copy instead of crashing. + # + # segv_handler_installed() is the primitive that drives that decision. Our + # handler is installed at native import time when fast copy is enabled, so the + # predicate can be verified deterministically here without waiting on the + # sampler's startup warmup window. import signal - import threading - import time - from ddtrace.profiling.profiler import Profiler + from ddtrace.internal.datadog.profiling import stack - prof = Profiler() - prof.start() + # Our handler is installed by the extension's constructor on import. + assert stack.segv_handler_installed() is True - # Overwrite the profiler's SIGSEGV handler with a foreign one. + # A foreign component overwrites the SIGSEGV disposition: ownership must flip. signal.signal(signal.SIGSEGV, signal.SIG_DFL) + assert stack.segv_handler_installed() is False - stop_evt = threading.Event() - - def churn() -> None: - def recurse(depth): - if depth <= 0: - return sum(len(str(i)) for i in range(8)) - return recurse(depth - 1) - - while not stop_evt.is_set(): - recurse(20) - - workers = [threading.Thread(target=churn, daemon=True) for _ in range(4)] - for w in workers: - w.start() - - # With the warmup disabled the sampler checks handler ownership every cycle and - # degrades on the first observation, so detection happens within a few sample - # intervals. Sample for a comfortable margin so the degrade reliably fires. - time.sleep(2.0) - - stop_evt.set() - for w in workers: - w.join(timeout=1.0) - prof.stop(flush=True) + # Reclaiming the handler is detected too. + stack.reinstall_segv_handler() + assert stack.segv_handler_installed() is True print("OK") From 12cae400877783b3184ff27ba85e4d8a18257daf Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Wed, 1 Jul 2026 21:17:17 -0400 Subject: [PATCH 03/28] test(profiling): update fast-copy stat test for the startup warmup test_fast_copy_memory_enabled asserted fast_copy_memory_enabled=True within a 3s run, but the foreign-handler fix (PROF-14568) now starts the sampler on the safe syscall-based copy for a startup warmup window and only upgrades to safe_memcpy afterwards. During warmup the metadata correctly reports False, so the short run failed. Update the test to reflect the warmup-then-upgrade behavior: poll the emitted metadata until the sampler upgrades (fast_copy_memory_enabled=True), asserting it ran on the syscall copy during warmup first. Breaks as soon as the upgrade is observed to keep runtime bounded. --- .../collector/test_copy_memory_stats.py | 45 +++++++++++++------ 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/tests/profiling/collector/test_copy_memory_stats.py b/tests/profiling/collector/test_copy_memory_stats.py index 25430c926c9..2b4047a2913 100644 --- a/tests/profiling/collector/test_copy_memory_stats.py +++ b/tests/profiling/collector/test_copy_memory_stats.py @@ -81,7 +81,13 @@ def test_fast_copy_memory_disabled(): err=None, ) def test_fast_copy_memory_enabled(): - """fast_copy_memory_enabled is True when _DD_PROFILING_STACK_FAST_COPY=1.""" + """With _DD_PROFILING_STACK_FAST_COPY=1 the sampler eventually uses safe_memcpy. + + It first runs on the safe syscall-based copy for a startup warmup window + (PROF-14568) and upgrades to safe_memcpy afterwards once it confirms it still + owns the SIGSEGV handler, so the emitted metadata reports + fast_copy_memory_enabled=False during warmup and True after the upgrade. + """ import glob import json import os @@ -92,19 +98,30 @@ def test_fast_copy_memory_enabled(): p = profiler.Profiler(tracer=tracer) p.start() - time.sleep(3) - p.stop() output_filename = os.environ["DD_PROFILING_OUTPUT_PPROF"] + "." + str(os.getpid()) - files = sorted(glob.glob(output_filename + ".*.internal_metadata.json")) - assert files, "Expected at least one internal_metadata.json file" - for i, f in enumerate(files): - is_last_file = i == len(files) - 1 - with open(f) as fp: - metadata = json.load(fp) - if not is_last_file: - assert "fast_copy_memory_enabled" in metadata, f"Missing fast_copy_memory_enabled in {f}: {metadata}" - assert metadata["fast_copy_memory_enabled"] is True, ( - f"Expected fast_copy_memory_enabled=true by default: {metadata}" - ) + # Poll the emitted metadata until the sampler upgrades to safe_memcpy. The warmup + # window is a fixed internal constant (kStackFastCopyWarmupSeconds in sampler.cpp); + # allow a generous margin above it before giving up. + saw_warmup = False + saw_upgrade = False + deadline = time.monotonic() + 45 + while time.monotonic() < deadline and not saw_upgrade: + time.sleep(1) + for f in glob.glob(output_filename + ".*.internal_metadata.json"): + try: + with open(f) as fp: + enabled = json.load(fp).get("fast_copy_memory_enabled") + except (json.JSONDecodeError, OSError): + # The most recent file may still be mid-write; skip and retry. + continue + if enabled is False: + saw_warmup = True + elif enabled is True: + saw_upgrade = True + + p.stop() + + assert saw_warmup, "Expected the sampler to run on the syscall copy during the warmup window" + assert saw_upgrade, "Expected the sampler to upgrade to safe_memcpy (fast_copy_memory_enabled=True) after warmup" From 2c5ea5fa00ce6b12fa10d082ddedc4bbbeee68d5 Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Fri, 3 Jul 2026 11:23:14 -0400 Subject: [PATCH 04/28] fix(profiling): check SIGSEGV and SIGBUS ownership; stop sampling when no safe fallback Address review feedback on the foreign-handler fix: - segv_handler_installed() now requires our SA_SIGINFO handler to own BOTH SIGSEGV and SIGBUS (init_segv_catcher installs both). Owning only one still lets safe_memcpy fault into a foreign handler and crash. - When the per-cycle fallback cannot install a safe copy method (e.g. process_vm_readv blocked on the host), stop stack sampling instead of leaving the unsafe safe_memcpy path active under a foreign handler. - Update comments, logs, docstrings and the repro env var to _DD_PROFILING_STACK_FAST_COPY, and extend the detection test to cover SIGBUS. --- .../datadog/profiling/stack/__init__.pyi | 6 +-- .../profiling/stack/echion/echion/danger.h | 11 ++--- .../profiling/stack/src/echion/danger.cc | 19 +++++++-- .../datadog/profiling/stack/src/sampler.cpp | 42 +++++++++++-------- .../datadog/profiling/stack/src/stack.cpp | 9 ++-- .../repro_prof_14568_foreign_handler.py | 9 ++-- tests/profiling/test_main.py | 17 ++++++-- 7 files changed, 73 insertions(+), 40 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/stack/__init__.pyi b/ddtrace/internal/datadog/profiling/stack/__init__.pyi index d8a6f148f4a..0f2a6abe094 100644 --- a/ddtrace/internal/datadog/profiling/stack/__init__.pyi +++ b/ddtrace/internal/datadog/profiling/stack/__init__.pyi @@ -53,10 +53,10 @@ def reinstall_segv_handler() -> None: ... def segv_handler_installed() -> bool: - """Return True if our SIGSEGV handler is the currently installed disposition. + """Return True if our handler is the installed disposition for SIGSEGV and SIGBUS. - Used to detect when another component (e.g. PyTorch/CUDA) has taken over the - handler, in which case safe_memcpy's fault recovery would no longer work. + Used to detect when another component (e.g. PyTorch/CUDA) has taken over either + signal, in which case safe_memcpy's fault recovery would no longer work. """ ... diff --git a/ddtrace/internal/datadog/profiling/stack/echion/echion/danger.h b/ddtrace/internal/datadog/profiling/stack/echion/echion/danger.h index eed1298df07..fedd8e2e419 100644 --- a/ddtrace/internal/datadog/profiling/stack/echion/echion/danger.h +++ b/ddtrace/internal/datadog/profiling/stack/echion/echion/danger.h @@ -27,11 +27,12 @@ init_segv_catcher(); void uninstall_segv_handler(); -// Returns true when our SIGSEGV handler (segv_handler) is the currently -// installed disposition. Used by the sampler to detect when another component -// (e.g. PyTorch/CUDA) has taken over the handler, in which case safe_memcpy's -// fault recovery would no longer work and we must fall back to a syscall-based -// copy. Returns false on any sigaction error. +// Returns true only when our fault-recovery handler (segv_handler) is the +// currently installed disposition for BOTH SIGSEGV and SIGBUS (init_segv_catcher +// installs it, with SA_SIGINFO, for both). Used by the sampler to detect when +// another component (e.g. PyTorch/CUDA) has taken over either signal, in which +// case safe_memcpy's fault recovery would no longer work and we must fall back to +// a syscall-based copy. Returns false on any sigaction error. bool segv_handler_installed(); diff --git a/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc b/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc index 13a96112ba8..bdfbc129f35 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc +++ b/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc @@ -143,11 +143,22 @@ init_segv_catcher() bool segv_handler_installed() { - struct sigaction current; - if (sigaction(SIGSEGV, nullptr, ¤t) != 0) { - return false; + // safe_memcpy's fault recovery relies on owning BOTH SIGSEGV and SIGBUS: + // init_segv_catcher installs our 3-arg (SA_SIGINFO) handler for both, and a + // copy fault can be delivered as either. If a foreign component has taken over + // either signal - or replaced our handler with a 1-arg (non SA_SIGINFO) one - + // recovery is no longer guaranteed, so report that our handler is not installed. + const int signals[] = { SIGSEGV, SIGBUS }; + for (int signo : signals) { + struct sigaction current; + if (sigaction(signo, nullptr, ¤t) != 0) { + return false; + } + if (current.sa_sigaction != segv_handler || (current.sa_flags & SA_SIGINFO) == 0) { + return false; + } } - return current.sa_sigaction == segv_handler; + return true; } void diff --git a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp index 239bfc7757d..fe51f4adab1 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp @@ -362,11 +362,11 @@ Sampler::sampling_thread(const uint64_t seq_num) auto sample_time_prev = steady_clock::now(); auto interval_adjust_time_prev = sample_time_prev; - // Foreign SIGSEGV handler handling for safe_memcpy (PROF-14568). + // Foreign SIGSEGV/SIGBUS handler handling for safe_memcpy (PROF-14568). // - // safe_memcpy's fault recovery only works while we own the SIGSEGV handler. - // Other components (PyTorch/CUDA, or abseil pulled in by vLLM/gRPC) install - // their own handler, most often during the import-heavy startup. We therefore: + // safe_memcpy's fault recovery only works while we own the SIGSEGV and SIGBUS + // handlers. Other components (PyTorch/CUDA, or abseil pulled in by vLLM/gRPC) + // install their own handler, most often during the import-heavy startup. We do: // 1. run on the safe syscall-based copy for a warmup window, so a fault during // startup can never crash regardless of who owns the handler, then upgrade // to safe_memcpy only if we still own the handler; and @@ -414,37 +414,45 @@ Sampler::sampling_thread(const uint64_t seq_num) auto wall_time_us = duration_cast(sample_time_now - sample_time_prev).count(); sample_time_prev = sample_time_now; - // Foreign SIGSEGV handler handling (see the notes before the loop). The - // ownership check is one cheap sigaction read; faulthandler's transient - // swaps are not a concern because the sampler is paused around them. + // Foreign SIGSEGV/SIGBUS handler handling (see the notes before the loop). + // The ownership check is one cheap sigaction read per signal; faulthandler's + // transient swaps are not a concern because the sampler is paused around them. if (fast_copy_desired) { if (!fast_copy_upgraded) { // Warmup window: still on the safe syscall copy. Once it elapses, - // upgrade to safe_memcpy only if we still own the handler. + // upgrade to safe_memcpy only if we still own the handlers. if (sample_time_now >= fast_copy_warmup_deadline) { fast_copy_upgraded = true; // decide once if (segv_handler_installed()) { set_fast_copy_enabled(true); } else { - // Another component already owns the handler; stay on the - // safe syscall copy for the life of the process. + // Another component already owns a handler; stay on the safe + // syscall copy (already active from warmup) for the life of + // the process. handler_fallback_done = true; - std::cerr << "ddtrace stack profiler: another component owns the SIGSEGV " + std::cerr << "ddtrace stack profiler: another component owns the SIGSEGV/SIGBUS " "handler; keeping the syscall-based memory copy to avoid crashing." << std::endl; } } } else if (fast_copy_active && !handler_fallback_done && !segv_handler_installed()) { - // A component took over the handler after we upgraded to safe_memcpy; + // A component took over a handler after we upgraded to safe_memcpy; // fall back permanently. A false positive only costs the slower copy - // path, so we do not debounce. Attempted/logged once: if the syscall - // copy is also unavailable, set_fast_copy_enabled reports it and there - // is nothing more to do. + // path, so we do not debounce. handler_fallback_done = true; - std::cerr << "ddtrace stack profiler: SIGSEGV handler was taken over by another " + std::cerr << "ddtrace stack profiler: SIGSEGV/SIGBUS handler was taken over by another " "component; falling back to syscall-based memory copy to avoid crashing." << std::endl; - set_fast_copy_enabled(false); + if (!set_fast_copy_enabled(false)) { + // No safe copy method is available to fall back to (e.g. + // process_vm_readv is blocked on this host), so safe_memcpy is + // still active. Continuing to read memory while a foreign handler + // owns the fault signals would crash, so stop sampling instead. + std::cerr << "ddtrace stack profiler: no safe memory-copy fallback available; " + "stopping stack sampling to avoid crashing." + << std::endl; + break; + } } } diff --git a/ddtrace/internal/datadog/profiling/stack/src/stack.cpp b/ddtrace/internal/datadog/profiling/stack/src/stack.cpp index a3aae48528f..598b1d3b813 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/stack.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/stack.cpp @@ -885,9 +885,10 @@ stack_reinstall_segv_handler(PyObject* Py_UNUSED(self), PyObject* Py_UNUSED(args static PyObject* stack_segv_handler_installed(PyObject* Py_UNUSED(self), PyObject* Py_UNUSED(args)) { - // Introspection used by tests: reports whether our SIGSEGV handler is the - // currently installed disposition. This is the same primitive the sampler - // uses to decide whether safe_memcpy's fault recovery is still trustworthy. + // Introspection used by tests: reports whether our fault-recovery handler is + // the currently installed disposition for both SIGSEGV and SIGBUS. This is the + // same primitive the sampler uses to decide whether safe_memcpy's fault + // recovery is still trustworthy. if (segv_handler_installed()) { Py_RETURN_TRUE; } @@ -979,7 +980,7 @@ static PyMethodDef stack_methods[] = { { "segv_handler_installed", stack_segv_handler_installed, METH_NOARGS, - "Return True if our SIGSEGV handler is the currently installed disposition" }, + "Return True if our handler is the currently installed disposition for SIGSEGV and SIGBUS" }, // Sampling pause/resume for safe signal handler swapping { "pause_sampling", stack_pause_sampling, diff --git a/scripts/profiling/repro_prof_14568_foreign_handler.py b/scripts/profiling/repro_prof_14568_foreign_handler.py index fbb341507a8..5257f93860d 100644 --- a/scripts/profiling/repro_prof_14568_foreign_handler.py +++ b/scripts/profiling/repro_prof_14568_foreign_handler.py @@ -20,11 +20,11 @@ (Linux) / ``mach_vm_read_overwrite`` (macOS), so the process survives. Usage: - DD_PROFILING_STACK_FAST_COPY=1 python scripts/profiling/repro_prof_14568_foreign_handler.py + _DD_PROFILING_STACK_FAST_COPY=1 python scripts/profiling/repro_prof_14568_foreign_handler.py Optionally import torch to install a *real* foreign handler instead of the synthetic one: - DD_PROFILING_STACK_FAST_COPY=1 python scripts/profiling/repro_prof_14568_foreign_handler.py --torch + _DD_PROFILING_STACK_FAST_COPY=1 python scripts/profiling/repro_prof_14568_foreign_handler.py --torch Exit code 0 means the process survived (fix working). A crash (negative exit / SIGSEGV from the parent's perspective) reproduces the bug. @@ -58,7 +58,10 @@ def main() -> int: args = parser.parse_args() os.environ.setdefault("DD_PROFILING_STACK_V2_ENABLED", "true") - os.environ.setdefault("DD_PROFILING_STACK_FAST_COPY", "1") + # The native fast-copy toggle is the private _DD_ variable that echion/vm.cc reads; + # force it on so the repro exercises the safe_memcpy + handler-fallback path even if + # the ambient environment disabled it. + os.environ["_DD_PROFILING_STACK_FAST_COPY"] = "1" from ddtrace.profiling.profiler import Profiler diff --git a/tests/profiling/test_main.py b/tests/profiling/test_main.py index 1ab48d11a7c..2d05db85182 100644 --- a/tests/profiling/test_main.py +++ b/tests/profiling/test_main.py @@ -257,10 +257,11 @@ def test_profiler_start_up_with_module_clean_up_in_protobuf_app() -> None: ) def test_stack_profiler_foreign_segv_handler_detection() -> None: # Regression test for PROF-14568: safe_memcpy's fault recovery only works while - # we own the SIGSEGV handler. If another component (PyTorch/CUDA, or abseil - # pulled in by vLLM/gRPC) installs its own handler, the native sampler must - # detect that it no longer owns the handler so it can stay on / fall back to the - # syscall-based memory copy instead of crashing. + # we own BOTH the SIGSEGV and SIGBUS handlers (a copy fault can be delivered as + # either). If another component (PyTorch/CUDA, or abseil pulled in by vLLM/gRPC) + # installs its own handler for either signal, the native sampler must detect that + # it no longer owns the handler so it can stay on / fall back to the syscall-based + # memory copy instead of crashing. # # segv_handler_installed() is the primitive that drives that decision. Our # handler is installed at native import time when fast copy is enabled, so the @@ -281,4 +282,12 @@ def test_stack_profiler_foreign_segv_handler_detection() -> None: stack.reinstall_segv_handler() assert stack.segv_handler_installed() is True + # Losing only SIGBUS is just as unsafe as losing SIGSEGV: the predicate must + # report we no longer own the handler even though SIGSEGV is still ours. + signal.signal(signal.SIGBUS, signal.SIG_DFL) + assert stack.segv_handler_installed() is False + + stack.reinstall_segv_handler() + assert stack.segv_handler_installed() is True + print("OK") From 96da0aabbbd94c5e98f09a7c03b6902266ef256a Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Mon, 6 Jul 2026 16:27:14 -0400 Subject: [PATCH 05/28] fix lints --- .../datadog/profiling/stack/src/echion/danger.cc | 6 ++---- .../internal/datadog/profiling/stack/src/sampler.cpp | 10 ++-------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc b/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc index bdfbc129f35..320d4a387f5 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc +++ b/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc @@ -69,8 +69,7 @@ segv_handler(int signo, siginfo_t*, void*) // handler: the handler chain has cycled back to us. Restore the // default disposition and re-raise to guarantee the process // terminates instead of looping forever. - struct sigaction dfl - {}; + struct sigaction dfl{}; dfl.sa_handler = SIG_DFL; sigemptyset(&dfl.sa_mask); dfl.sa_flags = 0; @@ -102,8 +101,7 @@ init_segv_catcher() return -1; } - struct sigaction sa - {}; + struct sigaction sa{}; sa.sa_sigaction = segv_handler; sigemptyset(&sa.sa_mask); // SA_SIGINFO for 3-arg handler; SA_ONSTACK to run on alt stack; SA_NODEFER to avoid having to use savemask diff --git a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp index fe51f4adab1..836ee44453e 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp @@ -93,10 +93,7 @@ uint64_t get_thread_cpu_time_us() { #if defined(__linux__) - struct timespec ts - { - 0, 0 - }; + struct timespec ts{ 0, 0 }; if (clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts) != 0) { return 0; @@ -126,10 +123,7 @@ void Sampler::adapt_sampling_interval() { #if defined(__linux__) - struct timespec ts - { - 0, 0 - }; + struct timespec ts{ 0, 0 }; clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts); auto new_process_count = static_cast(ts.tv_sec * 1'000'000ULL + ts.tv_nsec / 1000); From d3b315bd40b27855303e31030c93202ff4e17406 Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Mon, 6 Jul 2026 16:41:05 -0400 Subject: [PATCH 06/28] docs(profiling): tie foreign-handler fix to PROF-14568 findings - Note the intentional call_once on init_segv_catcher (auto-fallback policy, not reinstall-and-chain) so the handler-chaining races from PROF-14568 are not reintroduced. - Expand the release note to mention SIGBUS, the stop-sampling fallback, and the complementary faulthandler integration. --- .../datadog/profiling/stack/src/sampler.cpp | 7 +++++++ ...r-foreign-handler-fallback-3d8f1b6a0e25c947.yaml | 13 ++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp index 836ee44453e..0eead54cc19 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp @@ -347,6 +347,13 @@ Sampler::sampling_thread(const uint64_t seq_num) // libraries (faulthandler, Django, FastAPI) can overwrite them afterwards. // Re-installing here ensures our handler is active when the sampling thread runs. // Only do this once to avoid overwriting g_old_segv with our own handler. + // + // Intentionally call_once: for foreign handlers we cannot wrap (PyTorch/CUDA, + // abseil via vLLM/gRPC - see PROF-14568), the sampler does NOT keep reinstalling + // to stay on top. Instead the loop below detects loss of ownership and falls + // back to the syscall copy. Re-arming here on every cycle would reintroduce the + // handler-chaining races that made safe_memcpy recovery unreliable in the first + // place, so leave this as a one-time install. static std::once_flag segv_handler_once; if (fast_copy_active) { std::call_once(segv_handler_once, init_segv_catcher); diff --git a/releasenotes/notes/profiling-stack-sampler-foreign-handler-fallback-3d8f1b6a0e25c947.yaml b/releasenotes/notes/profiling-stack-sampler-foreign-handler-fallback-3d8f1b6a0e25c947.yaml index 021d4cf957b..9235bc67736 100644 --- a/releasenotes/notes/profiling-stack-sampler-foreign-handler-fallback-3d8f1b6a0e25c947.yaml +++ b/releasenotes/notes/profiling-stack-sampler-foreign-handler-fallback-3d8f1b6a0e25c947.yaml @@ -3,9 +3,12 @@ fixes: - | profiling: The stack profiler no longer crashes in environments where another component (for example PyTorch/CUDA, or abseil pulled in by vLLM/gRPC) installs - its own ``SIGSEGV`` handler. The sampler now runs on a syscall-based memory copy - during a short startup warmup window, upgrades to the faster fault-recovery copy - only if it still owns the ``SIGSEGV`` handler afterwards, and permanently falls - back to the syscall-based copy if a foreign handler is installed later. This - removes the need for the ``_DD_PROFILING_STACK_FAST_COPY=0`` workaround in those + its own ``SIGSEGV``/``SIGBUS`` handler that the profiler cannot wrap. The sampler + now runs on a syscall-based memory copy during a short startup warmup window, + upgrades to the faster fault-recovery copy only if it still owns both fault + handlers afterwards, and permanently falls back to the syscall-based copy (or + stops sampling if no safe copy method is available) if a foreign handler is + installed later. This complements the existing ``faulthandler`` integration, + which keeps the profiler's handler on top of components it can intercept, and + removes the need for the ``_DD_PROFILING_STACK_FAST_COPY=0`` workaround in these environments. From cd95947307a2f8a5056cd2365a82f6fa418ff8e3 Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Tue, 7 Jul 2026 09:47:20 -0400 Subject: [PATCH 07/28] fix(profiling): do not stomp a foreign fault handler at sampler start Address review feedback on PR #18798: - sampling_thread only (re)arms our SIGSEGV/SIGBUS handlers via init_segv_catcher when we still own them. If a foreign component (abseil via vLLM/gRPC, torch/CUDA) took over the dispositions before the sampling thread started, we no longer overwrite it - which had made segv_handler_installed() falsely report ownership and defeated the warmup/fallback. We now leave the foreign handler authoritative and stay on the safe syscall copy. - Reduce the fast-copy stat test deadline from 45s to 30s for faster CI feedback (warmup window is 15s). --- .../datadog/profiling/stack/src/sampler.cpp | 30 +++++++++++-------- .../collector/test_copy_memory_stats.py | 6 ++-- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp index 0eead54cc19..76116bb5860 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp @@ -342,21 +342,27 @@ Sampler::sampling_thread(const uint64_t seq_num) // Mark thread as running thread_running.store(true); - // Re-install SIGSEGV/SIGBUS handlers here, after Python initialization. - // The handlers may have been installed during static init, but Python or - // libraries (faulthandler, Django, FastAPI) can overwrite them afterwards. - // Re-installing here ensures our handler is active when the sampling thread runs. - // Only do this once to avoid overwriting g_old_segv with our own handler. + // (Re)arm our SIGSEGV/SIGBUS fault-recovery handlers here, after Python + // initialization, but ONLY if we still own them. Our handler is installed by + // the library constructor at import; Python or libraries may overwrite it + // afterwards. Done once, on a std::once_flag. // - // Intentionally call_once: for foreign handlers we cannot wrap (PyTorch/CUDA, - // abseil via vLLM/gRPC - see PROF-14568), the sampler does NOT keep reinstalling - // to stay on top. Instead the loop below detects loss of ownership and falls - // back to the syscall copy. Re-arming here on every cycle would reintroduce the - // handler-chaining races that made safe_memcpy recovery unreliable in the first - // place, so leave this as a one-time install. + // Crucially, do NOT overwrite a foreign handler. If a component we cannot wrap + // (abseil via vLLM/gRPC, PyTorch/CUDA - see PROF-14568) already owns the + // dispositions when the sampling thread starts, stomping it here would (a) + // reintroduce the handler-chaining races that made safe_memcpy recovery + // unreliable, and (b) make segv_handler_installed() falsely report ownership, + // defeating the warmup + per-cycle fallback below. When a foreign handler is + // present we leave it authoritative and let the sampler stay on / fall back to + // the safe syscall-based copy instead. (init_segv_catcher is a near no-op when + // we already own both signals - it just re-ensures the alternate stack.) static std::once_flag segv_handler_once; if (fast_copy_active) { - std::call_once(segv_handler_once, init_segv_catcher); + std::call_once(segv_handler_once, []() { + if (segv_handler_installed()) { + init_segv_catcher(); + } + }); } using namespace std::chrono; diff --git a/tests/profiling/collector/test_copy_memory_stats.py b/tests/profiling/collector/test_copy_memory_stats.py index 2b4047a2913..00ac059d009 100644 --- a/tests/profiling/collector/test_copy_memory_stats.py +++ b/tests/profiling/collector/test_copy_memory_stats.py @@ -102,11 +102,11 @@ def test_fast_copy_memory_enabled(): output_filename = os.environ["DD_PROFILING_OUTPUT_PPROF"] + "." + str(os.getpid()) # Poll the emitted metadata until the sampler upgrades to safe_memcpy. The warmup - # window is a fixed internal constant (kStackFastCopyWarmupSeconds in sampler.cpp); - # allow a generous margin above it before giving up. + # window is a fixed internal constant (kStackFastCopyWarmupSeconds in sampler.cpp, + # 15s); allow a margin above it before giving up while keeping CI feedback fast. saw_warmup = False saw_upgrade = False - deadline = time.monotonic() + 45 + deadline = time.monotonic() + 30 while time.monotonic() < deadline and not saw_upgrade: time.sleep(1) for f in glob.glob(output_filename + ".*.internal_metadata.json"): From 93a7c2605a3b28d362b327f07f96f891308c8b2f Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Tue, 7 Jul 2026 09:47:34 -0400 Subject: [PATCH 08/28] chore(profiling): drop PROF-14568 repro script from PR The foreign-handler repro is kept for reference on the stacked branch vlad/prof-14568-repro-script; it is not part of the shipped fix. --- .../repro_prof_14568_foreign_handler.py | 104 ------------------ 1 file changed, 104 deletions(-) delete mode 100644 scripts/profiling/repro_prof_14568_foreign_handler.py diff --git a/scripts/profiling/repro_prof_14568_foreign_handler.py b/scripts/profiling/repro_prof_14568_foreign_handler.py deleted file mode 100644 index 5257f93860d..00000000000 --- a/scripts/profiling/repro_prof_14568_foreign_handler.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python3 -"""Repro for the stack-profiler SIGSEGV under a foreign SIGSEGV handler (PROF-14568, Layer 1). - -``safe_memcpy``'s fault recovery only works while the profiler owns the -``SIGSEGV`` disposition. Libraries such as PyTorch/CUDA install their own -``SIGSEGV`` handler after the profiler has started. Once that happens, a fault on -a stale read (which the profiler would normally recover from via ``siglongjmp``) -is delivered to the foreign handler instead, and the process crashes. - -This script reproduces the condition: - - 1. Start the profiler with fast copy enabled. - 2. Overwrite the ``SIGSEGV`` disposition with a foreign handler (here, the - default disposition, standing in for torch/CUDA installing their own). - 3. Churn deep Python stacks while the sampler runs. - -On ``main`` the sampler keeps using ``safe_memcpy`` even though it no longer owns -the handler, so a fault terminates the process. With the fix, the sampler detects -within ~2s that it lost handler ownership and falls back to ``process_vm_readv`` -(Linux) / ``mach_vm_read_overwrite`` (macOS), so the process survives. - -Usage: - _DD_PROFILING_STACK_FAST_COPY=1 python scripts/profiling/repro_prof_14568_foreign_handler.py - -Optionally import torch to install a *real* foreign handler instead of the -synthetic one: - _DD_PROFILING_STACK_FAST_COPY=1 python scripts/profiling/repro_prof_14568_foreign_handler.py --torch - -Exit code 0 means the process survived (fix working). A crash (negative exit / -SIGSEGV from the parent's perspective) reproduces the bug. -""" - -from __future__ import annotations - -import argparse -import os -import signal -import sys -import threading -import time - - -def _busy_worker(stop_evt: threading.Event) -> None: - def recurse(depth: int) -> int: - if depth <= 0: - return sum(len(str(i)) for i in range(16)) - return recurse(depth - 1) - - while not stop_evt.is_set(): - recurse(28) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--threads", type=int, default=8) - parser.add_argument("--run", type=float, default=6.0, help="seconds to sample under the foreign handler") - parser.add_argument("--torch", action="store_true", help="import torch to install a real foreign handler") - args = parser.parse_args() - - os.environ.setdefault("DD_PROFILING_STACK_V2_ENABLED", "true") - # The native fast-copy toggle is the private _DD_ variable that echion/vm.cc reads; - # force it on so the repro exercises the safe_memcpy + handler-fallback path even if - # the ambient environment disabled it. - os.environ["_DD_PROFILING_STACK_FAST_COPY"] = "1" - - from ddtrace.profiling.profiler import Profiler - - prof = Profiler() - prof.start() - - if args.torch: - try: - import torch # noqa: F401 - - print("torch imported; its SIGSEGV handler (if any) is now installed") - except Exception as exc: # pragma: no cover - torch is optional - print(f"could not import torch ({exc}); installing synthetic foreign handler instead") - signal.signal(signal.SIGSEGV, signal.SIG_DFL) - else: - # Stand in for a library that installs its own SIGSEGV handler after start. - signal.signal(signal.SIGSEGV, signal.SIG_DFL) - print("installed synthetic foreign SIGSEGV handler (SIG_DFL)") - - stop_evt = threading.Event() - workers = [threading.Thread(target=_busy_worker, args=(stop_evt,), daemon=True) for _ in range(args.threads)] - for w in workers: - w.start() - - # Sample for longer than the handler-ownership check cadence (~2s) so the - # fixed sampler observes the foreign handler and falls back. - print(f"sampling for {args.run}s under a foreign SIGSEGV handler...") - time.sleep(args.run) - - stop_evt.set() - for w in workers: - w.join(timeout=1.0) - prof.stop(flush=True) - - print("OK: process survived under a foreign SIGSEGV handler") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) From 6cf6b1826374936a8135c4c090b13d5782701fba Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Tue, 7 Jul 2026 10:02:04 -0400 Subject: [PATCH 09/28] style: apply clang-format 18.1.5 to stack sampler fault-handler code Fixes the prechecks style step, which pins clang-format 18.1.5 and splits `struct X{}` aggregate initializers onto their own line. The local pre-commit hook used a different clang-format version (Apple 17), so the skew slipped through locally. --- .../datadog/profiling/stack/src/echion/danger.cc | 6 ++++-- .../internal/datadog/profiling/stack/src/sampler.cpp | 10 ++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc b/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc index 320d4a387f5..bdfbc129f35 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc +++ b/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc @@ -69,7 +69,8 @@ segv_handler(int signo, siginfo_t*, void*) // handler: the handler chain has cycled back to us. Restore the // default disposition and re-raise to guarantee the process // terminates instead of looping forever. - struct sigaction dfl{}; + struct sigaction dfl + {}; dfl.sa_handler = SIG_DFL; sigemptyset(&dfl.sa_mask); dfl.sa_flags = 0; @@ -101,7 +102,8 @@ init_segv_catcher() return -1; } - struct sigaction sa{}; + struct sigaction sa + {}; sa.sa_sigaction = segv_handler; sigemptyset(&sa.sa_mask); // SA_SIGINFO for 3-arg handler; SA_ONSTACK to run on alt stack; SA_NODEFER to avoid having to use savemask diff --git a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp index 76116bb5860..fcf2708ba59 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp @@ -93,7 +93,10 @@ uint64_t get_thread_cpu_time_us() { #if defined(__linux__) - struct timespec ts{ 0, 0 }; + struct timespec ts + { + 0, 0 + }; if (clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts) != 0) { return 0; @@ -123,7 +126,10 @@ void Sampler::adapt_sampling_interval() { #if defined(__linux__) - struct timespec ts{ 0, 0 }; + struct timespec ts + { + 0, 0 + }; clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts); auto new_process_count = static_cast(ts.tv_sec * 1'000'000ULL + ts.tv_nsec / 1000); From fcdf250a3411c063366714864718389bab48a352 Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Tue, 7 Jul 2026 10:16:54 -0400 Subject: [PATCH 10/28] stubs(profiling): add segv_handler_installed to _stack.pyi Mirror the segv_handler_installed declaration into the native _stack extension stub so both stubs agree (addresses Copilot review on 18798). --- ddtrace/internal/datadog/profiling/stack/_stack.pyi | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ddtrace/internal/datadog/profiling/stack/_stack.pyi b/ddtrace/internal/datadog/profiling/stack/_stack.pyi index ee7fdee446e..8109a3764c6 100644 --- a/ddtrace/internal/datadog/profiling/stack/_stack.pyi +++ b/ddtrace/internal/datadog/profiling/stack/_stack.pyi @@ -23,6 +23,13 @@ def set_interval(new_interval: float) -> None: ... # Memory copy strategy def set_fast_copy(enabled: bool) -> None: ... def is_safe_copy_failed() -> bool: ... +def segv_handler_installed() -> bool: + """Return True if our handler is the installed disposition for SIGSEGV and SIGBUS. + + Used to detect when another component (e.g. PyTorch/CUDA) has taken over either + signal, in which case safe_memcpy's fault recovery would no longer work. + """ + ... # span <-> profile association def link_span( From d99a702bda81d86369dabc4dad8bf1722dfdf738 Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Tue, 7 Jul 2026 10:22:08 -0400 Subject: [PATCH 11/28] shorten release note --- ...foreign-handler-fallback-3d8f1b6a0e25c947.yaml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/releasenotes/notes/profiling-stack-sampler-foreign-handler-fallback-3d8f1b6a0e25c947.yaml b/releasenotes/notes/profiling-stack-sampler-foreign-handler-fallback-3d8f1b6a0e25c947.yaml index 9235bc67736..0078356a47e 100644 --- a/releasenotes/notes/profiling-stack-sampler-foreign-handler-fallback-3d8f1b6a0e25c947.yaml +++ b/releasenotes/notes/profiling-stack-sampler-foreign-handler-fallback-3d8f1b6a0e25c947.yaml @@ -1,14 +1,7 @@ --- fixes: - | - profiling: The stack profiler no longer crashes in environments where another - component (for example PyTorch/CUDA, or abseil pulled in by vLLM/gRPC) installs - its own ``SIGSEGV``/``SIGBUS`` handler that the profiler cannot wrap. The sampler - now runs on a syscall-based memory copy during a short startup warmup window, - upgrades to the faster fault-recovery copy only if it still owns both fault - handlers afterwards, and permanently falls back to the syscall-based copy (or - stops sampling if no safe copy method is available) if a foreign handler is - installed later. This complements the existing ``faulthandler`` integration, - which keeps the profiler's handler on top of components it can intercept, and - removes the need for the ``_DD_PROFILING_STACK_FAST_COPY=0`` workaround in these - environments. + profiling: crash fix for stack sampler in case where another + component installs its own ``SIGSEGV``/``SIGBUS`` handler that the profiler cannot wrap (e.g. CUDA, PyTorch, etc.). + The sampler upgrades to the faster fault-recovery copy if it still owns both fault handlers afterwards. + Otherwise, it permanently falls back to the syscall-based copy. From d5248bd4b2739f81a69cc1ac093dbb94cf0a8cdc Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Tue, 7 Jul 2026 11:30:57 -0400 Subject: [PATCH 12/28] test(profiling): observe fast-copy upgrade via native introspection Replace the metadata-file scraping in test_fast_copy_memory_enabled with an in-process check and a fast, deterministic warmup: - Add fast_copy_memory_active() native getter so tests can read the live copy mode directly instead of polling *.internal_metadata.json (which raced with mid-write files and depended on the upload cadence). Mirrors the existing segv_handler_installed()/is_safe_copy_failed() introspection. - Add a test-only _set_fast_copy_warmup_seconds() knob (underscore-prefixed, accessed via the _stack submodule) so the warmup->safe_memcpy upgrade can be observed in ~1s instead of waiting out the 15s production default. The default warmup is now a Sampler member (15s) rather than a file-local constant. The test now shrinks the warmup, confirms the sampler runs on the syscall copy during warmup, then confirms it upgrades to safe_memcpy, all via the getter. --- .../datadog/profiling/stack/__init__.pyi | 13 +++++ .../datadog/profiling/stack/_stack.pyi | 13 +++++ .../profiling/stack/include/sampler.hpp | 13 +++++ .../datadog/profiling/stack/src/sampler.cpp | 10 +--- .../datadog/profiling/stack/src/stack.cpp | 45 +++++++++++++++++ .../collector/test_copy_memory_stats.py | 50 ++++++++++--------- 6 files changed, 112 insertions(+), 32 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/stack/__init__.pyi b/ddtrace/internal/datadog/profiling/stack/__init__.pyi index 0f2a6abe094..a8899d95708 100644 --- a/ddtrace/internal/datadog/profiling/stack/__init__.pyi +++ b/ddtrace/internal/datadog/profiling/stack/__init__.pyi @@ -42,6 +42,19 @@ def set_interval(new_interval: float) -> None: ... # Memory copy strategy def set_fast_copy(enabled: bool) -> None: ... def is_safe_copy_failed() -> bool: ... +def fast_copy_memory_active() -> bool: + """Return True if the fast safe_memcpy copy path is currently active. + + The sampler starts on the safe syscall-based copy during the startup warmup + window (PROF-14568) and flips this to True only after it upgrades, so tests can + observe the warmup -> upgrade transition without scraping metadata files. + """ + ... + +# Note: _set_fast_copy_warmup_seconds is intentionally not re-exported here; it is a +# test-only, underscore-prefixed native symbol accessed via the _stack submodule +# (see _stack.pyi), since `from ._stack import *` skips underscored names. + def uninstall_segv_handler() -> None: ... def reinstall_segv_handler() -> None: """Reinstall SIGSEGV/SIGBUS handlers after another component overwrites them. diff --git a/ddtrace/internal/datadog/profiling/stack/_stack.pyi b/ddtrace/internal/datadog/profiling/stack/_stack.pyi index 8109a3764c6..d66a66dc16b 100644 --- a/ddtrace/internal/datadog/profiling/stack/_stack.pyi +++ b/ddtrace/internal/datadog/profiling/stack/_stack.pyi @@ -23,6 +23,19 @@ def set_interval(new_interval: float) -> None: ... # Memory copy strategy def set_fast_copy(enabled: bool) -> None: ... def is_safe_copy_failed() -> bool: ... +def fast_copy_memory_active() -> bool: + """Return True if the fast safe_memcpy copy path is currently active. + + The sampler starts on the safe syscall-based copy during the startup warmup + window (PROF-14568) and flips this to True only after it upgrades, so tests can + observe the warmup -> upgrade transition without scraping metadata files. + """ + ... + +def _set_fast_copy_warmup_seconds(seconds: float) -> None: + """Test-only: set the fast-copy startup warmup duration (must be set before start).""" + ... + def segv_handler_installed() -> bool: """Return True if our handler is the installed disposition for SIGSEGV and SIGBUS. diff --git a/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp b/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp index 0ef9ecb0489..fff35e29924 100644 --- a/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp +++ b/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp @@ -104,6 +104,15 @@ class Sampler // Rolling window duration in seconds; controls the ring buffer capacity. uint32_t p_stable_window_s = 600; + // Seconds the sampler runs on the safe syscall-based memory copy before + // upgrading to the faster safe_memcpy path. Running on the safe path during + // the import-heavy startup window means a fault there can never crash, even + // if another component (PyTorch/CUDA, or abseil pulled in by vLLM/gRPC) + // installs its own SIGSEGV handler during that window (PROF-14568). Read once + // by the sampling thread at startup; set (before start) only by tests to keep + // the warmup fast. + double fast_copy_warmup_seconds = 15.0; + // Tracks whether the sampler was running when prefork was called, // so that postfork_parent/restart_after_fork can restore it. bool was_running_at_fork_{ false }; @@ -164,6 +173,10 @@ class Sampler // Set the percentile (0–100) used to compute p_stable from the rolling window. void set_p_stable_percentile(double percentile) { p_stable_percentile_frac = percentile / 100.0; } + // Set the fast-copy startup warmup duration (seconds). Test-only knob; must be + // called before the sampler is started so the sampling thread reads the value. + void set_fast_copy_warmup_seconds(double value) { fast_copy_warmup_seconds = value; } + // Delegates to the StackRenderer to clear its caches after fork void postfork_child(); diff --git a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp index fcf2708ba59..df7c53bad83 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp @@ -25,13 +25,6 @@ using namespace Datadog; -// Number of seconds the stack sampler runs on the safe syscall-based memory copy -// before upgrading to the faster safe_memcpy path. Running on the safe path during -// the import-heavy startup window means a fault there can never crash, even if -// another component (PyTorch/CUDA, or abseil pulled in by vLLM/gRPC) installs its -// own SIGSEGV handler during that window (PROF-14568). -static constexpr long kStackFastCopyWarmupSeconds = 15; - // Helper class for spawning a std::thread with control over its default stack size #ifdef __linux__ #include @@ -397,7 +390,8 @@ Sampler::sampling_thread(const uint64_t seq_num) const bool fast_copy_warmup = fast_copy_desired && syscall_copy_available; bool fast_copy_upgraded = !fast_copy_warmup; bool handler_fallback_done = false; - const auto fast_copy_warmup_deadline = sample_time_prev + seconds(kStackFastCopyWarmupSeconds); + const auto fast_copy_warmup_deadline = + sample_time_prev + duration_cast(duration(fast_copy_warmup_seconds)); if (fast_copy_warmup) { // Drop to the safe syscall copy for the startup window. set_fast_copy_enabled(false); diff --git a/ddtrace/internal/datadog/profiling/stack/src/stack.cpp b/ddtrace/internal/datadog/profiling/stack/src/stack.cpp index 598b1d3b813..434db1f1b75 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/stack.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/stack.cpp @@ -907,6 +907,43 @@ stack_is_safe_copy_failed(PyObject* Py_UNUSED(self), PyObject* Py_UNUSED(args)) Py_RETURN_FALSE; } +static PyObject* +stack_fast_copy_memory_active(PyObject* Py_UNUSED(self), PyObject* Py_UNUSED(args)) +{ + // Introspection used by tests: reports whether the fast safe_memcpy path is the + // currently active copy method. The sampler starts on the safe syscall-based + // copy during the startup warmup window (PROF-14568) and flips this to true only + // after upgrading, so tests can observe the warmup -> upgrade transition without + // scraping the emitted pprof metadata files. + if (fast_copy_active) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +static PyObject* +stack_set_fast_copy_warmup_seconds(PyObject* Py_UNUSED(self), PyObject* args) +{ + // Test-only knob: shrink the fast-copy startup warmup window so the + // warmup -> safe_memcpy upgrade can be observed quickly instead of waiting out + // the 15s production default. Must be set before the sampler starts. + double seconds_value = 0.0; + + if (!PyArg_ParseTuple(args, "d", &seconds_value)) { + return NULL; + } + + if (Sampler::get().is_running()) { + PyErr_SetString(PyExc_RuntimeError, + "_set_fast_copy_warmup_seconds must be called before the sampler is started"); + return NULL; + } + + Sampler::get().set_fast_copy_warmup_seconds(seconds_value); + + Py_RETURN_NONE; +} + static PyMethodDef stack_methods[] = { { "start", reinterpret_cast(stack_start), METH_VARARGS | METH_KEYWORDS, "Start the sampler" }, { "stop", stack_stop, METH_VARARGS, "Stop the sampler" }, @@ -969,6 +1006,14 @@ static PyMethodDef stack_methods[] = { stack_is_safe_copy_failed, METH_NOARGS, "Check if all safe copy methods failed to initialize" }, + { "fast_copy_memory_active", + stack_fast_copy_memory_active, + METH_NOARGS, + "Return True if the fast safe_memcpy copy path is currently active" }, + { "_set_fast_copy_warmup_seconds", + stack_set_fast_copy_warmup_seconds, + METH_VARARGS, + "Test-only: set the fast-copy startup warmup duration in seconds (before start)" }, { "uninstall_segv_handler", stack_uninstall_segv_handler, METH_NOARGS, diff --git a/tests/profiling/collector/test_copy_memory_stats.py b/tests/profiling/collector/test_copy_memory_stats.py index 00ac059d009..b8f93d61614 100644 --- a/tests/profiling/collector/test_copy_memory_stats.py +++ b/tests/profiling/collector/test_copy_memory_stats.py @@ -85,43 +85,45 @@ def test_fast_copy_memory_enabled(): It first runs on the safe syscall-based copy for a startup warmup window (PROF-14568) and upgrades to safe_memcpy afterwards once it confirms it still - owns the SIGSEGV handler, so the emitted metadata reports - fast_copy_memory_enabled=False during warmup and True after the upgrade. + owns the SIGSEGV handler. We read the live copy mode via the native + fast_copy_memory_active() introspection instead of scraping metadata files, and + shrink the warmup via the test-only _set_fast_copy_warmup_seconds() knob so the + warmup -> upgrade transition can be observed quickly. """ - import glob - import json - import os import time + # _set_fast_copy_warmup_seconds is underscore-prefixed, so it lives only on the + # native _stack submodule (`from ._stack import *` skips underscored names). + from ddtrace.internal.datadog.profiling.stack import _stack from ddtrace.profiling import profiler from ddtrace.trace import tracer + # Shrink the 15s production warmup so the upgrade happens quickly. Must be set + # before the sampler thread starts. + _stack._set_fast_copy_warmup_seconds(1.0) + p = profiler.Profiler(tracer=tracer) p.start() - output_filename = os.environ["DD_PROFILING_OUTPUT_PPROF"] + "." + str(os.getpid()) - - # Poll the emitted metadata until the sampler upgrades to safe_memcpy. The warmup - # window is a fixed internal constant (kStackFastCopyWarmupSeconds in sampler.cpp, - # 15s); allow a margin above it before giving up while keeping CI feedback fast. + # First confirm the sampler runs on the safe syscall copy during warmup + # (fast_copy_memory_active() is False), then that it upgrades to safe_memcpy + # (True). We only accept the upgrade after seeing the warmup so the brief + # constructor-time True (before the sampling thread drops to the warmup copy) + # can't be mistaken for the upgrade. saw_warmup = False saw_upgrade = False - deadline = time.monotonic() + 30 - while time.monotonic() < deadline and not saw_upgrade: - time.sleep(1) - for f in glob.glob(output_filename + ".*.internal_metadata.json"): - try: - with open(f) as fp: - enabled = json.load(fp).get("fast_copy_memory_enabled") - except (json.JSONDecodeError, OSError): - # The most recent file may still be mid-write; skip and retry. - continue - if enabled is False: + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + active = _stack.fast_copy_memory_active() + if not saw_warmup: + if active is False: saw_warmup = True - elif enabled is True: - saw_upgrade = True + elif active is True: + saw_upgrade = True + break + time.sleep(0.05) p.stop() assert saw_warmup, "Expected the sampler to run on the syscall copy during the warmup window" - assert saw_upgrade, "Expected the sampler to upgrade to safe_memcpy (fast_copy_memory_enabled=True) after warmup" + assert saw_upgrade, "Expected the sampler to upgrade to safe_memcpy (fast_copy_memory_active()=True) after warmup" From f70f110c758238c4b875fa976e2e5ce7502d3bfe Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Tue, 7 Jul 2026 11:53:55 -0400 Subject: [PATCH 13/28] shorten comments and improve tests --- .../datadog/profiling/stack/__init__.pyi | 19 ++----- .../datadog/profiling/stack/_stack.pyi | 21 ++------ .../profiling/stack/echion/echion/danger.h | 7 +-- .../profiling/stack/include/sampler.hpp | 10 +--- .../profiling/stack/src/echion/danger.cc | 7 +-- .../datadog/profiling/stack/src/sampler.cpp | 51 +++++-------------- .../datadog/profiling/stack/src/stack.cpp | 16 ++---- .../collector/test_copy_memory_stats.py | 36 ++++--------- tests/profiling/test_main.py | 14 ++--- 9 files changed, 42 insertions(+), 139 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/stack/__init__.pyi b/ddtrace/internal/datadog/profiling/stack/__init__.pyi index a8899d95708..2cdfb9591ef 100644 --- a/ddtrace/internal/datadog/profiling/stack/__init__.pyi +++ b/ddtrace/internal/datadog/profiling/stack/__init__.pyi @@ -42,18 +42,9 @@ def set_interval(new_interval: float) -> None: ... # Memory copy strategy def set_fast_copy(enabled: bool) -> None: ... def is_safe_copy_failed() -> bool: ... -def fast_copy_memory_active() -> bool: - """Return True if the fast safe_memcpy copy path is currently active. +def fast_copy_memory_active() -> bool: ... # test introspection: is safe_memcpy active? - The sampler starts on the safe syscall-based copy during the startup warmup - window (PROF-14568) and flips this to True only after it upgrades, so tests can - observe the warmup -> upgrade transition without scraping metadata files. - """ - ... - -# Note: _set_fast_copy_warmup_seconds is intentionally not re-exported here; it is a -# test-only, underscore-prefixed native symbol accessed via the _stack submodule -# (see _stack.pyi), since `from ._stack import *` skips underscored names. +# _set_fast_copy_warmup_seconds is test-only; accessed via _stack (import * skips it). def uninstall_segv_handler() -> None: ... def reinstall_segv_handler() -> None: @@ -66,11 +57,7 @@ def reinstall_segv_handler() -> None: ... def segv_handler_installed() -> bool: - """Return True if our handler is the installed disposition for SIGSEGV and SIGBUS. - - Used to detect when another component (e.g. PyTorch/CUDA) has taken over either - signal, in which case safe_memcpy's fault recovery would no longer work. - """ + """Return True if our handler is the installed disposition for SIGSEGV and SIGBUS.""" ... # Pause/resume sampling diff --git a/ddtrace/internal/datadog/profiling/stack/_stack.pyi b/ddtrace/internal/datadog/profiling/stack/_stack.pyi index d66a66dc16b..513fac05d66 100644 --- a/ddtrace/internal/datadog/profiling/stack/_stack.pyi +++ b/ddtrace/internal/datadog/profiling/stack/_stack.pyi @@ -23,25 +23,10 @@ def set_interval(new_interval: float) -> None: ... # Memory copy strategy def set_fast_copy(enabled: bool) -> None: ... def is_safe_copy_failed() -> bool: ... -def fast_copy_memory_active() -> bool: - """Return True if the fast safe_memcpy copy path is currently active. - - The sampler starts on the safe syscall-based copy during the startup warmup - window (PROF-14568) and flips this to True only after it upgrades, so tests can - observe the warmup -> upgrade transition without scraping metadata files. - """ - ... - -def _set_fast_copy_warmup_seconds(seconds: float) -> None: - """Test-only: set the fast-copy startup warmup duration (must be set before start).""" - ... - +def fast_copy_memory_active() -> bool: ... # test introspection: is safe_memcpy active? +def _set_fast_copy_warmup_seconds(seconds: float) -> None: ... # test-only; before start def segv_handler_installed() -> bool: - """Return True if our handler is the installed disposition for SIGSEGV and SIGBUS. - - Used to detect when another component (e.g. PyTorch/CUDA) has taken over either - signal, in which case safe_memcpy's fault recovery would no longer work. - """ + """Return True if our handler is the installed disposition for SIGSEGV and SIGBUS.""" ... # span <-> profile association diff --git a/ddtrace/internal/datadog/profiling/stack/echion/echion/danger.h b/ddtrace/internal/datadog/profiling/stack/echion/echion/danger.h index fedd8e2e419..7b21f2f705e 100644 --- a/ddtrace/internal/datadog/profiling/stack/echion/echion/danger.h +++ b/ddtrace/internal/datadog/profiling/stack/echion/echion/danger.h @@ -27,12 +27,7 @@ init_segv_catcher(); void uninstall_segv_handler(); -// Returns true only when our fault-recovery handler (segv_handler) is the -// currently installed disposition for BOTH SIGSEGV and SIGBUS (init_segv_catcher -// installs it, with SA_SIGINFO, for both). Used by the sampler to detect when -// another component (e.g. PyTorch/CUDA) has taken over either signal, in which -// case safe_memcpy's fault recovery would no longer work and we must fall back to -// a syscall-based copy. Returns false on any sigaction error. +// Returns true only if our segv_handler owns both SIGSEGV and SIGBUS; false on any error. bool segv_handler_installed(); diff --git a/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp b/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp index fff35e29924..f6b0e306be0 100644 --- a/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp +++ b/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp @@ -104,13 +104,7 @@ class Sampler // Rolling window duration in seconds; controls the ring buffer capacity. uint32_t p_stable_window_s = 600; - // Seconds the sampler runs on the safe syscall-based memory copy before - // upgrading to the faster safe_memcpy path. Running on the safe path during - // the import-heavy startup window means a fault there can never crash, even - // if another component (PyTorch/CUDA, or abseil pulled in by vLLM/gRPC) - // installs its own SIGSEGV handler during that window (PROF-14568). Read once - // by the sampling thread at startup; set (before start) only by tests to keep - // the warmup fast. + // Fast-copy startup warmup in seconds. double fast_copy_warmup_seconds = 15.0; // Tracks whether the sampler was running when prefork was called, @@ -173,8 +167,6 @@ class Sampler // Set the percentile (0–100) used to compute p_stable from the rolling window. void set_p_stable_percentile(double percentile) { p_stable_percentile_frac = percentile / 100.0; } - // Set the fast-copy startup warmup duration (seconds). Test-only knob; must be - // called before the sampler is started so the sampling thread reads the value. void set_fast_copy_warmup_seconds(double value) { fast_copy_warmup_seconds = value; } // Delegates to the StackRenderer to clear its caches after fork diff --git a/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc b/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc index bdfbc129f35..121d68b06e8 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc +++ b/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc @@ -143,11 +143,8 @@ init_segv_catcher() bool segv_handler_installed() { - // safe_memcpy's fault recovery relies on owning BOTH SIGSEGV and SIGBUS: - // init_segv_catcher installs our 3-arg (SA_SIGINFO) handler for both, and a - // copy fault can be delivered as either. If a foreign component has taken over - // either signal - or replaced our handler with a 1-arg (non SA_SIGINFO) one - - // recovery is no longer guaranteed, so report that our handler is not installed. + // Recovery needs our 3-arg (SA_SIGINFO) handler to own BOTH SIGSEGV and SIGBUS + // (a copy fault can arrive as either); anything else means we can't recover. const int signals[] = { SIGSEGV, SIGBUS }; for (int signo : signals) { struct sigaction current; diff --git a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp index df7c53bad83..2ae2391695c 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp @@ -341,20 +341,10 @@ Sampler::sampling_thread(const uint64_t seq_num) // Mark thread as running thread_running.store(true); - // (Re)arm our SIGSEGV/SIGBUS fault-recovery handlers here, after Python - // initialization, but ONLY if we still own them. Our handler is installed by - // the library constructor at import; Python or libraries may overwrite it - // afterwards. Done once, on a std::once_flag. - // - // Crucially, do NOT overwrite a foreign handler. If a component we cannot wrap - // (abseil via vLLM/gRPC, PyTorch/CUDA - see PROF-14568) already owns the - // dispositions when the sampling thread starts, stomping it here would (a) - // reintroduce the handler-chaining races that made safe_memcpy recovery - // unreliable, and (b) make segv_handler_installed() falsely report ownership, - // defeating the warmup + per-cycle fallback below. When a foreign handler is - // present we leave it authoritative and let the sampler stay on / fall back to - // the safe syscall-based copy instead. (init_segv_catcher is a near no-op when - // we already own both signals - it just re-ensures the alternate stack.) + // (Re)arm our SIGSEGV/SIGBUS handlers once, but ONLY if we still own them. If a + // foreign component we can't wrap (abseil via vLLM/gRPC, PyTorch/CUDA) already owns + // them, overwriting it would cause handler-chaining races and make + // segv_handler_installed() report incorrectly, so leave it authoritative. static std::once_flag segv_handler_once; if (fast_copy_active) { std::call_once(segv_handler_once, []() { @@ -368,25 +358,15 @@ Sampler::sampling_thread(const uint64_t seq_num) auto sample_time_prev = steady_clock::now(); auto interval_adjust_time_prev = sample_time_prev; - // Foreign SIGSEGV/SIGBUS handler handling for safe_memcpy (PROF-14568). - // - // safe_memcpy's fault recovery only works while we own the SIGSEGV and SIGBUS - // handlers. Other components (PyTorch/CUDA, or abseil pulled in by vLLM/gRPC) - // install their own handler, most often during the import-heavy startup. We do: - // 1. run on the safe syscall-based copy for a warmup window, so a fault during - // startup can never crash regardless of who owns the handler, then upgrade - // to safe_memcpy only if we still own the handler; and - // 2. keep checking ownership every cycle afterwards, permanently falling back - // to the syscall copy if a handler is installed later (e.g. lazy CUDA init - // on first GPU use). + // safe_memcpy recovery needs us to own both handlers (PROF-14568): warm up on the + // syscall copy, upgrade only if we still own them, then re-check and fall back. const bool fast_copy_desired = fast_copy_active; #if defined PL_LINUX const bool syscall_copy_available = process_vm_readv_available; #else const bool syscall_copy_available = true; // mach_vm_read_overwrite is always available #endif - // Only warm up when fast copy is wanted and a safe path exists to run on in the - // meantime; otherwise leave the active copy method untouched. + // Warm up only when fast copy is wanted and a safe fallback path exists to run on. const bool fast_copy_warmup = fast_copy_desired && syscall_copy_available; bool fast_copy_upgraded = !fast_copy_warmup; bool handler_fallback_done = false; @@ -421,9 +401,8 @@ Sampler::sampling_thread(const uint64_t seq_num) auto wall_time_us = duration_cast(sample_time_now - sample_time_prev).count(); sample_time_prev = sample_time_now; - // Foreign SIGSEGV/SIGBUS handler handling (see the notes before the loop). - // The ownership check is one cheap sigaction read per signal; faulthandler's - // transient swaps are not a concern because the sampler is paused around them. + // Foreign handler handling (see notes before the loop); faulthandler's + // transient swaps are safe since the sampler is paused around them. if (fast_copy_desired) { if (!fast_copy_upgraded) { // Warmup window: still on the safe syscall copy. Once it elapses, @@ -443,18 +422,16 @@ Sampler::sampling_thread(const uint64_t seq_num) } } } else if (fast_copy_active && !handler_fallback_done && !segv_handler_installed()) { - // A component took over a handler after we upgraded to safe_memcpy; - // fall back permanently. A false positive only costs the slower copy - // path, so we do not debounce. + // A handler was taken over after upgrading; fall back permanently + // (no debounce - a false positive just costs the slower copy path). handler_fallback_done = true; std::cerr << "ddtrace stack profiler: SIGSEGV/SIGBUS handler was taken over by another " "component; falling back to syscall-based memory copy to avoid crashing." << std::endl; if (!set_fast_copy_enabled(false)) { - // No safe copy method is available to fall back to (e.g. - // process_vm_readv is blocked on this host), so safe_memcpy is - // still active. Continuing to read memory while a foreign handler - // owns the fault signals would crash, so stop sampling instead. + // No safe fallback available (e.g. process_vm_readv blocked), so + // safe_memcpy is still active; reading under a foreign handler would + // crash - stop sampling instead. std::cerr << "ddtrace stack profiler: no safe memory-copy fallback available; " "stopping stack sampling to avoid crashing." << std::endl; diff --git a/ddtrace/internal/datadog/profiling/stack/src/stack.cpp b/ddtrace/internal/datadog/profiling/stack/src/stack.cpp index 434db1f1b75..6a39e65c13e 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/stack.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/stack.cpp @@ -885,10 +885,8 @@ stack_reinstall_segv_handler(PyObject* Py_UNUSED(self), PyObject* Py_UNUSED(args static PyObject* stack_segv_handler_installed(PyObject* Py_UNUSED(self), PyObject* Py_UNUSED(args)) { - // Introspection used by tests: reports whether our fault-recovery handler is - // the currently installed disposition for both SIGSEGV and SIGBUS. This is the - // same primitive the sampler uses to decide whether safe_memcpy's fault - // recovery is still trustworthy. + // Test introspection: do we still own both SIGSEGV and SIGBUS? (same primitive + // the sampler uses to decide whether safe_memcpy recovery is trustworthy) if (segv_handler_installed()) { Py_RETURN_TRUE; } @@ -910,11 +908,7 @@ stack_is_safe_copy_failed(PyObject* Py_UNUSED(self), PyObject* Py_UNUSED(args)) static PyObject* stack_fast_copy_memory_active(PyObject* Py_UNUSED(self), PyObject* Py_UNUSED(args)) { - // Introspection used by tests: reports whether the fast safe_memcpy path is the - // currently active copy method. The sampler starts on the safe syscall-based - // copy during the startup warmup window (PROF-14568) and flips this to true only - // after upgrading, so tests can observe the warmup -> upgrade transition without - // scraping the emitted pprof metadata files. + // Test introspection: is the fast safe_memcpy path active? (PROF-14568) if (fast_copy_active) { Py_RETURN_TRUE; } @@ -924,9 +918,7 @@ stack_fast_copy_memory_active(PyObject* Py_UNUSED(self), PyObject* Py_UNUSED(arg static PyObject* stack_set_fast_copy_warmup_seconds(PyObject* Py_UNUSED(self), PyObject* args) { - // Test-only knob: shrink the fast-copy startup warmup window so the - // warmup -> safe_memcpy upgrade can be observed quickly instead of waiting out - // the 15s production default. Must be set before the sampler starts. + // Test-only: shrink the fast-copy warmup; must be set before the sampler starts. double seconds_value = 0.0; if (!PyArg_ParseTuple(args, "d", &seconds_value)) { diff --git a/tests/profiling/collector/test_copy_memory_stats.py b/tests/profiling/collector/test_copy_memory_stats.py index b8f93d61614..53c76e20c4d 100644 --- a/tests/profiling/collector/test_copy_memory_stats.py +++ b/tests/profiling/collector/test_copy_memory_stats.py @@ -80,41 +80,27 @@ def test_fast_copy_memory_disabled(): ), err=None, ) -def test_fast_copy_memory_enabled(): - """With _DD_PROFILING_STACK_FAST_COPY=1 the sampler eventually uses safe_memcpy. - - It first runs on the safe syscall-based copy for a startup warmup window - (PROF-14568) and upgrades to safe_memcpy afterwards once it confirms it still - owns the SIGSEGV handler. We read the live copy mode via the native - fast_copy_memory_active() introspection instead of scraping metadata files, and - shrink the warmup via the test-only _set_fast_copy_warmup_seconds() knob so the - warmup -> upgrade transition can be observed quickly. - """ +def test_fast_copy_memory_enabled() -> None: + """Sampler runs on the syscall copy during warmup, then upgrades to safe_memcpy (PROF-14568).""" import time - # _set_fast_copy_warmup_seconds is underscore-prefixed, so it lives only on the - # native _stack submodule (`from ._stack import *` skips underscored names). + # Underscore-prefixed, so only on the _stack submodule (`import *` skips it). from ddtrace.internal.datadog.profiling.stack import _stack from ddtrace.profiling import profiler from ddtrace.trace import tracer - # Shrink the 15s production warmup so the upgrade happens quickly. Must be set - # before the sampler thread starts. _stack._set_fast_copy_warmup_seconds(1.0) - p = profiler.Profiler(tracer=tracer) + p: profiler.Profiler = profiler.Profiler(tracer=tracer) p.start() - # First confirm the sampler runs on the safe syscall copy during warmup - # (fast_copy_memory_active() is False), then that it upgrades to safe_memcpy - # (True). We only accept the upgrade after seeing the warmup so the brief - # constructor-time True (before the sampling thread drops to the warmup copy) - # can't be mistaken for the upgrade. - saw_warmup = False - saw_upgrade = False - deadline = time.monotonic() + 10 + # Require warmup (False) before accepting the upgrade (True), so the brief + # constructor-time True isn't mistaken for it. + saw_warmup: bool = False + saw_upgrade: bool = False + deadline: float = time.monotonic() + 10 while time.monotonic() < deadline: - active = _stack.fast_copy_memory_active() + active: bool = _stack.fast_copy_memory_active() if not saw_warmup: if active is False: saw_warmup = True @@ -126,4 +112,4 @@ def test_fast_copy_memory_enabled(): p.stop() assert saw_warmup, "Expected the sampler to run on the syscall copy during the warmup window" - assert saw_upgrade, "Expected the sampler to upgrade to safe_memcpy (fast_copy_memory_active()=True) after warmup" + assert saw_upgrade, "Expected the sampler to upgrade to safe_memcpy after warmup" diff --git a/tests/profiling/test_main.py b/tests/profiling/test_main.py index 2d05db85182..499aacd08ec 100644 --- a/tests/profiling/test_main.py +++ b/tests/profiling/test_main.py @@ -256,17 +256,9 @@ def test_profiler_start_up_with_module_clean_up_in_protobuf_app() -> None: err=None, ) def test_stack_profiler_foreign_segv_handler_detection() -> None: - # Regression test for PROF-14568: safe_memcpy's fault recovery only works while - # we own BOTH the SIGSEGV and SIGBUS handlers (a copy fault can be delivered as - # either). If another component (PyTorch/CUDA, or abseil pulled in by vLLM/gRPC) - # installs its own handler for either signal, the native sampler must detect that - # it no longer owns the handler so it can stay on / fall back to the syscall-based - # memory copy instead of crashing. - # - # segv_handler_installed() is the primitive that drives that decision. Our - # handler is installed at native import time when fast copy is enabled, so the - # predicate can be verified deterministically here without waiting on the - # sampler's startup warmup window. + # Regression test for PROF-14568: safe_memcpy's fault recovery needs us to own + # BOTH SIGSEGV and SIGBUS. segv_handler_installed() drives the sampler's + # detect-and-fallback decision; assert it flips when either signal is taken over. import signal from ddtrace.internal.datadog.profiling import stack From 8caced0b09797c5b45b2882e829d9282b979bd0d Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Tue, 7 Jul 2026 15:22:34 -0400 Subject: [PATCH 14/28] fix(profiling): reorder Sampler fields to satisfy clang-tidy padding check The clang-tidy profiling job fails on clang-analyzer-optin.performance.Padding (warnings-as-errors): class Datadog::Sampler had 35 padding bytes where 3 is optimal after adding fast_copy_warmup_seconds. Reorder the data members into the alignment-descending order clang-tidy reports as optimal (pointers/atomics/8-byte scalars, then vectors, mutexes, condvars, then 4-byte and bool fields). Layout only; no behavior change (the constructor initializes only echion, which stays first, so no -Wreorder). --- .../profiling/stack/include/sampler.hpp | 90 ++++++++++--------- 1 file changed, 49 insertions(+), 41 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp b/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp index f6b0e306be0..b86d34f365a 100644 --- a/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp +++ b/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp @@ -31,6 +31,10 @@ class Sampler // This class manages the initialization of echion as well as the sampling thread. // The underlying echion instance it manages keeps much of its state globally, so this class is a singleton in order // to keep it aligned with the echion state. + // + // NOTE: fields below are ordered largest-to-smallest alignment to minimize padding + // (enforced by clang-analyzer-optin.performance.Padding). Keep new members grouped + // by size rather than by logical role. std::unique_ptr echion; // The sampling interval is atomic because it needs to be safely propagated to the sampling thread @@ -41,13 +45,52 @@ class Sampler // stopped or started in a straightforward manner without finer-grained control (locks) std::atomic thread_seq_num{ 0 }; + // Internal perf counters + uint64_t process_count = 0; + uint64_t sampler_thread_count = 0; + + double target_overhead = g_target_overhead; + microsecond_t max_sampling_period_us = g_max_sampling_period_us; + std::minstd_rand rng{ std::random_device{}() }; + + // Ring-buffer head for the p_stable process_delta window (see process_delta_window below). + size_t process_delta_window_head = 0; + + // Baseline CPU budget (us per adapt window) corresponding to an absolute floor overhead. + // Derived from baseline_core_pct at configuration time; keeps the profiler sampling even + // when app CPU is near 0. + double baseline_cpu_us_per_adapt_window = 0.0; + + // Percentile (0..1) used for p_stable; configurable, default p95. + double p_stable_percentile_frac = 0.95; + + // Fast-copy startup warmup in seconds. + double fast_copy_warmup_seconds = 15.0; + + std::vector thread_candidates; + + // Rolling window for p_stable: ring buffer of process_delta values (us CPU per adapt window). + // p_stable is the p-th percentile of this buffer, giving a stable estimate of app CPU usage + // that doesn't collapse to zero during brief idle periods. + std::vector process_delta_window; + // Thread exit synchronization - allows stop() to wait for the sampling thread to exit. // The mutex + condition variable pair is used to avoid the "lost wake-up" race condition // where stop() could miss the notification and hang forever (or until timeout). - std::atomic thread_running{ false }; std::mutex thread_exit_mutex; + // Pause synchronization — allows the faulthandler wrapper to temporarily + // suspend sampling so signal handlers can be safely swapped without racing + // with in-flight safe_memcpy calls. + std::mutex pause_mutex_; std::condition_variable thread_exit_cv; + std::condition_variable pause_cv_; + + unsigned int max_threads_per_sample = g_default_max_threads_per_sample; + + // Rolling window duration in seconds; controls the ring buffer capacity. + uint32_t p_stable_window_s = 600; + std::atomic thread_running{ false }; // Whether the sampler is currently active. Unlike thread_running (which tracks // the thread's actual lifecycle) this is set synchronously in start/stop, so // it is not subject to the sampling-thread startup race. @@ -56,14 +99,13 @@ class Sampler // prefork reads this to decide whether to restart the sampler after fork, // ensuring a sampler that stopped due to an error is not silently restarted. std::atomic sampler_active_{ false }; - - // Pause synchronization — allows the faulthandler wrapper to temporarily - // suspend sampling so signal handlers can be safely swapped without racing - // with in-flight safe_memcpy calls. std::atomic pause_requested_{ false }; std::atomic paused_{ false }; - std::mutex pause_mutex_; - std::condition_variable pause_cv_; + bool do_adaptive_sampling = true; + + // Tracks whether the sampler was running when prefork was called, + // so that postfork_parent/restart_after_fork can restore it. + bool was_running_at_fork_{ false }; // This is a singleton, so no public constructor Sampler(); @@ -71,46 +113,12 @@ class Sampler // One-time setup of echion void one_time_setup(); - // Internal perf counters - uint64_t process_count = 0; - uint64_t sampler_thread_count = 0; - - bool do_adaptive_sampling = true; - double target_overhead = g_target_overhead; - microsecond_t max_sampling_period_us = g_max_sampling_period_us; - unsigned int max_threads_per_sample = g_default_max_threads_per_sample; - std::minstd_rand rng{ std::random_device{}() }; - std::vector thread_candidates; void adapt_sampling_interval(); // Captures one sampling cycle across all threads (or a reservoir-sampled subset thereof // when max_threads_per_sample is set). void capture_samples(microsecond_t wall_time_us); - // Rolling window for p_stable: ring buffer of process_delta values (us CPU per adapt window). - // p_stable is the p-th percentile of this buffer, giving a stable estimate of app CPU usage - // that doesn't collapse to zero during brief idle periods. - std::vector process_delta_window; - size_t process_delta_window_head = 0; - - // Baseline CPU budget (us per adapt window) corresponding to an absolute floor overhead. - // Derived from baseline_core_pct at configuration time; keeps the profiler sampling even - // when app CPU is near 0. - double baseline_cpu_us_per_adapt_window = 0.0; - - // Percentile (0..1) used for p_stable; configurable, default p95. - double p_stable_percentile_frac = 0.95; - - // Rolling window duration in seconds; controls the ring buffer capacity. - uint32_t p_stable_window_s = 600; - - // Fast-copy startup warmup in seconds. - double fast_copy_warmup_seconds = 15.0; - - // Tracks whether the sampler was running when prefork was called, - // so that postfork_parent/restart_after_fork can restore it. - bool was_running_at_fork_{ false }; - void atfork_child(); friend void stack_atfork_prepare(); friend void stack_atfork_parent(); From 263be8752db254138476617c02162911724ce66c Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Tue, 7 Jul 2026 16:29:32 -0400 Subject: [PATCH 15/28] fix(profiling): validate warmup setter input and skip test when stack unavailable Address review feedback on PR #18798: - _set_fast_copy_warmup_seconds now rejects non-finite (NaN/inf) and negative durations with a ValueError, instead of letting a nonsensical warmup deadline through this Python-exposed entry point. - test_stack_profiler_foreign_segv_handler_detection now skips when the native stack extension is unavailable (stack.is_available is False), avoiding an AttributeError on builds/platforms without it. --- ddtrace/internal/datadog/profiling/stack/src/stack.cpp | 7 +++++++ tests/profiling/test_main.py | 2 ++ 2 files changed, 9 insertions(+) diff --git a/ddtrace/internal/datadog/profiling/stack/src/stack.cpp b/ddtrace/internal/datadog/profiling/stack/src/stack.cpp index 6a39e65c13e..604599d8a2e 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/stack.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/stack.cpp @@ -8,6 +8,7 @@ #include "echion/echion_sampler.h" #include "echion/vm.h" +#include #include #include @@ -925,6 +926,12 @@ stack_set_fast_copy_warmup_seconds(PyObject* Py_UNUSED(self), PyObject* args) return NULL; } + if (!std::isfinite(seconds_value) || seconds_value < 0.0) { + PyErr_SetString(PyExc_ValueError, + "_set_fast_copy_warmup_seconds requires a finite, non-negative number of seconds"); + return NULL; + } + if (Sampler::get().is_running()) { PyErr_SetString(PyExc_RuntimeError, "_set_fast_copy_warmup_seconds must be called before the sampler is started"); diff --git a/tests/profiling/test_main.py b/tests/profiling/test_main.py index 499aacd08ec..ea78158fe29 100644 --- a/tests/profiling/test_main.py +++ b/tests/profiling/test_main.py @@ -6,6 +6,7 @@ import pytest +from ddtrace.internal.datadog.profiling import stack as _stack_ext from tests.profiling.collector import lock_utils from tests.profiling.collector import pprof_utils from tests.utils import call_program @@ -250,6 +251,7 @@ def test_profiler_start_up_with_module_clean_up_in_protobuf_app() -> None: @pytest.mark.skipif(sys.platform == "win32", reason="stack v2 profiler is not available on Windows") +@pytest.mark.skipif(not _stack_ext.is_available, reason="stack v2 native extension not available") @pytest.mark.subprocess( env=dict(_DD_PROFILING_STACK_FAST_COPY="1"), out="OK\n", From a73661d1ee3d12c521b6262c0f233c853cc9587a Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Wed, 8 Jul 2026 12:24:41 -0400 Subject: [PATCH 16/28] Thomas and bot comments addressed --- .../datadog/profiling/stack/__init__.pyi | 6 +- .../datadog/profiling/stack/_stack.pyi | 6 +- .../profiling/stack/echion/echion/danger.h | 2 +- .../profiling/stack/include/sampler.hpp | 94 +++++++++---------- .../datadog/profiling/stack/src/sampler.cpp | 7 +- 5 files changed, 59 insertions(+), 56 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/stack/__init__.pyi b/ddtrace/internal/datadog/profiling/stack/__init__.pyi index 2cdfb9591ef..b41e62760f2 100644 --- a/ddtrace/internal/datadog/profiling/stack/__init__.pyi +++ b/ddtrace/internal/datadog/profiling/stack/__init__.pyi @@ -57,7 +57,11 @@ def reinstall_segv_handler() -> None: ... def segv_handler_installed() -> bool: - """Return True if our handler is the installed disposition for SIGSEGV and SIGBUS.""" + """Return True if our handler is the installed disposition for SIGSEGV and SIGBUS. + + Primarily test introspection: it queries the live disposition via sigaction(2) + for both signals on every call, so it is not free. Do not call it on hot paths. + """ ... # Pause/resume sampling diff --git a/ddtrace/internal/datadog/profiling/stack/_stack.pyi b/ddtrace/internal/datadog/profiling/stack/_stack.pyi index 513fac05d66..d3551e3a78d 100644 --- a/ddtrace/internal/datadog/profiling/stack/_stack.pyi +++ b/ddtrace/internal/datadog/profiling/stack/_stack.pyi @@ -26,7 +26,11 @@ def is_safe_copy_failed() -> bool: ... def fast_copy_memory_active() -> bool: ... # test introspection: is safe_memcpy active? def _set_fast_copy_warmup_seconds(seconds: float) -> None: ... # test-only; before start def segv_handler_installed() -> bool: - """Return True if our handler is the installed disposition for SIGSEGV and SIGBUS.""" + """Return True if our handler is the installed disposition for SIGSEGV and SIGBUS. + + Primarily test introspection: it queries the live disposition via sigaction(2) + for both signals on every call, so it is not free. Do not call it on hot paths. + """ ... # span <-> profile association diff --git a/ddtrace/internal/datadog/profiling/stack/echion/echion/danger.h b/ddtrace/internal/datadog/profiling/stack/echion/echion/danger.h index 7b21f2f705e..17c49397b88 100644 --- a/ddtrace/internal/datadog/profiling/stack/echion/echion/danger.h +++ b/ddtrace/internal/datadog/profiling/stack/echion/echion/danger.h @@ -27,7 +27,7 @@ init_segv_catcher(); void uninstall_segv_handler(); -// Returns true only if our segv_handler owns both SIGSEGV and SIGBUS; false on any error. +// Returns true only if our signal handler owns both SIGSEGV and SIGBUS; false on any error. bool segv_handler_installed(); diff --git a/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp b/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp index b86d34f365a..00aef513686 100644 --- a/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp +++ b/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp @@ -31,10 +31,6 @@ class Sampler // This class manages the initialization of echion as well as the sampling thread. // The underlying echion instance it manages keeps much of its state globally, so this class is a singleton in order // to keep it aligned with the echion state. - // - // NOTE: fields below are ordered largest-to-smallest alignment to minimize padding - // (enforced by clang-analyzer-optin.performance.Padding). Keep new members grouped - // by size rather than by logical role. std::unique_ptr echion; // The sampling interval is atomic because it needs to be safely propagated to the sampling thread @@ -45,51 +41,9 @@ class Sampler // stopped or started in a straightforward manner without finer-grained control (locks) std::atomic thread_seq_num{ 0 }; - // Internal perf counters - uint64_t process_count = 0; - uint64_t sampler_thread_count = 0; - - double target_overhead = g_target_overhead; - microsecond_t max_sampling_period_us = g_max_sampling_period_us; - std::minstd_rand rng{ std::random_device{}() }; - - // Ring-buffer head for the p_stable process_delta window (see process_delta_window below). - size_t process_delta_window_head = 0; - - // Baseline CPU budget (us per adapt window) corresponding to an absolute floor overhead. - // Derived from baseline_core_pct at configuration time; keeps the profiler sampling even - // when app CPU is near 0. - double baseline_cpu_us_per_adapt_window = 0.0; - - // Percentile (0..1) used for p_stable; configurable, default p95. - double p_stable_percentile_frac = 0.95; - - // Fast-copy startup warmup in seconds. - double fast_copy_warmup_seconds = 15.0; - - std::vector thread_candidates; - - // Rolling window for p_stable: ring buffer of process_delta values (us CPU per adapt window). - // p_stable is the p-th percentile of this buffer, giving a stable estimate of app CPU usage - // that doesn't collapse to zero during brief idle periods. - std::vector process_delta_window; - // Thread exit synchronization - allows stop() to wait for the sampling thread to exit. // The mutex + condition variable pair is used to avoid the "lost wake-up" race condition // where stop() could miss the notification and hang forever (or until timeout). - std::mutex thread_exit_mutex; - // Pause synchronization — allows the faulthandler wrapper to temporarily - // suspend sampling so signal handlers can be safely swapped without racing - // with in-flight safe_memcpy calls. - std::mutex pause_mutex_; - std::condition_variable thread_exit_cv; - std::condition_variable pause_cv_; - - unsigned int max_threads_per_sample = g_default_max_threads_per_sample; - - // Rolling window duration in seconds; controls the ring buffer capacity. - uint32_t p_stable_window_s = 600; - std::atomic thread_running{ false }; // Whether the sampler is currently active. Unlike thread_running (which tracks // the thread's actual lifecycle) this is set synchronously in start/stop, so @@ -99,13 +53,16 @@ class Sampler // prefork reads this to decide whether to restart the sampler after fork, // ensuring a sampler that stopped due to an error is not silently restarted. std::atomic sampler_active_{ false }; + std::mutex thread_exit_mutex; + std::condition_variable thread_exit_cv; + + // Pause synchronization — allows the faulthandler wrapper to temporarily + // suspend sampling so signal handlers can be safely swapped without racing + // with in-flight safe_memcpy calls. std::atomic pause_requested_{ false }; std::atomic paused_{ false }; - bool do_adaptive_sampling = true; - - // Tracks whether the sampler was running when prefork was called, - // so that postfork_parent/restart_after_fork can restore it. - bool was_running_at_fork_{ false }; + std::mutex pause_mutex_; + std::condition_variable pause_cv_; // This is a singleton, so no public constructor Sampler(); @@ -113,12 +70,47 @@ class Sampler // One-time setup of echion void one_time_setup(); + // Internal perf counters + uint64_t process_count = 0; + uint64_t sampler_thread_count = 0; + + bool do_adaptive_sampling = true; + double target_overhead = g_target_overhead; + microsecond_t max_sampling_period_us = g_max_sampling_period_us; + unsigned int max_threads_per_sample = g_default_max_threads_per_sample; + std::minstd_rand rng{ std::random_device{}() }; + std::vector thread_candidates; void adapt_sampling_interval(); // Captures one sampling cycle across all threads (or a reservoir-sampled subset thereof // when max_threads_per_sample is set). void capture_samples(microsecond_t wall_time_us); + // Rolling window for p_stable: ring buffer of process_delta values (us CPU per adapt window). + // p_stable is the p-th percentile of this buffer, giving a stable estimate of app CPU usage + // that doesn't collapse to zero during brief idle periods. + std::vector process_delta_window; + size_t process_delta_window_head = 0; + + // Baseline CPU budget (us per adapt window) corresponding to an absolute floor overhead. + // Derived from baseline_core_pct at configuration time; keeps the profiler sampling even + // when app CPU is near 0. + double baseline_cpu_us_per_adapt_window = 0.0; + + // Percentile (0..1) used for p_stable; configurable, default p95. + double p_stable_percentile_frac = 0.95; + + // Fast-copy startup warmup in seconds. Grouped with the other doubles so it adds + // no struct padding (clang-analyzer-optin.performance.Padding). + double fast_copy_warmup_seconds = 15.0; + + // Rolling window duration in seconds; controls the ring buffer capacity. + uint32_t p_stable_window_s = 600; + + // Tracks whether the sampler was running when prefork was called, + // so that postfork_parent/restart_after_fork can restore it. + bool was_running_at_fork_{ false }; + void atfork_child(); friend void stack_atfork_prepare(); friend void stack_atfork_parent(); diff --git a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp index 2ae2391695c..a3ad4292dc0 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp @@ -341,7 +341,7 @@ Sampler::sampling_thread(const uint64_t seq_num) // Mark thread as running thread_running.store(true); - // (Re)arm our SIGSEGV/SIGBUS handlers once, but ONLY if we still own them. If a + // (Re)install our SIGSEGV/SIGBUS handlers once, but ONLY if we still own them. If a // foreign component we can't wrap (abseil via vLLM/gRPC, PyTorch/CUDA) already owns // them, overwriting it would cause handler-chaining races and make // segv_handler_installed() report incorrectly, so leave it authoritative. @@ -423,7 +423,10 @@ Sampler::sampling_thread(const uint64_t seq_num) } } else if (fast_copy_active && !handler_fallback_done && !segv_handler_installed()) { // A handler was taken over after upgrading; fall back permanently - // (no debounce - a false positive just costs the slower copy path). + // (no debounce). This is not free: it pins the process to the slower + // syscall copy for its remaining lifetime, which can meaningfully + // degrade sample quality (e.g. on asyncio workloads). We still prefer + // it over the alternative, which is crashing under a foreign handler. handler_fallback_done = true; std::cerr << "ddtrace stack profiler: SIGSEGV/SIGBUS handler was taken over by another " "component; falling back to syscall-based memory copy to avoid crashing." From 192efca9543f652b10db98ba1e5bb0a00a0cb4ec Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Thu, 9 Jul 2026 09:00:11 +0200 Subject: [PATCH 17/28] review: apply suggestion --- ddtrace/internal/datadog/profiling/stack/include/sampler.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp b/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp index 00aef513686..b54b8293666 100644 --- a/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp +++ b/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp @@ -100,8 +100,7 @@ class Sampler // Percentile (0..1) used for p_stable; configurable, default p95. double p_stable_percentile_frac = 0.95; - // Fast-copy startup warmup in seconds. Grouped with the other doubles so it adds - // no struct padding (clang-analyzer-optin.performance.Padding). + // Fast-copy startup warmup in seconds. double fast_copy_warmup_seconds = 15.0; // Rolling window duration in seconds; controls the ring buffer capacity. From 846d7984029a695291ae3c055a006bf071974750 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Thu, 9 Jul 2026 09:00:22 +0200 Subject: [PATCH 18/28] review: apply suggestion --- ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc b/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc index 121d68b06e8..be1b3ee500a 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc +++ b/ddtrace/internal/datadog/profiling/stack/src/echion/danger.cc @@ -143,7 +143,7 @@ init_segv_catcher() bool segv_handler_installed() { - // Recovery needs our 3-arg (SA_SIGINFO) handler to own BOTH SIGSEGV and SIGBUS + // Recovery needs our handler to own BOTH SIGSEGV and SIGBUS // (a copy fault can arrive as either); anything else means we can't recover. const int signals[] = { SIGSEGV, SIGBUS }; for (int signo : signals) { From b6e1d89e638a875e907101ecf3bada8859ac866b Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Thu, 9 Jul 2026 09:00:33 +0200 Subject: [PATCH 19/28] review: apply suggestion --- ddtrace/internal/datadog/profiling/stack/src/stack.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/ddtrace/internal/datadog/profiling/stack/src/stack.cpp b/ddtrace/internal/datadog/profiling/stack/src/stack.cpp index 604599d8a2e..6887993827f 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/stack.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/stack.cpp @@ -909,7 +909,6 @@ stack_is_safe_copy_failed(PyObject* Py_UNUSED(self), PyObject* Py_UNUSED(args)) static PyObject* stack_fast_copy_memory_active(PyObject* Py_UNUSED(self), PyObject* Py_UNUSED(args)) { - // Test introspection: is the fast safe_memcpy path active? (PROF-14568) if (fast_copy_active) { Py_RETURN_TRUE; } From 741dd844aff3b394cd09f2e626b4b48eb9adb2a1 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Thu, 9 Jul 2026 09:00:44 +0200 Subject: [PATCH 20/28] review: apply suggestion --- ddtrace/internal/datadog/profiling/stack/_stack.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/internal/datadog/profiling/stack/_stack.pyi b/ddtrace/internal/datadog/profiling/stack/_stack.pyi index d3551e3a78d..50449e05bb1 100644 --- a/ddtrace/internal/datadog/profiling/stack/_stack.pyi +++ b/ddtrace/internal/datadog/profiling/stack/_stack.pyi @@ -29,7 +29,7 @@ def segv_handler_installed() -> bool: """Return True if our handler is the installed disposition for SIGSEGV and SIGBUS. Primarily test introspection: it queries the live disposition via sigaction(2) - for both signals on every call, so it is not free. Do not call it on hot paths. + for both signals on every call, so it is costly. Do not call it on hot paths. """ ... From 8371811fa49a4ce970ee3ae0fd33700dd6b125e9 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Thu, 9 Jul 2026 09:00:54 +0200 Subject: [PATCH 21/28] review: apply suggestion --- ddtrace/internal/datadog/profiling/stack/src/stack.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/internal/datadog/profiling/stack/src/stack.cpp b/ddtrace/internal/datadog/profiling/stack/src/stack.cpp index 6887993827f..983171116c8 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/stack.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/stack.cpp @@ -1023,7 +1023,7 @@ static PyMethodDef stack_methods[] = { { "segv_handler_installed", stack_segv_handler_installed, METH_NOARGS, - "Return True if our handler is the currently installed disposition for SIGSEGV and SIGBUS" }, + "Return True if ddtrace's handler is the currently installed disposition for SIGSEGV and SIGBUS" }, // Sampling pause/resume for safe signal handler swapping { "pause_sampling", stack_pause_sampling, From 89ed7dbe2e66285cf302de3a3069d408cf31e292 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Thu, 9 Jul 2026 09:01:06 +0200 Subject: [PATCH 22/28] review: apply suggestion --- ddtrace/internal/datadog/profiling/stack/src/stack.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/stack/src/stack.cpp b/ddtrace/internal/datadog/profiling/stack/src/stack.cpp index 983171116c8..398d66ba19a 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/stack.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/stack.cpp @@ -886,8 +886,6 @@ stack_reinstall_segv_handler(PyObject* Py_UNUSED(self), PyObject* Py_UNUSED(args static PyObject* stack_segv_handler_installed(PyObject* Py_UNUSED(self), PyObject* Py_UNUSED(args)) { - // Test introspection: do we still own both SIGSEGV and SIGBUS? (same primitive - // the sampler uses to decide whether safe_memcpy recovery is trustworthy) if (segv_handler_installed()) { Py_RETURN_TRUE; } From 015616291344e7a4987cf191e44c8809615759b4 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Thu, 9 Jul 2026 09:01:19 +0200 Subject: [PATCH 23/28] review: apply suggestion --- ddtrace/internal/datadog/profiling/stack/src/stack.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/ddtrace/internal/datadog/profiling/stack/src/stack.cpp b/ddtrace/internal/datadog/profiling/stack/src/stack.cpp index 398d66ba19a..22f61d4ac42 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/stack.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/stack.cpp @@ -916,7 +916,6 @@ stack_fast_copy_memory_active(PyObject* Py_UNUSED(self), PyObject* Py_UNUSED(arg static PyObject* stack_set_fast_copy_warmup_seconds(PyObject* Py_UNUSED(self), PyObject* args) { - // Test-only: shrink the fast-copy warmup; must be set before the sampler starts. double seconds_value = 0.0; if (!PyArg_ParseTuple(args, "d", &seconds_value)) { From acc736357738848d92e0345e65f92f6993831c60 Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Fri, 10 Jul 2026 17:07:42 -0400 Subject: [PATCH 24/28] feat(profiling): add ProfilerStats metrics for fast-copy fallback observability --- .../dd_wrapper/include/profiler_stats.hpp | 18 ++++++ .../dd_wrapper/src/profiler_stats.cpp | 56 ++++++++++++++++++- .../profiling/stack/echion/echion/vm.h | 15 ++++- .../datadog/profiling/stack/src/echion/vm.cc | 5 ++ .../datadog/profiling/stack/src/sampler.cpp | 5 ++ .../datadog/profiling/stack/src/stack.cpp | 6 +- .../collector/test_copy_memory_stats.py | 23 ++++++++ 7 files changed, 124 insertions(+), 4 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/include/profiler_stats.hpp b/ddtrace/internal/datadog/profiling/dd_wrapper/include/profiler_stats.hpp index 49d91779a65..e629ecc1ff9 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/include/profiler_stats.hpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/include/profiler_stats.hpp @@ -31,6 +31,15 @@ class ProfilerStats // Whether fast_copy_memory (safe_memcpy) is enabled; unset until the sampler starts std::optional fast_copy_memory_enabled; + // User opted out of fast copy (env var or set_fast_copy(false)); static per process + std::optional fast_copy_memory_user_disabled; + + // Whether safe_memcpy initialized at startup; static per process + std::optional fast_copy_memory_capable; + + // Sticky: fell back to syscall copy (init failure, foreign handler, etc.) + std::optional fast_copy_memory_syscall_fallback; + // Number of copy_memory errors accumulated since the last profile reset (i.e. since the last upload) size_t copy_memory_error_count = 0; @@ -69,6 +78,15 @@ class ProfilerStats void set_fast_copy_memory_enabled(bool enabled); std::optional get_fast_copy_memory_enabled() const; + void set_fast_copy_memory_user_disabled(bool disabled); + std::optional get_fast_copy_memory_user_disabled() const; + + void set_fast_copy_memory_capable(bool capable); + std::optional get_fast_copy_memory_capable() const; + + void set_fast_copy_memory_syscall_fallback(bool fallback); + std::optional get_fast_copy_memory_syscall_fallback() const; + void add_copy_memory_error_count(size_t count); size_t get_copy_memory_error_count() const; diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/src/profiler_stats.cpp b/ddtrace/internal/datadog/profiling/dd_wrapper/src/profiler_stats.cpp index 7352673a13f..14dff03dee3 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/src/profiler_stats.cpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/src/profiler_stats.cpp @@ -12,6 +12,19 @@ append_to_string(std::string& s, size_t value) s.append(buf, ptr); } +void +append_optional_bool(std::string& s, const char* key, const std::optional& value) +{ + if (!value.has_value()) { + return; + } + s += '"'; + s += key; + s += "\": "; + s += *value ? "true" : "false"; + s += ','; +} + } // namespace void @@ -51,7 +64,7 @@ Datadog::ProfilerStats::reset_state() asyncio_task_count = std::nullopt; greenlet_count = std::nullopt; sample_capture_cpu_time_us = 0; - // fast_copy_memory_enabled is intentionally not reset: it reflects a static configuration + // fast_copy_memory_* static fields are intentionally not reset (see setters). } void @@ -66,6 +79,42 @@ Datadog::ProfilerStats::get_fast_copy_memory_enabled() const return fast_copy_memory_enabled; } +void +Datadog::ProfilerStats::set_fast_copy_memory_user_disabled(bool disabled) +{ + fast_copy_memory_user_disabled = disabled; +} + +std::optional +Datadog::ProfilerStats::get_fast_copy_memory_user_disabled() const +{ + return fast_copy_memory_user_disabled; +} + +void +Datadog::ProfilerStats::set_fast_copy_memory_capable(bool capable) +{ + fast_copy_memory_capable = capable; +} + +std::optional +Datadog::ProfilerStats::get_fast_copy_memory_capable() const +{ + return fast_copy_memory_capable; +} + +void +Datadog::ProfilerStats::set_fast_copy_memory_syscall_fallback(bool fallback) +{ + fast_copy_memory_syscall_fallback = fallback; +} + +std::optional +Datadog::ProfilerStats::get_fast_copy_memory_syscall_fallback() const +{ + return fast_copy_memory_syscall_fallback; +} + void Datadog::ProfilerStats::add_copy_memory_error_count(size_t count) { @@ -203,6 +252,11 @@ Datadog::ProfilerStats::get_internal_metadata_json() internal_metadata_json += ","; } + append_optional_bool(internal_metadata_json, "fast_copy_memory_user_disabled", fast_copy_memory_user_disabled); + append_optional_bool(internal_metadata_json, "fast_copy_memory_capable", fast_copy_memory_capable); + append_optional_bool( + internal_metadata_json, "fast_copy_memory_syscall_fallback", fast_copy_memory_syscall_fallback); + auto maybe_heap_tracker_count = get_heap_tracker_size(); if (maybe_heap_tracker_count) { internal_metadata_json += R"("heap_tracker_count": )"; diff --git a/ddtrace/internal/datadog/profiling/stack/echion/echion/vm.h b/ddtrace/internal/datadog/profiling/stack/echion/echion/vm.h index ccb54bbf334..a615a6a467d 100644 --- a/ddtrace/internal/datadog/profiling/stack/echion/echion/vm.h +++ b/ddtrace/internal/datadog/profiling/stack/echion/echion/vm.h @@ -54,8 +54,19 @@ inline kern_return_t (*safe_copy)(vm_map_read_t, // Whether safe_copy is currently set to the memcpy-based wrapper. inline bool fast_copy_active = false; -// Whether init_segv_catcher succeeded at constructor time. Persists even if -// fast_copy_active is later toggled off by set_fast_copy_enabled. +// User opted out via _DD_PROFILING_STACK_FAST_COPY or set_fast_copy(false). +inline bool fast_copy_user_disabled = false; + +// Sticky: fell back to syscall copy (init failure, foreign handler, warmup miss). +inline bool fast_copy_syscall_fallback = false; + +inline void +mark_fast_copy_syscall_fallback() +{ + fast_copy_syscall_fallback = true; +} + +// Set at init; survives toggling fast_copy_active. inline bool safe_memcpy_initialized = false; #if defined PL_LINUX diff --git a/ddtrace/internal/datadog/profiling/stack/src/echion/vm.cc b/ddtrace/internal/datadog/profiling/stack/src/echion/vm.cc index ef21bc207a1..a776e7b191c 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/echion/vm.cc +++ b/ddtrace/internal/datadog/profiling/stack/src/echion/vm.cc @@ -41,6 +41,7 @@ init_safe_copy() // Honor the fast-copy opt-out: when disabled via env var, skip installing // the SIGSEGV/SIGBUS handlers and alt stack entirely. if (fast_copy_env_disabled()) { + fast_copy_user_disabled = true; if (process_vm_readv_available) { safe_copy = process_vm_readv; } else { @@ -60,6 +61,7 @@ init_safe_copy() fprintf(stderr, "Failed to initialize segv catcher. Trying process_vm_readv.\n"); if (process_vm_readv_available) { safe_copy = process_vm_readv; + mark_fast_copy_syscall_fallback(); } else { fprintf(stderr, "Failed to initialize safe copy interface\n"); failed_safe_copy = true; @@ -72,6 +74,7 @@ init_safe_copy() { // Honor the fast-copy opt-out: skip installing signal handlers when disabled. if (fast_copy_env_disabled()) { + fast_copy_user_disabled = true; return; } @@ -84,6 +87,7 @@ init_safe_copy() // std::cerr might not be fully initialized at constructor time. fprintf(stderr, "Failed to initialize segv catcher. Using mach_vm_read_overwrite instead.\n"); + mark_fast_copy_syscall_fallback(); } #endif // PL_DARWIN @@ -99,6 +103,7 @@ set_fast_copy_enabled(bool enabled) } fprintf(stderr, "Warning: fast copy requested but safe_memcpy was not initialized; falling back to process_vm_readv\n"); + mark_fast_copy_syscall_fallback(); // Fall through to process_vm_readv. } diff --git a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp index a3ad4292dc0..9cbb94cd927 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp @@ -416,6 +416,7 @@ Sampler::sampling_thread(const uint64_t seq_num) // syscall copy (already active from warmup) for the life of // the process. handler_fallback_done = true; + mark_fast_copy_syscall_fallback(); std::cerr << "ddtrace stack profiler: another component owns the SIGSEGV/SIGBUS " "handler; keeping the syscall-based memory copy to avoid crashing." << std::endl; @@ -428,6 +429,7 @@ Sampler::sampling_thread(const uint64_t seq_num) // degrade sample quality (e.g. on asyncio workloads). We still prefer // it over the alternative, which is crashing under a foreign handler. handler_fallback_done = true; + mark_fast_copy_syscall_fallback(); std::cerr << "ddtrace stack profiler: SIGSEGV/SIGBUS handler was taken over by another " "component; falling back to syscall-based memory copy to avoid crashing." << std::endl; @@ -481,6 +483,9 @@ Sampler::sampling_thread(const uint64_t seq_num) borrow.stats().increment_sampling_event_count(); borrow.stats().set_string_table_count(echion->string_table().size()); + borrow.stats().set_fast_copy_memory_user_disabled(fast_copy_user_disabled); + borrow.stats().set_fast_copy_memory_capable(safe_memcpy_initialized); + borrow.stats().set_fast_copy_memory_syscall_fallback(fast_copy_syscall_fallback); borrow.stats().set_fast_copy_memory_enabled(fast_copy_active); borrow.stats().set_asyncio_task_count(echion->asyncio_task_count()); borrow.stats().set_greenlet_count(greenlet_count); diff --git a/ddtrace/internal/datadog/profiling/stack/src/stack.cpp b/ddtrace/internal/datadog/profiling/stack/src/stack.cpp index 22f61d4ac42..10e6f567c23 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/stack.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/stack.cpp @@ -851,7 +851,11 @@ stack_set_fast_copy(PyObject* Py_UNUSED(self), PyObject* args) return nullptr; } - set_fast_copy_enabled(static_cast(enabled)); + const bool want = static_cast(enabled); + if (!want) { + fast_copy_user_disabled = true; + } + set_fast_copy_enabled(want); Py_RETURN_NONE; } diff --git a/tests/profiling/collector/test_copy_memory_stats.py b/tests/profiling/collector/test_copy_memory_stats.py index 53c76e20c4d..b979a503eba 100644 --- a/tests/profiling/collector/test_copy_memory_stats.py +++ b/tests/profiling/collector/test_copy_memory_stats.py @@ -32,6 +32,13 @@ def test_copy_memory_error_count_present(): metadata = json.load(fp) assert "copy_memory_error_count" in metadata, f"Missing copy_memory_error_count in {f}: {metadata}" assert metadata["copy_memory_error_count"] >= 0, f"copy_memory_error_count must be non-negative: {metadata}" + assert "fast_copy_memory_user_disabled" in metadata, ( + f"Missing fast_copy_memory_user_disabled in {f}: {metadata}" + ) + assert "fast_copy_memory_capable" in metadata, f"Missing fast_copy_memory_capable in {f}: {metadata}" + assert "fast_copy_memory_syscall_fallback" in metadata, ( + f"Missing fast_copy_memory_syscall_fallback in {f}: {metadata}" + ) @pytest.mark.subprocess( @@ -70,6 +77,8 @@ def test_fast_copy_memory_disabled(): assert metadata["fast_copy_memory_enabled"] is False, ( f"Expected fast_copy_memory_enabled=false when _DD_PROFILING_STACK_FAST_COPY=false: {metadata}" ) + assert metadata["fast_copy_memory_user_disabled"] is True, metadata + assert metadata["fast_copy_memory_syscall_fallback"] is False, metadata @pytest.mark.subprocess( @@ -82,6 +91,9 @@ def test_fast_copy_memory_disabled(): ) def test_fast_copy_memory_enabled() -> None: """Sampler runs on the syscall copy during warmup, then upgrades to safe_memcpy (PROF-14568).""" + import glob + import json + import os import time # Underscore-prefixed, so only on the _stack submodule (`import *` skips it). @@ -111,5 +123,16 @@ def test_fast_copy_memory_enabled() -> None: p.stop() + output_filename = os.environ["DD_PROFILING_OUTPUT_PPROF"] + "." + str(os.getpid()) + files = sorted(glob.glob(output_filename + ".*.internal_metadata.json")) + assert files, "Expected at least one internal_metadata.json file" + + with open(files[-1]) as fp: + metadata = json.load(fp) + assert metadata["fast_copy_memory_user_disabled"] is False, metadata + assert metadata["fast_copy_memory_capable"] is True, metadata + assert metadata["fast_copy_memory_syscall_fallback"] is False, metadata + assert metadata["fast_copy_memory_enabled"] is True, metadata + assert saw_warmup, "Expected the sampler to run on the syscall copy during the warmup window" assert saw_upgrade, "Expected the sampler to upgrade to safe_memcpy after warmup" From 51c010547ee66cf82a0f921d0e61dfc11bbb46b2 Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Mon, 13 Jul 2026 15:04:56 -0400 Subject: [PATCH 25/28] fix(profiling): preserve fast-copy metadata across stats swaps --- .../dd_wrapper/include/profiler_stats.hpp | 3 +++ .../dd_wrapper/src/profiler_stats.cpp | 17 ++++++++++++++ .../dd_wrapper/src/uploader_builder.cpp | 1 + .../profiling/stack/include/sampler.hpp | 2 ++ .../datadog/profiling/stack/src/sampler.cpp | 22 +++++++++++++++---- .../datadog/profiling/stack/src/stack.cpp | 1 + 6 files changed, 42 insertions(+), 4 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/include/profiler_stats.hpp b/ddtrace/internal/datadog/profiling/dd_wrapper/include/profiler_stats.hpp index e629ecc1ff9..3802b83d1f1 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/include/profiler_stats.hpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/include/profiler_stats.hpp @@ -87,6 +87,9 @@ class ProfilerStats void set_fast_copy_memory_syscall_fallback(bool fallback); std::optional get_fast_copy_memory_syscall_fallback() const; + // fast_copy_memory_* are process-static; carry them across ProfilerStats swaps. + void copy_fast_copy_metadata_from(const ProfilerStats& other); + void add_copy_memory_error_count(size_t count); size_t get_copy_memory_error_count() const; diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/src/profiler_stats.cpp b/ddtrace/internal/datadog/profiling/dd_wrapper/src/profiler_stats.cpp index 14dff03dee3..f0c74973cf0 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/src/profiler_stats.cpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/src/profiler_stats.cpp @@ -115,6 +115,23 @@ Datadog::ProfilerStats::get_fast_copy_memory_syscall_fallback() const return fast_copy_memory_syscall_fallback; } +void +Datadog::ProfilerStats::copy_fast_copy_metadata_from(const ProfilerStats& other) +{ + if (auto value = other.get_fast_copy_memory_user_disabled()) { + set_fast_copy_memory_user_disabled(*value); + } + if (auto value = other.get_fast_copy_memory_capable()) { + set_fast_copy_memory_capable(*value); + } + if (auto value = other.get_fast_copy_memory_syscall_fallback()) { + set_fast_copy_memory_syscall_fallback(*value); + } + if (auto value = other.get_fast_copy_memory_enabled()) { + set_fast_copy_memory_enabled(*value); + } +} + void Datadog::ProfilerStats::add_copy_memory_error_count(size_t count) { diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/src/uploader_builder.cpp b/ddtrace/internal/datadog/profiling/dd_wrapper/src/uploader_builder.cpp index 24d5f3cce3f..e83cc5141a6 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/src/uploader_builder.cpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/src/uploader_builder.cpp @@ -205,6 +205,7 @@ Datadog::UploaderBuilder::build() // Swap the ProfilerStats (which replaces the one being written to with an empty state). // We do this first as we still want to reset ProfilerStats if the serialization fails. std::swap(stats, borrowed.stats()); + borrowed.stats().copy_fast_copy_metadata_from(stats); // Try to encode the Profile (which will also reset it) encoded = ddog_prof_Profile_serialize(&borrowed.profile(), nullptr, nullptr); diff --git a/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp b/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp index b54b8293666..3b68f0579f0 100644 --- a/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp +++ b/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp @@ -182,4 +182,6 @@ class Sampler bool restart_after_fork(); }; +void seed_fast_copy_profiler_stats(); + } // namespace Datadog diff --git a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp index 9cbb94cd927..891df23eb10 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp @@ -25,6 +25,21 @@ using namespace Datadog; +static void +update_fast_copy_stats(ProfilerStats& stats) +{ + stats.set_fast_copy_memory_user_disabled(fast_copy_user_disabled); + stats.set_fast_copy_memory_capable(safe_memcpy_initialized); + stats.set_fast_copy_memory_syscall_fallback(fast_copy_syscall_fallback); + stats.set_fast_copy_memory_enabled(fast_copy_active); +} + +void +Datadog::seed_fast_copy_profiler_stats() +{ + update_fast_copy_stats(Sample::profile_borrow().stats()); +} + // Helper class for spawning a std::thread with control over its default stack size #ifdef __linux__ #include @@ -341,6 +356,8 @@ Sampler::sampling_thread(const uint64_t seq_num) // Mark thread as running thread_running.store(true); + seed_fast_copy_profiler_stats(); + // (Re)install our SIGSEGV/SIGBUS handlers once, but ONLY if we still own them. If a // foreign component we can't wrap (abseil via vLLM/gRPC, PyTorch/CUDA) already owns // them, overwriting it would cause handler-chaining races and make @@ -483,10 +500,7 @@ Sampler::sampling_thread(const uint64_t seq_num) borrow.stats().increment_sampling_event_count(); borrow.stats().set_string_table_count(echion->string_table().size()); - borrow.stats().set_fast_copy_memory_user_disabled(fast_copy_user_disabled); - borrow.stats().set_fast_copy_memory_capable(safe_memcpy_initialized); - borrow.stats().set_fast_copy_memory_syscall_fallback(fast_copy_syscall_fallback); - borrow.stats().set_fast_copy_memory_enabled(fast_copy_active); + update_fast_copy_stats(borrow.stats()); borrow.stats().set_asyncio_task_count(echion->asyncio_task_count()); borrow.stats().set_greenlet_count(greenlet_count); diff --git a/ddtrace/internal/datadog/profiling/stack/src/stack.cpp b/ddtrace/internal/datadog/profiling/stack/src/stack.cpp index 10e6f567c23..b5777dace00 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/stack.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/stack.cpp @@ -33,6 +33,7 @@ stack_start_impl(PyObject* self, PyObject* args, PyObject* kwargs) Py_BEGIN_ALLOW_THREADS; OriginTaskLinks::get_instance().enable(); Py_END_ALLOW_THREADS; + seed_fast_copy_profiler_stats(); Py_RETURN_TRUE; } Py_RETURN_FALSE; From 5bd63f33e46f4ec4c570d7bcb47a72dd83be5678 Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Mon, 13 Jul 2026 16:01:29 -0400 Subject: [PATCH 26/28] style(profiling): clang-format sampler.hpp --- ddtrace/internal/datadog/profiling/stack/include/sampler.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp b/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp index 3b68f0579f0..9d3b4490c07 100644 --- a/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp +++ b/ddtrace/internal/datadog/profiling/stack/include/sampler.hpp @@ -182,6 +182,7 @@ class Sampler bool restart_after_fork(); }; -void seed_fast_copy_profiler_stats(); +void +seed_fast_copy_profiler_stats(); } // namespace Datadog From fd5fd49c2c808c83b8f108ebfe1207768742952e Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Mon, 13 Jul 2026 17:24:50 -0400 Subject: [PATCH 27/28] update comment and release note --- .../datadog/profiling/stack/src/sampler.cpp | 24 +++++++++---------- ...ign-handler-fallback-3d8f1b6a0e25c947.yaml | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp index 891df23eb10..ddb3f7fe704 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp @@ -101,10 +101,7 @@ uint64_t get_thread_cpu_time_us() { #if defined(__linux__) - struct timespec ts - { - 0, 0 - }; + struct timespec ts{ 0, 0 }; if (clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts) != 0) { return 0; @@ -134,10 +131,7 @@ void Sampler::adapt_sampling_interval() { #if defined(__linux__) - struct timespec ts - { - 0, 0 - }; + struct timespec ts{ 0, 0 }; clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts); auto new_process_count = static_cast(ts.tv_sec * 1'000'000ULL + ts.tv_nsec / 1000); @@ -358,10 +352,16 @@ Sampler::sampling_thread(const uint64_t seq_num) seed_fast_copy_profiler_stats(); - // (Re)install our SIGSEGV/SIGBUS handlers once, but ONLY if we still own them. If a - // foreign component we can't wrap (abseil via vLLM/gRPC, PyTorch/CUDA) already owns - // them, overwriting it would cause handler-chaining races and make - // segv_handler_installed() report incorrectly, so leave it authoritative. + // (Re)install our SIGSEGV/SIGBUS handlers once, but ONLY if we still own them. + // + // safe_memcpy recovers only when our handler owns BOTH signals (see danger.cc). + // We can chain on top of handlers we coordinate with (faulthandler, crashtracker: + // pause + uninstall/reinstall in stack.cpp / crashtracking.py). Libraries such as + // abseil (vLLM/gRPC) or PyTorch/CUDA install their own handlers independently—often + // lazily on other threads—so overwriting them breaks their crash path and faults + // during sampling may still reach their handler instead of our siglongjmp (PROF-14568). + // If a foreign owner is already authoritative, leave it in place and fall back to + // the syscall copy rather than reclaiming on top. static std::once_flag segv_handler_once; if (fast_copy_active) { std::call_once(segv_handler_once, []() { diff --git a/releasenotes/notes/profiling-stack-sampler-foreign-handler-fallback-3d8f1b6a0e25c947.yaml b/releasenotes/notes/profiling-stack-sampler-foreign-handler-fallback-3d8f1b6a0e25c947.yaml index 0078356a47e..cbd3305a823 100644 --- a/releasenotes/notes/profiling-stack-sampler-foreign-handler-fallback-3d8f1b6a0e25c947.yaml +++ b/releasenotes/notes/profiling-stack-sampler-foreign-handler-fallback-3d8f1b6a0e25c947.yaml @@ -2,6 +2,6 @@ fixes: - | profiling: crash fix for stack sampler in case where another - component installs its own ``SIGSEGV``/``SIGBUS`` handler that the profiler cannot wrap (e.g. CUDA, PyTorch, etc.). + component installs its own ``SIGSEGV``/``SIGBUS`` handler that the profiler cannot safely chain with (e.g. CUDA, PyTorch, abseil via vLLM/gRPC; see sampler.cpp). The sampler upgrades to the faster fault-recovery copy if it still owns both fault handlers afterwards. Otherwise, it permanently falls back to the syscall-based copy. From 61b81a023ed44841d89396d0b7aa7d4e2a6a8ec4 Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Thu, 23 Jul 2026 15:46:04 -0400 Subject: [PATCH 28/28] fix lint --- .../internal/datadog/profiling/stack/src/sampler.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp index ddb3f7fe704..c0f36cc2313 100644 --- a/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp +++ b/ddtrace/internal/datadog/profiling/stack/src/sampler.cpp @@ -101,7 +101,10 @@ uint64_t get_thread_cpu_time_us() { #if defined(__linux__) - struct timespec ts{ 0, 0 }; + struct timespec ts + { + 0, 0 + }; if (clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts) != 0) { return 0; @@ -131,7 +134,10 @@ void Sampler::adapt_sampling_interval() { #if defined(__linux__) - struct timespec ts{ 0, 0 }; + struct timespec ts + { + 0, 0 + }; clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts); auto new_process_count = static_cast(ts.tv_sec * 1'000'000ULL + ts.tv_nsec / 1000);