fix(profiling): maintain fast-copy and foreign-takeover fallback across forks (PROF-14568)

ddtrace/internal/datadog/profiling/dd_wrapper/include/profiler_state.hpp

@@ -86,6 +86,21 @@ class ProfilerState
     // every upload in UploaderBuilder::build, which would drop the value.
     std::string profiler_settings_info_json;
 
+    // Process-static fast-copy configuration snapshot. ProfilerStats is swapped on
+    // every upload, so metadata serialization falls back to this when per-window
+    // stats have not been seeded yet (e.g. heap upload before a sampling cycle).
+    struct FastCopyProfilerMetadata
+    {
+        bool snapshot_initialized = false;
+        bool user_disabled = false;
+        bool capable = false;
+        bool syscall_fallback = false;
+        bool enabled = false;
+        bool desired = false;
+        bool foreign_takeover = false;
+    };
+    FastCopyProfilerMetadata fast_copy_metadata{};
+
     // ========================================================================
     // Native call tracking state
     // ========================================================================

ddtrace/internal/datadog/profiling/dd_wrapper/include/profiler_stats.hpp

@@ -40,6 +40,12 @@ class ProfilerStats
     // Sticky: fell back to syscall copy (init failure, foreign handler, etc.)
     std::optional<bool> fast_copy_memory_syscall_fallback;
 
+    // Persistent intent to use safe_memcpy; not toggled by warmup/fallback
+    std::optional<bool> fast_copy_memory_desired;
+
+    // Sticky: foreign handler owns SIGSEGV/SIGBUS; blocks reclaim and re-warm
+    std::optional<bool> fast_copy_memory_foreign_takeover;
+
     // Number of copy_memory errors accumulated since the last profile reset (i.e. since the last upload)
     size_t copy_memory_error_count = 0;
 
@@ -49,8 +55,7 @@ class ProfilerStats
     // Samples dropped because the cap was reached (cumulative over tracker lifetime)
     std::optional<size_t> heap_tracker_cap_drops;
 
-    // Peak number of asyncio tasks seen across sampled threads in any single sampling
-    // cycle during the current profile period (see set_asyncio_task_count).
+    // Number of asyncio tasks seen across sampled threads in the last sampling cycle
     std::optional<size_t> asyncio_task_count;
 
     // Number of greenlets currently tracked by the stack profiler
@@ -87,6 +92,12 @@ class ProfilerStats
     void set_fast_copy_memory_syscall_fallback(bool fallback);
     std::optional<bool> get_fast_copy_memory_syscall_fallback() const;
 
+    void set_fast_copy_memory_desired(bool desired);
+    std::optional<bool> get_fast_copy_memory_desired() const;
+
+    void set_fast_copy_memory_foreign_takeover(bool takeover);
+    std::optional<bool> get_fast_copy_memory_foreign_takeover() const;
+
     // fast_copy_memory_* are process-static; carry them across ProfilerStats swaps.
     void copy_fast_copy_metadata_from(const ProfilerStats& other);
 

ddtrace/internal/datadog/profiling/dd_wrapper/src/profiler_stats.cpp

@@ -1,5 +1,7 @@
 #include "profiler_stats.hpp"
 
+#include "profiler_state.hpp"
+
 #include <charconv>
 
 namespace {
@@ -12,16 +14,37 @@ append_to_string(std::string& s, size_t value)
     s.append(buf, ptr);
 }
 
-void
-append_optional_bool(std::string& s, const char* key, const std::optional<bool>& value)
+std::optional<bool>
+resolve_fast_copy_bool(const std::optional<bool>& stat_value, bool snapshot_value, bool snapshot_initialized)
 {
-    if (!value.has_value()) {
-        return;
+    if (stat_value.has_value()) {
+        return stat_value;
     }
+    if (snapshot_initialized) {
+        return snapshot_value;
+    }
+    return std::nullopt;
+}
+
+bool
+resolve_fast_copy_bool_or_default(const std::optional<bool>& stat_value,
+                                  bool snapshot_value,
+                                  bool snapshot_initialized,
+                                  bool default_value)
+{
+    if (const auto resolved = resolve_fast_copy_bool(stat_value, snapshot_value, snapshot_initialized)) {
+        return *resolved;
+    }
+    return default_value;
+}
+
+void
+append_bool(std::string& s, const char* key, bool value)
+{
     s += '"';
     s += key;
     s += "\": ";
-    s += *value ? "true" : "false";
+    s += value ? "true" : "false";
     s += ',';
 }
 
@@ -115,6 +138,30 @@ Datadog::ProfilerStats::get_fast_copy_memory_syscall_fallback() const
     return fast_copy_memory_syscall_fallback;
 }
 
+void
+Datadog::ProfilerStats::set_fast_copy_memory_desired(bool desired)
+{
+    fast_copy_memory_desired = desired;
+}
+
+std::optional<bool>
+Datadog::ProfilerStats::get_fast_copy_memory_desired() const
+{
+    return fast_copy_memory_desired;
+}
+
+void
+Datadog::ProfilerStats::set_fast_copy_memory_foreign_takeover(bool takeover)
+{
+    fast_copy_memory_foreign_takeover = takeover;
+}
+
+std::optional<bool>
+Datadog::ProfilerStats::get_fast_copy_memory_foreign_takeover() const
+{
+    return fast_copy_memory_foreign_takeover;
+}
+
 void
 Datadog::ProfilerStats::copy_fast_copy_metadata_from(const ProfilerStats& other)
 {
@@ -127,6 +174,12 @@ Datadog::ProfilerStats::copy_fast_copy_metadata_from(const ProfilerStats& other)
     if (auto value = other.get_fast_copy_memory_syscall_fallback()) {
         set_fast_copy_memory_syscall_fallback(*value);
     }
+    if (auto value = other.get_fast_copy_memory_desired()) {
+        set_fast_copy_memory_desired(*value);
+    }
+    if (auto value = other.get_fast_copy_memory_foreign_takeover()) {
+        set_fast_copy_memory_foreign_takeover(*value);
+    }
     if (auto value = other.get_fast_copy_memory_enabled()) {
         set_fast_copy_memory_enabled(*value);
     }
@@ -262,17 +315,40 @@ Datadog::ProfilerStats::get_internal_metadata_json()
     append_to_string(internal_metadata_json, sampling_event_count);
     internal_metadata_json += ",";
 
-    auto maybe_fast_copy_enabled = get_fast_copy_memory_enabled();
-    if (maybe_fast_copy_enabled) {
-        internal_metadata_json += R"("fast_copy_memory_enabled": )";
-        internal_metadata_json += *maybe_fast_copy_enabled ? "true" : "false";
-        internal_metadata_json += ",";
-    }
-
-    append_optional_bool(internal_metadata_json, "fast_copy_memory_user_disabled", fast_copy_memory_user_disabled);
-    append_optional_bool(internal_metadata_json, "fast_copy_memory_capable", fast_copy_memory_capable);
-    append_optional_bool(
-      internal_metadata_json, "fast_copy_memory_syscall_fallback", fast_copy_memory_syscall_fallback);
+    const auto& fast_copy_snapshot = ProfilerState::get().fast_copy_metadata;
+    append_bool(
+      internal_metadata_json,
+      "fast_copy_memory_enabled",
+      resolve_fast_copy_bool_or_default(
+        fast_copy_memory_enabled, fast_copy_snapshot.enabled, fast_copy_snapshot.snapshot_initialized, false));
+    append_bool(internal_metadata_json,
+                "fast_copy_memory_user_disabled",
+                resolve_fast_copy_bool_or_default(fast_copy_memory_user_disabled,
+                                                  fast_copy_snapshot.user_disabled,
+                                                  fast_copy_snapshot.snapshot_initialized,
+                                                  false));
+    append_bool(
+      internal_metadata_json,
+      "fast_copy_memory_capable",
+      resolve_fast_copy_bool_or_default(
+        fast_copy_memory_capable, fast_copy_snapshot.capable, fast_copy_snapshot.snapshot_initialized, false));
+    append_bool(internal_metadata_json,
+                "fast_copy_memory_syscall_fallback",
+                resolve_fast_copy_bool_or_default(fast_copy_memory_syscall_fallback,
+                                                  fast_copy_snapshot.syscall_fallback,
+                                                  fast_copy_snapshot.snapshot_initialized,
+                                                  false));
+    append_bool(
+      internal_metadata_json,
+      "fast_copy_memory_desired",
+      resolve_fast_copy_bool_or_default(
+        fast_copy_memory_desired, fast_copy_snapshot.desired, fast_copy_snapshot.snapshot_initialized, false));
+    append_bool(internal_metadata_json,
+                "fast_copy_memory_foreign_takeover",
+                resolve_fast_copy_bool_or_default(fast_copy_memory_foreign_takeover,
+                                                  fast_copy_snapshot.foreign_takeover,
+                                                  fast_copy_snapshot.snapshot_initialized,
+                                                  false));
 
     auto maybe_heap_tracker_count = get_heap_tracker_size();
     if (maybe_heap_tracker_count) {

ddtrace/internal/datadog/profiling/stack/__init__.pyi

@@ -10,13 +10,8 @@ from ddtrace._trace import span as ddspan
 # Core stack v2 functions
 def start(min_interval: float = ...) -> bool: ...
 def stop() -> None: ...
-def is_origin_task_linking_enabled() -> bool: ...
 def _native_call_registry_size() -> int: ...
 
-# executor worker thread <-> originating asyncio task association
-def link_origin_task(task_id: int, task_name: str) -> None: ...
-def unlink_origin_task() -> None: ...
-
 # Sampling configuration
 def set_adaptive_sampling(do_adaptive_sampling: bool = False) -> None: ...
 def set_target_overhead(target_overhead: float) -> None: ...
@@ -42,9 +37,9 @@ def set_interval(new_interval: float) -> None: ...
 # Memory copy strategy
 def set_fast_copy(enabled: bool) -> None: ...
 def is_safe_copy_failed() -> bool: ...
-def fast_copy_memory_active() -> bool: ...  # test introspection: is safe_memcpy active?
+def fast_copy_memory_active() -> bool: ...
 
-# _set_fast_copy_warmup_seconds is test-only; accessed via _stack (import * skips it).
+# Test-only APIs live on _stack (import * skips them).
 
 def uninstall_segv_handler() -> None: ...
 def reinstall_segv_handler() -> None:
@@ -57,11 +52,7 @@ def reinstall_segv_handler() -> None:
     ...
 
 def segv_handler_installed() -> bool:
-    """Return True if our handler is the installed disposition for SIGSEGV and SIGBUS.
-
-    Primarily test introspection: it queries the live disposition via sigaction(2)
-    for both signals on every call, so it is not free. Do not call it on hot paths.
-    """
+    """True if our SIGSEGV/SIGBUS handlers are installed (test introspection; not cheap)."""
     ...
 
 # Pause/resume sampling

ddtrace/internal/datadog/profiling/stack/_stack.pyi

@@ -23,16 +23,18 @@ def set_interval(new_interval: float) -> None: ...
 # Memory copy strategy
 def set_fast_copy(enabled: bool) -> None: ...
 def is_safe_copy_failed() -> bool: ...
-def fast_copy_memory_active() -> bool: ...  # test introspection: is safe_memcpy active?
-def _set_fast_copy_warmup_seconds(seconds: float) -> None: ...  # test-only; before start
+def fast_copy_memory_active() -> bool: ...
+def _sampler_running() -> bool: ...
+def _sampling_paused() -> bool: ...
+def _take_prefork_pause_observation() -> bool: ...
+def _set_fast_copy_warmup_seconds(seconds: float) -> None: ...
 def segv_handler_installed() -> bool:
-    """Return True if our handler is the installed disposition for SIGSEGV and SIGBUS.
-
-    Primarily test introspection: it queries the live disposition via sigaction(2)
-    for both signals on every call, so it is costly. Do not call it on hot paths.
-    """
+    """True if our SIGSEGV/SIGBUS handlers are installed (test introspection; not cheap)."""
     ...
 
+def uninstall_segv_handler() -> None: ...
+def reinstall_segv_handler() -> None: ...
+
 # span <-> profile association
 def link_span(
     span_id: int,

ddtrace/internal/datadog/profiling/stack/echion/echion/interp.h

@@ -30,4 +30,7 @@ class InterpreterInfo
 };
 
 void
-for_each_interp(_PyRuntimeState* runtime, const std::function<void(InterpreterInfo& interp)>& callback);
+for_each_interp(
+  _PyRuntimeState* runtime,
+  const std::function<void(InterpreterInfo& interp)>& callback,
+  const std::function<bool()>& continue_sampling = []() { return true; });

ddtrace/internal/datadog/profiling/stack/echion/echion/threads.h

@@ -129,7 +129,11 @@ class ThreadInfo
 
 // ----------------------------------------------------------------------------
 
-using PyThreadStateCallback = std::function<void(PyThreadState*, ThreadInfo&)>;
+using PyThreadStateCallback = std::function<void(const PyThreadState*, ThreadInfo&)>;
 
 void
-for_each_thread(EchionSampler& echion, InterpreterInfo& interp, const PyThreadStateCallback& callback);
+for_each_thread(
+  EchionSampler& echion,
+  InterpreterInfo& interp,
+  const PyThreadStateCallback& callback,
+  const std::function<bool()>& continue_sampling = []() { return true; });

ddtrace/internal/datadog/profiling/stack/echion/echion/vm.h

@@ -51,8 +51,11 @@ inline kern_return_t (*safe_copy)(vm_map_read_t,
 
 #endif
 
-// Whether safe_copy is currently set to the memcpy-based wrapper.
-inline bool fast_copy_active = false;
+// Transient: safe_memcpy is the active copy path.
+inline std::atomic<bool> fast_copy_active{ false };
+
+// User wants fast copy (set at init); survives warmup toggling fast_copy_active.
+inline bool fast_copy_requested = false;
 
 // User opted out via _DD_PROFILING_STACK_FAST_COPY or set_fast_copy(false).
 inline bool fast_copy_user_disabled = false;
@@ -66,6 +69,18 @@ mark_fast_copy_syscall_fallback()
     fast_copy_syscall_fallback = true;
 }
 
+// Persistent intent; not toggled by warmup/fallback; survives fork.
+inline std::atomic<bool> fast_copy_desired{ false };
+// Sticky: foreign handler owns SIGSEGV/SIGBUS; blocks reclaim and re-warm.
+inline std::atomic<bool> fast_copy_foreign_takeover{ false };
+
+inline bool
+fast_copy_handler_ops_enabled()
+{
+    return fast_copy_desired.load(std::memory_order_relaxed) &&
+           !fast_copy_foreign_takeover.load(std::memory_order_relaxed);
+}
+
 // Set at init; survives toggling fast_copy_active.
 inline bool safe_memcpy_initialized = false;
 

ddtrace/internal/datadog/profiling/stack/include/sampler.hpp

@@ -19,6 +19,8 @@ class EchionSampler;
 
 namespace Datadog {
 
+class ProfilerStats;
+
 enum class PauseResult : std::uint8_t
 {
     Paused,     // sampler was running and is now paused
@@ -82,10 +84,6 @@ class Sampler
     std::vector<PyThreadState> thread_candidates;
     void adapt_sampling_interval();
 
-    // Captures one sampling cycle across all threads (or a reservoir-sampled subset thereof
-    // when max_threads_per_sample is set).
-    void capture_samples(microsecond_t wall_time_us);
-
     // Rolling window for p_stable: ring buffer of process_delta values (us CPU per adapt window).
     // p_stable is the p-th percentile of this buffer, giving a stable estimate of app CPU usage
     // that doesn't collapse to zero during brief idle periods.
@@ -100,15 +98,12 @@ class Sampler
     // Percentile (0..1) used for p_stable; configurable, default p95.
     double p_stable_percentile_frac = 0.95;
 
-    // Fast-copy startup warmup in seconds.
     double fast_copy_warmup_seconds = 15.0;
 
-    // Rolling window duration in seconds; controls the ring buffer capacity.
     uint32_t p_stable_window_s = 600;
 
-    // Tracks whether the sampler was running when prefork was called,
-    // so that postfork_parent/restart_after_fork can restore it.
     bool was_running_at_fork_{ false };
+    bool paused_for_fork_{ false };
 
     void atfork_child();
     friend void stack_atfork_prepare();
@@ -145,6 +140,8 @@ class Sampler
     // self-time, and we're not currently accounting for the echion self-time.
     void set_interval(double new_interval);
     bool is_running() const { return thread_running.load(); }
+    bool is_sampling_paused() const { return paused_.load(std::memory_order_acquire); }
+    bool take_prefork_pause_observation();
     void set_adaptive_sampling(bool value) { do_adaptive_sampling = value; }
     void set_target_overhead(double value) { target_overhead = value; }
     void set_max_sampling_period(microsecond_t max_interval_us)
@@ -177,12 +174,13 @@ class Sampler
     // Restart the sampling thread in the parent after fork
     void postfork_parent();
 
-    // Restart the sampler after fork if it was running.
-    // Returns true if start() was invoked and succeeded.
-    bool restart_after_fork();
+    // Restart the sampler after fork if it was running
+    void restart_after_fork();
 };
 
+// Publish the current echion fast-copy globals into ProfilerState and,
+// optionally, the active profile stats object.
 void
-seed_fast_copy_profiler_stats();
+publish_fast_copy_profiler_metadata(Datadog::ProfilerStats* stats = nullptr);
 
 } // namespace Datadog