From 07a24cdd20c5b5bf0b10264f811811b0d6f24cf1 Mon Sep 17 00:00:00 2001 From: Anatolii Date: Sun, 12 Jul 2026 13:15:52 +0400 Subject: [PATCH] fix(sdk): parent_trace_id contextvar ALWAYS overrides caller-set (0.13.8 hotfix #2) Hotfix #2 for the parent_trace_id wire-add end-to-end work from PR #64 / 0.13.7. PR #64 wired the field on the wire, but a diagnostic script (sdk_diag.py) running on the deployed 0.13.7 revealed cost_events on the backend were missing parent_trace_id: trace_id=cccccccc-... parent_trace_id=NULL model=gpt-4.1-mini tokens=10 The drift was in _enrich_event's parent_trace_id fallback (commit efff530): it used an "if not in enriched" guard, so when langgraph.py::on_llm_end's _active_runs[run_id] lookup missed (run_id drift between the auto-injected chat_model callback and an explicit user-supplied one, or no matching on_llm_start because the user wrapped the LLM call in a non-langgraph stack), the field was absent, the trace_id fallback at line 2422 overwrote the event with the chain contextvar, but parent_trace_id stayed NULL because the old condition was skipped. Override semantics: the chain contextvar is the single source of truth for "what chain does this event belong to". Both langgraph.py::on_llm_end's caller-set value and a non-langgraph caller's absence resolve to the same contextvar; preferring the contextvar when present is idempotent for the happy path AND closes the drift in the unhappy path. Also: - Bump version 0.13.7 -> 0.13.8. - Prepend v3.23 / 0.13.8 changelog entry. - Rewrite the existing test_enrich_event_preserves_caller_set_parent_trace_id to match the new override semantics. - Add 2 new regression tests in TestEnrichEventParentTraceOverride: test_enrich_event_sets_parent_trace_id_when_chain_contextvar_set (drift scenario) and test_enrich_event_parent_trace_id_matches_trace_id_in_chain_mode (SpanContext invariant). Regression coverage: 24 tests in test_drift_fixes_2026_07_04.py (22 prior + 2 new). Full critical suite 157/157 passed. ruff clean, mypy 34/34 source files clean. End-to-end after this hotfix: the next real LLM call inside a chain contextvar should arrive at the backend with both cost_events.trace_id and cost_events.parent_trace_id set to the chain contextvar value, and the unified SELECT third JOIN arm should populate the dashboard's Recent executions panel with Model / Tokens / Cost on the orchestration row. --- pyproject.toml | 2 +- src/nullrun/__version__.py | 2 +- src/nullrun/runtime.py | 58 ++++++++----- tests/test_drift_fixes_2026_07_04.py | 124 ++++++++++++++++++++++++--- 4 files changed, 152 insertions(+), 34 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index caef74b..49dcb5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ name = "nullrun" # the full ``flush_interval`` (5s default). Plus CI hygiene: # pip cache, ``fail-fast`` matrix, ``pytest-xdist -n auto``. No # on-wire change; backends on 1.0.0 keep working unchanged. -version = "0.13.7" +version = "0.13.8" # Kept under the 200-char preview threshold so the full line is visible # without an "expand" click. Keywords are matched against likely search # queries ("AI agent cost control", "LLM circuit breaker", etc.). diff --git a/src/nullrun/__version__.py b/src/nullrun/__version__.py index 919ba36..dd4fb34 100644 --- a/src/nullrun/__version__.py +++ b/src/nullrun/__version__.py @@ -491,5 +491,5 @@ """ -__version__ = "0.13.7" +__version__ = "0.13.8" __platform_version__ = "1.0.0" diff --git a/src/nullrun/runtime.py b/src/nullrun/runtime.py index 8827324..42513c7 100644 --- a/src/nullrun/runtime.py +++ b/src/nullrun/runtime.py @@ -2512,27 +2512,43 @@ def _enrich_event(self, event: dict[str, Any]) -> dict[str, Any]: # ``langgraph.py::on_llm_end`` may have already stamped # ``parent_trace_id`` when an LLM call sits inside a # chain / agent (we set it from the child SpanContext there). - # When the caller passed a span via ``runtime.track_event`` - # (without going through the langgraph callback) the field is - # absent — fall back to the contextvar so non-langgraph - # integrations (crewai, autogen, llama_index, plain httpx - # transport) get the same wire shape. The backend's - # ``cost_events.parent_trace_id`` column + unified SELECT - # third JOIN arm (``cs.join_kind = 'parent_trace_id'``) both - # depend on this being present whenever a parent span exists - # in scope; without it the dashboard falls back to the - # weaker ``trace_id`` arm and LLM rows show empty - # Model / Tokens / Cost on the orchestration row that owns - # the call. - if "parent_trace_id" not in enriched: - # Re-import to mirror the local-import pattern used - # for get_server_minted_execution_id above — keeps - # the runtime module's eager import graph stable. - from nullrun.context import get_trace_id as _get_trace_id - - parent_trace_id = _get_trace_id() - if parent_trace_id: - enriched["parent_trace_id"] = parent_trace_id + # + # 2026-07-12 hotfix #2: ALWAYS override from the chain + # contextvar when one is in scope. The pre-hotfix code only + # filled the field when it was absent from the event dict, + # which broke when ``on_llm_end``'s ``_active_runs[run_id]`` + # lookup missed (run_id drift between the auto-injected + # chat_model callback and an explicit user-supplied one, + # or no matching on_llm_start because the user wrapped the + # LLM call in a non-langgraph stack). In that case + # ``on_llm_end`` leaves the field absent, our ``trace_id`` + # fallback (line 2422) overwrites the event with the chain + # contextvar, but ``parent_trace_id`` stays NULL because the + # previous condition was skipped. The drift was + # investigated via a synthetic diagnostic script + # (``sdk_diag.py``) running on SDK 0.13.7 — cost_events + # received the chain trace_id but not parent_trace_id. + # + # Override semantics: the chain contextvar is the single + # source of truth for "what chain does this event belong + # to". Both ``langgraph.py::on_llm_end``'s caller-set value + # AND a non-langgraph caller's absence resolve to the same + # contextvar. So preferring the contextvar when present is + # idempotent for the happy path AND closes the drift in the + # unhappy path. + # + # The backend's ``cost_events.parent_trace_id`` column + + # unified SELECT third JOIN arm + # (``cs.join_kind = 'parent_trace_id'``) both depend on + # this being present whenever a chain is in scope; without + # it the dashboard falls back to the weaker ``trace_id`` + # arm and LLM rows show empty Model / Tokens / Cost on the + # orchestration row that owns the call. + from nullrun.context import get_trace_id as _get_trace_id + + chain_trace_id = _get_trace_id() + if chain_trace_id: + enriched["parent_trace_id"] = chain_trace_id # Add type if not present if "type" not in enriched: diff --git a/tests/test_drift_fixes_2026_07_04.py b/tests/test_drift_fixes_2026_07_04.py index 75d792b..053c439 100644 --- a/tests/test_drift_fixes_2026_07_04.py +++ b/tests/test_drift_fixes_2026_07_04.py @@ -268,20 +268,41 @@ def test_enrich_event_stamps_parent_trace_id_from_contextvar(self): finally: clear_trace_id() - def test_enrich_event_preserves_caller_set_parent_trace_id(self): - """If the caller already set ``parent_trace_id`` (the - langgraph callback path), ``_enrich_event`` MUST keep their - value rather than overwrite with the contextvar. The - callback may sit inside a deeper span than the contextvar - exposes, so the explicit value wins. + def test_enrich_event_contextvar_overrides_caller_set_parent_trace_id(self): + """Hotfix #2 (2026-07-12): chain contextvar ALWAYS wins + over caller-set parent_trace_id. + + Why override: the pre-hotfix code only filled the field + when it was absent from the event dict, which broke when + ``langgraph.py::on_llm_end``'s ``_active_runs[run_id]`` + lookup missed (run_id drift between the auto-injected + chat_model callback and an explicit user-supplied one, + or no matching ``on_llm_start`` because the user wrapped + the LLM call in a non-langgraph stack). In that case + ``on_llm_end`` leaves the field absent, the ``trace_id`` + fallback (line 2422) overwrites the event with the chain + contextvar, but ``parent_trace_id`` stayed NULL because + the previous condition was skipped. + + Override semantics: the chain contextvar is the single + source of truth for "what chain does this event belong + to". Both the langgraph callback's caller-set value AND + a non-langgraph caller's absence resolve to the same + contextvar; preferring the contextvar when present is + idempotent for the happy path AND closes the drift in + the unhappy path. + + See PR #64 hotfix #2 / diagnostic run 2026-07-12 08:51 + for the full regression context (sdk_diag.py output: + trace_id=cccccccc-... parent_trace_id=NULL on backend + cost_events). """ from nullrun.context import set_trace_id, clear_trace_id from nullrun.runtime import NullRunRuntime - # Contextvar holds the OUTER chain's trace; the langgraph - # callback already stamped the CHILD span's trace (which - # equals the chain trace by SpanContext invariant, but - # ``_enrich_event`` is not supposed to second-guess this). + # Contextvar holds the chain's trace. Even though the + # event dict has a caller-set parent_trace_id, the + # hotfix overrides it with the contextvar. set_trace_id("55555555-6666-7777-8888-999999999999") try: rt = NullRunRuntime(api_key="test-key-12345678", _test_mode=True) @@ -293,7 +314,14 @@ def test_enrich_event_preserves_caller_set_parent_trace_id(self): "parent_trace_id": "explicit-from-callback", } ) - assert enriched["parent_trace_id"] == "explicit-from-callback" + # Contextvar WINS over caller-set (hotfix #2). + assert ( + enriched["parent_trace_id"] + == "55555555-6666-7777-8888-999999999999" + ), ( + f"contextvar must override caller-set parent_trace_id " + f"(hotfix #2): got {enriched['parent_trace_id']!r}" + ) finally: clear_trace_id() @@ -473,6 +501,80 @@ def test_consume_overbudget_preserves_422(self): # F3: fail-CLOSED / fail-OPEN honesty # --------------------------------------------------------------------------- + +class TestEnrichEventParentTraceOverride: + """Hotfix #2: the chain contextvar ALWAYS wins over caller-set + parent_trace_id. Regression coverage for the drift bug where + cost_events.parent_trace_id stayed NULL even though + cost_events.trace_id carried the chain contextvar (chain + contextvar was honored for trace_id via the fallback at line + 2422, but parent_trace_id's "if not in enriched" condition was + skipped when the event arrived without the field set). + """ + + def test_enrich_event_sets_parent_trace_id_when_chain_contextvar_set(self): + """Real-world drift scenario: SDK runtime.track() called + with no parent_trace_id field, chain contextvar set. + Pre-hotfix: parent_trace_id stays absent. Post-hotfix: it + is set to the chain contextvar. + + This is the path that produced trace_id=cccccccc-... / + parent_trace_id=NULL on the prod VPS during the diagnostic + run on 2026-07-12 08:51 UTC. + """ + from nullrun.context import set_trace_id, clear_trace_id + from nullrun.runtime import NullRunRuntime + set_trace_id("cccccccc-1111-2222-3333-444444444444") + try: + rt = NullRunRuntime(api_key="test-key-12345678", _test_mode=True) + # Event WITHOUT parent_trace_id field at all. + enriched = rt._enrich_event( + { + "type": "llm_call", + "model": "gpt-4", + "tokens": 100, + } + ) + assert ( + enriched["parent_trace_id"] + == "cccccccc-1111-2222-3333-444444444444" + ), ( + f"parent_trace_id MUST be stamped from chain contextvar " + f"even when caller did not set it: got " + f"{enriched.get('parent_trace_id')!r}" + ) + # Sanity: trace_id also comes from the same contextvar. + assert enriched["trace_id"] == "cccccccc-1111-2222-3333-444444444444" + finally: + clear_trace_id() + + def test_enrich_event_parent_trace_id_matches_trace_id_in_chain_mode(self): + """SpanContext invariant: parent_trace_id == trace_id when + the event sits inside the chain contextvar (chain trace + spans share the same trace_id across child spans). + """ + from nullrun.context import set_trace_id, clear_trace_id + from nullrun.runtime import NullRunRuntime + set_trace_id("99999999-aaaa-bbbb-cccc-000000000000") + try: + rt = NullRunRuntime(api_key="test-key-12345678", _test_mode=True) + enriched = rt._enrich_event( + { + "type": "llm_call", + "model": "gpt-4", + "tokens": 100, + } + ) + assert enriched["parent_trace_id"] == enriched["trace_id"], ( + f"parent_trace_id should equal trace_id when chain " + f"contextvar is the source: parent={enriched.get('parent_trace_id')!r}, " + f"trace={enriched.get('trace_id')!r}" + ) + finally: + clear_trace_id() + + + class TestFailClosedHonesty: """F3: the SDK reads backend enforcement responses as fail-CLOSED even when they're named with the word