diff --git a/pyproject.toml b/pyproject.toml index 72df018..caef74b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,8 +28,7 @@ name = "nullrun" # the full ``flush_interval`` (5s default). Plus CI hygiene: # pip cache, ``fail-fast`` matrix, ``pytest-xdist -n auto``. No # on-wire change; backends on 1.0.0 keep working unchanged. -version = "0.13.6" -# Long form used by PyPI page meta-description and search snippets. +version = "0.13.7" # Kept under the 200-char preview threshold so the full line is visible # without an "expand" click. Keywords are matched against likely search # queries ("AI agent cost control", "LLM circuit breaker", etc.). diff --git a/src/nullrun/__version__.py b/src/nullrun/__version__.py index ca7bd29..919ba36 100644 --- a/src/nullrun/__version__.py +++ b/src/nullrun/__version__.py @@ -1,5 +1,79 @@ """NullRun Platform SDK. +v3.22 / 0.13.7 (2026-07-12) — wire ``parent_trace_id`` end-to-end on +``/track`` (v3 + legacy batch). + +Pre-fix (0.13.6): ``langgraph.py::on_llm_end`` set +``event["parent_trace_id"]`` on the llm_call cost event when an +LLM call sat inside a chain / agent, but two leaks dropped the +field on the wire: + + 1. ``runtime._enrich_event`` never stamped ``parent_trace_id`` + from the active span contextvar, so non-langgraph integrations + (crewai, autogen, llama_index, plain httpx transport) emitted + the field as ``None``. + 2. ``runtime._build_v3_track_payload`` did NOT map + ``parent_trace_id`` onto the v3 ``/track`` payload, so even + when the langgraph callback set it, the field dropped at the + SDK wire boundary. + +Result on production (VPS Postgres after deploy 2026-07-11): + + SELECT count(*), count(parent_trace_id) + FROM cost_events WHERE created_at > '2026-07-11 17:54:00'; + -- 28 | 0 + +Zero rows carried the parent trace — the backend's unified +SELECT third JOIN arm (``cs.join_kind = 'parent_trace_id'``) never +matched, and the workflow detail "Recent executions" panel showed +empty Model / Tokens / Cost on every orchestration row that owned +an LLM call. + +Fix (no public API change, no wire-format change — the field +was always wire-additive; just stop dropping it on the SDK side): + + 1. ``runtime._enrich_event``: stamp ``parent_trace_id`` from + ``get_trace_id()`` contextvar when the caller did NOT set it + explicitly. The langgraph callback's explicit value wins (no + second-guessing), preserving the existing contract. + + 2. ``runtime._build_v3_track_payload``: map ``parent_trace_id`` + from ``wire_event`` onto the v3 ``/track`` body, mirroring + the existing ``trace_id`` / ``span_id`` handling. + + 3. ``nullrun.context``: add ``set_trace_id`` / + ``reset_trace_id`` / ``clear_trace_id`` helpers. Tests that + pin the trace contextvar (mimicking ``@protect`` blocks) + need a way to set + restore. Matches the existing pattern + of ``set_/get_/clear_server_minted_execution_id``. + +Tests (7 new in ``test_drift_fixes_2026_07_04.py``, all passing): + + - ``test_build_v3_track_payload_includes_parent_trace_id`` + - ``test_build_v3_track_payload_omits_parent_trace_id_when_absent`` + - ``test_enrich_event_stamps_parent_trace_id_from_contextvar`` + - ``test_enrich_event_preserves_caller_set_parent_trace_id`` + - ``test_enrich_event_leaves_parent_trace_id_blank_when_no_contextvar`` + - ``test_enrich_event_omits_empty_string_parent_trace_id`` + - ``test_enrich_event_parent_trace_id_matches_existing_trace_id_field`` + +Verification locally: + + - ``pytest tests/test_drift_fixes_2026_07_04.py`` — 22/22 passed. + - ``pytest tests/ -n auto -q`` — 1142 passed, 1 pre-existing flake + (``test_is_paused_respects_cooldown``, NOT introduced by this + release). + - ``ruff check src/`` — All checks passed. + - ``mypy src/`` — Success: no issues found in 34 source files. + +No public API change. No ``SDK_MIN_VERSION`` bump. Backends on +1.0.0 keep working unchanged. Recommended: 0.13.6 → 0.13.7 +(patch). Required: backend must have ``cost_events.parent_trace_id`` +column from migration 217 (already deployed on prod as of +2026-07-11 12:52 UTC). + +--- + v3.21 / 0.13.6 (2026-07-11) — multi-agent span attachment (parent_trace_id). Pre-fix the langgraph callback's on_llm_start/on_llm_end handlers @@ -417,5 +491,5 @@ """ -__version__ = "0.13.6" +__version__ = "0.13.7" __platform_version__ = "1.0.0" diff --git a/src/nullrun/context.py b/src/nullrun/context.py index 737f3d4..f8a6212 100644 --- a/src/nullrun/context.py +++ b/src/nullrun/context.py @@ -74,6 +74,40 @@ def get_trace_id() -> str | None: return _trace_id_var.get() +def set_trace_id(trace_id: str | None) -> object: + """Pin the current trace_id on the context. + + Used by ``@protect`` blocks and by the langgraph callback + during ``on_chain_start`` to give downstream cost events a + stable parent-trace reference. Returns a token that the caller + passes to :func:`reset_trace_id` to restore the previous value + — this is the ``ContextVar`` contract, see + https://docs.python.org/3/library/contextvars.html#contextvars.ContextVar.set. + + Passing ``None`` clears the field. Tests should pair this with + a try/finally ``reset_trace_id`` to avoid bleeding state into + the next test (we observed this as the root cause of the + 2026-07-11 cross-test WAL-replay flake). + """ + return _trace_id_var.set(trace_id) + + +def reset_trace_id(token: object) -> None: + """Restore the previous trace_id state from a ``set_trace_id`` + token. See :func:`set_trace_id`.""" + _trace_id_var.reset(token) # type: ignore[arg-type] + + +def clear_trace_id() -> None: + """Clear the trace_id contextvar to its default (None). + + Convenience for tests + teardown paths that do not need to + capture the previous value. Equivalent to + ``set_trace_id(None)`` but with no return token to manage. + """ + _trace_id_var.set(None) + + def get_span_id() -> str | None: """Get current span ID from context.""" return _span_id_var.get() diff --git a/src/nullrun/runtime.py b/src/nullrun/runtime.py index e30baa8..8827324 100644 --- a/src/nullrun/runtime.py +++ b/src/nullrun/runtime.py @@ -2507,6 +2507,33 @@ def _enrich_event(self, event: dict[str, Any]) -> dict[str, Any]: if idem_key: enriched["idempotency_key"] = idem_key + # 2026-07-12 (multi-agent span attachment — SDK counterpart at + # nullrun-sdk-python release/0.13.5 commit efff530): + # ``langgraph.py::on_llm_end`` may have already stamped + # ``parent_trace_id`` when an LLM call sits inside a + # chain / agent (we set it from the child SpanContext there). + # When the caller passed a span via ``runtime.track_event`` + # (without going through the langgraph callback) the field is + # absent — fall back to the contextvar so non-langgraph + # integrations (crewai, autogen, llama_index, plain httpx + # transport) get the same wire shape. The backend's + # ``cost_events.parent_trace_id`` column + unified SELECT + # third JOIN arm (``cs.join_kind = 'parent_trace_id'``) both + # depend on this being present whenever a parent span exists + # in scope; without it the dashboard falls back to the + # weaker ``trace_id`` arm and LLM rows show empty + # Model / Tokens / Cost on the orchestration row that owns + # the call. + if "parent_trace_id" not in enriched: + # Re-import to mirror the local-import pattern used + # for get_server_minted_execution_id above — keeps + # the runtime module's eager import graph stable. + from nullrun.context import get_trace_id as _get_trace_id + + parent_trace_id = _get_trace_id() + if parent_trace_id: + enriched["parent_trace_id"] = parent_trace_id + # Add type if not present if "type" not in enriched: enriched["type"] = "event" @@ -3034,6 +3061,18 @@ def _build_v3_track_payload( payload["trace_id"] = wire_event["trace_id"] if "span_id" in wire_event and wire_event["span_id"]: payload["span_id"] = wire_event["span_id"] + # 2026-07-12 (multi-agent span attachment): the orchestration + # trace that owns this LLM call. Stamped by ``_enrich_event`` + # from the active span contextvar (or earlier by + # ``langgraph.py::on_llm_end`` when the call sits inside a chain + # / agent). Backend persists it on ``cost_events.parent_trace_id`` + # and the unified SELECT joins ``traces.trace_id`` directly via + # this column so the workflow detail "Recent executions" panel + # surfaces Model / Tokens / Cost on the orchestration row that + # owns the LLM call. Without this, the dashboard's 4/5-row + # empty-cells problem returns for every multi-agent workflow. + if "parent_trace_id" in wire_event and wire_event["parent_trace_id"]: + payload["parent_trace_id"] = wire_event["parent_trace_id"] # Optional downstream fields preserved verbatim (workflow-level # cost attribution, agent_id, etc.). Backend ignores unknown diff --git a/tests/conftest.py b/tests/conftest.py index 89bc842..5045651 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -168,3 +168,44 @@ def _make(**kwargs): return rt return _make + +@pytest.fixture +def make_test_runtime(monkeypatch, tmp_path): + """Factory for tests that build a real ``NullRunRuntime`` inline + (no ``mock_api`` indirection). + + Pins ``NULLRUN_WAL_PATH`` to a tmp_path-scoped file so the + constructor's ``Transport._replay_from_wal`` never reads the + default ``tempfile.gettempdir()/nullrun.wal`` (which may carry + real on-disk events from a previous test run or parallel + worker and would cause HTTP 401 → ``NullRunAuthError`` in + setup). Mirrors the ``test_runtime`` fixture in + ``test_protect_branches.py`` so all tests that build a runtime + directly get the same isolation. + + Stub ``_do_flush`` / ``_do_flush_locked`` / ``_client`` so any + real network attempt is no-op'd. Reset singleton around the + factory so test ordering is independent. + """ + from unittest.mock import MagicMock + from nullrun.runtime import NullRunRuntime + + NullRunRuntime.reset_instance() + # Pre-pin the WAL path before any runtime can be constructed + # (otherwise the default is captured at first construction). + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) + + def _factory(**overrides): + api_key = overrides.pop("api_key", "test-key-12345678") + rt = NullRunRuntime(api_key=api_key, _test_mode=True) + # Stub the network-facing pieces for tests that build a + # runtime inline (not via ``mock_api``). + rt._transport._do_flush = lambda: None + rt._transport._do_flush_locked = lambda: None + rt._transport._client = MagicMock() + for k, v in overrides.items(): + setattr(rt, k, v) + return rt + + yield _factory + NullRunRuntime.reset_instance() diff --git a/tests/test_drift_fixes_2026_07_04.py b/tests/test_drift_fixes_2026_07_04.py index a4f9a3a..75d792b 100644 --- a/tests/test_drift_fixes_2026_07_04.py +++ b/tests/test_drift_fixes_2026_07_04.py @@ -192,6 +192,185 @@ def test_build_v3_track_payload_omits_idempotency_key_when_absent( assert payload is not None assert "idempotency_key" not in payload + def test_build_v3_track_payload_includes_parent_trace_id(self): + """2026-07-12 (multi-agent span attachment): the v3 /track + payload mapper must surface ``parent_trace_id`` on the wire + when the enriched event carries it. Without this the backend's + ``cost_events.parent_trace_id`` column stays NULL and the + unified SELECT's third JOIN arm (``cs.join_kind = + 'parent_trace_id'``) misses the row — the dashboard falls + back to the weaker ``trace_id`` arm and the workflow detail + "Recent executions" panel shows empty Model / Tokens / Cost + on the orchestration row that owns the LLM call. + """ + from nullrun.runtime import _build_v3_track_payload + + payload = _build_v3_track_payload( + { + "workflow_id": "wf-123", + "tokens": 100, + "trace_id": "11111111-2222-3333-4444-555555555555", + "span_id": "22222222-3333-4444-5555-666666666666", + "parent_trace_id": "33333333-4444-5555-6666-777777777777", + }, + "01926e7a-3b3b-7ddd-9bdd-7f0d3b3b7b3b", + ) + + assert payload is not None + assert payload["parent_trace_id"] == "33333333-4444-5555-6666-777777777777" + # Sanity: existing fields still surface. + assert payload["trace_id"] == "11111111-2222-3333-4444-555555555555" + assert payload["span_id"] == "22222222-3333-4444-5555-666666666666" + + def test_build_v3_track_payload_omits_parent_trace_id_when_absent(self): + """Backward compat: when no parent chain / agent context is + active (single-shot /track outside @protect), the field must + be absent — not an empty string. Backend stores ``None`` / + missing-field identically, so the omission is the right + shape for the "no parent" case. + """ + from nullrun.runtime import _build_v3_track_payload + + payload = _build_v3_track_payload( + { + "workflow_id": "wf-123", + "tokens": 100, + "trace_id": "11111111-2222-3333-4444-555555555555", + }, + "01926e7a-3b3b-7ddd-9bdd-7f0d3b3b7b3b", + ) + + assert payload is not None + assert "parent_trace_id" not in payload + assert payload["trace_id"] == "11111111-2222-3333-4444-555555555555" + + def test_enrich_event_stamps_parent_trace_id_from_contextvar(self): + """When the caller did not pass ``parent_trace_id`` explicitly + on the event dict (e.g. plain httpx transport that does NOT + go through ``langgraph.py::on_llm_end``), ``_enrich_event`` + must stamp the field from the active span contextvar so the + wire shape is consistent regardless of caller integration. + """ + from nullrun.context import set_trace_id, clear_trace_id + from nullrun.runtime import NullRunRuntime + + # Pin the trace contextvar to a known value (mimics + # ``@protect`` block / chain mode). + set_trace_id("44444444-5555-6666-7777-888888888888") + try: + rt = NullRunRuntime(api_key="test-key-12345678", _test_mode=True) + enriched = rt._enrich_event( + {"type": "llm_call", "model": "gpt-4", "tokens": 100} + ) + assert enriched["parent_trace_id"] == ( + "44444444-5555-6666-7777-888888888888" + ) + finally: + clear_trace_id() + + def test_enrich_event_preserves_caller_set_parent_trace_id(self): + """If the caller already set ``parent_trace_id`` (the + langgraph callback path), ``_enrich_event`` MUST keep their + value rather than overwrite with the contextvar. The + callback may sit inside a deeper span than the contextvar + exposes, so the explicit value wins. + """ + from nullrun.context import set_trace_id, clear_trace_id + from nullrun.runtime import NullRunRuntime + + # Contextvar holds the OUTER chain's trace; the langgraph + # callback already stamped the CHILD span's trace (which + # equals the chain trace by SpanContext invariant, but + # ``_enrich_event`` is not supposed to second-guess this). + set_trace_id("55555555-6666-7777-8888-999999999999") + try: + rt = NullRunRuntime(api_key="test-key-12345678", _test_mode=True) + enriched = rt._enrich_event( + { + "type": "llm_call", + "model": "gpt-4", + "tokens": 100, + "parent_trace_id": "explicit-from-callback", + } + ) + assert enriched["parent_trace_id"] == "explicit-from-callback" + finally: + clear_trace_id() + + def test_enrich_event_leaves_parent_trace_id_blank_when_no_contextvar( + self, + ): + """Backward compat: legacy / pre-0.13.6 callers run with no + ``@protect`` block and no chain contextvar set. In that case + ``parent_trace_id`` MUST stay absent — never pick up a stale + value from a previous test, never default to ``trace_id`` + (the backend's JOIN keys off the explicit value, not the + trace_id column). + """ + from nullrun.context import clear_trace_id, set_trace_id + from nullrun.runtime import NullRunRuntime + + clear_trace_id() # belt + braces + try: + set_trace_id(None) + except Exception: + pass + try: + clear_trace_id() + except Exception: + pass + + rt = NullRunRuntime(api_key="test-key-12345678", _test_mode=True) + enriched = rt._enrich_event( + {"type": "llm_call", "model": "gpt-4", "tokens": 100} + ) + assert "parent_trace_id" not in enriched + + def test_enrich_event_omits_empty_string_parent_trace_id(self): + """Empty string ``""`` is a falsy ``parent_trace_id``. Treat + it like None so the wire payload stays clean (backend + parser would otherwise reject the field or store empty + string in a UUID column, depending on path). + """ + from nullrun.context import set_trace_id, clear_trace_id + from nullrun.runtime import NullRunRuntime + + set_trace_id("") # boundary value + try: + rt = NullRunRuntime(api_key="test-key-12345678", _test_mode=True) + enriched = rt._enrich_event( + {"type": "llm_call", "model": "gpt-4", "tokens": 100} + ) + # The contextvar was set to empty string; ``_enrich_event`` + # branches on truthy value, so the field is absent + # (not propagated as empty string). + assert "parent_trace_id" not in enriched + finally: + clear_trace_id() + + def test_enrich_event_parent_trace_id_matches_existing_trace_id_field( + self, + ): + """Invariant (see SpanContext): a child span inherits + ``trace_id`` from its parent and only differs in + ``span_id``. When the contextvar is set, ``parent_trace_id`` + and ``trace_id`` MUST point at the same value. This protects + the backend's JOIN from drifting — see + ``db/mod.rs::get_execution_records_for_workflow``. + """ + from nullrun.context import set_trace_id, clear_trace_id + from nullrun.runtime import NullRunRuntime + + set_trace_id("77777777-8888-9999-aaaa-bbbbbbbbbbbb") + try: + rt = NullRunRuntime(api_key="test-key-12345678", _test_mode=True) + enriched = rt._enrich_event( + {"type": "llm_call", "model": "gpt-4", "tokens": 100} + ) + assert enriched["trace_id"] == enriched["parent_trace_id"] + finally: + clear_trace_id() + # --------------------------------------------------------------------------- # F2: HTTP status_code on every decision exception diff --git a/tests/test_protect_branches.py b/tests/test_protect_branches.py index cd528cd..36cc2ba 100644 --- a/tests/test_protect_branches.py +++ b/tests/test_protect_branches.py @@ -35,11 +35,20 @@ @pytest.fixture -def test_runtime(monkeypatch): +def test_runtime(monkeypatch, tmp_path): """Provide a runtime in test mode so get_runtime returns without authenticating against a real server. + + Replays any WAL left over from previous test runs in a + tmp_path-scoped WAL file so the constructor's + ``_replay_from_wal`` never reads ``~/.nullrun/sdk.wal`` and + flushes real on-disk events to a live API. This avoids the + cross-Python-version flake seen on CI in 2026-07-11 where + 3.11 picked up a stale WAL from a 3.10/3.12 worker that + finished without explicitly clearing it. """ monkeypatch.setenv("NULLRUN_API_KEY", "test-key-12345678") + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) NullRunRuntime.reset_instance() rt = NullRunRuntime(api_key="test-key-12345678", _test_mode=True) rt.organization_id = "org-1" @@ -437,13 +446,13 @@ def f(): @pytest.mark.asyncio -async def test_protect_async_kill_re_raises_WorkflowKilledInterrupt(): +async def test_protect_async_kill_re_raises_WorkflowKilledInterrupt(make_test_runtime): """Async wrapper does NOT unify — kill signal propagates as-is so async frameworks can interrupt the event loop cleanly. """ from nullrun import decorators as dec_mod - rt = NullRunRuntime(api_key="test-key-12345678", _test_mode=True) + rt = make_test_runtime() rt.track_event = MagicMock() rt.check_control_plane = MagicMock( side_effect=WorkflowKilledInterrupt(workflow_id="wf-1", reason="x") @@ -542,12 +551,12 @@ def test_get_protected_runtime_returns_runtime(test_runtime): assert decorators.get_protected_runtime() is rt -def test_get_protected_runtime_falls_back_to_get_runtime(test_runtime, monkeypatch): +def test_get_protected_runtime_falls_back_to_get_runtime(monkeypatch, make_test_runtime): """When the decorator slot is empty, fall back to the global singleton.""" from nullrun import decorators decorators._runtime = None - NullRunRuntime._instance = NullRunRuntime(api_key="test-key-12345678", _test_mode=True) + NullRunRuntime._instance = make_test_runtime() try: out = decorators.get_protected_runtime() assert out is NullRunRuntime._instance diff --git a/tests/test_runtime_branches.py b/tests/test_runtime_branches.py index 3c034b1..d782618 100644 --- a/tests/test_runtime_branches.py +++ b/tests/test_runtime_branches.py @@ -30,7 +30,21 @@ def _reset_singleton(): def _make_test_runtime() -> NullRunRuntime: """Build a runtime that skips network I/O and returns from ``_authenticate`` with a stub organisation id. + + Pins ``NULLRUN_WAL_PATH`` to a per-call tmp dir so the + constructor's ``Transport._replay_from_wal`` never picks up a + stale WAL from a previous test run (which would replay real + events to a live API and cause HTTP 401 in setup). See + ``conftest::make_test_runtime`` for the fixture equivalent. """ + # Per-call isolation: each helper invocation owns its WAL. + # ``setdefault`` so an outer session-level pinning (from + # ``make_test_runtime`` fixture) is preserved if already set. + import os + import tempfile + if not os.environ.get("NULLRUN_WAL_PATH"): + wal_dir = tempfile.mkdtemp(prefix="nullrun-test-wal-") + os.environ["NULLRUN_WAL_PATH"] = os.path.join(wal_dir, "sdk.wal") rt = NullRunRuntime(api_key="test-key-12345678", _test_mode=True) rt.organization_id = "org-1" rt.workflow_id = "wf-1" @@ -357,8 +371,9 @@ def _trigger_shutdown(): # ─── get_instance credential rotation ────────────────────────────── -def test_get_instance_returns_singleton_when_no_change(monkeypatch): +def test_get_instance_returns_singleton_when_no_change(monkeypatch, tmp_path): monkeypatch.setenv("NULLRUN_API_KEY", "test-key-12345678") + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) NullRunRuntime.reset_instance() rt1 = NullRunRuntime(api_key="test-key-12345678", _test_mode=True) NullRunRuntime._instance = rt1 @@ -372,7 +387,16 @@ def test_get_instance_returns_singleton_when_no_change(monkeypatch): def _make_runtime_with_mocked_auth() -> NullRunRuntime: """Build a test-mode runtime and stub the transport client.post so we can drive ``_authenticate`` deterministically. + + Pins ``NULLRUN_WAL_PATH`` per call so we never read a stale + WAL from a previous run. ``setdefault`` preserves any + outer-session pinning set by a fixture. """ + import os + import tempfile + if not os.environ.get("NULLRUN_WAL_PATH"): + wal_dir = tempfile.mkdtemp(prefix="nullrun-test-wal-") + os.environ["NULLRUN_WAL_PATH"] = os.path.join(wal_dir, "sdk.wal") rt = NullRunRuntime(api_key="test-key-12345678", _test_mode=True) rt._transport._client = MagicMock() rt._fetch_policy = MagicMock() diff --git a/tests/test_toolbox_langgraph.py b/tests/test_toolbox_langgraph.py index 83d89e2..bab8dcf 100644 --- a/tests/test_toolbox_langgraph.py +++ b/tests/test_toolbox_langgraph.py @@ -15,10 +15,18 @@ @pytest.fixture(autouse=True) -def _test_runtime(monkeypatch): +def _test_runtime(monkeypatch, tmp_path): """Provide a runtime in test mode so get_runtime returns without - authenticating against a real server.""" + authenticating against a real server. + + Pins ``NULLRUN_WAL_PATH`` to a tmp_path-scoped file so the + constructor's ``Transport._replay_from_wal`` never picks up + a stale WAL left over from a previous test run (which would + replay real events to a live API and cause HTTP 401 in + setup). Mirrors ``conftest::make_test_runtime``. + """ monkeypatch.setenv("NULLRUN_API_KEY", "test-key-12345678") + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) NullRunRuntime.reset_instance() # Pre-build a test-mode singleton so get_runtime returns it without # hitting the network. Construct directly and store on the singleton