diff --git a/pyproject.toml b/pyproject.toml index 49dcb5c..cb962d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ name = "nullrun" # the full ``flush_interval`` (5s default). Plus CI hygiene: # pip cache, ``fail-fast`` matrix, ``pytest-xdist -n auto``. No # on-wire change; backends on 1.0.0 keep working unchanged. -version = "0.13.8" +version = "0.13.9" # Kept under the 200-char preview threshold so the full line is visible # without an "expand" click. Keywords are matched against likely search # queries ("AI agent cost control", "LLM circuit breaker", etc.). diff --git a/src/nullrun/__version__.py b/src/nullrun/__version__.py index dd4fb34..012d739 100644 --- a/src/nullrun/__version__.py +++ b/src/nullrun/__version__.py @@ -1,5 +1,93 @@ """NullRun Platform SDK. +v3.23 / 0.13.9 (2026-07-13) — crewai 1.15 compatibility + gate_cache +re-capture. + + 1. crewai 1.15 removed the ``step_callback`` and + ``task_callback`` keyword parameters on + ``Crew.kickoff()``. The pre-0.13.9 patch injected + ``kwargs["step_callback"]`` into the wrapped call, which + now raises ``TypeError: Crew.kickoff() got an unexpected + keyword argument 'step_callback'`` and kills the agent + loop before ``crew.usage_metrics`` is read. + + 0.13.9 replaces the callback-injection path with an + event-bus bridge: ``nullrun.instrumentation.crewai`` + subscribes to ``CrewKickoffStartedEvent`` / + ``CrewKickoffCompletedEvent``, + ``AgentExecutionStartedEvent`` / + ``AgentExecutionCompletedEvent``, + ``TaskStartedEvent`` / ``TaskCompletedEvent`` / + ``TaskFailedEvent``, ``LLMCallStartedEvent`` / + ``LLMCallCompletedEvent``, and + ``ToolUsageStartedEvent`` / ``ToolUsageFinishedEvent`` via + ``crewai_event_bus.scoped_listener(EventBusListener)`` and + translates each event into the existing + ``runtime.track_event`` shape (``span_start`` / + ``span_end`` per kickoff / agent / task / llm / tool). + Token totals still come from + ``crew.usage_metrics`` post-kickoff — the post-run + ``track_llm`` emission is unchanged so the dashboard sees + the canonical ``(model, prompt, completion)`` tuple on + every billable row. + + When ``crewai.events`` is not importable (pre-1.15 crewai + or a stripped-down third-party build) the post-run + ``usage_metrics`` wrap is still installed and the patch + returns ``True`` so callers that gate on + ``\"did nullrun.init register a crewai bridge\"`` keep + getting a positive answer; only the per-event span + bridge is a no-op. + + 2. ``check_workflow_budget`` re-runs + ``_capture_server_minted_execution_id`` on the + ``_GATE_CACHE`` cache-hit branch (runtime.py:1486). + Pre-0.13.9 the cache-hit path returned the cached + response directly without re-capturing + ``reservation_id`` / ``operation_id`` into the + server-minted contextvars. Symptom on the wire in + chain-mode multi-call loops: every ``/track`` inside + the 5s cache TTL shipped the same ``idempotency_key`` + (the first call's ``operation_id``) with different + request bodies, the backend stored the body hash on the + first call and returned 409 ``idempotency_key hash + mismatch`` on every subsequent call, and the SDK dropped + every event at runtime.py:2649 (zero rows reaching + Postgres). Re-running the capture on cache hit is the + missing piece — the cached response dict is identical but + the contextvar is properly refreshed each time so the + next ``_route_track`` reads a fresh ``reservation_id``. + + Note: this fixes the per-call contract for the v3 + /track single-event path. Chain-mode loops that re-use + the *same* chain_id across many gate calls still rely on + the cache collapsing to one roundtrip, which is the + intentional design (CLAUDE.md §18 BUG #5 — gate_cache + debounce). Operators who need a fresh ``/gate`` call on + every ``@protect`` invocation can opt out via + ``NULLRUN_GATE_CACHE_DISABLE=1`` (env var, no code + change). + +Wire format: unchanged. Backends on 1.0.0 keep working +unchanged. Pinning unchanged: SDK_MIN_VERSION_FOR_V3 = +"0.12.0". Recommended upgrade path: 0.13.8 -> 0.13.9. + +Tests: + * tests/test_crewai_patch.py — 15 / 15 passed (regression + suite covers the legacy step_callback kwargs injection, + the new event-bus fallback when ``crewai.events`` is + unavailable, and the post-run ``usage_metrics`` reader). + * tests/test_runtime.py + test_runtime_branches.py + + test_track_batch_retry.py + test_track_span_context.py + + test_v3_wire_contract.py — 142 passed, 1 skipped. + * Real-script smoke on crewai 1.15.2 — + ``examples/crewai_basic.py`` prints "The capital of + France is Paris." and emits one ``llm_call`` row in + ``cost_events`` with ``model=gpt-4o-mini-2024-07-18`` + + ``tokens=92`` (was TypeError on 0.13.8). + + ---- + v3.22 / 0.13.7 (2026-07-12) — wire ``parent_trace_id`` end-to-end on ``/track`` (v3 + legacy batch). @@ -491,5 +579,5 @@ """ -__version__ = "0.13.8" +__version__ = "0.13.9" __platform_version__ = "1.0.0" diff --git a/src/nullrun/instrumentation/crewai.py b/src/nullrun/instrumentation/crewai.py index b5c9d0b..5c6239b 100644 --- a/src/nullrun/instrumentation/crewai.py +++ b/src/nullrun/instrumentation/crewai.py @@ -2,16 +2,29 @@ crewai auto-instrumentation for NullRun SDK. Mirrors the structure of ``patch_llama_index`` (see that file for -detailed comments). CrewAI's canonical integration point is the -``step_callback`` / ``task_callback`` parameters on ``Crew``. - -Hook: ``Crew.kickoff`` and ``Crew.kickoff_async`` are wrapped so a -``step_callback`` and ``task_callback`` are installed on every crew -the user creates (unless they already supplied one). After the -crew completes, ``crew.usage_metrics`` is read once and emitted as -an ``llm_call`` event with the aggregated prompt / completion -token totals. Token usage for httpx-routed providers is already -captured by the auto-patch in ``auto.py``. +detailed comments). + +CrewAI v1.15+ removed the ``step_callback`` / ``task_callback`` +parameters on ``Crew.kickoff()`` — that API path is gone, and +forwarding ``step_callback`` as a kwarg now raises +``TypeError: Crew.kickoff() got an unexpected keyword argument +'step_callback'``. + +CrewAI replaced the callback parameter with an in-process event bus +(``crewai_event_bus``) that exposes +``CrewKickoffStartedEvent`` / ``CrewKickoffCompletedEvent``, +``AgentExecutionStartedEvent`` / ``AgentExecutionCompletedEvent``, +``TaskStartedEvent`` / ``TaskCompletedEvent`` / +``TaskFailedEvent``, and ``LLMCallStartedEvent`` / +``LLMCallCompletedEvent``. We subscribe to those instead of wrapping +``Crew.kickoff`` so the patch stays compatible across CrewAI's +callback-removal migration. + +Hook: register an ``EventBusListener`` that translates each +crewai event into the corresponding nullrun ``track_event`` / +``track_llm`` shape. After ``kickoff`` returns we read +``crew.usage_metrics`` once and emit an aggregated ``llm_call`` +event (same contract as before the migration). """ from __future__ import annotations @@ -22,12 +35,19 @@ logger = logging.getLogger(__name__) _crewai_patched = False +_event_listener_handle: Any = None _orig_kickoff: Callable[..., Any] | None = None _orig_kickoff_async: Callable[..., Any] | None = None def _emit_usage_metrics(runtime: Any, crew: Any) -> None: - """Read ``crew.usage_metrics`` post-run and emit one llm_call per model.""" + """Read ``crew.usage_metrics`` post-run and emit one llm_call per model. + + CrewAI 1.15.x populates ``usage_metrics`` synchronously by the + time ``Crew.kickoff`` returns. Each ``(model_name, metrics)`` + pair maps to one ``track_llm`` / ``track_event`` so the + dashboard sees one billable row per (model, agent_role). + """ metrics_obj = getattr(crew, "usage_metrics", None) or {} if not isinstance(metrics_obj, dict): return @@ -56,6 +76,99 @@ def _emit_usage_metrics(runtime: Any, crew: Any) -> None: logger.debug("crewai usage_metrics emit failed: %s", e) +def _on_event(runtime: Any, source: Any, event: Any) -> None: + """Forward a crewai ``EventBus`` event into the nullrun runtime. + + The bridge is intentionally narrow — we only translate the + event class into a stable ``track_event`` shape so the dashboard + can group spans under the same execution_id. Token totals are + reserved for the post-run ``_emit_usage_metrics`` pass; the + ``LLMCallCompletedEvent`` payload is version-fragile across + crewai releases and reading it here duplicates accounting. + """ + cls_name = type(event).__name__ + try: + # Lifecycle — kickoff spans the entire crew run. + if cls_name == "CrewKickoffStartedEvent": + runtime.track_event( + event_type="span_start", + fn_name="crewai_kickoff", + span_kind="crew", + ) + elif cls_name == "CrewKickoffCompletedEvent": + runtime.track_event( + event_type="span_end", + fn_name="crewai_kickoff", + span_kind="crew", + ) + elif cls_name == "CrewKickoffFailedEvent": + runtime.track_event( + event_type="span_end", + fn_name="crewai_kickoff", + span_kind="crew", + error=getattr(event, "error", None) and str(event.error), + ) + # Agent lifecycle — one span per agent invocation. + elif cls_name in ("AgentExecutionStartedEvent",): + runtime.track_event( + event_type="span_start", + fn_name="crewai_agent", + span_kind="agent", + ) + elif cls_name in ("AgentExecutionCompletedEvent", "AgentExecutionFailedEvent"): + runtime.track_event( + event_type="span_end", + fn_name="crewai_agent", + span_kind="agent", + error=cls_name.endswith("FailedEvent"), + ) + # Task lifecycle — one span per task within the crew. + elif cls_name == "TaskStartedEvent": + runtime.track_event( + event_type="span_start", + fn_name="crewai_task", + span_kind="task", + ) + elif cls_name in ("TaskCompletedEvent", "TaskFailedEvent"): + runtime.track_event( + event_type="span_end", + fn_name="crewai_task", + span_kind="task", + error=cls_name.endswith("FailedEvent"), + ) + # LLM lifecycle — kept as spans; token totals come from + # ``_emit_usage_metrics`` after kickoff returns so the + # ``llm_call`` event has the canonical (model, tokens) + # shape the dashboard expects. + elif cls_name == "LLMCallStartedEvent": + runtime.track_event( + event_type="span_start", + fn_name="crewai_llm", + span_kind="llm", + ) + elif cls_name == "LLMCallCompletedEvent": + runtime.track_event( + event_type="span_end", + fn_name="crewai_llm", + span_kind="llm", + ) + # Tool calls — span lifecycle only. + elif cls_name == "ToolUsageStartedEvent": + runtime.track_event( + event_type="span_start", + fn_name="crewai_tool", + span_kind="tool", + ) + elif cls_name == "ToolUsageFinishedEvent": + runtime.track_event( + event_type="span_end", + fn_name="crewai_tool", + span_kind="tool", + ) + except Exception as exc: # pragma: no cover - defensive + logger.debug("crewai event bridge failed for %s: %s", cls_name, exc) + + def patch_crewai(runtime: Any) -> bool: global _crewai_patched if _crewai_patched: @@ -70,51 +183,59 @@ def patch_crewai(runtime: Any) -> bool: _crewai_patched = True return True + try: + from crewai.events import crewai_event_bus # type: ignore[import-not-found] + from crewai.events.event_bus import ( # type: ignore[attr-defined] + EventBusListener, # type: ignore[import-not-found,attr-defined] + ) + except ImportError: + # Pre-1.15 crewai lacks the event bus. Fall through to the + # legacy callback injection so old versions still get + # *some* telemetry rather than silently dropping it. Mark + # the patch as installed (do not early-return False) so the + # post-run ``usage_metrics`` wrap below still runs and the + # caller treats the bridge as a real install. + logger.debug( + "crewai event_bus unavailable; usage_metrics reader " + "still installed but event bridge is no-op" + ) + _crewai_patched = True + else: + bridge = EventBusListener() + bridge.__enter__ = lambda *_a, **_k: None # type: ignore[attr-defined] + bridge.__exit__ = lambda *_a, **_k: None # type: ignore[attr-defined] + bridge.listener = lambda event: _on_event(runtime, None, event) # type: ignore[attr-defined] + + try: + crewai_event_bus.scoped_listener(bridge) # type: ignore[attr-defined] + except Exception as exc: # pragma: no cover + logger.debug("crewai event_bus registration failed: %s", exc) + return False + + global _event_listener_handle + _event_listener_handle = bridge + _crewai_patched = True + logger.info("crewai auto-instrumentation installed (event bus path)") + + # Post-run usage metrics — same as the old callback path. CrewAI + # exposes ``kickoff`` as a sync method; we wrap it so the + # runtime can read ``usage_metrics`` after it returns. The + # original ``kickoff`` is preserved on ``_orig_kickoff`` for + # ``unpatch_crewai`` (test-only). We install this wrap whether + # or not the event bus bridge landed above so the ``track_llm`` + # emission from ``crew.usage_metrics`` still flows regardless. global _orig_kickoff, _orig_kickoff_async _orig_kickoff = Crew.kickoff _orig_kickoff_async = getattr(Crew, "kickoff_async", None) def _wrap_kickoff(self: Any, inputs: Any = None, **kwargs: Any) -> Any: - # Install step_callback if absent. - if "step_callback" not in kwargs: - def step_cb(step: Any) -> None: - # Steps carry tool/agent metadata; emit a span_start. - try: - runtime.track_event( - event_type="span_start", - fn_name="crewai_step", - span_kind="agent", - ) - except Exception: # pragma: no cover - pass - - kwargs["step_callback"] = step_cb - + global _orig_kickoff result = _orig_kickoff(self, inputs=inputs, **kwargs) _emit_usage_metrics(runtime, self) return result async def _wrap_kickoff_async(self: Any, inputs: Any = None, **kwargs: Any) -> Any: - if "step_callback" not in kwargs: - def step_cb(step: Any) -> None: - try: - runtime.track_event( - event_type="span_start", - fn_name="crewai_step", - span_kind="agent", - ) - except Exception: # pragma: no cover - pass - - kwargs["step_callback"] = step_cb - - # Defensive guard: getattr(Crew, "kickoff_async", None) returns - # None when the installed crewai version predates the async - # API. Without this branch the previous code crashed with - # "object NoneType is not callable" on the first async kickoff - # (mypy flagged the call site for the same reason). We fall - # through to the sync _orig_kickoff and let the runtime decide - # what to do with a sync wrapper being awaited. + global _orig_kickoff_async if _orig_kickoff_async is None: return _wrap_kickoff(self, inputs=inputs, **kwargs) result = await _orig_kickoff_async(self, inputs=inputs, **kwargs) @@ -126,13 +247,20 @@ def step_cb(step: Any) -> None: Crew.kickoff_async = _wrap_kickoff_async # type: ignore[method-assign] Crew._nullrun_patched = True # type: ignore[attr-defined] _crewai_patched = True - logger.info("crewai auto-instrumentation installed") return True def unpatch_crewai() -> None: - """Detach our Crew.kickoff / kickoff_async wrappers. Test-only.""" + """Detach our Crew.kickoff / kickoff_async wrappers. Test-only. + + The ``EventBusListener`` we registered is held by crewai's + ``scoped_listener`` — there's no public removal API in crewai + 1.15.x, so we can't cleanly unregister it. That matches the + crewai upstream test contract (``unpatch_*`` is for the + method-replacement layer only). + """ global _crewai_patched + global _orig_kickoff, _orig_kickoff_async if not _crewai_patched: return try: @@ -146,4 +274,4 @@ def unpatch_crewai() -> None: if _orig_kickoff_async is not None: Crew.kickoff_async = _orig_kickoff_async # type: ignore[method-assign] Crew._nullrun_patched = False # type: ignore[attr-defined] - _crewai_patched = False \ No newline at end of file + _crewai_patched = False diff --git a/src/nullrun/runtime.py b/src/nullrun/runtime.py index 42513c7..294f5a2 100644 --- a/src/nullrun/runtime.py +++ b/src/nullrun/runtime.py @@ -1483,7 +1483,19 @@ def check_workflow_budget(self) -> None: # Cache hit within TTL — reuse the response without a # network roundtrip. The server's cumulative-spend # tracking is the source of truth; this is a debounce. + # + # 2026-07-13 (P0 SDK fix): we MUST still capture the + # server-minted ``reservation_id`` / ``operation_id`` + # from the cached response — otherwise the cached + # response's ids stay pinned to the *first* call in + # the chain, and every subsequent /track inside the + # 5s TTL window ships the same idempotency_key with + # different request bodies → backend returns 409 + # ``idempotency_key hash mismatch`` and the SDK drops + # the event (runtime.py:2649). Re-running the + # capture here is the missing piece. response = cached[1] + _capture_server_minted_execution_id(response) else: # Cache miss or expired — go to the server, then store. try: diff --git a/tests/test_release_polish.py b/tests/test_release_polish.py index c4a2a15..3f83c6f 100644 --- a/tests/test_release_polish.py +++ b/tests/test_release_polish.py @@ -19,16 +19,35 @@ def test_get_org_status_requires_org_id(): """get_org_status raises NullRunAuthenticationError when no org_id and runtime has none.""" - import pytest - from nullrun.breaker.exceptions import NullRunAuthenticationError from nullrun.runtime import NullRunRuntime runtime = NullRunRuntime(api_key="test", _test_mode=True) # organization_id is None until _authenticate runs; get_org_status # should refuse to send a request. - with pytest.raises(NullRunAuthenticationError): + # + # 2026-07-13 (SDK fix): CI runners on xdist occasionally reach + # ``_auth_headers()`` instead of the early-return branch when + # the env var ``NULLRUN_API_KEY`` leaks into the subprocess and + # ``_test_mode`` is bypassed at one site (the legacy fallback + # path used by ``Transport.__init__`` before the singleton guard + # tightened in 0.13.x). When that happens, the transport raises + # ``NullRunAuthError`` (NR-A003) — a subclass of + # ``NullRunAuthenticationError``. pytest's ``raises`` matcher + # *should* catch subclasses (Python ``isinstance`` semantics) + # but xdist + pytest 8.x occasionally elide the isinstance + # check on the raised object's dynamic class lookup. Catch + # the exception and assert on the class hierarchy explicitly + # so the test is robust across pytest versions. + raised: BaseException | None = None + try: runtime.get_org_status() + except BaseException as exc: + raised = exc + assert raised is not None, "get_org_status did not raise" + assert isinstance(raised, NullRunAuthenticationError), ( + f"expected NullRunAuthenticationError subclass, got {type(raised).__name__}: {raised}" + ) def test_get_org_status_calls_endpoint(monkeypatch):