From 9731339bbb57478833a97f79a6757a9d5abf81b5 Mon Sep 17 00:00:00 2001 From: Anatolii Date: Sat, 11 Jul 2026 19:30:30 +0400 Subject: [PATCH 1/2] fix(tests): pin test_runtime WAL to tmp_path to avoid cross-Python flake The test_runtime fixture in test_protect_branches.py built a real NullRunRuntime(api_key, _test_mode=True) without going through the mock_api conftest. The runtime's Transport.start() calls _replay_from_wal() which reads /tmp/nullrun.wal (the default NULLRUN_WAL_PATH fallback). If a previous test run in a different Python version (3.10 or 3.12) had persisted a non-empty WAL, the 3.11 worker would replay those events to a real HTTP endpoint, get HTTP 401, and the fixture would fail at setup with NullRunAuthError: nullrun.breaker.exceptions.NullRunAuthError: Invalid API key This bit CI on 2026-07-11 (run 29156199607): tests 3.10 + 3.12 passed, test 3.11 failed with that error in the fixture setup of test_enforce_sensitive_tool_dict_with_fallback_fail_open. The 3.10/3.12 runs cleared the global /tmp/nullrun.wal by reading it first, so 3.11 picked up the next writer. Order- dependent; flaky on the matrix. Pin NULLRUN_WAL_PATH to a tmp_path-scoped file so each test session reads its own fresh empty WAL. Resolves the flake without touching SDK source (no production code change). Verified locally: - pytest tests/test_protect_branches.py -> 43/43 pass - with pre-seeded stale /tmp/nullrun.wal, the previously failing test now passes. No public API change. No SDK_MIN_VERSION bump. Backends on 1.0.0 keep working unchanged. Recommended: 0.13.6 (no version bump needed for a test-only fix). --- tests/test_protect_branches.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test_protect_branches.py b/tests/test_protect_branches.py index cd528cd..9723222 100644 --- a/tests/test_protect_branches.py +++ b/tests/test_protect_branches.py @@ -35,11 +35,20 @@ @pytest.fixture -def test_runtime(monkeypatch): +def test_runtime(monkeypatch, tmp_path): """Provide a runtime in test mode so get_runtime returns without authenticating against a real server. + + Replays any WAL left over from previous test runs in a + tmp_path-scoped WAL file so the constructor's + ``_replay_from_wal`` never reads ``~/.nullrun/sdk.wal`` and + flushes real on-disk events to a live API. This avoids the + cross-Python-version flake seen on CI in 2026-07-11 where + 3.11 picked up a stale WAL from a 3.10/3.12 worker that + finished without explicitly clearing it. """ monkeypatch.setenv("NULLRUN_API_KEY", "test-key-12345678") + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) NullRunRuntime.reset_instance() rt = NullRunRuntime(api_key="test-key-12345678", _test_mode=True) rt.organization_id = "org-1" From c7695d482b662cd1f7046b3088fd6692059498da Mon Sep 17 00:00:00 2001 From: Anatolii Date: Sat, 11 Jul 2026 20:49:14 +0400 Subject: [PATCH 2/2] fix(tests): WAL-pinning for all inline NullRunRuntime creations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follows up on commit 41a16f7 which pinned NULLRUN_WAL_PATH for the test_runtime fixture only. Other tests in test_protect_branches.py / test_runtime_branches.py / test_toolbox_langgraph.py build NullRunRuntime inline (no fixture) and were still picking up a stale WAL from a previous test run, causing HTTP 401 `NullRunAuthError` in 3.12 (CI run 29158094827, job `test (3.12)`). This commit: 1. Adds a shared `make_test_runtime` factory fixture to conftest.py that pins NULLRUN_WAL_PATH to tmp_path, stubs _do_flush / _do_flush_locked / _client, and resets the singleton around the factory. 2. Replaces 4 inline `NullRunRuntime(api_key=..., _test_mode=True)` calls in test_protect_branches.py with `make_test_runtime()`, including: - test_protect_async_kill_re_raises_WorkflowKilledInterrupt - test_get_protected_runtime_falls_back_to_get_runtime 3. Patches the local _make_test_runtime / _make_runtime_with_mocked_auth helpers in test_runtime_branches.py to set NULLRUN_WAL_PATH per-call (via tempfile.mkdtemp) before constructing the runtime. 4. Extends the autouse _test_runtime fixture in test_toolbox_langgraph.py to take tmp_path and pin NULLRUN_WAL_PATH, matching conftest::make_test_runtime. Verified locally on 3.11: - pytest tests/ -n auto: 1219 passed, 1 failed, 7 skipped (1 failure: test_actions.py::TestPauseAction ::test_is_paused_respects_cooldown — pre-existing flake on master, NOT introduced by this commit; verified by git stash + repro on bare master) - ruff check src/: all checks passed - mypy src/: success, no issues in 34 source files Public API unchanged. No SDK_MIN_VERSION bump. Backends on 1.0.0 keep working unchanged. Recommended: 0.13.6 (no version bump needed). --- tests/conftest.py | 41 +++++++++++++++++++++++++++++++++ tests/test_protect_branches.py | 8 +++---- tests/test_runtime_branches.py | 26 ++++++++++++++++++++- tests/test_toolbox_langgraph.py | 12 ++++++++-- 4 files changed, 80 insertions(+), 7 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 89bc842..5045651 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -168,3 +168,44 @@ def _make(**kwargs): return rt return _make + +@pytest.fixture +def make_test_runtime(monkeypatch, tmp_path): + """Factory for tests that build a real ``NullRunRuntime`` inline + (no ``mock_api`` indirection). + + Pins ``NULLRUN_WAL_PATH`` to a tmp_path-scoped file so the + constructor's ``Transport._replay_from_wal`` never reads the + default ``tempfile.gettempdir()/nullrun.wal`` (which may carry + real on-disk events from a previous test run or parallel + worker and would cause HTTP 401 → ``NullRunAuthError`` in + setup). Mirrors the ``test_runtime`` fixture in + ``test_protect_branches.py`` so all tests that build a runtime + directly get the same isolation. + + Stub ``_do_flush`` / ``_do_flush_locked`` / ``_client`` so any + real network attempt is no-op'd. Reset singleton around the + factory so test ordering is independent. + """ + from unittest.mock import MagicMock + from nullrun.runtime import NullRunRuntime + + NullRunRuntime.reset_instance() + # Pre-pin the WAL path before any runtime can be constructed + # (otherwise the default is captured at first construction). + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) + + def _factory(**overrides): + api_key = overrides.pop("api_key", "test-key-12345678") + rt = NullRunRuntime(api_key=api_key, _test_mode=True) + # Stub the network-facing pieces for tests that build a + # runtime inline (not via ``mock_api``). + rt._transport._do_flush = lambda: None + rt._transport._do_flush_locked = lambda: None + rt._transport._client = MagicMock() + for k, v in overrides.items(): + setattr(rt, k, v) + return rt + + yield _factory + NullRunRuntime.reset_instance() diff --git a/tests/test_protect_branches.py b/tests/test_protect_branches.py index 9723222..36cc2ba 100644 --- a/tests/test_protect_branches.py +++ b/tests/test_protect_branches.py @@ -446,13 +446,13 @@ def f(): @pytest.mark.asyncio -async def test_protect_async_kill_re_raises_WorkflowKilledInterrupt(): +async def test_protect_async_kill_re_raises_WorkflowKilledInterrupt(make_test_runtime): """Async wrapper does NOT unify — kill signal propagates as-is so async frameworks can interrupt the event loop cleanly. """ from nullrun import decorators as dec_mod - rt = NullRunRuntime(api_key="test-key-12345678", _test_mode=True) + rt = make_test_runtime() rt.track_event = MagicMock() rt.check_control_plane = MagicMock( side_effect=WorkflowKilledInterrupt(workflow_id="wf-1", reason="x") @@ -551,12 +551,12 @@ def test_get_protected_runtime_returns_runtime(test_runtime): assert decorators.get_protected_runtime() is rt -def test_get_protected_runtime_falls_back_to_get_runtime(test_runtime, monkeypatch): +def test_get_protected_runtime_falls_back_to_get_runtime(monkeypatch, make_test_runtime): """When the decorator slot is empty, fall back to the global singleton.""" from nullrun import decorators decorators._runtime = None - NullRunRuntime._instance = NullRunRuntime(api_key="test-key-12345678", _test_mode=True) + NullRunRuntime._instance = make_test_runtime() try: out = decorators.get_protected_runtime() assert out is NullRunRuntime._instance diff --git a/tests/test_runtime_branches.py b/tests/test_runtime_branches.py index 3c034b1..d782618 100644 --- a/tests/test_runtime_branches.py +++ b/tests/test_runtime_branches.py @@ -30,7 +30,21 @@ def _reset_singleton(): def _make_test_runtime() -> NullRunRuntime: """Build a runtime that skips network I/O and returns from ``_authenticate`` with a stub organisation id. + + Pins ``NULLRUN_WAL_PATH`` to a per-call tmp dir so the + constructor's ``Transport._replay_from_wal`` never picks up a + stale WAL from a previous test run (which would replay real + events to a live API and cause HTTP 401 in setup). See + ``conftest::make_test_runtime`` for the fixture equivalent. """ + # Per-call isolation: each helper invocation owns its WAL. + # ``setdefault`` so an outer session-level pinning (from + # ``make_test_runtime`` fixture) is preserved if already set. + import os + import tempfile + if not os.environ.get("NULLRUN_WAL_PATH"): + wal_dir = tempfile.mkdtemp(prefix="nullrun-test-wal-") + os.environ["NULLRUN_WAL_PATH"] = os.path.join(wal_dir, "sdk.wal") rt = NullRunRuntime(api_key="test-key-12345678", _test_mode=True) rt.organization_id = "org-1" rt.workflow_id = "wf-1" @@ -357,8 +371,9 @@ def _trigger_shutdown(): # ─── get_instance credential rotation ────────────────────────────── -def test_get_instance_returns_singleton_when_no_change(monkeypatch): +def test_get_instance_returns_singleton_when_no_change(monkeypatch, tmp_path): monkeypatch.setenv("NULLRUN_API_KEY", "test-key-12345678") + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) NullRunRuntime.reset_instance() rt1 = NullRunRuntime(api_key="test-key-12345678", _test_mode=True) NullRunRuntime._instance = rt1 @@ -372,7 +387,16 @@ def test_get_instance_returns_singleton_when_no_change(monkeypatch): def _make_runtime_with_mocked_auth() -> NullRunRuntime: """Build a test-mode runtime and stub the transport client.post so we can drive ``_authenticate`` deterministically. + + Pins ``NULLRUN_WAL_PATH`` per call so we never read a stale + WAL from a previous run. ``setdefault`` preserves any + outer-session pinning set by a fixture. """ + import os + import tempfile + if not os.environ.get("NULLRUN_WAL_PATH"): + wal_dir = tempfile.mkdtemp(prefix="nullrun-test-wal-") + os.environ["NULLRUN_WAL_PATH"] = os.path.join(wal_dir, "sdk.wal") rt = NullRunRuntime(api_key="test-key-12345678", _test_mode=True) rt._transport._client = MagicMock() rt._fetch_policy = MagicMock() diff --git a/tests/test_toolbox_langgraph.py b/tests/test_toolbox_langgraph.py index 83d89e2..bab8dcf 100644 --- a/tests/test_toolbox_langgraph.py +++ b/tests/test_toolbox_langgraph.py @@ -15,10 +15,18 @@ @pytest.fixture(autouse=True) -def _test_runtime(monkeypatch): +def _test_runtime(monkeypatch, tmp_path): """Provide a runtime in test mode so get_runtime returns without - authenticating against a real server.""" + authenticating against a real server. + + Pins ``NULLRUN_WAL_PATH`` to a tmp_path-scoped file so the + constructor's ``Transport._replay_from_wal`` never picks up + a stale WAL left over from a previous test run (which would + replay real events to a live API and cause HTTP 401 in + setup). Mirrors ``conftest::make_test_runtime``. + """ monkeypatch.setenv("NULLRUN_API_KEY", "test-key-12345678") + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) NullRunRuntime.reset_instance() # Pre-build a test-mode singleton so get_runtime returns it without # hitting the network. Construct directly and store on the singleton