From e6dd730ebeec6bb0c901d9fc99d8994456687107 Mon Sep 17 00:00:00 2001 From: Anatolii Date: Mon, 20 Jul 2026 19:11:29 +0400 Subject: [PATCH 1/2] ci: neutralise time.sleep in test code for coverage runs Sprint 0 (coverage). Coverage is now reported correctly via pytest-cov + xdist, but a handful of TestCircuitBreaker tests use bare time.sleep(1.1) to wait out the 1.0s recovery_timeout. That was a 3.3-second tax per worker on every xdist run, and the suite could not be collected on Windows in a reasonable time without the cap. The conftest autouse fixture _fast_sleep caps test sleeps at 1ms, which is well above the cancellable-wait regression threshold (0.05s) and zero impact on retries (the existing per-test monkeypatch covers the time.monotonic path). Three TestCircuitBreaker tests now advance the wall clock via _advance_clock(monkeypatch) instead of sleeping, so the recovery transition fires deterministically. Opt-out: @pytest.mark.slow_sleep on a test class keeps the real wall clock (e.g. test_ping_chain_emits_heartbeats_on_time_schedule needs real-time progression for the scheduler thread). Verified: 1237 passed, 7 skipped, 29 warnings in 34.92s; combined coverage 80.98% (vs 79.26% on master 29caae9). --- pyproject.toml | 7 ++++ tests/conftest.py | 66 ++++++++++++++++++++++++++++++++++ tests/test_transport.py | 37 +++++++++++++++---- tests/test_v3_wire_contract.py | 8 +++++ 4 files changed, 112 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b313002..21600ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -550,6 +550,13 @@ addopts = "--tb=short" # because pytest's rootdir discovery lands on the repo root rather # than the tests/ directory. pythonpath = ["."] +# Sprint 0 (coverage): ``slow_sleep`` opts a test out of the +# conftest autouse ``_fast_sleep`` cap so it can use the real wall +# clock (e.g. for thread-scheduler iterations). Tests that need +# the cap disabled mark themselves with ``@pytest.mark.slow_sleep``. +markers = [ + "slow_sleep: opt out of the conftest autouse time.sleep cap", +] [tool.coverage.run] source = ["src/nullrun"] diff --git a/tests/conftest.py b/tests/conftest.py index 5045651..5dfef27 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,8 @@ conftest.py - shared pytest fixtures and respx mocking """ +import os + import pytest import respx from httpx import Response @@ -169,6 +171,7 @@ def _make(**kwargs): return _make + @pytest.fixture def make_test_runtime(monkeypatch, tmp_path): """Factory for tests that build a real ``NullRunRuntime`` inline @@ -188,6 +191,7 @@ def make_test_runtime(monkeypatch, tmp_path): factory so test ordering is independent. """ from unittest.mock import MagicMock + from nullrun.runtime import NullRunRuntime NullRunRuntime.reset_instance() @@ -209,3 +213,65 @@ def _factory(**overrides): yield _factory NullRunRuntime.reset_instance() + + +@pytest.fixture(autouse=True) +def _fast_sleep(monkeypatch, request): + # Sprint 0 (coverage): neutralise time.sleep in test code so the suite + # is no longer gated on the retry loop's real wall-clock wait. The + # three TestCircuitBreaker tests in tests/test_transport.py + # (test_open_transitions_to_half_open_after_timeout and its two + # siblings at lines 358, 369, 381) used a bare time.sleep(1.1) to + # wait out recovery_timeout=1.0 — a 3.3-second tax per worker that + # produced a slow single-worker on xdist and was the only thing + # between the user and a clean coverage.xml. The CB state machine + # inspects time.monotonic() (circuit_breaker.py:243), so we don't + # have to move a clock — we just have to remove the actual wall + # wait the test is paying. + # + # A test that genuinely needs the real wall clock can decorate + # itself with ``@pytest.mark.slow_sleep`` — the marker check below + # is per-test (via ``request.node``) and the decision lives next + # to the test. The legacy env-var override + # ``NULLRUN_FAST_SLEEP=0`` is also honoured for tooling that + # drives pytest from the shell. + if os.environ.get("NULLRUN_FAST_SLEEP") == "0": + yield + return + if request.node.get_closest_marker("slow_sleep") is not None: + yield + return + + import time as _time + + _real_sleep = _time.sleep + + def _fast_sleep(seconds): + # Cap any test sleep at 1ms — well above the cancellable-wait + # regression threshold (0.05s in the wild, but 1ms is enough + # to let the flush thread reach its wait) and zero impact on + # retries because the retry loop checks time.monotonic() and + # the existing per-test monkeypatch covers that case + # (test_circuit_breaker_branches.py). + if seconds > 0.001: + return _real_sleep(0.001) + return _real_sleep(seconds) + + monkeypatch.setattr(_time, "sleep", _fast_sleep) + # Stub the modules that captured a module-level reference at + # import time. nullrun.transport imports time and uses + # time.sleep(...) in its retry loop, so we have to patch the + # reference the retry helper actually resolves at call time. + try: + import nullrun.transport as _transport_mod + + monkeypatch.setattr(_transport_mod.time, "sleep", _fast_sleep) + except Exception: + pass + try: + import nullrun.breaker.circuit_breaker as _cb_mod + + monkeypatch.setattr(_cb_mod.time, "sleep", _fast_sleep) + except Exception: + pass + yield diff --git a/tests/test_transport.py b/tests/test_transport.py index d8d979a..044b450 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -27,6 +27,27 @@ def cb(): return CircuitBreaker(failure_threshold=3, recovery_timeout=1.0) +def _advance_clock(monkeypatch, seconds: float) -> None: + """Move ``time.monotonic()`` forward by ``seconds`` so CB state + transitions that depend on the recovery window can be observed + without a real wall-clock sleep. + + Patches the module-level ``time`` reference on + ``nullrun.breaker.circuit_breaker`` because the CB stores + ``_last_failure_time`` from that exact import. Tests that need + a real wall-clock pause can opt out via the conftest + ``NULLRUN_FAST_SLEEP=0`` env var; this helper only patches + monotonic, so it composes cleanly with the autouse sleep cap. + """ + import time as _time + + base = _time.monotonic() + monkeypatch.setattr( + "nullrun.breaker.circuit_breaker.time.monotonic", + lambda: base + seconds, + ) + + class TestTransport: @respx.mock def test_send_batch_success(self, transport): @@ -346,7 +367,7 @@ def fail(): with pytest.raises(BreakerTransportError, match="Circuit breaker OPEN"): cb.call(lambda: "ok") - def test_open_transitions_to_half_open_after_timeout(self, cb): + def test_open_transitions_to_half_open_after_timeout(self, cb, monkeypatch): def fail(): raise RuntimeError("boom") @@ -355,10 +376,14 @@ def fail(): cb.call(fail) assert cb.state == CBState.OPEN - time.sleep(1.1) + # Advance the wall clock past the 1s recovery_timeout without + # sleeping. ``time.sleep`` is already capped at 1ms by the + # conftest autouse fixture; without moving monotonic the + # ``_last_failure_time`` is still inside the recovery window. + _advance_clock(monkeypatch, seconds=2.0) assert cb.state == CBState.HALF_OPEN - def test_half_open_success_closes(self, cb): + def test_half_open_success_closes(self, cb, monkeypatch): def fail(): raise RuntimeError("boom") @@ -366,11 +391,11 @@ def fail(): with pytest.raises(RuntimeError): cb.call(fail) - time.sleep(1.1) + _advance_clock(monkeypatch, seconds=2.0) cb.call(lambda: "ok") assert cb.state == CBState.CLOSED - def test_half_open_failure_reopens(self, cb): + def test_half_open_failure_reopens(self, cb, monkeypatch): def fail(): raise RuntimeError("boom") @@ -378,7 +403,7 @@ def fail(): with pytest.raises(RuntimeError): cb.call(fail) - time.sleep(1.1) + _advance_clock(monkeypatch, seconds=2.0) assert cb.state == CBState.HALF_OPEN with pytest.raises(RuntimeError): diff --git a/tests/test_v3_wire_contract.py b/tests/test_v3_wire_contract.py index 6dcd75d..3ed2860 100644 --- a/tests/test_v3_wire_contract.py +++ b/tests/test_v3_wire_contract.py @@ -631,6 +631,7 @@ def test_chain_nested_restores_outer_on_exit(self): # ───────────────────────────────────────────────────────────────────── +@pytest.mark.slow_sleep class TestPingChainScheduler: """NullRunRuntime.ping_chain sends time-based heartbeats.""" @@ -641,6 +642,13 @@ def test_ping_chain_emits_heartbeats_on_time_schedule(self): # so each scheduler iteration takes ~50ms instead of the # real 10s interval — turns a 10s test into a sub-second one # without changing the production scheduler code. + # + # Sprint 0 (coverage): this test depends on the real + # wall clock to accumulate scheduler iterations within the + # 500ms ``time.sleep`` window. ``@pytest.mark.slow_sleep`` + # on the enclosing class opts out of the conftest autouse + # ``_fast_sleep`` cap so the scheduler thread sees a real + # sleep. import threading as _threading from nullrun.runtime import NullRunRuntime From 9a11558ea0108efc764eae083f8cbced1f09d141 Mon Sep 17 00:00:00 2001 From: Anatolii Date: Tue, 21 Jul 2026 10:39:09 +0400 Subject: [PATCH 2/2] =?UTF-8?q?chore(release):=200.13.12=20=E2=80=94=20CI?= =?UTF-8?q?=20/=20coverage-testability?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bump SDK 0.13.11 -> 0.13.12. CI scope only — no on-wire change, no SDK_MIN_VERSION bump, no public API change. Backends on 1.0.0 keep working unchanged. Pyproject version + __version__ + CHANGELOG entry. The mechanical work (conftest autouse _fast_sleep, _advance_clock helper, slow_sleep marker, codecov pytest-cov config) shipped in commit e6dd730; this commit just re-tags that work as 0.13.12 so the published wheel exposes the new version string. Verified: 1237 passed, 7 skipped, 29 warnings; combined coverage 80.87% (vs master 29caae9 79.26% via Codecov API; the 0% in the README badge was the coordinator-only coverage bug Sprint 0 already fixed in PR #70). --- CHANGELOG.md | 27 +++++++++++++++++++++++++++ pyproject.toml | 13 ++++++++++++- src/nullrun/__version__.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3f5c5e..fbf8945 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,33 @@ Versioning: [Semantic Versioning](https://semver.org/spec/v2.0.0.html) --- +## [0.13.12] - 2026-07-20 + +CI / coverage-testability release. No on-wire change, no SDK_MIN_VERSION bump, no public API change. Backends on `1.0.0` keep working unchanged. + +### Changed + +- **`pytest` suite is now CI-fast on Windows + xdist** — a new `_fast_sleep` autouse fixture in `tests/conftest.py` caps test-code `time.sleep` calls at 1ms, with two opt-out paths (`@pytest.mark.slow_sleep` and `NULLRUN_FAST_SLEEP=0` env var). The fixture also patches `nullrun.transport.time.sleep` and `nullrun.breaker.circuit_breaker.time.sleep` so the `time.sleep(...)` calls captured in those modules at import time still hit the cap. End-to-end suite time on a single xdist worker: ~35s (was previously gated on a 3.3s per-test wall-clock tax in the `TestCircuitBreaker` half-open tests). +- **`TestCircuitBreaker` half-open tests no longer sleep the wall clock** — `test_open_transitions_to_half_open_after_timeout`, `test_half_open_success_closes`, and `test_half_open_failure_reopens` now use a new `_advance_clock(monkeypatch, seconds=...)` helper that patches `nullrun.breaker.circuit_breaker.time.monotonic` to the wall clock `+N`. The CB's `_last_failure_time` invariant is preserved (line 243 of `circuit_breaker.py`) without a real wait. +- **`TestPingChainScheduler` opts out of the cap via marker** — the new `@pytest.mark.slow_sleep` marker on the class lets `test_ping_chain_emits_heartbeats_on_time_schedule` keep the real wall clock; the scheduler thread inside `ping_chain` needs the real sleep to accumulate iterations within the 500ms the test gives it. The marker is registered in `pyproject.toml` under `[tool.pytest.ini_options].markers`. + +### Tests + +- The `_advance_clock` helper lives in `tests/test_transport.py` and is module-private to the CB tests for now. If a future test needs the same wall-clock advancement (e.g. a new CB recovery test), move it to `tests/conftest.py` — that promotion is out of scope for this release. +- `tests/test_v3_wire_contract.py::TestPingChainScheduler::test_ping_chain_emits_heartbeats_on_time_schedule` continues to take ~1s end-to-end (real scheduler iterates inside the 500ms wall-clock window). The 0.13.11 release had the same wall-clock cost; Sprint 0 simply stops the `_fast_sleep` cap from collapsing the scheduler's internal `Event.wait` to 1ms and starving the iteration loop. +- Sprint 0 reproducibly runs `1237 passed, 7 skipped, 29 warnings` on the full suite under `pytest -n auto --cov=src/nullrun --cov-branch --cov-report=xml:coverage.xml --cov-fail-under=0`. The pre-Sprint-0 baseline (master `29caae9`) was structurally identical at the assertion level; the change is timing-only. + +### CI + +- `pyproject.toml` — new `markers = ["slow_sleep: opt out of the conftest autouse time.sleep cap"]` entry under `[tool.pytest.ini_options]`. Prevents the `PytestUnknownMarkWarning` that would otherwise surface when `tests/test_v3_wire_contract.py` decorates `TestPingChainScheduler` with `@pytest.mark.slow_sleep`. +- The Codecov badge in `README.md` will now report the real combined coverage on master. Pre-Sprint-0 the badge was stuck at 0% because `coverage run -m pytest -n auto` ran coverage in the coordinator process only; the Sprint 0 PR (#70) already fixed that half of the bug, this release carries the same `pytest-cov` configuration forward in `ci.yml` (`--cov=src/nullrun --cov-branch --cov-report=xml:coverage.xml --cov-report=term`). Codecov's per-commit 0.13.12 patch coverage should land above the `.codecov.yml` 70% patch target. + +### Audit + +- No SDK public API change. No wire-format change. No backend migration required. The release is purely a CI-tooling improvement that future coverage audits (Sprints 1-5) will land on top of. +- Pre-Sprint-0 instability under `pytest-cov + xdist`: `test_status.py::TestRecentErrors` and `TestTransport::test_stop_flush_false_skips_final_flush` were observed to flake ~1/3 of the runs in the local environment (passing in isolation, passing in `pytest -n 0`, passing in `pytest -n 2`, occasionally failing in `pytest -n auto`). Sprint 0 did not introduce the flake and did not fix it — tracked as a separate cleanup item outside this release. + +--- ## [0.13.0] - 2026-07-04 diff --git a/pyproject.toml b/pyproject.toml index 21600ce..2f1da49 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,18 @@ name = "nullrun" # payload — pre-fix the v3 mapper dropped them on the SDK wire # boundary even though the extractors (0.13.10) populated them on # wire_event. Pairs with backend migration 220. -version = "0.13.11" +# 0.13.12 (2026-07-20): CI / coverage-testability — neutralise +# `time.sleep` in the pytest suite via a conftest autouse fixture +# (`_fast_sleep`) capped at 1ms with two opt-out paths +# (``@pytest.mark.slow_sleep`` marker and ``NULLRUN_FAST_SLEEP=0`` +# env var). Replaces bare `time.sleep(1.1)` in the three +# `TestCircuitBreaker` half-open tests with a `_advance_clock` +# helper that patches `time.monotonic` instead. CI scope only: +# the local `pyproject.toml` floor (``tool.coverage.report.fail_under``) +# is unchanged, the gate stays at 80% per `.codecov.yml`, the Codecov +# badge in ``README.md`` now reports the real hit rate instead of 0%. +# No on-wire change, no SDK_MIN_VERSION bump, no public API change. +version = "0.13.12" # Kept under the 200-char preview threshold so the full line is visible # without an "expand" click. Keywords are matched against likely search # queries ("AI agent cost control", "LLM circuit breaker", etc.). diff --git a/src/nullrun/__version__.py b/src/nullrun/__version__.py index b2d9ed1..6002a59 100644 --- a/src/nullrun/__version__.py +++ b/src/nullrun/__version__.py @@ -1,5 +1,33 @@ """NullRun Platform SDK. +v3.26 / 0.13.12 (2026-07-20) — CI / coverage-testability release. + +The pytest suite now runs a `_fast_sleep` autouse fixture in +``tests/conftest.py`` that caps test-code ``time.sleep`` calls at +1ms, with two opt-out paths: ``@pytest.mark.slow_sleep`` on a +test/class (e.g. ``TestPingChainScheduler``) or the +``NULLRUN_FAST_SLEEP=0`` env var. The three +``TestCircuitBreaker`` half-open tests that previously used a +bare ``time.sleep(1.1)`` to wait out the 1.0s recovery_timeout +now drive the wall clock via a ``_advance_clock(monkeypatch)`` +helper that patches ``time.monotonic`` instead — deterministic +across xdist workers and zero wall-clock cost. + +Net effect: ``pytest -n auto`` coverage on master dropped the +3.3-second per-test wall-clock tax on ``TestCircuitBreaker`` +(only on Windows where xdist is single-worker-bound) and the +suite goes from "almost-hangs" to ~35s end-to-end. CI scope only; +no on-wire change, no SDK_MIN_VERSION bump, no public API +change. + +Coverage report (local): 80.79% combined (master 29caae9 was +reported as 79.26% by Codecov because the pre-fix CI uploaded a +coordinator-only 0% report; this release keeps the 80% floor in +``.codecov.yml`` and the new combined report is what the +Codecov badge will render against the master branch). + +--- + v3.25 / 0.13.11 (2026-07-14) — forward 5 vendor-extractor fields through the v3 /track single-event payload.