From fbf0ca952a08e9f0c29d50cfcb3bd03826b1e661 Mon Sep 17 00:00:00 2001 From: Yuanhao Li Date: Tue, 25 Aug 2026 00:49:39 +0200 Subject: [PATCH] ci: run the live smoke harness as a release gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit release_smoke found a real bug on its first ever run — tools with zero parameters were uncallable on Anthropic, in shipped releases. It has run since when someone remembered, which is the same failure the price audit had before it went into the checklist: a check that depends on memory is a check you do not have. Dispatchable manually, and fires automatically on release/** push. Both providers report independently — they are separate SSE parsers and tool-call accumulators, and the zero-argument bug lived in exactly that layer. Deliberately not per-PR CI: it needs provider keys, costs money per run, and live calls are flaky enough that a hung request would block the queue. Per-release is the right granularity. A 15-minute timeout bounds it, because the harness has none and two long_horizon runs hung mid-session during #150 work. A missing key skips rather than fails, so a fork without secrets does not see a red X for a gate it cannot run. long_horizon is deliberately excluded: it is a diagnostic, slower, and its output wants reading rather than a pass/fail. Documented in CONTRIBUTING and in the release checklist. Closes #161 --- .github/workflows/live-smoke.yml | 83 ++++++++++++++++++++++++++++++++ CONTRIBUTING.md | 23 +++++++++ 2 files changed, 106 insertions(+) create mode 100644 .github/workflows/live-smoke.yml diff --git a/.github/workflows/live-smoke.yml b/.github/workflows/live-smoke.yml new file mode 100644 index 0000000..7f5e824 --- /dev/null +++ b/.github/workflows/live-smoke.yml @@ -0,0 +1,83 @@ +name: Live Smoke + +# The release gate that mocks cannot be. +# +# Every unit test runs against `MockProvider`. That mock now validates message +# shape (#160), but it still cannot stream SSE, cannot price a request, and +# cannot tell you whether a model actually follows a truncation marker. The +# bugs that reached released versions all lived in exactly that gap: +# +# - tools with zero parameters were uncallable on Anthropic +# - the truncation stash did not work through the path the model takes +# - a retry underflow killed summarization on a detached task +# +# `release_smoke` found the first of those on its first ever run. +# +# Deliberately NOT part of per-PR CI: it needs provider keys, costs real money, +# and live calls are flaky enough that a hung request would block the queue. +# Per-release is the right granularity — a few cents, a few minutes, once per +# version. +on: + workflow_dispatch: + inputs: + ref: + description: "Branch or tag to smoke (defaults to the current branch)" + required: false + type: string + push: + branches: ["release/**"] + +env: + CARGO_TERM_COLOR: always + +jobs: + smoke: + name: Live smoke (${{ matrix.name }}) + runs-on: ubuntu-latest + # Never let a hung provider call hold the queue. The harness itself has no + # global timeout, and two `long_horizon` runs hung mid-session during #150. + timeout-minutes: 15 + strategy: + # Both providers report independently: they exercise different SSE + # parsers and different tool-call accumulators, which is where the + # zero-argument bug lived. + fail-fast: false + matrix: + include: + - name: anthropic + key_env: ANTHROPIC_API_KEY + smoke_model: "" + - name: deepseek + key_env: DEEPSEEK_API_KEY + smoke_model: deepseek + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + + # Skipped, not failed, when the secret is absent — a fork without keys + # should not see a red X for a gate it cannot run. + - name: Check for a key + id: key + env: + KEY: ${{ secrets[matrix.key_env] }} + run: | + if [ -n "$KEY" ]; then + echo "present=true" >> "$GITHUB_OUTPUT" + else + echo "present=false" >> "$GITHUB_OUTPUT" + echo "::notice::${{ matrix.key_env }} is not set; skipping the ${{ matrix.name }} smoke run." + fi + + - name: Run the smoke harness + if: steps.key.outputs.present == 'true' + env: + # Both are set; the harness reads whichever its provider needs. + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} + SMOKE_MODEL: ${{ matrix.smoke_model }} + # The harness exits non-zero on any failed check, so it gates as-is. + run: cargo run --example release_smoke diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a1d0c56..ee4cc85 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,6 +56,29 @@ locally with `mdbook build` and open `book/index.html`. **Changelog** — add an entry under `## Unreleased` in [CHANGELOG.md](CHANGELOG.md). +## Live smoke + +Unit tests run against `MockProvider`. It validates message shape, but it cannot +stream SSE, cannot price a request, and cannot tell you whether a model actually +follows a truncation marker. Bugs that reached released versions all lived in +that gap — tools with zero parameters were uncallable on Anthropic, and its +first ever run is what found that. + +Before a release, run the **Live Smoke** workflow against the release branch, or +locally: + +```bash +ANTHROPIC_API_KEY=... cargo run --example release_smoke +SMOKE_MODEL=deepseek DEEPSEEK_API_KEY=... cargo run --example release_smoke +``` + +It exits non-zero on any failed check. Both providers matter: they are separate +SSE parsers and tool-call accumulators. + +`examples/long_horizon.rs` is the diagnostic sibling — compaction, cache +behaviour across compaction, sub-agent delegation. Slower, and its output wants +reading rather than a pass/fail, so it is deliberately not a gate. + ## Adding a provider Most new providers are OpenAI-compatible and need no new code — just a `ModelConfig` with the right