From 17c68133af54c04753590c78e46b8de3f3837158 Mon Sep 17 00:00:00 2001 From: ruv Date: Tue, 4 Aug 2026 16:19:16 -0400 Subject: [PATCH] fix(ci): core-and-rest dropped all 99 excludes to a shell comment The packages value is a YAML folded scalar. YAML does not treat '#' as a comment inside '>-', so the note added in 578400d1d (2026-06-21) survived as literal text and was substituted into run: cargo nextest run --no-fail-fast ${{ matrix.packages }} where bash does treat ' #' as starting a comment. Every flag after it was discarded. The shard has been running a bare cargo nextest run --no-fail-fast --workspace for six weeks: the entire workspace, including all the heavy research crates the sharding exists to subtract, plus examples/. Measured, three runs, identical signature: main job 91974446705 11:36:32Z last Compiling, 3h45m47s silence PR #791 attempt 1 12:19:03Z last Compiling, 3h45m40s silence PR #791 attempt 3 16:21:37Z last Compiling, 3h45m20s silence All three cancelled at the 4h timeout-minutes cap with one rustc still alive, and zero tests executed -- no "Starting N tests" and no "Summary [" line in any of the three logs. It presented as a slow-test timeout, which is why the cap was raised 150 -> 180 -> 240 minutes over iters 231-232 without ever fixing it. Comments move above the packages key, where YAML treats them as comments. core-and-rest-wasm had the same pattern; its comment trailed all the -p flags so nothing was dropped, but it is fixed too. Verified: all 99 excludes now reach cargo (0 before). Two of the 99 names, ruvector-postgres and timesfm-wasm, are not workspace members -- cargo emits a warning rather than an error for those, confirmed by experiment, and ruvector-postgres is deliberately in the root Cargo.toml workspace exclude. Adds .github/scripts/lint_test_matrix.py as a guard: the failure is silent, so a convention will not hold it. Tested both ways -- passes on this tree, and on the pre-fix workflow reports core-and-rest 99 intended / 0 surviving. Co-Authored-By: claude-flow Claude-Session: https://claude.ai/code/session_01WfyBMvexdN4bhWVaqSXKvR --- .github/scripts/lint_test_matrix.py | 69 +++++++++++++++++++++++++++++ .github/workflows/ci.yml | 48 +++++++++++++++----- 2 files changed, 107 insertions(+), 10 deletions(-) create mode 100644 .github/scripts/lint_test_matrix.py diff --git a/.github/scripts/lint_test_matrix.py b/.github/scripts/lint_test_matrix.py new file mode 100644 index 0000000000..ddb0b17739 --- /dev/null +++ b/.github/scripts/lint_test_matrix.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Fail if a test-matrix `packages` scalar contains a '#'. + +`packages:` is a YAML folded scalar (`>-`). YAML does not recognise '#' as a +comment inside a block scalar, so any '#' written there is literal text. That +text is substituted verbatim into + + run: cargo nextest run --no-fail-fast ${{ matrix.packages }} + +where bash *does* treat ' #' as starting a comment — silently discarding every +flag after it. + +This is what happened to the `core-and-rest` shard between 578400d1d +(2026-06-21) and the commit that added this script: all 99 `--exclude` flags +were dropped, so the shard ran a bare `cargo nextest run --workspace`, compiled +the entire workspace including every crate the sharding exists to avoid, and +was cancelled at the 4h `timeout-minutes` cap having never started a test. It +failed this way on `main` and on every PR branch, for six weeks, while +presenting as a timeout rather than as a misconfiguration. + +Nothing about that is visible in a diff or a log — the job runs, it just runs +the wrong command — so it needs a check. +""" + +import sys +from pathlib import Path + +try: + import yaml +except ImportError: + sys.exit("PyYAML is required: pip install pyyaml") + +WORKFLOW = Path(__file__).resolve().parents[1] / "workflows" / "ci.yml" + + +def main() -> int: + doc = yaml.safe_load(WORKFLOW.read_text()) + include = doc["jobs"]["test"]["strategy"]["matrix"]["include"] + + failures = [] + for entry in include: + name = entry.get("name", "") + packages = entry.get("packages", "") + if "#" not in packages: + continue + intended = packages.count("--exclude") + packages.count("-p ") + surviving_text = packages.split(" #")[0] + surviving = surviving_text.count("--exclude") + surviving_text.count("-p ") + failures.append((name, intended, surviving, packages.index("#"))) + + if not failures: + print(f"ok: {len(include)} matrix shards, no '#' in any packages scalar") + return 0 + + print("FAIL: '#' found inside a matrix `packages` block scalar\n") + for name, intended, surviving, at in failures: + print(f" shard {name!r}: '#' at offset {at}") + print(f" flags intended: {intended}, flags reaching cargo: {surviving}") + if surviving < intended: + print(f" -> {intended - surviving} flag(s) SILENTLY DROPPED") + print( + "\nMove the comment above the `packages:` key, where YAML will treat it\n" + "as a comment. Do not put '#' inside the `>-` scalar." + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77b42c2c3e..dfe94bfd70 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,21 @@ env: CARGO_INCREMENTAL: 0 jobs: + # Guards the defect fixed in this commit: a '#' written inside one of the + # 'packages: >-' block scalars below is NOT a YAML comment, so it reaches the + # shell through ${{ matrix.packages }} and comments out every flag after it. + # It is silent — the job still "runs", it just runs the wrong command — so it + # needs a check rather than a convention. Runs in seconds and gates nothing. + matrix-lint: + name: Test matrix lint + runs-on: ubuntu-22.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + + - name: No '#' inside any matrix packages scalar + run: python3 .github/scripts/lint_test_matrix.py + fmt: name: Rustfmt runs-on: ubuntu-22.04 @@ -178,6 +193,18 @@ jobs: # iter-231 all cancelled at the timeout boundary). The wasm # crates are a natural sub-group: thin bindings on top of host # crates, easy to compile + test in isolation. + # + # Iter 233 — ruvllm-wasm is deliberately absent from the list + # below: 11 of its 195 tests (sona_instant + workers::feature_detect) + # fail or SIGABRT on native because they are wasm-target specific + # (they need wasm-bindgen-test). Surfaced by iter-232's split; + # previously masked by the iter-228..231 timeout cancellations of + # the mega-shard. Tracking as a workspace follow-up — the fix is to + # gate the affected modules behind cfg(target_arch = "wasm32") or + # migrate them to wasm-bindgen-test runners. + # + # DO NOT put a '#' comment inside the block scalar below. See the + # core-and-rest entry for what that costs. packages: >- -p neural-trader-wasm -p ruvector-acorn-wasm @@ -207,15 +234,6 @@ jobs: -p ruvector-tiny-dancer-wasm -p ruvector-verified-wasm -p ruvector-wasm - # Iter 233 — `ruvllm-wasm` excluded from native nextest: - # 11 of its 195 tests (sona_instant + workers::feature_detect) - # fail or SIGABRT on native because they're wasm-target - # specific (need wasm-bindgen-test). Surfaced by iter-232's - # split; previously masked by the iter-228..231 timeout - # cancellations of the megaShard. Tracking as workspace - # follow-up — fix is to gate the affected modules behind - # `#[cfg(target_arch = "wasm32")]` or migrate to - # wasm-bindgen-test runners. - name: research-nightly # Nightly research PoC crates (iter 240+). Kept in a separate shard # so their compile + test time doesn't inflate core-and-rest. @@ -239,9 +257,19 @@ jobs: # Everything else: core, delta, server/cluster, etc. # Uses --workspace + --exclude to subtract the groups above so we # don't have to enumerate ~100 crates by hand. + # + # The photonlayer/ruvector-* research crates below run in the + # research-nightly shard (iter 240+). + # + # DO NOT put a '#' comment inside this block scalar. YAML does not + # treat '#' as a comment inside '>-', so it survives into the shell + # via ${{ matrix.packages }} and comments out every flag after it. + # That is exactly what happened between 578400d1d (2026-06-21) and + # this commit: all 99 --exclude flags were silently dropped, the + # shard ran a bare 'cargo nextest run --workspace', and it was + # cancelled at the timeout having never started a single test. packages: >- --workspace - # Nightly research crates run in the research-nightly shard (iter 240+) --exclude photonlayer-core --exclude photonlayer-bench --exclude photonlayer-cli