From d2fab7ab7737bb7449bc8faf3090735b8563a1d2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 02:49:08 +0000 Subject: [PATCH 1/3] ci(test): e2e and live filename tiers move to a nightly run on main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One switch, OS_TEST_TIERS (queue | nightly, unset = queue), read once in scripts/nightly-tiers.mjs and applied in packages/cli's tier walk — the one config that owns tier files (60 *.e2e.test.ts; the tree owns no *.live.test.*). turbo.json hashes the variable into every test task so a nightly can never replay a queue-mode cache entry. ci.yml's shard step spells queue; test-nightly-tiers.yml runs the two tiers on main, two shards through the existing partitioner, and files or refreshes one deduplicated card on red. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_019RfFHiRCSs3JXLK4cwcfox --- .github/workflows/ci.yml | 8 + .github/workflows/test-nightly-tiers.yml | 529 ++++++++++++++++++++++ packages/cli/vitest-tiers.ts | 62 ++- packages/cli/vitest.config.ts | 72 ++- scripts/nightly-tiers.d.mts | 41 ++ scripts/nightly-tiers.mjs | 539 +++++++++++++++++++++++ turbo.json | 30 ++ 7 files changed, 1261 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/test-nightly-tiers.yml create mode 100644 scripts/nightly-tiers.d.mts create mode 100644 scripts/nightly-tiers.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08ad09856c..105b74669a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -575,6 +575,14 @@ jobs: - name: Run this shard's tests env: NODE_OPTIONS: --report-on-signal --report-signal=SIGUSR2 --report-directory=${{ runner.temp }}/stall-reports + # The `e2e` and `live` filename tiers run NIGHTLY on main + # (test-nightly-tiers.yml), not here: `queue` is the per-PR and + # merge-queue setting, read once in scripts/nightly-tiers.mjs. Spelled + # explicitly even though unset reads the same, so the setting this + # required check verifies is written where the check runs. + # turbo.json hashes it in the `test` task's `env`, which is what lets + # it reach vitest under strict env mode at all. + OS_TEST_TIERS: queue run: | if [ ! -s "$RUNNER_TEMP/shard-packages.txt" ]; then echo "No packages on this shard — nothing to test." diff --git a/.github/workflows/test-nightly-tiers.yml b/.github/workflows/test-nightly-tiers.yml new file mode 100644 index 0000000000..8f4d230248 --- /dev/null +++ b/.github/workflows/test-nightly-tiers.yml @@ -0,0 +1,529 @@ +name: Nightly Tiers + +# The `e2e` and `live` test tiers, run nightly on `main` (#16455). +# +# Maintainer direction (2026-09-07, verbatim): 「我想的是测试会不会太多，是否都是 +# 必要的，是不是应该砍，每次修改都要完整的测试吗」. The two tiers leave the per-PR +# and merge-queue runs (ci.yml's Test Core now runs under `OS_TEST_TIERS=queue`) +# and run here, under `OS_TEST_TIERS=nightly`, selected by their EXISTING +# filename tiers only — `*.e2e.test.*` and `*.live.test.*` — with no test file +# renamed, deleted or edited to move it. `scripts/nightly-tiers.mjs` is the one +# reader of the switch; its header carries the contract, the measured +# population and the reason it is one module rather than a line per config. +# +# ## Shape +# +# `coverage-nightly.yml`'s: `schedule` + `workflow_dispatch`, node 22, pnpm, +# RESTORE-ONLY caches (a scheduled run reads main's pnpm store and Build Core's +# turbo seeds; the per-push workflows own saving them, and a nightly that saved +# would churn the rationed 10 GB pool the way PR-side saves once did — see +# ci.yml's "Restore Turbo cache"). The turbo cache restore is what keeps the +# dependency closure's `build` tasks a replay: `OS_TEST_TIERS` is hashed into +# the `test` tasks ONLY (turbo.json), so the builds hit main's entries while +# every test task misses by construction and runs for real. +# +# ## Sharding — TWO shards, derived rather than picked +# +# The package set is not the workspace: `scripts/nightly-tiers.mjs --packages` +# enumerates exactly the packages that own a tier file, in the `turbo ls` +# document shape, and `scripts/partition-test-shards.mjs` splits THAT with the +# same measured weights, the same LPT placement and the same +# ` k/n` slice grammar Test Core uses. Measured at 6eba38f5a3: one +# package, `@objectstack/cli`, owning all 60 tier files; its whole suite was +# 1231s on a hosted runner (run 34009395649) and the e2e tier is ~75% of it +# (packages/cli/vitest.config.ts, the priced trade), so ≈ 15 min of tests plus +# ~5 min of restore/install per runner. Under 60 minutes on ONE shard already — +# but `FILE_SHARDED_PACKAGES` cuts the CLI into two vitest slices, and the +# partitioner REFUSES to place two slices of one package in one bin +# (`assertSlicesSpread`), so one shard is not a legal count and two is the +# smallest one that is. Each shard then carries one slice, ≈ 8 min of tests. +# Raise the matrix (three places below: `matrix.shard`, the job `name:`, and +# `--shard k/N`) only when a second tier-owning package arrives heavy enough +# for the partitioner's own balance line to say so. +# +# ## What lands where — the house idiom of `platform-checklist-watchdog.yml` +# +# On green this workflow files nothing, edits nothing and closes nothing. On +# red the `report` job — ONE writer, after both shards, never the shards +# themselves (two matrix legs racing the same lookup would mint two cards) — +# opens or refreshes ONE card: fixed title `nightly-tiers: red on main`, labels +# `bug` · `domain:devx` · `priority:p1` on create only (grading is the devx +# seat's and a refresh must not undo it), body carrying the run link and the +# failing files, de-duplicated by scanning OPEN issues for that exact title +# prefix through the repo-scoped REST API with this workflow's own token. A +# CLOSED card is not reopened: red again after it was answered is a regression, +# filed fresh. +# +# The body marker is plain text, ⛔ never an HTML comment: this platform's body +# sanitizer is measured to eat short angle-bracket fragments, and a de-dup key +# that can be swallowed files a duplicate a day. The failing files come from +# vitest's own JSON reporter (`--reporter=json`, written per package, collected +# per shard, read by `scripts/nightly-tiers.mjs --failing-files`), never from a +# regex over the log — the log is presentation, the report is the contract. +# +# ⛔ A run on any ref but `main` NEVER writes to the board — a +# `workflow_dispatch` on a branch proves the transport, the sharding and the +# rendering on a real runner and publishes the would-be card to the run +# summary instead. That is how the card that landed this was smoke-tested. +# +# ⛔ This job is NOT a required context and must never become one: it runs on +# no pull request at all, so a ruleset naming it would block every PR forever +# on a check that cannot report. + +on: + schedule: + # Nightly. The minute is offset off the top of the hour ON PURPOSE (scheduled + # workflows queue behind everyone else's `:00` cron) and off every sibling + # patrol's minute, so two of them never contend for the same runner minute; + # 05:29 sits after `rerun-safety-nightly` (04:00) and `coverage-nightly` + # (05:00) have started and well before `showcase-smoke` (07:00). + - cron: '29 5 * * *' + workflow_dispatch: {} + +# Least privilege. The shards read the repo and publish artifacts of their own +# run; the report job writes issues and nothing else — no label on anybody's +# PR, no state, no comment stream. +permissions: + contents: read + issues: write + +# One nightly at a time. A dispatch overlapping the schedule would have two +# `report` jobs racing the same lookup, and the loser would mint a duplicate. +concurrency: + group: test-nightly-tiers + cancel-in-progress: false + +env: + # The card's IDENTITY — the title prefix the de-dup scan keys on, and the + # plain-text body marker that is its second way in. Single-sourced here so + # the report step and a reader of this file see the same strings. + CARD_TITLE: 'nightly-tiers: red on main' + CARD_MARKER: os-nightly-tiers + # Applied on CREATE only, additively; never rewritten on a refresh. + CARD_LABELS: 'bug,domain:devx,priority:p1' + +jobs: + tiers: + name: Nightly tiers (${{ matrix.shard }}/2) + runs-on: ubuntu-latest + # Generous against the ≈ 13 min a shard is expected to take, and still + # inside the hour the card set as the whole run's bound. A shard killed here + # produces no vitest report; the report job says so rather than reading it + # as "no failing files". + timeout-minutes: 45 + strategy: + # Both shards run to completion: the card should carry every failing + # file, not the first shard's. + fail-fast: false + matrix: + shard: [1, 2] + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + # Kept as this job's own step rather than folded into the composite + # below: `scripts/check-node-version.mjs` scans `.github/workflows/*.yml` + # only and reports how many setup-node steps it audited, so a step moved + # out of sight would silently shrink its census. + - name: Setup Node.js + uses: actions/setup-node@v7 + with: + node-version: '22' + + - name: Setup pnpm + uses: ./.github/actions/setup-pnpm + + - name: Get pnpm store directory + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + + # Restore-only: scheduled runs read main's store cache; the per-push + # workflows own saving it. + - name: Restore pnpm cache + uses: actions/cache/restore@v6 + with: + path: ${{ env.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-v3-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store-v3- + + # Restore-only, from Build Core's namespace: that job builds the whole + # workspace on every main push, so its entries replay every `build` task + # this run depends on. The `test` tasks cannot replay from it — their + # hash carries `OS_TEST_TIERS`, which no push run has ever set to + # `nightly` — which is the property that makes a nightly a test run and + # not a cache read. + - name: Restore Turbo cache + uses: actions/cache/restore@v6 + with: + path: .turbo/cache + key: ${{ runner.os }}-turbo-build-core-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-turbo-build-core-${{ github.ref_name }}- + ${{ runner.os }}-turbo-build-core- + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + # Two readings before a single test runs, both loud on failure: + # + # --check every tier-owning package honours the switch, judged by what + # vitest COLLECTS under each setting (a package that adopted a + # tier without reading the switch would otherwise run its whole + # suite here under the nightly's name, and its tier files in the + # merge queue — both silently); + # --packages the tier-owning packages, in the `turbo ls` document shape + # the partitioner reads, so this run shards exactly them. + # + # An EMPTY shard short-circuits the test step: `turbo run test` with zero + # --filter args runs the entire workspace. + - name: Compute this shard's package set + run: | + node scripts/nightly-tiers.mjs --self-test + node scripts/nightly-tiers.mjs --check + node scripts/nightly-tiers.mjs --packages > "$RUNNER_TEMP/tier-packages.json" + node scripts/partition-test-shards.mjs "$RUNNER_TEMP/tier-packages.json" \ + --shard ${{ matrix.shard }}/2 \ + > "$RUNNER_TEMP/shard-packages.txt" + echo 'Items on this shard (a package name, or a package plus a k/n file-level slice):' + cat "$RUNNER_TEMP/shard-packages.txt" + + # The same leg loop as ci.yml's "Run this shard's tests", under the + # nightly setting, with vitest's JSON reporter added beside the default + # one: `--outputFile.json` is RELATIVE so each package writes its own + # report under its own root (an absolute path would make a second + # tier-owning package overwrite the first's), and the default reporter + # stays so the stall guard still sees output flush. + # + # `--log-order=stream` is MANDATORY under the guard and `--concurrency=4` + # matches the hosted runner's cores — both reasons are in ci.yml beside + # the step this mirrors; NODE_OPTIONS arms the stall guard's SIGUSR2 + # diagnostic reports the same way. + # + # ⛔ A failing leg STOPS the remaining ones, as in ci.yml: a second full + # leg on a job that is already red only risks turning an informative red + # into a killed job with no report at all. + - name: Run this shard's nightly tiers + env: + OS_TEST_TIERS: nightly + NODE_OPTIONS: --report-on-signal --report-signal=SIGUSR2 --report-directory=${{ runner.temp }}/stall-reports + run: | + if [ ! -s "$RUNNER_TEMP/shard-packages.txt" ]; then + echo "No packages on this shard — nothing to test." + exit 0 + fi + export VITEST_MAX_WORKERS="$(node scripts/vitest-worker-cap.mjs)" + mkdir -p "$RUNNER_TEMP/stall-reports" + + FILTERS="" + SLICES="" + while read -r PKG SLICE; do + [ -n "$PKG" ] || continue + if [ -n "$SLICE" ]; then + SLICES="$SLICES $PKG=$SLICE" + else + FILTERS="$FILTERS --filter=$PKG" + fi + done < "$RUNNER_TEMP/shard-packages.txt" + + REPORTER="--reporter=default --reporter=json --outputFile.json=.nightly-tiers/vitest-report.json" + STATUS=0 + LOGS="" + for LEG in __whole__ $SLICES; do + if [ "$LEG" = __whole__ ]; then + [ -n "$FILTERS" ] || continue + LOG="$RUNNER_TEMP/nightly-tiers-packages.log" + set -- pnpm turbo run test $FILTERS --concurrency=4 --summarize --log-order=stream -- $REPORTER + else + PKG="${LEG%%=*}" + SLICE="${LEG#*=}" + LOG="$RUNNER_TEMP/nightly-tiers-slice-$(printf '%s' "$PKG" | tr -c 'A-Za-z0-9' '-').log" + set -- pnpm turbo run test "--filter=$PKG" --concurrency=4 --summarize --log-order=stream -- "--shard=$SLICE" $REPORTER + fi + LOGS="$LOGS $LOG" + node scripts/run-with-stall-guard.mjs --log "$LOG" --stall-minutes 10 \ + --report-dir "$RUNNER_TEMP/stall-reports" -- "$@" || { STATUS=$?; break; } + done + + : > "$RUNNER_TEMP/nightly-tiers.log" + for LOG in $LOGS; do + if [ -f "$LOG" ]; then + cat "$LOG" >> "$RUNNER_TEMP/nightly-tiers.log" + fi + done + exit $STATUS + + # Gather what the report job needs, red or green: every package's vitest + # JSON report (named by the package directory so two packages never + # collide in the merged download) and the tail of this shard's log. + # `if: always()` — a red shard is exactly when this earns its keep. + - name: Collect this shard's reports + if: always() + run: | + OUT="$RUNNER_TEMP/nightly-reports/shard-${{ matrix.shard }}" + mkdir -p "$OUT" + find . -path '*/node_modules' -prune -o -path '*/.nightly-tiers/vitest-report.json' -print \ + | while read -r REPORT; do + PKG_DIR="$(dirname "$(dirname "$REPORT")")" + cp "$REPORT" "$OUT/$(printf '%s' "$PKG_DIR" | sed 's#^\./##' | tr -c 'A-Za-z0-9' '-').json" + done + if [ -f "$RUNNER_TEMP/nightly-tiers.log" ]; then + tail -n 200 "$RUNNER_TEMP/nightly-tiers.log" > "$OUT/log-tail.txt" + fi + echo "collected:" + ls -la "$OUT" + + - name: Publish this shard's reports + if: always() + uses: actions/upload-artifact@v7 + with: + name: nightly-tiers-reports-${{ matrix.shard }}-of-2 + path: ${{ runner.temp }}/nightly-reports/ + if-no-files-found: ignore + retention-days: 14 + + # The same two questions ci.yml's guard answers about a red suite — was + # every test vitest COUNTED actually run, and did every scheduled package + # report at all — asked of this shard's log against this shard's schedule. + - name: Test completeness guard + if: always() + run: | + if [ ! -f "$RUNNER_TEMP/nightly-tiers.log" ]; then + echo "No test log — the test step did not get far enough to produce one." + exit 0 + fi + node scripts/check-test-completeness.mjs "$RUNNER_TEMP/nightly-tiers.log" \ + --scheduled "$RUNNER_TEMP/shard-packages.txt" \ + --package-list "$RUNNER_TEMP/tier-packages.json" + + - name: Upload stall diagnostic reports + if: failure() + uses: actions/upload-artifact@v7 + with: + name: stall-reports-nightly-tiers-${{ matrix.shard }} + path: ${{ runner.temp }}/stall-reports/ + if-no-files-found: ignore + retention-days: 14 + + report: + name: File or refresh the red card + needs: tiers + # `always()` spelled out: this job must run precisely when a shard FAILED or + # was killed, which is the case the implicit `success()` wrapper would + # skip. The board write below is further gated on the result and the ref. + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Setup Node.js + uses: actions/setup-node@v7 + with: + node-version: '22' + + # No `pnpm install`: `scripts/nightly-tiers.mjs --failing-files` imports + # `node:` builtins and two repo-local helpers only, so installing the + # workspace here would buy nothing and would give an unattended nightly a + # lockfile it could fail on. + - name: Download the shards' reports + uses: actions/download-artifact@v8 + with: + pattern: nightly-tiers-reports-* + path: ${{ runner.temp }}/nightly-reports + merge-multiple: true + + # The verdict, read from the matrix job's RESULT — a status read, on + # purpose: `failure` is a red tier, `cancelled` is a shard killed by its + # timeout or by a human, and neither is a green nightly. The failing + # files come from vitest's own JSON reports; a red run that produced none + # (a build failure, a stall kill) says so in the body instead of reading + # as "nothing failed". + - name: Read the verdict + id: verdict + env: + TIERS_RESULT: ${{ needs.tiers.result }} + run: | + echo "tiers result: $TIERS_RESULT" + echo "result=$TIERS_RESULT" >> "$GITHUB_OUTPUT" + set +e + node scripts/nightly-tiers.mjs --failing-files "$RUNNER_TEMP/nightly-reports" \ + > "$RUNNER_TEMP/failing-files.txt" 2> "$RUNNER_TEMP/failing-files.err" + code=$? + set -e + # Captured with NO pipe in between: `cmd | tail` would report the + # pipe's status, and this exit code decides whether the list below is + # a reading or a refusal. + echo "failing_files_exit=$code" >> "$GITHUB_OUTPUT" + cat "$RUNNER_TEMP/failing-files.err" >&2 || true + echo "failing nightly-tier files:" + cat "$RUNNER_TEMP/failing-files.txt" + + - name: File or refresh the card + # RED only, on MAIN only. On green nothing in this job writes anything; + # on a branch (the smoke test a card's dev dispatches) the body goes to + # the run summary and the board is untouched. + if: steps.verdict.outputs.result != 'success' && github.ref == 'refs/heads/main' + uses: actions/github-script@v9 + env: + TIERS_RESULT: ${{ steps.verdict.outputs.result }} + FAILING_FILES_EXIT: ${{ steps.verdict.outputs.failing_files_exit }} + with: + # Delivery is retried, never assumed: this write is the entire product + # of a red run, and a transient answer from the issues endpoint would + # otherwise discard a completed sweep. + retries: 3 + script: | + const fs = require('fs'); + const path = require('path'); + + const owner = context.repo.owner; + const repo = context.repo.repo; + const title = process.env.CARD_TITLE; + const marker = process.env.CARD_MARKER; + const labels = process.env.CARD_LABELS.split(',').map((s) => s.trim()).filter(Boolean); + const result = process.env.TIERS_RESULT; + const runUrl = `${process.env.GITHUB_SERVER_URL}/${owner}/${repo}/actions/runs/${process.env.GITHUB_RUN_ID}`; + + const read = (name) => { + try { return fs.readFileSync(path.join(process.env.RUNNER_TEMP, name), 'utf8'); } + catch { return ''; } + }; + const failing = read('failing-files.txt').split('\n').map((s) => s.trim()).filter(Boolean); + const extractorExit = process.env.FAILING_FILES_EXIT; + + // The log tails, bounded: a GitHub issue body caps at 65536 chars, + // and the run link carries the whole log anyway. + const tails = []; + const reportsDir = path.join(process.env.RUNNER_TEMP, 'nightly-reports'); + const walk = (dir) => { + let entries = []; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + for (const e of entries) { + const p = path.join(dir, e.name); + if (e.isDirectory()) walk(p); + else if (e.name === 'log-tail.txt') tails.push({ shard: path.basename(dir), text: fs.readFileSync(p, 'utf8') }); + } + }; + walk(reportsDir); + const BUDGET = 40000; + const perTail = Math.max(2000, Math.floor(BUDGET / Math.max(1, tails.length))); + const tailBlocks = tails.sort((a, b) => a.shard.localeCompare(b.shard)).flatMap(({ shard, text }) => [ + `### ${shard} — last lines of the test log`, + '', + '```', + text.length > perTail ? `… (${text.length - perTail} chars elided) …\n${text.slice(-perTail)}` : text, + '```', + '', + ]); + + const filesSection = failing.length > 0 + ? ['## Failing files', '', ...failing.map((f) => `- \`${f}\``), ''] + : extractorExit !== '0' + ? ['## Failing files', '', `⚠️ The report reader exited ${extractorExit}; no per-file verdict is available. Read the run log.`, ''] + : ['## Failing files', '', '⚠️ No vitest report named a failing nightly-tier file, yet the tiers job did not succeed — the run died before or outside vitest (a build failure, a stall kill, a cancelled shard). Read the run log.', '']; + + const body = [ + `${marker} — machine-findable marker for this generated card. ⛔ Do not delete this line: it is how the nightly finds this card instead of filing a new one every night.`, + '', + `# ${title}`, + '', + `_Swept ${new Date().toISOString()} · [run log](${runUrl}) · commit \`${process.env.GITHUB_SHA}\` · trigger \`${context.eventName}\` · tiers job result \`${result}\`._`, + '', + 'The `e2e` and `live` test tiers — the files named `*.e2e.test.*` and `*.live.test.*` — run here', + 'nightly on `main` under `OS_TEST_TIERS=nightly` and nowhere else (the per-PR and merge-queue', + 'Test Core runs under `queue`). A red here is a real suite that no pull request will red on;', + 'this card is the channel that sees it. Nothing is blocked by it.', + '', + '⛔ The remedy is never to rename, skip or delete the failing test to make the nightly green.', + 'Reproduce locally with `OS_TEST_TIERS=nightly pnpm --filter test`, fix what it names, and', + 'let the next nightly refresh this card.', + '', + ...filesSection, + ...tailBlocks, + '_Filed by `.github/workflows/test-nightly-tiers.yml`. Generated by [Claude Code](https://claude.ai/code)_', + ].join('\n'); + + // ── De-dup: the OPEN cards carrying the title prefix ───────────── + // Repo-scoped REST, this workflow's token, bounded pages; a + // truncated scan cannot establish ABSENCE, and filing on an + // unestablished absence is how a nightly mints a duplicate a day. + const isThisCard = (i) => !i.pull_request + && (String(i.title ?? '').startsWith(title) || String(i.body ?? '').includes(marker)); + const MAX_PAGES = 10; + const candidates = []; + let scanComplete = true; + for (let page = 1; page <= MAX_PAGES; page += 1) { + const res = await github.rest.issues.listForRepo({ + owner, repo, state: 'open', sort: 'created', direction: 'asc', per_page: 100, page, + }); + candidates.push(...res.data.filter(isThisCard)); + if (res.data.length < 100) break; + if (page === MAX_PAGES) scanComplete = false; + } + if (!scanComplete) { + throw new Error(`the open-issue scan hit its ${MAX_PAGES}-page bound without completing — absence is NOT established, so nothing was filed. The failing files are in this run's summary.`); + } + + // The OLDEST open card wins — that is the one any duplicates were + // closed against. + const existing = candidates.sort((a, b) => a.number - b.number)[0] ?? null; + + if (existing) { + // Rewritten in place, never a comment per run. ⛔ Labels are NOT + // rewritten — grading is the devx seat's and a refresh must not + // undo it. + await github.rest.issues.update({ owner, repo, issue_number: existing.number, body }); + core.info(`refreshed nightly-tiers card #${existing.number} (${body.length} chars, tiers ${result})`); + core.notice(`nightly tiers are red on main — card #${existing.number} refreshed.`); + return; + } + + const created = await github.rest.issues.create({ + owner, repo, title, body, + // Additive on create; ⛔ nothing here ever replaces a whole label + // set (`check:whole-set-label-write` refuses that verb outright). + labels, + }); + core.info(`filed nightly-tiers card #${created.data.number} (${body.length} chars, tiers ${result})`); + core.notice(`nightly tiers are red on main — card #${created.data.number} filed.`); + + - name: Publish the verdict to the run summary + # Always: on green this IS the whole product; on a branch it is the + # smoke test's evidence; on a red main run it makes the run + # self-contained for whoever opens it from the card. + if: always() + env: + TIERS_RESULT: ${{ steps.verdict.outputs.result }} + run: | + { + echo "### Nightly tiers — tiers job result \`${TIERS_RESULT:-unread}\`" + echo + if [ "${{ github.ref }}" != "refs/heads/main" ]; then + echo "_Board write skipped: this run is on \`${{ github.ref }}\`, not \`main\`. A branch run proves the sharding, the switch and the rendering without filing anything._" + echo + fi + if [ "$TIERS_RESULT" = "success" ]; then + echo "Green: every nightly-tier file passed. Nothing filed, nothing edited, nothing closed." + else + echo "Failing nightly-tier files (from the vitest JSON reports; empty means the run died outside vitest — read the log):" + echo + echo '```' + cat "$RUNNER_TEMP/failing-files.txt" 2>/dev/null || echo '(no list produced)' + echo '```' + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Fail the run if the verdict was never read + # LAST, on purpose: the card is filed BEFORE this can turn the job red. + # A red TIER is not this job's failure — the shard already went red and + # the card carries it. What must never look green is a report job that + # finished without reading the matrix result at all: nothing downstream + # could tell that apart from a green nightly. + if: always() && steps.verdict.outputs.result == '' + run: | + echo "the verdict step recorded no result, so this run is NOT a reading about the nightly tiers -- it is neither a green one nor a red one. See this run's summary." + exit 1 diff --git a/packages/cli/vitest-tiers.ts b/packages/cli/vitest-tiers.ts index 79605a1376..cbcdccc4f9 100644 --- a/packages/cli/vitest-tiers.ts +++ b/packages/cli/vitest-tiers.ts @@ -85,11 +85,42 @@ * tokens that must track the predicate, which is the exact class of copy this * change exists to delete. Revisit only with a measurement that says it costs * something real. + * + * ## The NIGHTLY tiers are a second, orthogonal cut — by NAME (#16455) + * + * The two tiers above answer "what does this file DO" and decide which + * project collects it. `OS_TEST_TIERS` answers a different question — "which + * RUN is this" — and is decided by the file's NAME alone: `*.e2e.test.*` and + * `*.live.test.*` are the nightly tiers, everything else is the queue's. The + * switch is read ONCE, in `scripts/nightly-tiers.mjs` (the values, the default + * and the refusal of anything else live there), and applied here in + * `testFilesOnDisk()` — the one walk both `integrationTestFiles()` and + * `unitTestFiles()` derive from — so the setting narrows the POPULATION and + * the behavioural predicate then partitions whatever is left: + * + * OS_TEST_TIERS unset / queue population = every test file that is NOT nightly-tier + * OS_TEST_TIERS=nightly population = exactly the nightly-tier files + * + * Both projects stay a partition of that population BY CONSTRUCTION + * (`unit` = population − predicate, `integration` = population ∩ predicate), + * and `test/vitest-tiers-partition.test.ts` still measures it under whichever + * setting it runs in: its `vitest list` child inherits the switch through + * `childEnv()` and its filesystem walk is this function, so the two sides of + * every equality it asserts are read under the same setting. + * + * ⛔ The two cuts deliberately disagree on 5 files and that is not a defect: + * a file that carries the `.e2e` name and spawns plain node is nightly-tier + * (name) AND `unit` (behaviour); a file that spawns the CLI without the name + * is queue (name) AND `integration` (behaviour). The ruling that landed the + * nightly selects by the existing filename tiers only and renames nothing, so + * the name decides the run and the behaviour decides the project, and no file + * is renamed to make the two agree. */ import { readdirSync, readFileSync } from 'node:fs'; import { join, relative, sep } from 'node:path'; import { maskComments } from '../../scripts/js-comment-mask.mjs'; +import { readTierMode, selectTierFiles } from '../../scripts/nightly-tiers.mjs'; // --------------------------------------------------------------------------- // The predicate @@ -164,7 +195,13 @@ export function tierOfFile(pkgRoot: string, relPath: string): TierSignals { const TEST_FILE_RE = /\.(?:test|spec)\.[cm]?[jt]sx?$/; const SKIP_DIRS = new Set(['node_modules', 'dist', '.git', '.turbo', 'coverage']); -/** `pkgRoot`-relative, POSIX-separated paths of every test file on disk, sorted. */ +/** + * `pkgRoot`-relative, POSIX-separated paths of every test file on disk that + * the current `OS_TEST_TIERS` setting selects, sorted — the nightly-tier files + * under `nightly`, everything else under `queue` (the default). See "The + * NIGHTLY tiers" in the header: this is the ONE place the switch narrows this + * package's population, and both tier derivations below read from it. + */ export function testFilesOnDisk(pkgRoot: string): string[] { const out: string[] = []; const walk = (dir: string): void => { @@ -176,16 +213,33 @@ export function testFilesOnDisk(pkgRoot: string): string[] { } }; walk(pkgRoot); - return out.sort(); + return selectTierFiles(out.sort(), readTierMode()); } /** - * The integration tier: every test file on disk the predicate calls integration. + * The integration tier: every selected test file the predicate calls integration. * * This is what `vitest.config.ts` feeds to the `integration` project's - * `include` and the `unit` project's `exclude`, so the two projects stay a + * `include`; `unitTestFiles()` is its complement, so the two projects stay a * partition of the population by CONSTRUCTION rather than by maintenance. */ export function integrationTestFiles(pkgRoot: string): string[] { return testFilesOnDisk(pkgRoot).filter((file) => isIntegration(tierOfFile(pkgRoot, file))); } + +/** + * The unit tier: the selected population MINUS the integration tier — the exact + * complement of `integrationTestFiles()` over the same walk, handed to the + * `unit` project's `include`. Spelled as an include list rather than as + * `exclude: INTEGRATION_FILES` so that BOTH projects read the switched + * population above; an exclude-shaped unit tier would fall back to vitest's + * default `include` and collect nightly-tier files the queue must not run. + * + * `integration` defaults to a fresh derivation; the config passes the list it + * already derived so the second tier costs one walk (~10ms) and no second + * classification pass (~250ms measured on the box this landed on). + */ +export function unitTestFiles(pkgRoot: string, integration: readonly string[] = integrationTestFiles(pkgRoot)): string[] { + const integrationSet = new Set(integration); + return testFilesOnDisk(pkgRoot).filter((file) => !integrationSet.has(file)); +} diff --git a/packages/cli/vitest.config.ts b/packages/cli/vitest.config.ts index 312c087abd..f276b98307 100644 --- a/packages/cli/vitest.config.ts +++ b/packages/cli/vitest.config.ts @@ -563,27 +563,66 @@ // the PR that landed this section; re-measure it when the population moves, // and print the commit here. // +// ## THE NIGHTLY TIERS (#16455) — a second cut, by NAME, read from `OS_TEST_TIERS` +// +// Maintainer direction (2026-09-07, verbatim): 「我想的是测试会不会太多，是否都是 +// 必要的，是不是应该砍，每次修改都要完整的测试吗」. The `e2e` and `live` tiers +// leave the per-PR and merge-queue runs and run nightly on `main`; selection is +// by the EXISTING filename tiers only (`*.e2e.test.*`, `*.live.test.*`) and no +// file is renamed, deleted or edited to move it. Measured on 6eba38f5a3: this +// package owns every one of the tree's 60 `*.e2e.test.ts` files and the tree +// owns no `*.live.test.*` at all, so this is the one config that reads the +// switch today — read through `scripts/nightly-tiers.mjs`, the single reader of +// the variable, so a package that adopts a tier tomorrow imports rather than +// re-spells it. +// +// OS_TEST_TIERS unset / queue this package's population = the 212 non-tier files (180 unit + 32 integration) +// OS_TEST_TIERS=nightly this package's population = exactly the 60 tier files (1 unit + 59 integration) +// +// (272 test files on disk at 6eba38f5a3, read from `vitest list --filesOnly` +// under each setting and each `--project`; 212 + 60 = 272, no file in both.) +// +// The cut is applied in ONE place — `testFilesOnDisk()` in `vitest-tiers.ts`, +// the walk both derivations below read from — so the behavioural partition +// into `unit` / `integration` operates on whatever population the setting +// selected, and the two projects remain a partition of it by construction. +// That is also why the unit tier is now an INCLUDE list rather than +// `exclude: INTEGRATION_FILES`: an exclude-shaped unit project falls back to +// vitest's default `include`, which would collect the tier files the queue +// must not run. `test/vitest-tiers-partition.test.ts` measures all of this +// under whichever setting it runs in (its `vitest list` child inherits the +// switch); under `nightly` it is not itself collected — it carries no tier +// name — which is the ruled behaviour, not a gap. +// +// ⛔ The switch reaches vitest only because `turbo.json` names `OS_TEST_TIERS` +// in the `test` task's `env`: turbo 2.10 runs in STRICT env mode and strips +// every undeclared variable before the task's shell sees it. It sits in `env` +// (hashed) rather than `passThroughEnv` on purpose — a `test` task's cached +// outcome depends on the setting, so the setting is in the hash and a nightly +// can never replay a queue-mode cache entry as `>>> FULL TURBO`. +// // ⚠️ INLINE PROJECTS INHERIT NOTHING BY DEFAULT — `extends: true` is what // carries this file's `resolve.alias` table and `test.server.deps.external` // into each project (vitest 4.1.10: an inline project without it gets a fresh // Vite config, so the source aliases the gate above guards would be declared // here and enforced nowhere). `disableConsoleIntercept: true` is repeated // inside every project because `check:console-intercept-disarm` measured the -// root-level setting inert under projects. `exclude` for the unit tier spreads -// `configDefaults.exclude` first: an `exclude` that names only the integration -// files would drop the `node_modules` exclusion and start collecting -// dependencies' own test files. -import { configDefaults, defineConfig } from 'vitest/config'; +// root-level setting inert under projects. Both projects carry an explicit +// `include` of exact paths, so neither reaches vitest's default `include` +// (which would collect the whole tree) and neither needs its own +// `node_modules` exclusion: an exact-path list matches nothing it does not name. +import { defineConfig } from 'vitest/config'; import path from 'path'; -import { integrationTestFiles } from './vitest-tiers.js'; +import { integrationTestFiles, unitTestFiles } from './vitest-tiers.js'; -// The integration tier, DERIVED from what the files DO — never written down. -// `vitest-tiers.ts` holds the predicate, the walk and the argument for both; -// `test/vitest-tiers-partition.test.ts` pins what a derivation cannot pin -// about itself. Package-root-relative, POSIX-separated, sorted; each entry is -// an exact path, which is what lets the same array serve as the integration -// project's `include` and the unit project's `exclude`. +// The two tiers, DERIVED from what the files DO — never written down — over +// the population `OS_TEST_TIERS` selects. `vitest-tiers.ts` holds the +// predicate, the walk and the argument for both; `test/vitest-tiers-partition.test.ts` +// pins what a derivation cannot pin about itself. Package-root-relative, +// POSIX-separated, sorted; each entry is an exact path, which is what lets +// each array serve as its project's `include`. export const INTEGRATION_FILES = integrationTestFiles(__dirname); +export const UNIT_FILES = unitTestFiles(__dirname, INTEGRATION_FILES); export default defineConfig({ resolve: { @@ -662,9 +701,10 @@ export default defineConfig({ external: [/packages[\/]types[\/]dist/], }, }, - // The two tiers (#13504) — see the header section of the same name. Both - // `extends: true` so each project inherits the `resolve.alias` table and - // the `server.deps.external` entry above; each repeats the console-intercept + // The two tiers (#13504) — see the header section of the same name, and + // "THE NIGHTLY TIERS" for the population both read. Both `extends: true` + // so each project inherits the `resolve.alias` table and the + // `server.deps.external` entry above; each repeats the console-intercept // disarm because the root-level one is inert under projects. projects: [ { @@ -672,7 +712,7 @@ export default defineConfig({ test: { name: 'unit', disableConsoleIntercept: true, - exclude: [...configDefaults.exclude, ...INTEGRATION_FILES], + include: UNIT_FILES, }, }, { diff --git a/scripts/nightly-tiers.d.mts b/scripts/nightly-tiers.d.mts new file mode 100644 index 0000000000..1d83a4c351 --- /dev/null +++ b/scripts/nightly-tiers.d.mts @@ -0,0 +1,41 @@ +// Types for the `nightly-tiers.mjs` exports a TypeScript consumer reads -- the +// same problem, and the same fix, as `js-comment-mask.d.mts` next door (#5475). +// +// The module itself stays `.mjs`: it carries a `--self-test` / `--packages` / +// `--check` / `--failing-files` CLI run with bare `node` from the nightly +// workflow, and every root script here is authored that way. What needs the +// declaration is the other direction -- `packages/cli/vitest-tiers.ts` imports +// the switch from inside a tsc program (`tsconfig.test.json`), where an untyped +// `.mjs` import is TS7016 and `readTierMode` silently becomes `any`. +// +// PARTIAL BY DESIGN, the `check-regen-pending.d.mts` shape: the module exports +// more names than this declares, and importing an undeclared one is `TS2305` +// -- loud, red and immediate -- never a silent `any`. Keep this file in step +// with the module by hand; `check:declaration-mirrors` holds the names, kinds +// and required arities. + +/** The two tiers the switch moves off the per-PR and merge-queue runs. */ +export const NIGHTLY_TIERS: readonly string[]; + +/** The two legal spellings of `OS_TEST_TIERS`. */ +export const TIER_MODES: readonly string[]; + +/** The environment variable the switch is read from. */ +export const TIER_ENV: string; + +/** A test file in one of the nightly tiers, judged on its basename. */ +export const NIGHTLY_TIER_FILE_RE: RegExp; + +export function isNightlyTierFile(relPath: string): boolean; + +/** + * The switch's value: `queue` when unset or empty, else exactly `queue` or + * `nightly`; any other spelling throws. + */ +export function readTierMode(env?: Record): 'queue' | 'nightly'; + +/** + * The files `mode` selects out of `files`: under `queue` everything NOT in a + * nightly tier, under `nightly` exactly what is. Order preserved. + */ +export function selectTierFiles(files: string[], mode: 'queue' | 'nightly'): string[]; diff --git a/scripts/nightly-tiers.mjs b/scripts/nightly-tiers.mjs new file mode 100644 index 0000000000..0bbfd66549 --- /dev/null +++ b/scripts/nightly-tiers.mjs @@ -0,0 +1,539 @@ +#!/usr/bin/env node +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * nightly-tiers -- the ONE reader of `OS_TEST_TIERS`, the switch that moves the + * `e2e` and `live` test tiers off the per-PR and merge-queue runs and onto a + * nightly run on `main` (maintainer direction, 2026-09-07: 「我想的是测试会不会 + * 太多，是否都是必要的，是不是应该砍，每次修改都要完整的测试吗」). + * + * node scripts/nightly-tiers.mjs --self-test + * node scripts/nightly-tiers.mjs --packages # turbo-ls-shaped JSON of the packages that OWN tier files + * node scripts/nightly-tiers.mjs --check # every such package's vitest collection honours the switch + * node scripts/nightly-tiers.mjs --failing-files # tier files the vitest JSON reports under call failed + * + * ## The switch + * + * OS_TEST_TIERS unset / empty / `queue` -> the two tiers are EXCLUDED + * (pull request, merge queue, local default) + * `nightly` -> EXACTLY the two tiers run + * anything else -> refused loudly, naming the two legal values + * + * `queue` is the default so that a runner, a hook or a developer who never + * heard of the switch gets the cheaper run and never the nightly one by + * accident; the nightly setting is always a deliberate spelling. An unknown + * value is a refusal rather than a fallback because a typo that silently fell + * back to `queue` would be a nightly that tested nothing while reading green. + * + * ## Selection is by FILENAME tier, and by nothing else + * + * .e2e.test.ts .live.test.ts (and `.spec.`, and every vitest extension) + * + * The ruling on the card that landed this is explicit: no test file is renamed, + * deleted or edited to move it between runs; a file's run is decided by the + * tier its name already carries. The regex below is therefore the whole + * contract, and it is deliberately the narrow one: `**\/*.e2e.test.*` would + * also match a `.snap` beside the test, and under an `include` vitest would + * then try to RUN the snapshot. + * + * ⚠️ Where the population actually is, measured at 6eba38f5a3 (2026-09-07): + * `packages/cli` owns all 60 `*.e2e.test.ts` files in the tree, and NO package + * owns a `*.live.test.*` file at all. The 23 "live" test files a substring + * search finds (`live-mysql`, `live-dialect-matrix`, `lint-liveness-…`, + * `email-plugin.queue-delivery`) are not tier-named and are NOT selected -- + * `live` is a tier a package may adopt by naming a file into it, not a set of + * files this module goes looking for. + * + * ## Why one module and not a line in every vitest config + * + * Measured on the same commit: 72 `vitest.config.ts` files, every one a + * standalone `defineConfig`, none extending a shared config, no root + * `vitest.config.*`, no `vitest.workspace`. A switch spelled per config would + * be 72 copies of one predicate, or -- the honest count -- ONE copy today, + * because only one package owns tier files, and a copy in a package that owns + * none excludes nothing. So the predicate lives here, the one config that owns + * tier files reads it (`packages/cli/vitest-tiers.ts`, whose walk feeds both of + * that package's vitest projects), and a package that adopts a tier tomorrow + * imports this module rather than re-spelling it. + * + * ⛔ What that does NOT buy, and what `--check` is for. A package that adds its + * first `*.live.test.ts` without reading the switch would run that file in the + * merge queue (nothing excludes it) and its WHOLE suite under the nightly + * (nothing narrows it) -- both silent. `--check` measures every tier-owning + * package by what vitest actually COLLECTS under each setting, the way + * `packages/cli/test/vitest-tiers-partition.test.ts` judges its own config, + * and the nightly workflow refuses to run past a package that fails it. That + * makes the drift LOUD on the nightly; it does not make it red on the pull + * request that introduces it, which is the open question the landing card + * records. + * + * ## Where `OS_TEST_TIERS` has to be declared to reach vitest at all + * + * turbo 2.10 runs in STRICT env mode: a variable that `turbo.json` neither + * hashes (`env`) nor passes through is stripped before the task's shell sees + * it, and `vitest run` would read the switch as unset -- i.e. `queue` -- on a + * nightly runner that exported `nightly`. `turbo.json` therefore names + * `OS_TEST_TIERS` in the `test` task's `env`, and in `env` rather than + * `passThroughEnv` on purpose: a `test` task's cached outcome depends on the + * setting, so the setting must be in the hash, or a nightly could replay a + * queue-mode cache entry as `>>> FULL TURBO` and test nothing while green. + */ + +import { existsSync, mkdtempSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import process from 'node:process'; +import { execFileSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +import { isEntrypoint } from './invoked-as.mjs'; +import { workspacePackageDirs } from './workspace-enumerator.mjs'; + +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +// --------------------------------------------------------------------------- +// The contract +// --------------------------------------------------------------------------- + +/** The tiers the switch moves. Spelled once; the regex below is built from it. */ +export const NIGHTLY_TIERS = Object.freeze(['e2e', 'live']); + +/** The two legal spellings of the switch. */ +export const TIER_MODES = Object.freeze(['queue', 'nightly']); + +/** The environment variable. `OS_TEST_*` is the CI/test-only shape AGENTS.md reserves. */ +export const TIER_ENV = 'OS_TEST_TIERS'; + +/** + * A test file in one of the nightly tiers, judged on its basename. Anchored on + * vitest's own default extension set so a snapshot or a `.d.ts` sitting beside + * a tier test can never be selected. + */ +export const NIGHTLY_TIER_FILE_RE = new RegExp(`\\.(?:${NIGHTLY_TIERS.join('|')})\\.(?:test|spec)\\.[cm]?[jt]sx?$`); + +export function isNightlyTierFile(relPath) { + return NIGHTLY_TIER_FILE_RE.test(String(relPath)); +} + +/** + * The switch's value, read from `env` (default: this process's environment). + * Unset and empty read as `queue`; any spelling that is not exactly one of + * `TIER_MODES` is refused. + */ +export function readTierMode(env = process.env) { + const raw = env[TIER_ENV]; + if (raw === undefined || raw === '') return 'queue'; + if (TIER_MODES.includes(raw)) return raw; + throw new Error( + `${TIER_ENV}=${JSON.stringify(raw)} is not a test-tier setting. ` + + `Legal values: ${TIER_MODES.map((m) => JSON.stringify(m)).join(', ')} (unset reads as "queue"). ` + + 'Refusing to guess: a typo that fell back to "queue" would be a nightly that tested nothing.' + ); +} + +/** + * The files `mode` selects out of `files`: under `queue` everything that is + * NOT in a nightly tier, under `nightly` exactly what is. Order is preserved. + */ +export function selectTierFiles(files, mode) { + if (!TIER_MODES.includes(mode)) { + throw new Error(`selectTierFiles: mode must be one of ${TIER_MODES.join(', ')}, got ${JSON.stringify(mode)}`); + } + const wantTier = mode === 'nightly'; + return files.filter((f) => isNightlyTierFile(f) === wantTier); +} + +// --------------------------------------------------------------------------- +// The population: which packages own tier files +// --------------------------------------------------------------------------- + +const TEST_FILE_RE = /\.(?:test|spec)\.[cm]?[jt]sx?$/; +const SKIP_DIRS = new Set(['node_modules', 'dist', '.git', '.turbo', 'coverage', '.next', 'build']); + +/** + * Every workspace package that owns at least one nightly-tier test file, as + * `{ name, path, files }` -- `path` repo-relative POSIX, `files` package-relative + * POSIX and sorted. A package directory nested inside another package's + * directory is walked as its own package, never as part of the outer one. + */ +export function tierPackages(repoRoot = REPO_ROOT) { + const dirs = workspacePackageDirs(repoRoot); + const packageDirSet = new Set(dirs); + const out = []; + for (const dir of dirs) { + const abs = path.join(repoRoot, dir); + const files = []; + const walk = (current, rel) => { + for (const entry of readdirSync(current, { withFileTypes: true })) { + if (SKIP_DIRS.has(entry.name)) continue; + const childRel = rel ? `${rel}/${entry.name}` : entry.name; + if (entry.isDirectory()) { + if (packageDirSet.has(`${dir}/${childRel}`)) continue; // a nested package is its own package + walk(path.join(current, entry.name), childRel); + } else if (TEST_FILE_RE.test(entry.name) && isNightlyTierFile(entry.name)) { + files.push(childRel); + } + } + }; + walk(abs, ''); + if (files.length === 0) continue; + const manifest = JSON.parse(readFileSync(path.join(abs, 'package.json'), 'utf8')); + if (typeof manifest.name !== 'string' || manifest.name === '') { + throw new Error(`${dir}/package.json: owns nightly-tier test files but declares no "name" to filter on`); + } + out.push({ name: manifest.name, path: dir, files: files.sort() }); + } + return out; +} + +/** + * `tierPackages()` in the `turbo ls --output=json` shape + * `scripts/partition-test-shards.mjs` reads, so the nightly can shard exactly + * the tier-owning packages with the same partitioner and the same slice + * grammar the Test Core matrix uses. `count` is written because the + * partitioner checks it against `items.length`. + */ +export function packageListDocument(repoRoot = REPO_ROOT) { + const items = tierPackages(repoRoot).map(({ name, path: dir }) => ({ name, path: dir })); + return { packages: { count: items.length, items } }; +} + +// --------------------------------------------------------------------------- +// `--check`: judged by what vitest COLLECTS, never by re-reading a config +// --------------------------------------------------------------------------- + +/** + * The verdict for one package, from the three lists: the tier files on disk, + * and what `vitest list --filesOnly` returned under each setting. Pure, so the + * self-test pins every direction without spawning vitest. + * + * Returns the problems, empty when the package honours the switch. + */ +export function judgeCollection(name, tierFiles, queueListed, nightlyListed) { + const problems = []; + const tier = new Set(tierFiles); + const leakedIntoQueue = queueListed.filter((f) => tier.has(f)); + if (leakedIntoQueue.length > 0) { + problems.push( + `${name}: under ${TIER_ENV}=queue vitest still collects ${leakedIntoQueue.length} nightly-tier file(s) -- ` + + `they would run in the merge queue: ${leakedIntoQueue.slice(0, 5).join(', ')}${leakedIntoQueue.length > 5 ? ', …' : ''}` + ); + } + const strayInNightly = nightlyListed.filter((f) => !tier.has(f)); + if (strayInNightly.length > 0) { + problems.push( + `${name}: under ${TIER_ENV}=nightly vitest collects ${strayInNightly.length} file(s) outside the nightly tiers -- ` + + `the nightly would run more than the two tiers: ${strayInNightly.slice(0, 5).join(', ')}${strayInNightly.length > 5 ? ', …' : ''}` + ); + } + const listedNightly = new Set(nightlyListed); + const missingFromNightly = tierFiles.filter((f) => !listedNightly.has(f)); + if (missingFromNightly.length > 0) { + problems.push( + `${name}: under ${TIER_ENV}=nightly vitest does NOT collect ${missingFromNightly.length} nightly-tier file(s) on disk -- ` + + `they would run nowhere: ${missingFromNightly.slice(0, 5).join(', ')}${missingFromNightly.length > 5 ? ', …' : ''}` + ); + } + return problems; +} + +/** `vitest list --filesOnly` in `pkgDir` under `mode`, package-relative paths, sorted. */ +function vitestListedFiles(pkgDir, mode) { + const require = createRequire(path.join(pkgDir, 'package.json')); + let vitestEntry; + try { + vitestEntry = path.resolve(path.dirname(require.resolve('vitest/package.json')), 'vitest.mjs'); + } catch { + throw new Error(`${pkgDir}: vitest is not resolvable from this package -- install the workspace before --check`); + } + const out = execFileSync(process.execPath, [vitestEntry, 'list', '--filesOnly'], { + cwd: pkgDir, + env: { ...process.env, [TIER_ENV]: mode }, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + maxBuffer: 64 * 1024 * 1024, + }); + return out + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => line.replace(/^\[[^\]]+\]\s+/, '')) + .sort(); +} + +/** + * Every tier-owning package, measured under both settings. Returns the + * problems; an empty list means every such package honours the switch. + */ +export function checkTierPackages(repoRoot = REPO_ROOT) { + const problems = []; + const owners = tierPackages(repoRoot); + for (const { name, path: dir, files } of owners) { + const pkgDir = path.join(repoRoot, dir); + let queueListed; + let nightlyListed; + try { + queueListed = vitestListedFiles(pkgDir, 'queue'); + nightlyListed = vitestListedFiles(pkgDir, 'nightly'); + } catch (cause) { + problems.push(`${name}: could not ask vitest what it collects (${cause.message.split('\n')[0]})`); + continue; + } + problems.push(...judgeCollection(name, files, queueListed, nightlyListed)); + console.error( + `nightly-tiers --check: ${name} owns ${files.length} nightly-tier file(s); ` + + `vitest collects ${queueListed.length} under queue, ${nightlyListed.length} under nightly` + ); + } + return { owners, problems }; +} + +// --------------------------------------------------------------------------- +// `--failing-files`: the tier files a run's vitest JSON reports call failed +// --------------------------------------------------------------------------- + +/** Every `*.json` under `dir`, recursively, absolute, sorted. */ +function jsonFilesUnder(dir) { + const out = []; + const walk = (current) => { + for (const entry of readdirSync(current, { withFileTypes: true })) { + const abs = path.join(current, entry.name); + if (entry.isDirectory()) walk(abs); + else if (entry.name.endsWith('.json')) out.push(abs); + } + }; + if (existsSync(dir)) walk(dir); + return out.sort(); +} + +/** + * Map the ABSOLUTE `testResults[].name` a vitest JSON report carries (a path on + * the runner that wrote it) back to a repo-relative path, by the longest + * `/` suffix it ends with. Pure; `known` is the + * repo-relative tier-file list `tierPackages()` yields. + */ +export function failingTierFilesFromReports(reports, known) { + const knownSet = new Set(known); + const failed = new Set(); + const unmapped = new Set(); + for (const report of reports) { + const results = Array.isArray(report?.testResults) ? report.testResults : []; + for (const result of results) { + if (result?.status !== 'failed') continue; + const name = String(result.name ?? '').replace(/\\/g, '/'); + const hit = [...knownSet].filter((k) => name === k || name.endsWith(`/${k}`)).sort((a, b) => b.length - a.length)[0]; + if (hit) failed.add(hit); + else unmapped.add(name); + } + } + return { failed: [...failed].sort(), unmapped: [...unmapped].sort() }; +} + +export function failingTierFiles(reportDir, repoRoot = REPO_ROOT) { + const known = tierPackages(repoRoot).flatMap(({ path: dir, files }) => files.map((f) => `${dir}/${f}`)); + const reports = []; + for (const file of jsonFilesUnder(reportDir)) { + try { + reports.push(JSON.parse(readFileSync(file, 'utf8'))); + } catch (cause) { + throw new Error(`${file}: not a vitest JSON report (${cause.message})`); + } + } + return { reportsRead: reports.length, ...failingTierFilesFromReports(reports, known) }; +} + +// --------------------------------------------------------------------------- +// Self-test +// --------------------------------------------------------------------------- + +const SELF_TEST_VERDICT = Symbol('nightly-tiers self-test reached its verdict'); + +function assert(cond, msg) { + if (!cond) throw new Error(`nightly-tiers self-test: ${msg}`); +} + +function assertThrows(fn, re, msg) { + let threw = null; + try { + fn(); + } catch (e) { + threw = e; + } + assert(threw !== null, `${msg} -- did not throw`); + assert(re.test(threw.message), `${msg} -- threw the wrong message: ${threw.message}`); +} + +function eq(a, b) { + return JSON.stringify(a) === JSON.stringify(b); +} + +export function selfTest() { + let cases = 0; + const check = (fn) => { + fn(); + cases++; + }; + + // -- the regex -------------------------------------------------------------- + for (const [file, expected] of [ + ['test/foo.e2e.test.ts', true], + ['src/bar.live.test.ts', true], + ['a.e2e.spec.mts', true], + ['a.live.test.tsx', true], + ['a.e2e.test.cjs', true], + ['test/foo.e2e.test.ts.snap', false], // the loose `*.e2e.test.*` would match this + ['test/foo.e2e.test.d.ts', false], + ['test/foo.e2e.ts', false], + ['test/foo.test.ts', false], + ['src/live-dialect-matrix.isolation.test.ts', false], // substring "live", not a tier + ['src/seed-tenancy-backfill.live-mysql.test.ts', false], + ['src/lint-liveness-properties.test.ts', false], + ['src/email-plugin.queue-delivery.test.ts', false], + ['src/server-timing-e2e.test.ts', false], // "e2e" without the dotted tier + ['e2e/showcase-smoke.spec.ts', false], // a directory is not a tier + ]) { + check(() => assert(isNightlyTierFile(file) === expected, `isNightlyTierFile(${file}) should be ${expected}`)); + } + + // -- the switch ------------------------------------------------------------- + check(() => assert(readTierMode({}) === 'queue', 'unset reads as queue')); + check(() => assert(readTierMode({ [TIER_ENV]: '' }) === 'queue', 'empty reads as queue')); + check(() => assert(readTierMode({ [TIER_ENV]: 'queue' }) === 'queue', 'queue reads as queue')); + check(() => assert(readTierMode({ [TIER_ENV]: 'nightly' }) === 'nightly', 'nightly reads as nightly')); + for (const bad of ['NIGHTLY', ' nightly', 'night', 'e2e', 'true', '1']) { + check(() => assertThrows(() => readTierMode({ [TIER_ENV]: bad }), /is not a test-tier setting/, `refuses ${JSON.stringify(bad)}`)); + } + + // -- selection ---------------------------------------------------------------- + const pop = ['a.test.ts', 'b.e2e.test.ts', 'c.live.spec.ts', 'd/e.test.ts']; + check(() => assert(eq(selectTierFiles(pop, 'queue'), ['a.test.ts', 'd/e.test.ts']), 'queue drops the tiers')); + check(() => assert(eq(selectTierFiles(pop, 'nightly'), ['b.e2e.test.ts', 'c.live.spec.ts']), 'nightly keeps only the tiers')); + check(() => + assert( + eq([...selectTierFiles(pop, 'queue'), ...selectTierFiles(pop, 'nightly')].sort(), [...pop].sort()), + 'queue and nightly partition the population' + ) + ); + check(() => assertThrows(() => selectTierFiles(pop, 'all'), /mode must be one of/, 'refuses an unknown mode')); + + // -- the population, on a fixture workspace --------------------------------- + const root = mkdtempSync(path.join(tmpdir(), 'test-tiers-')); + try { + writeFileSync(path.join(root, 'pnpm-workspace.yaml'), 'packages:\n - packages/*\n - packages/outer/*\n'); + const mk = (dir, name, files) => { + mkdirSync(path.join(root, dir), { recursive: true }); + writeFileSync(path.join(root, dir, 'package.json'), JSON.stringify({ name })); + for (const f of files) { + mkdirSync(path.dirname(path.join(root, dir, f)), { recursive: true }); + writeFileSync(path.join(root, dir, f), ''); + } + }; + mk('packages/owner', '@x/owner', ['test/a.e2e.test.ts', 'test/b.test.ts', 'src/c.live.test.ts', 'node_modules/dep/d.e2e.test.ts', 'dist/e.e2e.test.js']); + mk('packages/plain', '@x/plain', ['src/only.test.ts', 'src/live-dialect.test.ts']); + mk('packages/outer', '@x/outer', ['outer.test.ts']); + mk('packages/outer/inner', '@x/inner', ['inner.live.test.ts']); + const owners = tierPackages(root); + check(() => assert(eq(owners.map((o) => o.name), ['@x/inner', '@x/owner']), `owners are the two tier-owning packages, got ${JSON.stringify(owners.map((o) => o.name))}`)); + check(() => assert(eq(owners.find((o) => o.name === '@x/owner').files, ['src/c.live.test.ts', 'test/a.e2e.test.ts']), 'node_modules and dist are skipped, files sorted')); + check(() => assert(eq(owners.find((o) => o.name === '@x/inner').files, ['inner.live.test.ts']), 'a nested package is walked as its own')); + check(() => assert(!owners.some((o) => o.name === '@x/outer'), 'the outer package does not inherit the nested package\'s tier file')); + const doc = packageListDocument(root); + check(() => assert(doc.packages.count === 2 && doc.packages.items.length === 2, 'the document counts what it lists')); + check(() => assert(eq(doc.packages.items[1], { name: '@x/owner', path: 'packages/owner' }), 'items carry name and repo-relative path')); + + // -- failing files, from a report shaped like vitest's JSON reporter ------- + const known = owners.flatMap((o) => o.files.map((f) => `${o.path}/${f}`)); + const report = { + testResults: [ + { name: '/home/runner/work/r/r/packages/owner/test/a.e2e.test.ts', status: 'failed' }, + { name: '/home/runner/work/r/r/packages/owner/src/c.live.test.ts', status: 'passed' }, + { name: 'C:\\w\\r\\packages\\outer\\inner\\inner.live.test.ts', status: 'failed' }, + { name: '/home/runner/work/r/r/packages/owner/test/b.test.ts', status: 'failed' }, + ], + }; + const verdict = failingTierFilesFromReports([report], known); + check(() => assert(eq(verdict.failed, ['packages/outer/inner/inner.live.test.ts', 'packages/owner/test/a.e2e.test.ts']), `failed files map by suffix, got ${JSON.stringify(verdict.failed)}`)); + check(() => assert(eq(verdict.unmapped, ['/home/runner/work/r/r/packages/owner/test/b.test.ts']), 'a failed non-tier file is reported as unmapped, never dropped')); + check(() => assert(eq(failingTierFilesFromReports([{}], known).failed, []), 'a report with no testResults yields nothing')); + const reportDir = path.join(root, 'reports'); + mkdirSync(path.join(reportDir, 'shard-1'), { recursive: true }); + writeFileSync(path.join(reportDir, 'shard-1', 'owner.json'), JSON.stringify(report)); + const fromDisk = failingTierFiles(reportDir, root); + check(() => assert(fromDisk.reportsRead === 1 && eq(fromDisk.failed, verdict.failed), 'the disk walk reads the same verdict')); + check(() => assert(failingTierFiles(path.join(root, 'no-such-dir'), root).reportsRead === 0, 'a missing report dir is zero reports, not a crash')); + writeFileSync(path.join(reportDir, 'shard-1', 'broken.json'), '{not json'); + check(() => assertThrows(() => failingTierFiles(reportDir, root), /not a vitest JSON report/, 'a malformed report is refused loudly')); + } finally { + rmSync(root, { recursive: true, force: true }); + } + + // -- the collection judge, every direction ------------------------------------ + const tier = ['test/a.e2e.test.ts', 'test/b.live.test.ts']; + check(() => assert(eq(judgeCollection('p', tier, ['test/c.test.ts'], tier), []), 'a package that honours the switch has no problems')); + check(() => assert(judgeCollection('p', tier, ['test/c.test.ts', 'test/a.e2e.test.ts'], tier).some((p) => /still collects 1 nightly-tier/.test(p)), 'a tier file collected under queue is a leak')); + check(() => assert(judgeCollection('p', tier, ['test/c.test.ts'], [...tier, 'test/c.test.ts']).some((p) => /outside the nightly tiers/.test(p)), 'a non-tier file collected under nightly is a stray')); + check(() => assert(judgeCollection('p', tier, ['test/c.test.ts'], ['test/a.e2e.test.ts']).some((p) => /does NOT collect 1 nightly-tier/.test(p)), 'a tier file missing from nightly runs nowhere')); + check(() => assert(judgeCollection('p', tier, [], []).length === 1, 'an empty nightly collection is exactly one problem: the tier files run nowhere')); + + const FLOOR = 45; + assert(cases >= FLOOR, `${cases} cases ran, below the pinned floor of ${FLOOR} -- cases stopped running`); + console.log(`\u2713 nightly-tiers self-test: ${cases} cases pass.`); + return SELF_TEST_VERDICT; +} + +// --------------------------------------------------------------------------- +// CLI +// --------------------------------------------------------------------------- + +function usage() { + console.error( + 'usage:\n' + + ' node scripts/nightly-tiers.mjs --self-test\n' + + ' node scripts/nightly-tiers.mjs --packages\n' + + ' node scripts/nightly-tiers.mjs --check\n' + + ' node scripts/nightly-tiers.mjs --failing-files ' + ); + process.exit(2); +} + +if (isEntrypoint(import.meta.url)) { + const argv = process.argv.slice(2); + if (argv.includes('--self-test')) { + if (selfTest() !== SELF_TEST_VERDICT) { + console.error( + '\n\u2717 nightly-tiers self-test: selfTest() returned without reaching its verdict,\n' + + 'so no success line was printed. Exiting 0 here would report a self-test\n' + + 'that never finished as a self-test that passed.\n' + ); + process.exit(1); + } + } else if (argv[0] === '--packages' && argv.length === 1) { + const doc = packageListDocument(); + for (const { name, files } of tierPackages()) { + console.error(`nightly-tiers: ${name} owns ${files.length} nightly-tier test file(s)`); + } + console.error(`nightly-tiers: ${doc.packages.count} package(s) own nightly-tier test files`); + process.stdout.write(`${JSON.stringify(doc, null, 2)}\n`); + } else if (argv[0] === '--check' && argv.length === 1) { + const { owners, problems } = checkTierPackages(); + if (problems.length > 0) { + console.error(`\n\u2717 nightly-tiers --check: ${problems.length} problem(s)\n`); + for (const p of problems) console.error(` - ${p}`); + console.error( + '\nA package that owns nightly-tier test files must select them through ' + + `${TIER_ENV} (read once, in scripts/nightly-tiers.mjs). Otherwise those files run in the ` + + 'merge queue and the whole suite runs under the nightly -- both silently.' + ); + process.exit(1); + } + console.log(`\u2713 nightly-tiers --check: ${owners.length} tier-owning package(s) honour ${TIER_ENV} (judged by what vitest collects).`); + } else if (argv[0] === '--failing-files' && argv.length === 2) { + const { reportsRead, failed, unmapped } = failingTierFiles(path.resolve(argv[1])); + console.error(`nightly-tiers: read ${reportsRead} vitest JSON report(s); ${failed.length} nightly-tier file(s) failed`); + for (const f of failed) process.stdout.write(`${f}\n`); + for (const u of unmapped) console.error(`nightly-tiers: failed file outside the nightly tiers (not mapped): ${u}`); + } else { + usage(); + } +} diff --git a/turbo.json b/turbo.json index 7a424f0721..57c50850f2 100644 --- a/turbo.json +++ b/turbo.json @@ -11,6 +11,7 @@ "test": { "dependsOn": ["^build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": ["$TURBO_DEFAULT$", "!dist/**", "!coverage/**", "!.turbo/**"] }, "typecheck": { @@ -26,6 +27,7 @@ "@objectstack/service-settings#test": { "dependsOn": ["^build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -38,6 +40,7 @@ "@objectstack/metadata#test": { "dependsOn": ["build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -50,6 +53,7 @@ "@objectstack/spec#test": { "dependsOn": ["^build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -90,6 +94,7 @@ "@objectstack/core#test": { "dependsOn": ["^build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -101,6 +106,7 @@ "@objectstack/types#test": { "dependsOn": ["^build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -112,11 +118,14 @@ "@objectstack/cli#test": { "dependsOn": ["build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", "!coverage/**", "!.turbo/**", + "$TURBO_ROOT$/scripts/nightly-tiers.mjs", + "$TURBO_ROOT$/scripts/nightly-tiers.d.mts", "$TURBO_ROOT$/packages/verify/src/**", "$TURBO_ROOT$/packages/plugins/plugin-security/src/**", "$TURBO_ROOT$/packages/services/service-cluster/src/**", @@ -150,6 +159,7 @@ "@objectstack/client#test": { "dependsOn": ["^build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -176,6 +186,7 @@ "@objectstack/lint#test": { "dependsOn": ["^build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -202,6 +213,7 @@ "^build" ], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -216,6 +228,7 @@ "^build" ], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -228,6 +241,7 @@ "@objectstack/platform-objects#test": { "dependsOn": ["^build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -243,6 +257,7 @@ "@objectstack/mcp#test": { "dependsOn": ["^build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -255,6 +270,7 @@ "@objectstack/runtime#test": { "dependsOn": ["^build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -269,6 +285,7 @@ "@objectstack/objectql#test": { "dependsOn": ["^build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -281,6 +298,7 @@ "@objectstack/driver-sql#test": { "dependsOn": ["^build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -293,6 +311,7 @@ "@objectstack/example-showcase#test": { "dependsOn": ["^build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -305,6 +324,7 @@ "@objectstack/plugin-auth#test": { "dependsOn": ["^build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -329,6 +349,7 @@ "@objectstack/trigger-api#test": { "dependsOn": ["^build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -340,6 +361,7 @@ }, "@objectstack/trigger-record-change#test": { "dependsOn": ["^build"], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -354,6 +376,7 @@ }, "@objectstack/plugin-approvals#test": { "dependsOn": ["^build"], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -368,6 +391,7 @@ "@objectstack/plugin-security#test": { "dependsOn": ["^build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -380,6 +404,7 @@ "@objectstack/dogfood#test": { "dependsOn": ["^build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -405,6 +430,7 @@ "@objectstack/rest#test": { "dependsOn": ["^build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -420,6 +446,7 @@ "@objectstack/formula#test": { "dependsOn": ["^build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -432,6 +459,7 @@ "@objectstack/metadata-protocol#test": { "dependsOn": ["^build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -443,6 +471,7 @@ "@objectstack/downstream-contract#test": { "dependsOn": ["^build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", @@ -455,6 +484,7 @@ "create-objectstack#test": { "dependsOn": ["^build"], "outputs": [], + "env": ["OS_TEST_TIERS"], "inputs": [ "$TURBO_DEFAULT$", "!dist/**", From 770dd18205962f48f35da1bec368a1ca882e25b7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 03:25:30 +0000 Subject: [PATCH 2/3] ci(nightly-tiers): pull_request paths trigger and a path-less population declaration dispatch-gates refuses a family reachable only from a scheduled workflow, so the nightly carries the patrol posture: a pull_request trigger paths-filtered to its own file and the switch reader. The reader declares no-path-population (its self-test drives a temp fixture) and resolves vitest's CLI entry by walking up from require.resolve('vitest') instead of spelling a subpath the derivation read as a population. Concurrency keyed per ref so a PR run never queues behind main's. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_019RfFHiRCSs3JXLK4cwcfox --- .github/workflows/test-nightly-tiers.yml | 36 ++++++++++++++++++------ scripts/nightly-tiers.mjs | 29 ++++++++++++++++--- 2 files changed, 53 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test-nightly-tiers.yml b/.github/workflows/test-nightly-tiers.yml index 8f4d230248..b399cb95e2 100644 --- a/.github/workflows/test-nightly-tiers.yml +++ b/.github/workflows/test-nightly-tiers.yml @@ -62,13 +62,14 @@ name: Nightly Tiers # regex over the log — the log is presentation, the report is the contract. # # ⛔ A run on any ref but `main` NEVER writes to the board — a -# `workflow_dispatch` on a branch proves the transport, the sharding and the -# rendering on a real runner and publishes the would-be card to the run -# summary instead. That is how the card that landed this was smoke-tested. +# `workflow_dispatch` on a branch, or the `pull_request` run below, proves the +# transport, the sharding and the rendering on a real runner and publishes the +# would-be card to the run summary instead. That is how the card that landed +# this was smoke-tested. # # ⛔ This job is NOT a required context and must never become one: it runs on -# no pull request at all, so a ruleset naming it would block every PR forever -# on a check that cannot report. +# no pull request but one editing its own two files, so a ruleset naming it +# would block every other PR forever on a check that cannot report. on: schedule: @@ -79,6 +80,23 @@ on: # (05:00) have started and well before `showcase-smoke` (07:00). - cron: '29 5 * * *' workflow_dispatch: {} + # ⛔ PATHS-FILTERED TO THIS FILE AND THE SWITCH READER IT INVOKES, and that + # filter is the whole reason this trigger is allowed to exist: a change to + # the nightly itself is exercised before it merges — the posture every patrol + # in this repo keeps, and the one `scripts/pm/dispatch-gates.mjs` pins + # tree-wide (a family reachable only from a scheduled workflow is refused + # there by name). A trigger that fires only on a pull request editing these + # two files puts ≈ 15 minutes of e2e on no unrelated PR's critical path. + # + # ⛔ Do NOT widen this list toward the tests themselves: a `packages/**` + # entry here would put the two tiers back on the per-PR path, which is the + # thing the card this landed removed. A `pull_request` run never writes to + # the board — its `github.ref` is the merge ref, not `main`, and the board + # write is gated on `main` by name. + pull_request: + paths: + - '.github/workflows/test-nightly-tiers.yml' + - 'scripts/nightly-tiers.mjs' # Least privilege. The shards read the repo and publish artifacts of their own # run; the report job writes issues and nothing else — no label on anybody's @@ -87,10 +105,12 @@ permissions: contents: read issues: write -# One nightly at a time. A dispatch overlapping the schedule would have two -# `report` jobs racing the same lookup, and the loser would mint a duplicate. +# One nightly at a time PER REF. A dispatch overlapping the schedule on `main` +# would have two `report` jobs racing the same lookup, and the loser would mint +# a duplicate; a `pull_request` run never writes, so it need not queue behind +# `main`'s run — keying the group by ref keeps exactly the writers serial. concurrency: - group: test-nightly-tiers + group: test-nightly-tiers-${{ github.ref }} cancel-in-progress: false env: diff --git a/scripts/nightly-tiers.mjs b/scripts/nightly-tiers.mjs index 0bbfd66549..7edf6b5a29 100644 --- a/scripts/nightly-tiers.mjs +++ b/scripts/nightly-tiers.mjs @@ -90,6 +90,8 @@ import { fileURLToPath } from 'node:url'; import { isEntrypoint } from './invoked-as.mjs'; import { workspacePackageDirs } from './workspace-enumerator.mjs'; +// dispatch-gates: no-path-population -- the --self-test (the one invocation a pull request runs, through test-nightly-tiers.yml's paths trigger) drives a temp fixture workspace and reads no tracked file; --packages, --check and --failing-files sweep the live workspace only inside the nightly run itself, which is placed by that workflow's own paths trigger, so a hint here would be a second spelling of one population + const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); // --------------------------------------------------------------------------- @@ -238,15 +240,34 @@ export function judgeCollection(name, tierFiles, queueListed, nightlyListed) { return problems; } -/** `vitest list --filesOnly` in `pkgDir` under `mode`, package-relative paths, sorted. */ -function vitestListedFiles(pkgDir, mode) { +/** + * The `vitest.mjs` CLI entry of the vitest THIS package resolves: from its main + * entry, walk up to the directory whose manifest is named `vitest`. Resolved + * rather than spelled as a subpath so the module carries no path-shaped + * literal for the dispatch derivation to read as a population declaration. + */ +function vitestCliEntry(pkgDir) { const require = createRequire(path.join(pkgDir, 'package.json')); - let vitestEntry; + let dir; try { - vitestEntry = path.resolve(path.dirname(require.resolve('vitest/package.json')), 'vitest.mjs'); + dir = path.dirname(require.resolve('vitest')); } catch { throw new Error(`${pkgDir}: vitest is not resolvable from this package -- install the workspace before --check`); } + for (;;) { + const manifest = path.join(dir, 'package.json'); + if (existsSync(manifest) && JSON.parse(readFileSync(manifest, 'utf8')).name === 'vitest') { + return path.join(dir, 'vitest.mjs'); + } + const parent = path.dirname(dir); + if (parent === dir) throw new Error(`${pkgDir}: walked to the filesystem root without finding vitest's package root`); + dir = parent; + } +} + +/** `vitest list --filesOnly` in `pkgDir` under `mode`, package-relative paths, sorted. */ +function vitestListedFiles(pkgDir, mode) { + const vitestEntry = vitestCliEntry(pkgDir); const out = execFileSync(process.execPath, [vitestEntry, 'list', '--filesOnly'], { cwd: pkgDir, env: { ...process.env, [TIER_ENV]: mode }, From 07464ab2279e4f5e18ccb3bb682359d14a706f67 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 04:44:39 +0000 Subject: [PATCH 3/3] test(cli): the secret-orphans driver contract declares the key its production-posture boot needs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test sets NODE_ENV=production and boots a real kernel; LocalCryptoProvider then needs a stable key, which this test never supplied. It was green only when an earlier dev-mode boot in the same job had persisted one to the runner's home directory (the ordering coupling serve-process.ts documents) — and with the e2e tier no longer in the queue run, that sibling is gone. Set OS_SECRET_KEY to a fresh random value in beforeAll, restored in afterAll beside NODE_ENV; nothing is written to disk and nothing is depended on from a sibling. Ruled by the seat for exactly this one file. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_019RfFHiRCSs3JXLK4cwcfox --- .../src/commands/secret/orphans.driver-contract.test.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/commands/secret/orphans.driver-contract.test.ts b/packages/cli/src/commands/secret/orphans.driver-contract.test.ts index 5a8a975016..7cc7e04053 100644 --- a/packages/cli/src/commands/secret/orphans.driver-contract.test.ts +++ b/packages/cli/src/commands/secret/orphans.driver-contract.test.ts @@ -44,6 +44,7 @@ */ import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'; +import { randomBytes } from 'node:crypto'; import { mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; @@ -117,10 +118,15 @@ describe('os secret orphans — the concrete driver behind both reads (#14843)', } savedEnv.OS_ARTIFACT_PATH = process.env.OS_ARTIFACT_PATH; savedEnv.NODE_ENV = process.env.NODE_ENV; + savedEnv.OS_SECRET_KEY = process.env.OS_SECRET_KEY; // Deliberately absent: no compiled artifact, so the boot is the bare data // stack plus the two plugins the command passes. process.env.OS_ARTIFACT_PATH = join(dir, 'dist', 'objectstack.json'); process.env.NODE_ENV = 'production'; + // The key this production-posture boot needs, declared here rather than + // inherited from a sibling's persisted `$HOME/.objectstack/dev-crypto-key` + // (#16491): a fresh value per run, never written to disk. + process.env.OS_SECRET_KEY = randomBytes(32).toString('hex'); // The command does not pass `projectRoot`, so its boot takes `process.cwd()` // for its state directory. Stand in the tempdir so the run under test keeps // its state there instead of in whatever directory vitest started in. @@ -154,7 +160,7 @@ describe('os secret orphans — the concrete driver behind both reads (#14843)', if (savedEnv[key] === undefined) delete process.env[key]; else process.env[key] = savedEnv[key]; } - for (const key of ['OS_ARTIFACT_PATH', 'NODE_ENV'] as const) { + for (const key of ['OS_ARTIFACT_PATH', 'NODE_ENV', 'OS_SECRET_KEY'] as const) { if (savedEnv[key] === undefined) delete process.env[key]; else process.env[key] = savedEnv[key]; }