diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08ad09856c..ab4a10def0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -350,15 +350,17 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - # PRs: only test packages affected by the diff against the PR base. - # spec sits at the root of the dependency graph, so spec-touching PRs - # still run (close to) everything — but the many PRs that don't touch - # spec skip the bulk of the 75-package matrix. Push to main and - # merge-queue builds partition the FULL package list instead: the queue - # result IS the next main, so it gets main's validation, not the PR's - # affected-only subset. (Spec's suite runs here plain / uninstrumented; - # the coverage-instrumented pass lives in the nightly coverage-nightly - # workflow.) + # PRs and merge-queue builds: only test packages affected by the diff — + # against the PR base on pull_request, against the group's own + # `base_sha` on merge_group (#16453). spec sits at the root of the + # dependency graph, so spec-touching diffs still run (close to) + # everything — but the many that don't touch spec skip the bulk of the + # 75-package matrix, and at ~120 queue merges a day that is the bulk of + # the queue's test compute. Push to main partitions the FULL package + # list: it gates nobody, and its full run is the ground truth the + # shard-timings refresh reads. (Spec's suite runs here plain / + # uninstrumented; the coverage-instrumented pass lives in the nightly + # coverage-nightly workflow.) # # !@objectstack/dogfood: the ~7½-minute dogfood suite is the dedicated # Dogfood job's whole purpose, and both jobs run under the same `core` @@ -415,127 +417,26 @@ jobs: # `turbo ls`'s internal choice of dot-ness — undocumented, and # `turbo ls` is experimental (see above). Resolving to a commit here # leaves turbo no choice to make. + # The selection itself lives in scripts/ci/select-shard-packages.sh: + # environment in, `$RUNNER_TEMP/turbo-ls.json` out. This step only + # exports the event's fields, so the same script -- every branch of + # it -- runs locally and under its self-test (#16453). Per event: + # pull_request affected set against merge-base(origin/, HEAD) + # merge_group affected set against the group's `base_sha`, unioned + # with the same cross-package scans. An EMPTY set is a + # legitimate docs-only group: it reaches the "No packages + # on this shard" exit below, every shard still attests, + # and Test Core is an honest green. It is NOT the #10057 + # case, which is pull_request-only (the script says why). + # push unchanged: the FULL list. - name: Compute this shard's package set env: - BASE_REF: ${{ github.event.pull_request.base.ref }} - PINNED_BASE_SHA: ${{ github.event.pull_request.base.sha }} + OS_SHARD_EVENT_NAME: ${{ github.event_name }} + OS_SHARD_PR_BASE_REF: ${{ github.event.pull_request.base.ref }} + OS_SHARD_PR_PINNED_BASE_SHA: ${{ github.event.pull_request.base.sha }} + OS_SHARD_MERGE_GROUP_BASE_SHA: ${{ github.event.merge_group.base_sha }} run: | - SCM_BASE='' - if [ "${{ github.event_name }}" = "pull_request" ]; then - if [ -z "$BASE_REF" ]; then - echo "::warning::This pull_request event carries no base branch, so the affected-set diff base cannot be computed." - else - # `fetch-depth: 0` above already makes this resolve — the fetch is - # the guard for the day that changes, not the normal path. - # - # `git cat-file -e` rather than the more idiomatic strict - # `git rev-parse --verify` spelling. That is history rather than - # style, and it is written down because it used to be a live - # hazard: the pre-#6589 check-shard-attestation.mjs classified a - # job as an aggregate GATE when the script's basename and its - # `--verify` flag merely CO-OCCURRED as substrings anywhere in - # the job's joined `run:` text. This job always carries the - # basename (its `--emit` step at the bottom), so spelling - # `--verify` anywhere in this step — comments included, since - # they were part of `run:` — silently reclassified the shard job - # as a gate. - # - # #6589 closed that by construction. Classification is now by - # INVOCATION: within one command the flag must follow the - # script's own name as an argument, the test is applied per STEP - # and never over the job's joined text, and the lexer drops shell - # comments — all three pinned by that script's `--self-test`, - # which is why this comment can now name the flag at all. Either - # spelling is safe here; `git cat-file -e` stays because - # churning it would buy nothing. - if ! git cat-file -e "refs/remotes/origin/$BASE_REF^{commit}" 2>/dev/null; then - git fetch --no-tags --quiet origin "+refs/heads/$BASE_REF:refs/remotes/origin/$BASE_REF" \ - || echo "::warning::Could not fetch origin/$BASE_REF; the merge-base resolution below will decide." - fi - # `if !` rather than a bare assignment on purpose: these steps run - # under `bash -e`, where a failing command substitution kills the - # step with no message at all. - if ! SCM_BASE=$(git merge-base "refs/remotes/origin/$BASE_REF" HEAD); then - SCM_BASE='' - fi - fi - fi - if [ -n "$SCM_BASE" ]; then - # The drift is printed, not just corrected: nothing in this log ever - # said which commit the affected diff started from, which is why the - # decay was invisible. - DRIFT=$(git rev-list --count "$PINNED_BASE_SHA..$SCM_BASE" 2>/dev/null || echo '?') - echo "Affected-set diff base: $SCM_BASE (merge-base of origin/$BASE_REF and HEAD)" - echo "Frozen payload base.sha: $PINNED_BASE_SHA -- $BASE_REF has moved $DRIFT commit(s) since it was frozen, and that drift is exactly what this step used to charge to this PR." - TURBO_SCM_BASE="$SCM_BASE" pnpm exec turbo ls --affected --output=json > "$RUNNER_TEMP/turbo-ls.json" - # `turbo ls --affected` answers "which packages does the dependency - # GRAPH reach from this diff" — and some suites read files the graph - # does not connect them to. spec's api-methods-batch-conformance scan - # walks every `*.object.ts` in the monorepo while spec declares no - # dependency on the packages it judges (nor should it: the scan reads - # source text precisely to avoid inverting the spec -> * direction). - # A platform-objects-only diff therefore left it unrun, and #7769 - # landed a violation on `main` that only PRs touching `spec` ever saw. - # Measured on turbo 2.10.7: 51 packages affected by that diff, spec - # not among them. - # - # So packages that declare a cross-package input radius are unioned - # back in when the diff touches it. The declarations, and the static - # detector that refuses to let a new cross-package scan go - # undeclared, live in the script (`pnpm check:cross-package-test-inputs`). - # - # Failure here falls back to the FULL package list, never to the - # affected-only set: same posture as the merge-base fallback below - # (#6195), and the same reason — the full list is a strict superset, - # so doubt costs minutes rather than coverage. This is the FILTER - # CONTRACT's half 1 applied one layer down. - if ! git diff --name-only "$SCM_BASE" HEAD > "$RUNNER_TEMP/changed-files.txt" \ - || ! node scripts/check-cross-package-test-inputs.mjs \ - --union-into "$RUNNER_TEMP/turbo-ls.json" \ - --changed "$RUNNER_TEMP/changed-files.txt"; then - echo "::warning::Could not union cross-package scans into the affected set; falling back to the full package list for this shard." - pnpm exec turbo ls --output=json > "$RUNNER_TEMP/turbo-ls.json" - # A fourth signal the failure branches do not cover: a producer that - # exits 0 with a wrong, plausible, EMPTY answer. Only that `git - # diff`'s exit STATUS was checked, never its emptiness, so a - # merge-base resolving to something wrong-but-valid gave an empty - # changed-file list -> zero affected packages -> a green shard that - # tested nothing, with every log line reading like a normal quiet PR - # (#10057). The shard attestation (#6082) does not cover it: it - # attests "shard N ran and every step passed", which is exactly what - # a shard that tested nothing does. - # - # Empty is decidable as BROKEN here, and only here: a pull_request - # always differs from its merge-base. At the partitioner zero is - # frequently the CORRECT answer (a docs-only PR genuinely affects no - # package, and with 6 shards a small change legitimately leaves - # shards empty), so a blanket "red on empty" belongs there least of - # all -- this is the one place selection-failed and nothing-selected - # can be told apart. - # - # Scoping this to pull_request needs no `github.event_name` test: - # SCM_BASE is assigned only inside the `pull_request` guard above, so - # this whole `[ -n "$SCM_BASE" ]` branch is unreachable on push and - # merge_group. Those keep taking the full-list path in the `else` - # below, by design. - elif [ ! -s "$RUNNER_TEMP/changed-files.txt" ]; then - echo "::warning::The diff against merge-base $SCM_BASE listed no changed files, which a pull_request cannot legitimately produce; falling back to the full package list for this shard rather than selecting nothing (#10057)." - pnpm exec turbo ls --output=json > "$RUNNER_TEMP/turbo-ls.json" - fi - else - # Falling back to the FULL package list, never to the frozen - # base.sha. This is not the #4690 silent-skip anti-pattern: that is - # about a gate PASSING on input it could not read, and the full list - # is a strict superset of the affected one — this shard still runs - # everything it would have run and more. Cost is minutes; the - # alternative is a red Test Core on a PR with nothing wrong with it. - # Push and merge-queue builds take this branch by design (the queue - # result IS the next main, so it gets main's validation). - if [ "${{ github.event_name }}" = "pull_request" ]; then - echo "::warning::Could not resolve merge-base(origin/$BASE_REF, HEAD); falling back to the full package list for this shard rather than diffing from the frozen base.sha (#6195)." - fi - pnpm exec turbo ls --output=json > "$RUNNER_TEMP/turbo-ls.json" - fi + bash scripts/ci/select-shard-packages.sh node scripts/partition-test-shards.mjs "$RUNNER_TEMP/turbo-ls.json" \ --shard ${{ matrix.shard }}/6 --exclude @objectstack/dogfood \ > "$RUNNER_TEMP/shard-packages.txt" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8f48537093..a3eaaa3eed 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -3235,6 +3235,20 @@ jobs: - name: Shard attestation gate run: pnpm check:shard-attestation + # Test Core package selection self-test (#16453). ci.yml's "Compute this + # shard's package set" step is a thin call into + # scripts/ci/select-shard-packages.sh — environment in, turbo-ls.json + # out — so that the `merge_group` branch, which by construction cannot + # run before a PR is already in the queue, is proved here instead: the + # self-test drives every branch of every event (pull_request, merge_group, + # push) offline, against a throwaway upstream + clone and a fake `turbo` + # on PATH, and pins which `turbo ls` calls were made and with what base, + # whether the cross-package union ran, every `::warning::` fallback and + # the absence of the ones that must not fire (an empty docs-only merge + # group selects nothing WITHOUT the #10057 complaint). Offline; ~2s. + - name: Test Core package selection self-test + run: pnpm check:select-shard-packages + # Aggregator roster gate (#10490). Three required contexts are aggregate # jobs standing in for a set of real jobs — `Test Core` and `Dogfood # Regression Gate` in ci.yml, `TypeScript Type Check` in this file — and diff --git a/package.json b/package.json index 4154ce7292..d9c3b03f81 100644 --- a/package.json +++ b/package.json @@ -132,6 +132,7 @@ "check:node-version": "node scripts/check-node-version.mjs", "check:pnpm-acquisition": "node scripts/check-pnpm-acquisition.mjs --self-test && node scripts/check-pnpm-acquisition.mjs", "check:workflow-status-functions": "node scripts/check-workflow-status-functions.mjs --self-test && node scripts/check-workflow-status-functions.mjs", + "check:select-shard-packages": "bash scripts/ci/select-shard-packages.selftest.sh", "check:shard-attestation": "node scripts/check-shard-attestation.mjs --self-test && node scripts/check-shard-attestation.mjs", "check:required-contexts": "node scripts/check-required-contexts.mjs --self-test && node scripts/check-required-contexts.mjs", "check:cross-package-test-inputs": "node scripts/check-cross-package-test-inputs.mjs --self-test && node scripts/check-cross-package-test-inputs.mjs", diff --git a/scripts/ci/select-shard-packages.selftest.sh b/scripts/ci/select-shard-packages.selftest.sh new file mode 100644 index 0000000000..f2e6c4a19e --- /dev/null +++ b/scripts/ci/select-shard-packages.selftest.sh @@ -0,0 +1,457 @@ +#!/usr/bin/env bash +# +# Self-test for scripts/ci/select-shard-packages.sh (#16453). +# +# The script decides the Test Core package set from the event, and every one +# of its fallbacks is a `::warning::` plus the FULL list -- so the property +# under test is not "does it exit 0" (it nearly always does) but WHICH branch +# it took: which `turbo ls` calls it made and with what `TURBO_SCM_BASE`, +# whether the cross-package union ran and over which changed-file list, which +# warnings it printed, and what landed in `turbo-ls.json`. Every case pins all +# of those, and pins the ABSENCE of the warnings it must not print (the +# #10057 guard on a merge group is the one that matters most: an empty +# docs-only group has to select nothing without complaint). +# +# Hermetic and offline: a throwaway upstream + clone (+ a shallow clone) under +# $TMPDIR, a fake `pnpm`/`turbo` pair on PATH that answers from control files +# and records every call, and a stub `check-cross-package-test-inputs.mjs` +# inside the fixture that records its argv and appends what it is told to. +# The REAL union script has its own self-test (`check:cross-package-test- +# inputs`); what is pinned here is that the selection script hands it the +# right files and ships its answer. Needs git, node and bash; ~2s. +# +# Run: bash scripts/ci/select-shard-packages.selftest.sh +set -uo pipefail +cd "$(git rev-parse --show-toplevel)" || exit 1 + +# The one repo path this self-test reads, spelled as a quoted repo-relative +# literal on purpose: the dispatch derivation (scripts/pm/dispatch-gates.mjs) +# reads a gate's quoted path literals as the population it watches, so a card +# touching the script schedules this family. Every other path below is a +# throwaway fixture under $TMPDIR, and none is spelled as a repo path -- a +# quoted literal with a slash in it would read as a declared population that reaches +# nothing (check:declared-population-live refuses exactly that). +SCRIPT_REL='scripts/ci/select-shard-packages.sh' +SCRIPT="$PWD/$SCRIPT_REL" +FIX=$(mktemp -d "${TMPDIR:-/tmp}/os-select-shard-selftest.XXXXXX") || exit 1 +trap 'rm -rf "$FIX"' EXIT INT TERM + +fail=0 +cases=0 +checks=0 + +git_q() { + git -c user.name=selftest -c user.email=selftest@example.invalid -c commit.gpgsign=false "$@" +} + +# ── The fakes ─────────────────────────────────────────────────────────────── +mkdir -p "$FIX/bin" +cat > "$FIX/bin/pnpm" <&2 + exit 97 +fi +shift 2 +exec "$FIX/bin/turbo" "\$@" +EOF +cat > "$FIX/bin/turbo" <> "$FIX/turbo-calls.log" + cat "$FIX/affected.json" + exit "\$(cat "$FIX/affected-exit")" + ;; + *) + printf 'full argv=[%s] TURBO_SCM_BASE=%s\n' "\$*" "\${TURBO_SCM_BASE:-unset}" >> "$FIX/turbo-calls.log" + cat "$FIX/full.json" + exit 0 + ;; +esac +EOF +chmod +x "$FIX/bin/pnpm" "$FIX/bin/turbo" + +# The stub union script, committed into the fixture upstream's scripts/ dir. +write_union_stub() { + mkdir -p "$1/scripts" + cat > "$1/scripts/check-cross-package-test-inputs.mjs" < "$FIX/full.json" + +# set_affected ... -- what the fake `turbo ls --affected` answers +set_affected() { + local items='' name + for name in "$@"; do + [ -n "$items" ] && items="$items," + items="$items{\"name\":\"$name\",\"path\":\"$name\"}" + done + printf '{"packageManager":"pnpm@10.0.0","packages":{"count":%d,"items":[%s]}}\n' "$#" "$items" > "$FIX/affected.json" +} +reset_controls() { + set_affected a + printf '0' > "$FIX/affected-exit" + : > "$FIX/union-add" + printf '0' > "$FIX/union-exit" +} + +# ── Fixture repositories ──────────────────────────────────────────────────── +UP="$FIX/upstream" +mkdir -p "$UP" +git_q -C "$UP" init -q +git_q -C "$UP" symbolic-ref HEAD refs/heads/main +mkdir -p "$UP/packages/a" "$UP/docs" +printf '{"name":"fixture","private":true}\n' > "$UP/package.json" +printf 'export const a = 1;\n' > "$UP/packages/a/index.ts" +printf '# guide\n' > "$UP/docs/guide.md" +# The stub union script is part of C0, so every commit carries it unchanged +# and no changed-file list below ever names it. +write_union_stub "$UP" +git_q -C "$UP" add -A +git_q -C "$UP" commit -q -m 'C0: root' +C0=$(git_q -C "$UP" rev-parse HEAD) +printf 'export const a = 2;\n' > "$UP/packages/a/index.ts" +git_q -C "$UP" commit -q -am 'C1: a moves' +C1=$(git_q -C "$UP" rev-parse HEAD) +# A merge group's base is fetched BY SHA; a local upstream has to be told to +# serve one (GitHub does so for every reachable commit -- actions/checkout +# itself fetches the merge ref by sha). +git_q -C "$UP" config uploadpack.allowReachableSHA1InWant true +git_q -C "$UP" config uploadpack.allowAnySHA1InWant true + +REPO="$FIX/repo" +git_q clone -q "$UP" "$REPO" +git_q -C "$REPO" checkout -q -b feature +printf 'export const leaf = 1;\n' > "$REPO/packages/a/leaf.ts" +git_q -C "$REPO" add -A +git_q -C "$REPO" commit -q -m 'F1: leaf change on the feature branch' +F1=$(git_q -C "$REPO" rev-parse HEAD) +# The queue's HEAD: this entry merged onto its base. +git_q -C "$REPO" checkout -q --detach "$C1" +git_q -C "$REPO" merge -q --no-ff -m 'M: feature merged onto C1' feature +M=$(git_q -C "$REPO" rev-parse HEAD) +# A docs-only entry on top of that. +printf '# guide, revised\n' > "$REPO/docs/guide.md" +git_q -C "$REPO" commit -q -am 'D: docs only' +D=$(git_q -C "$REPO" rev-parse HEAD) +# Created AFTER the clone, so the clone has no refs/remotes/origin/release: +# the pull_request fetch path has something to fetch. +git_q -C "$UP" branch release "$C1" + +SHALLOW="$FIX/shallow" +git_q clone -q --depth 1 "file://$UP" "$SHALLOW" 2>/dev/null + +ZEROS=0000000000000000000000000000000000000000 + +# ── The runner and the assertions ─────────────────────────────────────────── +at() { git_q -C "$REPO" checkout -q --detach "$1"; } + +RT='' +rc=0 +# run_case