benchmarks.yml

name: benchmarks

# Smoke-level gate for `benchmarks/` — proves the suite still COMPILES and
# still RUNS against the current `src/` API.  It does not measure anything:
# benchmark numbers from a shared CI runner are noise, and regressions in
# throughput are not what this catches.
#
# Why it exists: nothing used to look at `benchmarks/` at all.  `bun run
# typecheck` uses the build tsconfig, which deliberately EXCLUDES the
# benchmarks; `test.yml` does not even trigger on `benchmarks/**`.  So when
# the free `ask()` helper was removed in favour of `ref.ask()`, ten suites
# were left importing a non-existent export and `bun run bench` stayed dead
# for months without a single red build (#506).
#
# Triggers on `src/**` too, not just `benchmarks/**` — the failure mode is
# a *source* change orphaning a benchmark, which is invisible from the
# benchmark side of the diff.

on:
  # Run on every branch push (feature work merges locally, so a branch push
  # is often the only pre-merge signal), not just `develop`.
  push:
    branches: ['**']
    paths:
      - 'src/**'
      - 'benchmarks/**'
      - 'package.json'
      - '.bun-version'
      - 'bun.lock'
      - 'tsconfig.json'
      - 'tsconfig.bench.json'
      - '.github/workflows/benchmarks.yml'
  pull_request:
    branches: [main, develop]
    paths:
      - 'src/**'
      - 'benchmarks/**'
      - 'package.json'
      - '.bun-version'
      - 'bun.lock'
      - 'tsconfig.json'
      - 'tsconfig.bench.json'
      - '.github/workflows/benchmarks.yml'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

# Least privilege, stated rather than inherited: the repository default is
# read-only today, but a settings flip would otherwise hand a write token to
# a job that installs and runs the whole dependency tree.  #621
permissions:
  contents: read

jobs:
  benchmarks:
    runs-on: ubuntu-latest
    # Smoke mode runs the whole suite in well under a minute locally; the cap
    # is here so a benchmark that HANGS (a cluster that never converges, a
    # worker that never replies) fails the job instead of burning an hour.
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup Bun
        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
        with:
          bun-version-file: .bun-version

      # Frozen, like every other install in CI: an unfrozen resolve can
      # green-light a dependency set no lockfile records, so the check
      # passes against something nobody can reproduce.  #622
      - name: Install
        run: bun install --frozen-lockfile

      - name: Typecheck benchmarks
        # The stricter of the two checks for this failure class: a missing
        # barrel export is a TS2305 here even when the imported binding is
        # never called, whereas Bun happily elides an unused named import at
        # runtime.  Cheap — seconds — so it runs first and fails fast.
        run: bun run typecheck:bench

      - name: Run benchmarks (smoke mode)
        # ACTOR_TS_BENCH_SMOKE=1 collapses every case to a single unwarmed
        # iteration, so this is "does each suite still execute", not a
        # measurement.  Catches what the typecheck cannot: wiring that is
        # type-correct but wrong at runtime.
        #
        # `worker` is excluded on hosted runners for the same reason the
        # worker-thread multi-node test suites are quarantined there (see
        # test.yml + #538): Bun on GitHub's runners cannot respawn functional
        # worker threads after the first, and worker-count-scaling.ts spawns
        # 1 → 2 → 4 → 8 → auto in one process.  It runs locally via
        # `bun run bench:smoke`.  The exclusion shares the quarantine's exit
        # criterion — `nightly-flakes.yml` is what measures it.
        run: bun run bench:smoke -- --exclude=worker

  # The framework-comparison tree (#27) is invisible to every job above: it
  # carries its OWN package.json and lockfile, so `run-all.ts` skips it and
  # both benchmark tsconfigs exclude it.  Without this job it would be checked
  # by nothing at all — which is exactly how ten benchmark suites sat broken on
  # a removed export for months (#506), and the reason that tree's README
  # promises this job exists.
  comparison:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup Bun
        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
        with:
          bun-version-file: .bun-version

      # Two installs, two manifests.  The root one is what the arms import
      # actor-ts from; the nested one carries the frameworks it is measured
      # against, which deliberately never enter the root closure.  Both frozen,
      # for the reason the job above states.  #622
      - name: Install (root)
        run: bun install --frozen-lockfile

      - name: Install (comparison tree)
        run: bun install --frozen-lockfile --cwd benchmarks/comparison

      - name: Typecheck comparison arms
        # The only check that compiles this tree.  It cannot live in
        # `typecheck:bench` or `typecheck:dev`: those resolve imports against
        # the root manifest, and these arms import from the nested one.
        run: bun run typecheck:compare

      - name: Run comparison arms (smoke mode, JavaScript only)
        # `--javascript-only` skips the JVM and .NET arms, which need a JDK and
        # a .NET SDK this runner does not install.  Adding those toolchains
        # would buy nothing: benchmark numbers from a shared runner are noise,
        # and what this job checks is that the arms still EXECUTE against the
        # current `src/` API.
        #
        # Smoke mode writes no result files — a single unwarmed iteration
        # measures the JIT, and letting it reach `results/` would replace a
        # committed nine-round median with noise.
        run: ACTOR_TS_BENCH_SMOKE=1 bun run bench:compare -- --javascript-only