No-assets mode: the full benchmark with zero RCT2 files, Linux images, OpenAI-compatible lane

.dockerignore

@@ -2,3 +2,8 @@ bin/
 build/
 lib/
 obj/
+rust/orct2-agent/target/
+rust/coaster-bench/target/
+rust/coaster-site/target/
+evals/runs/
+evals/library-previews/

CLAUDE.md

@@ -86,6 +86,18 @@ Non-bundled binaries look for `data/` next to the exe. One-time setup:
   See issue #1 and the readme war story.
 - Stalls never get ratings (RatingsCalculationType::Stall); tracked rides need
   a completed test circuit (RideFlag::tested) before ratings compute.
+- No-assets mode: `eval --no-graphics` sets gOpenRCT2NoGraphics so the whole
+  scoring path (park load, placement, testing, ratings, drawability gate) runs
+  with zero RCT2 files — objects come from the bundled JSON pack; only pixels
+  need g1.dat. Screenshots/--render-library are refused, the MCP server drops
+  image tools server-side (Modalities::server_side), and the similarity
+  penalty is inert (no stock library; report says `similarity: null`).
+  Assetless scenario default: test/tests/testdata/parks/BigMapTest.sv6
+  (mostly-open flat grass, cash-rich, builds fine). driver.py mirrors the
+  flag (`--no-graphics`, design mode only) and also takes `--base-url` for
+  any OpenAI-compatible endpoint (e.g. `vllm serve`); `evals/ci/` has the
+  CPU-only Dockerfile, a protocol-success gate (check_run.py), and the CI
+  job shape.
 - Head-to-head driver: `uv run evals/driver.py` (needs ANTHROPIC_API_KEY, or
   `--vertex` with GCP ADC; project defaults from $ANTHROPIC_VERTEX_PROJECT_ID);
   results under `evals/runs/<timestamp>/`. Models get a validate_track_program

evals/ci/Dockerfile

@@ -0,0 +1,90 @@
+# CoasterBench game image for CI: openrct2-cli with the Rust agent bridge,
+# headless-only, zero RCT2 assets (runs eval/MCP with --no-graphics).
+#
+# The image is CPU-only and small (~hundreds of MB); build it once, push to a
+# registry, and pin the digest in CI. The model under test runs elsewhere
+# (e.g. `vllm serve` in a sibling container); this container only hosts the
+# game as an MCP server or batch eval.
+#
+#   docker build -f evals/ci/Dockerfile -t coasterbench-game .
+#   docker run --rm -p 8791:8791 coasterbench-game \
+#       openrct2-cli eval /opt/coasterbench/BigMapTest.sv6 --no-graphics \
+#       --serve 8791 --serve-bind 0.0.0.0
+#
+# Verified: builds and serves on Linux arm64 (podman, Ubuntu 24.04 base) —
+# a container-built oval places, tests, and rates identically to the macOS
+# build. x86_64 not yet exercised.
+
+FROM ubuntu:24.04 AS build
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential cmake ninja-build git curl ca-certificates pkg-config \
+        libpng-dev libzip-dev libssl-dev libcurl4-openssl-dev libicu-dev \
+        nlohmann-json3-dev libflac-dev libogg-dev libvorbis-dev libzstd-dev \
+        libbz2-dev duktape-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Rust toolchain for the agent staticlib (corrosion drives cargo from CMake).
+RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable
+ENV PATH=/root/.cargo/bin:$PATH
+
+COPY . /src
+
+# Headless CLI only: no SDL/OpenGL/UI. TTF and audio downloads are dead weight
+# for an assetless eval. DOWNLOAD_OBJECTS stays ON: the bundled JSON object
+# pack (~11 MB) is what lets parks load without any RCT2 install.
+# The prefix must be set at CONFIGURE time: the objects/asset downloads run
+# from install(CODE) strings that bake CMAKE_INSTALL_FULL_DATADIR when
+# configuring, so `cmake --install --prefix` would land them at the stale
+# /usr/local while everything else moves — exactly the invisible-objects bug.
+RUN cmake -S /src -B /build -G Ninja -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_INSTALL_PREFIX=/usr \
+        -DDISABLE_GUI=ON \
+        -DDISABLE_TTF=ON \
+        -DDISABLE_DISCORD_RPC=ON \
+        -DDISABLE_VERSION_CHECKER=ON \
+        -DDOWNLOAD_OPENSFX=OFF \
+        -DDOWNLOAD_OPENMUSIC=OFF \
+        -DDOWNLOAD_TITLE_SEQUENCES=OFF \
+    && cmake --build /build \
+    # DISABLE_GUI leaves the `graphics` target (g2/fonts/palettes/tracks.dat,
+    # generated by the sprite compiler from in-repo resources) orphaned, but
+    # the install manifest still expects the files — build it explicitly.
+    && cmake --build /build --target graphics \
+    && DESTDIR=/stage cmake --install /build
+
+FROM ubuntu:24.04 AS runtime
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        libpng16-16t64 libzip4t64 libicu74 libcurl4t64 libssl3t64 \
+        libflac12t64 libvorbisfile3 libzstd1 ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=build /stage/usr /usr
+# The checked-in assetless scenario and a known-good smoke program, so the
+# container is self-contained for both serving and zero-GPU smoke runs.
+COPY test/tests/testdata/parks/BigMapTest.sv6 evals/programs/test_oval.json /opt/coasterbench/
+
+EXPOSE 8791
+CMD ["openrct2-cli", "eval", "/opt/coasterbench/BigMapTest.sv6", "--no-graphics", "--serve", "8791", "--serve-bind", "0.0.0.0"]
+
+# Self-contained eval runner: game + driver + preinstalled deps, so a single
+# `docker run` plays a full benchmark against any OpenAI-compatible endpoint
+# with no network needs beyond the endpoint itself. This is the default build
+# target; the lean game-only image is `--target runtime`.
+FROM runtime AS eval
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        python3 python3-venv \
+    && rm -rf /var/lib/apt/lists/* \
+    && python3 -m venv /opt/venv \
+    && /opt/venv/bin/pip install --no-cache-dir 'anthropic[vertex]>=0.40' 'openai>=1.40' 'pillow>=10'
+
+COPY evals/driver.py evals/ci /opt/coasterbench/evals/
+COPY evals/programs /opt/coasterbench/evals/programs
+ENV COASTERBENCH_CLI=/usr/bin/openrct2-cli
+
+# docker run --network host coasterbench-eval \
+#   /opt/venv/bin/python /opt/coasterbench/evals/driver.py \
+#   --base-url http://localhost:8000/v1 --models <model> --no-graphics \
+#   --scenario /opt/coasterbench/BigMapTest.sv6

evals/ci/README.md

@@ -0,0 +1,85 @@
+# CoasterBench in CI (no RCT2 assets)
+
+Everything the eval scores — park loading, track placement, ride testing,
+ratings — runs without any RollerCoaster Tycoon 2 files via `--no-graphics`
+(objects come from the bundled JSON pack; only pixels need `g1.dat`). That
+makes the whole benchmark legally shippable in a CI image. What you give up:
+screenshots (feedback is the eval report only, so contenders run text-only)
+and the stock TD6 library (library mode is unavailable and the similarity
+penalty is inert, so scores are not comparable with asset-full leaderboard
+runs — `run.json` records `no_graphics: true`).
+
+## Pieces
+
+- `Dockerfile` — CPU-only game image (headless `openrct2-cli`, Rust agent,
+  bundled objects, checked-in scenario; ~240 MB). Build once, push, pin the
+  digest. Verified on Linux arm64 (podman): the containerized MCP server
+  builds, tests, and rates a coaster with ratings identical to the macOS
+  build. x86_64 not yet exercised.
+- `check_run.py` — pass/fail gate: every model needs ≥1 round that built and
+  completed a test circuit. Scores are metrics, not assertions (models are
+  nondeterministic; don't gate merges on excitement).
+
+## Shape of a job against vLLM
+
+Nightly / non-blocking is the realistic tier — the wall clock is model
+inference, the game sim is ~7s per 25k-tick round.
+
+```bash
+# 1. serve the model under test (own container/step; needs the GPU)
+vllm serve "$MODEL" --enable-auto-tool-choice --tool-call-parser hermes &
+
+# 2. game + driver are CPU-only
+uv run evals/driver.py \
+    --base-url http://localhost:8000/v1 \
+    --models "$MODEL" --rounds "${ROUNDS:-2}" --no-graphics \
+    --name "ci-${BUILD_NUMBER:-local}"
+
+# 3. gate on protocol success, keep the run dir as the artifact
+python3 evals/ci/check_run.py evals/runs/*-"ci-${BUILD_NUMBER:-local}"
+```
+
+What this exercises end-to-end that parser unit tests don't: forced and named
+`tool_choice` (guided decoding), multi-turn tool loops with tool_result
+round-trips, and large JSON tool arguments (a 148-piece track program is a
+few KB of structured output).
+
+There is also a zero-GPU smoke: replay a canned program with no model at all —
+`openrct2-cli eval test/tests/testdata/parks/BigMapTest.sv6 --no-graphics
+--ticks 25000 --program evals/programs/test_oval.json --out report.json`
+must produce `program.ok == true` and a tested ride.
+
+## Field notes: what the first live runs caught (vLLM 0.25.1, A100)
+
+Findings from the eval's first day out, with honest attribution — kept here
+because they are exactly the failure classes this job exists to surface:
+
+1. **Confirmed vLLM bug — named `tool_choice` silently unenforced**
+   (`poolside_v1` parser, Laguna-S-2.1, thinking disabled): a request
+   forcing `submit_track_program` by name got `validate_track_program`
+   back, 3/3 reproducible probes. The contract requires enforcement or
+   rejection, not a different function. The driver copes by giving the
+   forced-submit step three attempts.
+2. **Unconfirmed anomaly — `tool_choice: "required"` returned zero tool
+   calls** once, on the server's first-ever structured-output request
+   (Qwen2.5-7B, hermes parser). Never reproduced (0/15 probes) and the
+   failing response body wasn't captured, so it is not claimable as a bug.
+   The driver retries up to 3×.
+3. **Harness bug (ours, fixed) — fixed `max_tokens` starves reasoning
+   models**: Laguna spent the entire 8k (then 24k) completion budget on
+   interleaved thinking and hit `finish_reason: "length"` with no tool
+   call, masquerading as finding #2. vLLM behaved correctly. The driver
+   now fails fast with the real cause and takes `--max-tokens` /
+   `--chat-template-kwargs '{"enable_thinking": false}'`. Follow-up ruled
+   out every confound: at TP=8, 512k serving context, a 131k budget, an
+   hour-long client timeout, pod-to-pod networking, and reasoning passed
+   back across turns (vLLM renders assistant `reasoning` into the
+   template — verified via prompt_tokens), turn-one thinking still never
+   terminated, and it also failed to terminate with no tools attached.
+   Conclusion: unbounded reasoning is a property of monolithic one-shot
+   design prompts; thinking-tier contenders belong in the interactive
+   per-piece mode, and one-shot runs should disable thinking.
+4. **Config foot-gun — `generation_config.json` overrides server sampling
+   defaults** (vLLM warns but serves): Qwen ships temp 0.7 / top-p 0.8 /
+   rep-penalty 1.05, so "default" runs are not the sampling you assumed.
+   Pin `--generation-config vllm` for vLLM defaults.

evals/ci/check_run.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+"""CI pass/fail gate for a CoasterBench run directory.
+
+Scores are model quality, not infrastructure health, so CI asserts protocol
+success only: every model must have at least one round whose program built OK
+and whose ride completed a test circuit. Score stays a reported metric.
+
+  usage: check_run.py evals/runs/<run-dir>
+"""
+import json
+import sys
+from pathlib import Path
+
+
+def main() -> int:
+    if len(sys.argv) != 2:
+        print(__doc__, file=sys.stderr)
+        return 2
+    run_dir = Path(sys.argv[1])
+    run = json.loads((run_dir / "run.json").read_text())
+    failures = []
+    for model in run["models"]:
+        reports = sorted(run_dir.glob(f"{model.replace('/', '_')}/round_*/report.json"))
+        tested = []
+        for path in reports:
+            report = json.loads(path.read_text())
+            # Batch runs name the program's ride; MCP-harness runs have no
+            # program, so any tested ride in the round's report counts.
+            ride_id = (report.get("program") or {}).get("ride_id")
+            tested += [
+                r
+                for r in report.get("rides", []) or []
+                if r.get("tested") and (ride_id is None or r["id"] == ride_id)
+            ]
+        best = max((r.get("excitement") or 0.0 for r in tested), default=None)
+        if not reports:
+            failures.append(f"{model}: no rounds ran")
+        elif not tested:
+            failures.append(f"{model}: {len(reports)} round(s), none produced a tested coaster")
+        else:
+            print(f"ok: {model}: {len(tested)}/{len(reports)} rounds tested, best excitement {best:.2f}")
+    for failure in failures:
+        print(f"FAIL: {failure}", file=sys.stderr)
+    return 1 if failures else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())