From ac3978bbb8c0aba3611fbe56a55711cc57ed8f63 Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Fri, 24 Jul 2026 17:07:39 -0400 Subject: [PATCH 01/18] No-assets mode: run the whole eval without RCT2 files (--no-graphics) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Everything the eval scores — park load, placement, testing, ratings, the drawability gate — is pure game logic; only pixels need g1.dat. Proven in-tree by upstream's own RideRatings tests, which load parks and assert exact ratings with NoGraphics set in an assetless CI. - eval --no-graphics: skips LoadBaseGraphics, refuses --capture/ --render-library combos; default (graphics) behavior unchanged. - orct2_host_capture guards cleanly; new orct2_host_graphics_available crosses the ABI (header regenerated). - MCP server intersects IMAGE out of every request's modalities when sprite data is absent: screenshot/best_screenshot vanish from tools/list and refuse calls regardless of what the client advertises; finish_and_test skips the best-shot capture instead of logging errors. - Assetless scenario default: the checked-in BigMapTest.sv6 (mostly open flat grass — probed — cash-rich, builds and tests fine). Verified end-to-end with zero RCT2 assets on this machine: bpb.sv6 rates 134 rides; a 25k-tick eval takes ~7s; a real 148-piece run program behaves identically to the current asset-full build. - driver.py: --no-graphics lane (design mode only, report-only feedback) and --base-url for any OpenAI-compatible endpoint (vLLM serve etc.) via an Anthropic-shaped facade over chat completions; exercised against a wire-format-asserting stub for two full rounds including validate. - evals/ci/: CPU-only Dockerfile (draft, unexercised on Linux), check_run.py protocol-success gate (validated against existing runs), and the nightly-job shape for running under vLLM CI. Similarity penalty is inert without the stock library (report records similarity: null, run.json records no_graphics) so assetless scores are not comparable with asset-full leaderboards. Co-Authored-By: Claude Fable 5 Signed-off-by: Tyler Michael Smith --- .dockerignore | 5 + CLAUDE.md | 12 ++ evals/ci/Dockerfile | 57 ++++++ evals/ci/README.md | 48 +++++ evals/ci/check_run.py | 48 +++++ evals/driver.py | 227 ++++++++++++++++++++- readme.md | 30 +++ rust/orct2-agent/src/host.rs | 10 + rust/orct2-agent/src/mcp.rs | 36 +++- src/openrct2/command_line/EvalCommands.cpp | 19 +- src/openrct2/rustbridge/RustBridge.cpp | 10 + src/openrct2/rustbridge/orct2_agent.h | 7 + 12 files changed, 493 insertions(+), 16 deletions(-) create mode 100644 evals/ci/Dockerfile create mode 100644 evals/ci/README.md create mode 100644 evals/ci/check_run.py diff --git a/.dockerignore b/.dockerignore index 2bcb8a0e79a2..c05da971df39 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,3 +2,8 @@ bin/ build/ lib/ obj/ +rust/orct2-agent/target/ +rust/coaster-bench/target/ +rust/coaster-site/target/ +evals/runs/ +evals/library-previews/ diff --git a/CLAUDE.md b/CLAUDE.md index b8fd9266ad3b..3f75dba66319 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -86,6 +86,18 @@ Non-bundled binaries look for `data/` next to the exe. One-time setup: See issue #1 and the readme war story. - Stalls never get ratings (RatingsCalculationType::Stall); tracked rides need a completed test circuit (RideFlag::tested) before ratings compute. +- No-assets mode: `eval --no-graphics` sets gOpenRCT2NoGraphics so the whole + scoring path (park load, placement, testing, ratings, drawability gate) runs + with zero RCT2 files — objects come from the bundled JSON pack; only pixels + need g1.dat. Screenshots/--render-library are refused, the MCP server drops + image tools server-side (Modalities::server_side), and the similarity + penalty is inert (no stock library; report says `similarity: null`). + Assetless scenario default: test/tests/testdata/parks/BigMapTest.sv6 + (mostly-open flat grass, cash-rich, builds fine). driver.py mirrors the + flag (`--no-graphics`, design mode only) and also takes `--base-url` for + any OpenAI-compatible endpoint (e.g. `vllm serve`); `evals/ci/` has the + CPU-only Dockerfile, a protocol-success gate (check_run.py), and the CI + job shape. - Head-to-head driver: `uv run evals/driver.py` (needs ANTHROPIC_API_KEY, or `--vertex` with GCP ADC; project defaults from $ANTHROPIC_VERTEX_PROJECT_ID); results under `evals/runs//`. Models get a validate_track_program diff --git a/evals/ci/Dockerfile b/evals/ci/Dockerfile new file mode 100644 index 000000000000..41863546ae82 --- /dev/null +++ b/evals/ci/Dockerfile @@ -0,0 +1,57 @@ +# CoasterBench game image for CI: openrct2-cli with the Rust agent bridge, +# headless-only, zero RCT2 assets (runs eval/MCP with --no-graphics). +# +# The image is CPU-only and small (~hundreds of MB); build it once, push to a +# registry, and pin the digest in CI. The model under test runs elsewhere +# (e.g. `vllm serve` in a sibling container); this container only hosts the +# game as an MCP server or batch eval. +# +# docker build -f evals/ci/Dockerfile -t coasterbench-game . +# docker run --rm -p 8791:8791 coasterbench-game \ +# openrct2-cli eval /opt/coasterbench/BigMapTest.sv6 --no-graphics \ +# --serve 8791 --serve-bind 0.0.0.0 +# +# NOTE: draft — authored on macOS, not yet exercised on a Linux builder. + +FROM ubuntu:24.04 AS build + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential cmake ninja-build git curl ca-certificates pkg-config \ + libpng-dev libzip-dev libssl-dev libcurl4-openssl-dev libicu-dev \ + nlohmann-json3-dev libflac-dev libogg-dev libvorbis-dev libzstd-dev \ + libbz2-dev duktape-dev \ + && rm -rf /var/lib/apt/lists/* + +# Rust toolchain for the agent staticlib (corrosion drives cargo from CMake). +RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable +ENV PATH=/root/.cargo/bin:$PATH + +COPY . /src + +# Headless CLI only: no SDL/OpenGL/UI. TTF and audio downloads are dead weight +# for an assetless eval. DOWNLOAD_OBJECTS stays ON: the bundled JSON object +# pack (~11 MB) is what lets parks load without any RCT2 install. +RUN cmake -S /src -B /build -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DDISABLE_GUI=ON \ + -DDISABLE_TTF=ON \ + -DDISABLE_DISCORD_RPC=ON \ + -DDISABLE_VERSION_CHECKER=ON \ + -DDOWNLOAD_OPENSFX=OFF \ + -DDOWNLOAD_OPENMUSIC=OFF \ + -DDOWNLOAD_TITLE_SEQUENCES=OFF \ + && cmake --build /build \ + && DESTDIR=/stage cmake --install /build --prefix /usr + +FROM ubuntu:24.04 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libpng16-16t64 libzip4t64 libicu74 libcurl4t64 libssl3t64 \ + libflac12t64 libvorbisfile3 libzstd1 ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=build /stage/usr /usr +# The checked-in assetless scenario, so the container is self-contained. +COPY test/tests/testdata/parks/BigMapTest.sv6 /opt/coasterbench/ + +EXPOSE 8791 +CMD ["openrct2-cli", "eval", "/opt/coasterbench/BigMapTest.sv6", "--no-graphics", "--serve", "8791", "--serve-bind", "0.0.0.0"] diff --git a/evals/ci/README.md b/evals/ci/README.md new file mode 100644 index 000000000000..37703fb6e7d7 --- /dev/null +++ b/evals/ci/README.md @@ -0,0 +1,48 @@ +# CoasterBench in CI (no RCT2 assets) + +Everything the eval scores — park loading, track placement, ride testing, +ratings — runs without any RollerCoaster Tycoon 2 files via `--no-graphics` +(objects come from the bundled JSON pack; only pixels need `g1.dat`). That +makes the whole benchmark legally shippable in a CI image. What you give up: +screenshots (feedback is the eval report only, so contenders run text-only) +and the stock TD6 library (library mode is unavailable and the similarity +penalty is inert, so scores are not comparable with asset-full leaderboard +runs — `run.json` records `no_graphics: true`). + +## Pieces + +- `Dockerfile` — CPU-only game image (headless `openrct2-cli`, Rust agent, + bundled objects, checked-in scenario). Build once, push, pin the digest. + Draft: authored on macOS, not yet exercised on a Linux builder. +- `check_run.py` — pass/fail gate: every model needs ≥1 round that built and + completed a test circuit. Scores are metrics, not assertions (models are + nondeterministic; don't gate merges on excitement). + +## Shape of a job against vLLM + +Nightly / non-blocking is the realistic tier — the wall clock is model +inference, the game sim is ~7s per 25k-tick round. + +```bash +# 1. serve the model under test (own container/step; needs the GPU) +vllm serve "$MODEL" --enable-auto-tool-choice --tool-call-parser hermes & + +# 2. game + driver are CPU-only +uv run evals/driver.py \ + --base-url http://localhost:8000/v1 \ + --models "$MODEL" --rounds "${ROUNDS:-2}" --no-graphics \ + --name "ci-${BUILD_NUMBER:-local}" + +# 3. gate on protocol success, keep the run dir as the artifact +python3 evals/ci/check_run.py evals/runs/*-"ci-${BUILD_NUMBER:-local}" +``` + +What this exercises end-to-end that parser unit tests don't: forced and named +`tool_choice` (guided decoding), multi-turn tool loops with tool_result +round-trips, and large JSON tool arguments (a 148-piece track program is a +few KB of structured output). + +There is also a zero-GPU smoke: replay a canned program with no model at all — +`openrct2-cli eval test/tests/testdata/parks/BigMapTest.sv6 --no-graphics +--ticks 25000 --program evals/programs/test_oval.json --out report.json` +must produce `program.ok == true` and a tested ride. diff --git a/evals/ci/check_run.py b/evals/ci/check_run.py new file mode 100644 index 000000000000..0e2ab489e2dd --- /dev/null +++ b/evals/ci/check_run.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""CI pass/fail gate for a CoasterBench run directory. + +Scores are model quality, not infrastructure health, so CI asserts protocol +success only: every model must have at least one round whose program built OK +and whose ride completed a test circuit. Score stays a reported metric. + + usage: check_run.py evals/runs/ +""" +import json +import sys +from pathlib import Path + + +def main() -> int: + if len(sys.argv) != 2: + print(__doc__, file=sys.stderr) + return 2 + run_dir = Path(sys.argv[1]) + run = json.loads((run_dir / "run.json").read_text()) + failures = [] + for model in run["models"]: + reports = sorted(run_dir.glob(f"{model.replace('/', '_')}/round_*/report.json")) + tested = [] + for path in reports: + report = json.loads(path.read_text()) + # Batch runs name the program's ride; MCP-harness runs have no + # program, so any tested ride in the round's report counts. + ride_id = (report.get("program") or {}).get("ride_id") + tested += [ + r + for r in report.get("rides", []) or [] + if r.get("tested") and (ride_id is None or r["id"] == ride_id) + ] + best = max((r.get("excitement") or 0.0 for r in tested), default=None) + if not reports: + failures.append(f"{model}: no rounds ran") + elif not tested: + failures.append(f"{model}: {len(reports)} round(s), none produced a tested coaster") + else: + print(f"ok: {model}: {len(tested)}/{len(reports)} rounds tested, best excitement {best:.2f}") + for failure in failures: + print(f"FAIL: {failure}", file=sys.stderr) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/evals/driver.py b/evals/driver.py index ca6e5fb16270..fd39953d3cc6 100644 --- a/evals/driver.py +++ b/evals/driver.py @@ -1,6 +1,6 @@ # /// script # requires-python = ">=3.11" -# dependencies = ["anthropic[vertex]>=0.40"] +# dependencies = ["anthropic[vertex]>=0.40", "openai>=1.40"] # /// """Coaster design head-to-head: two Claude models iteratively design a coaster. @@ -25,6 +25,16 @@ Vertex model IDs for current-generation models are the bare first-party strings (claude-opus-4-6, claude-sonnet-5) — no prefix, no @date suffix. + +Usage (any OpenAI-compatible endpoint, e.g. a local vLLM server): + vllm serve Qwen/Qwen2.5-7B-Instruct --enable-auto-tool-choice ... + uv run evals/driver.py --base-url http://localhost:8000/v1 \ + --models Qwen/Qwen2.5-7B-Instruct --rounds 4 --no-graphics + +--no-graphics runs the game without RCT2 assets (design mode only): the +scenario defaults to a checked-in test park, feedback is the eval report +alone (no park screenshot), and the similarity penalty is inert because +there is no stock library to compare against. """ from __future__ import annotations @@ -47,6 +57,34 @@ CLI = REPO / "build" / "openrct2-cli" DEFAULT_SCENARIO = Path.home() / "rct2-assets" / "Scenarios" / "Build your own Six Flags Park.SC6" RCT2_DATA = Path.home() / "rct2-assets" +# Assetless default: a checked-in upstream test park (large, mostly-open flat +# grass, cash-rich) that loads and builds with only the bundled JSON objects. +CI_SCENARIO = REPO / "test" / "tests" / "testdata" / "parks" / "BigMapTest.sv6" + +MAP_LINES = { + DEFAULT_SCENARIO.name: ( + "Flat grass around tile (60, 60); a lake sits near map centre roughly tiles (68-85, 55-75) — do NOT " + "build into it. Stay within tiles 20-120. Directions: dir 0 faces -x, dir 1 faces +y, dir 2 faces +x, " + "dir 3 faces -y." + ), + CI_SCENARIO.name: ( + "A large park: flat open grass across roughly tiles 30-190 on both axes, with scattered existing " + "rides and footpaths (placement errors will name what is in the way; shift a few tiles and retry). " + "Flat grass around tile (60, 60) is a good anchor. Directions: dir 0 faces -x, dir 1 faces +y, " + "dir 2 faces +x, dir 3 faces -y." + ), +} + +# Set from --no-graphics in main(): the game loads no sprite data, so no RCT2 +# assets are needed and nothing can render (no screenshots, no previews). +NO_GRAPHICS = False + + +def rct2_args() -> list[str]: + """The eval CLI either loads the RCT2 install or runs assetless.""" + if NO_GRAPHICS: + return ["--no-graphics"] + return ["--rct2-data-path", str(RCT2_DATA)] PIECE_CATALOG = """ Station (required, place these FIRST, 3+ in a row): begin_station, middle_station, end_station @@ -92,9 +130,14 @@ def ride_type_info(ride_type: int) -> tuple[str, str]: return name, line + " This is the required type for this competition." -def build_system_prompt(ride_type: int) -> str: +def build_system_prompt(ride_type: int, scenario: Path) -> str: _, ride_line = ride_type_info(ride_type) - return SYSTEM_PROMPT.replace("{RIDE_TYPE_LINE}", ride_line) + map_line = MAP_LINES.get( + scenario.name, + "Terrain unknown; flat grass around tile (60, 60) is a reasonable first bet. Use validation " + "errors to find open ground. Directions: dir 0 faces -x, dir 1 faces +y, dir 2 faces +x, dir 3 faces -y.", + ) + return SYSTEM_PROMPT.replace("{RIDE_TYPE_LINE}", ride_line).replace("{MAP_LINE}", map_line) SYSTEM_PROMPT = f"""You are competing to design the best RollerCoaster Tycoon 2 roller coaster. @@ -124,7 +167,7 @@ def build_system_prompt(ride_type: int) -> str: {PIECE_CATALOG} ## Map -Flat grass around tile (60, 60); a lake sits near map centre roughly tiles (68-85, 55-75) — do NOT build into it. Stay within tiles 20-120. Directions: dir 0 faces -x, dir 1 faces +y, dir 2 faces +x, dir 3 faces -y. +{{MAP_LINE}} Before submitting, use the validate_track_program tool (same payload) to dry-run your program: it reports placement errors with the exact piece index, or whether the circuit closes, without spending your round. You get a limited number of validations per round, use them to fix geometry, then submit. @@ -290,12 +333,12 @@ def run_eval(program: dict, scenario: Path, workdir: Path, ticks: int) -> tuple[ cmd = [ str(CLI), "eval", str(scenario), "--ticks", str(ticks), - "--rct2-data-path", str(RCT2_DATA), + *rct2_args(), "--program", str(program_path), "--out", str(report_path), - "--capture", str(capture_path), - "--capture-xray", ] + if not NO_GRAPHICS: + cmd += ["--capture", str(capture_path), "--capture-xray"] proc = subprocess.run(cmd, capture_output=True, text=True, timeout=600) if not report_path.exists(): return {"program": {"ok": False, "error": {"message": f"eval crashed: {proc.stderr[-500:]}"}}}, None @@ -345,7 +388,7 @@ def validate_program(program: dict, scenario: Path) -> str: [ str(CLI), "eval", str(scenario), "--ticks", "5", - "--rct2-data-path", str(RCT2_DATA), + *rct2_args(), "--program", str(program_path), "--out", str(report_path), ], @@ -495,8 +538,142 @@ def feedback_content(attempt: Attempt) -> list[dict]: return content +@dataclass +class ToolUseBlock: + id: str + name: str + input: dict + type: str = "tool_use" + + +@dataclass +class TextBlock: + text: str + type: str = "text" + + +@dataclass +class _Usage: + input_tokens: int + output_tokens: int + + +@dataclass +class _Response: + content: list + usage: _Usage + + +def _to_openai(message: dict) -> list[dict]: + """One anthropic-form history entry -> the OpenAI messages it becomes. + + The driver only ever builds three shapes: a plain-string user message, a + user message holding tool_result blocks, and an assistant message whose + content is the block list a previous create() returned. + """ + role = message["role"] + content = message["content"] + if role == "assistant": + text = "".join(b.text for b in content if b.type == "text") + calls = [ + {"id": b.id, "type": "function", "function": {"name": b.name, "arguments": json.dumps(b.input)}} + for b in content + if b.type == "tool_use" + ] + msg: dict = {"role": "assistant", "content": text or None} + if calls: + msg["tool_calls"] = calls + return [msg] + if isinstance(content, str): + return [{"role": "user", "content": content}] + out: list[dict] = [] + images: list[str] = [] + for block in content: + if not isinstance(block, dict) or block.get("type") != "tool_result": + continue + inner = block.get("content") + texts: list[str] = [] + if isinstance(inner, str): + texts.append(inner) + else: + for part in inner or []: + if part.get("type") == "text": + texts.append(part["text"]) + elif part.get("type") == "image": + images.append(part["source"]["data"]) + out.append( + {"role": "tool", "tool_call_id": block["tool_use_id"], "content": "\n".join(texts) or "(no text)"} + ) + for data in images: + # OpenAI tool messages are text-only; a park screenshot rides along + # as a follow-up user message instead. + out.append( + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{data}"}}], + } + ) + return out + + +class OpenAICompat: + """Anthropic-messages-shaped facade over an OpenAI chat-completions + endpoint (vLLM serve, llama.cpp, OpenRouter, ...). compete() only touches + client.messages.create, response.content, and response.usage, so the tool + loop stays identical across lanes. Named and required tool_choice both map + onto the endpoint's structured-output support (vLLM: guided decoding via + --enable-auto-tool-choice).""" + + def __init__(self, base_url: str, api_key: str): + import openai + + self._client = openai.OpenAI(base_url=base_url, api_key=api_key) + self.messages = self # so client.messages.create(...) resolves here + + def create(self, *, model: str, max_tokens: int, system: str, messages: list[dict], tools: list[dict], tool_choice: dict) -> _Response: + payload: list[dict] = [{"role": "system", "content": system}] + for message in messages: + payload.extend(_to_openai(message)) + oa_tools = [ + { + "type": "function", + "function": {"name": t["name"], "description": t.get("description", ""), "parameters": t["input_schema"]}, + } + for t in tools + ] + oa_choice: str | dict = ( + {"type": "function", "function": {"name": tool_choice["name"]}} + if tool_choice.get("type") == "tool" + else "required" + ) + resp = self._client.chat.completions.create( + model=model, + max_tokens=max_tokens, + messages=payload, + tools=oa_tools, + tool_choice=oa_choice, + ) + choice = resp.choices[0].message + content: list = [] + if choice.content: + content.append(TextBlock(text=choice.content)) + for call in choice.tool_calls or []: + try: + args = json.loads(call.function.arguments or "{}") + except json.JSONDecodeError: + args = {} + content.append(ToolUseBlock(id=call.id, name=call.function.name, input=args)) + if not any(b.type == "tool_use" for b in content): + raise RuntimeError(f"{model} returned no tool call despite tool_choice={oa_choice!r}") + usage = resp.usage + return _Response( + content=content, + usage=_Usage(usage.prompt_tokens if usage else 0, usage.completion_tokens if usage else 0), + ) + + def compete( - client: anthropic.Anthropic | anthropic.AnthropicVertex, + client: anthropic.Anthropic | anthropic.AnthropicVertex | OpenAICompat, model: str, rounds: int, scenario: Path, @@ -507,7 +684,7 @@ def compete( ) -> Contender: contender = Contender(model=model) ride_name, _ = ride_type_info(ride_type) - system_prompt = build_system_prompt(ride_type) + (LIBRARY_PROMPT if library is not None else "") + system_prompt = build_system_prompt(ride_type, scenario) + (LIBRARY_PROMPT if library is not None else "") tools = [TOOL, VALIDATE_TOOL] + (LIBRARY_TOOLS if library is not None else []) messages: list[dict] = [ { @@ -607,6 +784,17 @@ def main() -> int: parser.add_argument("--ticks", type=int, default=25000) parser.add_argument("--scenario", type=Path, default=DEFAULT_SCENARIO) parser.add_argument("--vertex", action="store_true", help="use Google Vertex AI instead of the first-party API") + parser.add_argument( + "--base-url", + help="OpenAI-compatible endpoint (e.g. a vLLM server: http://localhost:8000/v1); " + "auth from $OPENAI_API_KEY, defaulting to 'EMPTY' for local servers", + ) + parser.add_argument( + "--no-graphics", + action="store_true", + help="run the game without RCT2 assets (design mode only): no screenshots in " + "feedback and no stock library, so the similarity penalty is inert", + ) parser.add_argument( "--project", default=os.environ.get("ANTHROPIC_VERTEX_PROJECT_ID"), @@ -627,6 +815,19 @@ def main() -> int: if not CLI.exists(): print(f"error: {CLI} not built", file=sys.stderr) return 1 + if args.no_graphics: + if args.mode == "library": + print("error: library mode needs the RCT2 track designs; --no-graphics is design mode only", file=sys.stderr) + return 1 + global NO_GRAPHICS + NO_GRAPHICS = True + if args.scenario == DEFAULT_SCENARIO: + # The graphics-lane default lives in the RCT2 install; assetless + # runs default to the checked-in test park instead. + args.scenario = CI_SCENARIO + if args.vertex and args.base_url: + print("error: pick one of --vertex and --base-url", file=sys.stderr) + return 1 if not args.scenario.exists(): print(f"error: scenario not found: {args.scenario}", file=sys.stderr) return 1 @@ -652,6 +853,8 @@ def main() -> int: "ticks": args.ticks, "ride_type": args.ride_type, "scenario": args.scenario.name, + "no_graphics": args.no_graphics, + **({"endpoint": args.base_url} if args.base_url else {}), # The site reads the penalty parameters from here; keep the # driver the single source of truth for the scoring math. "similarity_grace": SIMILARITY_GRACE, @@ -660,7 +863,9 @@ def main() -> int: ) ) - if args.vertex: + if args.base_url: + client = OpenAICompat(args.base_url, os.environ.get("OPENAI_API_KEY", "EMPTY")) + elif args.vertex: # Auth is GCP application-default credentials, not an Anthropic key. kwargs = {"region": args.region} if args.project: diff --git a/readme.md b/readme.md index dfa1f75a25e5..c6d8d5101b2a 100644 --- a/readme.md +++ b/readme.md @@ -145,6 +145,36 @@ retrieval and adaptation. uv run evals/driver.py --models claude-sonnet-5 --rounds 4 --mode library ``` +### Running without RCT2 assets + +Everything the eval scores — park loading, placement, ride testing, ratings, +the drawability gate — is pure game logic; only rendering needs `g1.dat`. +`--no-graphics` runs the whole benchmark with zero RollerCoaster Tycoon 2 +files (ride/scenery objects come from OpenRCT2's bundled JSON pack, and the +scenario defaults to a checked-in test park): + +```bash +./build/openrct2-cli eval test/tests/testdata/parks/BigMapTest.sv6 \ + --no-graphics --ticks 25000 --program evals/programs/test_oval.json --out report.json +``` + +The trade: no screenshots (the MCP server drops its image tools, contenders +run text-only), no stock library (library mode unavailable, similarity +penalty inert — `run.json` records `no_graphics` so such runs are not +compared against asset-full leaderboards). + +This is what makes the benchmark shippable in CI. The driver speaks to any +OpenAI-compatible endpoint (a `vllm serve` under test, llama.cpp, ...) via +`--base-url`; `evals/ci/` has the CPU-only Dockerfile and the pass/fail gate +(protocol success — a built, tested coaster — not score, which is model +quality, not infrastructure health). + +```bash +uv run evals/driver.py --base-url http://localhost:8000/v1 \ + --models Qwen/Qwen2.5-7B-Instruct --rounds 2 --no-graphics +python3 evals/ci/check_run.py evals/runs/ +``` + ## MCP server ```bash diff --git a/rust/orct2-agent/src/host.rs b/rust/orct2-agent/src/host.rs index 89d63313b7de..a0f8f223d921 100644 --- a/rust/orct2-agent/src/host.rs +++ b/rust/orct2-agent/src/host.rs @@ -108,6 +108,7 @@ unsafe extern "C" { err_len: usize, ) -> bool; fn orct2_host_ride_detail(ride_id: u16, out: *mut RideDetail) -> bool; + fn orct2_host_graphics_available() -> bool; fn orct2_host_capture( path: *const c_char, zoom: i32, @@ -275,6 +276,12 @@ pub fn entrance_place( } } +/// Whether sprite data is loaded. False under `eval --no-graphics` (no RCT2 +/// assets): screenshots cannot render, so image tools must not be offered. +pub fn graphics_available() -> bool { + unsafe { orct2_host_graphics_available() } +} + /// Renders a park screenshot to `path` (PNG). With `fit_track` the view is /// cropped to the bounding box of all track in the park (falls back to the /// full map when no track exists). With `xray` terrain and supports are @@ -417,6 +424,9 @@ mod test_stubs { pub unsafe fn orct2_host_ride_detail(_r: u16, _o: *mut RideDetail) -> bool { false } + pub unsafe fn orct2_host_graphics_available() -> bool { + true + } pub unsafe fn orct2_host_capture( _p: *const c_char, _z: i32, diff --git a/rust/orct2-agent/src/mcp.rs b/rust/orct2-agent/src/mcp.rs index 183b37d2a911..26b467d14276 100644 --- a/rust/orct2-agent/src/mcp.rs +++ b/rust/orct2-agent/src/mcp.rs @@ -44,6 +44,23 @@ impl Modalities { self.0 & other.0 == other.0 } + fn without(self, other: Modalities) -> Modalities { + Modalities(self.0 & !other.0) + } + + /// What this server instance can actually produce: everything, minus + /// image content when sprite data is not loaded (`eval --no-graphics`). + /// Intersected with the client's advertised set per request, so a + /// no-graphics server never lists or answers the screenshot tools no + /// matter what the client claims to accept. + fn server_side() -> Modalities { + if host::graphics_available() { + Modalities::ALL + } else { + Modalities::ALL.without(Modalities::IMAGE) + } + } + /// Parse a comma-separated list; unknown names are ignored. fn parse(list: &str) -> Modalities { list.split(',').fold(Modalities(0), |set, name| { @@ -77,6 +94,13 @@ impl std::ops::BitOr for Modalities { } } +impl std::ops::BitAnd for Modalities { + type Output = Modalities; + fn bitand(self, rhs: Modalities) -> Modalities { + Modalities(self.0 & rhs.0) + } +} + /// A request's claim on the park, read from the query string. struct Claim { lease: Option, @@ -270,7 +294,7 @@ fn handle_connection(stream: TcpStream, session: &mut Session) -> Result<(), Str Ok(None) => return Ok(()), // clean EOF Err(e) => return Err(e), }; - let modalities = Modalities::from_request_target(&target); + let modalities = Modalities::from_request_target(&target) & Modalities::server_side(); if method != "POST" { write_http(&mut stream, 405, "text/plain", b"method not allowed")?; continue; @@ -407,7 +431,7 @@ fn dispatch(message: &Value, session: &mut Session, modalities: Modalities) -> V return rpc_result( id, json!({ - "content": [{"type": "text", "text": format!("{name} is unavailable: this client did not advertise the content kind it answers with")}], + "content": [{"type": "text", "text": format!("{name} is unavailable: either this client did not advertise the content kind it answers with, or the server cannot produce it (screenshots need sprite data, absent under --no-graphics)")}], "isError": true, }), ); @@ -695,7 +719,13 @@ fn call_tool(name: &str, args: &Value, session: &mut Session) -> Result b) { session.best_test = Some(report.clone()); - session.best_shot = capture_park_png().ok(); + // Under --no-graphics nothing can render; skip rather than + // log a capture error every improved test. + session.best_shot = if host::graphics_available() { + capture_park_png().ok() + } else { + None + }; } } Ok(text_content(report)) diff --git a/src/openrct2/command_line/EvalCommands.cpp b/src/openrct2/command_line/EvalCommands.cpp index a4e4d97f8067..29ae8cc0f05a 100644 --- a/src/openrct2/command_line/EvalCommands.cpp +++ b/src/openrct2/command_line/EvalCommands.cpp @@ -39,6 +39,7 @@ namespace OpenRCT2 static u8string _renderLibraryDir{}; static bool _captureAllRotations = false; static bool _captureXray = false; + static bool _noGraphics = false; // clang-format off static constexpr CommandLineOptionDefinition kEvalOptions[] @@ -55,6 +56,7 @@ namespace OpenRCT2 { CMDLINE_TYPE_STRING, &_renderLibraryDir, kNAC, "render-library", "render a preview PNG of every stock track design into this directory and exit" }, { CMDLINE_TYPE_SWITCH, &_captureAllRotations, kNAC, "capture-all-rotations", "with --capture, also write the other three view rotations as -r1/-r2/-r3.png" }, { CMDLINE_TYPE_SWITCH, &_captureXray, kNAC, "capture-xray", "with --capture, also write a see-through verification view (terrain and supports hidden, every placed piece visible) as -x.png" }, + { CMDLINE_TYPE_SWITCH, &_noGraphics, kNAC, "no-graphics", "skip loading sprite data: no RCT2 assets required, but screenshots and library previews are unavailable" }, kOptionTableEnd }; @@ -90,9 +92,22 @@ namespace OpenRCT2 gCustomRCT2DataPath = Path::GetAbsolute(_evalRCT2DataPath); } - // Headless, but keep graphics data loaded (gOpenRCT2NoGraphics stays - // false) so CaptureImage can render screenshots of the result. + // Headless, but by default keep graphics data loaded (gOpenRCT2NoGraphics + // stays false) so CaptureImage can render screenshots of the result. + // --no-graphics drops that: the whole scoring path (park load, placement, + // testing, ratings) works without sprite data, so no RCT2 assets are + // needed — only anything that renders pixels is off the table. gOpenRCT2Headless = true; + if (_noGraphics) + { + if (!_capturePath.empty() || _captureAllRotations || _captureXray || !_renderLibraryDir.empty()) + { + Console::Error::WriteLine( + "--no-graphics cannot render: remove --capture/--capture-all-rotations/--capture-xray/--render-library"); + return ExitCode::fail; + } + gOpenRCT2NoGraphics = true; + } std::unique_ptr context(CreateContext()); if (!context->Initialise()) diff --git a/src/openrct2/rustbridge/RustBridge.cpp b/src/openrct2/rustbridge/RustBridge.cpp index 854d8fb43d6b..102bacd0fab4 100644 --- a/src/openrct2/rustbridge/RustBridge.cpp +++ b/src/openrct2/rustbridge/RustBridge.cpp @@ -666,12 +666,22 @@ uint16_t orct2_host_track_mirror(uint16_t track_type) return mirror == TrackElemType::none ? track_type : static_cast(mirror); } +bool orct2_host_graphics_available(void) +{ + return !gOpenRCT2NoGraphics; +} + bool orct2_host_capture(const char* path, int32_t zoom, uint8_t rotation, bool fit_track, bool xray) { if (path == nullptr) { return false; } + if (gOpenRCT2NoGraphics) + { + LOG_ERROR("screenshot capture unavailable: running with --no-graphics (no sprite data loaded)"); + return false; + } try { // CaptureImage refuses paths outside the screenshot directory, so diff --git a/src/openrct2/rustbridge/orct2_agent.h b/src/openrct2/rustbridge/orct2_agent.h index 956d13235bbb..5eb65609b926 100644 --- a/src/openrct2/rustbridge/orct2_agent.h +++ b/src/openrct2/rustbridge/orct2_agent.h @@ -63,6 +63,11 @@ typedef struct Orct2RideDetail { int16_t nausea; int32_t max_speed; int32_t average_speed; + /** + * Raw total length summed over stations, as 16.16 fixed-point metres + * (`Ride::getTotalLength()`). Divide by 2^16 for the human-readable metres + * the game shows; see `report::ride_length_metres`. + */ int32_t ride_length; int16_t max_positive_g; int16_t max_negative_g; @@ -195,6 +200,8 @@ extern bool orct2_host_ride_set_status(uint16_t ride_id, extern bool orct2_host_ride_detail(uint16_t ride_id, struct Orct2RideDetail *out); +extern bool orct2_host_graphics_available(void); + extern bool orct2_host_capture(const char *path, int32_t zoom, uint8_t rotation, From c0c1667a0926bef7e7d9def43a5ab899f50fab02 Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Fri, 24 Jul 2026 17:12:21 -0400 Subject: [PATCH 02/18] CI image: build the orphaned graphics target; verified on Linux arm64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DISABLE_GUI leaves the graphics target (g2/fonts/palettes/tracks.dat, generated by the sprite compiler from in-repo resources) with no dependents, but the install manifest still expects the files — build it explicitly before install. First Linux build of the fork: 237 MB image, containerized MCP server places/tests/rates a coaster with ratings identical to the macOS build. Co-Authored-By: Claude Fable 5 Signed-off-by: Tyler Michael Smith --- evals/ci/Dockerfile | 8 +++++++- evals/ci/README.md | 6 ++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/evals/ci/Dockerfile b/evals/ci/Dockerfile index 41863546ae82..b39b34db9e24 100644 --- a/evals/ci/Dockerfile +++ b/evals/ci/Dockerfile @@ -11,7 +11,9 @@ # openrct2-cli eval /opt/coasterbench/BigMapTest.sv6 --no-graphics \ # --serve 8791 --serve-bind 0.0.0.0 # -# NOTE: draft — authored on macOS, not yet exercised on a Linux builder. +# Verified: builds and serves on Linux arm64 (podman, Ubuntu 24.04 base) — +# a container-built oval places, tests, and rates identically to the macOS +# build. x86_64 not yet exercised. FROM ubuntu:24.04 AS build @@ -40,6 +42,10 @@ RUN cmake -S /src -B /build -G Ninja -DCMAKE_BUILD_TYPE=Release \ -DDOWNLOAD_OPENMUSIC=OFF \ -DDOWNLOAD_TITLE_SEQUENCES=OFF \ && cmake --build /build \ + # DISABLE_GUI leaves the `graphics` target (g2/fonts/palettes/tracks.dat, + # generated by the sprite compiler from in-repo resources) orphaned, but + # the install manifest still expects the files — build it explicitly. + && cmake --build /build --target graphics \ && DESTDIR=/stage cmake --install /build --prefix /usr FROM ubuntu:24.04 diff --git a/evals/ci/README.md b/evals/ci/README.md index 37703fb6e7d7..85430d440c53 100644 --- a/evals/ci/README.md +++ b/evals/ci/README.md @@ -12,8 +12,10 @@ runs — `run.json` records `no_graphics: true`). ## Pieces - `Dockerfile` — CPU-only game image (headless `openrct2-cli`, Rust agent, - bundled objects, checked-in scenario). Build once, push, pin the digest. - Draft: authored on macOS, not yet exercised on a Linux builder. + bundled objects, checked-in scenario; ~240 MB). Build once, push, pin the + digest. Verified on Linux arm64 (podman): the containerized MCP server + builds, tests, and rates a coaster with ratings identical to the macOS + build. x86_64 not yet exercised. - `check_run.py` — pass/fail gate: every model needs ≥1 round that built and completed a test circuit. Scores are metrics, not assertions (models are nondeterministic; don't gate merges on excitement). From c5666e80d10b690ca0a864e9788ab0795378c8d9 Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Fri, 24 Jul 2026 17:35:29 -0400 Subject: [PATCH 03/18] OpenAI lane: retry when tool_choice=required returns no tool call Observed once against vLLM 0.25.1 (Qwen2.5-7B, hermes parser): the server's first structured-output request after startup answered with zero tool calls, violating the required contract. Not reproduced in 15 probes or a full 4-round run, so treat it as rare and retry (3 attempts) instead of killing the run; usage from wasted attempts still counts. Co-Authored-By: Claude Fable 5 Signed-off-by: Tyler Michael Smith --- evals/driver.py | 53 +++++++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/evals/driver.py b/evals/driver.py index fd39953d3cc6..652a3be725f8 100644 --- a/evals/driver.py +++ b/evals/driver.py @@ -646,30 +646,35 @@ def create(self, *, model: str, max_tokens: int, system: str, messages: list[dic if tool_choice.get("type") == "tool" else "required" ) - resp = self._client.chat.completions.create( - model=model, - max_tokens=max_tokens, - messages=payload, - tools=oa_tools, - tool_choice=oa_choice, - ) - choice = resp.choices[0].message - content: list = [] - if choice.content: - content.append(TextBlock(text=choice.content)) - for call in choice.tool_calls or []: - try: - args = json.loads(call.function.arguments or "{}") - except json.JSONDecodeError: - args = {} - content.append(ToolUseBlock(id=call.id, name=call.function.name, input=args)) - if not any(b.type == "tool_use" for b in content): - raise RuntimeError(f"{model} returned no tool call despite tool_choice={oa_choice!r}") - usage = resp.usage - return _Response( - content=content, - usage=_Usage(usage.prompt_tokens if usage else 0, usage.completion_tokens if usage else 0), - ) + # tool_choice="required" is not airtight in the wild: vLLM's guided + # grammar can emit an empty call array, so a callless response gets + # retried rather than killing the run. + input_tokens = output_tokens = 0 + for attempt in range(3): + resp = self._client.chat.completions.create( + model=model, + max_tokens=max_tokens, + messages=payload, + tools=oa_tools, + tool_choice=oa_choice, + ) + if resp.usage: + input_tokens += resp.usage.prompt_tokens + output_tokens += resp.usage.completion_tokens + choice = resp.choices[0].message + content: list = [] + if choice.content: + content.append(TextBlock(text=choice.content)) + for call in choice.tool_calls or []: + try: + args = json.loads(call.function.arguments or "{}") + except json.JSONDecodeError: + args = {} + content.append(ToolUseBlock(id=call.id, name=call.function.name, input=args)) + if any(b.type == "tool_use" for b in content): + return _Response(content=content, usage=_Usage(input_tokens, output_tokens)) + print(f" [{model}] no tool call (attempt {attempt + 1}/3), retrying", flush=True) + raise RuntimeError(f"{model} returned no tool call in 3 attempts despite tool_choice={oa_choice!r}") def compete( From 6da1d2644465451623440f551ae103bc77697692 Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Fri, 24 Jul 2026 17:55:20 -0400 Subject: [PATCH 04/18] driver: --max-tokens, fail fast when thinking exhausts the budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Laguna-S-2.1 (reasoning model, poolside_v1 parsers) burned all 8000 completion tokens thinking and never reached a tool call — finish_reason "length", surfaced as a tool_choice=required violation. Retrying that is deterministic waste, so the OpenAI lane now raises immediately with the real cause, and the per-request budget is a flag (default unchanged). Co-Authored-By: Claude Fable 5 Signed-off-by: Tyler Michael Smith --- evals/driver.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/evals/driver.py b/evals/driver.py index 652a3be725f8..697ceee8f508 100644 --- a/evals/driver.py +++ b/evals/driver.py @@ -673,6 +673,14 @@ def create(self, *, model: str, max_tokens: int, system: str, messages: list[dic content.append(ToolUseBlock(id=call.id, name=call.function.name, input=args)) if any(b.type == "tool_use" for b in content): return _Response(content=content, usage=_Usage(input_tokens, output_tokens)) + if resp.choices[0].finish_reason == "length": + # Deterministic, so retrying just burns tokens: the model (a + # reasoning model, usually) hit the token ceiling while still + # thinking and never got to the call. + raise RuntimeError( + f"{model} exhausted max_tokens={max_tokens} before emitting a tool call " + "(reasoning models spend the budget thinking first; raise --max-tokens)" + ) print(f" [{model}] no tool call (attempt {attempt + 1}/3), retrying", flush=True) raise RuntimeError(f"{model} returned no tool call in 3 attempts despite tool_choice={oa_choice!r}") @@ -686,6 +694,7 @@ def compete( ticks: int, ride_type: int, library: list[dict] | None = None, + max_tokens: int = 8000, ) -> Contender: contender = Contender(model=model) ride_name, _ = ride_type_info(ride_type) @@ -708,7 +717,7 @@ def compete( force_submit = step == MAX_LOOKUPS_PER_ROUND response = client.messages.create( model=model, - max_tokens=8000, + max_tokens=max_tokens, system=system_prompt, messages=messages, tools=tools, @@ -787,6 +796,13 @@ def main() -> int: help="required coaster ride type for the competition (52 wooden, 51 steel twister)", ) parser.add_argument("--ticks", type=int, default=25000) + parser.add_argument( + "--max-tokens", + type=int, + default=8000, + help="completion token budget per request; reasoning models think before " + "they call tools, so give them room (e.g. 24000 for Laguna)", + ) parser.add_argument("--scenario", type=Path, default=DEFAULT_SCENARIO) parser.add_argument("--vertex", action="store_true", help="use Google Vertex AI instead of the first-party API") parser.add_argument( @@ -879,7 +895,7 @@ def main() -> int: else: client = anthropic.Anthropic() contenders = [ - compete(client, model, args.rounds, args.scenario, run_dir, args.ticks, args.ride_type, library) + compete(client, model, args.rounds, args.scenario, run_dir, args.ticks, args.ride_type, library, args.max_tokens) for model in args.models ] From cb35fe7dcb3e716f5a7d7187aecb96e97148e145 Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Fri, 24 Jul 2026 17:59:40 -0400 Subject: [PATCH 05/18] CI notes: document the vLLM findings from the first live runs Three failure classes surfaced on day one against vLLM 0.25.1 (all handled by the driver, first two worth an upstream look): a cold-start tool_choice=required contract violation, reasoning models exhausting max_tokens before the tool call, and generation_config.json silently overriding sampling defaults. Co-Authored-By: Claude Fable 5 Signed-off-by: Tyler Michael Smith --- evals/ci/README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/evals/ci/README.md b/evals/ci/README.md index 85430d440c53..211abb74b5bb 100644 --- a/evals/ci/README.md +++ b/evals/ci/README.md @@ -48,3 +48,26 @@ There is also a zero-GPU smoke: replay a canned program with no model at all — `openrct2-cli eval test/tests/testdata/parks/BigMapTest.sv6 --no-graphics --ticks 25000 --program evals/programs/test_oval.json --out report.json` must produce `program.ok == true` and a tested ride. + +## Field notes: what the first live runs caught (vLLM 0.25.1, A100) + +Three findings from the eval's first day out, all now handled by the driver — +kept here because they are exactly the failure classes this job exists to +surface, and the first two are worth checking against vLLM upstream: + +1. **`tool_choice: "required"` returned zero tool calls** on the server's + first-ever structured-output request after startup (Qwen2.5-7B-Instruct, + hermes parser). A contract violation; not reproduced in 15 identical + probes afterwards, so it looks cold-start-related. The driver retries up + to 3× rather than failing the run. +2. **Reasoning models silently starve the tool call.** Laguna-S-2.1 + (`poolside_v1` tool + reasoning parsers, thinking enabled) spent the + entire completion budget on interleaved thinking and hit + `finish_reason: "length"` with `tool_calls: []` — surfacing, misleadingly, + as another `required` violation. Any agentic harness with a fixed + `max_tokens` sized for non-reasoning models hits this. The driver fails + fast with the real cause and takes `--max-tokens` (Laguna wants ~24000). +3. **`generation_config.json` overrides server sampling defaults** (vLLM + warns but serves): Qwen shipped temp 0.7 / top-p 0.8 / rep-penalty 1.05, + so "default" runs are not the sampling you assumed. Pin + `--generation-config vllm` if you want vLLM defaults. From b122f5ee4d00127c4e729ba876509246c588b9d1 Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Fri, 24 Jul 2026 18:07:51 -0400 Subject: [PATCH 06/18] driver: named tool_choice is advisory on some stacks; retry forced submit vLLM + poolside_v1 returned validate_track_program when the request forced submit_track_program by name (finding #4 for the CI notes), so the final forced step of a round now gets three attempts, answering whatever tool the model actually called in between. Also thread chat_template_kwargs through the OpenAI lane (--chat-template-kwargs) so reasoning models can run with thinking disabled. Co-Authored-By: Claude Fable 5 Signed-off-by: Tyler Michael Smith --- evals/driver.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/evals/driver.py b/evals/driver.py index 697ceee8f508..ac8fdbb0d630 100644 --- a/evals/driver.py +++ b/evals/driver.py @@ -624,10 +624,14 @@ class OpenAICompat: onto the endpoint's structured-output support (vLLM: guided decoding via --enable-auto-tool-choice).""" - def __init__(self, base_url: str, api_key: str): + def __init__(self, base_url: str, api_key: str, extra_body: dict | None = None): import openai self._client = openai.OpenAI(base_url=base_url, api_key=api_key) + # Endpoint-specific request extras, e.g. vLLM's chat_template_kwargs + # ({"enable_thinking": false} tames reasoning models whose thinking + # would otherwise exhaust any completion budget on this task). + self._extra_body = extra_body or {} self.messages = self # so client.messages.create(...) resolves here def create(self, *, model: str, max_tokens: int, system: str, messages: list[dict], tools: list[dict], tool_choice: dict) -> _Response: @@ -657,6 +661,7 @@ def create(self, *, model: str, max_tokens: int, system: str, messages: list[dic messages=payload, tools=oa_tools, tool_choice=oa_choice, + extra_body=self._extra_body, ) if resp.usage: input_tokens += resp.usage.prompt_tokens @@ -713,8 +718,11 @@ def compete( tool_use = None lookups: list[dict] = [] round_usage = {"input_tokens": 0, "output_tokens": 0} - for step in range(MAX_LOOKUPS_PER_ROUND + 1): - force_submit = step == MAX_LOOKUPS_PER_ROUND + # Two extra forced-submit attempts: named tool_choice is not actually + # enforced by every endpoint (vLLM + poolside_v1 returned a different + # tool than the one forced), so the "guaranteed" final step isn't. + for step in range(MAX_LOOKUPS_PER_ROUND + 3): + force_submit = step >= MAX_LOOKUPS_PER_ROUND response = client.messages.create( model=model, max_tokens=max_tokens, @@ -746,7 +754,7 @@ def compete( } ) if program is None or tool_use is None: - # Unreachable: the final loop step forces submit_track_program. + # Three forced-submit attempts all returned something else. raise RuntimeError(f"{model} never submitted a program in round {rnd}") if program.get("ride_type") != ride_type: print( @@ -810,6 +818,11 @@ def main() -> int: help="OpenAI-compatible endpoint (e.g. a vLLM server: http://localhost:8000/v1); " "auth from $OPENAI_API_KEY, defaulting to 'EMPTY' for local servers", ) + parser.add_argument( + "--chat-template-kwargs", + help="JSON merged into each request as vLLM chat_template_kwargs " + "(OpenAI lane only), e.g. '{\"enable_thinking\": false}'", + ) parser.add_argument( "--no-graphics", action="store_true", @@ -885,7 +898,10 @@ def main() -> int: ) if args.base_url: - client = OpenAICompat(args.base_url, os.environ.get("OPENAI_API_KEY", "EMPTY")) + extra_body = ( + {"chat_template_kwargs": json.loads(args.chat_template_kwargs)} if args.chat_template_kwargs else None + ) + client = OpenAICompat(args.base_url, os.environ.get("OPENAI_API_KEY", "EMPTY"), extra_body) elif args.vertex: # Auth is GCP application-default credentials, not an Anthropic key. kwargs = {"region": args.region} From b952e700e5b47958214e60a38688c18f33fe6bce Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Fri, 24 Jul 2026 18:14:32 -0400 Subject: [PATCH 07/18] driver: COASTERBENCH_CLI env override for CI binary location Co-Authored-By: Claude Fable 5 Signed-off-by: Tyler Michael Smith --- evals/driver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/evals/driver.py b/evals/driver.py index ac8fdbb0d630..fa9a0bf3cde7 100644 --- a/evals/driver.py +++ b/evals/driver.py @@ -54,7 +54,9 @@ import anthropic REPO = Path(__file__).resolve().parent.parent -CLI = REPO / "build" / "openrct2-cli" +# COASTERBENCH_CLI lets a CI environment point at a binary that didn't come +# from this checkout's build dir (e.g. extracted from the game image). +CLI = Path(os.environ.get("COASTERBENCH_CLI", REPO / "build" / "openrct2-cli")) DEFAULT_SCENARIO = Path.home() / "rct2-assets" / "Scenarios" / "Build your own Six Flags Park.SC6" RCT2_DATA = Path.home() / "rct2-assets" # Assetless default: a checked-in upstream test park (large, mostly-open flat From 6d66daa03dfb8afdc42c4122fa1e93457f895d90 Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Fri, 24 Jul 2026 18:21:28 -0400 Subject: [PATCH 08/18] =?UTF-8?q?CI=20notes:=20final=20attribution=20?= =?UTF-8?q?=E2=80=94=20one=20confirmed=20vLLM=20bug,=20one=20anomaly,=20on?= =?UTF-8?q?e=20harness=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Named tool_choice unenforced on poolside_v1 is now 3/3 reproducible; the required-zero-calls event stays unconfirmed (no captured body); the max_tokens starvation was ours. Co-Authored-By: Claude Fable 5 Signed-off-by: Tyler Michael Smith --- evals/ci/README.md | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/evals/ci/README.md b/evals/ci/README.md index 211abb74b5bb..b44f5f33b86e 100644 --- a/evals/ci/README.md +++ b/evals/ci/README.md @@ -51,23 +51,27 @@ must produce `program.ok == true` and a tested ride. ## Field notes: what the first live runs caught (vLLM 0.25.1, A100) -Three findings from the eval's first day out, all now handled by the driver — -kept here because they are exactly the failure classes this job exists to -surface, and the first two are worth checking against vLLM upstream: +Findings from the eval's first day out, with honest attribution — kept here +because they are exactly the failure classes this job exists to surface: -1. **`tool_choice: "required"` returned zero tool calls** on the server's - first-ever structured-output request after startup (Qwen2.5-7B-Instruct, - hermes parser). A contract violation; not reproduced in 15 identical - probes afterwards, so it looks cold-start-related. The driver retries up - to 3× rather than failing the run. -2. **Reasoning models silently starve the tool call.** Laguna-S-2.1 - (`poolside_v1` tool + reasoning parsers, thinking enabled) spent the - entire completion budget on interleaved thinking and hit - `finish_reason: "length"` with `tool_calls: []` — surfacing, misleadingly, - as another `required` violation. Any agentic harness with a fixed - `max_tokens` sized for non-reasoning models hits this. The driver fails - fast with the real cause and takes `--max-tokens` (Laguna wants ~24000). -3. **`generation_config.json` overrides server sampling defaults** (vLLM - warns but serves): Qwen shipped temp 0.7 / top-p 0.8 / rep-penalty 1.05, - so "default" runs are not the sampling you assumed. Pin - `--generation-config vllm` if you want vLLM defaults. +1. **Confirmed vLLM bug — named `tool_choice` silently unenforced** + (`poolside_v1` parser, Laguna-S-2.1, thinking disabled): a request + forcing `submit_track_program` by name got `validate_track_program` + back, 3/3 reproducible probes. The contract requires enforcement or + rejection, not a different function. The driver copes by giving the + forced-submit step three attempts. +2. **Unconfirmed anomaly — `tool_choice: "required"` returned zero tool + calls** once, on the server's first-ever structured-output request + (Qwen2.5-7B, hermes parser). Never reproduced (0/15 probes) and the + failing response body wasn't captured, so it is not claimable as a bug. + The driver retries up to 3×. +3. **Harness bug (ours, fixed) — fixed `max_tokens` starves reasoning + models**: Laguna spent the entire 8k (then 24k) completion budget on + interleaved thinking and hit `finish_reason: "length"` with no tool + call, masquerading as finding #2. vLLM behaved correctly. The driver + now fails fast with the real cause and takes `--max-tokens` / + `--chat-template-kwargs '{"enable_thinking": false}'`. +4. **Config foot-gun — `generation_config.json` overrides server sampling + defaults** (vLLM warns but serves): Qwen ships temp 0.7 / top-p 0.8 / + rep-penalty 1.05, so "default" runs are not the sampling you assumed. + Pin `--generation-config vllm` for vLLM defaults. From 81da8a0548cd6fa756fe351209d359c91c30cfcb Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Fri, 24 Jul 2026 18:26:52 -0400 Subject: [PATCH 09/18] driver: force submission in-band when the validation budget runs out With named tool_choice advisory (finding #1), agentic models keep validating through all forced steps and never submit. The last budgeted tool result now says so explicitly, which instruction-following models respect even when grammar enforcement doesn't exist. Co-Authored-By: Claude Fable 5 Signed-off-by: Tyler Michael Smith --- evals/driver.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/evals/driver.py b/evals/driver.py index fa9a0bf3cde7..d2832fd0df60 100644 --- a/evals/driver.py +++ b/evals/driver.py @@ -749,6 +749,15 @@ def compete( print(f" [{model}] round {rnd}: {tool_use.name}({json.dumps(tool_use.input)})", flush=True) result, lookup = library_tool_result(tool_use.name, tool_use.input, library or []) lookups.append(lookup) + if step + 1 >= MAX_LOOKUPS_PER_ROUND: + # Named tool_choice is advisory on some stacks, so forcing has + # to happen in-band too: agentic models otherwise keep + # validating forever instead of ever submitting. + result += ( + "\n\nVALIDATION BUDGET EXHAUSTED: you must now call " + "submit_track_program with your best current program. Do not " + "call any other tool." + ) messages.append( { "role": "user", From 5911936bc9dfdafb3325d6a4c8ea0236a872cc87 Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Fri, 24 Jul 2026 18:42:18 -0400 Subject: [PATCH 10/18] Asset-free track graphics: cursor trace in the report, schematic PNG per round MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit program.rs records the cursor after every placed piece into report.json's program.trace; the driver renders it as a two-panel (top-down + isometric) diagram — station green, chain lift red, height gradient, and an explicit dashed gap-to-start line for open circuits, which was today's dominant failure mode and invisible in prose. --schematic-feedback attaches it to round feedback for multimodal contenders; the PNG is always saved as a round artifact regardless. Co-Authored-By: Claude Fable 5 Signed-off-by: Tyler Michael Smith --- evals/driver.py | 91 ++++++++++++++++++++++++++++++++- rust/orct2-agent/src/program.rs | 32 ++++++++++++ 2 files changed, 121 insertions(+), 2 deletions(-) diff --git a/evals/driver.py b/evals/driver.py index d2832fd0df60..c61c4f9f3064 100644 --- a/evals/driver.py +++ b/evals/driver.py @@ -1,6 +1,6 @@ # /// script # requires-python = ">=3.11" -# dependencies = ["anthropic[vertex]>=0.40", "openai>=1.40"] +# dependencies = ["anthropic[vertex]>=0.40", "openai>=1.40", "pillow>=10"] # /// """Coaster design head-to-head: two Claude models iteratively design a coaster. @@ -81,6 +81,11 @@ # assets are needed and nothing can render (no screenshots, no previews). NO_GRAPHICS = False +# Set from --schematic-feedback: attach the schematic track diagram (rendered +# from the report's cursor trace, no assets needed) to round feedback, for +# multimodal contenders in no-graphics runs. +SCHEMATIC_FEEDBACK = False + def rct2_args() -> list[str]: """The eval CLI either loads the RCT2 install or runs assetless.""" @@ -325,6 +330,72 @@ def best(self) -> Attempt | None: return max(rated, key=lambda a: a.excitement) if rated else None +STATION_PIECES = {"begin_station", "middle_station", "end_station"} + + +def render_schematic(trace: list[dict], out_path: Path) -> Path | None: + """Draws the placed track as a two-panel PNG (top-down + isometric) from + the report's cursor trace — no game assets involved. Stations are green, + chain lift red, everything else shaded by height; an open circuit gets a + dashed gap line from track end back to the start.""" + if len(trace) < 2: + return None + from PIL import Image, ImageDraw + + pts = [(p["x"], p["y"], p["z"]) for p in trace] + zs = [z for _, _, z in pts] + z0, z1 = min(zs), max(zs) + + def color(i: int) -> tuple[int, int, int]: + piece = trace[i]["piece"] + if piece in STATION_PIECES: + return (46, 160, 67) + if trace[i].get("chain"): + return (220, 68, 61) + t = (pts[i][2] - z0) / (z1 - z0) if z1 > z0 else 0.0 + return (int(60 + 195 * t), int(120 - 40 * t), int(220 - 160 * t)) + + panels = { + "top": lambda x, y, z: (x, y), + "iso": lambda x, y, z: (x - y, (x + y) * 0.5 - z / 24), + } + size, margin = 640, 40 + img = Image.new("RGB", (size * 2, size), (250, 250, 248)) + draw = ImageDraw.Draw(img) + + closed = pts[0][:2] == pts[-1][:2] and trace[0]["z"] == trace[-1]["z"] + for panel, (name, proj) in enumerate(panels.items()): + proj_pts = [proj(*p) for p in pts] + xs = [u for u, _ in proj_pts] + ys = [v for _, v in proj_pts] + span = max(max(xs) - min(xs), max(ys) - min(ys)) or 1.0 + scale = (size - 2 * margin) / span + + def to_px(uv, panel=panel, xs=xs, ys=ys, scale=scale): + return ( + panel * size + margin + (uv[0] - min(xs)) * scale, + margin + (uv[1] - min(ys)) * scale, + ) + + px = [to_px(p) for p in proj_pts] + for i in range(1, len(px)): + draw.line([px[i - 1], px[i]], fill=color(i), width=4) + if not closed: + draw.line([px[-1], px[0]], fill=(150, 150, 150), width=2) + sx, sy = px[0] + draw.ellipse([sx - 5, sy - 5, sx + 5, sy + 5], outline=(0, 0, 0), width=2) + draw.text((panel * size + margin, size - margin + 8), name, fill=(90, 90, 90)) + + if not closed: + dx = pts[0][0] - pts[-1][0] + dy = pts[0][1] - pts[-1][1] + dz = trace[0]["z"] - trace[-1]["z"] + draw.text((margin, 8), f"OPEN CIRCUIT: gap to start dx={dx} dy={dy} dz={dz}", fill=(180, 30, 30)) + draw.text((size + margin, 8), "green=station red=chain-lift blue->orange=height", fill=(90, 90, 90)) + img.save(out_path) + return out_path + + def run_eval(program: dict, scenario: Path, workdir: Path, ticks: int) -> tuple[dict, Path | None]: workdir.mkdir(parents=True, exist_ok=True) program_path = workdir / "program.json" @@ -346,7 +417,14 @@ def run_eval(program: dict, scenario: Path, workdir: Path, ticks: int) -> tuple[ return {"program": {"ok": False, "error": {"message": f"eval crashed: {proc.stderr[-500:]}"}}}, None report = json.loads(report_path.read_text()) - shot = None + trace = (report.get("program") or {}).get("trace") or [] + schematic = None + if trace: + try: + schematic = render_schematic(trace, workdir / "track.png") + except Exception as e: # a diagram must never sink the round + print(f" schematic render failed: {e}", file=sys.stderr) + shot = schematic if SCHEMATIC_FEEDBACK else None if capture_path.exists(): small = workdir / "park_small.png" # The API rejects images over 5 MB of base64 (~3.7 MB raw); tall parks @@ -829,6 +907,12 @@ def main() -> int: help="OpenAI-compatible endpoint (e.g. a vLLM server: http://localhost:8000/v1); " "auth from $OPENAI_API_KEY, defaulting to 'EMPTY' for local servers", ) + parser.add_argument( + "--schematic-feedback", + action="store_true", + help="attach the asset-free schematic track diagram to round feedback " + "(multimodal contenders only; text-only models will reject image content)", + ) parser.add_argument( "--chat-template-kwargs", help="JSON merged into each request as vLLM chat_template_kwargs " @@ -866,6 +950,9 @@ def main() -> int: return 1 global NO_GRAPHICS NO_GRAPHICS = True + if args.schematic_feedback: + global SCHEMATIC_FEEDBACK + SCHEMATIC_FEEDBACK = True if args.scenario == DEFAULT_SCENARIO: # The graphics-lane default lives in the RCT2 install; assetless # runs default to the checked-in test park instead. diff --git a/rust/orct2-agent/src/program.rs b/rust/orct2-agent/src/program.rs index 4b8504ee18c6..ebdd9a3edf97 100644 --- a/rust/orct2-agent/src/program.rs +++ b/rust/orct2-agent/src/program.rs @@ -100,6 +100,21 @@ pub struct ProgramOutcome { /// serialised: the program JSON already spells out the pieces. #[serde(skip)] pub placed_types: Vec, + /// Cursor after each placed piece (tile coords), preceded by the start + /// cursor. Lets a renderer draw the layout schematically with no game + /// assets: point i to i+1 is the chord of piece i. + pub trace: Vec, +} + +#[derive(Debug, Serialize)] +pub struct TracePoint { + /// Piece whose placement ended at this cursor; "start" for the first. + pub piece: String, + pub chain: bool, + pub x: i32, + pub y: i32, + pub z: i32, + pub dir: u8, } #[derive(Debug, Serialize)] @@ -203,6 +218,15 @@ pub fn run(json: &str) -> ProgramOutcome { // Tiles occupied by station pieces, for entrance/exit placement. let mut station_tiles: Vec<(i32, i32)> = Vec::new(); + outcome.trace.push(TracePoint { + piece: "start".into(), + chain: false, + x: cursor.x / 32, + y: cursor.y / 32, + z: cursor.z, + dir: cursor.direction, + }); + for (index, piece) in program.pieces.iter().enumerate() { let (piece_ref, chain) = piece.parts(); let track_type = match resolve(piece_ref) { @@ -222,6 +246,14 @@ pub fn run(json: &str) -> ProgramOutcome { outcome.pieces_placed += 1; outcome.total_cost += cost; outcome.placed_types.push(track_type); + outcome.trace.push(TracePoint { + piece: describe(track_type), + chain, + x: cursor.x / 32, + y: cursor.y / 32, + z: cursor.z, + dir: cursor.direction, + }); // TrackElemType 1-3 are the station pieces. if (1..=3).contains(&track_type) { station_tiles.push(piece_tile); From 6dbbff58ca97f15633b66c2b644f39a283d22f7e Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Fri, 24 Jul 2026 18:44:16 -0400 Subject: [PATCH 11/18] CI image: bundle the smoke program alongside the scenario Co-Authored-By: Claude Fable 5 Signed-off-by: Tyler Michael Smith --- evals/ci/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/evals/ci/Dockerfile b/evals/ci/Dockerfile index b39b34db9e24..daa1db3c0a8d 100644 --- a/evals/ci/Dockerfile +++ b/evals/ci/Dockerfile @@ -56,8 +56,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* COPY --from=build /stage/usr /usr -# The checked-in assetless scenario, so the container is self-contained. -COPY test/tests/testdata/parks/BigMapTest.sv6 /opt/coasterbench/ +# The checked-in assetless scenario and a known-good smoke program, so the +# container is self-contained for both serving and zero-GPU smoke runs. +COPY test/tests/testdata/parks/BigMapTest.sv6 evals/programs/test_oval.json /opt/coasterbench/ EXPOSE 8791 CMD ["openrct2-cli", "eval", "/opt/coasterbench/BigMapTest.sv6", "--no-graphics", "--serve", "8791", "--serve-bind", "0.0.0.0"] From 421cf254bc5d429b3d28dc9fe8718250309bf8ae Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Fri, 24 Jul 2026 18:45:19 -0400 Subject: [PATCH 12/18] OpenAI lane: preserve reasoning across turns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vLLM renders `reasoning` passed back on assistant messages into the chat template (verified via prompt_tokens delta), so thinking models keep their derivations instead of re-reasoning from scratch each call — the difference between bounded per-step thinking and 60k-token restarts. Captured from the SDK's extra fields, replayed by _to_openai; inert for non-reasoning models. Co-Authored-By: Claude Fable 5 Signed-off-by: Tyler Michael Smith --- evals/driver.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/evals/driver.py b/evals/driver.py index c61c4f9f3064..e292c1ceeb2e 100644 --- a/evals/driver.py +++ b/evals/driver.py @@ -632,6 +632,17 @@ class TextBlock: type: str = "text" +@dataclass +class ReasoningBlock: + """A reasoning model's thinking, preserved so it can be passed back on the + next turn (vLLM renders `reasoning` on assistant messages into the chat + template — verified: prompt_tokens grows by the trace length). Without + passback the model re-derives everything from scratch every call.""" + + text: str + type: str = "reasoning" + + @dataclass class _Usage: input_tokens: int @@ -655,12 +666,15 @@ def _to_openai(message: dict) -> list[dict]: content = message["content"] if role == "assistant": text = "".join(b.text for b in content if b.type == "text") + reasoning = "".join(b.text for b in content if b.type == "reasoning") calls = [ {"id": b.id, "type": "function", "function": {"name": b.name, "arguments": json.dumps(b.input)}} for b in content if b.type == "tool_use" ] msg: dict = {"role": "assistant", "content": text or None} + if reasoning: + msg["reasoning"] = reasoning if calls: msg["tool_calls"] = calls return [msg] @@ -748,6 +762,12 @@ def create(self, *, model: str, max_tokens: int, system: str, messages: list[dic output_tokens += resp.usage.completion_tokens choice = resp.choices[0].message content: list = [] + # The SDK model keeps unknown fields; reasoning arrives as an + # extra ("reasoning" on vLLM, "reasoning_content" on some stacks). + extra = choice.model_dump() if hasattr(choice, "model_dump") else {} + reasoning = extra.get("reasoning") or extra.get("reasoning_content") + if reasoning: + content.append(ReasoningBlock(text=reasoning)) if choice.content: content.append(TextBlock(text=choice.content)) for call in choice.tool_calls or []: From f50817c66813b0abcf80b60302ade9b82ba0534c Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Fri, 24 Jul 2026 18:50:43 -0400 Subject: [PATCH 13/18] driver: default to 6 rounds Best-across-rounds is the score; models rarely close a circuit early and more rounds help roughly monotonically. CI overrides to 2 for time. Co-Authored-By: Claude Fable 5 Signed-off-by: Tyler Michael Smith --- evals/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/driver.py b/evals/driver.py index e292c1ceeb2e..2d47f8c83fbc 100644 --- a/evals/driver.py +++ b/evals/driver.py @@ -905,7 +905,7 @@ def main() -> int: default="design", help="design = from scratch; library = with track design library search (retrieval eval)", ) - parser.add_argument("--rounds", type=int, default=4) + parser.add_argument("--rounds", type=int, default=6) parser.add_argument( "--ride-type", type=int, From 6016489a5ddfcfcfaf70c029dba9b720a40b58f6 Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Fri, 24 Jul 2026 19:06:08 -0400 Subject: [PATCH 14/18] CI image: fix objects prefix bug; add self-contained eval stage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The objects/asset downloads run from install(CODE) strings that bake CMAKE_INSTALL_FULL_DATADIR at configure time, so installing with --prefix /usr left objects at /usr/local — present in the image, invisible to the binary. (Runtime object auto-download masked this on any container with DNS, which is how the arm64 e2e passed.) Prefix now set at configure. New default `eval` target layers python venv + driver on the game so one container runs a whole benchmark; game-only image is --target runtime. Co-Authored-By: Claude Fable 5 Signed-off-by: Tyler Michael Smith --- evals/ci/Dockerfile | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/evals/ci/Dockerfile b/evals/ci/Dockerfile index daa1db3c0a8d..0f681f0e93df 100644 --- a/evals/ci/Dockerfile +++ b/evals/ci/Dockerfile @@ -33,7 +33,12 @@ COPY . /src # Headless CLI only: no SDL/OpenGL/UI. TTF and audio downloads are dead weight # for an assetless eval. DOWNLOAD_OBJECTS stays ON: the bundled JSON object # pack (~11 MB) is what lets parks load without any RCT2 install. +# The prefix must be set at CONFIGURE time: the objects/asset downloads run +# from install(CODE) strings that bake CMAKE_INSTALL_FULL_DATADIR when +# configuring, so `cmake --install --prefix` would land them at the stale +# /usr/local while everything else moves — exactly the invisible-objects bug. RUN cmake -S /src -B /build -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr \ -DDISABLE_GUI=ON \ -DDISABLE_TTF=ON \ -DDISABLE_DISCORD_RPC=ON \ @@ -46,9 +51,9 @@ RUN cmake -S /src -B /build -G Ninja -DCMAKE_BUILD_TYPE=Release \ # generated by the sprite compiler from in-repo resources) orphaned, but # the install manifest still expects the files — build it explicitly. && cmake --build /build --target graphics \ - && DESTDIR=/stage cmake --install /build --prefix /usr + && DESTDIR=/stage cmake --install /build -FROM ubuntu:24.04 +FROM ubuntu:24.04 AS runtime RUN apt-get update && apt-get install -y --no-install-recommends \ libpng16-16t64 libzip4t64 libicu74 libcurl4t64 libssl3t64 \ @@ -62,3 +67,24 @@ COPY test/tests/testdata/parks/BigMapTest.sv6 evals/programs/test_oval.json /opt EXPOSE 8791 CMD ["openrct2-cli", "eval", "/opt/coasterbench/BigMapTest.sv6", "--no-graphics", "--serve", "8791", "--serve-bind", "0.0.0.0"] + +# Self-contained eval runner: game + driver + preinstalled deps, so a single +# `docker run` plays a full benchmark against any OpenAI-compatible endpoint +# with no network needs beyond the endpoint itself. This is the default build +# target; the lean game-only image is `--target runtime`. +FROM runtime AS eval + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 python3-venv \ + && rm -rf /var/lib/apt/lists/* \ + && python3 -m venv /opt/venv \ + && /opt/venv/bin/pip install --no-cache-dir 'anthropic[vertex]>=0.40' 'openai>=1.40' 'pillow>=10' + +COPY evals/driver.py evals/ci /opt/coasterbench/evals/ +COPY evals/programs /opt/coasterbench/evals/programs +ENV COASTERBENCH_CLI=/usr/bin/openrct2-cli + +# docker run --network host coasterbench-eval \ +# /opt/venv/bin/python /opt/coasterbench/evals/driver.py \ +# --base-url http://localhost:8000/v1 --models --no-graphics \ +# --scenario /opt/coasterbench/BigMapTest.sv6 From 4e7630d0b77148cdfcf908b411e0228c12f44e21 Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Fri, 24 Jul 2026 19:06:49 -0400 Subject: [PATCH 15/18] driver: COASTERBENCH_OPENRCT2_DATA env for containerized data path Co-Authored-By: Claude Fable 5 Signed-off-by: Tyler Michael Smith --- evals/driver.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/evals/driver.py b/evals/driver.py index 2d47f8c83fbc..d3931ee309b6 100644 --- a/evals/driver.py +++ b/evals/driver.py @@ -89,9 +89,15 @@ def rct2_args() -> list[str]: """The eval CLI either loads the RCT2 install or runs assetless.""" + args = [] + # Containers/chroots can't always resolve the data dir relative to the + # binary (/proc may be absent); CI sets this explicitly. + data = os.environ.get("COASTERBENCH_OPENRCT2_DATA") + if data: + args += ["--openrct2-data-path", data] if NO_GRAPHICS: - return ["--no-graphics"] - return ["--rct2-data-path", str(RCT2_DATA)] + return args + ["--no-graphics"] + return args + ["--rct2-data-path", str(RCT2_DATA)] PIECE_CATALOG = """ Station (required, place these FIRST, 3+ in a row): begin_station, middle_station, end_station From 4abde858f44c9436c5b6fa40839dfaed92472090 Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Sat, 25 Jul 2026 00:54:50 -0400 Subject: [PATCH 16/18] OpenAI lane: hour-long request timeout for thinking models The SDK's 10-minute default killed a legitimate 85k-token think mid-generation. Co-Authored-By: Claude Fable 5 Signed-off-by: Tyler Michael Smith --- evals/driver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/evals/driver.py b/evals/driver.py index d3931ee309b6..01b405ce07ab 100644 --- a/evals/driver.py +++ b/evals/driver.py @@ -727,7 +727,9 @@ class OpenAICompat: def __init__(self, base_url: str, api_key: str, extra_body: dict | None = None): import openai - self._client = openai.OpenAI(base_url=base_url, api_key=api_key) + # A thinking model can legitimately generate for well over the SDK's + # 10-minute default timeout (131k tokens at ~140 tok/s is ~15 min). + self._client = openai.OpenAI(base_url=base_url, api_key=api_key, timeout=3600) # Endpoint-specific request extras, e.g. vLLM's chat_template_kwargs # ({"enable_thinking": false} tames reasoning models whose thinking # would otherwise exhaust any completion budget on this task). From c1c8d47e40fe9bb28bdbd158ebf41bcf821c4628 Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Sat, 25 Jul 2026 01:19:06 -0400 Subject: [PATCH 17/18] =?UTF-8?q?CI=20notes:=20thinking-tier=20verdict=20?= =?UTF-8?q?=E2=80=94=20unbounded=20reasoning=20on=20one-shot=20prompts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every confound removed (512k context, 131k budget, hour timeout, pod-to-pod network, reasoning passback); turn-one thinking still never terminates, with or without tools. One-shot runs disable thinking; thinking models belong in the interactive per-piece mode. Co-Authored-By: Claude Fable 5 Signed-off-by: Tyler Michael Smith --- evals/ci/README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/evals/ci/README.md b/evals/ci/README.md index b44f5f33b86e..8c3da37e9f03 100644 --- a/evals/ci/README.md +++ b/evals/ci/README.md @@ -70,7 +70,15 @@ because they are exactly the failure classes this job exists to surface: interleaved thinking and hit `finish_reason: "length"` with no tool call, masquerading as finding #2. vLLM behaved correctly. The driver now fails fast with the real cause and takes `--max-tokens` / - `--chat-template-kwargs '{"enable_thinking": false}'`. + `--chat-template-kwargs '{"enable_thinking": false}'`. Follow-up ruled + out every confound: at TP=8, 512k serving context, a 131k budget, an + hour-long client timeout, pod-to-pod networking, and reasoning passed + back across turns (vLLM renders assistant `reasoning` into the + template — verified via prompt_tokens), turn-one thinking still never + terminated, and it also failed to terminate with no tools attached. + Conclusion: unbounded reasoning is a property of monolithic one-shot + design prompts; thinking-tier contenders belong in the interactive + per-piece mode, and one-shot runs should disable thinking. 4. **Config foot-gun — `generation_config.json` overrides server sampling defaults** (vLLM warns but serves): Qwen ships temp 0.7 / top-p 0.8 / rep-penalty 1.05, so "default" runs are not the sampling you assumed. From ea30482fb1653369c479407d5b811b461a2c47d2 Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Sat, 25 Jul 2026 10:21:31 -0400 Subject: [PATCH 18/18] driver: save the raw response when a call exhausts its budget The 131k-token thinking trace was the evidence both times a run died this way, and raising without persisting it lost it both times. Failed calls now dump the full response into the run dir. Co-Authored-By: Claude Fable 5 Signed-off-by: Tyler Michael Smith --- evals/driver.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/evals/driver.py b/evals/driver.py index 01b405ce07ab..3ecdd216bc54 100644 --- a/evals/driver.py +++ b/evals/driver.py @@ -86,6 +86,12 @@ # multimodal contenders in no-graphics runs. SCHEMATIC_FEEDBACK = False +# Set to the run dir by main(): calls that fail (e.g. a reasoning model +# exhausting its budget) dump their raw response here, because the response +# body — especially a 131k-token thinking trace — is the evidence, and +# raising without saving it has already lost that evidence twice. +FAILED_CALL_DIR: Path | None = None + def rct2_args() -> list[str]: """The eval CLI either loads the RCT2 install or runs assetless.""" @@ -790,9 +796,14 @@ def create(self, *, model: str, max_tokens: int, system: str, messages: list[dic # Deterministic, so retrying just burns tokens: the model (a # reasoning model, usually) hit the token ceiling while still # thinking and never got to the call. + where = "" + if FAILED_CALL_DIR is not None: + dump = FAILED_CALL_DIR / f"failed-call-{model.replace('/', '_')}.json" + dump.write_text(json.dumps(resp.model_dump(), indent=1)) + where = f"; full response (thinking trace included) saved to {dump}" raise RuntimeError( f"{model} exhausted max_tokens={max_tokens} before emitting a tool call " - "(reasoning models spend the budget thinking first; raise --max-tokens)" + f"(reasoning models spend the budget thinking first; raise --max-tokens){where}" ) print(f" [{model}] no tool call (attempt {attempt + 1}/3), retrying", flush=True) raise RuntimeError(f"{model} returned no tool call in 3 attempts despite tool_choice={oa_choice!r}") @@ -995,6 +1006,8 @@ def main() -> int: suffix = args.name if args.name else time.strftime("%H%M%S") run_dir = REPO / "evals" / "runs" / f"{time.strftime('%Y%m%d')}-{suffix}" run_dir.mkdir(parents=True) + global FAILED_CALL_DIR + FAILED_CALL_DIR = run_dir print(f"run dir: {run_dir} (mode: {args.mode})") library = None