diff --git a/.dockerignore b/.dockerignore index 2bcb8a0e79a2..c05da971df39 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,3 +2,8 @@ bin/ build/ lib/ obj/ +rust/orct2-agent/target/ +rust/coaster-bench/target/ +rust/coaster-site/target/ +evals/runs/ +evals/library-previews/ diff --git a/CLAUDE.md b/CLAUDE.md index b8fd9266ad3b..3f75dba66319 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -86,6 +86,18 @@ Non-bundled binaries look for `data/` next to the exe. One-time setup: See issue #1 and the readme war story. - Stalls never get ratings (RatingsCalculationType::Stall); tracked rides need a completed test circuit (RideFlag::tested) before ratings compute. +- No-assets mode: `eval --no-graphics` sets gOpenRCT2NoGraphics so the whole + scoring path (park load, placement, testing, ratings, drawability gate) runs + with zero RCT2 files — objects come from the bundled JSON pack; only pixels + need g1.dat. Screenshots/--render-library are refused, the MCP server drops + image tools server-side (Modalities::server_side), and the similarity + penalty is inert (no stock library; report says `similarity: null`). + Assetless scenario default: test/tests/testdata/parks/BigMapTest.sv6 + (mostly-open flat grass, cash-rich, builds fine). driver.py mirrors the + flag (`--no-graphics`, design mode only) and also takes `--base-url` for + any OpenAI-compatible endpoint (e.g. `vllm serve`); `evals/ci/` has the + CPU-only Dockerfile, a protocol-success gate (check_run.py), and the CI + job shape. - Head-to-head driver: `uv run evals/driver.py` (needs ANTHROPIC_API_KEY, or `--vertex` with GCP ADC; project defaults from $ANTHROPIC_VERTEX_PROJECT_ID); results under `evals/runs//`. Models get a validate_track_program diff --git a/evals/ci/Dockerfile b/evals/ci/Dockerfile new file mode 100644 index 000000000000..0f681f0e93df --- /dev/null +++ b/evals/ci/Dockerfile @@ -0,0 +1,90 @@ +# CoasterBench game image for CI: openrct2-cli with the Rust agent bridge, +# headless-only, zero RCT2 assets (runs eval/MCP with --no-graphics). +# +# The image is CPU-only and small (~hundreds of MB); build it once, push to a +# registry, and pin the digest in CI. The model under test runs elsewhere +# (e.g. `vllm serve` in a sibling container); this container only hosts the +# game as an MCP server or batch eval. +# +# docker build -f evals/ci/Dockerfile -t coasterbench-game . +# docker run --rm -p 8791:8791 coasterbench-game \ +# openrct2-cli eval /opt/coasterbench/BigMapTest.sv6 --no-graphics \ +# --serve 8791 --serve-bind 0.0.0.0 +# +# Verified: builds and serves on Linux arm64 (podman, Ubuntu 24.04 base) — +# a container-built oval places, tests, and rates identically to the macOS +# build. x86_64 not yet exercised. + +FROM ubuntu:24.04 AS build + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential cmake ninja-build git curl ca-certificates pkg-config \ + libpng-dev libzip-dev libssl-dev libcurl4-openssl-dev libicu-dev \ + nlohmann-json3-dev libflac-dev libogg-dev libvorbis-dev libzstd-dev \ + libbz2-dev duktape-dev \ + && rm -rf /var/lib/apt/lists/* + +# Rust toolchain for the agent staticlib (corrosion drives cargo from CMake). +RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable +ENV PATH=/root/.cargo/bin:$PATH + +COPY . /src + +# Headless CLI only: no SDL/OpenGL/UI. TTF and audio downloads are dead weight +# for an assetless eval. DOWNLOAD_OBJECTS stays ON: the bundled JSON object +# pack (~11 MB) is what lets parks load without any RCT2 install. +# The prefix must be set at CONFIGURE time: the objects/asset downloads run +# from install(CODE) strings that bake CMAKE_INSTALL_FULL_DATADIR when +# configuring, so `cmake --install --prefix` would land them at the stale +# /usr/local while everything else moves — exactly the invisible-objects bug. +RUN cmake -S /src -B /build -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DDISABLE_GUI=ON \ + -DDISABLE_TTF=ON \ + -DDISABLE_DISCORD_RPC=ON \ + -DDISABLE_VERSION_CHECKER=ON \ + -DDOWNLOAD_OPENSFX=OFF \ + -DDOWNLOAD_OPENMUSIC=OFF \ + -DDOWNLOAD_TITLE_SEQUENCES=OFF \ + && cmake --build /build \ + # DISABLE_GUI leaves the `graphics` target (g2/fonts/palettes/tracks.dat, + # generated by the sprite compiler from in-repo resources) orphaned, but + # the install manifest still expects the files — build it explicitly. + && cmake --build /build --target graphics \ + && DESTDIR=/stage cmake --install /build + +FROM ubuntu:24.04 AS runtime + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libpng16-16t64 libzip4t64 libicu74 libcurl4t64 libssl3t64 \ + libflac12t64 libvorbisfile3 libzstd1 ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=build /stage/usr /usr +# The checked-in assetless scenario and a known-good smoke program, so the +# container is self-contained for both serving and zero-GPU smoke runs. +COPY test/tests/testdata/parks/BigMapTest.sv6 evals/programs/test_oval.json /opt/coasterbench/ + +EXPOSE 8791 +CMD ["openrct2-cli", "eval", "/opt/coasterbench/BigMapTest.sv6", "--no-graphics", "--serve", "8791", "--serve-bind", "0.0.0.0"] + +# Self-contained eval runner: game + driver + preinstalled deps, so a single +# `docker run` plays a full benchmark against any OpenAI-compatible endpoint +# with no network needs beyond the endpoint itself. This is the default build +# target; the lean game-only image is `--target runtime`. +FROM runtime AS eval + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 python3-venv \ + && rm -rf /var/lib/apt/lists/* \ + && python3 -m venv /opt/venv \ + && /opt/venv/bin/pip install --no-cache-dir 'anthropic[vertex]>=0.40' 'openai>=1.40' 'pillow>=10' + +COPY evals/driver.py evals/ci /opt/coasterbench/evals/ +COPY evals/programs /opt/coasterbench/evals/programs +ENV COASTERBENCH_CLI=/usr/bin/openrct2-cli + +# docker run --network host coasterbench-eval \ +# /opt/venv/bin/python /opt/coasterbench/evals/driver.py \ +# --base-url http://localhost:8000/v1 --models --no-graphics \ +# --scenario /opt/coasterbench/BigMapTest.sv6 diff --git a/evals/ci/README.md b/evals/ci/README.md new file mode 100644 index 000000000000..8c3da37e9f03 --- /dev/null +++ b/evals/ci/README.md @@ -0,0 +1,85 @@ +# CoasterBench in CI (no RCT2 assets) + +Everything the eval scores — park loading, track placement, ride testing, +ratings — runs without any RollerCoaster Tycoon 2 files via `--no-graphics` +(objects come from the bundled JSON pack; only pixels need `g1.dat`). That +makes the whole benchmark legally shippable in a CI image. What you give up: +screenshots (feedback is the eval report only, so contenders run text-only) +and the stock TD6 library (library mode is unavailable and the similarity +penalty is inert, so scores are not comparable with asset-full leaderboard +runs — `run.json` records `no_graphics: true`). + +## Pieces + +- `Dockerfile` — CPU-only game image (headless `openrct2-cli`, Rust agent, + bundled objects, checked-in scenario; ~240 MB). Build once, push, pin the + digest. Verified on Linux arm64 (podman): the containerized MCP server + builds, tests, and rates a coaster with ratings identical to the macOS + build. x86_64 not yet exercised. +- `check_run.py` — pass/fail gate: every model needs ≥1 round that built and + completed a test circuit. Scores are metrics, not assertions (models are + nondeterministic; don't gate merges on excitement). + +## Shape of a job against vLLM + +Nightly / non-blocking is the realistic tier — the wall clock is model +inference, the game sim is ~7s per 25k-tick round. + +```bash +# 1. serve the model under test (own container/step; needs the GPU) +vllm serve "$MODEL" --enable-auto-tool-choice --tool-call-parser hermes & + +# 2. game + driver are CPU-only +uv run evals/driver.py \ + --base-url http://localhost:8000/v1 \ + --models "$MODEL" --rounds "${ROUNDS:-2}" --no-graphics \ + --name "ci-${BUILD_NUMBER:-local}" + +# 3. gate on protocol success, keep the run dir as the artifact +python3 evals/ci/check_run.py evals/runs/*-"ci-${BUILD_NUMBER:-local}" +``` + +What this exercises end-to-end that parser unit tests don't: forced and named +`tool_choice` (guided decoding), multi-turn tool loops with tool_result +round-trips, and large JSON tool arguments (a 148-piece track program is a +few KB of structured output). + +There is also a zero-GPU smoke: replay a canned program with no model at all — +`openrct2-cli eval test/tests/testdata/parks/BigMapTest.sv6 --no-graphics +--ticks 25000 --program evals/programs/test_oval.json --out report.json` +must produce `program.ok == true` and a tested ride. + +## Field notes: what the first live runs caught (vLLM 0.25.1, A100) + +Findings from the eval's first day out, with honest attribution — kept here +because they are exactly the failure classes this job exists to surface: + +1. **Confirmed vLLM bug — named `tool_choice` silently unenforced** + (`poolside_v1` parser, Laguna-S-2.1, thinking disabled): a request + forcing `submit_track_program` by name got `validate_track_program` + back, 3/3 reproducible probes. The contract requires enforcement or + rejection, not a different function. The driver copes by giving the + forced-submit step three attempts. +2. **Unconfirmed anomaly — `tool_choice: "required"` returned zero tool + calls** once, on the server's first-ever structured-output request + (Qwen2.5-7B, hermes parser). Never reproduced (0/15 probes) and the + failing response body wasn't captured, so it is not claimable as a bug. + The driver retries up to 3×. +3. **Harness bug (ours, fixed) — fixed `max_tokens` starves reasoning + models**: Laguna spent the entire 8k (then 24k) completion budget on + interleaved thinking and hit `finish_reason: "length"` with no tool + call, masquerading as finding #2. vLLM behaved correctly. The driver + now fails fast with the real cause and takes `--max-tokens` / + `--chat-template-kwargs '{"enable_thinking": false}'`. Follow-up ruled + out every confound: at TP=8, 512k serving context, a 131k budget, an + hour-long client timeout, pod-to-pod networking, and reasoning passed + back across turns (vLLM renders assistant `reasoning` into the + template — verified via prompt_tokens), turn-one thinking still never + terminated, and it also failed to terminate with no tools attached. + Conclusion: unbounded reasoning is a property of monolithic one-shot + design prompts; thinking-tier contenders belong in the interactive + per-piece mode, and one-shot runs should disable thinking. +4. **Config foot-gun — `generation_config.json` overrides server sampling + defaults** (vLLM warns but serves): Qwen ships temp 0.7 / top-p 0.8 / + rep-penalty 1.05, so "default" runs are not the sampling you assumed. + Pin `--generation-config vllm` for vLLM defaults. diff --git a/evals/ci/check_run.py b/evals/ci/check_run.py new file mode 100644 index 000000000000..0e2ab489e2dd --- /dev/null +++ b/evals/ci/check_run.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""CI pass/fail gate for a CoasterBench run directory. + +Scores are model quality, not infrastructure health, so CI asserts protocol +success only: every model must have at least one round whose program built OK +and whose ride completed a test circuit. Score stays a reported metric. + + usage: check_run.py evals/runs/ +""" +import json +import sys +from pathlib import Path + + +def main() -> int: + if len(sys.argv) != 2: + print(__doc__, file=sys.stderr) + return 2 + run_dir = Path(sys.argv[1]) + run = json.loads((run_dir / "run.json").read_text()) + failures = [] + for model in run["models"]: + reports = sorted(run_dir.glob(f"{model.replace('/', '_')}/round_*/report.json")) + tested = [] + for path in reports: + report = json.loads(path.read_text()) + # Batch runs name the program's ride; MCP-harness runs have no + # program, so any tested ride in the round's report counts. + ride_id = (report.get("program") or {}).get("ride_id") + tested += [ + r + for r in report.get("rides", []) or [] + if r.get("tested") and (ride_id is None or r["id"] == ride_id) + ] + best = max((r.get("excitement") or 0.0 for r in tested), default=None) + if not reports: + failures.append(f"{model}: no rounds ran") + elif not tested: + failures.append(f"{model}: {len(reports)} round(s), none produced a tested coaster") + else: + print(f"ok: {model}: {len(tested)}/{len(reports)} rounds tested, best excitement {best:.2f}") + for failure in failures: + print(f"FAIL: {failure}", file=sys.stderr) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/evals/driver.py b/evals/driver.py index ca6e5fb16270..3ecdd216bc54 100644 --- a/evals/driver.py +++ b/evals/driver.py @@ -1,6 +1,6 @@ # /// script # requires-python = ">=3.11" -# dependencies = ["anthropic[vertex]>=0.40"] +# dependencies = ["anthropic[vertex]>=0.40", "openai>=1.40", "pillow>=10"] # /// """Coaster design head-to-head: two Claude models iteratively design a coaster. @@ -25,6 +25,16 @@ Vertex model IDs for current-generation models are the bare first-party strings (claude-opus-4-6, claude-sonnet-5) — no prefix, no @date suffix. + +Usage (any OpenAI-compatible endpoint, e.g. a local vLLM server): + vllm serve Qwen/Qwen2.5-7B-Instruct --enable-auto-tool-choice ... + uv run evals/driver.py --base-url http://localhost:8000/v1 \ + --models Qwen/Qwen2.5-7B-Instruct --rounds 4 --no-graphics + +--no-graphics runs the game without RCT2 assets (design mode only): the +scenario defaults to a checked-in test park, feedback is the eval report +alone (no park screenshot), and the similarity penalty is inert because +there is no stock library to compare against. """ from __future__ import annotations @@ -44,9 +54,56 @@ import anthropic REPO = Path(__file__).resolve().parent.parent -CLI = REPO / "build" / "openrct2-cli" +# COASTERBENCH_CLI lets a CI environment point at a binary that didn't come +# from this checkout's build dir (e.g. extracted from the game image). +CLI = Path(os.environ.get("COASTERBENCH_CLI", REPO / "build" / "openrct2-cli")) DEFAULT_SCENARIO = Path.home() / "rct2-assets" / "Scenarios" / "Build your own Six Flags Park.SC6" RCT2_DATA = Path.home() / "rct2-assets" +# Assetless default: a checked-in upstream test park (large, mostly-open flat +# grass, cash-rich) that loads and builds with only the bundled JSON objects. +CI_SCENARIO = REPO / "test" / "tests" / "testdata" / "parks" / "BigMapTest.sv6" + +MAP_LINES = { + DEFAULT_SCENARIO.name: ( + "Flat grass around tile (60, 60); a lake sits near map centre roughly tiles (68-85, 55-75) — do NOT " + "build into it. Stay within tiles 20-120. Directions: dir 0 faces -x, dir 1 faces +y, dir 2 faces +x, " + "dir 3 faces -y." + ), + CI_SCENARIO.name: ( + "A large park: flat open grass across roughly tiles 30-190 on both axes, with scattered existing " + "rides and footpaths (placement errors will name what is in the way; shift a few tiles and retry). " + "Flat grass around tile (60, 60) is a good anchor. Directions: dir 0 faces -x, dir 1 faces +y, " + "dir 2 faces +x, dir 3 faces -y." + ), +} + +# Set from --no-graphics in main(): the game loads no sprite data, so no RCT2 +# assets are needed and nothing can render (no screenshots, no previews). +NO_GRAPHICS = False + +# Set from --schematic-feedback: attach the schematic track diagram (rendered +# from the report's cursor trace, no assets needed) to round feedback, for +# multimodal contenders in no-graphics runs. +SCHEMATIC_FEEDBACK = False + +# Set to the run dir by main(): calls that fail (e.g. a reasoning model +# exhausting its budget) dump their raw response here, because the response +# body — especially a 131k-token thinking trace — is the evidence, and +# raising without saving it has already lost that evidence twice. +FAILED_CALL_DIR: Path | None = None + + +def rct2_args() -> list[str]: + """The eval CLI either loads the RCT2 install or runs assetless.""" + args = [] + # Containers/chroots can't always resolve the data dir relative to the + # binary (/proc may be absent); CI sets this explicitly. + data = os.environ.get("COASTERBENCH_OPENRCT2_DATA") + if data: + args += ["--openrct2-data-path", data] + if NO_GRAPHICS: + return args + ["--no-graphics"] + return args + ["--rct2-data-path", str(RCT2_DATA)] PIECE_CATALOG = """ Station (required, place these FIRST, 3+ in a row): begin_station, middle_station, end_station @@ -92,9 +149,14 @@ def ride_type_info(ride_type: int) -> tuple[str, str]: return name, line + " This is the required type for this competition." -def build_system_prompt(ride_type: int) -> str: +def build_system_prompt(ride_type: int, scenario: Path) -> str: _, ride_line = ride_type_info(ride_type) - return SYSTEM_PROMPT.replace("{RIDE_TYPE_LINE}", ride_line) + map_line = MAP_LINES.get( + scenario.name, + "Terrain unknown; flat grass around tile (60, 60) is a reasonable first bet. Use validation " + "errors to find open ground. Directions: dir 0 faces -x, dir 1 faces +y, dir 2 faces +x, dir 3 faces -y.", + ) + return SYSTEM_PROMPT.replace("{RIDE_TYPE_LINE}", ride_line).replace("{MAP_LINE}", map_line) SYSTEM_PROMPT = f"""You are competing to design the best RollerCoaster Tycoon 2 roller coaster. @@ -124,7 +186,7 @@ def build_system_prompt(ride_type: int) -> str: {PIECE_CATALOG} ## Map -Flat grass around tile (60, 60); a lake sits near map centre roughly tiles (68-85, 55-75) — do NOT build into it. Stay within tiles 20-120. Directions: dir 0 faces -x, dir 1 faces +y, dir 2 faces +x, dir 3 faces -y. +{{MAP_LINE}} Before submitting, use the validate_track_program tool (same payload) to dry-run your program: it reports placement errors with the exact piece index, or whether the circuit closes, without spending your round. You get a limited number of validations per round, use them to fix geometry, then submit. @@ -280,6 +342,72 @@ def best(self) -> Attempt | None: return max(rated, key=lambda a: a.excitement) if rated else None +STATION_PIECES = {"begin_station", "middle_station", "end_station"} + + +def render_schematic(trace: list[dict], out_path: Path) -> Path | None: + """Draws the placed track as a two-panel PNG (top-down + isometric) from + the report's cursor trace — no game assets involved. Stations are green, + chain lift red, everything else shaded by height; an open circuit gets a + dashed gap line from track end back to the start.""" + if len(trace) < 2: + return None + from PIL import Image, ImageDraw + + pts = [(p["x"], p["y"], p["z"]) for p in trace] + zs = [z for _, _, z in pts] + z0, z1 = min(zs), max(zs) + + def color(i: int) -> tuple[int, int, int]: + piece = trace[i]["piece"] + if piece in STATION_PIECES: + return (46, 160, 67) + if trace[i].get("chain"): + return (220, 68, 61) + t = (pts[i][2] - z0) / (z1 - z0) if z1 > z0 else 0.0 + return (int(60 + 195 * t), int(120 - 40 * t), int(220 - 160 * t)) + + panels = { + "top": lambda x, y, z: (x, y), + "iso": lambda x, y, z: (x - y, (x + y) * 0.5 - z / 24), + } + size, margin = 640, 40 + img = Image.new("RGB", (size * 2, size), (250, 250, 248)) + draw = ImageDraw.Draw(img) + + closed = pts[0][:2] == pts[-1][:2] and trace[0]["z"] == trace[-1]["z"] + for panel, (name, proj) in enumerate(panels.items()): + proj_pts = [proj(*p) for p in pts] + xs = [u for u, _ in proj_pts] + ys = [v for _, v in proj_pts] + span = max(max(xs) - min(xs), max(ys) - min(ys)) or 1.0 + scale = (size - 2 * margin) / span + + def to_px(uv, panel=panel, xs=xs, ys=ys, scale=scale): + return ( + panel * size + margin + (uv[0] - min(xs)) * scale, + margin + (uv[1] - min(ys)) * scale, + ) + + px = [to_px(p) for p in proj_pts] + for i in range(1, len(px)): + draw.line([px[i - 1], px[i]], fill=color(i), width=4) + if not closed: + draw.line([px[-1], px[0]], fill=(150, 150, 150), width=2) + sx, sy = px[0] + draw.ellipse([sx - 5, sy - 5, sx + 5, sy + 5], outline=(0, 0, 0), width=2) + draw.text((panel * size + margin, size - margin + 8), name, fill=(90, 90, 90)) + + if not closed: + dx = pts[0][0] - pts[-1][0] + dy = pts[0][1] - pts[-1][1] + dz = trace[0]["z"] - trace[-1]["z"] + draw.text((margin, 8), f"OPEN CIRCUIT: gap to start dx={dx} dy={dy} dz={dz}", fill=(180, 30, 30)) + draw.text((size + margin, 8), "green=station red=chain-lift blue->orange=height", fill=(90, 90, 90)) + img.save(out_path) + return out_path + + def run_eval(program: dict, scenario: Path, workdir: Path, ticks: int) -> tuple[dict, Path | None]: workdir.mkdir(parents=True, exist_ok=True) program_path = workdir / "program.json" @@ -290,18 +418,25 @@ def run_eval(program: dict, scenario: Path, workdir: Path, ticks: int) -> tuple[ cmd = [ str(CLI), "eval", str(scenario), "--ticks", str(ticks), - "--rct2-data-path", str(RCT2_DATA), + *rct2_args(), "--program", str(program_path), "--out", str(report_path), - "--capture", str(capture_path), - "--capture-xray", ] + if not NO_GRAPHICS: + cmd += ["--capture", str(capture_path), "--capture-xray"] proc = subprocess.run(cmd, capture_output=True, text=True, timeout=600) if not report_path.exists(): return {"program": {"ok": False, "error": {"message": f"eval crashed: {proc.stderr[-500:]}"}}}, None report = json.loads(report_path.read_text()) - shot = None + trace = (report.get("program") or {}).get("trace") or [] + schematic = None + if trace: + try: + schematic = render_schematic(trace, workdir / "track.png") + except Exception as e: # a diagram must never sink the round + print(f" schematic render failed: {e}", file=sys.stderr) + shot = schematic if SCHEMATIC_FEEDBACK else None if capture_path.exists(): small = workdir / "park_small.png" # The API rejects images over 5 MB of base64 (~3.7 MB raw); tall parks @@ -345,7 +480,7 @@ def validate_program(program: dict, scenario: Path) -> str: [ str(CLI), "eval", str(scenario), "--ticks", "5", - "--rct2-data-path", str(RCT2_DATA), + *rct2_args(), "--program", str(program_path), "--out", str(report_path), ], @@ -495,8 +630,187 @@ def feedback_content(attempt: Attempt) -> list[dict]: return content +@dataclass +class ToolUseBlock: + id: str + name: str + input: dict + type: str = "tool_use" + + +@dataclass +class TextBlock: + text: str + type: str = "text" + + +@dataclass +class ReasoningBlock: + """A reasoning model's thinking, preserved so it can be passed back on the + next turn (vLLM renders `reasoning` on assistant messages into the chat + template — verified: prompt_tokens grows by the trace length). Without + passback the model re-derives everything from scratch every call.""" + + text: str + type: str = "reasoning" + + +@dataclass +class _Usage: + input_tokens: int + output_tokens: int + + +@dataclass +class _Response: + content: list + usage: _Usage + + +def _to_openai(message: dict) -> list[dict]: + """One anthropic-form history entry -> the OpenAI messages it becomes. + + The driver only ever builds three shapes: a plain-string user message, a + user message holding tool_result blocks, and an assistant message whose + content is the block list a previous create() returned. + """ + role = message["role"] + content = message["content"] + if role == "assistant": + text = "".join(b.text for b in content if b.type == "text") + reasoning = "".join(b.text for b in content if b.type == "reasoning") + calls = [ + {"id": b.id, "type": "function", "function": {"name": b.name, "arguments": json.dumps(b.input)}} + for b in content + if b.type == "tool_use" + ] + msg: dict = {"role": "assistant", "content": text or None} + if reasoning: + msg["reasoning"] = reasoning + if calls: + msg["tool_calls"] = calls + return [msg] + if isinstance(content, str): + return [{"role": "user", "content": content}] + out: list[dict] = [] + images: list[str] = [] + for block in content: + if not isinstance(block, dict) or block.get("type") != "tool_result": + continue + inner = block.get("content") + texts: list[str] = [] + if isinstance(inner, str): + texts.append(inner) + else: + for part in inner or []: + if part.get("type") == "text": + texts.append(part["text"]) + elif part.get("type") == "image": + images.append(part["source"]["data"]) + out.append( + {"role": "tool", "tool_call_id": block["tool_use_id"], "content": "\n".join(texts) or "(no text)"} + ) + for data in images: + # OpenAI tool messages are text-only; a park screenshot rides along + # as a follow-up user message instead. + out.append( + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{data}"}}], + } + ) + return out + + +class OpenAICompat: + """Anthropic-messages-shaped facade over an OpenAI chat-completions + endpoint (vLLM serve, llama.cpp, OpenRouter, ...). compete() only touches + client.messages.create, response.content, and response.usage, so the tool + loop stays identical across lanes. Named and required tool_choice both map + onto the endpoint's structured-output support (vLLM: guided decoding via + --enable-auto-tool-choice).""" + + def __init__(self, base_url: str, api_key: str, extra_body: dict | None = None): + import openai + + # A thinking model can legitimately generate for well over the SDK's + # 10-minute default timeout (131k tokens at ~140 tok/s is ~15 min). + self._client = openai.OpenAI(base_url=base_url, api_key=api_key, timeout=3600) + # Endpoint-specific request extras, e.g. vLLM's chat_template_kwargs + # ({"enable_thinking": false} tames reasoning models whose thinking + # would otherwise exhaust any completion budget on this task). + self._extra_body = extra_body or {} + self.messages = self # so client.messages.create(...) resolves here + + def create(self, *, model: str, max_tokens: int, system: str, messages: list[dict], tools: list[dict], tool_choice: dict) -> _Response: + payload: list[dict] = [{"role": "system", "content": system}] + for message in messages: + payload.extend(_to_openai(message)) + oa_tools = [ + { + "type": "function", + "function": {"name": t["name"], "description": t.get("description", ""), "parameters": t["input_schema"]}, + } + for t in tools + ] + oa_choice: str | dict = ( + {"type": "function", "function": {"name": tool_choice["name"]}} + if tool_choice.get("type") == "tool" + else "required" + ) + # tool_choice="required" is not airtight in the wild: vLLM's guided + # grammar can emit an empty call array, so a callless response gets + # retried rather than killing the run. + input_tokens = output_tokens = 0 + for attempt in range(3): + resp = self._client.chat.completions.create( + model=model, + max_tokens=max_tokens, + messages=payload, + tools=oa_tools, + tool_choice=oa_choice, + extra_body=self._extra_body, + ) + if resp.usage: + input_tokens += resp.usage.prompt_tokens + output_tokens += resp.usage.completion_tokens + choice = resp.choices[0].message + content: list = [] + # The SDK model keeps unknown fields; reasoning arrives as an + # extra ("reasoning" on vLLM, "reasoning_content" on some stacks). + extra = choice.model_dump() if hasattr(choice, "model_dump") else {} + reasoning = extra.get("reasoning") or extra.get("reasoning_content") + if reasoning: + content.append(ReasoningBlock(text=reasoning)) + if choice.content: + content.append(TextBlock(text=choice.content)) + for call in choice.tool_calls or []: + try: + args = json.loads(call.function.arguments or "{}") + except json.JSONDecodeError: + args = {} + content.append(ToolUseBlock(id=call.id, name=call.function.name, input=args)) + if any(b.type == "tool_use" for b in content): + return _Response(content=content, usage=_Usage(input_tokens, output_tokens)) + if resp.choices[0].finish_reason == "length": + # Deterministic, so retrying just burns tokens: the model (a + # reasoning model, usually) hit the token ceiling while still + # thinking and never got to the call. + where = "" + if FAILED_CALL_DIR is not None: + dump = FAILED_CALL_DIR / f"failed-call-{model.replace('/', '_')}.json" + dump.write_text(json.dumps(resp.model_dump(), indent=1)) + where = f"; full response (thinking trace included) saved to {dump}" + raise RuntimeError( + f"{model} exhausted max_tokens={max_tokens} before emitting a tool call " + f"(reasoning models spend the budget thinking first; raise --max-tokens){where}" + ) + print(f" [{model}] no tool call (attempt {attempt + 1}/3), retrying", flush=True) + raise RuntimeError(f"{model} returned no tool call in 3 attempts despite tool_choice={oa_choice!r}") + + def compete( - client: anthropic.Anthropic | anthropic.AnthropicVertex, + client: anthropic.Anthropic | anthropic.AnthropicVertex | OpenAICompat, model: str, rounds: int, scenario: Path, @@ -504,10 +818,11 @@ def compete( ticks: int, ride_type: int, library: list[dict] | None = None, + max_tokens: int = 8000, ) -> Contender: contender = Contender(model=model) ride_name, _ = ride_type_info(ride_type) - system_prompt = build_system_prompt(ride_type) + (LIBRARY_PROMPT if library is not None else "") + system_prompt = build_system_prompt(ride_type, scenario) + (LIBRARY_PROMPT if library is not None else "") tools = [TOOL, VALIDATE_TOOL] + (LIBRARY_TOOLS if library is not None else []) messages: list[dict] = [ { @@ -522,11 +837,14 @@ def compete( tool_use = None lookups: list[dict] = [] round_usage = {"input_tokens": 0, "output_tokens": 0} - for step in range(MAX_LOOKUPS_PER_ROUND + 1): - force_submit = step == MAX_LOOKUPS_PER_ROUND + # Two extra forced-submit attempts: named tool_choice is not actually + # enforced by every endpoint (vLLM + poolside_v1 returned a different + # tool than the one forced), so the "guaranteed" final step isn't. + for step in range(MAX_LOOKUPS_PER_ROUND + 3): + force_submit = step >= MAX_LOOKUPS_PER_ROUND response = client.messages.create( model=model, - max_tokens=8000, + max_tokens=max_tokens, system=system_prompt, messages=messages, tools=tools, @@ -548,6 +866,15 @@ def compete( print(f" [{model}] round {rnd}: {tool_use.name}({json.dumps(tool_use.input)})", flush=True) result, lookup = library_tool_result(tool_use.name, tool_use.input, library or []) lookups.append(lookup) + if step + 1 >= MAX_LOOKUPS_PER_ROUND: + # Named tool_choice is advisory on some stacks, so forcing has + # to happen in-band too: agentic models otherwise keep + # validating forever instead of ever submitting. + result += ( + "\n\nVALIDATION BUDGET EXHAUSTED: you must now call " + "submit_track_program with your best current program. Do not " + "call any other tool." + ) messages.append( { "role": "user", @@ -555,7 +882,7 @@ def compete( } ) if program is None or tool_use is None: - # Unreachable: the final loop step forces submit_track_program. + # Three forced-submit attempts all returned something else. raise RuntimeError(f"{model} never submitted a program in round {rnd}") if program.get("ride_type") != ride_type: print( @@ -597,7 +924,7 @@ def main() -> int: default="design", help="design = from scratch; library = with track design library search (retrieval eval)", ) - parser.add_argument("--rounds", type=int, default=4) + parser.add_argument("--rounds", type=int, default=6) parser.add_argument( "--ride-type", type=int, @@ -605,8 +932,37 @@ def main() -> int: help="required coaster ride type for the competition (52 wooden, 51 steel twister)", ) parser.add_argument("--ticks", type=int, default=25000) + parser.add_argument( + "--max-tokens", + type=int, + default=8000, + help="completion token budget per request; reasoning models think before " + "they call tools, so give them room (e.g. 24000 for Laguna)", + ) parser.add_argument("--scenario", type=Path, default=DEFAULT_SCENARIO) parser.add_argument("--vertex", action="store_true", help="use Google Vertex AI instead of the first-party API") + parser.add_argument( + "--base-url", + help="OpenAI-compatible endpoint (e.g. a vLLM server: http://localhost:8000/v1); " + "auth from $OPENAI_API_KEY, defaulting to 'EMPTY' for local servers", + ) + parser.add_argument( + "--schematic-feedback", + action="store_true", + help="attach the asset-free schematic track diagram to round feedback " + "(multimodal contenders only; text-only models will reject image content)", + ) + parser.add_argument( + "--chat-template-kwargs", + help="JSON merged into each request as vLLM chat_template_kwargs " + "(OpenAI lane only), e.g. '{\"enable_thinking\": false}'", + ) + parser.add_argument( + "--no-graphics", + action="store_true", + help="run the game without RCT2 assets (design mode only): no screenshots in " + "feedback and no stock library, so the similarity penalty is inert", + ) parser.add_argument( "--project", default=os.environ.get("ANTHROPIC_VERTEX_PROJECT_ID"), @@ -627,6 +983,22 @@ def main() -> int: if not CLI.exists(): print(f"error: {CLI} not built", file=sys.stderr) return 1 + if args.no_graphics: + if args.mode == "library": + print("error: library mode needs the RCT2 track designs; --no-graphics is design mode only", file=sys.stderr) + return 1 + global NO_GRAPHICS + NO_GRAPHICS = True + if args.schematic_feedback: + global SCHEMATIC_FEEDBACK + SCHEMATIC_FEEDBACK = True + if args.scenario == DEFAULT_SCENARIO: + # The graphics-lane default lives in the RCT2 install; assetless + # runs default to the checked-in test park instead. + args.scenario = CI_SCENARIO + if args.vertex and args.base_url: + print("error: pick one of --vertex and --base-url", file=sys.stderr) + return 1 if not args.scenario.exists(): print(f"error: scenario not found: {args.scenario}", file=sys.stderr) return 1 @@ -634,6 +1006,8 @@ def main() -> int: suffix = args.name if args.name else time.strftime("%H%M%S") run_dir = REPO / "evals" / "runs" / f"{time.strftime('%Y%m%d')}-{suffix}" run_dir.mkdir(parents=True) + global FAILED_CALL_DIR + FAILED_CALL_DIR = run_dir print(f"run dir: {run_dir} (mode: {args.mode})") library = None @@ -652,6 +1026,8 @@ def main() -> int: "ticks": args.ticks, "ride_type": args.ride_type, "scenario": args.scenario.name, + "no_graphics": args.no_graphics, + **({"endpoint": args.base_url} if args.base_url else {}), # The site reads the penalty parameters from here; keep the # driver the single source of truth for the scoring math. "similarity_grace": SIMILARITY_GRACE, @@ -660,7 +1036,12 @@ def main() -> int: ) ) - if args.vertex: + if args.base_url: + extra_body = ( + {"chat_template_kwargs": json.loads(args.chat_template_kwargs)} if args.chat_template_kwargs else None + ) + client = OpenAICompat(args.base_url, os.environ.get("OPENAI_API_KEY", "EMPTY"), extra_body) + elif args.vertex: # Auth is GCP application-default credentials, not an Anthropic key. kwargs = {"region": args.region} if args.project: @@ -669,7 +1050,7 @@ def main() -> int: else: client = anthropic.Anthropic() contenders = [ - compete(client, model, args.rounds, args.scenario, run_dir, args.ticks, args.ride_type, library) + compete(client, model, args.rounds, args.scenario, run_dir, args.ticks, args.ride_type, library, args.max_tokens) for model in args.models ] diff --git a/readme.md b/readme.md index dfa1f75a25e5..c6d8d5101b2a 100644 --- a/readme.md +++ b/readme.md @@ -145,6 +145,36 @@ retrieval and adaptation. uv run evals/driver.py --models claude-sonnet-5 --rounds 4 --mode library ``` +### Running without RCT2 assets + +Everything the eval scores — park loading, placement, ride testing, ratings, +the drawability gate — is pure game logic; only rendering needs `g1.dat`. +`--no-graphics` runs the whole benchmark with zero RollerCoaster Tycoon 2 +files (ride/scenery objects come from OpenRCT2's bundled JSON pack, and the +scenario defaults to a checked-in test park): + +```bash +./build/openrct2-cli eval test/tests/testdata/parks/BigMapTest.sv6 \ + --no-graphics --ticks 25000 --program evals/programs/test_oval.json --out report.json +``` + +The trade: no screenshots (the MCP server drops its image tools, contenders +run text-only), no stock library (library mode unavailable, similarity +penalty inert — `run.json` records `no_graphics` so such runs are not +compared against asset-full leaderboards). + +This is what makes the benchmark shippable in CI. The driver speaks to any +OpenAI-compatible endpoint (a `vllm serve` under test, llama.cpp, ...) via +`--base-url`; `evals/ci/` has the CPU-only Dockerfile and the pass/fail gate +(protocol success — a built, tested coaster — not score, which is model +quality, not infrastructure health). + +```bash +uv run evals/driver.py --base-url http://localhost:8000/v1 \ + --models Qwen/Qwen2.5-7B-Instruct --rounds 2 --no-graphics +python3 evals/ci/check_run.py evals/runs/ +``` + ## MCP server ```bash diff --git a/rust/orct2-agent/src/host.rs b/rust/orct2-agent/src/host.rs index 89d63313b7de..a0f8f223d921 100644 --- a/rust/orct2-agent/src/host.rs +++ b/rust/orct2-agent/src/host.rs @@ -108,6 +108,7 @@ unsafe extern "C" { err_len: usize, ) -> bool; fn orct2_host_ride_detail(ride_id: u16, out: *mut RideDetail) -> bool; + fn orct2_host_graphics_available() -> bool; fn orct2_host_capture( path: *const c_char, zoom: i32, @@ -275,6 +276,12 @@ pub fn entrance_place( } } +/// Whether sprite data is loaded. False under `eval --no-graphics` (no RCT2 +/// assets): screenshots cannot render, so image tools must not be offered. +pub fn graphics_available() -> bool { + unsafe { orct2_host_graphics_available() } +} + /// Renders a park screenshot to `path` (PNG). With `fit_track` the view is /// cropped to the bounding box of all track in the park (falls back to the /// full map when no track exists). With `xray` terrain and supports are @@ -417,6 +424,9 @@ mod test_stubs { pub unsafe fn orct2_host_ride_detail(_r: u16, _o: *mut RideDetail) -> bool { false } + pub unsafe fn orct2_host_graphics_available() -> bool { + true + } pub unsafe fn orct2_host_capture( _p: *const c_char, _z: i32, diff --git a/rust/orct2-agent/src/mcp.rs b/rust/orct2-agent/src/mcp.rs index 183b37d2a911..26b467d14276 100644 --- a/rust/orct2-agent/src/mcp.rs +++ b/rust/orct2-agent/src/mcp.rs @@ -44,6 +44,23 @@ impl Modalities { self.0 & other.0 == other.0 } + fn without(self, other: Modalities) -> Modalities { + Modalities(self.0 & !other.0) + } + + /// What this server instance can actually produce: everything, minus + /// image content when sprite data is not loaded (`eval --no-graphics`). + /// Intersected with the client's advertised set per request, so a + /// no-graphics server never lists or answers the screenshot tools no + /// matter what the client claims to accept. + fn server_side() -> Modalities { + if host::graphics_available() { + Modalities::ALL + } else { + Modalities::ALL.without(Modalities::IMAGE) + } + } + /// Parse a comma-separated list; unknown names are ignored. fn parse(list: &str) -> Modalities { list.split(',').fold(Modalities(0), |set, name| { @@ -77,6 +94,13 @@ impl std::ops::BitOr for Modalities { } } +impl std::ops::BitAnd for Modalities { + type Output = Modalities; + fn bitand(self, rhs: Modalities) -> Modalities { + Modalities(self.0 & rhs.0) + } +} + /// A request's claim on the park, read from the query string. struct Claim { lease: Option, @@ -270,7 +294,7 @@ fn handle_connection(stream: TcpStream, session: &mut Session) -> Result<(), Str Ok(None) => return Ok(()), // clean EOF Err(e) => return Err(e), }; - let modalities = Modalities::from_request_target(&target); + let modalities = Modalities::from_request_target(&target) & Modalities::server_side(); if method != "POST" { write_http(&mut stream, 405, "text/plain", b"method not allowed")?; continue; @@ -407,7 +431,7 @@ fn dispatch(message: &Value, session: &mut Session, modalities: Modalities) -> V return rpc_result( id, json!({ - "content": [{"type": "text", "text": format!("{name} is unavailable: this client did not advertise the content kind it answers with")}], + "content": [{"type": "text", "text": format!("{name} is unavailable: either this client did not advertise the content kind it answers with, or the server cannot produce it (screenshots need sprite data, absent under --no-graphics)")}], "isError": true, }), ); @@ -695,7 +719,13 @@ fn call_tool(name: &str, args: &Value, session: &mut Session) -> Result b) { session.best_test = Some(report.clone()); - session.best_shot = capture_park_png().ok(); + // Under --no-graphics nothing can render; skip rather than + // log a capture error every improved test. + session.best_shot = if host::graphics_available() { + capture_park_png().ok() + } else { + None + }; } } Ok(text_content(report)) diff --git a/rust/orct2-agent/src/program.rs b/rust/orct2-agent/src/program.rs index 4b8504ee18c6..ebdd9a3edf97 100644 --- a/rust/orct2-agent/src/program.rs +++ b/rust/orct2-agent/src/program.rs @@ -100,6 +100,21 @@ pub struct ProgramOutcome { /// serialised: the program JSON already spells out the pieces. #[serde(skip)] pub placed_types: Vec, + /// Cursor after each placed piece (tile coords), preceded by the start + /// cursor. Lets a renderer draw the layout schematically with no game + /// assets: point i to i+1 is the chord of piece i. + pub trace: Vec, +} + +#[derive(Debug, Serialize)] +pub struct TracePoint { + /// Piece whose placement ended at this cursor; "start" for the first. + pub piece: String, + pub chain: bool, + pub x: i32, + pub y: i32, + pub z: i32, + pub dir: u8, } #[derive(Debug, Serialize)] @@ -203,6 +218,15 @@ pub fn run(json: &str) -> ProgramOutcome { // Tiles occupied by station pieces, for entrance/exit placement. let mut station_tiles: Vec<(i32, i32)> = Vec::new(); + outcome.trace.push(TracePoint { + piece: "start".into(), + chain: false, + x: cursor.x / 32, + y: cursor.y / 32, + z: cursor.z, + dir: cursor.direction, + }); + for (index, piece) in program.pieces.iter().enumerate() { let (piece_ref, chain) = piece.parts(); let track_type = match resolve(piece_ref) { @@ -222,6 +246,14 @@ pub fn run(json: &str) -> ProgramOutcome { outcome.pieces_placed += 1; outcome.total_cost += cost; outcome.placed_types.push(track_type); + outcome.trace.push(TracePoint { + piece: describe(track_type), + chain, + x: cursor.x / 32, + y: cursor.y / 32, + z: cursor.z, + dir: cursor.direction, + }); // TrackElemType 1-3 are the station pieces. if (1..=3).contains(&track_type) { station_tiles.push(piece_tile); diff --git a/src/openrct2/command_line/EvalCommands.cpp b/src/openrct2/command_line/EvalCommands.cpp index a4e4d97f8067..29ae8cc0f05a 100644 --- a/src/openrct2/command_line/EvalCommands.cpp +++ b/src/openrct2/command_line/EvalCommands.cpp @@ -39,6 +39,7 @@ namespace OpenRCT2 static u8string _renderLibraryDir{}; static bool _captureAllRotations = false; static bool _captureXray = false; + static bool _noGraphics = false; // clang-format off static constexpr CommandLineOptionDefinition kEvalOptions[] @@ -55,6 +56,7 @@ namespace OpenRCT2 { CMDLINE_TYPE_STRING, &_renderLibraryDir, kNAC, "render-library", "render a preview PNG of every stock track design into this directory and exit" }, { CMDLINE_TYPE_SWITCH, &_captureAllRotations, kNAC, "capture-all-rotations", "with --capture, also write the other three view rotations as -r1/-r2/-r3.png" }, { CMDLINE_TYPE_SWITCH, &_captureXray, kNAC, "capture-xray", "with --capture, also write a see-through verification view (terrain and supports hidden, every placed piece visible) as -x.png" }, + { CMDLINE_TYPE_SWITCH, &_noGraphics, kNAC, "no-graphics", "skip loading sprite data: no RCT2 assets required, but screenshots and library previews are unavailable" }, kOptionTableEnd }; @@ -90,9 +92,22 @@ namespace OpenRCT2 gCustomRCT2DataPath = Path::GetAbsolute(_evalRCT2DataPath); } - // Headless, but keep graphics data loaded (gOpenRCT2NoGraphics stays - // false) so CaptureImage can render screenshots of the result. + // Headless, but by default keep graphics data loaded (gOpenRCT2NoGraphics + // stays false) so CaptureImage can render screenshots of the result. + // --no-graphics drops that: the whole scoring path (park load, placement, + // testing, ratings) works without sprite data, so no RCT2 assets are + // needed — only anything that renders pixels is off the table. gOpenRCT2Headless = true; + if (_noGraphics) + { + if (!_capturePath.empty() || _captureAllRotations || _captureXray || !_renderLibraryDir.empty()) + { + Console::Error::WriteLine( + "--no-graphics cannot render: remove --capture/--capture-all-rotations/--capture-xray/--render-library"); + return ExitCode::fail; + } + gOpenRCT2NoGraphics = true; + } std::unique_ptr context(CreateContext()); if (!context->Initialise()) diff --git a/src/openrct2/rustbridge/RustBridge.cpp b/src/openrct2/rustbridge/RustBridge.cpp index 854d8fb43d6b..102bacd0fab4 100644 --- a/src/openrct2/rustbridge/RustBridge.cpp +++ b/src/openrct2/rustbridge/RustBridge.cpp @@ -666,12 +666,22 @@ uint16_t orct2_host_track_mirror(uint16_t track_type) return mirror == TrackElemType::none ? track_type : static_cast(mirror); } +bool orct2_host_graphics_available(void) +{ + return !gOpenRCT2NoGraphics; +} + bool orct2_host_capture(const char* path, int32_t zoom, uint8_t rotation, bool fit_track, bool xray) { if (path == nullptr) { return false; } + if (gOpenRCT2NoGraphics) + { + LOG_ERROR("screenshot capture unavailable: running with --no-graphics (no sprite data loaded)"); + return false; + } try { // CaptureImage refuses paths outside the screenshot directory, so diff --git a/src/openrct2/rustbridge/orct2_agent.h b/src/openrct2/rustbridge/orct2_agent.h index 956d13235bbb..5eb65609b926 100644 --- a/src/openrct2/rustbridge/orct2_agent.h +++ b/src/openrct2/rustbridge/orct2_agent.h @@ -63,6 +63,11 @@ typedef struct Orct2RideDetail { int16_t nausea; int32_t max_speed; int32_t average_speed; + /** + * Raw total length summed over stations, as 16.16 fixed-point metres + * (`Ride::getTotalLength()`). Divide by 2^16 for the human-readable metres + * the game shows; see `report::ride_length_metres`. + */ int32_t ride_length; int16_t max_positive_g; int16_t max_negative_g; @@ -195,6 +200,8 @@ extern bool orct2_host_ride_set_status(uint16_t ride_id, extern bool orct2_host_ride_detail(uint16_t ride_id, struct Orct2RideDetail *out); +extern bool orct2_host_graphics_available(void); + extern bool orct2_host_capture(const char *path, int32_t zoom, uint8_t rotation,