diff --git a/.agents/skills/shellbench-research-runbook/references/runbook.md b/.agents/skills/shellbench-research-runbook/references/runbook.md index dd8a1f0..4fd7452 100644 --- a/.agents/skills/shellbench-research-runbook/references/runbook.md +++ b/.agents/skills/shellbench-research-runbook/references/runbook.md @@ -75,6 +75,13 @@ The code source of truth is `scripts/native_eval/models.py` and | Node | `22.23.1` | | LiteLLM | `1.93.0` | +Fleet bootstrap installs only the harness assigned to each lease, alongside +shared Docker, Node, uv, and LiteLLM dependencies. The toolchain manifest records +the selected `harness` and its installed version; unused harness keys are absent. +For manual bootstrap, pass exactly one harness: +`bash scripts/native_eval/bootstrap_beast.sh openclaw` (or `codex`, `claude-code`, +`hermes`). Omitting the selection is an error. + Before every campaign: 1. compare this table with the code pins diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e24f1d..17106b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Fixed +- Install only the assigned native harness on each fleet lease and record only its version in the toolchain manifest (#65). - Refresh Python runtime, MLflow, lint, and HF mirror dependencies, including websockets 17 with a real gateway socket regression test, while preserving Python 3.11 NumPy support. - Record and enforce GPT reasoning effort in native runs, preserve it across reruns, and stabilize OpenClaw and Hermes trace completion. diff --git a/scripts/native_eval/bootstrap_beast.sh b/scripts/native_eval/bootstrap_beast.sh index 8309721..bca2e2b 100755 --- a/scripts/native_eval/bootstrap_beast.sh +++ b/scripts/native_eval/bootstrap_beast.sh @@ -1,6 +1,12 @@ #!/usr/bin/env bash set -euo pipefail +HARNESS="${1:-}" +if [[ $# -ne 1 ]] || [[ ! "$HARNESS" =~ ^(openclaw|codex|claude-code|hermes)$ ]]; then + echo "Usage: $0 {openclaw|codex|claude-code|hermes}" >&2 + exit 2 +fi + TOOLCHAIN_ROOT="${TOOLCHAIN_ROOT:-/opt/shellbench-native}" NODE_VERSION="${NODE_VERSION:-22.23.1}" OPENCLAW_VERSION="${OPENCLAW_VERSION:-2026.7.1-2}" @@ -65,7 +71,7 @@ EOF docker run --rm hello-world >/dev/null } -install_node_tools() { +install_node() { local node_root="$TOOLCHAIN_ROOT/node" if [[ ! -x "$node_root/bin/node" ]] || \ [[ "$("$node_root/bin/node" --version)" != "v$NODE_VERSION" ]]; then @@ -79,14 +85,11 @@ install_node_tools() { fi export PATH="$node_root/bin:$PATH" - rm -rf "$TOOLCHAIN_ROOT/npm-packages" - npm install --prefix "$TOOLCHAIN_ROOT/npm-packages" \ - "openclaw@$OPENCLAW_VERSION" \ - "@openai/codex@$CODEX_VERSION" \ - "@anthropic-ai/claude-code@$CLAUDE_CODE_VERSION" } install_uv() { + export UV_PYTHON_INSTALL_DIR="$TOOLCHAIN_ROOT/uv-python" + export UV_CACHE_DIR="$TOOLCHAIN_ROOT/uv-cache" install -d -m 0755 "$TOOLCHAIN_ROOT/bin" if [[ ! -x "$TOOLCHAIN_ROOT/bin/uv" ]]; then curl -LsSf https://astral.sh/uv/install.sh -o /tmp/install-uv.sh @@ -99,8 +102,6 @@ install_hermes() { export HOME="$TOOLCHAIN_ROOT/home" export HERMES_HOME="$TOOLCHAIN_ROOT/hermes-home" export HERMES_INSTALL_DIR="$TOOLCHAIN_ROOT/hermes-agent" - export UV_PYTHON_INSTALL_DIR="$TOOLCHAIN_ROOT/uv-python" - export UV_CACHE_DIR="$TOOLCHAIN_ROOT/uv-cache" export PATH="$TOOLCHAIN_ROOT/node/bin:$TOOLCHAIN_ROOT/bin:$PATH" install -d -m 0755 "$HOME" "$HERMES_HOME" @@ -124,35 +125,51 @@ install_litellm() { "litellm[proxy]==$LITELLM_VERSION" } +install_harness() { + local package + case "$HARNESS" in + openclaw) package="openclaw@$OPENCLAW_VERSION" ;; + codex) package="@openai/codex@$CODEX_VERSION" ;; + claude-code) package="@anthropic-ai/claude-code@$CLAUDE_CODE_VERSION" ;; + hermes) install_hermes; return ;; + esac + rm -rf "$TOOLCHAIN_ROOT/npm-packages" + npm install --prefix "$TOOLCHAIN_ROOT/npm-packages" "$package" +} + write_manifest() { export PATH="$TOOLCHAIN_ROOT/node/bin:$TOOLCHAIN_ROOT/npm-packages/node_modules/.bin:$TOOLCHAIN_ROOT/home/.local/bin:$PATH" + local harness_version + case "$HARNESS" in + openclaw) harness_version=$(openclaw --version) ;; + codex) harness_version=$(codex --version) ;; + claude-code) harness_version=$(claude --version) ;; + hermes) harness_version=$(hermes version) ;; + esac jq -n \ --arg created_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ --arg node "$(node --version)" \ - --arg openclaw "$(openclaw --version | head -1)" \ - --arg codex "$(codex --version | head -1)" \ - --arg claude_code "$(claude --version | head -1)" \ - --arg hermes "$(hermes version | head -1)" \ + --arg harness "$HARNESS" \ + --arg harness_key "${HARNESS//-/_}" \ + --arg harness_version "${harness_version%%$'\n'*}" \ --arg litellm "$("$TOOLCHAIN_ROOT/litellm-venv/bin/python" -c 'from importlib.metadata import version; print(version("litellm"))')" \ --arg hermes_commit "$HERMES_COMMIT" \ '{ created_at_utc: $created_at, node: $node, - openclaw: $openclaw, - codex: $codex, - claude_code: $claude_code, - hermes: $hermes, - hermes_commit: $hermes_commit, + harness: $harness, + ($harness_key): $harness_version, litellm: $litellm - }' > "$TOOLCHAIN_ROOT/manifest.json" + } + (if $harness == "hermes" then {hermes_commit: $hermes_commit} else {} end)' \ + > "$TOOLCHAIN_ROOT/manifest.json" chmod -R a+rX "$TOOLCHAIN_ROOT" } install_base_packages install_docker -install_node_tools +install_node install_uv -install_hermes +install_harness install_litellm write_manifest diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index 433d775..a8e73f0 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -564,7 +564,7 @@ def _execute_entry(self, run_label: str) -> bool: if self._local_artifacts(run.run_label): raise FleetError("local artifacts exist but the remote run state is missing") if not entry.get("bootstrapped_at_utc"): - self._hydrate_lease(lease) + self._hydrate_lease(lease, run) self._dispatch(lease, run) elif remote_state == "stale": raise FleetError("remote run state is stale and cannot be overwritten") @@ -823,7 +823,7 @@ def _detect_region(self, lease: Lease) -> Lease: region=region, ) - def _hydrate_lease(self, lease: Lease) -> None: + def _hydrate_lease(self, lease: Lease, run: RunSpec) -> None: assert self.runner_archive is not None remote_runner_archive = f"/tmp/{lease.slug}-runner.tar.gz" remote_task_archive = f"/tmp/{lease.slug}-tasks.tar.gz" @@ -856,6 +856,7 @@ def _hydrate_lease(self, lease: Lease) -> None: runner_commit=$5 runner_sha256=$6 task_sha256=$7 +harness=$8 printf '%s %s\n' "$runner_sha256" "$runner_archive" | sha256sum -c - printf '%s %s\n' "$task_sha256" "$task_archive" | sha256sum -c - rm -rf "$root/runner.new" "$root/public-tasks.new" @@ -868,7 +869,7 @@ def _hydrate_lease(self, lease: Lease) -> None: mv "$root/public-tasks.new" "$root/public-tasks" printf '%s\n' "$runner_commit" > "$root/runner.commit" rm -f "$runner_archive" "$task_archive" "$source_env" -bash "$root/runner/scripts/native_eval/bootstrap_beast.sh" +bash "$root/runner/scripts/native_eval/bootstrap_beast.sh" "$harness" """ self._checked( self._ssh_command( @@ -885,6 +886,7 @@ def _hydrate_lease(self, lease: Lease) -> None: self.runner_commit, self.runner_archive_sha256, self.task_archive_sha256, + run.harness, ], ), capture_output=False, diff --git a/tests/test_native_eval_bootstrap.py b/tests/test_native_eval_bootstrap.py new file mode 100644 index 0000000..e8c6d47 --- /dev/null +++ b/tests/test_native_eval_bootstrap.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + + +BOOTSTRAP = Path(__file__).resolve().parents[1] / "scripts/native_eval/bootstrap_beast.sh" + + +@pytest.fixture +def bootstrap_env(tmp_path: Path) -> dict[str, str]: + if shutil.which("jq") is None: + pytest.skip("bootstrap manifest checks require jq") + root = tmp_path / "toolchain" + for relative, output in ( + ("node/bin/node", "v22.23.1"), + ("litellm-venv/bin/python", "1.93.0"), + ): + executable = root / relative + executable.parent.mkdir(parents=True, exist_ok=True) + executable.write_text(f"#!/bin/sh\nprintf '%s\\n' '{output}'\n") + executable.chmod(0o755) + uv = root / "bin/uv" + uv.parent.mkdir(parents=True) + uv.write_text("""#!/bin/sh +test "$UV_PYTHON_INSTALL_DIR" = "$TOOLCHAIN_ROOT/uv-python" || exit 1 +test "$UV_CACHE_DIR" = "$TOOLCHAIN_ROOT/uv-cache" || exit 1 +printf 'uv %s\\n' "$*" >> "$BOOTSTRAP_CALLS" +""") + uv.chmod(0o755) + stubs = tmp_path / "commands.sh" + stubs.write_text(""" +record() { printf '%s\\n' "$*" >> "$BOOTSTRAP_CALLS"; } +id() { printf '0\\n'; } +apt-get() { record "apt-get $*"; } +rm() { :; } +chmod() { :; } +docker() { printf 'docker-test\\n'; } +curl() { record "curl $*"; } +bash() { record "hermes-install $*"; } +npm() { record "npm $*"; } +openclaw() { record 'probe openclaw'; printf 'openclaw-test\\n'; } +codex() { record 'probe codex'; printf 'codex-test\\n'; } +claude() { record 'probe claude-code'; printf 'claude-code-test\\n'; } +hermes() { record 'probe hermes'; printf 'hermes-test\\n'; } +""") + return { + **os.environ, + "BASH_ENV": str(stubs), + "TOOLCHAIN_ROOT": str(root), + "BOOTSTRAP_CALLS": str(tmp_path / "calls.txt"), + "NODE_VERSION": "22.23.1", + "OPENCLAW_VERSION": "2026.7.1-2", + "CODEX_VERSION": "0.145.0", + "CLAUDE_CODE_VERSION": "2.1.220", + "HERMES_COMMIT": "test-hermes-commit", + } + + +@pytest.mark.parametrize("harness,package", ( + ("openclaw", "openclaw@2026.7.1-2"), + ("codex", "@openai/codex@0.145.0"), + ("claude-code", "@anthropic-ai/claude-code@2.1.220"), + ("hermes", None), +)) +def test_bootstrap_installs_and_probes_only_selected_harness( + bootstrap_env: dict[str, str], harness: str, package: str | None, +) -> None: + result = subprocess.run( + ["/bin/bash", str(BOOTSTRAP), harness], env=bootstrap_env, + capture_output=True, text=True, check=True, + ) + calls = Path(bootstrap_env["BOOTSTRAP_CALLS"]).read_text().splitlines() + npm_calls = [line for line in calls if line.startswith("npm ")] + if package: + assert len(npm_calls) == 1 + assert npm_calls[0].split()[4:] == [package] + else: + assert npm_calls == [] + assert sum(line.startswith("hermes-install ") for line in calls) == (harness == "hermes") + assert [line for line in calls if line.startswith("probe ")] == [f"probe {harness}"] + assert sum(line.startswith("uv ") for line in calls) == 2 + manifest = json.loads(result.stdout[result.stdout.index("{"):]) + key = harness.replace("-", "_") + assert manifest[key] == f"{harness}-test" + assert manifest["harness"] == harness + assert manifest["node"] == "v22.23.1" + assert manifest["litellm"] == "1.93.0" + assert set(manifest) == { + "created_at_utc", "node", "litellm", "harness", key, + } | ({"hermes_commit"} if harness == "hermes" else set()) + + +@pytest.mark.parametrize("arguments", [[], ["unknown"], ["openclaw", "codex"]]) +def test_bootstrap_rejects_invalid_selection_before_installing( + bootstrap_env: dict[str, str], arguments: list[str], +) -> None: + result = subprocess.run( + ["/bin/bash", str(BOOTSTRAP), *arguments], env=bootstrap_env, + capture_output=True, text=True, + ) + assert result.returncode == 2 + assert "Usage:" in result.stderr + assert not Path(bootstrap_env["BOOTSTRAP_CALLS"]).exists() diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index b9855ff..82b285d 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -564,6 +564,25 @@ def test_controller_runs_bounded_wave_and_stops_after_verified_export( assert "TOPSECRET" not in "\n".join(" ".join(command) for command in executor.commands) +@pytest.mark.parametrize("harness", ["openclaw", "codex", "claude-code", "hermes"]) +def test_controller_hydrates_selected_harness(tmp_path: Path, harness: str) -> None: + label = f"{harness}-gpt55-full-2-r1-20260727" + entry = {**_planned(_run_spec(label)), "harness": harness} + run_index = tmp_path / "manifests" / "run_index.json" + _write_index(run_index, [entry]) + config = _config(tmp_path, run_index) + executor = FakeExecutor(config.local_root, expected_counts={label: 2}) + + assert FleetController(config, executor=executor).run() == 0 + + hydration = next( + shlex.split(command[-1]) for command in executor.commands + if command[0] == "ssh" and "fleet-hydrate" in command[-1] + ) + assert hydration[-1] == harness + assert 'bootstrap_beast.sh" "$harness"' in hydration[2] + + def test_controller_dispatches_only_matching_parity_scope(tmp_path: Path) -> None: label = "openclaw-gpt55-full-2-r1-20260727" run_index = tmp_path / "manifests" / "run_index.json"