fix(native-eval): bootstrap only the assigned harness

.agents/skills/shellbench-research-runbook/references/runbook.md

@@ -75,6 +75,13 @@ The code source of truth is `scripts/native_eval/models.py` and
 | Node | `22.23.1` |
 | LiteLLM | `1.93.0` |
 
+Fleet bootstrap installs only the harness assigned to each lease, alongside
+shared Docker, Node, uv, and LiteLLM dependencies. The toolchain manifest records
+the selected `harness` and its installed version; unused harness keys are absent.
+For manual bootstrap, pass exactly one harness:
+`bash scripts/native_eval/bootstrap_beast.sh openclaw` (or `codex`, `claude-code`,
+`hermes`). Omitting the selection is an error.
+
 Before every campaign:
 
 1. compare this table with the code pins

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Fixed
 
+- Install only the assigned native harness on each fleet lease and record only its version in the toolchain manifest (#65).
 - Refresh Python runtime, MLflow, lint, and HF mirror dependencies, including websockets 17 with a real gateway socket regression test, while preserving Python 3.11 NumPy support.
 - Record and enforce GPT reasoning effort in native runs, preserve it across
   reruns, and stabilize OpenClaw and Hermes trace completion.

scripts/native_eval/bootstrap_beast.sh

@@ -1,6 +1,12 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+HARNESS="${1:-}"
+if [[ $# -ne 1 ]] || [[ ! "$HARNESS" =~ ^(openclaw|codex|claude-code|hermes)$ ]]; then
+  echo "Usage: $0 {openclaw|codex|claude-code|hermes}" >&2
+  exit 2
+fi
+
 TOOLCHAIN_ROOT="${TOOLCHAIN_ROOT:-/opt/shellbench-native}"
 NODE_VERSION="${NODE_VERSION:-22.23.1}"
 OPENCLAW_VERSION="${OPENCLAW_VERSION:-2026.7.1-2}"
@@ -65,7 +71,7 @@ EOF
   docker run --rm hello-world >/dev/null
 }
 
-install_node_tools() {
+install_node() {
   local node_root="$TOOLCHAIN_ROOT/node"
   if [[ ! -x "$node_root/bin/node" ]] || \
     [[ "$("$node_root/bin/node" --version)" != "v$NODE_VERSION" ]]; then
@@ -79,14 +85,11 @@ install_node_tools() {
   fi
 
   export PATH="$node_root/bin:$PATH"
-  rm -rf "$TOOLCHAIN_ROOT/npm-packages"
-  npm install --prefix "$TOOLCHAIN_ROOT/npm-packages" \
-    "openclaw@$OPENCLAW_VERSION" \
-    "@openai/codex@$CODEX_VERSION" \
-    "@anthropic-ai/claude-code@$CLAUDE_CODE_VERSION"
 }
 
 install_uv() {
+  export UV_PYTHON_INSTALL_DIR="$TOOLCHAIN_ROOT/uv-python"
+  export UV_CACHE_DIR="$TOOLCHAIN_ROOT/uv-cache"
   install -d -m 0755 "$TOOLCHAIN_ROOT/bin"
   if [[ ! -x "$TOOLCHAIN_ROOT/bin/uv" ]]; then
     curl -LsSf https://astral.sh/uv/install.sh -o /tmp/install-uv.sh
@@ -99,8 +102,6 @@ install_hermes() {
   export HOME="$TOOLCHAIN_ROOT/home"
   export HERMES_HOME="$TOOLCHAIN_ROOT/hermes-home"
   export HERMES_INSTALL_DIR="$TOOLCHAIN_ROOT/hermes-agent"
-  export UV_PYTHON_INSTALL_DIR="$TOOLCHAIN_ROOT/uv-python"
-  export UV_CACHE_DIR="$TOOLCHAIN_ROOT/uv-cache"
   export PATH="$TOOLCHAIN_ROOT/node/bin:$TOOLCHAIN_ROOT/bin:$PATH"
   install -d -m 0755 "$HOME" "$HERMES_HOME"
 
@@ -124,35 +125,51 @@ install_litellm() {
     "litellm[proxy]==$LITELLM_VERSION"
 }
 
+install_harness() {
+  local package
+  case "$HARNESS" in
+    openclaw) package="openclaw@$OPENCLAW_VERSION" ;;
+    codex) package="@openai/codex@$CODEX_VERSION" ;;
+    claude-code) package="@anthropic-ai/claude-code@$CLAUDE_CODE_VERSION" ;;
+    hermes) install_hermes; return ;;
+  esac
+  rm -rf "$TOOLCHAIN_ROOT/npm-packages"
+  npm install --prefix "$TOOLCHAIN_ROOT/npm-packages" "$package"
+}
+
 write_manifest() {
   export PATH="$TOOLCHAIN_ROOT/node/bin:$TOOLCHAIN_ROOT/npm-packages/node_modules/.bin:$TOOLCHAIN_ROOT/home/.local/bin:$PATH"
+  local harness_version
+  case "$HARNESS" in
+    openclaw) harness_version=$(openclaw --version) ;;
+    codex) harness_version=$(codex --version) ;;
+    claude-code) harness_version=$(claude --version) ;;
+    hermes) harness_version=$(hermes version) ;;
+  esac
   jq -n \
     --arg created_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
     --arg node "$(node --version)" \
-    --arg openclaw "$(openclaw --version | head -1)" \
-    --arg codex "$(codex --version | head -1)" \
-    --arg claude_code "$(claude --version | head -1)" \
-    --arg hermes "$(hermes version | head -1)" \
+    --arg harness "$HARNESS" \
+    --arg harness_key "${HARNESS//-/_}" \
+    --arg harness_version "${harness_version%%$'\n'*}" \
     --arg litellm "$("$TOOLCHAIN_ROOT/litellm-venv/bin/python" -c 'from importlib.metadata import version; print(version("litellm"))')" \
     --arg hermes_commit "$HERMES_COMMIT" \
     '{
       created_at_utc: $created_at,
       node: $node,
-      openclaw: $openclaw,
-      codex: $codex,
-      claude_code: $claude_code,
-      hermes: $hermes,
-      hermes_commit: $hermes_commit,
+      harness: $harness,
+      ($harness_key): $harness_version,
       litellm: $litellm
-    }' > "$TOOLCHAIN_ROOT/manifest.json"
+    } + (if $harness == "hermes" then {hermes_commit: $hermes_commit} else {} end)' \
+    > "$TOOLCHAIN_ROOT/manifest.json"
   chmod -R a+rX "$TOOLCHAIN_ROOT"
 }
 
 install_base_packages
 install_docker
-install_node_tools
+install_node
 install_uv
-install_hermes
+install_harness
 install_litellm
 write_manifest
 

scripts/native_eval/fleet.py

@@ -564,7 +564,7 @@ def _execute_entry(self, run_label: str) -> bool:
                 if self._local_artifacts(run.run_label):
                     raise FleetError("local artifacts exist but the remote run state is missing")
                 if not entry.get("bootstrapped_at_utc"):
-                    self._hydrate_lease(lease)
+                    self._hydrate_lease(lease, run)
                 self._dispatch(lease, run)
             elif remote_state == "stale":
                 raise FleetError("remote run state is stale and cannot be overwritten")
@@ -823,7 +823,7 @@ def _detect_region(self, lease: Lease) -> Lease:
             region=region,
         )
 
-    def _hydrate_lease(self, lease: Lease) -> None:
+    def _hydrate_lease(self, lease: Lease, run: RunSpec) -> None:
         assert self.runner_archive is not None
         remote_runner_archive = f"/tmp/{lease.slug}-runner.tar.gz"
         remote_task_archive = f"/tmp/{lease.slug}-tasks.tar.gz"
@@ -856,6 +856,7 @@ def _hydrate_lease(self, lease: Lease) -> None:
 runner_commit=$5
 runner_sha256=$6
 task_sha256=$7
+harness=$8
 printf '%s  %s\n' "$runner_sha256" "$runner_archive" | sha256sum -c -
 printf '%s  %s\n' "$task_sha256" "$task_archive" | sha256sum -c -
 rm -rf "$root/runner.new" "$root/public-tasks.new"
@@ -868,7 +869,7 @@ def _hydrate_lease(self, lease: Lease) -> None:
 mv "$root/public-tasks.new" "$root/public-tasks"
 printf '%s\n' "$runner_commit" > "$root/runner.commit"
 rm -f "$runner_archive" "$task_archive" "$source_env"
-bash "$root/runner/scripts/native_eval/bootstrap_beast.sh"
+bash "$root/runner/scripts/native_eval/bootstrap_beast.sh" "$harness"
 """
         self._checked(
             self._ssh_command(
@@ -885,6 +886,7 @@ def _hydrate_lease(self, lease: Lease) -> None:
                     self.runner_commit,
                     self.runner_archive_sha256,
                     self.task_archive_sha256,
+                    run.harness,
                 ],
             ),
             capture_output=False,

tests/test_native_eval_bootstrap.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+
+BOOTSTRAP = Path(__file__).resolve().parents[1] / "scripts/native_eval/bootstrap_beast.sh"
+
+
+@pytest.fixture
+def bootstrap_env(tmp_path: Path) -> dict[str, str]:
+    if shutil.which("jq") is None:
+        pytest.skip("bootstrap manifest checks require jq")
+    root = tmp_path / "toolchain"
+    for relative, output in (
+        ("node/bin/node", "v22.23.1"),
+        ("litellm-venv/bin/python", "1.93.0"),
+    ):
+        executable = root / relative
+        executable.parent.mkdir(parents=True, exist_ok=True)
+        executable.write_text(f"#!/bin/sh\nprintf '%s\\n' '{output}'\n")
+        executable.chmod(0o755)
+    uv = root / "bin/uv"
+    uv.parent.mkdir(parents=True)
+    uv.write_text("""#!/bin/sh
+test "$UV_PYTHON_INSTALL_DIR" = "$TOOLCHAIN_ROOT/uv-python" || exit 1
+test "$UV_CACHE_DIR" = "$TOOLCHAIN_ROOT/uv-cache" || exit 1
+printf 'uv %s\\n' "$*" >> "$BOOTSTRAP_CALLS"
+""")
+    uv.chmod(0o755)
+    stubs = tmp_path / "commands.sh"
+    stubs.write_text("""
+record() { printf '%s\\n' "$*" >> "$BOOTSTRAP_CALLS"; }
+id() { printf '0\\n'; }
+apt-get() { record "apt-get $*"; }
+rm() { :; }
+chmod() { :; }
+docker() { printf 'docker-test\\n'; }
+curl() { record "curl $*"; }
+bash() { record "hermes-install $*"; }
+npm() { record "npm $*"; }
+openclaw() { record 'probe openclaw'; printf 'openclaw-test\\n'; }
+codex() { record 'probe codex'; printf 'codex-test\\n'; }
+claude() { record 'probe claude-code'; printf 'claude-code-test\\n'; }
+hermes() { record 'probe hermes'; printf 'hermes-test\\n'; }
+""")
+    return {
+        **os.environ,
+        "BASH_ENV": str(stubs),
+        "TOOLCHAIN_ROOT": str(root),
+        "BOOTSTRAP_CALLS": str(tmp_path / "calls.txt"),
+        "NODE_VERSION": "22.23.1",
+        "OPENCLAW_VERSION": "2026.7.1-2",
+        "CODEX_VERSION": "0.145.0",
+        "CLAUDE_CODE_VERSION": "2.1.220",
+        "HERMES_COMMIT": "test-hermes-commit",
+    }
+
+
+@pytest.mark.parametrize("harness,package", (
+    ("openclaw", "openclaw@2026.7.1-2"),
+    ("codex", "@openai/codex@0.145.0"),
+    ("claude-code", "@anthropic-ai/claude-code@2.1.220"),
+    ("hermes", None),
+))
+def test_bootstrap_installs_and_probes_only_selected_harness(
+    bootstrap_env: dict[str, str], harness: str, package: str | None,
+) -> None:
+    result = subprocess.run(
+        ["/bin/bash", str(BOOTSTRAP), harness], env=bootstrap_env,
+        capture_output=True, text=True, check=True,
+    )
+    calls = Path(bootstrap_env["BOOTSTRAP_CALLS"]).read_text().splitlines()
+    npm_calls = [line for line in calls if line.startswith("npm ")]
+    if package:
+        assert len(npm_calls) == 1
+        assert npm_calls[0].split()[4:] == [package]
+    else:
+        assert npm_calls == []
+    assert sum(line.startswith("hermes-install ") for line in calls) == (harness == "hermes")
+    assert [line for line in calls if line.startswith("probe ")] == [f"probe {harness}"]
+    assert sum(line.startswith("uv ") for line in calls) == 2
+    manifest = json.loads(result.stdout[result.stdout.index("{"):])
+    key = harness.replace("-", "_")
+    assert manifest[key] == f"{harness}-test"
+    assert manifest["harness"] == harness
+    assert manifest["node"] == "v22.23.1"
+    assert manifest["litellm"] == "1.93.0"
+    assert set(manifest) == {
+        "created_at_utc", "node", "litellm", "harness", key,
+    } | ({"hermes_commit"} if harness == "hermes" else set())
+
+
+@pytest.mark.parametrize("arguments", [[], ["unknown"], ["openclaw", "codex"]])
+def test_bootstrap_rejects_invalid_selection_before_installing(
+    bootstrap_env: dict[str, str], arguments: list[str],
+) -> None:
+    result = subprocess.run(
+        ["/bin/bash", str(BOOTSTRAP), *arguments], env=bootstrap_env,
+        capture_output=True, text=True,
+    )
+    assert result.returncode == 2
+    assert "Usage:" in result.stderr
+    assert not Path(bootstrap_env["BOOTSTRAP_CALLS"]).exists()

tests/test_native_eval_fleet.py

@@ -564,6 +564,25 @@ def test_controller_runs_bounded_wave_and_stops_after_verified_export(
     assert "TOPSECRET" not in "\n".join(" ".join(command) for command in executor.commands)
 
 
+@pytest.mark.parametrize("harness", ["openclaw", "codex", "claude-code", "hermes"])
+def test_controller_hydrates_selected_harness(tmp_path: Path, harness: str) -> None:
+    label = f"{harness}-gpt55-full-2-r1-20260727"
+    entry = {**_planned(_run_spec(label)), "harness": harness}
+    run_index = tmp_path / "manifests" / "run_index.json"
+    _write_index(run_index, [entry])
+    config = _config(tmp_path, run_index)
+    executor = FakeExecutor(config.local_root, expected_counts={label: 2})
+
+    assert FleetController(config, executor=executor).run() == 0
+
+    hydration = next(
+        shlex.split(command[-1]) for command in executor.commands
+        if command[0] == "ssh" and "fleet-hydrate" in command[-1]
+    )
+    assert hydration[-1] == harness
+    assert 'bootstrap_beast.sh" "$harness"' in hydration[2]
+
+
 def test_controller_dispatches_only_matching_parity_scope(tmp_path: Path) -> None:
     label = "openclaw-gpt55-full-2-r1-20260727"
     run_index = tmp_path / "manifests" / "run_index.json"