diff --git a/AGENTS.md b/AGENTS.md index 578e9995..47134cb5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -166,6 +166,7 @@ simulated_user_model_config: datasets/model_configs/gemini_2.5_pro_model.yaml runners: agent_runners: 10 + prompt_timeout_seconds: 300 scorers: trajectory_matcher: {} @@ -183,11 +184,12 @@ reporting: ``` ### 2. Model Configuration (e.g., `gemini_cli_model.yaml`) -Specifies tested version, model ID, and environment variables. +Specifies tested version, model ID, environment variables, and prompt timeout. ```yaml gemini_cli_version: "@google/gemini-cli@0.36.0" generator: gemini_cli +prompt_timeout_seconds: 180 env: GOOGLE_CLOUD_PROJECT: "my-evaluation-project" GOOGLE_CLOUD_LOCATION: "us-central1" @@ -211,7 +213,8 @@ Contains the test cases. "conversation_plan": "Ensure the agent accurately calls list_instances. Verify the output is returned correctly.", "expected_trajectory": ["cloud-sql__list_instances"], "env": { "GOOGLE_CLOUD_PROJECT": "my-evaluation-project" }, - "max_turns": 4 + "max_turns": 4, + "prompt_timeout_seconds": 120 } ] } diff --git a/docs/agy_cli_agent_testing.md b/docs/agy_cli_agent_testing.md index f34cfade..97aa044a 100644 --- a/docs/agy_cli_agent_testing.md +++ b/docs/agy_cli_agent_testing.md @@ -167,6 +167,7 @@ Specifies the generator, model label, execution timeouts, and environment: | `generator` | Yes | Must be `agy_cli` | | `model` | Optional | Model label (e.g. `"Gemini 3.1 Pro (Low)"` or `"Gemini 3.5 Flash (Medium)"`). Omit to use agy's default. | | `timeout` | Optional | CLI turn timeout string (e.g. `"20m"`, passed to `--print-timeout`). Defaults to 5m. | +| `prompt_timeout_seconds` | Optional | Timeout limit in seconds for each CLI prompt turn (e.g., `180`). | | `env` | Optional | Environment block. Set `GOOGLE_CLOUD_PROJECT` (see below); `GOOGLE_CLOUD_LOCATION` defaults to `global`. | | `setup` | Optional | Tool setup block for `mcp_servers`, `skills`, or `fake_mcp_servers`. | diff --git a/docs/claude_code_agent_testing.md b/docs/claude_code_agent_testing.md index ea20ec1e..a0e78115 100644 --- a/docs/claude_code_agent_testing.md +++ b/docs/claude_code_agent_testing.md @@ -206,6 +206,7 @@ The model config defines the Claude Code CLI version, model, auth, environment, | `vertex_project_id` | If `use_vertex` | GCP project for Vertex AI | | `vertex_region` | If `use_vertex` | Vertex region (e.g., `us-east5`) | | `env` | Optional | Environment variables passed to the CLI process | +| `prompt_timeout_seconds` | Optional | Timeout limit in seconds for each CLI prompt turn (e.g., `180`) | | `setup.mcp_servers` | Optional | MCP server configurations (see [MCP Servers](#mcp-servers)) | | `allowed_tools` | Optional | List of tool names to allow (e.g., `["Bash", "mcp__cloud-sql"]`) | diff --git a/docs/codex_cli_agent_testing.md b/docs/codex_cli_agent_testing.md index c634e4e9..755e9d71 100644 --- a/docs/codex_cli_agent_testing.md +++ b/docs/codex_cli_agent_testing.md @@ -213,6 +213,7 @@ The model config defines the Codex CLI version, model, auth, sandbox/approval po | `profile` | Optional | Codex profile name (forwarded as `--profile `) | | `json_flag` | Optional | `"--json"` (default, newer Codex versions) or `"--experimental-json"` (older versions). Codex requires NDJSON for the eval pipeline to extract tool calls and tokens. | | `pricing` | Optional | Per-model rates used to compute `cost_usd` per turn. See [Pricing & Cost Tracking](#pricing--cost-tracking). | +| `prompt_timeout_seconds` | Optional | Timeout limit in seconds for each CLI prompt turn (e.g., `180`) | | `env` | Optional | Environment variables passed to the CLI process (e.g., `GOOGLE_CLOUD_PROJECT` for Cloud SQL MCP) | | `setup.mcp_servers` | Optional | MCP server configurations (see [MCP Servers](#mcp-servers)) | | `setup.config` | Optional | Free-form key/value pairs written to the top of `~/.codex/config.toml`. Merged on top of the default `forced_login_method = "api"`. | diff --git a/docs/configs/model-config.md b/docs/configs/model-config.md index 3a70f0e1..b675f513 100644 --- a/docs/configs/model-config.md +++ b/docs/configs/model-config.md @@ -15,6 +15,7 @@ These settings are passed to all generators, regardless of the specific engine u | `max_tokens` | Optional | N/A | Specifies the maximum number of tokens the model can generate in a single output. | | `execs_per_minute` | Optional | `60` | Sets the maximum number of executions allowed per minute. If not provided, it defaults to `60`. This helps throttle the rate of query generation. | | `max_attempts` | Optional | `3` | Specifies the maximum number of attempts for query generation in case of failures. Defaults to `3` if not provided. | +| `prompt_timeout_seconds` | Optional | N/A | Timeout limit in seconds for each CLI prompt execution turn (e.g. `180`). Overrides run config `runners.prompt_timeout_seconds`. | ## GCP Specific Configuration diff --git a/docs/configs/run-config.md b/docs/configs/run-config.md index 0c5fcfd6..fbb303cf 100644 --- a/docs/configs/run-config.md +++ b/docs/configs/run-config.md @@ -19,6 +19,7 @@ This section defines the primary resources used during evaluation, including the | `num_trials` | Optional | Number of trials to run for each prompt. | | `scenarios` | Optional | A list of specific scenario IDs to run (only applies to scenario-based agentic datasets like `gemini-cli-format` or `cortado-format`). Defaults to empty (runs all scenarios). | | `scenario_pattern` | Optional | A glob pattern of scenario IDs to run (only applies to scenario-based agentic datasets). Defaults to None (runs all scenarios). | +| `runners` | Optional | Dictionary configuring concurrency (`agent_runners`, default: 10) and prompt timeouts (`prompt_timeout_seconds`, e.g. `300` seconds per CLI execution turn). | --- ## 2. Prompt and Generation Modules diff --git a/docs/gemini_cli_agent_testing.md b/docs/gemini_cli_agent_testing.md index b3a37991..7fe02244 100644 --- a/docs/gemini_cli_agent_testing.md +++ b/docs/gemini_cli_agent_testing.md @@ -255,6 +255,7 @@ The evalset JSON file defines the test scenarios. Each scenario represents an ag | `max_turns` | Yes | Maximum number of conversation turns before the evaluation stops | | `env` | Optional | Per-scenario environment variables (merged with model config env) | | `kind` | Optional | Category label (e.g., `"tools"`) | +| `prompt_timeout_seconds` | Optional | Timeout limit in seconds for each CLI prompt turn in this scenario. Overrides model config and run config timeout settings. | #### Tool name format diff --git a/evalbench/evaluator/agentevaluator.py b/evalbench/evaluator/agentevaluator.py index f464a83b..3526af18 100644 --- a/evalbench/evaluator/agentevaluator.py +++ b/evalbench/evaluator/agentevaluator.py @@ -141,6 +141,12 @@ def process_scenario( "Declared env file not found in session: %s", src_path ) + prompt_timeout = ( + scenario.get("prompt_timeout_seconds") + or getattr(self.generator, "prompt_timeout_seconds", None) + or self.config.get("runners", {}).get("prompt_timeout_seconds") + ) + session_id = None for turn in range(max_turns): logging.info( @@ -153,6 +159,7 @@ def process_scenario( resume=(turn > 0), session_id=session_id, cwd=resolved_work_dir, + timeout=prompt_timeout, ) try: result = self.generator.safe_generate(cli_cmd) diff --git a/evalbench/generators/models/agent_cli.py b/evalbench/generators/models/agent_cli.py index 41010e6c..2ebb9c8b 100644 --- a/evalbench/generators/models/agent_cli.py +++ b/evalbench/generators/models/agent_cli.py @@ -1,13 +1,17 @@ from abc import abstractmethod +from contextlib import contextmanager +import logging +import os +import shutil +import signal +import subprocess +import threading from mcp import types as mcp_types from . import mcp_client from .generator import QueryGenerator from .tool_naming import canonical_tool_name -import logging -import os -import shutil class AgentCliGenerator(QueryGenerator): @@ -91,7 +95,7 @@ def version(self) -> str: @abstractmethod def create_command( self, cli: str, prompt: str, env: dict = None, resume: bool = False, - session_id: str = None, cwd: str = None, + session_id: str = None, cwd: str = None, timeout: float | int = None, ): raise NotImplementedError("Subclasses must implement this method") @@ -110,3 +114,70 @@ def extract_tools(self, stdout: str) -> list: @abstractmethod def extract_skills(self, stdout: str) -> list: raise NotImplementedError("Subclasses must implement this method") + + +def parse_timeout_seconds(timeout: float | int | str | None) -> float | None: + if timeout is None: + return None + if isinstance(timeout, (int, float)): + return float(timeout) + if isinstance(timeout, str): + s = timeout.strip() + unit = 1.0 + if s.lower().endswith("s"): + s = s[:-1] + elif s.lower().endswith("m"): + s = s[:-1] + unit = 60.0 + elif s.lower().endswith("h"): + s = s[:-1] + unit = 3600.0 + + try: + return float(s) * unit + except ValueError as e: + logging.warning("Failed to parse timeout string %r: %s", timeout, e) + return None + + +def _kill_process_group(proc: subprocess.Popen): + """Terminates proc's process group with SIGKILL, falling back to proc.kill().""" + try: + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + return + except (ProcessLookupError, OSError) as e: + logging.warning("os.killpg failed for pid %s: %s; trying proc.kill()", proc.pid, e) + except Exception as e: + logging.warning("Unexpected error in os.killpg for pid %s: %s; trying proc.kill()", proc.pid, e) + + try: + proc.kill() + except (ProcessLookupError, OSError) as e: + logging.warning("proc.kill() failed for pid %s: %s", proc.pid, e) + except Exception as e: + logging.warning("Unexpected error in proc.kill() for pid %s: %s", proc.pid, e) + + +@contextmanager +def process_timeout(proc: subprocess.Popen, timeout: float | int | str | None): + """Context manager that sets a timer to kill ``proc``'s process group on timeout. + + Yields a callable ``is_timed_out() -> bool``. + """ + timed_out = False + timer = None + timeout_sec = parse_timeout_seconds(timeout) + if timeout_sec: + def _on_timeout(): + nonlocal timed_out + timed_out = True + _kill_process_group(proc) + + timer = threading.Timer(timeout_sec, _on_timeout) + timer.start() + + try: + yield lambda: timed_out + finally: + if timer: + timer.cancel() diff --git a/evalbench/generators/models/agy_cli.py b/evalbench/generators/models/agy_cli.py index 37eee849..9764d9e7 100644 --- a/evalbench/generators/models/agy_cli.py +++ b/evalbench/generators/models/agy_cli.py @@ -1,4 +1,4 @@ -from .agent_cli import AgentCliGenerator +from .agent_cli import AgentCliGenerator, process_timeout from .tool_naming import canonicalize_agy_tool_name, parse_agy_mcp_tool_call import subprocess import os @@ -35,12 +35,13 @@ def _shred_credential(path: str) -> None: class CLICommand: - def __init__(self, cli, prompt, env=None, resume=False, cwd=None): + def __init__(self, cli, prompt, env=None, resume=False, cwd=None, timeout=None): self.cli = cli self.prompt = prompt self.env = env if env else {} self.resume = resume self.cwd = cwd + self.timeout = timeout class AgyCliGenerator(AgentCliGenerator): @@ -872,17 +873,27 @@ def generate_internal(self, cli_cmd): return self._run_agy_cli(cli_cmd) def _execute_cli_command( - self, command, env=None, cwd=None + self, command, env=None, cwd=None, timeout=None ) -> subprocess.CompletedProcess: try: - return subprocess.run( + proc = subprocess.Popen( command, - stdin=subprocess.DEVNULL, capture_output=True, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, - check=False, env=env, cwd=cwd if cwd else self.fake_home, + start_new_session=True, ) + with process_timeout(proc, timeout) as is_timed_out: + stdout, stderr = proc.communicate() + + if is_timed_out(): + stderr = (stderr + "\n" if stderr else "") + f"Error: Command timed out after {timeout} seconds." + return subprocess.CompletedProcess(command, 124, stdout or "", stderr) + + return subprocess.CompletedProcess(command, proc.returncode, stdout or "", stderr) except FileNotFoundError: return subprocess.CompletedProcess( command, 127, "", f"Error: Command not found: {command[0]}" @@ -904,7 +915,7 @@ def _run_agy_cli(self, cli_cmd: CLICommand): timeout=self.timeout, ) cwd = cli_cmd.cwd if cli_cmd.cwd else self.fake_home - result = self._execute_cli_command(command, env=env, cwd=cwd) + result = self._execute_cli_command(command, env=env, cwd=cwd, timeout=cli_cmd.timeout) # Parse whenever agy emitted a stream, even on a non-zero exit: a # timed-out/errored run still ends in a ``result`` event carrying real @@ -1212,7 +1223,7 @@ def safe_generate( def create_command( self, cli: str, prompt: str, env: dict = None, resume: bool = False, - session_id: str = None, cwd: str = None, + session_id: str = None, cwd: str = None, timeout: float | int = None, ) -> CLICommand: # The executable is always this session's sandbox binary # (self.agy_bin); the ``cli`` argument -- the agent_version label "agy" @@ -1222,4 +1233,4 @@ def create_command( # environment are layered in once at invocation time by # ``_run_agy_cli`` via ``_merged_env``. return CLICommand(cli=self.agy_bin, prompt=prompt, env=env or {}, - resume=resume, cwd=cwd) + resume=resume, cwd=cwd, timeout=timeout) diff --git a/evalbench/generators/models/claude_code.py b/evalbench/generators/models/claude_code.py index 0d546ec9..da946215 100644 --- a/evalbench/generators/models/claude_code.py +++ b/evalbench/generators/models/claude_code.py @@ -1,4 +1,4 @@ -from .agent_cli import AgentCliGenerator +from .agent_cli import AgentCliGenerator, process_timeout from .tool_naming import canonicalize_claude_tool_name import subprocess import os @@ -14,7 +14,7 @@ class CLICommand: - def __init__(self, cli, prompt, env=None, resume=False, session_id=None, allowedTools=None, cwd=None): + def __init__(self, cli, prompt, env=None, resume=False, session_id=None, allowedTools=None, cwd=None, timeout=None): self.cli = cli self.prompt = prompt self.env = env if env else {} @@ -22,6 +22,7 @@ def __init__(self, cli, prompt, env=None, resume=False, session_id=None, allowed self.session_id = session_id self.allowedTools = allowedTools self.cwd = cwd + self.timeout = timeout class ClaudeCodeGenerator(AgentCliGenerator): @@ -606,14 +607,27 @@ def generate_internal(self, cli_cmd): def _execute_cli_command( self, command: list[str], env: dict[str, str] | None = None, - cwd: str | None = None, + cwd: str | None = None, timeout: float | int | None = None, ) -> subprocess.CompletedProcess: try: - result = subprocess.run( - command, capture_output=True, text=True, check=False, env=env, - cwd=cwd if cwd else self.fake_home, stdin=subprocess.DEVNULL + proc = subprocess.Popen( + command, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + cwd=cwd if cwd else self.fake_home, + start_new_session=True, ) - return result + with process_timeout(proc, timeout) as is_timed_out: + stdout, stderr = proc.communicate() + + if is_timed_out(): + stderr = (stderr + "\n" if stderr else "") + f"Error: Command timed out after {timeout} seconds." + return subprocess.CompletedProcess(command, 124, stdout or "", stderr) + + return subprocess.CompletedProcess(command, proc.returncode, stdout or "", stderr) except FileNotFoundError: return subprocess.CompletedProcess( command, 127, "", f"Error: Command not found: {command[0]}" @@ -697,7 +711,7 @@ def _run_claude_code(self, cli_cmd: CLICommand): logging.info(f"Running Claude Code CLI: {' '.join(command)}") - result = self._execute_cli_command(command, env=env, cwd=cli_cmd.cwd) + result = self._execute_cli_command(command, env=env, cwd=cli_cmd.cwd, timeout=cli_cmd.timeout) if result.stdout: result.stdout = self._parse_stream_json(result.stdout) @@ -1089,12 +1103,12 @@ def safe_generate(self, cli_cmd: CLICommand) -> subprocess.CompletedProcess: def create_command( self, cli: str, prompt: str, env: dict = None, resume: bool = False, - session_id: str = None, cwd: str = None, + session_id: str = None, cwd: str = None, timeout: float | int = None, ) -> CLICommand: merged_env = self.env.copy() if env: merged_env.update(env) return CLICommand( cli=cli, prompt=prompt, env=merged_env, - resume=resume, session_id=session_id, cwd=cwd, + resume=resume, session_id=session_id, cwd=cwd, timeout=timeout, ) diff --git a/evalbench/generators/models/codex_cli.py b/evalbench/generators/models/codex_cli.py index 35f6e4ab..479e562f 100644 --- a/evalbench/generators/models/codex_cli.py +++ b/evalbench/generators/models/codex_cli.py @@ -1,4 +1,4 @@ -from .agent_cli import AgentCliGenerator +from .agent_cli import AgentCliGenerator, process_timeout from .tool_naming import canonical_tool_name import subprocess import os @@ -42,13 +42,14 @@ def _fetch_secret_manager(path: str) -> str: class CLICommand: - def __init__(self, cli, prompt, env=None, resume=False, session_id=None, cwd=None): + def __init__(self, cli, prompt, env=None, resume=False, session_id=None, cwd=None, timeout=None): self.cli = cli self.prompt = prompt self.env = env if env else {} self.resume = resume self.session_id = session_id self.cwd = cwd + self.timeout = timeout class CodexCliGenerator(AgentCliGenerator): @@ -719,6 +720,7 @@ def generate_internal(self, cli_cmd): def _execute_cli_command( self, command: list[str], env: dict[str, str] | None = None, cwd: str | None = None, + timeout: float | int | None = None, ) -> tuple[subprocess.CompletedProcess, dict[str, int]]: """Runs the Codex CLI with line-streamed stdout so we can stamp the wall-clock time at which each NDJSON event arrives. @@ -736,6 +738,7 @@ def _execute_cli_command( stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=1, # line-buffered so events arrive as Codex flushes them + start_new_session=True, ) except FileNotFoundError: return subprocess.CompletedProcess( @@ -762,22 +765,30 @@ def _drain_stderr(): started_at_ms: dict[str, float] = {} tool_durations: dict[str, int] = {} - try: - for line in proc.stdout: - arrival_ms = time.monotonic() * 1000 - stdout_lines.append(line) - self._stamp_tool_event( - line, arrival_ms, started_at_ms, tool_durations, - ) - except Exception as e: - logging.warning(f"stdout stream read failed: {e}") + with process_timeout(proc, timeout) as is_timed_out: + try: + for line in proc.stdout: + arrival_ms = time.monotonic() * 1000 + stdout_lines.append(line) + self._stamp_tool_event( + line, arrival_ms, started_at_ms, tool_durations, + ) + except Exception as e: + logging.warning(f"stdout stream read failed: {e}") - proc.wait() - stderr_thread.join(timeout=5) + proc.wait() + stderr_thread.join(timeout=5) + + stderr_str = "".join(stderr_chunks) + if is_timed_out(): + stderr_str = (stderr_str + "\n" if stderr_str else "") + f"Error: Command timed out after {timeout} seconds." + proc_returncode = 124 + else: + proc_returncode = proc.returncode completed = subprocess.CompletedProcess( - command, proc.returncode, - "".join(stdout_lines), "".join(stderr_chunks), + command, proc_returncode, + "".join(stdout_lines), stderr_str, ) return completed, tool_durations @@ -866,7 +877,7 @@ def _run_codex_cli(self, cli_cmd: CLICommand): logging.info(f"Running Codex CLI: {' '.join(command)}") start_ms = time.monotonic() - result, tool_durations = self._execute_cli_command(command, env=env, cwd=cli_cmd.cwd) + result, tool_durations = self._execute_cli_command(command, env=env, cwd=cli_cmd.cwd, timeout=cli_cmd.timeout) duration_ms = int((time.monotonic() - start_ms) * 1000) if result.stdout: result.stdout = self._parse_stream_json( @@ -1304,12 +1315,12 @@ def safe_generate(self, cli_cmd: CLICommand) -> subprocess.CompletedProcess: def create_command( self, cli: str, prompt: str, env: dict = None, resume: bool = False, - session_id: str = None, cwd: str = None, + session_id: str = None, cwd: str = None, timeout: float | int = None, ) -> CLICommand: merged_env = self.env.copy() if env: merged_env.update(env) return CLICommand( cli=cli, prompt=prompt, env=merged_env, - resume=resume, session_id=session_id, cwd=cwd, + resume=resume, session_id=session_id, cwd=cwd, timeout=timeout, ) diff --git a/evalbench/generators/models/gemini_cli.py b/evalbench/generators/models/gemini_cli.py index a72e1231..2ee2e168 100644 --- a/evalbench/generators/models/gemini_cli.py +++ b/evalbench/generators/models/gemini_cli.py @@ -1,4 +1,4 @@ -from .agent_cli import AgentCliGenerator +from .agent_cli import AgentCliGenerator, process_timeout from .tool_naming import canonicalize_gemini_tool_name import subprocess import os @@ -11,13 +11,14 @@ class CLICommand: - def __init__(self, cli, prompt, env=None, resume=False, yolo=True, cwd=None): + def __init__(self, cli, prompt, env=None, resume=False, yolo=True, cwd=None, timeout=None): self.cli = cli self.prompt = prompt self.env = env if env else {} self.resume = resume self.yolo = yolo self.cwd = cwd + self.timeout = timeout class GeminiCliGenerator(AgentCliGenerator): @@ -790,20 +791,37 @@ def generate_internal(self, cli_cmd: CLICommand | str): return self._run_gemini_cli(cli_cmd) def _execute_cli_command( - self, command: list[str], env: dict[str, str] | None = None, cwd: str | None = None + self, command: list[str], env: dict[str, str] | None = None, cwd: str | None = None, + timeout: float | int | None = None, ) -> subprocess.CompletedProcess: try: - result = subprocess.run(command, capture_output=True, text=True, check=False, env=env, cwd=cwd if cwd else self.fake_home) + proc = subprocess.Popen( + command, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + cwd=cwd if cwd else self.fake_home, + start_new_session=True, + ) + with process_timeout(proc, timeout) as is_timed_out: + stdout, stderr = proc.communicate() + # Filter out benign schema warnings from json decoder from stderr to reduce noise - if result.stderr: - result.stderr = "\n".join( + if stderr: + stderr = "\n".join( [ - line - for line in result.stderr.splitlines() + line for line in stderr.splitlines() if 'unknown format "google-duration" ignored' not in line ] ) - return result + + if is_timed_out(): + stderr = (stderr + "\n" if stderr else "") + f"Error: Command timed out after {timeout} seconds." + return subprocess.CompletedProcess(command, 124, stdout or "", stderr) + + return subprocess.CompletedProcess(command, proc.returncode, stdout or "", stderr) except FileNotFoundError: return subprocess.CompletedProcess( command, 127, "", f"Error: Command not found: {command[0]}" @@ -847,8 +865,8 @@ def _run_gemini_cli(self, cli_cmd: CLICommand): ] ) - result = self._execute_cli_command(command, env=env, cwd=cli_cmd.cwd) - if result.returncode == 0 and result.stdout: + result = self._execute_cli_command(command, env=env, cwd=cli_cmd.cwd, timeout=cli_cmd.timeout) + if result.stdout: result.stdout = self._parse_stream_json(result.stdout) return result @@ -1083,7 +1101,7 @@ def safe_generate(self, cli_cmd: CLICommand) -> subprocess.CompletedProcess: def create_command( self, cli: str, prompt: str, env: dict = None, resume: bool = False, - session_id: str = None, cwd: str = None, + session_id: str = None, cwd: str = None, timeout: float | int = None, ) -> CLICommand: merged_env = self.env.copy() @@ -1094,4 +1112,4 @@ def create_command( if env: merged_env.update(env) - return CLICommand(cli=cli, prompt=prompt, env=merged_env, resume=resume, cwd=cwd) + return CLICommand(cli=cli, prompt=prompt, env=merged_env, resume=resume, cwd=cwd, timeout=timeout) diff --git a/evalbench/generators/models/generator.py b/evalbench/generators/models/generator.py index e2e3fe2c..cf660e7c 100644 --- a/evalbench/generators/models/generator.py +++ b/evalbench/generators/models/generator.py @@ -11,6 +11,9 @@ def __init__(self, querygenerator_config): "execs_per_minute") or None self.max_attempts = querygenerator_config.get("max_attempts") or 3 self.semaphore = Semaphore(self.execs_per_minute or 1) + self.prompt_timeout_seconds = querygenerator_config.get( + "prompt_timeout_seconds" + ) def generate(self, prompt): try: diff --git a/evalbench/test/agentevaluator_test.py b/evalbench/test/agentevaluator_test.py index de3176dc..e52f4243 100644 --- a/evalbench/test/agentevaluator_test.py +++ b/evalbench/test/agentevaluator_test.py @@ -88,6 +88,55 @@ def test_process_scenario_copies_env_files(self, mock_get_generator): content = f.read() self.assertEqual(content, "print('mock sleep')") + @patch("evaluator.agentevaluator.get_generator") + def test_process_scenario_passes_prompt_timeout(self, mock_get_generator): + mock_generator = MagicMock(spec=AgentCliGenerator) + mock_generator.fake_home = self.fake_home + mock_generator.name = "mock_agent_cli" + mock_generator.version = "1.0" + mock_get_generator.return_value = mock_generator + + config = { + "model_config": "dummy_model_config.yaml", + "runners": {"agent_runners": 1, "prompt_timeout_seconds": 120} + } + evaluator = AgentEvaluator(config) + + scenario = { + "id": "test_timeout_scenario", + "starting_prompt": "hello", + "max_turns": 1, + "prompt_timeout_seconds": 45, + } + + mock_result = MagicMock() + mock_result.stdout = '{"session_id": "s1"}' + mock_result.stderr = '' + mock_result.returncode = 0 + mock_generator.safe_generate.return_value = mock_result + mock_generator.parse_response.return_value = {} + mock_generator.extract_tools.return_value = [] + mock_generator.extract_skills.return_value = [] + evaluator._finalize_scenario = MagicMock() + + evaluator.process_scenario( + scenario=scenario, + eval_result=MagicMock(), + job_id="job1", + metadata={} + ) + + # Verify create_command received timeout=45 from scenario JSON override + mock_generator.create_command.assert_called_once_with( + cli="1.0", + prompt="hello", + env={}, + resume=False, + session_id=None, + cwd=None, + timeout=45, + ) + if __name__ == "__main__": unittest.main() diff --git a/evalbench/test/agy_cli_test.py b/evalbench/test/agy_cli_test.py index 80b236e8..996c0ed4 100644 --- a/evalbench/test/agy_cli_test.py +++ b/evalbench/test/agy_cli_test.py @@ -55,11 +55,24 @@ def skip_agy_install(request): @pytest.fixture def mock_run(): - """Patches the generator's ``subprocess.run`` with a success-by-default + """Patches the generator's ``subprocess.run`` and ``subprocess.Popen`` with a success-by-default mock. Tests needing custom behavior set ``side_effect``.""" - with patch("generators.models.agy_cli.subprocess.run") as m: - m.return_value = MagicMock(returncode=0, stdout="", stderr="") - yield m + with patch("generators.models.agy_cli.subprocess.run") as m_run, \ + patch("generators.models.agy_cli.subprocess.Popen") as m_popen: + m_run.return_value = MagicMock(returncode=0, stdout="", stderr="") + + def popen_side_effect(*args, **kwargs): + res = m_run(*args, **kwargs) + proc = MagicMock() + proc.pid = 12345 + proc.returncode = getattr(res, "returncode", 0) if not isinstance(getattr(res, "returncode", 0), MagicMock) else 0 + stdout = res.stdout if isinstance(getattr(res, "stdout", None), str) else "" + stderr = res.stderr if isinstance(getattr(res, "stderr", None), str) else "" + proc.communicate.return_value = (stdout, stderr) + return proc + + m_popen.side_effect = popen_side_effect + yield m_run def _install_calls(mock_run): @@ -326,6 +339,21 @@ def test_run_command_argv_shape_with_timeout(mock_run, sandbox): ] +def test_run_command_with_int_prompt_timeout_routes_to_executor_not_argv(mock_run, sandbox): + """An integer timeout on CLICommand (e.g. from prompt_timeout_seconds) must + NOT be appended to argv as --print-timeout , which crashes Popen. + It must be routed to _execute_cli_command(timeout=...).""" + generator = AgyCliGenerator({}) + cmd = CLICommand(cli="agy", prompt="hello world", timeout=180) + + with patch.object(generator, "_execute_cli_command", return_value=MagicMock(stdout="", stderr="", returncode=0)) as mock_exec: + generator._run_agy_cli(cmd) + mock_exec.assert_called_once() + assert mock_exec.call_args.kwargs.get("timeout") == 180 + sent_argv = mock_exec.call_args.args[0] + assert "--print-timeout" not in sent_argv + + def test_run_agy_cli_parses_stream_on_nonzero_exit(mock_run, sandbox): """A timed-out/errored run exits non-zero but still emits a full stream ending in an ERROR result -- its usage tokens and tool calls must be kept, diff --git a/evalbench/test/claude_code_test.py b/evalbench/test/claude_code_test.py index fa4d6bc7..fbb9816f 100644 --- a/evalbench/test/claude_code_test.py +++ b/evalbench/test/claude_code_test.py @@ -1,4 +1,5 @@ import os +import signal import sys from unittest.mock import MagicMock, patch, ANY @@ -88,3 +89,40 @@ def test_install_plugin_runs_init_and_install(mock_open, mock_makedirs, monkeypa assert "plugins" in second_call_args assert "install" in second_call_args assert "my-plugin" in second_call_args + + +def test_execute_cli_command_timeout_preserves_stdout_and_kills_pg(monkeypatch): + monkeypatch.setenv("HOME", "/fake/real_home") + + with ( + patch('generators.models.claude_code.os.makedirs'), + patch('generators.models.claude_code.open', create=True), + ): + generator = ClaudeCodeGenerator({}) + + mock_proc = MagicMock() + mock_proc.pid = 54321 + stream_output = '{"type":"assistant","message":{"content":[{"type":"text","text":"claude response"}]}}\n' + mock_proc.communicate.return_value = (stream_output, "") + + with ( + patch('generators.models.claude_code.subprocess.Popen', return_value=mock_proc) as mock_popen, + patch('generators.models.agent_cli.os.killpg') as mock_killpg, + patch('generators.models.agent_cli.os.getpgid', return_value=54321), + patch('generators.models.agent_cli.threading.Timer') as mock_timer_cls, + ): + def fake_timer(interval, function): + function() + return MagicMock() + + mock_timer_cls.side_effect = fake_timer + + cli_cmd = generator.create_command("claude", "prompt", timeout=2) + res = generator._run_claude_code(cli_cmd) + + mock_popen.assert_called_once() + assert mock_popen.call_args.kwargs.get("start_new_session") is True + mock_killpg.assert_called_once_with(54321, signal.SIGKILL) + assert res.returncode == 124 + assert "Error: Command timed out after 2 seconds." in res.stderr + assert "claude response" in res.stdout diff --git a/evalbench/test/codex_cli_test.py b/evalbench/test/codex_cli_test.py index f6dbf515..8f9ba61b 100644 --- a/evalbench/test/codex_cli_test.py +++ b/evalbench/test/codex_cli_test.py @@ -1,3 +1,4 @@ +import errno import os import sys from unittest.mock import MagicMock, patch @@ -108,3 +109,100 @@ def test_write_config_toml_escapes_plugin_id(monkeypatch, tmp_path): content = config_file.read_text() assert '[plugins."dak@evalbench-local-marketplace"]' in content assert '[plugins.clean_plugin]' in content + + +@patch('generators.models.codex_cli.subprocess.Popen') +def test_execute_cli_command_timeout_wall_clock(mock_popen, monkeypatch): + monkeypatch.setenv("HOME", "/fake/real_home") + + with ( + patch('generators.models.codex_cli.os.makedirs'), + patch('generators.models.codex_cli.open', create=True), + ): + generator = CodexCliGenerator({"model": "gpt-4"}) + + r, w = os.pipe() + stdout_file = os.fdopen(r, "r") + + mock_proc = MagicMock() + mock_proc.stdout = stdout_file + mock_proc.stderr = [] + mock_proc.returncode = None + + def mock_kill(): + try: + os.close(w) + except OSError as e: + # Ignore EBADF if the write pipe descriptor was already closed + if e.errno != errno.EBADF: + raise + + mock_proc.kill.side_effect = mock_kill + mock_popen.return_value = mock_proc + + import time + start = time.monotonic() + res, _ = generator._execute_cli_command(["codex"], timeout=0.2) + elapsed = time.monotonic() - start + + assert res.returncode == 124 + assert "timed out after 0.2 seconds" in res.stderr + mock_proc.kill.assert_called_once() + assert elapsed < 0.6 + + +def test_create_command_passes_timeout(): + with ( + patch('generators.models.codex_cli.os.makedirs'), + patch('generators.models.codex_cli.open', create=True), + ): + generator = CodexCliGenerator({"model": "gpt-4"}) + + cli_cmd = generator.create_command("codex", "do something", timeout=120) + assert cli_cmd.timeout == 120 + + with patch.object(generator, '_execute_cli_command', return_value=(MagicMock(stdout="", stderr="", returncode=0), {})) as mock_exec: + generator._run_codex_cli(cli_cmd) + mock_exec.assert_called_once() + assert mock_exec.call_args.kwargs.get("timeout") == 120 + + +@patch('generators.models.codex_cli.subprocess.Popen') +def test_execute_cli_command_timeout_when_stdout_closes_before_proc_exit(mock_popen, monkeypatch): + monkeypatch.setenv("HOME", "/fake/real_home") + + with ( + patch('generators.models.codex_cli.os.makedirs'), + patch('generators.models.codex_cli.open', create=True), + ): + generator = CodexCliGenerator({"model": "gpt-4"}) + + mock_proc = MagicMock() + mock_proc.stdout = [] # EOF immediately on stdout + mock_proc.stderr = [] + mock_proc.returncode = None + mock_proc.pid = 99999 + + import threading + wait_event = threading.Event() + + def mock_wait(): + wait_event.wait(timeout=2.0) + return 0 + + def mock_kill(): + wait_event.set() + + mock_proc.wait.side_effect = mock_wait + mock_proc.kill.side_effect = mock_kill + mock_popen.return_value = mock_proc + + import time + start = time.monotonic() + res, _ = generator._execute_cli_command(["codex"], timeout=0.2) + elapsed = time.monotonic() - start + + assert res.returncode == 124 + assert "timed out after 0.2 seconds" in res.stderr + mock_proc.kill.assert_called_once() + assert elapsed < 0.6 diff --git a/evalbench/test/gemini_cli_test.py b/evalbench/test/gemini_cli_test.py index ce2552bf..2cde7040 100644 --- a/evalbench/test/gemini_cli_test.py +++ b/evalbench/test/gemini_cli_test.py @@ -1,4 +1,5 @@ import os +import signal import sys from unittest.mock import MagicMock, patch @@ -337,3 +338,41 @@ def test_setup_skill_dict_disable( "my-skill", ] assert expected_cmd in calls + + +def test_execute_cli_command_timeout_preserves_stdout_and_kills_pg(monkeypatch): + monkeypatch.setenv("HOME", "/fake/real_home") + + with ( + patch('generators.models.gemini_cli.os.makedirs'), + patch('generators.models.gemini_cli.os.path.exists', return_value=False), + patch('generators.models.gemini_cli.open', create=True), + ): + generator = GeminiCliGenerator({}) + + mock_proc = MagicMock() + mock_proc.pid = 12345 + stream_output = '{"type":"init","session_id":"s1"}\n{"type":"message","role":"assistant","content":"hello"}\n' + mock_proc.communicate.return_value = (stream_output, "") + + with ( + patch('generators.models.gemini_cli.subprocess.Popen', return_value=mock_proc) as mock_popen, + patch('generators.models.agent_cli.os.killpg') as mock_killpg, + patch('generators.models.agent_cli.os.getpgid', return_value=12345), + patch('generators.models.agent_cli.threading.Timer') as mock_timer_cls, + ): + def fake_timer(interval, function): + function() + return MagicMock() + + mock_timer_cls.side_effect = fake_timer + + cli_cmd = generator.create_command("gemini", "prompt", timeout=1) + res = generator._run_gemini_cli(cli_cmd) + + mock_popen.assert_called_once() + assert mock_popen.call_args.kwargs.get("start_new_session") is True + mock_killpg.assert_called_once_with(12345, signal.SIGKILL) + assert res.returncode == 124 + assert "Error: Command timed out after 1 seconds." in res.stderr + assert "hello" in res.stdout