diff --git a/agent/engine.py b/agent/engine.py index 2f9948f9..393c1d58 100644 --- a/agent/engine.py +++ b/agent/engine.py @@ -1,16 +1,13 @@ -import os -import time -import traceback -from threading import Event, Thread - from chipcompiler.data import StateEnum, WorkspaceStep -from chipcompiler.engine.flow import EngineFlow, get_process_rss_mb, track_current_process_memory -from chipcompiler.utility.log import redirect_stdio_to_file +from chipcompiler.engine.flow import EngineFlow from .tools import run_step as run_agent_step class AgentEngineFlow(EngineFlow): + """Flow Agent engine: the canonical step lifecycle lives in EngineFlow; + only the tool runner and the agent's result vocabulary differ.""" + def build_default_steps(self): super().build_default_steps() steps = self.workspace.flow.data["steps"] @@ -20,76 +17,16 @@ def build_default_steps(self): steps.insert(filler_index, self.init_flow_step("DRC", "ecc", StateEnum.Unstart)) self.save() - def run_step(self, workspace_step: WorkspaceStep | str, *, rerun: bool = False) -> StateEnum: - if isinstance(workspace_step, str): - workspace_step = self.get_workspace_step(workspace_step) - if workspace_step is None: - return StateEnum.Invalid - step_tag = f"{workspace_step.name}({workspace_step.tool})" - if not rerun and self.check_state( - name=workspace_step.name, tool=workspace_step.tool, state=StateEnum.Success - ): - self.workspace.logger.info("[SKIP] %s already succeeded", step_tag) - self.clear_db_engine_after_step(workspace_step, StateEnum.Success) - return StateEnum.Success - - start_time = time.time() - timing_constraints = self.timing_constraint_facts() - self.set_state(name=workspace_step.name, tool=workspace_step.tool, state=StateEnum.Ongoing) - self._redirect_step_stdio(workspace_step) - start_memory, peak_memory, stop_monitor, monitor = self._start_memory_monitor() - result = False - try: - result = run_agent_step( - workspace=self.workspace, step=workspace_step, ecc_module=self.engine_db.engine - ) - self.workspace.logger.info("[STEP] %s finished result=%s", step_tag, result) - except Exception: - self.workspace.logger.error("[STEP] %s failed with exception", step_tag) - traceback.print_exc() - finally: - self._stop_memory_monitor(stop_monitor, monitor) - - elapsed = time.time() - start_time - state = self._step_state(workspace_step, result) - self._finish_step( - workspace_step, - state, - elapsed, - timing_constraints, - max(0, round(peak_memory[0] - start_memory, 3)), - ) - return state - - def _redirect_step_stdio(self, workspace_step: WorkspaceStep) -> None: - log_file = workspace_step.log.file or "" - if not log_file: - return - try: - log_file = os.path.abspath(log_file) - os.makedirs(os.path.dirname(log_file) or ".", exist_ok=True) - redirect_stdio_to_file(log_file) - except Exception: - traceback.print_exc() - - def _start_memory_monitor(self) -> tuple[float, list[float], Event, Thread]: - start_memory = get_process_rss_mb(os.getpid()) - peak_memory = [start_memory] - stop_monitor = Event() - monitor = Thread( - target=track_current_process_memory, - args=(os.getpid(), stop_monitor, peak_memory), - daemon=True, + def _invoke_step_tool(self, workspace_step: WorkspaceStep): + return run_agent_step( + workspace=self.workspace, step=workspace_step, ecc_module=self.engine_db.engine ) - monitor.start() - return start_memory, peak_memory, stop_monitor, monitor - @staticmethod - def _stop_memory_monitor(stop_monitor: Event, monitor: Thread) -> None: - stop_monitor.set() - monitor.join() - - def _step_state(self, workspace_step: WorkspaceStep, result: object) -> StateEnum: + def _derive_step_state( + self, workspace_step: WorkspaceStep, result, *, raised: bool + ) -> StateEnum: + if raised: + return StateEnum.Imcomplete if result is StateEnum.Invalid: return StateEnum.Invalid if result is True or result is StateEnum.Success: @@ -99,52 +36,3 @@ def _step_state(self, workspace_step: WorkspaceStep, result: object) -> StateEnu else StateEnum.Imcomplete ) return StateEnum.Imcomplete - - def _finish_step( - self, - workspace_step: WorkspaceStep, - state: StateEnum, - elapsed: float, - timing_constraints: dict, - peak_memory_mb: float, - ) -> None: - runtime = f"{int(elapsed // 3600)}:{int((elapsed % 3600) // 60)}:{int(elapsed % 60)}" - self.set_state( - name=workspace_step.name, - tool=workspace_step.tool, - state=state, - runtime=runtime, - peak_memory=peak_memory_mb, - ) - if state == StateEnum.Success: - self._save_agent_step_facts( - workspace_step, - state, - elapsed, - peak_memory_mb, - timing_constraints, - ) - self.clear_db_engine_after_step(workspace_step, state) - - def _save_agent_step_facts( - self, - workspace_step: WorkspaceStep, - state: StateEnum, - elapsed: float, - peak_memory: float, - timing_constraints: dict, - ) -> None: - from chipcompiler.tools import build_step_metrics, save_layout_image - - if self.save_step_flow_facts( - workspace_step=workspace_step, - state=state, - runtime_seconds=elapsed, - peak_memory_mb=peak_memory, - timing_constraints=timing_constraints, - ): - try: - build_step_metrics(workspace=self.workspace, step=workspace_step) - except Exception: - self.workspace.logger.exception("[QOR] failed to refresh analysis") - save_layout_image(workspace=self.workspace, step=workspace_step) diff --git a/agent/test/test_engine.py b/agent/test/test_engine.py index 2e80a2de..7b44d834 100644 --- a/agent/test/test_engine.py +++ b/agent/test/test_engine.py @@ -38,3 +38,98 @@ def run_step(**_kwargs): assert flow.run_step(step) is expected_state assert flow.check_state("route", "ecc", expected_state) + + +def _marker_workspace(tmp_path): + workspace = Workspace(directory=tmp_path, flow=Flow(path=tmp_path / "flow.json")) + flow = AgentEngineFlow(workspace) + workspace.flow.data = {"steps": [{"name": "route", "tool": "ecc", "state": "Unstart"}]} + step = EccStep(name="route", directory=tmp_path, tool="ecc") + flow.workspace_steps = [step] + flow.engine_db = SimpleNamespace(engine=None) + return flow, step + + +def test_agent_engine_emits_markers_around_step_writes(monkeypatch, tmp_path): + """The agent executor frames the step stream with markers and never + redirects stdio into a step log file.""" + flow, step = _marker_workspace(tmp_path) + events = [] + monkeypatch.setattr(flow, "check_step_result", lambda **_kwargs: True) + monkeypatch.setattr( + "agent.engine.run_agent_step", + lambda **kwargs: events.append(("tool", None)) or True, + ) + monkeypatch.setattr(flow, "save_step_flow_facts", lambda **_kwargs: False) + monkeypatch.setattr( + "chipcompiler.tools.save_layout_image", + lambda **_kwargs: events.append(("layout", None)), + ) + monkeypatch.setattr( + "chipcompiler.runtime.log_stream.emit_step_marker", + lambda event, *, step, tool: events.append(("marker", event)), + ) + + assert flow.run_step(step) is StateEnum.Success + end_index = events.index(("marker", "end")) + assert events.index(("marker", "begin")) < events.index(("tool", None)) + assert events.index(("layout", None)) < end_index + + +def test_agent_engine_suppresses_end_marker_when_final_save_fails(monkeypatch, tmp_path): + """A failed final save downgrades the record and suppresses the end + marker, matching the base engine's authoritative-save contract.""" + flow, step = _marker_workspace(tmp_path) + markers = [] + monkeypatch.setattr(flow, "check_step_result", lambda **_kwargs: True) + monkeypatch.setattr("agent.engine.run_agent_step", lambda **kwargs: True) + monkeypatch.setattr( + "chipcompiler.runtime.log_stream.emit_step_marker", + lambda event, *, step, tool: markers.append(event), + ) + + real_save = flow.save + save_calls = [] + + def save_failing_on_final(): + save_calls.append(len(save_calls) + 1) + if len(save_calls) == 1: + return real_save() # the Ongoing save persists + return False # the one final save fails + + monkeypatch.setattr(flow, "save", save_failing_on_final) + + assert flow.run_step(step) is StateEnum.Imcomplete + assert save_calls == [1, 2] + assert markers == ["begin"] + record = flow.get_step("route", "ecc") + assert record["state"] == StateEnum.Imcomplete.value + + +def test_agent_engine_inherits_lifecycle_and_never_opens_step_logs(monkeypatch, tmp_path): + """The inherited base lifecycle drives the agent tool hook end to end, + and no step log file is opened even when the step declares a log path.""" + flow, step = _marker_workspace(tmp_path) + declared_log = tmp_path / "route_ecc" / "log" / "route.log" + step.log.file = declared_log + tool_calls = [] + monkeypatch.setattr(flow, "check_step_result", lambda **_kwargs: True) + monkeypatch.setattr( + "agent.engine.run_agent_step", + lambda **kwargs: tool_calls.append(kwargs) or True, + ) + monkeypatch.setattr(flow, "save_step_flow_facts", lambda **_kwargs: False) + monkeypatch.setattr("chipcompiler.tools.save_layout_image", lambda **_kwargs: True) + markers = [] + monkeypatch.setattr( + "chipcompiler.runtime.log_stream.emit_step_marker", + lambda event, *, step, tool: markers.append(event), + ) + + assert flow.run_step(step) is StateEnum.Success + # The agent runner hook was invoked through the base lifecycle. + assert len(tool_calls) == 1 + assert tool_calls[0]["step"] is step + assert markers == ["begin", "end"] + assert not declared_log.exists() + assert flow.check_state("route", "ecc", StateEnum.Success) diff --git a/agent/test/test_workspace_api.py b/agent/test/test_workspace_api.py index a400f3f7..7b623e5b 100644 --- a/agent/test/test_workspace_api.py +++ b/agent/test/test_workspace_api.py @@ -1,6 +1,10 @@ +import json +import os from pathlib import Path from types import SimpleNamespace +import pytest + from agent.requests import CandidateRerunRequest from agent.workspace_api import FlowAgentRuntimeApi, _candidate_step_artifact_dirs from chipcompiler.data import StateEnum @@ -112,6 +116,61 @@ def test_candidate_rerun_uses_the_agent_flow_and_replays_its_receipts(monkeypatc assert not list(cts_output.iterdir()) +def test_candidate_step_exception_reconciles_the_record(monkeypatch, tmp_path, capfd): + """A run_step raising after its begin marker must downgrade the record + before the exception propagates — not leave Success over a partial log.""" + (tmp_path / "home").mkdir() + steps = [{"name": "place", "tool": "dreamplace", "state": "Success"}] + (tmp_path / "home" / "flow.json").write_text(json.dumps({"steps": steps})) + workspace = SimpleNamespace( + directory=tmp_path, + flow=SimpleNamespace(data={"steps": [dict(s) for s in steps]}), + ) + place_output = tmp_path / "place_dreamplace" / "output" + place_output.mkdir(parents=True) + step = SimpleNamespace( + name="place", + tool="dreamplace", + output=EccOutput(dir=place_output), + analysis={}, + ) + flow = _Flow(workspace, (step,)) + + from chipcompiler.runtime.log_stream import emit_step_marker + + def raising_run_step(step, *, rerun): + flow.run_calls.append((step.name, rerun)) + emit_step_marker("begin", step=step.name, tool=step.tool) + os.write(2, b"partial\n") + raise RuntimeError("layout save blew up") + + flow.run_step = raising_run_step + flow.set_state = lambda name, tool, state: ( + workspace.flow.data["steps"][0].update({"state": state.value}) + ) + + api = FlowAgentRuntimeApi(_EccApi(workspace)) + monkeypatch.setattr("agent.workspace_api.build_agent_flow_for_workspace", lambda _ws: flow) + monkeypatch.setattr( + "agent.workspace_api._init_db_engine_for_workspace_step", lambda _flow, _step: None + ) + + with pytest.raises(RuntimeError, match="layout save blew up"): + api.candidate_rerun( + CandidateRerunRequest( + workspace_id="workspace-1", + target_step="place", + end_step="place", + candidate_id=None, + patch=None, + execution_scope="single_step", + ) + ) + + assert workspace.flow.data["steps"][0]["state"] == StateEnum.Imcomplete.value + assert "ECC-STEP" not in capfd.readouterr().err + + class _EccApi: def __init__(self, workspace): self.session = SimpleNamespace(workspace=workspace, db_handle=None) diff --git a/agent/workspace_api.py b/agent/workspace_api.py index 9c310a17..c8a2b893 100644 --- a/agent/workspace_api.py +++ b/agent/workspace_api.py @@ -3,6 +3,7 @@ from hashlib import sha256 from pathlib import Path +from chipcompiler.data import StateEnum from chipcompiler.runtime.requests import WorkspaceIdRequest from chipcompiler.runtime.workspace_api import ( RuntimeApiError, @@ -262,8 +263,40 @@ def _clear_candidate_artifact_dir(workspace_root: Path, directory: Path, step_na def _run_candidate_step(flow, step) -> None: + from chipcompiler.runtime.log_stream import archive_own_step_logs + _init_db_engine_for_workspace_step(flow, step) - state = flow.run_step(step, rerun=True) + # In-process execution is still executor+client in one process: route the + # own fd-2 stream through the reader so markers are consumed and the + # step's bytes land in its archive (echoed to the real stderr). + reader = None + try: + with archive_own_step_logs(flow.workspace.directory) as active_reader: + reader = active_reader + state = flow.run_step(step, rerun=True) + except BaseException: + # A step raising after its begin marker (post-processing, the end + # write) leaves the reader holding an active step while flow.json may + # already say Success; reconcile before propagating. + if reader is not None and ( + reader.state.error is not None or reader.state.active_step is not None + ): + flow.set_state(step.name, step.tool, StateEnum.Imcomplete) + raise + # An archive failure or unmatched begin must not report success while the + # step's log is missing; downgrade so a later rerun rebuilds it. A None + # reader means an outer client owns the stream (passthrough) — nothing to + # reconcile here. + if reader is not None and ( + reader.state.error is not None or reader.state.active_step is not None + ): + # set_state owns the authoritative save; a failed save is logged there. + flow.set_state(step.name, step.tool, StateEnum.Imcomplete) + raise RuntimeApiError( + "command_failed", + f"candidate rerun step {step.name} log archival failed: " + f"{reader.state.error or 'unmatched begin marker'}", + ) if _state_value(state) != "Success": raise RuntimeApiError( "command_failed", diff --git a/chipcompiler/cli/command_handlers/project.py b/chipcompiler/cli/command_handlers/project.py index 020bf3ae..3ba31286 100644 --- a/chipcompiler/cli/command_handlers/project.py +++ b/chipcompiler/cli/command_handlers/project.py @@ -1,9 +1,18 @@ import contextlib import os -import shlex import shutil import sys +from chipcompiler.cli.command_handlers.workspace_run import ( # noqa: F401 + _load_valid_steps, + _make_run_operation, + _preflight_selected_tools, + _read_flow_data, + _run_flow_via_worker, + _run_worker_calls, + _run_workspace, + _workspace_run_outcome, +) from chipcompiler.cli.core.inputs import CheckInput, InitInput, RunInput from chipcompiler.cli.core.output import disclosure_cmd from chipcompiler.cli.core.records import error_record @@ -187,6 +196,15 @@ def _canonically_inside(path: str, anchor: str) -> bool: return real == real_base or real.startswith(real_base.rstrip(os.sep) + os.sep) +def _worker_binary_missing_error() -> str | None: + from chipcompiler.runtime.worker_operation import _default_worker_argv + + argv = _default_worker_argv() + if not os.path.isfile(argv[0]): + return f"worker binary not found: {argv[0]}" + return None + + def run(command_input: RunInput, ctx: CommandContext) -> CommandResult: if command_input.workspace is not None: return _run_workspace(command_input, ctx) @@ -425,22 +443,32 @@ def run(command_input: RunInput, ctx: CommandContext) -> CommandResult: ) if should_enable_run_progress(ctx, sys.stderr): - flow_ok = run_flow_with_progress(engine_flow, ctx, project, sys.stderr) + op_result = run_flow_with_progress( + run_dir, + ctx, + project, + sys.stderr, + run_operation=lambda **callbacks: _run_flow_via_worker(run_dir, **callbacks), + ) else: - flow_ok = engine_flow.run_steps() + op_result = _run_flow_via_worker(run_dir) + flow_ok = op_result.success if not flow_ok: - return CommandResult.err( - [ - { - "run": run_name, - "status": "failed", - "workspace": run_dir, - "inspect_cmd": disclosure_cmd("ecc status", project, ctx.run_id), - "log": disclosure_cmd("ecc log", project, ctx.run_id), - } - ] - ) + error_record = { + "run": run_name, + "status": "failed", + "workspace": run_dir, + "inspect_cmd": disclosure_cmd("ecc status", project, ctx.run_id), + "log": disclosure_cmd("ecc log", project, ctx.run_id), + } + if op_result.error: + error_record["error"] = op_result.error + if op_result.exit_code is not None: + error_record["exit_code"] = op_result.exit_code + if op_result.repaired_steps: + error_record["repaired_steps"] = op_result.repaired_steps + return CommandResult.err([error_record]) except Exception as exc: return CommandResult.err( [ @@ -465,84 +493,3 @@ def run(command_input: RunInput, ctx: CommandContext) -> CommandResult: } ] ) - - -def _run_workspace(command_input: RunInput, ctx: CommandContext) -> CommandResult: - def error(kind: str, **fields) -> CommandResult: - return CommandResult.err([{"kind": "error", "error": kind, **fields}]) - - if ctx.project is not None or command_input.project.run_id is not None: - return error("project_workspace_conflict") - if command_input.overwrite: - return error("overwrite_requires_project") - if command_input.param_set: - return error("set_requires_project") - selectors = sum( - ( - command_input.resume, - command_input.from_step is not None, - command_input.only is not None, - ) - ) - if selectors > 1: - return error("selector_conflict") - if command_input.force and command_input.only is None: - return error("force_requires_only") - - from chipcompiler.data import load_workspace - from chipcompiler.engine import EngineFlow, rerun - - workspace_path = os.path.abspath(os.path.expanduser(command_input.workspace)) - try: - workspace = load_workspace(workspace_path) - except Exception as exc: - return error("invalid_workspace", workspace=workspace_path, reason=str(exc)) - if workspace is None: - return error("invalid_workspace", workspace=workspace_path) - - try: - engine_flow = EngineFlow(workspace=workspace) - except Exception as exc: - return error("invalid_workspace", workspace=workspace_path, reason=str(exc)) - if not engine_flow.has_init(): - return error("missing_flow", workspace=workspace_path) - - try: - selected = rerun.selected_step_names( - engine_flow, - from_step=command_input.from_step, - only=command_input.only, - force=command_input.force, - ) - except ValueError as exc: - return error("unknown_step", workspace=workspace_path, reason=str(exc)) - - from chipcompiler.cli.rendering.progress import preserve_cli_stdio - - try: - with preserve_cli_stdio(): - if selected: - engine_flow.create_step_workspaces(executable_steps=set(selected)) - if command_input.only is not None: - result = rerun.run_only(engine_flow, command_input.only, force=command_input.force) - elif command_input.from_step is not None: - result = rerun.run_from(engine_flow, command_input.from_step) - else: - result = rerun.run_resume(engine_flow) - except ValueError as exc: - return error("step_unavailable", workspace=workspace_path, reason=str(exc)) - except Exception as exc: - return error("flow_failed", workspace=workspace_path, reason=str(exc)) - - record = { - "run": "workspace", - "status": "success" if result.ok else "failed", - "workspace": workspace_path, - "executed_steps": list(result.executed), - "no_op": result.ok and not result.executed, - } - if result.ok: - return CommandResult.ok([record]) - record["failed_step"] = result.failed - record["resume_cmd"] = f"ecc run --workspace {shlex.quote(workspace_path)} --resume" - return CommandResult.err([record]) diff --git a/chipcompiler/cli/command_handlers/workspace_run.py b/chipcompiler/cli/command_handlers/workspace_run.py new file mode 100644 index 00000000..1ce9fad0 --- /dev/null +++ b/chipcompiler/cli/command_handlers/workspace_run.py @@ -0,0 +1,265 @@ +"""Workspace-run orchestration behind `ecc run --workspace`. + +Owns the worker-call construction, tool preflight, and outcome +reconciliation for `--resume`/`--from`/`--only` selections. The CLI handler +in project.py stays a thin dispatcher. +""" + +import os +import shlex + +from chipcompiler.cli.core.inputs import RunInput +from chipcompiler.cli.core.types import CommandContext, CommandResult + + +def _worker_binary_missing_error() -> str | None: + from chipcompiler.runtime.worker_operation import _default_worker_argv + + argv = _default_worker_argv() + if not os.path.isfile(argv[0]): + return f"worker binary not found: {argv[0]}" + return None + + +def _read_flow_data(workspace_dir: str) -> dict | None: + """Read home/flow.json, tolerating a missing or corrupt file.""" + from chipcompiler.cli.inspection.discovery import CORRUPT_FLOW_JSON, read_flow_json + + flow_data = read_flow_json(workspace_dir) + if flow_data is None or flow_data is CORRUPT_FLOW_JSON: + return None + return flow_data + + +def _load_valid_steps(workspace_dir: str) -> set[tuple[str, str]] | None: + flow_data = _read_flow_data(workspace_dir) + if flow_data is None: + return None + return { + (s["name"], s["tool"]) + for s in flow_data.get("steps", []) + if isinstance(s, dict) and "name" in s and "tool" in s + } + + +def _preflight_selected_tools(engine_flow, selected: list[str]) -> str | None: + """Fail before any mutation if a selected step's tool is unavailable.""" + from chipcompiler.tools.eda import load_eda_module + + tools = { + step["name"]: step.get("tool") + for step in engine_flow.workspace.flow.data.get("steps", []) + if isinstance(step, dict) and "name" in step + } + for name in selected: + tool = tools.get(name) + if not tool: + continue + try: + if load_eda_module(tool, check_dependency=True) is None: + return f"tool unavailable for step {name}: {tool}" + except Exception as exc: + # Dependency checks raise (e.g. yosys is_eda_exist) rather than + # returning None; both shapes mean unavailable. + return f"tool unavailable for step {name}: {tool} ({exc})" + if tool == "sizer": + from chipcompiler.tools.ecc_sizer import is_sizer_runtime_exist + + try: + if not is_sizer_runtime_exist(): + return f"sizer runtime incomplete for step {name}: missing src/sizer_os.tcl" + except Exception as exc: + return f"sizer runtime check failed for step {name}: {exc}" + return None + + +def _make_run_operation(workspace_dir: str, *, on_output=None, on_step_event=None): + """Build a RunOperation for a workspace with step-log archiving wired in.""" + from pathlib import Path + + from chipcompiler.runtime.log_stream import step_log_archive_resolver + from chipcompiler.runtime.worker_operation import RunOperation + + return RunOperation( + workspace_dir=Path(workspace_dir), + flow_json_path=Path(workspace_dir) / "home" / "flow.json", + log_path_resolver=step_log_archive_resolver(workspace_dir), + on_output=on_output, + on_step_event=on_step_event, + valid_steps=_load_valid_steps(workspace_dir), + ) + + +def _run_worker_calls(workspace_dir: str, calls: list[tuple[str, dict]], **callbacks): + """Execute an ordered RPC sequence through the workspace's worker. + + A missing worker binary is a structured failure, never a crash. + """ + from chipcompiler.runtime.worker_operation import OperationResult + + missing = _worker_binary_missing_error() + if missing is not None: + return OperationResult(success=False, error=missing) + + op = _make_run_operation(workspace_dir, **callbacks) + return op.run_sequence(calls) + + +def _run_flow_via_worker(workspace_dir: str, *, on_output=None, on_step_event=None): + """Execute flow.run through an isolated worker process.""" + return _run_worker_calls( + workspace_dir, + [("flow.run", {"rerun": False})], + on_output=on_output, + on_step_event=on_step_event, + ) + + +def _run_workspace(command_input: RunInput, ctx: CommandContext) -> CommandResult: + def error(kind: str, **fields) -> CommandResult: + return CommandResult.err([{"kind": "error", "error": kind, **fields}]) + + if ctx.project is not None or command_input.project.run_id is not None: + return error("project_workspace_conflict") + if command_input.overwrite: + return error("overwrite_requires_project") + if command_input.param_set: + return error("set_requires_project") + selectors = sum( + ( + command_input.resume, + command_input.from_step is not None, + command_input.only is not None, + ) + ) + if selectors > 1: + return error("selector_conflict") + if command_input.force and command_input.only is None: + return error("force_requires_only") + + from chipcompiler.data import load_workspace + from chipcompiler.engine import EngineFlow, rerun + + workspace_path = os.path.abspath(os.path.expanduser(command_input.workspace)) + try: + workspace = load_workspace(workspace_path) + except Exception as exc: + return error("invalid_workspace", workspace=workspace_path, reason=str(exc)) + if workspace is None: + return error("invalid_workspace", workspace=workspace_path) + + try: + engine_flow = EngineFlow(workspace=workspace) + except Exception as exc: + return error("invalid_workspace", workspace=workspace_path, reason=str(exc)) + if not engine_flow.has_init(): + return error("missing_flow", workspace=workspace_path) + + try: + selected = rerun.selected_step_names( + engine_flow, + from_step=command_input.from_step, + only=command_input.only, + force=command_input.force, + ) + except ValueError as exc: + return error("unknown_step", workspace=workspace_path, reason=str(exc)) + + def no_op_result() -> CommandResult: + return CommandResult.ok( + [ + { + "run": "workspace", + "status": "success", + "workspace": workspace_path, + "executed_steps": [], + "no_op": True, + } + ] + ) + + if not selected: + # --only on an already-successful step without --force, or --resume + # with every step successful: nothing to execute. + return no_op_result() + + # Preflight the whole selected suffix before anything is invalidated: + # the first worker call resets and clears the suffix, so discovering an + # unavailable tool mid-sequence would leave the workspace mutated. + tool_error = _preflight_selected_tools(engine_flow, selected) + if tool_error is not None: + return error("config_error", workspace=workspace_path, reason=tool_error) + + target = selected[0] + if command_input.only is not None: + # An executed --only step always reruns with clean artifacts; the + # --force distinction only gates whether a successful step qualifies. + # Downstream steps keep their outputs but are marked Unstart. + calls = [("flow.run_step", {"step": target, "rerun": True, "invalidate_dependents": True})] + else: + # --resume/--from run exactly the selected suffix, step by step. A + # trailing unscoped flow.run would resume from the FIRST non-success + # step — possibly before the --from boundary — so the suffix is + # driven as explicit run_step calls instead. + calls = [("flow.run_step", {"step": target, "rerun": True, "reset_dependents": True})] + calls += [("flow.run_step", {"step": name, "rerun": True}) for name in selected[1:]] + + op_result = _run_worker_calls(workspace_path, calls) + + if op_result.success: + return CommandResult.ok( + [ + { + "run": "workspace", + "status": "success", + "workspace": workspace_path, + "executed_steps": list(selected), + "no_op": False, + } + ] + ) + + executed, failed_step = _workspace_run_outcome(workspace_path, selected) + record = { + "run": "workspace", + "status": "failed", + "workspace": workspace_path, + "executed_steps": executed, + "no_op": False, + "resume_cmd": f"ecc run --workspace {shlex.quote(workspace_path)} --resume", + } + if failed_step is not None: + record["failed_step"] = failed_step + if op_result.error: + record["error"] = op_result.error + if op_result.exit_code is not None: + record["exit_code"] = op_result.exit_code + if op_result.repaired_steps: + record["repaired_steps"] = op_result.repaired_steps + return CommandResult.err([record]) + + +def _workspace_run_outcome( + workspace_path: str, selected: list[str] +) -> tuple[list[str], str | None]: + """Derive executed steps and the failed step from post-run flow.json. + + After a stopped sequence the selected suffix reads as: a Success prefix + that did execute, the step that failed, and an Unstart remainder that was + invalidated but never ran. + """ + flow_data = _read_flow_data(workspace_path) + if flow_data is None: + return [], None + + states = { + record["name"]: record.get("state") + for record in flow_data.get("steps", []) + if isinstance(record, dict) and "name" in record + } + executed = [] + for name in selected: + if states.get(name) != "Success": + return executed, name + executed.append(name) + return executed, None diff --git a/chipcompiler/cli/rendering/progress.py b/chipcompiler/cli/rendering/progress.py index 5241de37..b2b2e476 100644 --- a/chipcompiler/cli/rendering/progress.py +++ b/chipcompiler/cli/rendering/progress.py @@ -1,10 +1,6 @@ -import contextlib -import multiprocessing import os import re import shutil -import sys -import threading import time from chipcompiler.cli.core.output import disclosure_cmd, normalize_state, normalize_step_name @@ -17,8 +13,6 @@ ) from chipcompiler.cli.rendering.pretty import BOLD, CYAN, DIM, GREEN, RED, RESET from chipcompiler.cli.rendering.pretty import style as _style -from chipcompiler.data import StateEnum, log_flow -from chipcompiler.utility.log import flush_cstdio, redirect_stdio_to_file def supports_color(stream, mode, env=None): @@ -43,8 +37,7 @@ def should_enable_run_progress(ctx, stderr): _CONTROL_RE = re.compile(r"[\r\n\t]+") _C0_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") _MULTI_SPACE_RE = re.compile(r" {2,}") -_LOG_POLL_INTERVAL = 0.5 -_LOG_STALE_AFTER = 10.0 +_LIVE_LINE_MIN_INTERVAL = 0.1 def sanitize_log_line(line): @@ -108,132 +101,6 @@ def format_error_context(log_path, context_lines, log_cmd, *, color=True): return "\n".join(lines) + "\n" -def latest_log_line(path): - if not path or not os.path.isfile(path): - return None - try: - with open(path, errors="replace") as f: - lines = f.readlines() - except OSError: - return None - for line in reversed(lines): - sanitized = sanitize_log_line(line) - if sanitized: - return sanitized - return None - - -class _IncrementalLogTail: - def __init__(self, path, step_name, stale_after=10.0): - self.path = path - self.step_name = step_name - self.stale_after = stale_after - self.file_id = None - self.change_id = None - self.fingerprint = None - self.head = None - self.offset = 0 - self.partial = "" - self.started_at = None - self.last_line = None - self.last_update_at = None - - def poll(self, now=None): - now = time.monotonic() if now is None else now - if self.started_at is None: - self.started_at = now - - for line in self._read_new_lines(): - sanitized = sanitize_log_line(line) - if sanitized: - self.last_line = sanitized - self.last_update_at = now - - if self.last_line is None: - elapsed = max(0, now - self.started_at) - return f"running {self.step_name}, waiting for step log {int(elapsed)}s..." - - elapsed = max(0, now - self.last_update_at) - if elapsed >= self.stale_after: - return f"running {self.step_name}, last log {int(elapsed)}s ago: {self.last_line}" - - return self.last_line - - def _read_new_lines(self): - if not self.path or not os.path.isfile(self.path): - return [] - - try: - stat = os.stat(self.path) - except OSError: - return [] - - file_id = (stat.st_dev, stat.st_ino) - change_id = (stat.st_mtime_ns, stat.st_ctime_ns) - replaced = False - if self.file_id == file_id and self.offset > 0: - head = self._head() - if stat.st_size < self.offset: - replaced = True - elif stat.st_size == self.offset: - fingerprint = self._fingerprint(stat.st_size) - replaced = fingerprint != self.fingerprint - elif self.head is not None: - compare_len = min(len(self.head), len(head), self.offset) - replaced = head[:compare_len] != self.head[:compare_len] - if self.file_id != file_id or replaced: - self.file_id = file_id - self.offset = 0 - self.partial = "" - self.change_id = change_id - - try: - with open(self.path, encoding="utf-8", errors="replace") as f: - f.seek(self.offset) - chunk = f.read() - self.offset = f.tell() - except OSError: - return [] - - self.fingerprint = self._fingerprint(stat.st_size) - self.head = self._head() - if not chunk: - return [] - - text = self.partial + chunk - lines = text.splitlines(keepends=True) - if lines and not lines[-1].endswith(("\n", "\r")): - self.partial = lines.pop() - else: - self.partial = "" - - return lines - - def _head(self): - if not self.path or not os.path.isfile(self.path): - return None - try: - with open(self.path, "rb") as f: - return f.read(4096) - except OSError: - return None - - def _fingerprint(self, size): - if not self.path or not os.path.isfile(self.path): - return None - try: - with open(self.path, "rb") as f: - head = f.read(4096) - if size > 4096: - f.seek(max(0, size - 4096)) - tail = f.read(4096) - else: - tail = b"" - except OSError: - return None - return size, head, tail - - def terminal_width(fallback=80): cols, _ = shutil.get_terminal_size(fallback=(fallback, 24)) return max(cols, 1) @@ -255,50 +122,6 @@ def _stable_stream_from(stream): return os.fdopen(dup_fd, "w", encoding=encoding, errors=errors, buffering=1, closefd=True) -@contextlib.contextmanager -def preserve_cli_stdio(): - saved_stdout = sys.stdout - saved_stderr = sys.stderr - saved_stdout_fd = None - saved_stderr_fd = None - - for stream in (sys.stdout, sys.stderr): - with contextlib.suppress(Exception): - stream.flush() - - try: - saved_stdout_fd = os.dup(1) - saved_stderr_fd = os.dup(2) - except OSError: - if saved_stdout_fd is not None: - os.close(saved_stdout_fd) - try: - yield - finally: - sys.stdout = saved_stdout - sys.stderr = saved_stderr - return - - try: - yield - finally: - for stream in (sys.stdout, sys.stderr): - with contextlib.suppress(Exception): - stream.flush() - # Drain C/C++ buffered output while fds still point at the step log, - # or it leaks to the terminal on the next flush after restore. - flush_cstdio() - - try: - os.dup2(saved_stdout_fd, 1) - os.dup2(saved_stderr_fd, 2) - finally: - os.close(saved_stdout_fd) - os.close(saved_stderr_fd) - sys.stdout = saved_stdout - sys.stderr = saved_stderr - - class RunProgressRenderer: def __init__(self, stream, width_fn=None, *, color=False): self._stream = stream @@ -358,151 +181,90 @@ def render_failure_context(self, block): self._stream.flush() -def _monitor_log_progress( - renderer, - log_path, - step_name, - stop_event, - interval=_LOG_POLL_INTERVAL, - stale_after=_LOG_STALE_AFTER, -): - tail = _IncrementalLogTail(log_path, step_name, stale_after=stale_after) - while not stop_event.is_set(): - renderer.running(tail.poll()) - stop_event.wait(interval) - - -def _poll_log(renderer, log_path, stop_event, interval=_LOG_POLL_INTERVAL): - _monitor_log_progress(renderer, log_path, "step", stop_event, interval=interval) - - -def _start_log_monitor( - renderer, - log_path, - step_name, - *, - isolated=False, - interval=_LOG_POLL_INTERVAL, - stale_after=_LOG_STALE_AFTER, -): - if isolated: - ctx = multiprocessing.get_context("fork") - stop_event = ctx.Event() - monitor = ctx.Process( - target=_monitor_log_progress, - args=(renderer, log_path, step_name, stop_event), - kwargs={"interval": interval, "stale_after": stale_after}, - daemon=True, - ) - else: - stop_event = threading.Event() - monitor = threading.Thread( - target=_monitor_log_progress, - args=(renderer, log_path, step_name, stop_event), - kwargs={"interval": interval, "stale_after": stale_after}, - daemon=True, - ) - monitor.start() - return stop_event, monitor - - -def _stop_log_monitor(stop_event, monitor, timeout=2.0): - stop_event.set() - monitor.join(timeout=timeout) - if monitor.is_alive() and hasattr(monitor, "terminate"): - monitor.terminate() - monitor.join(timeout=timeout) - - -def run_flow_with_progress(engine_flow, ctx, project, stderr): +def run_flow_with_progress(workspace_dir, ctx, project, stderr, run_operation): + """Render TTY progress for a worker-driven flow execution. + + run_operation receives the reader callbacks (on_output, on_step_event) + and returns an OperationResult. Step transitions and the live log line + are driven by those callbacks; per-step final states are refreshed from + flow.json on each begin marker and once more when the operation ends. + """ color = supports_color(stderr, ctx.output_mode) progress_stream = _stable_stream_from(stderr) try: renderer = RunProgressRenderer(progress_stream, color=color) - engine_flow.workspace.home.reset() - - run_dir = engine_flow.workspace.directory run_name = ctx.run_id or "default" - renderer.start_run(run_name, run_dir) - - for workspace_step in engine_flow.workspace_steps: - step_token = normalize_step_name(workspace_step.name) - tool = workspace_step.tool - log_path = workspace_step.log.file or "" - - engine_flow.workspace.logger.log_section( - f"{workspace_step.tool} - begin step - {workspace_step.name}" - ) - - renderer.start_step(step_token, tool) - renderer.running("starting step...") - - stop_event, monitor = _start_log_monitor( - renderer, - log_path, - step_token, - isolated=progress_stream is not stderr, - ) - - start = time.time() - - try: - with preserve_cli_stdio(): - if not engine_flow.check_state( - name=workspace_step.name, - tool=workspace_step.tool, - state=StateEnum.Success, - ): - init_log_stream = None - if log_path: - try: - abs_log_path = os.path.abspath(log_path) - os.makedirs(os.path.dirname(abs_log_path) or ".", exist_ok=True) - init_log_stream = redirect_stdio_to_file(abs_log_path) - except OSError: - init_log_stream = None - try: - engine_flow.init_db_engine() - finally: - if init_log_stream is not None: - init_log_stream.close() - state = engine_flow.run_step(workspace_step) - finally: - _stop_log_monitor(stop_event, monitor) - renderer.clear() - - log_flow(workspace=engine_flow.workspace) - engine_flow.workspace.logger.log_section( - f"{workspace_step.tool} - end step - {workspace_step.name}" - ) - - elapsed = time.time() - start - hours = int(elapsed // 3600) - minutes = int((elapsed % 3600) // 60) - seconds = int(elapsed % 60) - runtime = f"{hours}:{minutes:02d}:{seconds:02d}" - - status = normalize_state(state.value) - - rel_log = "" - if log_path: + renderer.start_run(run_name, workspace_dir) + + from chipcompiler.runtime.log_stream import step_log_archive_resolver + + resolve_log = step_log_archive_resolver(workspace_dir) + rendered = set() + live = {"written_at": 0.0} + + def on_output(data: bytes) -> None: + now = time.monotonic() + if now - live["written_at"] < _LIVE_LINE_MIN_INTERVAL: + return + text = sanitize_log_line(data.decode("utf-8", errors="replace")) + if not text: + return + live["written_at"] = now + renderer.running(text) + + def refresh_final_states() -> None: + from chipcompiler.cli.inspection.discovery import CORRUPT_FLOW_JSON, read_flow_json + + flow_data = read_flow_json(workspace_dir) + if flow_data is None or flow_data is CORRUPT_FLOW_JSON: + return + for record in flow_data.get("steps", []): + if not isinstance(record, dict): + continue + name = record.get("name") + tool = record.get("tool") or "" + state = record.get("state") + if not name or (name, tool) in rendered: + continue + if state not in ("Success", "Imcomplete", "Incomplete", "Invalid"): + continue + rendered.add((name, tool)) + runtime = record.get("runtime") or "0:00:00" + step_token = normalize_step_name(name) + log_path = str(resolve_log(name, tool)) try: - rel_log = os.path.relpath(log_path, engine_flow.workspace.directory) + rel_log = os.path.relpath(log_path, workspace_dir) except ValueError: rel_log = log_path - - inspect = disclosure_cmd(f"ecc log {step_token}", project, ctx.run_id) - - is_success = state == StateEnum.Success - renderer.finish_step(step_token, tool, status, runtime, rel_log, inspect, is_success) - - if not is_success: - _maybe_render_failure_context( - renderer, log_path, rel_log, step_token, project, ctx.run_id, color + inspect = disclosure_cmd(f"ecc log {step_token}", project, ctx.run_id) + success = state == "Success" + renderer.finish_step( + step_token, + tool, + normalize_state(state), + runtime, + rel_log, + inspect, + success, ) - return False + if not success: + _maybe_render_failure_context( + renderer, log_path, rel_log, step_token, project, ctx.run_id, color + ) + + def on_step_event(event: str, step: str, tool: str) -> None: + if event != "begin": + return + # The previous step's final state is persisted before this begin + # marker is written, so flow.json already reflects it here. + refresh_final_states() + renderer.start_step(normalize_step_name(step), tool) + renderer.running("starting step...") - return True + result = run_operation(on_output=on_output, on_step_event=on_step_event) + refresh_final_states() + renderer.clear() + return result finally: if progress_stream is not stderr: progress_stream.close() diff --git a/chipcompiler/engine/flow.py b/chipcompiler/engine/flow.py index 3a0175a2..b241a9c5 100644 --- a/chipcompiler/engine/flow.py +++ b/chipcompiler/engine/flow.py @@ -1,19 +1,16 @@ #!/usr/bin/env python -import hashlib import logging import os -import time -from threading import Event, Thread -from chipcompiler.data import EccOutput, StateEnum, StepEnum, Workspace, WorkspaceStep, log_flow +from chipcompiler.data import EccOutput, StateEnum, StepEnum, Workspace, WorkspaceStep from chipcompiler.engine import EngineDB +from chipcompiler.engine.runner import EngineFlowRunner from chipcompiler.engine.signoff import ( SignoffPackageCollector, SignoffPackageOptions, SignoffPackageResult, ) -from chipcompiler.utility.log import redirect_stdio_to_file logger = logging.getLogger(__name__) @@ -28,33 +25,13 @@ StepEnum.LEGALIZATION.value, StepEnum.ROUTING.value, StepEnum.DRC.value, + StepEnum.LVS.value, StepEnum.FILLER.value, } ) -def get_process_rss_mb(pid: int) -> float: - peak_memory = 0 - try: - with open(f"/proc/{pid}/status") as f: - for line in f: - if line.startswith("VmRSS:"): - rss_kb = int(line.split()[1]) - peak_memory = rss_kb / 1024 - break - except (OSError, ValueError): - pass - return peak_memory - - -def track_current_process_memory(pid: int, stop_event: Event, peak_memory: list[float]): - while not stop_event.is_set(): - peak_memory[0] = max(peak_memory[0], get_process_rss_mb(pid)) - stop_event.wait(0.1) - peak_memory[0] = max(peak_memory[0], get_process_rss_mb(pid)) - - -class EngineFlow: +class EngineFlow(EngineFlowRunner): def __init__(self, workspace: Workspace, engine_db: EngineDB = None): self.workspace = workspace self.workspace_steps = [] @@ -172,6 +149,7 @@ def set_state( tool, state_value, ) + return False return True return False @@ -202,7 +180,6 @@ def check_step_result(self, workspace_step: WorkspaceStep): """ check step output exist """ - import os success = False output = workspace_step.output @@ -342,298 +319,3 @@ def init_db_engine(self) -> bool: break return self.engine_db.create_db_engine(step=workspace_step) - - def clear_db_engine_after_step(self, workspace_step: WorkspaceStep, state: StateEnum) -> None: - if workspace_step.tool == "sizer" and state == StateEnum.Success: - engine_db = self.engine_db - self.engine_db = None - if engine_db is not None: - close = getattr(engine_db, "close", None) - if callable(close): - close() - - def timing_constraint_facts(self) -> dict: - sdc_path = self.workspace.pdk.sdc - if sdc_path is None: - return {"availability": "missing_source"} - - try: - path = os.fspath(sdc_path) - size_bytes = os.path.getsize(path) - digest = hashlib.sha256() - with open(path, "rb") as sdc_file: - for chunk in iter(lambda: sdc_file.read(1024 * 1024), b""): - digest.update(chunk) - except OSError: - return {"availability": "unreadable"} - - return { - "availability": "available", - "sha256": digest.hexdigest(), - "size_bytes": size_bytes, - } - - def save_step_flow_facts( - self, - workspace_step: WorkspaceStep, - state: StateEnum, - runtime_seconds: float, - peak_memory_mb: float, - timing_constraints: dict, - ) -> bool: - feature_path = getattr(workspace_step.feature, "step", None) - if feature_path is None or feature_path == "": - return False - - from chipcompiler.utility import JsonReadError, json_read_strict, json_write - - try: - existing = json_read_strict(feature_path) - except (FileNotFoundError, JsonReadError): - existing = {} - payload = existing if isinstance(existing, dict) else {} - payload["run"] = { - "state": state.value, - "runtime_seconds": round(runtime_seconds, 3), - "peak_memory_mb": round(peak_memory_mb, 3), - } - payload["constraints"] = {"sdc": timing_constraints} - return json_write(file_path=feature_path, data=payload) - - return True - - def run_steps(self, *, rerun: bool = False, observer=None) -> bool: - """ - run all flow steps - """ - - for workspace_step in self.workspace_steps: - self.workspace.logger.log_section( - f"{workspace_step.tool} - begin step - {workspace_step.name}" - ) - self.init_db_engine() - state = ( - self.run_step(workspace_step, rerun=rerun) - if observer is None - else self.run_step(workspace_step, rerun=rerun, observer=observer) - ) - - log_flow(workspace=self.workspace) - self.workspace.logger.log_section( - f"{workspace_step.tool} - end step - {workspace_step.name}" - ) - - match state: - case StateEnum.Success: - continue - case StateEnum.Invalid: - return False - case StateEnum.Unstart: - return False - case StateEnum.Imcomplete: - return False - case StateEnum.Pending: - return False - case StateEnum.Ongoing: - return False - - total_steps = len(self.workspace.flow.data.get("steps", [])) - if len(self.workspace_steps) < total_steps: - self.workspace.logger.error( - "Flow incomplete: %d of %d steps were created; remaining steps could not be set up", - len(self.workspace_steps), - total_steps, - ) - return False - - return True - - def run_step( - self, - workspace_step: WorkspaceStep | str, - *, - rerun: bool = False, - observer=None, - ) -> StateEnum: - """ - run single step - """ - if isinstance(workspace_step, str): - workspace_step = self.get_workspace_step(workspace_step) - if workspace_step is None: - return StateEnum.Invalid - - step_tag = f"{workspace_step.name}({workspace_step.tool})" - - if not rerun and self.check_state( - name=workspace_step.name, tool=workspace_step.tool, state=StateEnum.Success - ): - self.workspace.logger.info("[SKIP] %s already succeeded", step_tag) - self.clear_db_engine_after_step(workspace_step, StateEnum.Success) - _notify_flow_observer(observer, "on_step_skipped", workspace_step) - return StateEnum.Success - - # set state ongoing - start_time = time.time() - timing_constraints = self.timing_constraint_facts() - self.set_state(name=workspace_step.name, tool=workspace_step.tool, state=StateEnum.Ongoing) - _notify_flow_observer(observer, "on_step_started", workspace_step) - - # run step - log_file = workspace_step.log.file or "" - if log_file: - log_file = os.path.abspath(log_file) - try: - os.makedirs(os.path.dirname(log_file) or ".", exist_ok=True) - redirect_stdio_to_file(log_file) - except Exception: - logger.exception("Failed to redirect stdio to log file: %s", log_file) - - step_tag = f"{workspace_step.name}({workspace_step.tool})" - self.workspace.logger.info(f"[STEP] {step_tag} pid={os.getpid()} started") - - pid = os.getpid() - start_memory_mb = get_process_rss_mb(pid) - peak_memory = [start_memory_mb] - stop_memory_monitor = Event() - memory_monitor = Thread( - target=track_current_process_memory, - args=(pid, stop_memory_monitor, peak_memory), - daemon=True, - ) - memory_monitor.start() - previous_observer = getattr(self.workspace, "_runtime_flow_observer", None) - if observer is not None: - self.workspace._runtime_flow_observer = observer - step_raised_exception = False - try: - from chipcompiler.tools import run_step as run_tool_step - - result = run_tool_step( - workspace=self.workspace, step=workspace_step, ecc_module=self.engine_db.engine - ) - self.workspace.logger.info(f"[STEP] {step_tag} finished result={result}") - except Exception: - step_raised_exception = True - self.workspace.logger.error(f"[STEP] {step_tag} failed with exception") - self.workspace.logger.exception(f"[STEP] {step_tag} exception details") - finally: - stop_memory_monitor.set() - memory_monitor.join() - if observer is not None: - if previous_observer is None: - delattr(self.workspace, "_runtime_flow_observer") - else: - self.workspace._runtime_flow_observer = previous_observer - - # compute metrics - peak_memory_mb = peak_memory[0] - start_memory_mb - peak_memory_mb = 0 if peak_memory_mb < 0 else round(peak_memory_mb, 3) - elapsed = time.time() - start_time - runtime = f"{int(elapsed // 3600)}:{int((elapsed % 3600) // 60)}:{int(elapsed % 60)}" - - # determine and save state - if step_raised_exception: - state = StateEnum.Imcomplete - else: - state = ( - StateEnum.Success - if self.check_step_result(workspace_step=workspace_step) - else StateEnum.Imcomplete - ) - - self.set_state( - name=workspace_step.name, - tool=workspace_step.tool, - state=state, - runtime=runtime, - peak_memory=peak_memory_mb, - ) - self.workspace.logger.info( - "[RESULT] %s state=%s runtime=%s mem=%sMB exitcode=%s", - step_tag, - state.value, - runtime, - peak_memory_mb, - 0, - ) - - # save layout snapshot on success - if state == StateEnum.Success: - if self.save_step_flow_facts( - workspace_step=workspace_step, - state=state, - runtime_seconds=elapsed, - peak_memory_mb=peak_memory_mb, - timing_constraints=timing_constraints, - ): - try: - from chipcompiler.tools import build_step_metrics - - if build_step_metrics(workspace=self.workspace, step=workspace_step) is None: - self.workspace.logger.warning( - "[QOR] %s run facts were saved but analysis refresh is unavailable", - step_tag, - ) - except Exception: - self.workspace.logger.exception( - "[QOR] %s failed to refresh analysis after saving run facts", - step_tag, - ) - else: - self.workspace.logger.warning( - "[QOR] %s has no step feature path; run facts were not saved", - step_tag, - ) - from chipcompiler.tools import save_layout_image - - save_layout_image(workspace=self.workspace, step=workspace_step) - - self.clear_db_engine_after_step(workspace_step, state) - _notify_flow_observer(observer, "on_step_completed", workspace_step, state) - if state == StateEnum.Success and not _wait_for_step_rendered( - observer, - workspace_step, - state, - ): - return StateEnum.Invalid - - return state - - def init_db_engine_for_step(self, workspace_step: WorkspaceStep) -> bool: - """Initialize the native DB engine from an explicitly selected step.""" - if self.engine_db is None: - self.engine_db = EngineDB(workspace=self.workspace) - elif self.engine_db.has_init(): - return True - - return self.engine_db.create_db_engine(step=workspace_step) - - -def _notify_flow_observer(observer, method_name: str, *args) -> None: - """Keep optional GUI observers outside the flow engine's failure domain.""" - if observer is None: - return - callback = getattr(observer, method_name, None) - if not callable(callback): - return - try: - callback(*args) - except Exception: - # Runtime observers must never turn a completed tool execution into a - # failed flow. The coordinator records transport failures separately. - logging.getLogger(__name__).exception("flow observer callback failed: %s", method_name) - - -def _wait_for_step_rendered(observer, workspace_step: WorkspaceStep, state: StateEnum) -> bool: - if observer is None or state != StateEnum.Success: - return True - callback = getattr(observer, "wait_for_step_rendered", None) - if not callable(callback): - return True - try: - return bool(callback(workspace_step, state)) - except Exception: - # Fail-open: observer bugs must not invalidate successful tool results. - logging.getLogger(__name__).exception("flow observer render gate failed") - return True diff --git a/chipcompiler/engine/rerun.py b/chipcompiler/engine/rerun.py index 0885fded..1add3d8f 100644 --- a/chipcompiler/engine/rerun.py +++ b/chipcompiler/engine/rerun.py @@ -7,21 +7,16 @@ from an existing workspace; execution itself stays in ``EngineFlow.run_step``. """ -import logging -import os import shutil from pathlib import Path from typing import TYPE_CHECKING, NamedTuple from chipcompiler.data import StateEnum, Workspace, WorkspaceStep, log_flow -from chipcompiler.utility.log import redirect_stdio_to_file from chipcompiler.utility.path import path_is_within if TYPE_CHECKING: from chipcompiler.engine.flow import EngineFlow -logger = logging.getLogger(__name__) - class StepRunResult(NamedTuple): ok: bool @@ -118,25 +113,83 @@ def _run_selected(flow: "EngineFlow", selected: list[tuple[WorkspaceStep, Path]] if flow.engine_db is not None: flow.engine_db.close() + # Direct in-process runs are executor and client in one process: route + # the own fd 1/2 stream through the archiver so step bytes land in + # per-step logs and markers never reach the caller's terminal. + from chipcompiler.runtime.log_stream import archive_own_step_logs + executed = [] - for workspace_step, output_dir in selected: - flow.workspace.logger.log_section( - f"{workspace_step.tool} - begin step - {workspace_step.name}" - ) - _reset_output_dir(output_dir) - _redirect_to_step_log(workspace_step) - flow.init_db_engine_for_step(workspace_step) - state = flow.run_step(workspace_step, rerun=True) - log_flow(workspace=flow.workspace) - flow.workspace.logger.log_section( - f"{workspace_step.tool} - end step - {workspace_step.name}" - ) - if state != StateEnum.Success: - return StepRunResult(ok=False, executed=tuple(executed), failed=workspace_step.name) - executed.append(workspace_step.name) + failed = None + reader = None + try: + with archive_own_step_logs(flow.workspace.directory) as active_reader: + reader = active_reader + for workspace_step, output_dir in selected: + flow.workspace.logger.log_section( + f"{workspace_step.tool} - begin step - {workspace_step.name}" + ) + _reset_output_dir(output_dir) + flow.init_db_engine_for_step(workspace_step) + state = flow.run_step(workspace_step, rerun=True) + log_flow(workspace=flow.workspace) + flow.workspace.logger.log_section( + f"{workspace_step.tool} - end step - {workspace_step.name}" + ) + if state != StateEnum.Success: + failed = workspace_step.name + break + executed.append(workspace_step.name) + except BaseException: + # A step that raised after its begin marker (post-processing, marker + # write) still needs archive reconciliation before propagating. + if reader is not None: + downgrade_unarchived_step(flow, reader, executed) + raise + + # The reader drained at context exit. An archive failure or an unmatched + # begin must not leave a Success record whose log is missing: downgrade + # the affected step so a later resume reruns it and rebuilds the archive. + if reader is not None and ( + reader.state.error is not None or reader.state.active_step is not None + ): + target = downgrade_unarchived_step(flow, reader, executed) + return StepRunResult(ok=False, executed=tuple(executed), failed=failed or target) + if failed is not None: + return StepRunResult(ok=False, executed=tuple(executed), failed=failed) return StepRunResult(ok=True, executed=tuple(executed)) +def downgrade_unarchived_step(flow: "EngineFlow", reader, executed: list[str]) -> str | None: + """Downgrade the step whose archive failed or whose end marker never came. + + Uses set_state so the downgrade persists through the single authoritative + save. If that save fails, the disk record still claims Success — log the + incomplete repair explicitly so the stale record is not silently trusted. + """ + target = reader.state.error_step or reader.state.active_step + target_tool = reader.state.error_tool or reader.state.active_tool + if target is None and executed: + target = executed[-1] + target_tool = None + if target is None: + return None + for record in flow.workspace.flow.data.get("steps", []): + if record.get("name") != target: + continue + # A flow may carry the same step name under two tools; only the + # evidenced (name, tool) pair may be downgraded. + if target_tool is not None and record.get("tool") != target_tool: + continue + if not flow.set_state(target, record.get("tool", ""), StateEnum.Imcomplete): + flow.workspace.logger.error( + "archive downgrade for step %s could not be persisted; " + "flow.json may still claim Success — repair it before resuming", + target, + ) + break + return target + + def _validated_output_dirs(workspace: Workspace, steps: list[WorkspaceStep]) -> list[Path]: """Validate that each step output is its canonical ``/output`` dir. @@ -166,19 +219,6 @@ def _validated_output_dirs(workspace: Workspace, steps: list[WorkspaceStep]) -> return output_dirs -def _redirect_to_step_log(workspace_step: WorkspaceStep) -> None: - """Redirect stdio before DB init so its warnings land in the step log.""" - log_file = workspace_step.log.file or "" - if not log_file: - return - log_file = os.path.abspath(log_file) - try: - os.makedirs(os.path.dirname(log_file) or ".", exist_ok=True) - redirect_stdio_to_file(log_file) - except Exception: - logger.exception("Failed to redirect stdio to log file: %s", log_file) - - def _reset_output_dir(output_dir: Path) -> None: if output_dir.exists(): shutil.rmtree(output_dir) diff --git a/chipcompiler/engine/runner.py b/chipcompiler/engine/runner.py new file mode 100644 index 00000000..b0c137ed --- /dev/null +++ b/chipcompiler/engine/runner.py @@ -0,0 +1,426 @@ +"""Step execution lifecycle for EngineFlow. + +Owns the protocol-critical ordering for a single step or a full flow: +markers, memory tracking, the authoritative final save, post-processing, +db cleanup, observer callbacks, and the render gate. EngineFlow inherits +this mixin; the data/state/build methods stay in flow.py. +""" + +import hashlib +import logging +import os +import time +from contextlib import nullcontext +from threading import Event, Thread + +from chipcompiler.data import StateEnum, WorkspaceStep, log_flow +from chipcompiler.engine import EngineDB + + +def get_process_rss_mb(pid: int) -> float: + peak_memory = 0 + try: + with open(f"/proc/{pid}/status") as f: + for line in f: + if line.startswith("VmRSS:"): + rss_kb = int(line.split()[1]) + peak_memory = rss_kb / 1024 + break + except (OSError, ValueError): + pass + return peak_memory + + +def track_current_process_memory(pid: int, stop_event: Event, peak_memory: list[float]): + while not stop_event.is_set(): + peak_memory[0] = max(peak_memory[0], get_process_rss_mb(pid)) + stop_event.wait(0.1) + peak_memory[0] = max(peak_memory[0], get_process_rss_mb(pid)) + + +class EngineFlowRunner: + """Mixin: step/flow execution lifecycle; requires the EngineFlow spine.""" + + def clear_db_engine_after_step(self, workspace_step: WorkspaceStep, state: StateEnum) -> None: + if workspace_step.tool == "sizer" and state == StateEnum.Success: + engine_db = self.engine_db + self.engine_db = None + if engine_db is not None: + close = getattr(engine_db, "close", None) + if callable(close): + close() + + def timing_constraint_facts(self) -> dict: + sdc_path = self.workspace.pdk.sdc + if sdc_path is None: + return {"availability": "missing_source"} + + try: + path = os.fspath(sdc_path) + size_bytes = os.path.getsize(path) + digest = hashlib.sha256() + with open(path, "rb") as sdc_file: + for chunk in iter(lambda: sdc_file.read(1024 * 1024), b""): + digest.update(chunk) + except OSError: + return {"availability": "unreadable"} + + return { + "availability": "available", + "sha256": digest.hexdigest(), + "size_bytes": size_bytes, + } + + def save_step_flow_facts( + self, + workspace_step: WorkspaceStep, + state: StateEnum, + runtime_seconds: float, + peak_memory_mb: float, + timing_constraints: dict, + ) -> bool: + feature_path = getattr(workspace_step.feature, "step", None) + if feature_path is None or feature_path == "": + return False + + from chipcompiler.utility import JsonReadError, json_read_strict, json_write + + try: + existing = json_read_strict(feature_path) + except (FileNotFoundError, JsonReadError): + existing = {} + payload = existing if isinstance(existing, dict) else {} + payload["run"] = { + "state": state.value, + "runtime_seconds": round(runtime_seconds, 3), + "peak_memory_mb": round(peak_memory_mb, 3), + } + payload["constraints"] = {"sdc": timing_constraints} + return json_write(file_path=feature_path, data=payload) + + return True + + def run_steps(self, *, rerun: bool = False, observer=None) -> bool: + """ + run all flow steps + """ + from chipcompiler.runtime.log_stream import archive_own_step_logs + + from .rerun import downgrade_unarchived_step + + # Direct in-process runs (documented Python API) self-archive so step + # logs exist and markers stay off the caller's terminal; inside a + # worker/sidecar process the outer client owns the stream and this + # context passes through. + succeeded = True + directory = self.workspace.directory + with archive_own_step_logs(directory) if directory is not None else nullcontext() as reader: + try: + for workspace_step in self.workspace_steps: + self.workspace.logger.log_section( + f"{workspace_step.tool} - begin step - {workspace_step.name}" + ) + self.init_db_engine() + state = ( + self.run_step(workspace_step, rerun=rerun) + if observer is None + else self.run_step(workspace_step, rerun=rerun, observer=observer) + ) + + log_flow(workspace=self.workspace) + self.workspace.logger.log_section( + f"{workspace_step.tool} - end step - {workspace_step.name}" + ) + + match state: + case StateEnum.Success: + continue + case _: + succeeded = False + break + except BaseException: + # Reconcile archive evidence before the exception propagates. + if reader is not None: + downgrade_unarchived_step(self, reader, []) + raise + + # An archive failure or unmatched begin must not report success over a + # missing step log; reconcile after the reader drains. (reader is None + # when an outer client owns the stream — nothing to reconcile here.) + if reader is not None and ( + reader.state.error is not None or reader.state.active_step is not None + ): + downgrade_unarchived_step(self, reader, []) + succeeded = False + if not succeeded: + return False + + total_steps = len(self.workspace.flow.data.get("steps", [])) + if len(self.workspace_steps) < total_steps: + self.workspace.logger.error( + "Flow incomplete: %d of %d steps were created; remaining steps could not be set up", + len(self.workspace_steps), + total_steps, + ) + return False + + return True + + def run_step( + self, + workspace_step: WorkspaceStep | str, + *, + rerun: bool = False, + observer=None, + ) -> StateEnum: + """ + run single step + """ + if isinstance(workspace_step, str): + workspace_step = self.get_workspace_step(workspace_step) + if workspace_step is None: + return StateEnum.Invalid + + from chipcompiler.runtime.log_stream import archive_own_step_logs + + from .rerun import downgrade_unarchived_step + + # Direct callers get client-side archival too; inside a worker/sidecar + # process or an explicit archive context this passes through. + if self.workspace.directory is None: + return self._run_step_body(workspace_step, rerun=rerun, observer=observer) + reader = None + try: + with archive_own_step_logs(self.workspace.directory) as active_reader: + reader = active_reader + state = self._run_step_body(workspace_step, rerun=rerun, observer=observer) + except BaseException: + # Reconcile archive evidence before the exception propagates. + if reader is not None: + downgrade_unarchived_step(self, reader, []) + raise + # An archive failure or unmatched begin must not report Success over a + # missing log; reconcile after the reader drains. + if reader is not None and ( + reader.state.error is not None or reader.state.active_step is not None + ): + downgrade_unarchived_step(self, reader, []) + return StateEnum.Imcomplete + return state + + def _run_step_body( + self, + workspace_step: WorkspaceStep, + *, + rerun: bool = False, + observer=None, + ) -> StateEnum: + step_tag = f"{workspace_step.name}({workspace_step.tool})" + + if not rerun and self.check_state( + name=workspace_step.name, tool=workspace_step.tool, state=StateEnum.Success + ): + self.workspace.logger.info("[SKIP] %s already succeeded", step_tag) + self.clear_db_engine_after_step(workspace_step, StateEnum.Success) + _notify_flow_observer(observer, "on_step_skipped", workspace_step) + return StateEnum.Success + + # set state ongoing + start_time = time.time() + timing_constraints = self.timing_constraint_facts() + self.set_state(name=workspace_step.name, tool=workspace_step.tool, state=StateEnum.Ongoing) + _notify_flow_observer(observer, "on_step_started", workspace_step) + + self.workspace.logger.info(f"[STEP] {step_tag} pid={os.getpid()} started") + + from chipcompiler.runtime.log_stream import emit_step_marker + + try: + emit_step_marker("begin", step=workspace_step.name, tool=workspace_step.tool) + except OSError: + # fd 2 is closed or the reader pipe is broken: the marker never + # reached any client, so recovery could never identify this step + # from the stream. Downgrade the persisted Ongoing now instead of + # leaving a permanent Ongoing no repair pass can find. + self.set_state( + name=workspace_step.name, tool=workspace_step.tool, state=StateEnum.Imcomplete + ) + raise + + pid = os.getpid() + start_memory_mb = get_process_rss_mb(pid) + peak_memory = [start_memory_mb] + stop_memory_monitor = Event() + memory_monitor = Thread( + target=track_current_process_memory, + args=(pid, stop_memory_monitor, peak_memory), + daemon=True, + ) + memory_monitor.start() + previous_observer = getattr(self.workspace, "_runtime_flow_observer", None) + if observer is not None: + self.workspace._runtime_flow_observer = observer + step_raised_exception = False + result = None + try: + result = self._invoke_step_tool(workspace_step) + self.workspace.logger.info(f"[STEP] {step_tag} finished result={result}") + except Exception: + step_raised_exception = True + self.workspace.logger.error(f"[STEP] {step_tag} failed with exception") + self.workspace.logger.exception(f"[STEP] {step_tag} exception details") + finally: + stop_memory_monitor.set() + memory_monitor.join() + if observer is not None: + if previous_observer is None: + delattr(self.workspace, "_runtime_flow_observer") + else: + self.workspace._runtime_flow_observer = previous_observer + + # compute metrics + peak_memory_mb = peak_memory[0] - start_memory_mb + peak_memory_mb = 0 if peak_memory_mb < 0 else round(peak_memory_mb, 3) + elapsed = time.time() - start_time + runtime = f"{int(elapsed // 3600)}:{int((elapsed % 3600) // 60)}:{int(elapsed % 60)}" + + # determine and save state + state = self._derive_step_state(workspace_step, result, raised=step_raised_exception) + + persisted = self.set_state( + name=workspace_step.name, + tool=workspace_step.tool, + state=state, + runtime=runtime, + peak_memory=peak_memory_mb, + ) + if not persisted: + # The marker protocol guarantees the final state is persisted + # before the end marker; a failed save makes the run's result + # untrustworthy. Downgrade the canonical in-memory record (the + # downgrade itself is not persisted — the save just failed), + # suppress the end marker, and report the step incomplete. + state = StateEnum.Imcomplete + record = self.get_step(workspace_step.name, workspace_step.tool) + if record is not None: + record["state"] = StateEnum.Imcomplete.value + self.workspace.logger.error( + "[RESULT] %s final state could not be persisted; marking step Imcomplete", + step_tag, + ) + self.workspace.logger.info( + "[RESULT] %s state=%s runtime=%s mem=%sMB exitcode=%s", + step_tag, + state.value, + runtime, + peak_memory_mb, + 0, + ) + + # save layout snapshot on success + if state == StateEnum.Success: + if self.save_step_flow_facts( + workspace_step=workspace_step, + state=state, + runtime_seconds=elapsed, + peak_memory_mb=peak_memory_mb, + timing_constraints=timing_constraints, + ): + try: + from chipcompiler.tools import build_step_metrics + + if build_step_metrics(workspace=self.workspace, step=workspace_step) is None: + self.workspace.logger.warning( + "[QOR] %s run facts were saved but analysis refresh is unavailable", + step_tag, + ) + except Exception: + self.workspace.logger.exception( + "[QOR] %s failed to refresh analysis after saving run facts", + step_tag, + ) + else: + self.workspace.logger.warning( + "[QOR] %s has no step feature path; run facts were not saved", + step_tag, + ) + from chipcompiler.tools import save_layout_image + + save_layout_image(workspace=self.workspace, step=workspace_step) + + self.clear_db_engine_after_step(workspace_step, state) + # The end marker closes the step's byte stream only after every + # step-scoped write (state persistence, [RESULT], QOR, layout, db + # cleanup) has flushed, and always before the completion notification. + # When the final state could not be persisted, the marker stays + # unwritten: consumers treat the step as crashed and repair its state. + if persisted: + emit_step_marker("end", step=workspace_step.name, tool=workspace_step.tool) + _notify_flow_observer(observer, "on_step_completed", workspace_step, state) + if state == StateEnum.Success and not _wait_for_step_rendered( + observer, + workspace_step, + state, + ): + return StateEnum.Invalid + + return state + + def init_db_engine_for_step(self, workspace_step: WorkspaceStep) -> bool: + """Initialize the native DB engine from an explicitly selected step.""" + if self.engine_db is None: + self.engine_db = EngineDB(workspace=self.workspace) + elif self.engine_db.has_init(): + return True + + return self.engine_db.create_db_engine(step=workspace_step) + + def _invoke_step_tool(self, workspace_step: WorkspaceStep): + """Run the step's tool. Subclasses redirect to their own runner.""" + from chipcompiler.tools import run_step as run_tool_step + + return run_tool_step( + workspace=self.workspace, step=workspace_step, ecc_module=self.engine_db.engine + ) + + def _derive_step_state( + self, workspace_step: WorkspaceStep, result, *, raised: bool + ) -> StateEnum: + """Map the tool result to the step state. Subclasses keep their own + result vocabulary; the base engine trusts the artifact check.""" + if raised: + return StateEnum.Imcomplete + return ( + StateEnum.Success + if self.check_step_result(workspace_step=workspace_step) + else StateEnum.Imcomplete + ) + + +def _notify_flow_observer(observer, method_name: str, *args) -> None: + """Keep optional GUI observers outside the flow engine's failure domain.""" + if observer is None: + return + callback = getattr(observer, method_name, None) + if not callable(callback): + return + try: + callback(*args) + except Exception: + # Runtime observers must never turn a completed tool execution into a + # failed flow. The coordinator records transport failures separately. + logging.getLogger(__name__).exception("flow observer callback failed: %s", method_name) + + +def _wait_for_step_rendered(observer, workspace_step: WorkspaceStep, state: StateEnum) -> bool: + if observer is None or state != StateEnum.Success: + return True + callback = getattr(observer, "wait_for_step_rendered", None) + if not callable(callback): + return True + try: + return bool(callback(workspace_step, state)) + except Exception: + # Fail-open: observer bugs must not invalidate successful tool results. + logging.getLogger(__name__).exception("flow observer render gate failed") + return True diff --git a/chipcompiler/runtime/events.py b/chipcompiler/runtime/events.py deleted file mode 100644 index de7aaff9..00000000 --- a/chipcompiler/runtime/events.py +++ /dev/null @@ -1,51 +0,0 @@ -import os -import sys -from contextlib import contextmanager, suppress - - -@contextmanager -def redirect_stdout_to_stderr(): - saved_stdout = sys.stdout - saved_stderr = sys.stderr - saved_stdout_fd = None - saved_stderr_fd = None - - with suppress(Exception): - sys.stdout.flush() - with suppress(Exception): - sys.stderr.flush() - - try: - saved_stdout_fd = os.dup(1) - saved_stderr_fd = os.dup(2) - os.dup2(2, 1) - sys.stdout = sys.stderr - except OSError: - with suppress(Exception): - if saved_stdout_fd is not None: - os.close(saved_stdout_fd) - if saved_stderr_fd is not None: - os.close(saved_stderr_fd) - sys.stdout = sys.stderr - try: - yield - finally: - sys.stdout = saved_stdout - sys.stderr = saved_stderr - return - - try: - yield - finally: - with suppress(Exception): - sys.stdout.flush() - with suppress(Exception): - sys.stderr.flush() - try: - os.dup2(saved_stdout_fd, 1) - os.dup2(saved_stderr_fd, 2) - finally: - os.close(saved_stdout_fd) - os.close(saved_stderr_fd) - sys.stdout = saved_stdout - sys.stderr = saved_stderr diff --git a/chipcompiler/runtime/log_stream.py b/chipcompiler/runtime/log_stream.py new file mode 100644 index 00000000..1fdc081a --- /dev/null +++ b/chipcompiler/runtime/log_stream.py @@ -0,0 +1,485 @@ +"""Step marker protocol and log stream archive. + +The worker emits step markers on stderr using a Record Separator prefix: + \\x1eECC-STEP {"v":1,"event":"begin","step":"Synthesis","tool":"yosys"}\\n + \\x1eECC-STEP {"v":1,"event":"end","step":"Synthesis","tool":"yosys"}\\n + +The client-side LogStreamReader drains worker stderr, parses markers to +switch between step log files, and archives raw tool bytes to the correct +step log path. +""" + +import json +import os +import threading +from collections.abc import Callable +from contextlib import contextmanager, suppress +from dataclasses import dataclass, field +from pathlib import Path +from typing import BinaryIO + +from chipcompiler.utility.path import path_is_within + +MARKER_PREFIX = b"\x1eECC-STEP " +MARKER_VERSION = 1 + + +@dataclass +class StepMarker: + event: str + step: str + tool: str + + +def emit_step_marker(event: str, step: str, tool: str) -> None: + """Write a step marker to stderr using a single os.write() call.""" + import sys + + from chipcompiler.utility.log import flush_cstdio + + sys.stdout.flush() + sys.stderr.flush() + # C/C++ buffers must drain before the marker, or pending native output + # lands on the wrong side of the step boundary. + flush_cstdio() + payload = json.dumps( + {"v": MARKER_VERSION, "event": event, "step": step, "tool": tool}, + separators=(",", ":"), + ) + line = MARKER_PREFIX + payload.encode("utf-8") + b"\n" + os.write(2, line) + + +def step_log_archive_resolver(workspace_dir) -> Callable[[str, str], Path]: + """Resolve the archive path for a step's tool log inside a workspace.""" + base = Path(workspace_dir) + + def resolve(step: str, tool: str) -> Path: + # Mirror the step-directory layout the builders create: the sizer + # builder sanitizes its directory name (whitespace runs become + # underscores, lowercased) while the other builders use the raw + # "_" form. + if tool == "sizer": + directory = f"{'_'.join(step.split()).lower()}_sizer" + else: + directory = f"{step}_{tool}" + return base / directory / "log" / f"{step}.log" + + return resolve + + +def parse_marker(line: bytes) -> StepMarker | None: + """Parse a complete line as a step marker, or return None if invalid.""" + if not line.startswith(MARKER_PREFIX): + return None + payload = line[len(MARKER_PREFIX) :] + if payload.endswith(b"\n"): + payload = payload[:-1] + try: + data = json.loads(payload) + except (json.JSONDecodeError, UnicodeDecodeError): + return None + if not isinstance(data, dict): + return None + version = data.get("v") + # JSON true/false are bool in Python; bool == 1 is True, so exclude it. + if isinstance(version, bool) or version != MARKER_VERSION: + return None + event = data.get("event") + step = data.get("step") + tool = data.get("tool") + if not isinstance(event, str) or not isinstance(step, str) or not isinstance(tool, str): + return None + return StepMarker(event=event, step=step, tool=tool) + + +@dataclass +class LogStreamState: + """Mutable state maintained by the log stream reader.""" + + tail_bytes: bytes = b"" + archive_file: BinaryIO | None = field(default=None, repr=False) + active_step: str | None = None + active_tool: str | None = None + bytes_archived: int = 0 + steps_seen: list[str] = field(default_factory=list) + error: Exception | None = None + # The step being archived when the first error was recorded, so failure + # paths can reconcile exactly that record even after its end marker. + error_step: str | None = None + # Display-callback failures live apart from archival failures: a broken + # renderer must never read as a broken archive. + display_error: Exception | None = None + error_tool: str | None = None + + +class LogStreamReader: + """Drains worker stderr, parses markers, and archives raw bytes to step logs. + + The reader runs in a dedicated thread. Marker lines are consumed (not archived). + Non-marker bytes are written to the current step's log file. A callback is + invoked for display purposes with the decoded text. + """ + + def __init__( + self, + stderr: BinaryIO, + *, + log_path_resolver: Callable[[str, str], Path | None] | None = None, + on_output: Callable[[bytes], None] | None = None, + on_step_event: Callable[[str, str, str], None] | None = None, + tail_size: int = 4096, + valid_steps: set[tuple[str, str]] | None = None, + workspace_dir: Path | None = None, + ): + self._stderr = stderr + self._resolve_path = log_path_resolver + self._on_output = on_output + self._on_output_disabled = False + self._on_step_event = on_step_event + self._on_step_event_disabled = False + self._tail_size = tail_size + self._valid_steps = valid_steps + self._workspace_dir = workspace_dir + self._state = LogStreamState() + self._thread: threading.Thread | None = None + self._stop = threading.Event() + + @property + def state(self) -> LogStreamState: + return self._state + + def _record_error(self, exc: Exception) -> None: + if self._state.error is None: + self._state.error = exc + self._state.error_step = self._state.active_step + self._state.error_tool = self._state.active_tool + + def start(self) -> None: + self._thread = threading.Thread(target=self._drain_loop, name="ecc-log-reader", daemon=True) + self._thread.start() + + def join(self, timeout: float | None = None) -> None: + if self._thread is not None: + self._thread.join(timeout=timeout) + + @property + def completed(self) -> bool: + """True if the drain thread has finished (or was never started).""" + if self._thread is None: + return True + return not self._thread.is_alive() + + def stop(self) -> None: + self._stop.set() + + def _drain_loop(self) -> None: + # read1 returns whatever the pipe currently holds; read(8192) would + # block until the buffer fills or EOF, stalling live progress for + # steps that emit less than 8 KiB while still running. + read_chunk = getattr(self._stderr, "read1", None) or self._stderr.read + buf = b"" + try: + while not self._stop.is_set(): + chunk = read_chunk(8192) + if not chunk: + break + buf += chunk + buf = self._process_buffer(buf) + if buf: + self._emit_data(buf) + except Exception as exc: + self._record_error(exc) + finally: + self._close_archive() + + def _process_buffer(self, buf: bytes) -> bytes: + while True: + idx = buf.find(MARKER_PREFIX) + if idx < 0: + # No candidate frame: hold back only a trailing partial prefix. + tail = buf.rfind(MARKER_PREFIX[:1]) + if tail >= 0 and MARKER_PREFIX.startswith(buf[tail:]): + if tail: + self._emit_data(buf[:tail]) + return buf[tail:] + if buf: + self._emit_data(buf) + return b"" + if idx > 0: + # Bytes before a marker candidate are ordinary stream data. + self._emit_data(buf[:idx]) + buf = buf[idx:] + continue + nl = buf.find(b"\n") + if nl < 0: + if len(buf) <= 512: + return buf + # An overlong candidate without a newline is not a marker: + # emit the prefix's first byte and rescan the remainder. + self._emit_data(buf[:1]) + buf = buf[1:] + continue + frame = buf[: nl + 1] + marker = parse_marker(frame) + if marker is not None: + self._handle_marker(marker, frame) + else: + self._emit_data(frame) + buf = buf[nl + 1 :] + + def _is_allowed_step(self, step: str, tool: str) -> bool: + if self._valid_steps is None: + return True + return (step, tool) in self._valid_steps + + def _validated_archive_path(self, step: str, tool: str) -> Path | None: + """Resolve and validate the archive path for a begin marker. + + Returns the validated path, or None when archiving is not configured + (resolver absent) or when the marker must degrade to ordinary bytes + (unsafe names, resolver failure, containment violation). + """ + if self._resolve_path is None: + return None + for value in (step, tool): + if not value or "/" in value or "\\" in value or ".." in value: + self._record_error(ValueError(f"unsafe step marker name: {value!r}")) + return None + try: + path = self._resolve_path(step, tool) + except Exception as exc: + self._record_error(exc) + return None + if path is None: + return None + if self._workspace_dir is not None and not path_is_within(path, self._workspace_dir): + self._record_error(ValueError(f"archive path escapes workspace: {path}")) + return None + return path + + def _handle_marker(self, marker: StepMarker, raw_line: bytes) -> None: + if marker.event == "begin": + if not self._is_allowed_step(marker.step, marker.tool): + self._emit_data(raw_line) + return + if self._state.active_step is not None: + self._emit_data(raw_line) + return + archive_path = self._validated_archive_path(marker.step, marker.tool) + if self._resolve_path is not None and archive_path is None: + # Validation failed before activation: keep the marker's + # identity on the error so failure-path reconciliation can + # still find and downgrade this step. + if self._state.error_step is None: + self._state.error_step = marker.step + self._state.error_tool = marker.tool + self._emit_data(raw_line) + return + self._state.active_step = marker.step + self._state.active_tool = marker.tool + self._state.steps_seen.append(marker.step) + if archive_path is not None: + self._open_archive(archive_path) + self._emit_step_event("begin", marker.step, marker.tool) + elif marker.event == "end": + if marker.step == self._state.active_step and marker.tool == self._state.active_tool: + self._close_archive() + self._state.active_step = None + self._state.active_tool = None + self._emit_step_event("end", marker.step, marker.tool) + else: + self._emit_data(raw_line) + else: + self._emit_data(raw_line) + + def _emit_step_event(self, event: str, step: str, tool: str) -> None: + if self._on_step_event is None or self._on_step_event_disabled: + return + try: + self._on_step_event(event, step, tool) + except Exception as exc: + if self._state.display_error is None: + self._state.display_error = exc + self._on_step_event_disabled = True + + def _emit_data(self, data: bytes) -> None: + if self._state.archive_file is not None: + try: + self._state.archive_file.write(data) + self._state.bytes_archived += len(data) + except OSError as exc: + self._record_error(exc) + with suppress(OSError): + self._state.archive_file.close() + self._state.archive_file = None + self._update_tail(data) + if self._on_output is not None and not self._on_output_disabled: + try: + self._on_output(data) + except Exception as exc: + if self._state.display_error is None: + self._state.display_error = exc + self._on_output_disabled = True + + def _update_tail(self, data: bytes) -> None: + combined = self._state.tail_bytes + data + if len(combined) > 2 * self._tail_size: + combined = combined[-self._tail_size :] + self._state.tail_bytes = combined + + def _open_archive(self, path: Path) -> None: + try: + path.parent.mkdir(parents=True, exist_ok=True) + self._state.archive_file = path.open("wb") # noqa: SIM115 + except OSError as exc: + self._record_error(exc) + self._state.archive_file = None + + def _close_archive(self) -> None: + if self._state.archive_file is not None: + try: + self._state.archive_file.flush() + except OSError as exc: + self._record_error(exc) + finally: + try: + self._state.archive_file.close() + except OSError as exc: + self._record_error(exc) + self._state.archive_file = None + + +# Set by the stdio/RPC server entry point: in that process an outer client +# (the CLI parent or the Electron archiver) owns the fd stream, so in-process +# self-archiving must stay off to keep the single-producer invariant. +_EXTERNAL_LOG_CLIENT = False +_SELF_ARCHIVE_ACTIVE = False + + +def mark_external_log_client() -> None: + """Mark that this process's log stream is owned by an outer client.""" + global _EXTERNAL_LOG_CLIENT + _EXTERNAL_LOG_CLIENT = True + + +@contextmanager +def archive_own_step_logs(workspace_dir, *, echo: bool = True): + """Archive this process's own fd 1+2 streams into per-step log files. + + In-process executor runs (no separate client process, e.g. agent + candidate reruns or the documented direct EngineFlow examples) still + must not write step log files from executor code. This context redirects + fd 1 and fd 2 through one pipe — the same merged stream the CLI worker's + stdio isolation produces — so a LogStreamReader (the client role) + archives step-scoped bytes and consumes markers, while echoing all bytes + to the original stderr. Yields the reader so callers can inspect + ``reader.state`` after the block. + """ + import sys + + from chipcompiler.utility.json import json_read + from chipcompiler.utility.log import flush_cstdio + + global _SELF_ARCHIVE_ACTIVE + if _EXTERNAL_LOG_CLIENT or _SELF_ARCHIVE_ACTIVE: + # An outer client owns this stream (worker/sidecar process), or an + # outer archive_own_step_logs is already active (e.g. run_steps + # auto-archives around an explicit caller): pass through untouched. + yield None + return + _SELF_ARCHIVE_ACTIVE = True + + workspace_dir = Path(workspace_dir) + flow_data = json_read(workspace_dir / "home" / "flow.json") + valid_steps = { + (step["name"], step["tool"]) + for step in flow_data.get("steps", []) + if isinstance(step, dict) and "name" in step and "tool" in step + } + + real_stdout = -1 + real_stderr = -1 + read_fd = -1 + write_fd = -1 + stream = None + try: + sys.stdout.flush() + sys.stderr.flush() + flush_cstdio() + real_stdout = os.dup(1) + real_stderr = os.dup(2) + read_fd, write_fd = os.pipe() + os.dup2(write_fd, 1) + os.dup2(write_fd, 2) + os.close(write_fd) + write_fd = -1 + stream = os.fdopen(read_fd, "rb") + read_fd = -1 # the stream owns it now + + def _echo(data: bytes) -> None: + os.write(real_stderr, data) + + reader = LogStreamReader( + stream, + log_path_resolver=step_log_archive_resolver(workspace_dir), + on_output=_echo if echo else None, + valid_steps=valid_steps or None, + workspace_dir=workspace_dir, + ) + reader.start() + except BaseException: + # A setup failure must not poison later runs: restore any redirected + # descriptors, close what was opened, and release the guard. + if real_stdout >= 0: + with suppress(OSError): + os.dup2(real_stdout, 1) + if real_stderr >= 0: + with suppress(OSError): + os.dup2(real_stderr, 2) + if write_fd >= 0: + with suppress(OSError): + os.close(write_fd) + if stream is not None: + with suppress(OSError): + stream.close() + elif read_fd >= 0: + with suppress(OSError): + os.close(read_fd) + if real_stdout >= 0: + with suppress(OSError): + os.close(real_stdout) + if real_stderr >= 0: + with suppress(OSError): + os.close(real_stderr) + _SELF_ARCHIVE_ACTIVE = False + raise + try: + yield reader + finally: + # Flush everything, restore both descriptors so the pipe sees EOF, + # and only then wait for the reader to drain the tail — the echo + # callback writes to real_stderr, so it must stay open until the + # drain finishes. The pipe's read stream closes too: nothing else + # owns it, and leaked pipes accumulate into EMFILE over many reruns. + # Every step here can fail on a broken descriptor, so the guard is + # released in the innermost finally regardless. + try: + with suppress(Exception): + sys.stdout.flush() + sys.stderr.flush() + flush_cstdio() + with suppress(OSError): + os.dup2(real_stdout, 1) + with suppress(OSError): + os.dup2(real_stderr, 2) + reader.join(timeout=5.0) + reader.stop() + with suppress(OSError): + stream.close() + with suppress(OSError): + os.close(real_stdout) + with suppress(OSError): + os.close(real_stderr) + finally: + _SELF_ARCHIVE_ACTIVE = False diff --git a/chipcompiler/runtime/operations.py b/chipcompiler/runtime/operations.py index ddba330c..86d5d0ee 100644 --- a/chipcompiler/runtime/operations.py +++ b/chipcompiler/runtime/operations.py @@ -4,31 +4,14 @@ import time from collections.abc import Callable from dataclasses import dataclass, field -from pathlib import Path from typing import Any from uuid import uuid4 -_LOG_POLL_INTERVAL_SECONDS = 0.25 -_MAX_LOG_CHUNK_BYTES = 16 * 1024 -_MAX_FINAL_LOG_BYTES = 64 * 1024 _RENDER_ACK_RETRY_SECONDS = 5.0 _RENDER_ACK_PAUSE_SECONDS = 30.0 _RENDER_ACK_ABORT_SECONDS = 300.0 -@dataclass -class _StepLogTail: - """A bounded worker-side reader for one active step log.""" - - operation_id: str - path: Path - step: str - tool: str - cursor: int - stopped: threading.Event = field(default_factory=threading.Event) - thread: threading.Thread | None = None - - class RuntimeOperationConflict(RuntimeError): """A workspace already owns a non-terminal runtime operation.""" @@ -80,7 +63,6 @@ def __init__(self, publisher: Callable[[dict[str, Any]], None] | None = None): self._operations: dict[str, RuntimeOperation] = {} self._active_by_workspace: dict[str, str] = {} self._idempotency: dict[tuple[str, str], str] = {} - self._step_log_tails: dict[str, _StepLogTail] = {} self._runtime_instance_id = uuid4().hex self._workspace_sequences: dict[str, int] = {} @@ -304,12 +286,10 @@ def _run( operation.updated_at = time.time() event = self._new_event_locked(operation, event_type, {"error": operation.error}) self._publish(event) - self._stop_step_log_tail(operation_id) with self._lock: self._active_by_workspace.pop(self._operations[operation_id].workspace_id, None) def step_started(self, operation_id: str, workspace_step: Any) -> None: - self._stop_step_log_tail(operation_id) with self._lock: operation = self._operations[operation_id] operation.current_step = str(getattr(workspace_step, "name", "")) @@ -324,24 +304,7 @@ def step_started(self, operation_id: str, workspace_step: Any) -> None: "state": "Ongoing", }, ) - log_tail = _step_log_tail_for( - operation_id, - getattr(workspace_step, "log", None), - operation.current_step, - operation.current_tool, - ) - if log_tail is not None: - self._step_log_tails[operation_id] = log_tail self._publish(event) - if log_tail is not None: - thread = threading.Thread( - target=self._tail_step_log, - args=(log_tail,), - name=f"ecc-runtime-log-{operation_id}", - daemon=True, - ) - log_tail.thread = thread - thread.start() def rerun_prepared( self, @@ -367,16 +330,13 @@ def rerun_prepared( self._publish(event) def step_completed(self, operation_id: str, workspace_step: Any, state: Any) -> None: - self._stop_step_log_tail(operation_id) state_value = str(getattr(state, "value", state)) - final_log = _read_final_log(getattr(workspace_step, "log", None)) with self._render_gate: operation = self._operations[operation_id] operation.current_step = str(getattr(workspace_step, "name", "")) operation.current_tool = str(getattr(workspace_step, "tool", "")) operation.updated_at = time.time() payload: dict[str, Any] = { - "finalLog": final_log, "step": operation.current_step, "tool": operation.current_tool, "state": state_value, @@ -423,7 +383,6 @@ def subflow_stage( self._publish(event) def step_skipped(self, operation_id: str, workspace_step: Any) -> None: - self._stop_step_log_tail(operation_id) with self._lock: operation = self._operations[operation_id] operation.current_step = str(getattr(workspace_step, "name", "")) @@ -517,57 +476,6 @@ def wait_for_step_rendered(self, operation_id: str) -> bool: if replay_event is not None: self._publish(replay_event) - def _tail_step_log(self, log_tail: _StepLogTail) -> None: - while not log_tail.stopped.is_set(): - self._publish_step_log_delta(log_tail) - log_tail.stopped.wait(_LOG_POLL_INTERVAL_SECONDS) - - def _publish_step_log_delta(self, log_tail: _StepLogTail) -> None: - try: - size = log_tail.path.stat().st_size - if size < log_tail.cursor: - # A rerun may truncate or replace a log file. The renderer treats - # this as a new bounded stream for the same step attempt. - log_tail.cursor = 0 - if size <= log_tail.cursor: - return - with log_tail.path.open("rb") as log_file: - log_file.seek(log_tail.cursor) - chunk = log_file.read(_MAX_LOG_CHUNK_BYTES) - except OSError: - return - - if not chunk: - return - log_tail.cursor += len(chunk) - text = chunk.decode("utf-8", errors="replace") - with self._lock: - if self._step_log_tails.get(log_tail.operation_id) is not log_tail: - return - operation = self._operations.get(log_tail.operation_id) - if operation is None: - return - event = self._new_event_locked( - operation, - "step.log", - { - "chunk": text, - "cursor": log_tail.cursor, - "step": log_tail.step, - "tool": log_tail.tool, - }, - ) - self._publish(event) - - def _stop_step_log_tail(self, operation_id: str) -> None: - with self._lock: - log_tail = self._step_log_tails.pop(operation_id, None) - if log_tail is None: - return - log_tail.stopped.set() - if log_tail.thread is not None and log_tail.thread is not threading.current_thread(): - log_tail.thread.join(timeout=_LOG_POLL_INTERVAL_SECONDS + 0.25) - def _new_event_locked( self, operation: RuntimeOperation, @@ -661,40 +569,3 @@ def on_step_skipped(self, workspace_step: Any) -> None: def wait_for_step_rendered(self, _workspace_step: Any, _state: Any) -> bool: return self._manager.wait_for_step_rendered(self._operation_id) - - -def _read_final_log(log: Any) -> str: - path = getattr(log, "file", None) - if not path: - return "" - try: - with Path(path).open("rb") as log_file: - log_file.seek(0, 2) - size = log_file.tell() - log_file.seek(max(0, size - _MAX_FINAL_LOG_BYTES)) - return log_file.read().decode("utf-8", errors="replace") - except OSError: - return "" - - -def _step_log_tail_for( - operation_id: str, - log: Any, - step: str, - tool: str, -) -> _StepLogTail | None: - path = getattr(log, "file", None) - if not path: - return None - log_path = Path(path) - try: - cursor = log_path.stat().st_size - except OSError: - cursor = 0 - return _StepLogTail( - operation_id=operation_id, - path=log_path, - step=step, - tool=tool, - cursor=cursor, - ) diff --git a/chipcompiler/runtime/requests.py b/chipcompiler/runtime/requests.py index d8a602c2..05bceb65 100644 --- a/chipcompiler/runtime/requests.py +++ b/chipcompiler/runtime/requests.py @@ -67,6 +67,8 @@ class FlowRunStepRequest: workspace_id: str step: str rerun: bool = False + reset_dependents: bool = False + invalidate_dependents: bool = False @dataclass(frozen=True) @@ -177,6 +179,7 @@ def __init__(self, reason: str): "workspaceRevision": "workspace_revision", "idempotencyKey": "idempotency_key", "resetDependents": "reset_dependents", + "invalidateDependents": "invalidate_dependents", "configPath": "config_path", "outputPath": "output_path", "infoId": "info_id", @@ -213,7 +216,8 @@ def parse_request_model(model: type, params: object): if required and _is_missing(values[field.name]): raise RequestValidationError(f"missing required field: {field.name}") - if field.name in {"rerun", "reset_dependents"} and not isinstance(values[field.name], bool): + bool_fields = {"rerun", "reset_dependents", "invalidate_dependents"} + if field.name in bool_fields and not isinstance(values[field.name], bool): raise RequestValidationError(f"{field.name} must be a boolean") return model(**values) diff --git a/chipcompiler/runtime/rerun_prepare.py b/chipcompiler/runtime/rerun_prepare.py new file mode 100644 index 00000000..b932bec6 --- /dev/null +++ b/chipcompiler/runtime/rerun_prepare.py @@ -0,0 +1,214 @@ +"""Rerun preparation: atomic record invalidation plus artifact cleanup. + +Extracted from workspace_api.py so the state-transition machinery lives in a +focused module and the API class stays an adapter. `prepare_steps_for_rerun` +is one all-or-nothing transition: validate artifact directories, snapshot +affected flow records, apply the target reset plus dependent state +invalidation, persist once, and only then delete the target's artifacts. +""" + +import shutil +from pathlib import Path + +from chipcompiler.utility.path import path_is_within + + +def _runtime_api_error(message: str): + # Local import: workspace_api imports this module, so the shared error + # type is resolved lazily to keep the dependency one-directional. + from chipcompiler.runtime.workspace_api import RuntimeApiError + + return RuntimeApiError("command_failed", message) + + +def rerun_affected_steps(engine_flow, workspace_step, *, reset_dependents: bool): + if not reset_dependents: + return [workspace_step] + workspace_steps = list(getattr(engine_flow, "workspace_steps", [])) + try: + start_index = workspace_steps.index(workspace_step) + except ValueError: + return [workspace_step] + return workspace_steps[start_index:] + + +def prepare_step_for_rerun(workspace, engine_flow, workspace_step) -> None: + prepare_steps_for_rerun(workspace, engine_flow, [workspace_step]) + + +def prepare_steps_for_rerun( + workspace, + engine_flow, + workspace_steps, + *, + invalidate_only_steps=(), +) -> None: + workspace_root = Path(workspace.directory).resolve() + unique_steps = [] + known_step_keys = set() + for workspace_step in workspace_steps: + key = ( + str(getattr(workspace_step, "name", "")), + str(getattr(workspace_step, "tool", "")), + ) + if key in known_step_keys: + continue + known_step_keys.add(key) + unique_steps.append(workspace_step) + + artifact_directories = [] + known_directories = set() + for workspace_step in unique_steps: + for directory in _step_artifact_dirs(workspace_step): + resolved = _validate_step_artifact_dir( + workspace_root, + directory, + workspace_step.name, + ) + if resolved in known_directories: + continue + known_directories.add(resolved) + artifact_directories.append((workspace_step.name, directory)) + + # One all-or-nothing state transition: apply the full reset for the + # rerun targets and the state-only invalidation for their dependents, + # then persist once. A failed save restores the record snapshots so + # neither disk nor the live session is left half-mutated, and no + # artifact is deleted before the new states are durable. + snapshots = [] + for workspace_step in (*unique_steps, *invalidate_only_steps): + record = engine_flow.get_step(workspace_step.name, workspace_step.tool) + if record is not None: + snapshots.append((record, dict(record))) + for workspace_step in unique_steps: + record = engine_flow.get_step(workspace_step.name, workspace_step.tool) + if record is None: + continue + record.update( + { + "state": "Unstart", + "runtime": "", + "peak memory (mb)": 0, + "info": {}, + } + ) + for workspace_step in invalidate_only_steps: + record = engine_flow.get_step(workspace_step.name, workspace_step.tool) + if record is None: + continue + record.update({"state": "Unstart", "runtime": "", "peak memory (mb)": 0}) + if snapshots and not engine_flow.save(): + for record, snapshot in snapshots: + record.clear() + record.update(snapshot) + raise _runtime_api_error("failed to persist step invalidation; refusing to modify outputs") + + # Post-save cleanup can still fail midway (artifact delete, subflow or + # checklist reset). Cleanup is irreversible for steps already cleared, + # so those steps keep their persisted Unstart records (their outputs are + # gone); only steps not yet touched roll back to their snapshots. + try: + for workspace_step in unique_steps: + for step_name, directory in artifact_directories: + if step_name != workspace_step.name: + continue + _clear_step_artifact_dir( + workspace_root, + directory, + step_name, + ) + _reset_step_subflow(workspace_step) + _reset_step_checklist(workspace_step) + except Exception: + # Steps up to and including the one mid-cleanup have lost artifacts; + # their persisted Unstart must stay. Later steps roll back intact. + cleaned = { + str(getattr(step, "name", "")) + for step in unique_steps[: unique_steps.index(workspace_step) + 1] + } + for record, snapshot in snapshots: + if record.get("name") in cleaned: + continue + record.clear() + record.update(snapshot) + engine_flow.save() + raise + + +def _reset_step_subflow(workspace_step) -> None: + from chipcompiler.utility import json_read, json_write + + subflow = getattr(workspace_step, "subflow", None) + path = getattr(subflow, "path", None) + if not path: + return + subflow_path = Path(path) + data = json_read(subflow_path) + steps = data.get("steps", []) if isinstance(data, dict) else [] + if not isinstance(steps, list): + return + for step in steps: + if not isinstance(step, dict): + continue + step.update( + { + "state": "Unstart", + "runtime": "", + "peak memory (mb)": 0, + "info": {}, + } + ) + json_write(subflow_path, {"path": str(subflow_path), "steps": steps}) + subflow.steps = steps + + +def _reset_step_checklist(workspace_step) -> None: + from chipcompiler.data import Checklist + + checklist = getattr(workspace_step, "checklist", None) + path = getattr(checklist, "path", None) + if not path: + return + checklist_path = Path(path) + Checklist(checklist_path).replace([]) + checklist.checklist = [] + + +def _step_artifact_dirs(step) -> tuple[Path, ...]: + directories: list[Path] = [] + for field in ("output", "data", "feature", "analysis", "report", "log"): + value = getattr(step, field, {}) + directory = value.get("dir") if isinstance(value, dict) else getattr(value, "dir", None) + if directory: + directories.append(Path(directory)) + return tuple(dict.fromkeys(directories)) + + +def _clear_step_artifact_dir( + workspace_root: Path, + directory: Path, + step_name: str, +) -> None: + _validate_step_artifact_dir(workspace_root, directory, step_name) + if directory.exists(): + if not directory.is_dir(): + raise _runtime_api_error(f"step artifact is not a directory: {step_name}") + shutil.rmtree(directory) + directory.mkdir(parents=True, exist_ok=True) + + +def _validate_step_artifact_dir( + workspace_root: Path, + directory: Path, + step_name: str, +) -> Path: + resolved = directory.resolve() + if ( + resolved == workspace_root + or not path_is_within(resolved, workspace_root) + or directory.is_symlink() + ): + raise _runtime_api_error(f"step artifact escapes workspace: {step_name}") + if directory.exists() and not directory.is_dir(): + raise _runtime_api_error(f"step artifact is not a directory: {step_name}") + return resolved diff --git a/chipcompiler/runtime/rpc_dispatch.py b/chipcompiler/runtime/rpc_dispatch.py index 89ad3f91..b2cae712 100644 --- a/chipcompiler/runtime/rpc_dispatch.py +++ b/chipcompiler/runtime/rpc_dispatch.py @@ -5,8 +5,6 @@ from jsonrpcserver import Success, dispatch from oslash.either import Left, Right -from chipcompiler.runtime.events import redirect_stdout_to_stderr - JsonRpcHandler = Callable[..., Any] @@ -31,8 +29,7 @@ def dispatch(self, payload: bytes | str) -> str: def _wrap_handler(self, handler: JsonRpcHandler) -> JsonRpcHandler: @wraps(handler) def wrapped(*args: Any, **kwargs: Any): - with redirect_stdout_to_stderr(): - result = handler(*args, **kwargs) + result = handler(*args, **kwargs) if isinstance(result, Left | Right): return result return Success(result) diff --git a/chipcompiler/runtime/stdio_isolation.py b/chipcompiler/runtime/stdio_isolation.py new file mode 100644 index 00000000..baa5f889 --- /dev/null +++ b/chipcompiler/runtime/stdio_isolation.py @@ -0,0 +1,55 @@ +"""Permanent stdio isolation for the worker process. + +At worker startup, the protocol output stream is duplicated to a safe fd, +then fd 1 is permanently redirected to fd 2. This ensures that any Python, +C/C++, glog, or subprocess output goes to stderr (the log stream), while +the protocol writer uses only the saved fd for RPC frames. +""" + +import os +import sys +from typing import BinaryIO + + +class StdioIsolation: + """Installs permanent fd-level stdio isolation for the worker process. + + After install(): + - protocol_stream: a binary stream on the original stdout fd (for RPC frames only) + - fd 1 and sys.stdout: permanently point to stderr (for tool/EDA output) + """ + + def __init__(self): + self._protocol_stream: BinaryIO | None = None + self._installed = False + + @property + def protocol_stream(self) -> BinaryIO: + if self._protocol_stream is None: + raise RuntimeError("stdio isolation not installed") + return self._protocol_stream + + @property + def installed(self) -> bool: + return self._installed + + def install(self) -> BinaryIO: + """Install permanent stdio isolation. Must be called once at worker startup.""" + if self._installed: + return self._protocol_stream # type: ignore[return-value] + + sys.stdout.flush() + sys.stderr.flush() + + protocol_fd = os.dup(1) + os.dup2(2, 1) + sys.stdout = sys.stderr + + self._protocol_stream = os.fdopen(protocol_fd, "wb", buffering=0) + self._installed = True + return self._protocol_stream + + def close(self) -> None: + if self._protocol_stream is not None: + self._protocol_stream.close() + self._protocol_stream = None diff --git a/chipcompiler/runtime/stdio_server.py b/chipcompiler/runtime/stdio_server.py index de0f9ced..5035f39b 100644 --- a/chipcompiler/runtime/stdio_server.py +++ b/chipcompiler/runtime/stdio_server.py @@ -113,8 +113,18 @@ def _read_chunk(input_stream: BinaryIO) -> bytes: def main(*, persistent_db_enabled: bool = False) -> int: - return run_stdio_server( - sys.stdin.buffer, - sys.stdout.buffer, - persistent_db_enabled=persistent_db_enabled, - ) + from chipcompiler.runtime.log_stream import mark_external_log_client + from chipcompiler.runtime.stdio_isolation import StdioIsolation + + # This process's fd stream belongs to the parent client's archiver. + mark_external_log_client() + isolation = StdioIsolation() + protocol_stream = isolation.install() + try: + return run_stdio_server( + sys.stdin.buffer, + protocol_stream, + persistent_db_enabled=persistent_db_enabled, + ) + finally: + isolation.close() diff --git a/chipcompiler/runtime/worker.py b/chipcompiler/runtime/worker.py new file mode 100644 index 00000000..3f9fb421 --- /dev/null +++ b/chipcompiler/runtime/worker.py @@ -0,0 +1,289 @@ +import json +import os +import signal +import subprocess +from collections import deque +from contextlib import suppress +from dataclasses import dataclass +from pathlib import Path + +from chipcompiler.runtime.transport import ( + ContentLengthDecoder, + TransportError, + encode_content_length_frame, +) +from chipcompiler.utility.json import json_read, json_write + + +@dataclass(frozen=True) +class WorkerResult: + success: bool + response: dict | None = None + exit_code: int | None = None + signal_number: int | None = None + error: str | None = None + + +class WorkerProcessError(Exception): + pass + + +_GRACEFUL_WAIT = 2.0 +_FORCEFUL_WAIT = 3.0 + + +class WorkerClient: + """Manages a worker subprocess running `ecc rpc serve --stdio`.""" + + def __init__(self, worker_argv: list[str]): + self._argv = worker_argv + self._process: subprocess.Popen | None = None + self._pgid: int | None = None + self._decoder = ContentLengthDecoder() + self._notifications: deque[dict] = deque() + self._pending_responses: dict[int, dict] = {} + + def start(self) -> subprocess.Popen: + self._process = subprocess.Popen( + self._argv, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=True, + ) + self._pgid = self._process.pid + return self._process + + @property + def process(self) -> subprocess.Popen | None: + return self._process + + def send_request(self, method: str, params: dict, request_id: int = 1) -> None: + if self._process is None or self._process.stdin is None: + raise WorkerProcessError("worker not started") + payload = json.dumps( + {"jsonrpc": "2.0", "method": method, "params": params, "id": request_id}, + separators=(",", ":"), + ) + frame = encode_content_length_frame(payload) + try: + self._process.stdin.write(frame) + self._process.stdin.flush() + except OSError as exc: + raise WorkerProcessError(f"failed to send request: {exc}") from exc + + def read_response(self, request_id: int = 1) -> dict: + """Read the next RPC response matching request_id. + + Checks the pending-response buffer first (for out-of-order delivery). + Non-matching responses are stored by id for later retrieval. + Notifications (no 'id') are queued separately. + Invalid envelopes raise WorkerProcessError. + """ + if request_id in self._pending_responses: + return self._pending_responses.pop(request_id) + if self._process is None or self._process.stdout is None: + raise WorkerProcessError("worker not started") + while True: + read1 = getattr(self._process.stdout, "read1", None) + chunk = read1(8192) if read1 is not None else self._process.stdout.read(8192) + if not chunk: + raise WorkerProcessError("worker stdout closed before response") + try: + messages = self._decoder.feed(chunk) + except TransportError as exc: + raise WorkerProcessError(f"protocol error: {exc}") from exc + found: dict | None = None + for raw in messages: + try: + msg = json.loads(raw) + except (json.JSONDecodeError, ValueError) as exc: + raise WorkerProcessError(f"malformed JSON from worker: {exc}") from exc + if not isinstance(msg, dict): + raise WorkerProcessError("expected JSON object from worker") + if "id" not in msg: + self._notifications.append(msg) + else: + _validate_response_envelope(msg) + if msg["id"] == request_id and found is None: + found = msg + else: + self._pending_responses[msg["id"]] = msg + if found is not None: + return found + + def pop_notification(self) -> dict | None: + if self._notifications: + return self._notifications.popleft() + return None + + def request(self, method: str, params: dict, request_id: int = 1) -> WorkerResult: + try: + self.send_request(method, params, request_id) + response = self.read_response(request_id) + except WorkerProcessError as exc: + return WorkerResult(success=False, error=str(exc)) + if "error" in response: + err_msg = response["error"].get("message", "rpc error") + return WorkerResult(success=False, response=response, error=err_msg) + return WorkerResult(success=True, response=response) + + def terminate(self) -> int | None: + proc = self._process + if proc is None: + return None + return _terminate_process_group(proc, self._pgid) + + def is_alive(self) -> bool: + if self._process is None: + return False + return self._process.poll() is None + + +def _terminate_process_group(proc: subprocess.Popen, pgid: int | None = None) -> int: + """Escalate signals to the worker process group. + + pgid is cached at start time (the worker pid, since start_new_session=True). + After each signal, waits for the process group to exit within a deadline + before escalating to the next signal. The leader is reaped each iteration + so its zombie does not keep the group visible to killpg. + """ + import time + + if pgid is None: + pgid = proc.pid + + def _group_alive() -> bool: + proc.poll() + try: + os.killpg(pgid, 0) + return True + except OSError: + return False + + def _signal_group(sig: int) -> None: + with suppress(OSError): + os.killpg(pgid, sig) + + def _wait_group_exit(timeout: float) -> bool: + """Poll group liveness until dead or timeout. Returns True if dead.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if not _group_alive(): + return True + time.sleep(0.05) + return not _group_alive() + + if proc.poll() is None or _group_alive(): + _signal_group(signal.SIGINT) + if not _wait_group_exit(_GRACEFUL_WAIT): + _signal_group(signal.SIGTERM) + if not _wait_group_exit(_FORCEFUL_WAIT): + _signal_group(signal.SIGKILL) + _wait_group_exit(_FORCEFUL_WAIT) + + if proc.poll() is None: + proc.wait() + + return proc.returncode + + +def _validate_response_envelope(msg: dict) -> None: + """Validate a JSON-RPC 2.0 response envelope. + + Requires jsonrpc=="2.0", exactly one of "result" or "error", + and id must be int|str|None (no booleans, lists, or dicts). + Error objects must have "code" (int) and "message" (str). + Raises WorkerProcessError on invalid envelopes. + """ + if msg.get("jsonrpc") != "2.0": + raise WorkerProcessError( + f"invalid JSON-RPC response: missing or wrong 'jsonrpc' version: {msg!r}" + ) + msg_id = msg.get("id") + if isinstance(msg_id, bool) or not isinstance(msg_id, (int, str, type(None))): + raise WorkerProcessError( + f"invalid JSON-RPC response: 'id' must be int, str, or null: {msg!r}" + ) + has_result = "result" in msg + has_error = "error" in msg + if has_result == has_error: + raise WorkerProcessError( + f"invalid JSON-RPC response: must have exactly one of 'result' or 'error': {msg!r}" + ) + if has_error: + err = msg["error"] + if ( + not isinstance(err, dict) + or not isinstance(err.get("code"), int) + or not isinstance(err.get("message"), str) + ): + raise WorkerProcessError( + f"invalid JSON-RPC error object: requires 'code' (int) and 'message' (str): {msg!r}" + ) + + +def classify_worker_exit(proc: subprocess.Popen) -> WorkerResult: + """Classify how the worker exited after it is no longer running.""" + code = proc.returncode + if code is None: + return WorkerResult(success=False, error="worker still running") + if code == 0: + return WorkerResult(success=True, exit_code=0) + if code < 0: + sig = -code + try: + sig_name = signal.Signals(sig).name + except ValueError: + sig_name = str(sig) + return WorkerResult( + success=False, + exit_code=code, + signal_number=sig, + error=f"worker killed by {sig_name}", + ) + return WorkerResult(success=False, exit_code=code, error=f"worker exited with code {code}") + + +def repair_flow_state( + flow_json_path: str | Path, *, active_step: str, active_tool: str | None = None +) -> list[str]: + """Repair the step left unfinished by a crashed worker, setting it to Incomplete. + + Operation-scoped: only the active step is repaired. The caller must identify + which step was owned by the crashed operation. Both Ongoing and Success + records are repaired: a Success without a completed end marker means the + crash interrupted the step's post-processing, so its persisted result is + not trustworthy. When active_tool is given, only the matching (name, tool) + record is touched — a flow may carry the same step name under two tools. + + Returns the list of step names that were repaired. + Raises OSError if the repaired state cannot be persisted. + """ + path = Path(flow_json_path) + data = json_read(path) + if not data: + return [] + + steps = data.get("steps") + if not isinstance(steps, list): + return [] + + repaired: list[str] = [] + for step in steps: + if not isinstance(step, dict): + continue + if step.get("state") not in ("Ongoing", "Success"): + continue + step_name = step.get("name", "") + if step_name != active_step: + continue + if active_tool is not None and step.get("tool") != active_tool: + continue + step["state"] = "Incomplete" + repaired.append(step_name) + + if repaired and not json_write(path, data): + raise OSError(f"failed to persist repaired flow state: {path}") + + return repaired diff --git a/chipcompiler/runtime/worker_operation.py b/chipcompiler/runtime/worker_operation.py new file mode 100644 index 00000000..b4060969 --- /dev/null +++ b/chipcompiler/runtime/worker_operation.py @@ -0,0 +1,351 @@ +"""Typed operation orchestrator for CLI/headless EDA execution via worker. + +Integrates WorkerClient, LogStreamReader, process-group cleanup, and +flow-state repair into one typed OperationResult boundary. All EDA +execution from CLI entry points should go through RunOperation. +""" + +import os +import signal +import subprocess +import sys +from collections.abc import Callable +from contextlib import suppress +from dataclasses import dataclass, field +from pathlib import Path + +from chipcompiler.runtime.log_stream import LogStreamReader, LogStreamState +from chipcompiler.runtime.worker import ( + WorkerClient, + WorkerResult, + classify_worker_exit, + repair_flow_state, +) + +PROTOCOL_VERSION = 1 + + +def _default_worker_argv() -> list[str]: + ecc_bin = os.path.join(os.path.dirname(sys.executable), "ecc") + return [ecc_bin, "rpc", "serve", "--stdio", "--persistent-db"] + + +@dataclass(frozen=True) +class OperationResult: + """Typed outcome of a single worker operation (run or run_step).""" + + success: bool + rpc_result: dict | None = None + exit_code: int | None = None + signal_number: int | None = None + error: str | None = None + repaired_steps: list[str] = field(default_factory=list) + archive_error: Exception | None = field(default=None, repr=False) + log_state: LogStreamState | None = field(default=None, repr=False) + + +class RunOperation: + """Orchestrates a single EDA flow execution through an isolated worker. + + Implements the real session lifecycle: + rpc.hello → workspace.open → caller's method → rpc.shutdown → EOF wait. + + Usage: + op = RunOperation( + workspace_dir=Path("/path/to/workspace"), + flow_json_path=Path("/path/to/flow.json"), + ) + result = op.run(method="flow.run", params={...}) + """ + + def __init__( + self, + *, + workspace_dir: Path, + flow_json_path: Path, + worker_argv: list[str] | None = None, + log_path_resolver: Callable[[str, str], Path | None] | None = None, + on_output: Callable[[bytes], None] | None = None, + on_step_event: Callable[[str, str, str], None] | None = None, + valid_steps: set[tuple[str, str]] | None = None, + ): + self._workspace_dir = workspace_dir + self._flow_json_path = flow_json_path + self._worker_argv = worker_argv or _default_worker_argv() + self._log_path_resolver = log_path_resolver + self._on_output = on_output + self._on_step_event = on_step_event + self._valid_steps = valid_steps + + def run(self, method: str, params: dict, *, request_id: int = 1) -> OperationResult: + """Execute one RPC method against the worker and return a typed result. + + Session sequence: hello → workspace.open → method → rpc.shutdown → EOF. + """ + return self.run_sequence([(method, params)], request_id=request_id) + + def run_sequence( + self, + calls: list[tuple[str, dict]], + *, + request_id: int = 1, + ) -> OperationResult: + """Execute an ordered list of (method, params) calls in one session. + + The calls share a single worker session: + hello → workspace.open → call 1 → ... → call N → rpc.shutdown → EOF. + + Execution stops at the first failed RPC: remaining calls are skipped, + the session is still shut down gracefully and drained, and the + returned OperationResult describes the failing call. On success the + result describes the last call. + """ + client = WorkerClient(self._worker_argv) + reader: LogStreamReader | None = None + + # The worker runs in its own process group (start_new_session), so a + # parent SIGTERM would otherwise leave it and its EDA descendants + # mutating the workspace. Convert SIGTERM into a KeyboardInterrupt on + # the main thread, which routes through crash recovery and + # terminates the worker group. (Main thread only; harmless no-op + # elsewhere.) + previous_sigterm = None + try: + previous_sigterm = signal.signal( + signal.SIGTERM, lambda _sig, _frame: (_ for _ in ()).throw(KeyboardInterrupt()) + ) + except (ValueError, OSError): + previous_sigterm = None + + try: + proc = client.start() + + reader = LogStreamReader( + proc.stderr, + log_path_resolver=self._log_path_resolver, + on_output=self._on_output, + on_step_event=self._on_step_event, + valid_steps=self._valid_steps, + workspace_dir=self._workspace_dir, + ) + reader.start() + + hello_result = client.request("rpc.hello", {"version": PROTOCOL_VERSION}, request_id=0) + if not hello_result.success: + return self._handle_protocol_or_crash(client, reader, hello_result) + + open_result = client.request( + "workspace.open", + {"directory": str(self._workspace_dir)}, + request_id=0, + ) + if not open_result.success: + return self._handle_protocol_or_crash(client, reader, open_result) + + workspace_id = open_result.response["result"]["workspaceId"] + + rpc_result: WorkerResult | None = None + for index, (method, params) in enumerate(calls): + full_params = {**params, "workspace_id": workspace_id} + rpc_result = client.request(method, full_params, request_id + index) + if not rpc_result.success: + break + + if rpc_result is not None and not rpc_result.success: + return self._handle_protocol_or_crash(client, reader, rpc_result) + + shutdown_ok = self._graceful_shutdown(client) + + reader.join(timeout=5.0) + reader.stop() + + log_state = reader.state + + error_parts: list[str] = [] + if log_state.error is not None: + error_parts.append(f"archive error: {log_state.error}") + if not reader.completed: + error_parts.append("log reader did not complete") + if log_state.active_step is not None: + error_parts.append(f"unmatched begin marker for step: {log_state.active_step}") + if not shutdown_ok: + error_parts.append("worker did not exit cleanly after shutdown") + + if error_parts: + # The reported failure must match the persisted state: a step + # left Success in flow.json would be skipped by a later resume + # while its archive is missing or incomplete. + repaired, repair_error = self._reconcile_step_state(log_state) + if repair_error is not None: + error_parts.append(repair_error) + return OperationResult( + success=False, + rpc_result=rpc_result.response if rpc_result else None, + exit_code=client.process.returncode if client.process else None, + error="; ".join(error_parts), + repaired_steps=repaired, + archive_error=log_state.error, + log_state=log_state, + ) + + return OperationResult( + success=True, + rpc_result=rpc_result.response if rpc_result else None, + exit_code=0, + log_state=log_state, + ) + + except KeyboardInterrupt: + return self._handle_crash(client, reader, "operation interrupted") + except Exception as exc: + return self._handle_crash(client, reader, str(exc)) + finally: + if previous_sigterm is not None: + with suppress(OSError, ValueError): + signal.signal(signal.SIGTERM, previous_sigterm) + + def _handle_protocol_or_crash( + self, + client: WorkerClient, + reader: LogStreamReader | None, + result: WorkerResult, + ) -> OperationResult: + """Route a failed WorkerResult to crash recovery or RPC error.""" + if result.response is None or not client.is_alive(): + error = result.error or "protocol failure" + return self._handle_crash(client, reader, error) + + self._graceful_shutdown(client) + if reader is not None: + reader.join(timeout=2.0) + reader.stop() + + log_state = reader.state if reader else None + # A live-worker RPC error can still leave a step unmatched: the flow + # raised after the begin marker, so flow.json may hold a stale Ongoing + # record. Repair it exactly as crash recovery does. + # A live-worker RPC error can leave an unmatched Ongoing step; an + # archive failure on an already-ended step leaves a stale Success. + # Both reconcile through the same repair path. + repaired, repair_error = self._reconcile_step_state(log_state) + error = result.error + if repair_error is not None: + error = f"{error}; {repair_error}" + return OperationResult( + success=False, + rpc_result=result.response, + exit_code=client.process.returncode if client.process else None, + error=error, + repaired_steps=repaired, + archive_error=log_state.error if log_state else None, + log_state=log_state, + ) + + def _graceful_shutdown(self, client: WorkerClient) -> bool: + """Send rpc.shutdown and wait for graceful EOF + zero exit.""" + try: + result = client.request("rpc.shutdown", {}, request_id=0) + except Exception: + client.terminate() + return False + + if not result.success: + client.terminate() + return False + + ok = (result.response or {}).get("result", {}).get("ok") + if ok is not True: + client.terminate() + return False + + proc = client.process + if proc is None: + return True + + try: + proc.wait(timeout=10.0) + except subprocess.TimeoutExpired: + client.terminate() + return False + + return proc.returncode == 0 + + def _reconcile_step_state( + self, log_state: LogStreamState | None + ) -> tuple[list[str], str | None]: + """Repair the step whose marker/archive evidence failed, if any. + + Picks the unmatched active step first, then the step being archived + when the first reader error fired, then the last seen step on an + archive error. With no stream evidence at all (a crash between the + Ongoing save and the begin marker), falls back to the persisted + Ongoing record — a worker session runs at most one step at a time. + Returns (repaired_steps, error_text): reconciling to Incomplete keeps + a later resume from trusting a stale record whose run never finished. + """ + if log_state is None: + return [], None + step = log_state.active_step or log_state.error_step + tool = log_state.active_tool or log_state.error_tool + if step is None and log_state.error is not None and log_state.steps_seen: + step = log_state.steps_seen[-1] + if step is None and self._flow_json_path.exists(): + from chipcompiler.utility import json_read + + data = json_read(self._flow_json_path) + ongoing = [ + record + for record in data.get("steps", []) + if isinstance(record, dict) and record.get("state") == "Ongoing" + ] + if len(ongoing) == 1: + step = ongoing[0].get("name") + tool = ongoing[0].get("tool") + if step is None or not self._flow_json_path.exists(): + return [], None + try: + return repair_flow_state(self._flow_json_path, active_step=step, active_tool=tool), None + except OSError as exc: + return [], f"state repair failed: {exc}" + + def _handle_crash( + self, + client: WorkerClient, + reader: LogStreamReader | None, + error: str, + ) -> OperationResult: + """Crash recovery: terminate, drain, repair, return failure.""" + exit_code: int | None = None + signal_number: int | None = None + log_state: LogStreamState | None = None + + try: + client.terminate() + proc = client.process + if proc is not None: + exit_result = classify_worker_exit(proc) + exit_code = exit_result.exit_code + signal_number = exit_result.signal_number + except Exception: + pass + + if reader is not None: + reader.join(timeout=2.0) + reader.stop() + log_state = reader.state + + # The crashed step (still active) or a step whose archive failed + # before the crash both reconcile to Incomplete through one path. + repaired, repair_error = self._reconcile_step_state(log_state) + if repair_error is not None: + error = f"{error}; {repair_error}" + + return OperationResult( + success=False, + exit_code=exit_code, + signal_number=signal_number, + error=error, + repaired_steps=repaired, + archive_error=log_state.error if log_state else None, + log_state=log_state, + ) diff --git a/chipcompiler/runtime/workspace_api.py b/chipcompiler/runtime/workspace_api.py index 38525718..cb797da3 100644 --- a/chipcompiler/runtime/workspace_api.py +++ b/chipcompiler/runtime/workspace_api.py @@ -39,6 +39,7 @@ WorkspaceOpenRequest, WorkspaceSyncConfigRequest, ) +from chipcompiler.runtime.rerun_prepare import prepare_steps_for_rerun, rerun_affected_steps from chipcompiler.runtime.sessions import ( LayoutEditSession, WorkspaceSession, @@ -249,9 +250,22 @@ def run(session: WorkspaceSession) -> dict: self._release_session_db(session) previous_db = None + if request.rerun or not session.workspace.flow.data.get("steps"): + # A full rerun executes every step; a fresh workspace has no + # persisted states yet, so every step is verified. + executable_steps = None + else: + executable_steps = { + record["name"] + for record in session.workspace.flow.data.get("steps", []) + if isinstance(record, dict) + and "name" in record + and record.get("state") != "Success" + } engine_flow = self._build_flow_for_session( session, attach_session_db=should_capture and not request.rerun, + executable_steps=executable_steps, ) if request.rerun: affected_steps = list(getattr(engine_flow, "workspace_steps", [])) @@ -287,7 +301,7 @@ def run(session: WorkspaceSession) -> dict: return self._with_session_mutation_lock(request.workspace_id, run) def flow_run_step(self, request: FlowRunStepRequest) -> dict: - return self._flow_run_step(request) + return self._flow_run_step(request, reset_dependents=request.reset_dependents) def _flow_run_step( self, @@ -306,6 +320,7 @@ def run_step(session: WorkspaceSession) -> dict: engine_flow = self._build_flow_for_session( session, attach_session_db=should_capture and not request.rerun, + executable_steps={request.step}, ) if request.rerun: if session.layout_edit_session is not None: @@ -319,15 +334,22 @@ def run_step(session: WorkspaceSession) -> dict: if workspace_step is None: raise RuntimeApiError("command_failed", f"step not found: {request.step}") if request.rerun: - affected_steps = self._rerun_affected_steps( + affected_steps = rerun_affected_steps( engine_flow, workspace_step, - reset_dependents=reset_dependents, + reset_dependents=reset_dependents or request.invalidate_dependents, ) - self._prepare_steps_for_rerun( + if request.invalidate_dependents and not reset_dependents: + # Clear only the target's artifacts; downstream steps keep + # their outputs but are marked Unstart for a later resume. + prepare_steps, invalidate_steps = affected_steps[:1], affected_steps[1:] + else: + prepare_steps, invalidate_steps = affected_steps, [] + prepare_steps_for_rerun( session.workspace, engine_flow, - affected_steps, + prepare_steps, + invalidate_only_steps=invalidate_steps, ) self._notify_rerun_prepared( observer, @@ -350,6 +372,12 @@ def run_step(session: WorkspaceSession) -> dict: rerun=request.rerun, observer=observer, ) + # Keep the global flow/status log entries that the in-process + # rerun helpers used to append after every selected step. + if getattr(session.workspace, "logger", None) is not None: + from chipcompiler.data import log_flow + + log_flow(workspace=session.workspace) finally: if should_capture: self._capture_flow_db( @@ -836,8 +864,9 @@ def _build_flow_for_session( session: WorkspaceSession, *, attach_session_db: bool, + executable_steps: set[str] | None = None, ): - engine_flow = build_flow_for_workspace(session.workspace) + engine_flow = build_flow_for_workspace(session.workspace, executable_steps=executable_steps) if attach_session_db: engine_flow.engine_db = session.db_handle return engine_flow @@ -892,17 +921,6 @@ def _prepare_workspace_for_rerun( preserve_user_inputs=preserve_user_inputs, ) - @staticmethod - def _rerun_affected_steps(engine_flow, workspace_step, *, reset_dependents: bool): - if not reset_dependents: - return [workspace_step] - workspace_steps = list(getattr(engine_flow, "workspace_steps", [])) - try: - start_index = workspace_steps.index(workspace_step) - except ValueError: - return [workspace_step] - return workspace_steps[start_index:] - @staticmethod def _notify_rerun_prepared( observer, @@ -920,159 +938,6 @@ def _notify_rerun_prepared( target_step=target_step, ) - @staticmethod - def _prepare_step_for_rerun(workspace, engine_flow, workspace_step) -> None: - WorkspaceRuntimeApi._prepare_steps_for_rerun( - workspace, - engine_flow, - [workspace_step], - ) - - @staticmethod - def _prepare_steps_for_rerun(workspace, engine_flow, workspace_steps) -> None: - workspace_root = Path(workspace.directory).resolve() - unique_steps = [] - known_step_keys = set() - for workspace_step in workspace_steps: - key = ( - str(getattr(workspace_step, "name", "")), - str(getattr(workspace_step, "tool", "")), - ) - if key in known_step_keys: - continue - known_step_keys.add(key) - unique_steps.append(workspace_step) - - artifact_directories = [] - known_directories = set() - for workspace_step in unique_steps: - for directory in WorkspaceRuntimeApi._step_artifact_dirs(workspace_step): - resolved = WorkspaceRuntimeApi._validate_step_artifact_dir( - workspace_root, - directory, - workspace_step.name, - ) - if resolved in known_directories: - continue - known_directories.add(resolved) - artifact_directories.append((workspace_step.name, directory)) - - for step_name, directory in artifact_directories: - WorkspaceRuntimeApi._clear_step_artifact_dir( - workspace_root, - directory, - step_name, - ) - - updated_record = False - for workspace_step in unique_steps: - record = engine_flow.get_step(workspace_step.name, workspace_step.tool) - if record is None: - continue - record.update( - { - "state": "Unstart", - "runtime": "", - "peak memory (mb)": 0, - "info": {}, - } - ) - updated_record = True - if updated_record: - engine_flow.save() - - for workspace_step in unique_steps: - WorkspaceRuntimeApi._reset_step_subflow(workspace_step) - WorkspaceRuntimeApi._reset_step_checklist(workspace_step) - - @staticmethod - def _reset_step_subflow(workspace_step) -> None: - from chipcompiler.utility import json_read, json_write - - subflow = getattr(workspace_step, "subflow", None) - path = getattr(subflow, "path", None) - if not path: - return - subflow_path = Path(path) - data = json_read(subflow_path) - steps = data.get("steps", []) if isinstance(data, dict) else [] - if not isinstance(steps, list): - return - for step in steps: - if not isinstance(step, dict): - continue - step.update( - { - "state": "Unstart", - "runtime": "", - "peak memory (mb)": 0, - "info": {}, - } - ) - json_write(subflow_path, {"path": str(subflow_path), "steps": steps}) - subflow.steps = steps - - @staticmethod - def _reset_step_checklist(workspace_step) -> None: - from chipcompiler.data import Checklist - - checklist = getattr(workspace_step, "checklist", None) - path = getattr(checklist, "path", None) - if not path: - return - checklist_path = Path(path) - Checklist(checklist_path).replace([]) - checklist.checklist = [] - - @staticmethod - def _step_artifact_dirs(step) -> tuple[Path, ...]: - directories: list[Path] = [] - for field in ("output", "data", "feature", "analysis", "report", "log"): - value = getattr(step, field, {}) - directory = value.get("dir") if isinstance(value, dict) else getattr(value, "dir", None) - if directory: - directories.append(Path(directory)) - return tuple(dict.fromkeys(directories)) - - @staticmethod - def _clear_step_artifact_dir( - workspace_root: Path, - directory: Path, - step_name: str, - ) -> None: - WorkspaceRuntimeApi._validate_step_artifact_dir(workspace_root, directory, step_name) - if directory.exists(): - if not directory.is_dir(): - raise RuntimeApiError( - "command_failed", - f"step artifact is not a directory: {step_name}", - ) - shutil.rmtree(directory) - directory.mkdir(parents=True, exist_ok=True) - - @staticmethod - def _validate_step_artifact_dir( - workspace_root: Path, - directory: Path, - step_name: str, - ) -> Path: - resolved = directory.resolve() - if ( - resolved == workspace_root - or not path_is_within(resolved, workspace_root) - or directory.is_symlink() - ): - raise RuntimeApiError( - "command_failed", - f"step artifact escapes workspace: {step_name}", - ) - if directory.exists() and not directory.is_dir(): - raise RuntimeApiError( - "command_failed", - f"step artifact is not a directory: {step_name}", - ) - return resolved - def _layout_edit_begin_result(edit_session: LayoutEditSession, *, reused: bool) -> dict: return { @@ -2013,7 +1878,12 @@ def _artifact_fingerprint(paths: tuple[Path, ...]) -> str: return digest.hexdigest() -def build_flow_for_workspace(workspace, *, create_step_workspaces: bool = True): +def build_flow_for_workspace( + workspace, + *, + create_step_workspaces: bool = True, + executable_steps: set[str] | None = None, +): import chipcompiler.engine as engine_api import chipcompiler.rtl2gds as rtl2gds_api @@ -2023,7 +1893,7 @@ def build_flow_for_workspace(workspace, *, create_step_workspaces: bool = True): engine_flow.add_step(step=step, tool=tool, state=state) if create_step_workspaces: - engine_flow.create_step_workspaces() + engine_flow.create_step_workspaces(executable_steps=executable_steps) return engine_flow diff --git a/chipcompiler/tools/ecc/runner.py b/chipcompiler/tools/ecc/runner.py index 561f0c91..e9dc9828 100644 --- a/chipcompiler/tools/ecc/runner.py +++ b/chipcompiler/tools/ecc/runner.py @@ -38,6 +38,7 @@ StepEnum.LEGALIZATION.value, StepEnum.ROUTING.value, StepEnum.DRC.value, + StepEnum.LVS.value, StepEnum.FILLER.value, StepEnum.RCX.value, StepEnum.STA.value, diff --git a/chipcompiler/tools/ecc_dreamplace/module.py b/chipcompiler/tools/ecc_dreamplace/module.py index e35eefc9..757f71a7 100644 --- a/chipcompiler/tools/ecc_dreamplace/module.py +++ b/chipcompiler/tools/ecc_dreamplace/module.py @@ -2,7 +2,6 @@ import json import logging -import os import sys from contextlib import contextmanager from pathlib import Path @@ -60,25 +59,15 @@ def _build_params(self, params_cls, *, legalize_only: bool): return params - def _log_path(self, *, legalize_only: bool) -> str: - log_name = "dreamplace_legalization.log" if legalize_only else "dreamplace_placement.log" - return os.path.join(self.result_dir, log_name) - @contextmanager - def _configure_root_logging(self, *, legalize_only: bool): + def _configure_root_logging(self): root_logger = logging.getLogger() original_handlers = root_logger.handlers[:] original_level = root_logger.level - log_file = self.step.log.file or self._log_path(legalize_only=legalize_only) - os.makedirs(os.path.dirname(log_file) or ".", exist_ok=True) - formatter = logging.Formatter("[%(levelname)-7s] %(message)s") - file_handler = logging.FileHandler(log_file, mode="w", encoding="utf-8") - file_handler.setFormatter(formatter) stdout_handler = logging.StreamHandler(sys.stdout) stdout_handler.setFormatter(formatter) - root_logger.addHandler(file_handler) root_logger.addHandler(stdout_handler) if original_level > logging.INFO: root_logger.setLevel(logging.INFO) @@ -86,9 +75,7 @@ def _configure_root_logging(self, *, legalize_only: bool): try: yield finally: - root_logger.removeHandler(file_handler) root_logger.removeHandler(stdout_handler) - file_handler.close() stdout_handler.close() root_logger.setLevel(original_level) for handler in original_handlers: @@ -99,7 +86,7 @@ def _run(self, *, legalize_only: bool) -> bool: from dreamplace.Params import Params from dreamplace.Placer import PlacementEngine - with self._configure_root_logging(legalize_only=legalize_only): + with self._configure_root_logging(): params = self._build_params(Params, legalize_only=legalize_only) engine = PlacementEngine(params) diff --git a/chipcompiler/tools/ecc_sizer/runner.py b/chipcompiler/tools/ecc_sizer/runner.py index 046cdb2d..df413503 100644 --- a/chipcompiler/tools/ecc_sizer/runner.py +++ b/chipcompiler/tools/ecc_sizer/runner.py @@ -43,19 +43,15 @@ def run_step( output_dir = step.data.workdir_for(step.name) or "" os.makedirs(output_dir, exist_ok=True) - log_path = step.log.file or "" - os.makedirs(os.path.dirname(log_path), exist_ok=True) os.makedirs(os.path.dirname(step.output.def_ or ""), exist_ok=True) command = get_sizer_command() + ["-env", str(env_path), "-f", str(cmd_path)] - with open(log_path, "w", encoding="utf-8") as log_file: - result = subprocess.run( - command, - cwd=str(output_dir), - stdout=log_file, - stderr=subprocess.STDOUT, - check=False, - ) + result = subprocess.run( + command, + cwd=str(output_dir), + stderr=subprocess.STDOUT, + check=False, + ) if result.returncode == 0 and _has_required_outputs(step): sub_flow.update_step(step_name=run_sizer_step, state=StateEnum.Success) diff --git a/chipcompiler/tools/yosys/runner.py b/chipcompiler/tools/yosys/runner.py index 9024c8bb..f5267752 100644 --- a/chipcompiler/tools/yosys/runner.py +++ b/chipcompiler/tools/yosys/runner.py @@ -45,17 +45,11 @@ def run_step(workspace: Workspace, step: YosysStep, ecc_module=None) -> bool: True if synthesis succeeded, False otherwise """ sub_flow = YosysSubFlow(workspace=workspace, workspace_step=step) - log_path = step.log.file or "" yosys_cmd, yosys_env = get_yosys_runtime() if not yosys_cmd: sub_flow.update_step(step_name="run yosys", state=StateEnum.Invalid) error_msg = "Error: yosys is not available (bundled runtime or PATH)." - try: - with open(log_path, "w") as log_file: - log_file.write(error_msg + "\n") - except Exception: - pass logger.error(error_msg) return False @@ -76,24 +70,21 @@ def run_step(workspace: Workspace, step: YosysStep, ecc_module=None) -> bool: cmd = yosys_cmd + ["yosys_synthesis.tcl"] - with open(log_path, "w") as log_file: - step_data = getattr(step, "data", None) - if getattr(step_data, "requires_slang", True) and not check_slang_support( - yosys_cmd=yosys_cmd, - cwd_dir=cwd_dir, - yosys_env=yosys_env, - log_file=log_file, - ): - sub_flow.update_step(step_name="run yosys", state=StateEnum.Invalid) - return False - - result = subprocess.run( - cmd, - cwd=cwd_dir, - env=yosys_env, - stdout=log_file, - stderr=subprocess.STDOUT, - ) + step_data = getattr(step, "data", None) + if getattr(step_data, "requires_slang", True) and not check_slang_support( + yosys_cmd=yosys_cmd, + cwd_dir=cwd_dir, + yosys_env=yosys_env, + ): + sub_flow.update_step(step_name="run yosys", state=StateEnum.Invalid) + return False + + result = subprocess.run( + cmd, + cwd=cwd_dir, + env=yosys_env, + stderr=subprocess.STDOUT, + ) if os.path.exists(step.output.verilog or ""): sub_flow.update_step(step_name="run yosys", state=StateEnum.Success) diff --git a/chipcompiler/tools/yosys/utility.py b/chipcompiler/tools/yosys/utility.py index 9af1ced4..986c1e28 100644 --- a/chipcompiler/tools/yosys/utility.py +++ b/chipcompiler/tools/yosys/utility.py @@ -109,7 +109,7 @@ def get_yosys_runtime() -> tuple[list[str], dict[str, str]]: def check_slang_support( - yosys_cmd: list[str], cwd_dir: str, yosys_env: dict[str, str], log_file, timeout: int = 60 + yosys_cmd: list[str], cwd_dir: str, yosys_env: dict[str, str], timeout: int = 60 ) -> bool: """ Run a lightweight slang frontend availability check. @@ -134,7 +134,6 @@ def check_slang_support( yosys_cmd + ["-Q", "-T", "-p", "plugin -i slang"], cwd=cwd_dir, env=yosys_env, - stdout=log_file, stderr=subprocess.STDOUT, timeout=timeout, ) @@ -146,7 +145,6 @@ def check_slang_support( "Neither builtin read_slang nor a loadable slang plugin was found. " "Please use a yosys build with slang support." ) - log_file.write(error_msg + "\n") print(error_msg) return False diff --git a/chipcompiler/utility/__init__.py b/chipcompiler/utility/__init__.py index abf98fb8..7c45c1d2 100644 --- a/chipcompiler/utility/__init__.py +++ b/chipcompiler/utility/__init__.py @@ -10,11 +10,7 @@ from .json import JsonReadError, dict_to_str, json_read, json_read_strict, json_write from .log import ( Logger, - build_timestamped_log_file, create_logger, - init_api_runtime_log, - redirect_stdio_to_file, - rotate_log_on_start, ) from .plot import plot_bar_chart, plot_csv_bar_chart, plot_csv_map, plot_csv_table, plot_metrics from .util import track_process_memory @@ -28,10 +24,6 @@ "dict_to_str", "Logger", "create_logger", - "build_timestamped_log_file", - "rotate_log_on_start", - "redirect_stdio_to_file", - "init_api_runtime_log", "track_process_memory", "plot_csv_map", "plot_metrics", diff --git a/chipcompiler/utility/log.py b/chipcompiler/utility/log.py index 12578add..ec0d6b51 100644 --- a/chipcompiler/utility/log.py +++ b/chipcompiler/utility/log.py @@ -6,57 +6,10 @@ import sys import time from contextlib import suppress -from datetime import datetime from logging.handlers import RotatingFileHandler -from typing import TextIO # TODO: Move some functions to Logger Module -def build_timestamped_log_file(log_file: str, pid: int | None = None) -> str: - """ - Build a timestamped log file path from a base path. - Example: - /tmp/chipcompiler-api-server.log - -> /tmp/chipcompiler-api-server-20260211-090428-12345.log - """ - resolved_path = os.path.abspath(os.path.expanduser(log_file)) - base_dir = os.path.dirname(resolved_path) or "." - base_name = os.path.basename(resolved_path) - stem, ext = os.path.splitext(base_name) - - timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") - pid_value = os.getpid() if pid is None else pid - - file_name = f"{stem}-{timestamp}-{pid_value}{ext}" if ext else f"{stem}-{timestamp}-{pid_value}" - - return os.path.join(base_dir, file_name) - - -def rotate_log_on_start(log_file: str, max_bytes: int, backup_count: int) -> None: - """Rotate log file at startup if it exceeds max_bytes.""" - if max_bytes <= 0 or not os.path.exists(log_file): - return - try: - if os.path.getsize(log_file) < max_bytes: - return - except OSError: - return - - if backup_count <= 0: - os.remove(log_file) - return - - # Shift existing backups: .5 -> delete, .4 -> .5, ... .1 -> .2 - oldest = f"{log_file}.{backup_count}" - if os.path.exists(oldest): - os.remove(oldest) - for i in range(backup_count - 1, 0, -1): - src, dst = f"{log_file}.{i}", f"{log_file}.{i + 1}" - if os.path.exists(src): - os.replace(src, dst) - os.replace(log_file, f"{log_file}.1") - - def flush_cstdio() -> None: """Flush C stdio buffers (printf/std::cout/glog) that Python-level flushes miss. @@ -68,36 +21,6 @@ def flush_cstdio() -> None: ctypes.CDLL(None).fflush(None) -def redirect_stdio_to_file(log_file: str) -> TextIO: - """Redirect process stdout/stderr to log_file at file-descriptor level.""" - # The stream intentionally stays open: its fd is dup2'd onto stdout/stderr below. - log_stream = open(log_file, "a", encoding="utf-8", buffering=1) # noqa: SIM115 - - for stream in (sys.stdout, sys.stderr): - with suppress(Exception): - stream.flush() - flush_cstdio() - - os.dup2(log_stream.fileno(), 1) - os.dup2(log_stream.fileno(), 2) - sys.stdout = os.fdopen(1, "w", encoding="utf-8", buffering=1, closefd=False) - sys.stderr = os.fdopen(2, "w", encoding="utf-8", buffering=1, closefd=False) - return log_stream - - -def init_api_runtime_log( - log_file: str, - max_bytes: int = 20 * 1024 * 1024, - backup_count: int = 5, -) -> str: - """Initialize API runtime logging: rotate if needed, redirect stdio.""" - resolved = os.path.abspath(os.path.expanduser(log_file)) - os.makedirs(os.path.dirname(resolved) or ".", exist_ok=True) - rotate_log_on_start(resolved, max_bytes, backup_count) - redirect_stdio_to_file(resolved) - return resolved - - class Logger: def __init__( self, diff --git a/docs/examples/gcd/README.cn.md b/docs/examples/gcd/README.cn.md index 8ed8f221..2db9b47d 100644 --- a/docs/examples/gcd/README.cn.md +++ b/docs/examples/gcd/README.cn.md @@ -114,7 +114,12 @@ if not engine_flow.has_init(): # 创建步骤工作空间并运行 engine_flow.create_step_workspaces() -engine_flow.run_steps() +# 进程内直接运行时，用 archive_own_step_logs 把本进程的 fd 1/2 交给客户端归档器， +# 每个步骤的日志文件照常生成，marker 不会泄漏到终端。 +from chipcompiler.runtime.log_stream import archive_own_step_logs + +with archive_own_step_logs(workspace.directory): + engine_flow.run_steps() ``` 定义的流程如下： diff --git a/docs/examples/gcd/README.md b/docs/examples/gcd/README.md index e73a4672..6c29a111 100644 --- a/docs/examples/gcd/README.md +++ b/docs/examples/gcd/README.md @@ -114,7 +114,13 @@ if not engine_flow.has_init(): # Create step workspaces and run engine_flow.create_step_workspaces() -engine_flow.run_steps() +# In-process runs still archive per-step logs client-side: the engine emits +# step markers on fd 2, and this context routes the process's own stream +# through the archiver (markers never reach the terminal). +from chipcompiler.runtime.log_stream import archive_own_step_logs + +with archive_own_step_logs(workspace.directory): + engine_flow.run_steps() ``` The flow we defined is: diff --git a/docs/examples/gcd/ics55flow.py b/docs/examples/gcd/ics55flow.py index 5acfa46b..1ab8deaf 100644 --- a/docs/examples/gcd/ics55flow.py +++ b/docs/examples/gcd/ics55flow.py @@ -6,6 +6,7 @@ get_pdk, ) from chipcompiler.engine import EngineFlow +from chipcompiler.runtime.log_stream import archive_own_step_logs # Setup paths workspace_dir = "./gcd_workspace" @@ -85,4 +86,7 @@ # Create step workspaces and run engine_flow.create_step_workspaces() -engine_flow.run_steps() +# archive_own_step_logs routes this process's fd 1/2 through the client-side +# archiver so each step's log file is written and markers stay off the terminal. +with archive_own_step_logs(workspace_dir): + engine_flow.run_steps() diff --git a/docs/examples/gcd/ics55flow_with_filelist.py b/docs/examples/gcd/ics55flow_with_filelist.py index b7a505b6..d661bba6 100644 --- a/docs/examples/gcd/ics55flow_with_filelist.py +++ b/docs/examples/gcd/ics55flow_with_filelist.py @@ -21,6 +21,7 @@ get_pdk, ) from chipcompiler.engine import EngineFlow +from chipcompiler.runtime.log_stream import archive_own_step_logs # Setup paths workspace_dir = "./gcd_workspace_with_filelist" @@ -158,7 +159,10 @@ # - Runs remaining steps via subprocess for isolation # - Updates state and runtime after each step # - Stops if any step fails (state = Incomplete) -engine_flow.run_steps() +# archive_own_step_logs routes this process's fd 1/2 through the client-side +# archiver so each step's log file is written and markers stay off the terminal. +with archive_own_step_logs(workspace_dir): + engine_flow.run_steps() print("\nFlow completed successfully!") print(f"Check logs and outputs in: {workspace_dir}") diff --git a/docs/specification/marker-protocol.md b/docs/specification/marker-protocol.md new file mode 100644 index 00000000..14594cf6 --- /dev/null +++ b/docs/specification/marker-protocol.md @@ -0,0 +1,175 @@ +# Step Marker Protocol + +This document is the normative specification of the step marker protocol: the +byte-stream convention an executor process (CLI worker or GUI sidecar) uses to +delimit per-step tool output on its stderr stream, and the rules clients +follow to archive that output into per-step log files. + +Status: v1, normative. + +## Design Invariant + +An executor process never opens, writes, or tails step log files. It writes +bytes to fd 1/2 and emits step markers on fd 2. Workspace state files +(flow.json, home.json) and the workspace logger (`log/.log`) are +exempt from this rule; it applies to step tool logs only. + +Each client (CLI, GUI) archives the byte stream into per-step log files as a +client-side result. Files are archival results: any tool that later reads a +step log (GUI history view, `ecc log`, failure-context extraction) is a +read-only consumer and does not care which client wrote it. + +Markers are private to the byte-stream layer. A marker is only ever meaningful +between the producing executor and the consuming archiver in its client. +Matched markers are consumed by the archiver and are never archived, forwarded +to any user-visible surface, or shown to users. + +## Frame Format + +A marker frame is exactly one line on fd 2: + +``` +\x1eECC-STEP {"v":1,"event":"begin","step":"Synthesis","tool":"yosys"}\n +``` + +- `\x1e` is the ASCII Record Separator control character (one byte, `0x1e`). +- The literal text `ECC-STEP ` (with one trailing space) follows immediately, + with no space between `\x1e` and `ECC-STEP`. +- The payload is a single JSON object serialized without insignificant + whitespace. +- The frame is terminated by a single `\n`. + +Payload (version 1): + +```json +{"v":1,"event":"begin","step":"Synthesis","tool":"yosys"} +{"v":1,"event":"end","step":"Synthesis","tool":"yosys"} +``` + +Fields: + +- `v` (number, required): protocol version. This document defines version `1`. +- `event` (string, required): `begin` or `end`. +- `step` (string, required): the flow step name (e.g. `Synthesis`). +- `tool` (string, required): the tool identifier (e.g. `yosys`, `ecc`). + +## Semantics + +Producer rules (executor): + +- `begin` is emitted when a step starts executing, before any of the step's + tool output. +- `end` is emitted according to the ordering guarantee below. +- Emission is unconditional: every execution path has a consumer, so no mode + flag exists. + +Consumer rules (client archiver): + +- Consumers scan for the reserved prefix `\x1eECC-STEP ` at **arbitrary byte + boundaries**, not only at line starts: the producer does not insert a + newline before a frame, so a frame may immediately follow output that lacks + a trailing newline. Bytes preceding a candidate frame are ordinary stream + data. +- A candidate is only consumed when it is a complete, newline-terminated, + valid v1 frame. An incomplete candidate at the end of the buffered stream + is held back (bounded: any candidate longer than 512 bytes without a + newline is not a frame; its bytes are data). +- A `begin` frame while no step is active: open/activate archival for + `(step, tool)`. An `end` frame matching the active `(step, tool)`: close + archival. Both frames are consumed (not archived). +- Any of the following is treated as ordinary stream bytes, never as a + marker: a `begin` while a step is active, an `end` that does not match the + active `(step, tool)`, an unknown `event`, a missing or unsupported `v`, or + a malformed frame (bad JSON, non-dict payload, missing/wrong-typed fields). +- Bytes received while no step is active are *unscoped*: they are not + attributed to any step and must not be written to any step log archive. +- Consumers must drain the stream continuously with a bounded buffer, and must + bound every wait they place on marker arrival (an executor crash may mean an + `end` frame never arrives). + +## Ordering Guarantee + +Within a step, the executor writes the `end` marker: + +- **after** all step-scoped writes have flushed — final state persistence, + `[RESULT]` logging, QOR/metrics refresh, layout snapshot, and db cleanup — + so that a consumer that has read the `end` marker has seen every byte of + that step; and +- **before** the step completion notification (`step.completed`) is published. + +Because the previous step's final state is persisted before its `end` marker, +and the `end` marker precedes the next step's `begin` marker, a consumer may +refresh per-step final states from flow.json on each `begin` marker and once +more when the operation ends. + +If a step's execution or post-processing raises, no `end` marker is emitted +for that step; consumers treat the missing `end` as a crash signal and must +not wait indefinitely for it. + +## Single-Producer Invariant + +At most one flow executes per executor process at a time: + +- GUI: one sidecar per workspace, with `RuntimeOperationConflict` enforced per + workspace. +- CLI: one worker per run operation. + +Frames from different steps therefore never interleave on the stream; a +`begin` while a step is active indicates a protocol violation or foreign +output and is handled as ordinary bytes (see Semantics). + +## Archive Path Layout + +Clients archive step output to: + +``` +/_/log/.log +``` + +Consumers must sanitize `step`/`tool` (reject path separators, `..`, and +empty names) and must verify the resolved path stays inside the workspace +directory before opening it. A frame that fails sanitization or containment +is degraded to ordinary bytes; its bytes are never written outside the +workspace. + +The archive is opened with truncation on each accepted `begin`, so a rerun +starts a fresh byte stream with cursor 0. + +## Allowlist + +On top of sanitization and containment, consumers should restrict archival +to `(step, tool)` pairs read from the workspace's flow.json: a begin marker +whose pair is not in the allowlist degrades to ordinary bytes. The allowlist +is loaded at workspace open and refreshed when an operation starts and when +a rerun is prepared. + +When flow.json cannot be read, each consumer picks its failure default and +documents it here: + +- the **GUI** archiver fails closed (empty allowlist): all markers degrade + to unscoped bytes, which still land in the sidecar log file, so no output + is lost; +- the **CLI** reader fails open (no allowlist): markers are honored after + sanitization and containment only, because the archived step logs are the + CLI's only record of the run. + +Both defaults are safe: containment bounds where bytes can be written, and +the bytes always land in a visible place. + + +## Protocol Change: Live-Log Events Are Client-Synthesized + +As of this protocol version, ecc no longer emits `step.log` notifications or +attaches `finalLog` to `step.completed` on the runtime event channel. Live +step-log events are synthesized by the client that archives the stream: + +- the CLI archives silently (its terminal UI renders from the same stream); +- the GUI's Electron process synthesizes `step.log` / `finalLog` events for + its renderer from the archived bytes. + +Consequence (DEC-1, accepted 2026-08-18): the GUI is the only supported +consumer of operation live logs. The event shapes the GUI renderer consumes +(`step.log` payload `{chunk, cursor, step, tool}`; `finalLog` on +`step.completed`) are unchanged — only their producer moved, from ecc +file-tailing to client-side synthesis. Versions are pinned via the +superproject's ecc submodule, so producer and consumer always match. diff --git a/test/cli/commands/conftest.py b/test/cli/commands/conftest.py index 93cb7cd5..d4677395 100644 --- a/test/cli/commands/conftest.py +++ b/test/cli/commands/conftest.py @@ -56,6 +56,13 @@ def fake_create_workspace(**kwargs): capture["create_kwargs"] = kwargs return workspace_obj + def fake_run_flow_via_worker(workspace_dir): + from chipcompiler.runtime.worker_operation import OperationResult + + for inst in DummyFlow.instances: + inst.run_called = True + return OperationResult(success=DummyFlow.run_steps_value, exit_code=0) + monkeypatch.setattr("chipcompiler.data.create_workspace", fake_create_workspace) monkeypatch.setattr("chipcompiler.engine.EngineFlow", DummyFlow) monkeypatch.setattr( @@ -66,5 +73,9 @@ def fake_create_workspace(**kwargs): "chipcompiler.cli.project.config._validate_pdk_contents", lambda name, root, overrides=None: None, ) + monkeypatch.setattr( + "chipcompiler.cli.command_handlers.project._run_flow_via_worker", + fake_run_flow_via_worker, + ) return SimpleNamespace(capture=capture, flow=DummyFlow) diff --git a/test/cli/commands/test_run.py b/test/cli/commands/test_run.py index 2c716870..a6870cbb 100644 --- a/test/cli/commands/test_run.py +++ b/test/cli/commands/test_run.py @@ -5,7 +5,9 @@ import pytest from chipcompiler.cli import main as cli_main -from chipcompiler.engine import StepRunResult +from chipcompiler.cli.command_handlers import project as project_module + +_REAL_RUN_FLOW_VIA_WORKER = project_module._run_flow_via_worker def _set_flow_preset(project_dir, preset): @@ -240,18 +242,20 @@ def test_run_forwards_absolute_paths_with_relative_env_pdk_root( class TestWorkspaceRun: @pytest.fixture - def workspace_mocks(self, monkeypatch): + def workspace_mocks(self, monkeypatch, tmp_path): + from chipcompiler.runtime.worker_operation import OperationResult + seen = SimpleNamespace( load_path=None, has_init=True, - selected_error=None, - selected=None, - create_calls=0, - executable=None, - only=None, - from_step=None, - resume=False, - result=StepRunResult(ok=True, executed=("place",)), + calls=None, + result=OperationResult(success=True, exit_code=0), + binary_error=None, + steps=[ + {"name": "Synthesis", "tool": "yosys", "state": "Success"}, + {"name": "place", "tool": "ecc", "state": "Imcomplete"}, + {"name": "CTS", "tool": "ecc", "state": "Unstart"}, + ], ) class Flow: @@ -261,48 +265,40 @@ def __init__(self, workspace): def has_init(self): return seen.has_init - def create_step_workspaces(self, *, executable_steps=None): - seen.create_calls += 1 - seen.executable = executable_steps - - def selected_step_names(flow, *, from_step=None, only=None, force=False): - if seen.selected_error is not None: - raise seen.selected_error - seen.selected = {"from_step": from_step, "only": only, "force": force} - if only is not None: - return [] if not force else [only] - if from_step is not None: - return [from_step, "CTS"] - return ["place", "CTS"] - - def run_only(flow, name, *, force=False): - seen.only = (name, force) - return seen.result - - def run_from(flow, name): - seen.from_step = name - return seen.result - - def run_resume(flow): - seen.resume = True - return seen.result - def fake_load_workspace(path): seen.load_path = path - return SimpleNamespace(name="workspace") + return SimpleNamespace( + name="workspace", + flow=SimpleNamespace(data={"steps": [dict(step) for step in seen.steps]}), + ) + + class FakeOperation: + def run_sequence(self, calls): + seen.calls = calls + return seen.result monkeypatch.setattr("chipcompiler.data.load_workspace", fake_load_workspace) monkeypatch.setattr("chipcompiler.engine.EngineFlow", Flow) - monkeypatch.setattr("chipcompiler.engine.rerun.selected_step_names", selected_step_names) - monkeypatch.setattr("chipcompiler.engine.rerun.run_only", run_only) - monkeypatch.setattr("chipcompiler.engine.rerun.run_from", run_from) - monkeypatch.setattr("chipcompiler.engine.rerun.run_resume", run_resume) + monkeypatch.setattr( + "chipcompiler.cli.command_handlers.workspace_run._make_run_operation", + lambda workspace_path, **kwargs: FakeOperation(), + ) + monkeypatch.setattr( + "chipcompiler.cli.command_handlers.workspace_run._worker_binary_missing_error", + lambda: seen.binary_error, + ) monkeypatch.setattr( "chipcompiler.data.create_workspace", lambda **_kwargs: pytest.fail("workspace mode must not create a workspace"), ) return seen + def _write_post_run_flow(self, workspace, steps): + home = os.path.join(workspace, "home") + os.makedirs(home, exist_ok=True) + with open(os.path.join(home, "flow.json"), "w") as f: + json.dump({"steps": steps}, f) + def test_only_force_wiring(self, workspace_mocks, tmp_path, capsys): workspace = str(tmp_path / "workspace") @@ -311,40 +307,37 @@ def test_only_force_wiring(self, workspace_mocks, tmp_path, capsys): record = json.loads(capsys.readouterr().out)["records"][0] assert rc == 0 assert workspace_mocks.load_path == workspace - assert workspace_mocks.selected == {"from_step": None, "only": "place", "force": True} - assert workspace_mocks.executable == {"place"} - assert workspace_mocks.only == ("place", True) + assert workspace_mocks.calls == [ + ("flow.run_step", {"step": "place", "rerun": True, "invalidate_dependents": True}) + ] assert record["run"] == "workspace" assert record["status"] == "success" assert record["workspace"] == workspace assert record["executed_steps"] == ["place"] assert record["no_op"] is False - def test_default_selector_is_resume(self, workspace_mocks, tmp_path): - rc = cli_main.run(["run", "--workspace", str(tmp_path / "workspace"), "--plain"]) - - assert rc == 0 - assert workspace_mocks.selected == {"from_step": None, "only": None, "force": False} - assert workspace_mocks.executable == {"place", "CTS"} - assert workspace_mocks.resume is True + def test_only_without_force_runs_step(self, workspace_mocks, tmp_path): + workspace = str(tmp_path / "workspace") - def test_from_step_wiring(self, workspace_mocks, tmp_path): - rc = cli_main.run(["run", "--workspace", str(tmp_path / "workspace"), "--from", "CTS"]) + rc = cli_main.run(["run", "--workspace", workspace, "--only", "place", "--json"]) assert rc == 0 - assert workspace_mocks.from_step == "CTS" - assert workspace_mocks.executable == {"CTS"} - - def test_noop_selection_skips_workspace_rebuild(self, workspace_mocks, tmp_path, capsys): - workspace_mocks.result = StepRunResult(ok=True, executed=()) + assert workspace_mocks.calls == [ + ("flow.run_step", {"step": "place", "rerun": True, "invalidate_dependents": True}) + ] + + def test_only_success_step_without_force_is_noop(self, workspace_mocks, tmp_path, capsys): + workspace_mocks.steps = [ + {"name": "Synthesis", "tool": "yosys", "state": "Success"}, + {"name": "place", "tool": "ecc", "state": "Success"}, + ] workspace = str(tmp_path / "workspace") rc = cli_main.run(["run", "--workspace", workspace, "--only", "place", "--json"]) record = json.loads(capsys.readouterr().out)["records"][0] assert rc == 0 - assert workspace_mocks.create_calls == 0 - assert workspace_mocks.only == ("place", False) + assert workspace_mocks.calls is None assert record == { "run": "workspace", "status": "success", @@ -353,23 +346,156 @@ def test_noop_selection_skips_workspace_rebuild(self, workspace_mocks, tmp_path, "no_op": True, } + def test_default_selector_is_resume(self, workspace_mocks, tmp_path, capsys): + workspace = str(tmp_path / "workspace") + + rc = cli_main.run(["run", "--workspace", workspace, "--json"]) + + record = json.loads(capsys.readouterr().out)["records"][0] + assert rc == 0 + # The persisted suffix is driven step by step: an unscoped flow.run + # would resume from the first non-success step, which may sit before + # the selected boundary. + assert workspace_mocks.calls == [ + ("flow.run_step", {"step": "place", "rerun": True, "reset_dependents": True}), + ("flow.run_step", {"step": "CTS", "rerun": True}), + ] + assert record["executed_steps"] == ["place", "CTS"] + + def test_from_step_wiring(self, workspace_mocks, tmp_path, capsys): + workspace = str(tmp_path / "workspace") + + rc = cli_main.run(["run", "--workspace", workspace, "--from", "CTS", "--json"]) + + record = json.loads(capsys.readouterr().out)["records"][0] + assert rc == 0 + assert workspace_mocks.calls == [ + ("flow.run_step", {"step": "CTS", "rerun": True, "reset_dependents": True}), + ] + assert record["executed_steps"] == ["CTS"] + + def test_preflight_rejects_an_unavailable_tool_before_any_mutation( + self, workspace_mocks, tmp_path, capsys, monkeypatch + ): + """A later step with an unavailable tool fails before the first + worker call — nothing is invalidated or deleted.""" + import chipcompiler.tools.eda as eda_module + + real_load = eda_module.load_eda_module + + def fake_load(tool, *, check_dependency=True): + if tool == "ecc" and check_dependency: + return None # CTS's tool is unavailable + return real_load(tool, check_dependency=check_dependency) + + monkeypatch.setattr("chipcompiler.tools.eda.load_eda_module", fake_load) + workspace = str(tmp_path / "workspace") + + rc = cli_main.run(["run", "--workspace", workspace, "--from", "place", "--json"]) + + record = json.loads(capsys.readouterr().out)["records"][0] + assert rc != 0 + assert record["error"] == "config_error" + assert workspace_mocks.calls is None or workspace_mocks.calls == [] + + def test_from_step_never_runs_steps_before_the_boundary( + self, workspace_mocks, tmp_path, capsys + ): + # Regression: with a failed step BEFORE the --from boundary, the + # previous run_step+flow.run sequence let flow.run resume from that + # earlier step, executing outside the requested suffix. + workspace_mocks.steps = [ + {"name": "Synthesis", "tool": "yosys", "state": "Imcomplete"}, + {"name": "place", "tool": "ecc", "state": "Imcomplete"}, + {"name": "CTS", "tool": "ecc", "state": "Unstart"}, + ] + workspace = str(tmp_path / "workspace") + + rc = cli_main.run(["run", "--workspace", workspace, "--from", "place", "--json"]) + + record = json.loads(capsys.readouterr().out)["records"][0] + assert rc == 0 + assert workspace_mocks.calls == [ + ("flow.run_step", {"step": "place", "rerun": True, "reset_dependents": True}), + ("flow.run_step", {"step": "CTS", "rerun": True}), + ] + assert record["executed_steps"] == ["place", "CTS"] + + def test_resume_all_success_is_noop(self, workspace_mocks, tmp_path, capsys): + workspace_mocks.steps = [ + {"name": "Synthesis", "tool": "yosys", "state": "Success"}, + {"name": "place", "tool": "ecc", "state": "Success"}, + ] + workspace = str(tmp_path / "workspace") + + rc = cli_main.run(["run", "--workspace", workspace, "--resume", "--json"]) + + record = json.loads(capsys.readouterr().out)["records"][0] + assert rc == 0 + assert workspace_mocks.calls is None + assert record["no_op"] is True + assert record["executed_steps"] == [] + def test_failed_run_reports_failed_step_and_resume(self, workspace_mocks, tmp_path, capsys): - workspace_mocks.result = StepRunResult(ok=False, executed=(), failed="place") + from chipcompiler.runtime.worker_operation import OperationResult + + workspace_mocks.result = OperationResult( + success=False, error="run step place failed with state Imcomplete" + ) workspace = str(tmp_path / "workspace") + self._write_post_run_flow( + workspace, + [ + {"name": "Synthesis", "tool": "yosys", "state": "Success"}, + {"name": "place", "tool": "ecc", "state": "Imcomplete"}, + {"name": "CTS", "tool": "ecc", "state": "Unstart"}, + ], + ) - rc = cli_main.run(["run", "--workspace", workspace, "--only", "place", "--json"]) + rc = cli_main.run(["run", "--workspace", workspace, "--only", "place", "--force", "--json"]) record = json.loads(capsys.readouterr().out)["records"][0] assert rc == 1 - assert record == { - "run": "workspace", - "status": "failed", - "workspace": workspace, - "executed_steps": [], - "no_op": False, - "failed_step": "place", - "resume_cmd": f"ecc run --workspace {workspace} --resume", - } + assert record["status"] == "failed" + assert record["executed_steps"] == [] + assert record["failed_step"] == "place" + assert record["resume_cmd"] == f"ecc run --workspace {workspace} --resume" + assert "place" in record["error"] + + def test_failed_suffix_run_reports_executed_prefix(self, workspace_mocks, tmp_path, capsys): + from chipcompiler.runtime.worker_operation import OperationResult + + workspace_mocks.result = OperationResult(success=False, error="run flow failed") + workspace = str(tmp_path / "workspace") + self._write_post_run_flow( + workspace, + [ + {"name": "Synthesis", "tool": "yosys", "state": "Success"}, + {"name": "place", "tool": "ecc", "state": "Success"}, + {"name": "CTS", "tool": "ecc", "state": "Imcomplete"}, + ], + ) + + rc = cli_main.run(["run", "--workspace", workspace, "--resume", "--json"]) + + record = json.loads(capsys.readouterr().out)["records"][0] + assert rc == 1 + assert record["executed_steps"] == ["place"] + assert record["failed_step"] == "CTS" + + def test_missing_worker_binary_returns_structured_failure( + self, workspace_mocks, tmp_path, capsys + ): + workspace_mocks.binary_error = "worker binary not found: /missing/ecc" + workspace = str(tmp_path / "workspace") + + rc = cli_main.run(["run", "--workspace", workspace, "--json"]) + + record = json.loads(capsys.readouterr().out)["records"][0] + assert rc == 1 + assert record["status"] == "failed" + assert "not found" in record["error"] + assert workspace_mocks.calls is None def test_invalid_workspace(self, tmp_path, capsys): rc = cli_main.run(["run", "--workspace", str(tmp_path / "missing"), "--json"]) @@ -388,8 +514,6 @@ def test_missing_flow(self, workspace_mocks, tmp_path, capsys): assert record["error"] == "missing_flow" def test_unknown_step(self, workspace_mocks, tmp_path, capsys): - workspace_mocks.selected_error = ValueError("unknown step 'bogus'") - rc = cli_main.run( ["run", "--workspace", str(tmp_path / "workspace"), "--only", "bogus", "--json"] ) @@ -426,3 +550,17 @@ def test_option_conflicts(self, argv, error, capsys, monkeypatch): record = json.loads(capsys.readouterr().out)["records"][0] assert rc == 1 assert record["error"] == error + + +class TestRunFlowViaWorkerFailure: + def test_missing_binary_returns_structured_failure(self, tmp_path, monkeypatch): + """A missing worker binary is a typed failure, not a crash.""" + monkeypatch.setattr( + "chipcompiler.runtime.worker_operation._default_worker_argv", + lambda: [str(tmp_path / "nonexistent_ecc"), "rpc", "serve", "--stdio"], + ) + + result = _REAL_RUN_FLOW_VIA_WORKER(str(tmp_path)) + + assert result.success is False + assert "not found" in result.error diff --git a/test/cli/commands/test_run_worker.py b/test/cli/commands/test_run_worker.py new file mode 100644 index 00000000..b68f7386 --- /dev/null +++ b/test/cli/commands/test_run_worker.py @@ -0,0 +1,296 @@ +"""CLI-level run tests that execute against a real worker subprocess.""" + +import json +import os +import sys +import textwrap +from types import SimpleNamespace + +import pytest + +from chipcompiler.cli import main as cli_main +from chipcompiler.cli.command_handlers import project as project_module + +# Captured at import time, before the autouse fixture replaces the module attr. +_REAL_RUN_FLOW_VIA_WORKER = project_module._run_flow_via_worker + +_FAKE_WORKER = textwrap.dedent("""\ + import sys, os, json + + def read_request(): + data = b"" + while True: + chunk = sys.stdin.buffer.read(1) + if not chunk: + return None + data += chunk + if b"\\r\\n\\r\\n" in data: + header, _, body_start = data.partition(b"\\r\\n\\r\\n") + length = int(header.split(b":")[1]) + while len(body_start) < length: + body_start += sys.stdin.buffer.read(1) + return json.loads(body_start[:length]) + + def send_response(resp): + payload = json.dumps(resp) + frame = f"Content-Length: {len(payload)}\\r\\n\\r\\n{payload}" + sys.stdout.buffer.write(frame.encode()) + sys.stdout.buffer.flush() + + def make_marker(event, step, tool): + payload = json.dumps({"v": 1, "event": event, "step": step, "tool": tool}) + return chr(0x1e).encode() + b"ECC-STEP " + payload.encode() + b"\\n" + + def flow_json_path(ws_dir): + return os.path.join(ws_dir, "home", "flow.json") + + def run_one_step(ws_dir, name, tool): + os.write(2, make_marker("begin", name, tool)) + os.write(2, ("output of " + name + "\\n").encode()) + os.write(2, make_marker("end", name, tool)) + path = flow_json_path(ws_dir) + with open(path) as handle: + data = json.load(handle) + for record in data["steps"]: + if record["name"] == name: + record["state"] = "Success" + record["runtime"] = "0:00:01" + with open(path, "w") as handle: + json.dump(data, handle) + + def run_pending(ws_dir): + path = flow_json_path(ws_dir) + with open(path) as handle: + data = json.load(handle) + for record in data["steps"]: + if record.get("state") != "Success": + run_one_step(ws_dir, record["name"], record.get("tool", "ecc")) + + ws_dir = "" + while True: + req = read_request() + if req is None: + break + method = req.get("method", "") + req_id = req.get("id") + if method == "rpc.hello": + send_response({"jsonrpc": "2.0", "result": {"version": 1}, "id": req_id}) + elif method == "workspace.open": + ws_dir = req["params"]["directory"] + result = {"workspaceId": "fake-worker"} + send_response({"jsonrpc": "2.0", "result": result, "id": req_id}) + elif method == "flow.run_step": + params = req["params"] + step = params["step"] + tool = "ecc" + with open(flow_json_path(ws_dir)) as handle: + flow_data = json.load(handle) + for record in flow_data["steps"]: + if record["name"] == step: + tool = record.get("tool", "ecc") + if params.get("invalidate_dependents"): + seen_target = False + for record in flow_data["steps"]: + if record["name"] == step: + seen_target = True + continue + if seen_target: + record["state"] = "Unstart" + with open(flow_json_path(ws_dir), "w") as handle: + json.dump(flow_data, handle) + run_one_step(ws_dir, step, tool) + result = {"step": step, "state": "Success"} + send_response({"jsonrpc": "2.0", "result": result, "id": req_id}) + elif method == "flow.run": + run_pending(ws_dir) + send_response({"jsonrpc": "2.0", "result": {"rerun": False}, "id": req_id}) + elif method == "rpc.shutdown": + send_response({"jsonrpc": "2.0", "result": {"ok": True}, "id": req_id}) + break + else: + err = {"code": -32601, "message": "unknown method"} + send_response({"jsonrpc": "2.0", "error": err, "id": req_id}) +""") + + +def _write_flow_json(workspace_dir, steps): + home = os.path.join(workspace_dir, "home") + os.makedirs(home, exist_ok=True) + with open(os.path.join(home, "flow.json"), "w") as handle: + json.dump({"steps": steps}, handle) + + +def _read_flow_states(workspace_dir): + with open(os.path.join(workspace_dir, "home", "flow.json")) as handle: + return {record["name"]: record["state"] for record in json.load(handle)["steps"]} + + +@pytest.fixture +def fake_worker(tmp_path, monkeypatch): + script = tmp_path / "fake_worker.py" + script.write_text(_FAKE_WORKER) + monkeypatch.setattr( + "chipcompiler.runtime.worker_operation._default_worker_argv", + lambda: [sys.executable, str(script)], + ) + return script + + +class TestWorkspaceRunWithRealWorker: + @pytest.fixture + def validation_mocks(self, monkeypatch): + class Flow: + def __init__(self, workspace): + self.workspace = workspace + + def has_init(self): + return True + + def fake_load_workspace(path): + return SimpleNamespace( + name="workspace", + flow=SimpleNamespace( + data={ + "steps": [ + {"name": "Synthesis", "tool": "yosys", "state": "Success"}, + {"name": "place", "tool": "ecc", "state": "Imcomplete"}, + {"name": "CTS", "tool": "ecc", "state": "Unstart"}, + ] + } + ), + ) + + monkeypatch.setattr("chipcompiler.data.load_workspace", fake_load_workspace) + monkeypatch.setattr("chipcompiler.engine.EngineFlow", Flow) + + def test_resume_runs_suffix_through_worker_and_archives( + self, fake_worker, validation_mocks, tmp_path, capsys + ): + workspace = str(tmp_path / "workspace") + _write_flow_json( + workspace, + [ + {"name": "Synthesis", "tool": "yosys", "state": "Success"}, + {"name": "place", "tool": "ecc", "state": "Imcomplete"}, + {"name": "CTS", "tool": "ecc", "state": "Unstart"}, + ], + ) + + rc = cli_main.run(["run", "--workspace", workspace, "--resume", "--json"]) + + record = json.loads(capsys.readouterr().out)["records"][0] + assert rc == 0 + assert record["status"] == "success" + assert record["executed_steps"] == ["place", "CTS"] + assert _read_flow_states(workspace) == { + "Synthesis": "Success", + "place": "Success", + "CTS": "Success", + } + for step, tool in (("place", "ecc"), ("CTS", "ecc")): + log_path = os.path.join(workspace, f"{step}_{tool}", "log", f"{step}.log") + with open(log_path, "rb") as handle: + content = handle.read() + assert f"output of {step}\n".encode() in content + assert b"ECC-STEP" not in content + + def test_only_executes_single_step(self, fake_worker, validation_mocks, tmp_path, capsys): + workspace = str(tmp_path / "workspace") + _write_flow_json( + workspace, + [ + {"name": "Synthesis", "tool": "yosys", "state": "Success"}, + {"name": "place", "tool": "ecc", "state": "Imcomplete"}, + {"name": "CTS", "tool": "ecc", "state": "Unstart"}, + ], + ) + + rc = cli_main.run(["run", "--workspace", workspace, "--only", "place", "--json"]) + + record = json.loads(capsys.readouterr().out)["records"][0] + assert rc == 0 + assert record["executed_steps"] == ["place"] + assert _read_flow_states(workspace) == { + "Synthesis": "Success", + "place": "Success", + "CTS": "Unstart", + } + assert not os.path.exists(os.path.join(workspace, "CTS_ecc")) + + def test_only_force_marks_downstream_unstart_but_keeps_outputs( + self, fake_worker, tmp_path, capsys, monkeypatch + ): + workspace = str(tmp_path / "workspace") + _write_flow_json( + workspace, + [ + {"name": "Synthesis", "tool": "yosys", "state": "Success"}, + {"name": "place", "tool": "ecc", "state": "Success"}, + {"name": "CTS", "tool": "ecc", "state": "Success"}, + ], + ) + cts_output = os.path.join(workspace, "CTS_ecc", "output") + os.makedirs(cts_output) + with open(os.path.join(cts_output, "result.def"), "w") as handle: + handle.write("old") + + class Flow: + def __init__(self, workspace): + self.workspace = workspace + + def has_init(self): + return True + + def fake_load_workspace(path): + return SimpleNamespace( + name="workspace", + flow=SimpleNamespace( + data={ + "steps": [ + {"name": "Synthesis", "tool": "yosys", "state": "Success"}, + {"name": "place", "tool": "ecc", "state": "Success"}, + {"name": "CTS", "tool": "ecc", "state": "Success"}, + ] + } + ), + ) + + monkeypatch.setattr("chipcompiler.data.load_workspace", fake_load_workspace) + monkeypatch.setattr("chipcompiler.engine.EngineFlow", Flow) + + rc = cli_main.run(["run", "--workspace", workspace, "--only", "place", "--force", "--json"]) + + record = json.loads(capsys.readouterr().out)["records"][0] + assert rc == 0 + assert record["executed_steps"] == ["place"] + assert _read_flow_states(workspace) == { + "Synthesis": "Success", + "place": "Success", + "CTS": "Unstart", + } + with open(os.path.join(cts_output, "result.def")) as handle: + assert handle.read() == "old" + + +class TestFlowRunViaWorkerArchival: + def test_non_tty_run_archives_step_logs_without_markers( + self, fake_worker, tmp_path, monkeypatch + ): + workspace = str(tmp_path / "workspace") + _write_flow_json( + workspace, + [ + {"name": "Synthesis", "tool": "yosys", "state": "Unstart"}, + {"name": "place", "tool": "ecc", "state": "Unstart"}, + ], + ) + + result = _REAL_RUN_FLOW_VIA_WORKER(workspace) + + assert result.success is True + for step, tool in (("Synthesis", "yosys"), ("place", "ecc")): + log_path = os.path.join(workspace, f"{step}_{tool}", "log", f"{step}.log") + with open(log_path, "rb") as handle: + content = handle.read() + assert f"output of {step}\n".encode() in content + assert b"ECC-STEP" not in content diff --git a/test/cli/conftest.py b/test/cli/conftest.py index 5855c8f0..4d301bbf 100644 --- a/test/cli/conftest.py +++ b/test/cli/conftest.py @@ -196,3 +196,18 @@ def factory(): mock_pdk_validation(monkeypatch) return factory + + +@pytest.fixture(autouse=True) +def _disable_worker_routing(monkeypatch): + """Simulate successful worker execution in CLI tests by default. + + Tests that need to exercise the worker path should override this by + patching _run_flow_via_worker themselves (e.g. via flow_mocks fixture). + """ + from chipcompiler.runtime.worker_operation import OperationResult + + monkeypatch.setattr( + "chipcompiler.cli.command_handlers.project._run_flow_via_worker", + lambda workspace_dir, **_kwargs: OperationResult(success=True, exit_code=0), + ) diff --git a/test/cli/rendering/test_progress.py b/test/cli/rendering/test_progress.py index 442049e8..bc154a8c 100644 --- a/test/cli/rendering/test_progress.py +++ b/test/cli/rendering/test_progress.py @@ -1,4 +1,5 @@ import io +import json import os import re import sys @@ -6,8 +7,6 @@ import time from pathlib import Path -import pytest - import chipcompiler.cli.rendering.progress as progress from chipcompiler.cli.core.types import CommandContext, OutputMode from chipcompiler.cli.inspection.log_view import LineKind, LogLine @@ -15,7 +14,6 @@ from chipcompiler.cli.rendering.progress import ( RunProgressRenderer, format_error_context, - latest_log_line, run_flow_with_progress, sanitize_log_line, should_enable_run_progress, @@ -23,8 +21,7 @@ supports_color, truncate_to_width, ) -from chipcompiler.data import LogPaths, StateEnum -from chipcompiler.utility.log import redirect_stdio_to_file +from chipcompiler.runtime.worker_operation import OperationResult _ANSI_RE = re.compile(r"\x1b\[[0-9;]*[a-zA-Z]") @@ -202,178 +199,6 @@ def test_small_width(self): assert truncate_to_width("hello", 2) == "he" -# -- latest_log_line -- - - -class TestLatestLogLine: - def test_returns_last_nonempty_line(self, tmp_path): - log = tmp_path / "test.log" - log.write_text("line one\nline two\n\n") - assert latest_log_line(str(log)) == "line two" - - def test_returns_none_for_missing_file(self): - assert latest_log_line("/nonexistent/file.log") is None - - def test_returns_none_for_empty_file(self, tmp_path): - log = tmp_path / "empty.log" - log.write_text("") - assert latest_log_line(str(log)) is None - - def test_returns_none_for_none_path(self): - assert latest_log_line(None) is None - - def test_sanitizes_ansi_in_line(self, tmp_path): - log = tmp_path / "ansi.log" - log.write_text("\x1b[32mprogress\x1b[0m\n") - assert latest_log_line(str(log)) == "progress" - - def test_trailing_newlines_only(self, tmp_path): - log = tmp_path / "nl.log" - log.write_text("\n\n\n") - assert latest_log_line(str(log)) is None - - -# -- incremental log tail -- - - -class TestIncrementalLogTail: - def test_reads_only_appended_complete_lines(self, tmp_path): - log = tmp_path / "step.log" - log.write_text("first\n") - tail = progress._IncrementalLogTail(str(log), "floorplan", stale_after=10.0) - - assert tail.poll(now=0.0) == "first" - - log.write_text("first\nsecond\n") - - assert tail.poll(now=1.0) == "second" - - def test_carries_partial_line_until_newline_arrives(self, tmp_path): - log = tmp_path / "step.log" - log.write_text("partial") - tail = progress._IncrementalLogTail(str(log), "floorplan", stale_after=10.0) - - assert tail.poll(now=0.0) == "running floorplan, waiting for step log 0s..." - - log.write_text("partial line\n") - - assert tail.poll(now=1.0) == "partial line" - - def test_ignores_empty_or_pure_control_lines(self, tmp_path): - log = tmp_path / "step.log" - log.write_text("\x1b[31m\x1b[0m\n\nreal\n") - tail = progress._IncrementalLogTail(str(log), "floorplan", stale_after=10.0) - - assert tail.poll(now=0.0) == "real" - - def test_restarts_when_file_is_truncated(self, tmp_path): - log = tmp_path / "step.log" - log.write_text("old\n") - tail = progress._IncrementalLogTail(str(log), "floorplan", stale_after=10.0) - assert tail.poll(now=0.0) == "old" - - log.write_text("new\n") - - assert tail.poll(now=1.0) == "new" - - def test_restarts_when_replaced_file_grows_beyond_previous_offset(self, tmp_path): - log = tmp_path / "step.log" - log.write_text("old\n") - tail = progress._IncrementalLogTail(str(log), "floorplan", stale_after=10.0) - assert tail.poll(now=0.0) == "old" - - log.write_text("replacement line\n") - - assert tail.poll(now=1.0) == "replacement line" - - def test_reports_stale_status_without_losing_last_line(self, tmp_path): - log = tmp_path / "step.log" - log.write_text("StaDataPropagation.cc:710] data bwd propagation start\n") - tail = progress._IncrementalLogTail(str(log), "fixfanout", stale_after=5.0) - assert tail.poll(now=10.0) == "StaDataPropagation.cc:710] data bwd propagation start" - - assert ( - tail.poll(now=16.0) == "running fixfanout, last log 6s ago: " - "StaDataPropagation.cc:710] data bwd propagation start" - ) - assert tail.last_line == "StaDataPropagation.cc:710] data bwd propagation start" - - -class TestMonitorLogProgress: - def test_late_created_log_updates_after_initial_waiting_status(self, tmp_path): - log = tmp_path / "late.log" - renderer = RecordingRenderer() - stop_event = threading.Event() - monitor = threading.Thread( - target=progress._monitor_log_progress, - args=(renderer, str(log), "floorplan", stop_event), - kwargs={"interval": 0.01, "stale_after": 10.0}, - daemon=True, - ) - - monitor.start() - try: - assert _wait_until( - lambda: renderer.has_line_containing("waiting for step log"), timeout=1.0 - ) - log.write_text("first appended line\n") - assert _wait_until( - lambda: renderer.has_line_containing("first appended line"), timeout=1.0 - ) - finally: - stop_event.set() - monitor.join(timeout=1.0) - - def test_silent_log_switches_from_banner_to_stale_status(self, tmp_path): - log = tmp_path / "silent.log" - log.write_text("|_| |_/_/\\_\\ |_|\n") - renderer = RecordingRenderer() - stop_event = threading.Event() - monitor = threading.Thread( - target=progress._monitor_log_progress, - args=(renderer, str(log), "fixfanout", stop_event), - kwargs={"interval": 0.01, "stale_after": 0.03}, - daemon=True, - ) - - monitor.start() - try: - assert _wait_until(lambda: renderer.has_line_containing("|_| |_"), timeout=1.0) - assert _wait_until(lambda: renderer.has_line_containing("last log"), timeout=1.0) - assert renderer.has_line_containing("running fixfanout") - finally: - stop_event.set() - monitor.join(timeout=1.0) - - def test_isolated_monitor_renders_stale_status_while_main_thread_is_busy(self, tmp_path): - log = tmp_path / "silent.log" - output = tmp_path / "progress.txt" - log.write_text("|_| |_/_/\\_\\ |_|\n") - - with open(output, "w", encoding="utf-8", buffering=1) as stream: - renderer = RunProgressRenderer(stream, color=False) - stop_event, monitor = progress._start_log_monitor( - renderer, - str(log), - "fixfanout", - isolated=True, - interval=0.01, - stale_after=0.03, - ) - previous_interval = sys.getswitchinterval() - try: - sys.setswitchinterval(0.5) - deadline = time.time() + 0.15 - while time.time() < deadline: - pass - finally: - sys.setswitchinterval(previous_interval) - stop_event.set() - monitor.join(timeout=1.0) - - assert "running fixfanout, last log" in output.read_text() - - # -- RunProgressRenderer -- @@ -579,270 +404,337 @@ def test_preserves_fd_stream_error_handler(self, tmp_path): assert "\\u2713" in path.read_text() -class TestPreserveCliStdio: - def test_restores_fd_stdout_stderr_after_redirect(self, tmp_path, capfd): - log_file = tmp_path / "step.log" +# --------------------------------------------------------------------------- +# Failure context block formatting (AC-5) +# --------------------------------------------------------------------------- - with progress.preserve_cli_stdio(): - redirected = redirect_stdio_to_file(str(log_file)) - print("inside stdout") - sys.stderr.write("inside stderr\n") - redirected.flush() - print("after stdout") - sys.stderr.write("after stderr\n") +class TestFormatErrorContext: + def test_first_line_is_error_log_path(self): + ctx_lines = [LogLine(10, LineKind.ERROR, "Error: something")] + out = format_error_context("log/synthesis.log", ctx_lines, "ecc log synthesis", color=False) + assert out.startswith("error: log/synthesis.log") - captured = capfd.readouterr() - assert "after stdout" in captured.out - assert "after stderr" in captured.err - assert "after stdout" not in log_file.read_text() - assert "after stderr" not in log_file.read_text() + def test_includes_numbered_context_lines(self): + ctx_lines = [ + LogLine(8, LineKind.INFO, "INFO: before"), + LogLine(9, LineKind.WARNING, "Warning: careful"), + LogLine(10, LineKind.ERROR, "Error: failed"), + ] + out = format_error_context("log/synthesis.log", ctx_lines, "ecc log synthesis", color=False) + for ll in ctx_lines: + assert str(ll.line_no) in out + assert ll.text in out - def test_restores_fd_stdout_stderr_after_exception(self, tmp_path, capfd): - log_file = tmp_path / "step.log" + def test_compact_kind_labels(self): + ctx_lines = [ + LogLine(5, LineKind.ERROR, "bad"), + LogLine(6, LineKind.WARNING, "meh"), + LogLine(7, LineKind.TRACEBACK, " File ..."), + LogLine(8, LineKind.INFO, "ok"), + ] + out = format_error_context("log/p.log", ctx_lines, "ecc log step", color=False) + assert "ERROR" in out + assert "WARN" in out + assert "TRACE" in out + assert "INFO" in out - with pytest.raises(RuntimeError, match="boom"), progress.preserve_cli_stdio(): - redirect_stdio_to_file(str(log_file)) - raise RuntimeError("boom") + def test_footer_includes_for_more_log_info(self): + ctx_lines = [LogLine(1, LineKind.ERROR, "failed")] + out = format_error_context( + "log/p.log", ctx_lines, "ecc log synthesis --project myproj", color=False + ) + assert "For more log info:" in out + assert "ecc log synthesis --project myproj" in out - print("after stdout") - sys.stderr.write("after stderr\n") + def test_footer_includes_command_grep_field(self): + ctx_lines = [LogLine(1, LineKind.ERROR, "failed")] + log_cmd = "ecc log synthesis --project myproj --run-id abc123" + out = format_error_context("log/p.log", ctx_lines, log_cmd, color=False) + assert 'command="ecc log synthesis --project myproj --run-id abc123"' in out - captured = capfd.readouterr() - assert "after stdout" in captured.out - assert "after stderr" in captured.err - assert "after stdout" not in log_file.read_text() - assert "after stderr" not in log_file.read_text() - - -# -- run_flow_with_progress -- - - -def _make_ws(directory="/tmp", log_section_fn=None): - section_fn = log_section_fn or (lambda self, msg: None) - return type( - "WS", - (), - { - "home": type("Home", (), {"reset": lambda self: None})(), - "logger": type( - "L", - (), - { - "info": lambda *a, **k: None, - "log_section": section_fn, - "log_separator": lambda *a, **k: None, - }, - )(), - "flow": type("F", (), {"data": {"steps": []}, "path": ""})(), - "directory": directory, - }, - )() - - -def _make_step(name, tool, log_file=""): - log = LogPaths(file=Path(log_file)) if log_file else LogPaths() - return type("WSS", (), {"name": name, "tool": tool, "log": log})() - - -def _make_flow(ws, steps, run_step_fn, init_db_engine_fn=None, check_state_fn=None): - if init_db_engine_fn is None: - - def init_db_engine_fn(self): - return None - - if check_state_fn is None: - - def check_state_fn(self, name, tool, state): - return False - - return type( - "EF", - (), - { - "workspace": ws, - "workspace_steps": steps, - "init_db_engine": init_db_engine_fn, - "run_step": run_step_fn, - "check_state": check_state_fn, - }, - )() + def test_project_and_run_id_preserved_in_footer(self): + ctx_lines = [LogLine(1, LineKind.ERROR, "failed")] + log_cmd = "ecc log synthesis --project /path/to/proj --run-id run42" + out = format_error_context("log/p.log", ctx_lines, log_cmd, color=False) + assert "--project /path/to/proj" in out + assert "--run-id run42" in out + + def test_color_gating_no_ansi_when_disabled(self): + ctx_lines = [LogLine(10, LineKind.ERROR, "Error: bad")] + out = format_error_context("log/p.log", ctx_lines, "ecc log step", color=False) + assert "\x1b[" not in out + + def test_color_gating_ansi_when_enabled(self): + ctx_lines = [LogLine(10, LineKind.ERROR, "Error: bad")] + out = format_error_context("log/p.log", ctx_lines, "ecc log step", color=True) + assert "\x1b[" in out + + def test_line_number_padding_consistent(self): + ctx_lines = [ + LogLine(1, LineKind.PLAIN, "first"), + LogLine(10, LineKind.ERROR, "error"), + LogLine(100, LineKind.PLAIN, "hundred"), + ] + out = format_error_context("log/p.log", ctx_lines, "ecc log step", color=False) + lines = out.strip().split("\n") + context_lines = [ + line + for line in lines + if line.strip() + and not line.startswith("error:") + and not line.startswith("For") + and not line.startswith("command=") + ] + for line in context_lines: + assert line.startswith(" ") + + def test_empty_context(self): + out = format_error_context("log/p.log", [], "ecc log step", color=False) + assert "error: log/p.log" in out + assert "For more log info:" in out + + +# --------------------------------------------------------------------------- +# run_flow_with_progress (worker-driven) +# --------------------------------------------------------------------------- + + +def _write_flow_json(workspace_dir, steps): + home = Path(workspace_dir) / "home" + home.mkdir(parents=True, exist_ok=True) + (home / "flow.json").write_text(json.dumps({"steps": steps})) + + +def _step_record(name, tool, state, runtime="0:00:01"): + return {"name": name, "tool": tool, "state": state, "runtime": runtime} + + +def _write_archived_log(workspace_dir, step, tool, content): + log_path = Path(workspace_dir) / f"{step}_{tool}" / "log" / f"{step}.log" + log_path.parent.mkdir(parents=True, exist_ok=True) + log_path.write_text(content) + return log_path + + +def _make_operation(workspace_dir, events, result): + """Replay reader-callback events against a flow.json progression.""" + + def run_operation(on_output, on_step_event): + for event in events: + kind = event[0] + if kind == "begin": + on_step_event("begin", event[1], event[2]) + elif kind == "output": + on_output(event[1]) + elif kind == "states": + _write_flow_json(workspace_dir, event[1]) + return result + + return run_operation class TestRunFlowWithProgress: def test_success_summary_format(self, tmp_path): - flow = _make_flow( - _make_ws(str(tmp_path)), - [_make_step("Synthesis", "yosys", str(tmp_path / "synth.log"))], - lambda self, s: StateEnum.Success, + workspace = tmp_path / "workspace" + _write_flow_json(workspace, [_step_record("Synthesis", "yosys", "Unstart")]) + operation = _make_operation( + workspace, + [ + ("begin", "Synthesis", "yosys"), + ("states", [_step_record("Synthesis", "yosys", "Success")]), + ], + OperationResult(success=True, exit_code=0), ) buf = FakeTTYStderr(isatty_value=True) - result = run_flow_with_progress(flow, _make_ctx(), None, buf) - assert result is True + result = run_flow_with_progress(str(workspace), _make_ctx(), None, buf, operation) + + assert result.success is True output = "".join(buf.written) assert "✓ synthesis (yosys)" in output - assert "status=success" not in output - def test_stops_on_failure(self): - call_count = [0] + def test_result_passed_through(self, tmp_path): + workspace = tmp_path / "workspace" + _write_flow_json(workspace, []) + failure = OperationResult(success=False, error="worker exploded") + operation = _make_operation(workspace, [], failure) - def fake_run_step(self, s): - call_count[0] += 1 - if s.name == "Synthesis": - return StateEnum.Success - return StateEnum.Imcomplete + buf = FakeTTYStderr(isatty_value=True) + result = run_flow_with_progress(str(workspace), _make_ctx(), None, buf, operation) - flow = _make_flow( - _make_ws(), - [_make_step("Synthesis", "yosys"), _make_step("Floorplan", "ecc")], - fake_run_step, - ) + assert result is failure - buf = FakeTTYStderr(isatty_value=True) - result = run_flow_with_progress(flow, _make_ctx(), None, buf) - assert result is False - assert call_count[0] == 2 - - def test_summary_includes_inspect_detail_line(self): - flow = _make_flow( - _make_ws(), - [_make_step("Synthesis", "yosys", "/tmp/synth.log")], - lambda self, s: StateEnum.Success, + def test_step_headers_emitted(self, tmp_path): + workspace = tmp_path / "workspace" + _write_flow_json(workspace, []) + operation = _make_operation( + workspace, + [ + ("begin", "Synthesis", "yosys"), + ("begin", "Floorplan", "ecc"), + ], + OperationResult(success=True, exit_code=0), ) buf = FakeTTYStderr(isatty_value=True) - run_flow_with_progress(flow, _make_ctx(), "myproject", buf) + run_flow_with_progress(str(workspace), _make_ctx(), None, buf, operation) plain = _strip_ansi("".join(buf.written)) - assert " inspect: ecc log synthesis --project myproject\n" in plain - - def test_summary_includes_log_detail_line(self, tmp_path): - log_file = tmp_path / "synth.log" - log_file.write_text("content\n") + assert "> synthesis (yosys)\n" in plain + assert "> floorplan (ecc)\n" in plain - flow = _make_flow( - _make_ws(str(tmp_path)), - [_make_step("Synthesis", "yosys", str(log_file))], - lambda self, s: StateEnum.Success, - ) + def test_run_header_emitted(self, tmp_path): + workspace = tmp_path / "workspace" + _write_flow_json(workspace, []) + operation = _make_operation(workspace, [], OperationResult(success=True, exit_code=0)) buf = FakeTTYStderr(isatty_value=True) - run_flow_with_progress(flow, _make_ctx(), None, buf) + run_flow_with_progress(str(workspace), _make_ctx(), None, buf, operation) output = "".join(buf.written) - assert " log:" in output + assert "[run]" in output + assert "workspace=" in output def test_run_label_uses_ctx_run_id(self, tmp_path): - run_dir = tmp_path / "sweeps" / "s1" / "r4" - flow = _make_flow( - _make_ws(str(run_dir)), - [_make_step("Synthesis", "yosys", str(tmp_path / "synth.log"))], - lambda self, s: StateEnum.Success, - ) + workspace = tmp_path / "workspace" + _write_flow_json(workspace, []) + operation = _make_operation(workspace, [], OperationResult(success=True, exit_code=0)) buf = FakeTTYStderr(isatty_value=True) - result = run_flow_with_progress(flow, _make_ctx(run_id="sweeps/s1/r4"), None, buf) + run_flow_with_progress( + str(workspace), _make_ctx(run_id="sweeps/s1/r4"), None, buf, operation + ) + plain = _strip_ansi("".join(buf.written)) + assert f"[run] sweeps/s1/r4 workspace={workspace}\n" in plain + + def test_summary_includes_inspect_detail_line(self, tmp_path): + workspace = tmp_path / "workspace" + _write_flow_json(workspace, [_step_record("Synthesis", "yosys", "Unstart")]) + operation = _make_operation( + workspace, + [ + ("begin", "Synthesis", "yosys"), + ("states", [_step_record("Synthesis", "yosys", "Success")]), + ], + OperationResult(success=True, exit_code=0), + ) - assert result is True + buf = FakeTTYStderr(isatty_value=True) + run_flow_with_progress(str(workspace), _make_ctx(), "myproject", buf, operation) plain = _strip_ansi("".join(buf.written)) - assert f"[run] sweeps/s1/r4 workspace={run_dir}\n" in plain + assert " inspect: ecc log synthesis --project myproject\n" in plain - def test_inspect_detail_carries_run_id(self): - flow = _make_flow( - _make_ws(), - [_make_step("Synthesis", "yosys", "/tmp/synth.log")], - lambda self, s: StateEnum.Success, + def test_inspect_detail_carries_run_id(self, tmp_path): + workspace = tmp_path / "workspace" + _write_flow_json(workspace, [_step_record("Synthesis", "yosys", "Unstart")]) + operation = _make_operation( + workspace, + [("states", [_step_record("Synthesis", "yosys", "Success")])], + OperationResult(success=True, exit_code=0), ) buf = FakeTTYStderr(isatty_value=True) - run_flow_with_progress(flow, _make_ctx(run_id="exp1"), "myproject", buf) + run_flow_with_progress( + str(workspace), _make_ctx(run_id="exp1"), "myproject", buf, operation + ) plain = _strip_ansi("".join(buf.written)) assert " inspect: ecc log synthesis --project myproject --run-id exp1\n" in plain - def test_step_headers_emitted(self): - flow = _make_flow( - _make_ws(), + def test_summary_includes_log_detail_line(self, tmp_path): + workspace = tmp_path / "workspace" + _write_flow_json(workspace, [_step_record("Synthesis", "yosys", "Unstart")]) + operation = _make_operation( + workspace, [ - _make_step("Synthesis", "yosys"), - _make_step("Floorplan", "ecc"), + ("begin", "Synthesis", "yosys"), + ("states", [_step_record("Synthesis", "yosys", "Success")]), ], - lambda self, s: StateEnum.Success, + OperationResult(success=True, exit_code=0), ) buf = FakeTTYStderr(isatty_value=True) - result = run_flow_with_progress(flow, _make_ctx(), None, buf) - assert result is True + run_flow_with_progress(str(workspace), _make_ctx(), None, buf, operation) plain = _strip_ansi("".join(buf.written)) - assert "> synthesis (yosys)\n" in plain - assert "> floorplan (ecc)\n" in plain + assert " log: Synthesis_yosys/log/Synthesis.log" in plain - def test_run_header_emitted(self, tmp_path): - flow = _make_flow( - _make_ws(str(tmp_path)), - [_make_step("Synthesis", "yosys")], - lambda self, s: StateEnum.Success, + def test_block_separator_between_steps(self, tmp_path): + workspace = tmp_path / "workspace" + _write_flow_json(workspace, []) + operation = _make_operation( + workspace, + [ + ("begin", "Synthesis", "yosys"), + ("states", [_step_record("Synthesis", "yosys", "Success")]), + ("begin", "Floorplan", "ecc"), + ], + OperationResult(success=True, exit_code=0), ) buf = FakeTTYStderr(isatty_value=True) - run_flow_with_progress(flow, _make_ctx(), None, buf) + run_flow_with_progress(str(workspace), _make_ctx(), None, buf, operation) output = "".join(buf.written) - assert "[run]" in output - assert "workspace=" in output - - def test_block_separator_between_steps(self): - flow = _make_flow( - _make_ws(), + synth_summary = output.find("✓ synthesis") + fp_header = output.find("> floorplan") + assert synth_summary >= 0 + assert fp_header >= 0 + assert "\n\n" in output[synth_summary:fp_header] + + def test_previous_step_rendered_before_next_header(self, tmp_path): + workspace = tmp_path / "workspace" + _write_flow_json(workspace, []) + operation = _make_operation( + workspace, [ - _make_step("Synthesis", "yosys"), - _make_step("Floorplan", "ecc"), + ("begin", "Synthesis", "yosys"), + ("states", [_step_record("Synthesis", "yosys", "Success")]), + ("begin", "Floorplan", "ecc"), ], - lambda self, s: StateEnum.Success, + OperationResult(success=True, exit_code=0), ) buf = FakeTTYStderr(isatty_value=True) - result = run_flow_with_progress(flow, _make_ctx(), None, buf) - assert result is True + run_flow_with_progress(str(workspace), _make_ctx(), None, buf, operation) output = "".join(buf.written) synth_summary = output.find("✓ synthesis") fp_header = output.find("> floorplan") - between = output[synth_summary:fp_header] - assert "\n\n" in between - - def test_failure_summary_includes_status(self): - def fake_run_step(self, s): - if s.name == "Synthesis": - return StateEnum.Success - return StateEnum.Imcomplete - - flow = _make_flow( - _make_ws(), - [_make_step("Synthesis", "yosys"), _make_step("Floorplan", "ecc")], - fake_run_step, + assert synth_summary >= 0 + assert fp_header > synth_summary + + def test_failure_summary_includes_status(self, tmp_path): + workspace = tmp_path / "workspace" + _write_flow_json(workspace, []) + operation = _make_operation( + workspace, + [ + ("begin", "Synthesis", "yosys"), + ("states", [_step_record("Synthesis", "yosys", "Success")]), + ("begin", "Floorplan", "ecc"), + ("states", [_step_record("Floorplan", "ecc", "Imcomplete", "0:00:02")]), + ], + OperationResult(success=False, error="run flow failed"), ) buf = FakeTTYStderr(isatty_value=True) - result = run_flow_with_progress(flow, _make_ctx(), None, buf) - assert result is False + result = run_flow_with_progress(str(workspace), _make_ctx(), None, buf, operation) + assert result.success is False plain = _strip_ansi("".join(buf.written)) assert "✗ floorplan (ecc)" in plain - assert "incomplete" in plain + assert "imcomplete" in plain def test_transient_line_shows_log_content(self, tmp_path): - log_file = tmp_path / "synth.log" - - def fake_run_step(self, s): - log_file.write_text("Synthesizing module top\n") - time.sleep(1.0) - return StateEnum.Success - - flow = _make_flow( - _make_ws(str(tmp_path)), - [_make_step("Synthesis", "yosys", str(log_file))], - fake_run_step, + workspace = tmp_path / "workspace" + _write_flow_json(workspace, [_step_record("Synthesis", "yosys", "Unstart")]) + operation = _make_operation( + workspace, + [ + ("begin", "Synthesis", "yosys"), + ("output", b"Synthesizing module top\n"), + ("states", [_step_record("Synthesis", "yosys", "Success")]), + ], + OperationResult(success=True, exit_code=0), ) buf = FakeTTYStderr(isatty_value=True) - result = run_flow_with_progress(flow, _make_ctx(), None, buf) - assert result is True + run_flow_with_progress(str(workspace), _make_ctx(), None, buf, operation) plain = _strip_ansi("".join(buf.written)) assert "Synthesizing module top" in plain @@ -852,402 +744,135 @@ def fake_run_step(self, s): assert summary_pos >= 0 assert log_pos < summary_pos - def test_transient_shows_waiting_when_no_log(self): - def fake_run_step(self, s): - time.sleep(1.0) - return StateEnum.Success - - flow = _make_flow( - _make_ws(), - [_make_step("Synthesis", "yosys", "/tmp/nonexistent_synth.log")], - fake_run_step, + def test_output_burst_is_throttled(self, tmp_path): + workspace = tmp_path / "workspace" + _write_flow_json(workspace, [_step_record("Synthesis", "yosys", "Unstart")]) + operation = _make_operation( + workspace, + [ + ("begin", "Synthesis", "yosys"), + ("output", b"line one\n"), + ("output", b"line two\n"), + ], + OperationResult(success=True, exit_code=0), ) buf = FakeTTYStderr(isatty_value=True) - result = run_flow_with_progress(flow, _make_ctx(), None, buf) - assert result is True + run_flow_with_progress(str(workspace), _make_ctx(), None, buf, operation) plain = _strip_ansi("".join(buf.written)) - assert " log: running synthesis, waiting for step log" in plain - - def test_log_section_markers_emitted(self, tmp_path): - sections = [] - flow = _make_flow( - _make_ws(str(tmp_path), log_section_fn=lambda self, msg: sections.append(msg)), - [_make_step("Synthesis", "yosys")], - lambda self, s: StateEnum.Success, - ) - - buf = FakeTTYStderr(isatty_value=True) - run_flow_with_progress(flow, _make_ctx(), None, buf) - - assert "yosys - begin step - Synthesis" in sections - assert "yosys - end step - Synthesis" in sections - assert sections.index("yosys - begin step - Synthesis") < sections.index( - "yosys - end step - Synthesis" - ) - - def test_log_section_markers_around_run_step(self, tmp_path): - call_order = [] - - def fake_run_step(self, s): - call_order.append(("run_step", s.name)) - return StateEnum.Success - - flow = _make_flow( - _make_ws( - str(tmp_path), log_section_fn=lambda self, msg: call_order.append(("section", msg)) - ), - [_make_step("Floorplan", "ecc")], - fake_run_step, - ) - - buf = FakeTTYStderr(isatty_value=True) - run_flow_with_progress(flow, _make_ctx(), None, buf) - - begin_idx = call_order.index(("section", "ecc - begin step - Floorplan")) - run_idx = call_order.index(("run_step", "Floorplan")) - end_idx = call_order.index(("section", "ecc - end step - Floorplan")) - assert begin_idx < run_idx < end_idx - - def test_init_db_engine_called_before_run_step(self, tmp_path): - call_order = [] - - def fake_init_db_engine(self): - call_order.append(("init_db_engine",)) - - def fake_run_step(self, s): - call_order.append(("run_step", s.name)) - return StateEnum.Success - - flow = _make_flow( - _make_ws( - str(tmp_path), log_section_fn=lambda self, msg: call_order.append(("section", msg)) - ), - [_make_step("Synthesis", "yosys")], - fake_run_step, - init_db_engine_fn=fake_init_db_engine, - ) - - buf = FakeTTYStderr(isatty_value=True) - run_flow_with_progress(flow, _make_ctx(), None, buf) - - begin_idx = call_order.index(("section", "yosys - begin step - Synthesis")) - init_idx = call_order.index(("init_db_engine",)) - run_idx = call_order.index(("run_step", "Synthesis")) - end_idx = call_order.index(("section", "yosys - end step - Synthesis")) - assert begin_idx < init_idx < run_idx < end_idx - - def test_restores_progress_output_after_run_step_redirects_stdio(self, tmp_path, capfd): - log_file = tmp_path / "place.log" - call_order = [] - - def fake_init_db_engine(self): - call_order.append(("init_db_engine",)) - - def fake_run_step(self, s): - call_order.append(("run_step", s.name)) - redirected = redirect_stdio_to_file(str(log_file)) - print("raw tool stdout") - sys.stderr.write("Plotting array maps: 57%\n") - redirected.flush() - time.sleep(1.0) - return StateEnum.Success - - flow = _make_flow( - _make_ws( - str(tmp_path), log_section_fn=lambda self, msg: call_order.append(("section", msg)) - ), - [_make_step("placement", "dreamplace", str(log_file))], - fake_run_step, - init_db_engine_fn=fake_init_db_engine, - ) - - result = run_flow_with_progress(flow, _make_ctx(), "myproj", sys.stderr) - print("after progress stdout") - sys.stderr.write("after progress stderr\n") - - captured = capfd.readouterr() - terminal = _strip_ansi(captured.err) - step_log = log_file.read_text() - - assert result is True - assert "> placement (dreamplace)\n" in terminal - assert "Plotting array maps: 57%" in terminal - assert "✓ placement (dreamplace)" in terminal - assert "after progress stdout" in captured.out - assert "after progress stderr" in captured.err - - assert "raw tool stdout" in step_log - assert "Plotting array maps: 57%" in step_log - assert "> placement (dreamplace)" not in step_log - assert "log: waiting for log..." not in step_log - assert "✓ placement (dreamplace)" not in step_log - assert "after progress stdout" not in step_log - assert "after progress stderr" not in step_log - - begin_idx = call_order.index(("section", "dreamplace - begin step - placement")) - init_idx = call_order.index(("init_db_engine",)) - run_idx = call_order.index(("run_step", "placement")) - end_idx = call_order.index(("section", "dreamplace - end step - placement")) - assert begin_idx < init_idx < run_idx < end_idx - - def test_captures_init_db_engine_output_in_step_log(self, tmp_path, capfd): - log_file = tmp_path / "floorplan.log" - - def fake_init_db_engine(self): - print("raw init stdout") - sys.stderr.write("raw init stderr\n") - sys.stderr.flush() - - def fake_run_step(self, s): - time.sleep(1.0) - return StateEnum.Success - - flow = _make_flow( - _make_ws(str(tmp_path)), - [_make_step("Floorplan", "ecc", str(log_file))], - fake_run_step, - init_db_engine_fn=fake_init_db_engine, - ) - - result = run_flow_with_progress(flow, _make_ctx(), "myproj", sys.stderr) - - captured = capfd.readouterr() - terminal = _strip_ansi(captured.err) - step_log = log_file.read_text() - - assert result is True - assert "raw init stdout" in step_log - assert "raw init stderr" in step_log - assert "raw init stdout" not in captured.out - assert "log: raw init stderr" in terminal - assert "\nraw init stderr\n" not in terminal - - def test_does_not_initialize_db_for_skipped_progress_step(self, tmp_path): - synth_log = tmp_path / "synth.log" - floorplan_log = tmp_path / "floorplan.log" - init_calls = [] - - def fake_check_state(self, name, tool, state): - return name == "Synthesis" and state == StateEnum.Success - - def fake_init_db_engine(self): - init_calls.append("init_db_engine") - print("init for step") - - def fake_run_step(self, s): - return StateEnum.Success - - flow = _make_flow( - _make_ws(str(tmp_path)), + assert "line one" in plain + assert "line two" not in plain + + def test_marker_text_never_rendered(self, tmp_path): + workspace = tmp_path / "workspace" + _write_flow_json(workspace, [_step_record("Synthesis", "yosys", "Unstart")]) + operation = _make_operation( + workspace, [ - _make_step("Synthesis", "yosys", str(synth_log)), - _make_step("Floorplan", "ecc", str(floorplan_log)), + ("begin", "Synthesis", "yosys"), + ("states", [_step_record("Synthesis", "yosys", "Success")]), ], - fake_run_step, - init_db_engine_fn=fake_init_db_engine, - check_state_fn=fake_check_state, + OperationResult(success=True, exit_code=0), ) buf = FakeTTYStderr(isatty_value=True) - result = run_flow_with_progress(flow, _make_ctx(), "myproj", buf) - - assert result is True - assert init_calls == ["init_db_engine"] - assert not synth_log.exists() - assert "init for step" in floorplan_log.read_text() - - def test_monitor_cleanup_on_run_step_exception(self, tmp_path): - def raising_run_step(self, s): - raise RuntimeError("tool crashed") - - flow = _make_flow( - _make_ws(str(tmp_path)), - [_make_step("Synthesis", "yosys")], - raising_run_step, - ) - - buf = FakeTTYStderr(isatty_value=True) - with pytest.raises(RuntimeError, match="tool crashed"): - run_flow_with_progress(flow, _make_ctx(), None, buf) - - output = "".join(buf.written) - assert "\r\x1b[K" in output + run_flow_with_progress(str(workspace), _make_ctx(), None, buf, operation) + plain = _strip_ansi("".join(buf.written)) + assert "ECC-STEP" not in plain - def test_color_enabled_for_tty_text(self, monkeypatch): + def test_color_enabled_for_tty_text(self, tmp_path, monkeypatch): monkeypatch.delenv("NO_COLOR", raising=False) monkeypatch.setenv("TERM", "xterm-256color") - - flow = _make_flow( - _make_ws(), - [_make_step("Synthesis", "yosys")], - lambda self, s: StateEnum.Success, + workspace = tmp_path / "workspace" + _write_flow_json(workspace, []) + operation = _make_operation( + workspace, + [("begin", "Synthesis", "yosys")], + OperationResult(success=True, exit_code=0), ) buf = FakeTTYStderr(isatty_value=True) - run_flow_with_progress(flow, _make_ctx(), None, buf) + run_flow_with_progress(str(workspace), _make_ctx(), None, buf, operation) output = "".join(buf.written) assert "\x1b[36m" in output # cyan for step header - def test_color_disabled_for_non_tty(self): - flow = _make_flow( - _make_ws(), - [_make_step("Synthesis", "yosys")], - lambda self, s: StateEnum.Success, + def test_color_disabled_for_non_tty(self, tmp_path): + workspace = tmp_path / "workspace" + _write_flow_json(workspace, []) + operation = _make_operation( + workspace, + [("begin", "Synthesis", "yosys")], + OperationResult(success=True, exit_code=0), ) buf = FakeTTYStderr(isatty_value=False) - run_flow_with_progress(flow, _make_ctx(), None, buf) + run_flow_with_progress(str(workspace), _make_ctx(), None, buf, operation) output = "".join(buf.written) for code in (BOLD, CYAN, GREEN, RED, DIM): assert code not in output # --------------------------------------------------------------------------- -# Failure context block formatting (AC-5) -# --------------------------------------------------------------------------- - - -class TestFormatErrorContext: - def test_first_line_is_error_log_path(self): - ctx_lines = [LogLine(10, LineKind.ERROR, "Error: something")] - out = format_error_context("log/synthesis.log", ctx_lines, "ecc log synthesis", color=False) - assert out.startswith("error: log/synthesis.log") - - def test_includes_numbered_context_lines(self): - ctx_lines = [ - LogLine(8, LineKind.INFO, "INFO: before"), - LogLine(9, LineKind.WARNING, "Warning: careful"), - LogLine(10, LineKind.ERROR, "Error: failed"), - ] - out = format_error_context("log/synthesis.log", ctx_lines, "ecc log synthesis", color=False) - for ll in ctx_lines: - assert str(ll.line_no) in out - assert ll.text in out - - def test_compact_kind_labels(self): - ctx_lines = [ - LogLine(5, LineKind.ERROR, "bad"), - LogLine(6, LineKind.WARNING, "meh"), - LogLine(7, LineKind.TRACEBACK, " File ..."), - LogLine(8, LineKind.INFO, "ok"), - ] - out = format_error_context("log/p.log", ctx_lines, "ecc log step", color=False) - assert "ERROR" in out - assert "WARN" in out - assert "TRACE" in out - assert "INFO" in out - - def test_footer_includes_for_more_log_info(self): - ctx_lines = [LogLine(1, LineKind.ERROR, "failed")] - out = format_error_context( - "log/p.log", ctx_lines, "ecc log synthesis --project myproj", color=False - ) - assert "For more log info:" in out - assert "ecc log synthesis --project myproj" in out - - def test_footer_includes_command_grep_field(self): - ctx_lines = [LogLine(1, LineKind.ERROR, "failed")] - log_cmd = "ecc log synthesis --project myproj --run-id abc123" - out = format_error_context("log/p.log", ctx_lines, log_cmd, color=False) - assert 'command="ecc log synthesis --project myproj --run-id abc123"' in out - - def test_project_and_run_id_preserved_in_footer(self): - ctx_lines = [LogLine(1, LineKind.ERROR, "failed")] - log_cmd = "ecc log synthesis --project /path/to/proj --run-id run42" - out = format_error_context("log/p.log", ctx_lines, log_cmd, color=False) - assert "--project /path/to/proj" in out - assert "--run-id run42" in out - - def test_color_gating_no_ansi_when_disabled(self): - ctx_lines = [LogLine(10, LineKind.ERROR, "Error: bad")] - out = format_error_context("log/p.log", ctx_lines, "ecc log step", color=False) - assert "\x1b[" not in out - - def test_color_gating_ansi_when_enabled(self): - ctx_lines = [LogLine(10, LineKind.ERROR, "Error: bad")] - out = format_error_context("log/p.log", ctx_lines, "ecc log step", color=True) - assert "\x1b[" in out - - def test_line_number_padding_consistent(self): - ctx_lines = [ - LogLine(1, LineKind.PLAIN, "first"), - LogLine(10, LineKind.ERROR, "error"), - LogLine(100, LineKind.PLAIN, "hundred"), - ] - out = format_error_context("log/p.log", ctx_lines, "ecc log step", color=False) - lines = out.strip().split("\n") - context_lines = [ - line - for line in lines - if line.strip() - and not line.startswith("error:") - and not line.startswith("For") - and not line.startswith("command=") - ] - for line in context_lines: - assert line.startswith(" ") - - def test_empty_context(self): - out = format_error_context("log/p.log", [], "ecc log step", color=False) - assert "error: log/p.log" in out - assert "For more log info:" in out - - -# --------------------------------------------------------------------------- -# Failure context progress integration (AC-6) +# Failure context progress integration # --------------------------------------------------------------------------- class TestFailureContextIntegration: def test_failed_step_prints_context_block(self, tmp_path): - log_file = tmp_path / "synth.log" - log_file.write_text("line 1\nline 2\nError: something failed\nline 4\n") - - def fail_step(self, s): - return StateEnum.Imcomplete - - flow = _make_flow( - _make_ws(str(tmp_path)), - [_make_step("Synthesis", "yosys", str(log_file))], - fail_step, + workspace = tmp_path / "workspace" + _write_flow_json(workspace, [_step_record("Synthesis", "yosys", "Unstart")]) + _write_archived_log( + workspace, "Synthesis", "yosys", "line 1\nline 2\nError: something failed\nline 4\n" + ) + operation = _make_operation( + workspace, + [ + ("begin", "Synthesis", "yosys"), + ("states", [_step_record("Synthesis", "yosys", "Imcomplete")]), + ], + OperationResult(success=False, error="run flow failed"), ) buf = FakeTTYStderr(isatty_value=True) - result = run_flow_with_progress(flow, _make_ctx(), "myproj", buf) - assert result is False + result = run_flow_with_progress(str(workspace), _make_ctx(), "myproj", buf, operation) + assert result.success is False plain = _strip_ansi("".join(buf.written)) assert "error:" in plain assert "For more log info:" in plain assert 'command="' in plain def test_successful_step_no_context_block(self, tmp_path): - log_file = tmp_path / "synth.log" - log_file.write_text("line 1\nline 2\nall good\n") - - flow = _make_flow( - _make_ws(str(tmp_path)), - [_make_step("Synthesis", "yosys", str(log_file))], - lambda self, s: StateEnum.Success, + workspace = tmp_path / "workspace" + _write_flow_json(workspace, [_step_record("Synthesis", "yosys", "Unstart")]) + _write_archived_log(workspace, "Synthesis", "yosys", "line 1\nline 2\nall good\n") + operation = _make_operation( + workspace, + [ + ("begin", "Synthesis", "yosys"), + ("states", [_step_record("Synthesis", "yosys", "Success")]), + ], + OperationResult(success=True, exit_code=0), ) buf = FakeTTYStderr(isatty_value=True) - result = run_flow_with_progress(flow, _make_ctx(), "myproj", buf) - assert result is True + result = run_flow_with_progress(str(workspace), _make_ctx(), "myproj", buf, operation) + assert result.success is True plain = _strip_ansi("".join(buf.written)) assert "error:" not in plain assert "For more log info:" not in plain - def test_missing_log_no_context_block(self): - flow = _make_flow( - _make_ws(), - [_make_step("Synthesis", "yosys", "/nonexistent/synth.log")], - lambda self, s: StateEnum.Imcomplete, + def test_missing_log_no_context_block(self, tmp_path): + workspace = tmp_path / "workspace" + _write_flow_json(workspace, [_step_record("Synthesis", "yosys", "Unstart")]) + operation = _make_operation( + workspace, + [("states", [_step_record("Synthesis", "yosys", "Imcomplete")])], + OperationResult(success=False, error="run flow failed"), ) buf = FakeTTYStderr(isatty_value=True) - result = run_flow_with_progress(flow, _make_ctx(), "myproj", buf) - assert result is False + result = run_flow_with_progress(str(workspace), _make_ctx(), "myproj", buf, operation) + assert result.success is False plain = _strip_ansi("".join(buf.written)) assert "error:" not in plain assert "For more log info:" not in plain @@ -1255,51 +880,53 @@ def test_missing_log_no_context_block(self): assert "inspect:" in plain def test_empty_log_no_context_block(self, tmp_path): - log_file = tmp_path / "empty.log" - log_file.write_text("") - - flow = _make_flow( - _make_ws(str(tmp_path)), - [_make_step("Synthesis", "yosys", str(log_file))], - lambda self, s: StateEnum.Imcomplete, + workspace = tmp_path / "workspace" + _write_flow_json(workspace, [_step_record("Synthesis", "yosys", "Unstart")]) + _write_archived_log(workspace, "Synthesis", "yosys", "") + operation = _make_operation( + workspace, + [("states", [_step_record("Synthesis", "yosys", "Imcomplete")])], + OperationResult(success=False, error="run flow failed"), ) buf = FakeTTYStderr(isatty_value=True) - result = run_flow_with_progress(flow, _make_ctx(), "myproj", buf) - assert result is False + result = run_flow_with_progress(str(workspace), _make_ctx(), "myproj", buf, operation) + assert result.success is False plain = _strip_ansi("".join(buf.written)) assert "For more log info:" not in plain def test_existing_log_and_inspect_lines_remain(self, tmp_path): - log_file = tmp_path / "synth.log" - log_file.write_text("line 1\nError: fail\nline 3\n") - - flow = _make_flow( - _make_ws(str(tmp_path)), - [_make_step("Synthesis", "yosys", str(log_file))], - lambda self, s: StateEnum.Imcomplete, + workspace = tmp_path / "workspace" + _write_flow_json(workspace, [_step_record("Synthesis", "yosys", "Unstart")]) + _write_archived_log(workspace, "Synthesis", "yosys", "line 1\nError: fail\nline 3\n") + operation = _make_operation( + workspace, + [("states", [_step_record("Synthesis", "yosys", "Imcomplete")])], + OperationResult(success=False, error="run flow failed"), ) buf = FakeTTYStderr(isatty_value=True) - result = run_flow_with_progress(flow, _make_ctx(), "myproj", buf) - assert result is False + result = run_flow_with_progress(str(workspace), _make_ctx(), "myproj", buf, operation) + assert result.success is False plain = _strip_ansi("".join(buf.written)) assert "log:" in plain assert "inspect:" in plain def test_context_block_no_blank_lines_between_rows(self, tmp_path): - log_file = tmp_path / "synth.log" - log_file.write_text("line one\nline two\nError: boom\nline four\n") - - flow = _make_flow( - _make_ws(str(tmp_path)), - [_make_step("Synthesis", "yosys", str(log_file))], - lambda self, s: StateEnum.Imcomplete, + workspace = tmp_path / "workspace" + _write_flow_json(workspace, [_step_record("Synthesis", "yosys", "Unstart")]) + _write_archived_log( + workspace, "Synthesis", "yosys", "line one\nline two\nError: boom\nline four\n" + ) + operation = _make_operation( + workspace, + [("states", [_step_record("Synthesis", "yosys", "Imcomplete")])], + OperationResult(success=False, error="run flow failed"), ) buf = FakeTTYStderr(isatty_value=True) - result = run_flow_with_progress(flow, _make_ctx(), "myproj", buf) - assert result is False + result = run_flow_with_progress(str(workspace), _make_ctx(), "myproj", buf, operation) + assert result.success is False raw = "".join(buf.written) header_pos = raw.find("error:") diff --git a/test/integration/conftest.py b/test/integration/conftest.py index a5fda155..d98c07b9 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -1,13 +1,9 @@ -import os -import sys -from contextlib import suppress from pathlib import Path import pytest from chipcompiler.data import create_workspace, get_design_parameters, get_pdk from chipcompiler.engine import EngineDB, EngineFlow -from chipcompiler.utility.log import flush_cstdio REPO_ROOT = Path(__file__).resolve().parents[2] @@ -52,23 +48,10 @@ def run_workspace_flow( engine_flow.create_step_workspaces() - # EngineFlow.run_step dup2's fd 1/2 into each step's log file and never - # restores them. Save/restore around the flow so pytest's own reporting - # is not swallowed by the last step's log. - saved_fds = (os.dup(1), os.dup(2)) - saved_streams = (sys.stdout, sys.stderr) - try: - return engine_flow.run_steps() - finally: - with suppress(Exception): - sys.stdout.flush() - sys.stderr.flush() - flush_cstdio() - os.dup2(saved_fds[0], 1) - os.dup2(saved_fds[1], 2) - os.close(saved_fds[0]) - os.close(saved_fds[1]) - sys.stdout, sys.stderr = saved_streams + # run_steps self-archives: the engine emits step markers on fd 1/2, and + # the built-in client-side archiver writes per-step logs without leaking + # markers into pytest's own output. + return engine_flow.run_steps() @pytest.fixture diff --git a/test/runtime/test_events.py b/test/runtime/test_events.py deleted file mode 100644 index 971ee5f3..00000000 --- a/test/runtime/test_events.py +++ /dev/null @@ -1,35 +0,0 @@ -import os -import sys - -from chipcompiler.runtime.events import redirect_stdout_to_stderr - - -def test_redirect_stdout_to_stderr_restores_fd_1_and_fd_2(tmp_path): - stdout_target = tmp_path / "stdout.txt" - stderr_target = tmp_path / "stderr.txt" - redirected_target = tmp_path / "redirected.txt" - - saved_stdout_fd = os.dup(1) - saved_stderr_fd = os.dup(2) - with open(stdout_target, "wb") as stdout_file, open(stderr_target, "wb") as stderr_file: - os.dup2(stdout_file.fileno(), 1) - os.dup2(stderr_file.fileno(), 2) - try: - with redirect_stdout_to_stderr(), open(redirected_target, "wb") as redirected_file: - os.dup2(redirected_file.fileno(), 2) - os.write(1, b"captured stdout\n") - os.write(2, b"captured stderr\n") - - os.write(1, b"restored stdout\n") - os.write(2, b"restored stderr\n") - finally: - os.dup2(saved_stdout_fd, 1) - os.dup2(saved_stderr_fd, 2) - os.close(saved_stdout_fd) - os.close(saved_stderr_fd) - sys.stdout = sys.__stdout__ - sys.stderr = sys.__stderr__ - - assert stdout_target.read_text() == "restored stdout\n" - assert stderr_target.read_text() == "captured stdout\nrestored stderr\n" - assert redirected_target.read_text() == "captured stderr\n" diff --git a/test/runtime/test_log_stream.py b/test/runtime/test_log_stream.py new file mode 100644 index 00000000..4a89d660 --- /dev/null +++ b/test/runtime/test_log_stream.py @@ -0,0 +1,519 @@ +"""Tests for chipcompiler.runtime.log_stream — reader archiving and resilience.""" + +import io + +import pytest + +from chipcompiler.runtime.log_stream import LogStreamReader + + +class TestLogStreamReader: + def _make_stream(self, chunks: list[bytes]) -> io.BytesIO: + return io.BytesIO(b"".join(chunks)) + + def test_archives_to_step_file(self, tmp_path): + log_path = tmp_path / "synth.log" + + def resolver(step, tool): + return log_path + + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"Synthesis","tool":"yosys"}\n' + b"yosys output line 1\n" + b"yosys output line 2\n" + b'\x1eECC-STEP {"v":1,"event":"end","step":"Synthesis","tool":"yosys"}\n' + ) + stream = io.BytesIO(stream_data) + reader = LogStreamReader(stream, log_path_resolver=resolver) + reader.start() + reader.join(timeout=5) + + assert log_path.exists() + content = log_path.read_bytes() + assert b"yosys output line 1\n" in content + assert b"yosys output line 2\n" in content + assert b"ECC-STEP" not in content + + def test_markers_not_archived(self, tmp_path): + log_path = tmp_path / "step.log" + + def resolver(step, tool): + return log_path + + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"S","tool":"T"}\n' + b"data\n" + b'\x1eECC-STEP {"v":1,"event":"end","step":"S","tool":"T"}\n' + ) + reader = LogStreamReader(io.BytesIO(stream_data), log_path_resolver=resolver) + reader.start() + reader.join(timeout=5) + assert log_path.read_bytes() == b"data\n" + + def test_multiple_steps(self, tmp_path): + paths = {} + + def resolver(step, tool): + p = tmp_path / f"{step}.log" + paths[step] = p + return p + + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"A","tool":"t"}\n' + b"output A\n" + b'\x1eECC-STEP {"v":1,"event":"end","step":"A","tool":"t"}\n' + b'\x1eECC-STEP {"v":1,"event":"begin","step":"B","tool":"t"}\n' + b"output B\n" + b'\x1eECC-STEP {"v":1,"event":"end","step":"B","tool":"t"}\n' + ) + reader = LogStreamReader(io.BytesIO(stream_data), log_path_resolver=resolver) + reader.start() + reader.join(timeout=5) + assert paths["A"].read_bytes() == b"output A\n" + assert paths["B"].read_bytes() == b"output B\n" + assert reader.state.steps_seen == ["A", "B"] + + def test_non_utf8_bytes_preserved(self, tmp_path): + log_path = tmp_path / "step.log" + + def resolver(step, tool): + return log_path + + raw = b"\x80\x81\xff\xfe binary data\n" + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"S","tool":"T"}\n' + + raw + + b'\x1eECC-STEP {"v":1,"event":"end","step":"S","tool":"T"}\n' + ) + reader = LogStreamReader(io.BytesIO(stream_data), log_path_resolver=resolver) + reader.start() + reader.join(timeout=5) + assert log_path.read_bytes() == raw + + def test_malformed_marker_treated_as_data(self, tmp_path): + log_path = tmp_path / "step.log" + + def resolver(step, tool): + return log_path + + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"S","tool":"T"}\n' + b"\x1eECC-STEP {bad json}\n" + b'\x1eECC-STEP {"v":1,"event":"end","step":"S","tool":"T"}\n' + ) + reader = LogStreamReader(io.BytesIO(stream_data), log_path_resolver=resolver) + reader.start() + reader.join(timeout=5) + assert b"{bad json}" in log_path.read_bytes() + + def test_on_output_callback(self): + received = [] + stream_data = b"hello world\n" + reader = LogStreamReader(io.BytesIO(stream_data), on_output=received.append) + reader.start() + reader.join(timeout=5) + assert b"hello world\n" in b"".join(received) + + def test_tail_bytes_maintained(self): + data = b"x" * 8000 + b"\n" + reader = LogStreamReader(io.BytesIO(data), tail_size=100) + reader.start() + reader.join(timeout=5) + assert len(reader.state.tail_bytes) == 100 + + def test_unknown_marker_event_archived_as_data(self, tmp_path): + """A valid marker with an unrecognized event must be archived as raw data.""" + log_path = tmp_path / "step.log" + + def resolver(step, tool): + return log_path + + unknown_line = b'\x1eECC-STEP {"v":1,"event":"pause","step":"S","tool":"T"}\n' + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"S","tool":"T"}\n' + + unknown_line + + b'\x1eECC-STEP {"v":1,"event":"end","step":"S","tool":"T"}\n' + ) + reader = LogStreamReader(io.BytesIO(stream_data), log_path_resolver=resolver) + reader.start() + reader.join(timeout=5) + assert log_path.read_bytes() == unknown_line + + def test_unversioned_marker_archived_as_data(self, tmp_path): + """A marker frame without a supported version is archived as raw data.""" + log_path = tmp_path / "step.log" + + def resolver(step, tool): + return log_path + + unversioned_begin = b'\x1eECC-STEP {"event":"begin","step":"S","tool":"T"}\n' + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"S","tool":"T"}\n' + + unversioned_begin + + b'\x1eECC-STEP {"v":1,"event":"end","step":"S","tool":"T"}\n' + ) + reader = LogStreamReader(io.BytesIO(stream_data), log_path_resolver=resolver) + reader.start() + reader.join(timeout=5) + assert log_path.read_bytes() == unversioned_begin + assert reader.state.steps_seen == ["S"] + + def test_archive_write_error_surfaces_in_state(self, tmp_path): + """An OSError during archive write must be captured in state.error.""" + log_path = tmp_path / "step.log" + + def resolver(step, tool): + return log_path + + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"S","tool":"T"}\n' + b"some output\n" + b'\x1eECC-STEP {"v":1,"event":"end","step":"S","tool":"T"}\n' + ) + reader = LogStreamReader(io.BytesIO(stream_data), log_path_resolver=resolver) + reader.start() + reader.join(timeout=5) + + log_path.unlink() + log_path.mkdir() + + stream_data2 = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"S","tool":"T"}\n' + b"more output\n" + b'\x1eECC-STEP {"v":1,"event":"end","step":"S","tool":"T"}\n' + ) + reader2 = LogStreamReader(io.BytesIO(stream_data2), log_path_resolver=resolver) + reader2.start() + reader2.join(timeout=5) + assert reader2.state.error is not None + assert isinstance(reader2.state.error, OSError) + + def test_archive_open_error_surfaces_in_state(self, tmp_path): + """An OSError when opening an archive must be captured in state.error.""" + + def resolver(step, tool): + return tmp_path / "nonexistent_dir" / "sub" / "step.log" + + (tmp_path / "nonexistent_dir").write_text("not a directory") + + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"S","tool":"T"}\n' + b"output\n" + b'\x1eECC-STEP {"v":1,"event":"end","step":"S","tool":"T"}\n' + ) + reader = LogStreamReader(io.BytesIO(stream_data), log_path_resolver=resolver) + reader.start() + reader.join(timeout=5) + assert reader.state.error is not None + assert isinstance(reader.state.error, OSError) + + def test_mismatched_end_marker_does_not_close_archive(self, tmp_path): + """begin A -> end B -> data -> end A: data must be in A's archive.""" + log_path = tmp_path / "step.log" + + def resolver(step, tool): + return log_path + + mismatched_end = b'\x1eECC-STEP {"v":1,"event":"end","step":"B","tool":"T"}\n' + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"A","tool":"T"}\n' + b"before\n" + + mismatched_end + + b"after\n" + + b'\x1eECC-STEP {"v":1,"event":"end","step":"A","tool":"T"}\n' + ) + reader = LogStreamReader(io.BytesIO(stream_data), log_path_resolver=resolver) + reader.start() + reader.join(timeout=5) + content = log_path.read_bytes() + assert b"before\n" in content + assert mismatched_end in content + assert b"after\n" in content + + def test_active_step_tracked_in_state(self, tmp_path): + """State tracks the active step/tool during archiving.""" + log_path = tmp_path / "step.log" + + def resolver(step, tool): + return log_path + + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"Synthesis","tool":"yosys"}\ndata\n' + ) + reader = LogStreamReader(io.BytesIO(stream_data), log_path_resolver=resolver) + reader.start() + reader.join(timeout=5) + assert reader.state.active_step == "Synthesis" + assert reader.state.active_tool == "yosys" + + def test_duplicate_begin_does_not_switch_archive(self, tmp_path): + """begin A -> data -> begin B -> data -> end A: all data stays in A's archive.""" + log_path = tmp_path / "step.log" + + def resolver(step, tool): + return log_path + + begin_b = b'\x1eECC-STEP {"v":1,"event":"begin","step":"B","tool":"T"}\n' + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"A","tool":"T"}\n' + b"before\n" + + begin_b + + b"after\n" + + b'\x1eECC-STEP {"v":1,"event":"end","step":"A","tool":"T"}\n' + ) + reader = LogStreamReader(io.BytesIO(stream_data), log_path_resolver=resolver) + reader.start() + reader.join(timeout=5) + content = log_path.read_bytes() + assert b"before\n" in content + assert begin_b in content + assert b"after\n" in content + assert reader.state.active_step is None + assert reader.state.steps_seen == ["A"] + + +class TestLiveStreaming: + def test_short_output_flows_without_waiting_for_8kib(self): + """read1 delivers available bytes immediately; read(8192) would block + until the buffer fills or EOF and stall live progress.""" + import os + import time + + read_fd, write_fd = os.pipe() + received = [] + reader = LogStreamReader(os.fdopen(read_fd, "rb"), on_output=received.append) + reader.start() + try: + os.write(write_fd, b"short line\n") + deadline = time.monotonic() + 2.0 + while not received and time.monotonic() < deadline: + time.sleep(0.01) + assert received == [b"short line\n"] + finally: + os.close(write_fd) + reader.join(timeout=5) + reader.stop() + + +class TestArchiveOwnStepLogs: + """In-process executor runs self-archive through the fd-2 pipe.""" + + def test_archives_step_bytes_and_echoes_without_markers(self, tmp_path, capfd): + import json + import os + + from chipcompiler.runtime.log_stream import archive_own_step_logs, emit_step_marker + + workspace = tmp_path / "ws" + (workspace / "home").mkdir(parents=True) + (workspace / "home" / "flow.json").write_text( + json.dumps({"steps": [{"name": "S", "tool": "T", "state": "Ongoing"}]}) + ) + + with archive_own_step_logs(workspace) as reader: + emit_step_marker("begin", step="S", tool="T") + os.write(2, b"tool output\n") + os.write(1, b"stdout line\n") + emit_step_marker("end", step="S", tool="T") + os.write(2, b"unscoped tail\n") + + assert reader.state.error is None + # fd 1 and fd 2 bytes both land in the archive, markers never do. + assert (workspace / "S_T" / "log" / "S.log").read_bytes() == (b"tool output\nstdout line\n") + echoed = capfd.readouterr().err + assert "tool output" in echoed + assert "stdout line" in echoed + assert "unscoped tail" in echoed + assert "ECC-STEP" not in echoed + + +class TestArchiveOwnStepLogsPassthrough: + def test_external_client_or_nesting_passes_through(self, tmp_path, capfd): + """A worker/sidecar process (marked) or an already-active outer context + must not double-wrap: markers keep flowing to the outer client.""" + import os + + import chipcompiler.runtime.log_stream as log_stream_module + from chipcompiler.runtime.log_stream import archive_own_step_logs, emit_step_marker + + workspace = tmp_path / "ws" + (workspace / "home").mkdir(parents=True) + + # Nested: the outer context archives; the inner one passes through. + with ( + archive_own_step_logs(workspace), + archive_own_step_logs(workspace) as inner, + ): + assert inner is None + emit_step_marker("begin", step="S", tool="T") + os.write(2, b"bytes\n") + emit_step_marker("end", step="S", tool="T") + assert (workspace / "S_T" / "log" / "S.log").read_bytes() == b"bytes\n" + assert "ECC-STEP" not in capfd.readouterr().err + + # Marked external client: no self-archive at all, bytes pass through. + log_stream_module.mark_external_log_client() + try: + with archive_own_step_logs(workspace) as reader: + assert reader is None + os.write(2, b"raw\n") + finally: + log_stream_module._EXTERNAL_LOG_CLIENT = False + assert "raw" in capfd.readouterr().err + + +class TestArchiveTeardownFailure: + def test_teardown_flush_failure_still_releases_the_guard(self, tmp_path, monkeypatch, capfd): + """A teardown flush raising (closed stream) must not leave the guard + set or the fds attached to the pipe.""" + import os + + import chipcompiler.runtime.log_stream as log_stream_module + from chipcompiler.runtime.log_stream import archive_own_step_logs, emit_step_marker + + workspace = tmp_path / "ws" + (workspace / "home").mkdir(parents=True) + (workspace / "home" / "flow.json").write_text( + '{"steps": [{"name": "S", "tool": "T", "state": "Ongoing"}]}' + ) + + # Fail the C-stdio flush at teardown only: armed just before the + # context exits, so entry and marker flushes still succeed. + import chipcompiler.utility.log as utility_log + + armed = [] + + def flaky_c_flush(): + if armed: + raise OSError("broken stdio buffer") + + monkeypatch.setattr(utility_log, "flush_cstdio", flaky_c_flush) + with archive_own_step_logs(workspace): + emit_step_marker("begin", step="S", tool="T") + os.write(2, b"bytes\n") + emit_step_marker("end", step="S", tool="T") + armed.append(True) + + assert log_stream_module._SELF_ARCHIVE_ACTIVE is False + assert (workspace / "S_T" / "log" / "S.log").read_bytes() == b"bytes\n" + # fd 2 is restored to the terminal. + os.write(2, b"restored\n") + assert "restored" in capfd.readouterr().err + + +class TestArchiveSetupFailure: + def test_setup_failure_restores_fds_and_releases_the_guard(self, tmp_path, monkeypatch, capfd): + """A failure during setup (pipe/dup/reader start) must restore fd 1/2 + and release the guard so later runs still archive.""" + import os + + import chipcompiler.runtime.log_stream as log_stream_module + from chipcompiler.runtime.log_stream import archive_own_step_logs, emit_step_marker + + workspace = tmp_path / "ws" + (workspace / "home").mkdir(parents=True) + (workspace / "home" / "flow.json").write_text( + '{"steps": [{"name": "S", "tool": "T", "state": "Ongoing"}]}' + ) + + monkeypatch.setattr( + log_stream_module.os, "pipe", lambda: (_ for _ in ()).throw(OSError("no fds")) + ) + with ( + pytest.raises(OSError, match="no fds"), + archive_own_step_logs(workspace), + ): + pass + assert log_stream_module._SELF_ARCHIVE_ACTIVE is False + # fd 2 still reaches the terminal after the failed setup. + os.write(2, b"still alive\n") + assert "still alive" in capfd.readouterr().err + + monkeypatch.undo() + with archive_own_step_logs(workspace): + emit_step_marker("begin", step="S", tool="T") + os.write(2, b"bytes\n") + emit_step_marker("end", step="S", tool="T") + assert (workspace / "S_T" / "log" / "S.log").read_bytes() == b"bytes\n" + + +class TestLogStreamResilience: + def test_resolver_exception_disables_archive_continues_drain(self): + """A resolver that raises must not kill the drain thread.""" + received = [] + call_count = [0] + + def failing_resolver(step, tool): + call_count[0] += 1 + raise RuntimeError("resolver failed") + + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"S","tool":"T"}\n' + b"output after failed resolver\n" + b'\x1eECC-STEP {"v":1,"event":"end","step":"S","tool":"T"}\n' + b"trailing data\n" + ) + reader = LogStreamReader( + io.BytesIO(stream_data), + log_path_resolver=failing_resolver, + on_output=received.append, + ) + reader.start() + reader.join(timeout=5) + assert reader.completed + assert isinstance(reader.state.error, RuntimeError) + assert "resolver failed" in str(reader.state.error) + combined = b"".join(received) + assert b"output after failed resolver\n" in combined + assert b"trailing data\n" in combined + + def test_callback_exception_disables_callback_continues_drain(self, tmp_path): + """An on_output callback that raises must not kill archiving.""" + log_path = tmp_path / "step.log" + call_count = [0] + + def failing_callback(data): + call_count[0] += 1 + if call_count[0] == 1: + raise ValueError("callback exploded") + + def resolver(step, tool): + return log_path + + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"S","tool":"T"}\n' + b"line 1\n" + b"line 2\n" + b'\x1eECC-STEP {"v":1,"event":"end","step":"S","tool":"T"}\n' + ) + reader = LogStreamReader( + io.BytesIO(stream_data), + log_path_resolver=resolver, + on_output=failing_callback, + ) + reader.start() + reader.join(timeout=5) + assert reader.completed + assert isinstance(reader.state.display_error, ValueError) + content = log_path.read_bytes() + assert b"line 1\n" in content + assert b"line 2\n" in content + + def test_drain_completes_after_callback_disabled(self): + """After callback is disabled, remaining data is still drained.""" + call_count = [0] + + def failing_callback(data): + call_count[0] += 1 + raise RuntimeError("always fails") + + stream_data = b"line 1\nline 2\nline 3\n" + reader = LogStreamReader( + io.BytesIO(stream_data), + on_output=failing_callback, + ) + reader.start() + reader.join(timeout=5) + assert reader.completed + assert call_count[0] == 1 + assert b"line 3\n" in reader.state.tail_bytes diff --git a/test/runtime/test_log_stream_markers.py b/test/runtime/test_log_stream_markers.py new file mode 100644 index 00000000..bca98bf3 --- /dev/null +++ b/test/runtime/test_log_stream_markers.py @@ -0,0 +1,242 @@ +"""Tests for chipcompiler.runtime.log_stream — marker protocol parsing and emission.""" + +import io +import os + +import pytest + +from chipcompiler.runtime.log_stream import ( + MARKER_PREFIX, + LogStreamReader, + StepMarker, + emit_step_marker, + parse_marker, +) + + +class _ChunkedStream: + """A binary stream that returns fixed-size chunks regardless of read size.""" + + def __init__(self, data: bytes, chunk_size: int): + self._data = data + self._chunk_size = chunk_size + self._pos = 0 + + def read(self, n: int = -1) -> bytes: + if self._pos >= len(self._data): + return b"" + size = self._chunk_size if n < 0 else min(n, self._chunk_size) + part = self._data[self._pos : self._pos + size] + self._pos += size + return part + + +class TestParseMarker: + def test_valid_begin(self): + line = b'\x1eECC-STEP {"v":1,"event":"begin","step":"Synthesis","tool":"yosys"}\n' + m = parse_marker(line) + assert m == StepMarker(event="begin", step="Synthesis", tool="yosys") + + def test_valid_end(self): + line = b'\x1eECC-STEP {"v":1,"event":"end","step":"Placement","tool":"ecc"}\n' + m = parse_marker(line) + assert m == StepMarker(event="end", step="Placement", tool="ecc") + + def test_no_prefix(self): + assert parse_marker(b"normal log line\n") is None + + def test_malformed_json(self): + assert parse_marker(b"\x1eECC-STEP {bad json}\n") is None + + def test_missing_fields(self): + line = b'\x1eECC-STEP {"v":1,"event":"begin"}\n' + assert parse_marker(line) is None + + def test_wrong_field_types(self): + line = b'\x1eECC-STEP {"v":1,"event":1,"step":"A","tool":"B"}\n' + assert parse_marker(line) is None + + def test_no_trailing_newline(self): + line = b'\x1eECC-STEP {"v":1,"event":"begin","step":"S","tool":"T"}' + m = parse_marker(line) + assert m is not None + assert m.event == "begin" + + def test_missing_version_rejected(self): + line = b'\x1eECC-STEP {"event":"begin","step":"S","tool":"T"}\n' + assert parse_marker(line) is None + + def test_unsupported_version_rejected(self): + line = b'\x1eECC-STEP {"v":2,"event":"begin","step":"S","tool":"T"}\n' + assert parse_marker(line) is None + + def test_string_version_rejected(self): + line = b'\x1eECC-STEP {"v":"1","event":"begin","step":"S","tool":"T"}\n' + assert parse_marker(line) is None + + def test_boolean_version_rejected(self): + line = b'\x1eECC-STEP {"v":true,"event":"begin","step":"S","tool":"T"}\n' + assert parse_marker(line) is None + + def test_non_utf8_payload_rejected(self): + line = b'\x1eECC-STEP {"v":1,"event":"begin","step":"S\xff","tool":"T"}\n' + assert parse_marker(line) is None + + @pytest.mark.parametrize("payload", ["[]", "null", "42", '"hello"', "true"]) + def test_non_object_payload_rejected(self, payload): + line = f"\x1eECC-STEP {payload}\n".encode() + assert parse_marker(line) is None + + +class TestEmitStepMarker: + def test_payload_carries_version_and_round_trips(self, monkeypatch): + written = [] + real_write = os.write + + def fake_write(fd, data): + if fd == 2: + written.append(data) + return len(data) + return real_write(fd, data) + + monkeypatch.setattr(os, "write", fake_write) + emit_step_marker("begin", step="Synthesis", tool="yosys") + + assert written == [ + MARKER_PREFIX + b'{"v":1,"event":"begin","step":"Synthesis","tool":"yosys"}\n' + ] + assert parse_marker(written[0]) == StepMarker(event="begin", step="Synthesis", tool="yosys") + + def test_c_stdio_buffer_drains_before_marker(self, tmp_path): + """Native buffered output must reach fd 2 ahead of the marker bytes.""" + import ctypes + + libc = ctypes.CDLL(None) + libc.fdopen.restype = ctypes.c_void_p + libc.fdopen.argtypes = [ctypes.c_int, ctypes.c_char_p] + libc.setvbuf.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int, ctypes.c_size_t] + libc.fputs.argtypes = [ctypes.c_char_p, ctypes.c_void_p] + libc.fclose.argtypes = [ctypes.c_void_p] + _IOFBF = 0 + + sink = tmp_path / "fd2.bin" + saved_fd = os.dup(2) + try: + with sink.open("wb") as handle: + os.dup2(handle.fileno(), 2) + # A fully buffered FILE* targeting fd 2: fputs bytes stay in the C + # buffer until something flushes the process streams. + stream = libc.fdopen(os.dup(2), b"w") + assert stream + assert libc.setvbuf(stream, None, _IOFBF, 4096) == 0 + libc.fputs(b"native-before-end\n", stream) + emit_step_marker("end", step="S", tool="T") + libc.fclose(stream) + finally: + os.dup2(saved_fd, 2) + os.close(saved_fd) + + content = sink.read_bytes() + assert content == ( + b"native-before-end\n" + + MARKER_PREFIX + + b'{"v":1,"event":"end","step":"S","tool":"T"}\n' + ) + + +class TestMarkerBoundaryScanning: + """Markers are recognized wherever the reserved prefix appears.""" + + def test_end_marker_after_unterminated_output(self, tmp_path): + log_path = tmp_path / "step.log" + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"S","tool":"T"}\n' + b"last line without newline" + b'\x1eECC-STEP {"v":1,"event":"end","step":"S","tool":"T"}\n' + ) + reader = LogStreamReader( + io.BytesIO(stream_data), log_path_resolver=lambda step, tool: log_path + ) + reader.start() + reader.join(timeout=5) + assert log_path.read_bytes() == b"last line without newline" + assert reader.state.active_step is None + assert reader.state.steps_seen == ["S"] + + def test_end_marker_split_after_unterminated_output(self, tmp_path): + log_path = tmp_path / "step.log" + end_frame = b'\x1eECC-STEP {"v":1,"event":"end","step":"S","tool":"T"}\n' + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"S","tool":"T"}\nunterminated' + end_frame + ) + reader = LogStreamReader( + _ChunkedStream(stream_data, 7), log_path_resolver=lambda step, tool: log_path + ) + reader.start() + reader.join(timeout=5) + assert log_path.read_bytes() == b"unterminated" + assert reader.state.active_step is None + + def test_invalid_marker_mid_line_is_data(self, tmp_path): + log_path = tmp_path / "step.log" + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"S","tool":"T"}\n' + b"glued text \x1eECC-STEP {bad json}\n" + b'\x1eECC-STEP {"v":1,"event":"end","step":"S","tool":"T"}\n' + ) + reader = LogStreamReader( + io.BytesIO(stream_data), log_path_resolver=lambda step, tool: log_path + ) + reader.start() + reader.join(timeout=5) + content = log_path.read_bytes() + assert b"glued text " in content + assert b"{bad json}" in content + assert reader.state.active_step is None + + def test_overlong_candidate_recovers_following_marker(self, tmp_path): + log_path = tmp_path / "step.log" + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"S","tool":"T"}\n' + b"\x1eECC-STEP " + + b"a" * 600 + + b'\x1eECC-STEP {"v":1,"event":"end","step":"S","tool":"T"}\n' + ) + reader = LogStreamReader( + _ChunkedStream(stream_data, 100), log_path_resolver=lambda step, tool: log_path + ) + reader.start() + reader.join(timeout=5) + content = log_path.read_bytes() + assert b"a" * 600 in content + assert b'"event":"end"' not in content + assert reader.state.active_step is None + + def test_candidate_at_exactly_512_bytes_is_held(self, tmp_path): + """A 512-byte candidate without its newline is held, then consumed.""" + log_path = tmp_path / "step.log" + wrapper = b'{"v":1,"event":"begin","step":"%s","tool":"T"}' + pad = 512 - len(b"\x1eECC-STEP ") - (len(wrapper) - 2) + payload = wrapper % (b"S" * pad) + frame_head = b"\x1eECC-STEP " + payload + assert len(frame_head) == 512 + reader = LogStreamReader( + _ChunkedStream( + frame_head + b"\nbody\n" + frame_head.replace(b"begin", b"end") + b"\n", 512 + ), + log_path_resolver=lambda step, tool: log_path, + ) + reader.start() + reader.join(timeout=5) + assert log_path.read_bytes() == b"body\n" + assert reader.state.active_step is None + + def test_candidate_beyond_512_bytes_degrades(self, tmp_path): + received = [] + overlong = b"\x1eECC-STEP " + b"a" * 503 + assert len(overlong) > 512 + reader = LogStreamReader(_ChunkedStream(overlong, 64), on_output=received.append) + reader.start() + reader.join(timeout=5) + assert b"".join(received) == overlong + assert reader.state.active_step is None diff --git a/test/runtime/test_log_stream_targets.py b/test/runtime/test_log_stream_targets.py new file mode 100644 index 00000000..50fa2741 --- /dev/null +++ b/test/runtime/test_log_stream_targets.py @@ -0,0 +1,284 @@ +"""Tests for chipcompiler.runtime.log_stream — archive targets and step events.""" + +import io + +from chipcompiler.runtime.log_stream import LogStreamReader + + +class TestLogStreamAllowlist: + def test_unknown_step_marker_treated_as_data(self, tmp_path): + """A begin marker for a pair not in valid_steps is archived as data.""" + log_path = tmp_path / "step.log" + + def resolver(step, tool): + return log_path + + valid = {("Synthesis", "yosys")} + unknown_begin = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"../../escape","tool":"evil"}\n' + ) + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"Synthesis","tool":"yosys"}\n' + + unknown_begin + + b"normal data\n" + + b'\x1eECC-STEP {"v":1,"event":"end","step":"Synthesis","tool":"yosys"}\n' + ) + reader = LogStreamReader( + io.BytesIO(stream_data), log_path_resolver=resolver, valid_steps=valid + ) + reader.start() + reader.join(timeout=5) + content = log_path.read_bytes() + assert unknown_begin in content + assert b"normal data\n" in content + assert reader.state.steps_seen == ["Synthesis"] + + def test_unknown_step_before_any_active_is_data(self): + """An unknown begin marker with no active step is sent to callback as data.""" + received = [] + valid = {("Place", "ecc")} + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"Bogus","tool":"fake"}\ntrailing\n' + ) + reader = LogStreamReader( + io.BytesIO(stream_data), on_output=received.append, valid_steps=valid + ) + reader.start() + reader.join(timeout=5) + combined = b"".join(received) + assert b"Bogus" in combined + assert b"trailing\n" in combined + assert reader.state.active_step is None + + def test_path_escape_does_not_open_archive(self, tmp_path): + """A resolved path outside workspace_dir must not open an archive file.""" + escape_target = tmp_path / "outside.log" + + def resolver(step, tool): + return escape_target + + workspace = tmp_path / "workspace" + workspace.mkdir() + valid = {("Escape", "evil")} + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"Escape","tool":"evil"}\n' + b"should not be written\n" + b'\x1eECC-STEP {"v":1,"event":"end","step":"Escape","tool":"evil"}\n' + ) + reader = LogStreamReader( + io.BytesIO(stream_data), + log_path_resolver=resolver, + valid_steps=valid, + workspace_dir=workspace, + ) + reader.start() + reader.join(timeout=5) + assert not escape_target.exists() + assert reader.state.error is not None + assert "escapes workspace" in str(reader.state.error) + + def test_contained_path_opens_normally(self, tmp_path): + """A path that resolves inside workspace_dir opens and archives.""" + workspace = tmp_path / "workspace" + workspace.mkdir() + log_path = workspace / "Synthesis_yosys" / "log" / "Synthesis.log" + + def resolver(step, tool): + return log_path + + valid = {("Synthesis", "yosys")} + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"Synthesis","tool":"yosys"}\n' + b"tool output\n" + b'\x1eECC-STEP {"v":1,"event":"end","step":"Synthesis","tool":"yosys"}\n' + ) + reader = LogStreamReader( + io.BytesIO(stream_data), + log_path_resolver=resolver, + valid_steps=valid, + workspace_dir=workspace, + ) + reader.start() + reader.join(timeout=5) + assert log_path.read_bytes() == b"tool output\n" + assert reader.state.error is None + + +class TestArchiveTargetSanitization: + def test_separator_in_name_degrades_to_data(self, tmp_path): + """An allowlisted begin with a path separator is ordinary bytes, not a marker.""" + workspace = tmp_path / "workspace" + workspace.mkdir() + received = [] + + def resolver(step, tool): + base = workspace + return base / f"{step}_{tool}" / "log" / f"{step}.log" + + unsafe_begin = b'\x1eECC-STEP {"v":1,"event":"begin","step":"foo/bar","tool":"ecc"}\n' + stream_data = unsafe_begin + b"body bytes\n" + reader = LogStreamReader( + io.BytesIO(stream_data), + log_path_resolver=resolver, + valid_steps={("foo/bar", "ecc")}, + workspace_dir=workspace, + on_output=received.append, + ) + reader.start() + reader.join(timeout=5) + combined = b"".join(received) + assert unsafe_begin in combined + assert b"body bytes\n" in combined + assert reader.state.active_step is None + assert reader.state.steps_seen == [] + assert isinstance(reader.state.error, ValueError) + assert not (workspace / "foo").exists() + + def test_dotdot_in_name_degrades_to_data(self, tmp_path): + workspace = tmp_path / "workspace" + workspace.mkdir() + received = [] + unsafe_begin = b'\x1eECC-STEP {"v":1,"event":"begin","step":"..","tool":".."}\n' + reader = LogStreamReader( + io.BytesIO(unsafe_begin), + log_path_resolver=lambda step, tool: workspace / "x" / "x.log", + valid_steps={("..", "..")}, + workspace_dir=workspace, + on_output=received.append, + ) + reader.start() + reader.join(timeout=5) + assert b"".join(received) == unsafe_begin + assert reader.state.active_step is None + assert isinstance(reader.state.error, ValueError) + + def test_containment_violation_degrades_begin_to_data(self, tmp_path): + """A begin whose archive escapes the workspace is forwarded as data.""" + workspace = tmp_path / "workspace" + workspace.mkdir() + escape_target = tmp_path / "outside.log" + received = [] + unsafe_begin = b'\x1eECC-STEP {"v":1,"event":"begin","step":"Escape","tool":"evil"}\n' + stream_data = unsafe_begin + b"not archived\n" + reader = LogStreamReader( + io.BytesIO(stream_data), + log_path_resolver=lambda step, tool: escape_target, + valid_steps={("Escape", "evil")}, + workspace_dir=workspace, + on_output=received.append, + ) + reader.start() + reader.join(timeout=5) + combined = b"".join(received) + assert unsafe_begin in combined + assert b"not archived\n" in combined + assert reader.state.active_step is None + assert not escape_target.exists() + assert "escapes workspace" in str(reader.state.error) + # The marker's identity survives on the error so failure-path + # reconciliation can find and downgrade this step. + assert reader.state.error_step == "Escape" + assert reader.state.error_tool == "evil" + + +class TestStepLogArchiveResolver: + def test_resolver_produces_canonical_step_log_path(self, tmp_path): + from chipcompiler.runtime.log_stream import step_log_archive_resolver + + resolver = step_log_archive_resolver(tmp_path) + assert resolver("Synthesis", "yosys") == ( + tmp_path / "Synthesis_yosys" / "log" / "Synthesis.log" + ) + assert resolver("Floorplan", "ecc") == ( + tmp_path / "Floorplan_ecc" / "log" / "Floorplan.log" + ) + + def test_resolver_mirrors_the_sizer_builder_layout(self, tmp_path): + """The sizer builder sanitizes its step directory; the archive must + land in the same directory the built step owns.""" + from chipcompiler.runtime.log_stream import step_log_archive_resolver + + resolver = step_log_archive_resolver(tmp_path) + assert resolver("Timing optimization", "sizer") == ( + tmp_path / "timing_optimization_sizer" / "log" / "Timing optimization.log" + ) + + +class TestOnStepEvent: + def test_fires_on_matched_begin_and_end(self, tmp_path): + events = [] + log_path = tmp_path / "step.log" + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"S","tool":"T"}\n' + b"data\n" + b'\x1eECC-STEP {"v":1,"event":"end","step":"S","tool":"T"}\n' + ) + reader = LogStreamReader( + io.BytesIO(stream_data), + log_path_resolver=lambda step, tool: log_path, + on_step_event=lambda event, step, tool: events.append((event, step, tool)), + ) + reader.start() + reader.join(timeout=5) + assert events == [("begin", "S", "T"), ("end", "S", "T")] + + def test_does_not_fire_on_unmatched_markers(self, tmp_path): + events = [] + log_path = tmp_path / "step.log" + nested_begin = b'\x1eECC-STEP {"v":1,"event":"begin","step":"B","tool":"T"}\n' + mismatched_end = b'\x1eECC-STEP {"v":1,"event":"end","step":"X","tool":"T"}\n' + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"A","tool":"T"}\n' + + nested_begin + + mismatched_end + + b'\x1eECC-STEP {"v":1,"event":"end","step":"A","tool":"T"}\n' + ) + reader = LogStreamReader( + io.BytesIO(stream_data), + log_path_resolver=lambda step, tool: log_path, + on_step_event=lambda event, step, tool: events.append((event, step, tool)), + ) + reader.start() + reader.join(timeout=5) + assert events == [("begin", "A", "T"), ("end", "A", "T")] + + def test_does_not_fire_on_disallowed_marker(self, tmp_path): + events = [] + stream_data = b'\x1eECC-STEP {"v":1,"event":"begin","step":"Bogus","tool":"fake"}\n' + reader = LogStreamReader( + io.BytesIO(stream_data), + on_step_event=lambda event, step, tool: events.append((event, step, tool)), + valid_steps={("Real", "tool")}, + ) + reader.start() + reader.join(timeout=5) + assert events == [] + + def test_callback_exception_disables_callback_continues_drain(self, tmp_path): + log_path = tmp_path / "step.log" + calls = [0] + + def failing_callback(event, step, tool): + calls[0] += 1 + raise RuntimeError("step event exploded") + + stream_data = ( + b'\x1eECC-STEP {"v":1,"event":"begin","step":"S","tool":"T"}\n' + b"line 1\n" + b'\x1eECC-STEP {"v":1,"event":"end","step":"S","tool":"T"}\n' + b'\x1eECC-STEP {"v":1,"event":"begin","step":"S2","tool":"T"}\n' + b"line 2\n" + ) + reader = LogStreamReader( + io.BytesIO(stream_data), + log_path_resolver=lambda step, tool: log_path, + on_step_event=failing_callback, + ) + reader.start() + reader.join(timeout=5) + assert reader.completed + assert calls[0] == 1 + assert isinstance(reader.state.display_error, RuntimeError) + # The second begin re-opened (truncated) the shared log path, so its + # content proves archiving continued after the callback was disabled. + assert log_path.read_bytes() == b"line 2\n" diff --git a/test/runtime/test_operations.py b/test/runtime/test_operations.py index 02e28ace..0935608b 100644 --- a/test/runtime/test_operations.py +++ b/test/runtime/test_operations.py @@ -364,10 +364,8 @@ def runner(observer): assert _wait_for_terminal(manager, started["operationId"])["state"] == "cancelled" -def test_step_log_events_stream_only_new_log_bytes_and_keep_final_tail(tmp_path): +def test_server_never_emits_step_log_and_step_completed_has_no_final_log(tmp_path): events = [] - step_started = threading.Event() - complete_step = threading.Event() manager = RuntimeOperationManager(events.append) log_file = tmp_path / "Synthesis.log" log_file.write_text("previous run\n", encoding="utf-8") @@ -379,8 +377,8 @@ def test_step_log_events_stream_only_new_log_bytes_and_keep_final_tail(tmp_path) def runner(observer): observer.on_step_started(step) - step_started.set() - assert complete_step.wait(timeout=2) + with log_file.open("a", encoding="utf-8") as handle: + handle.write("live line one\nlive line two\n") observer.on_step_completed(step, StateEnum.Success) assert observer.wait_for_step_rendered(step, StateEnum.Success) return {"rerun": False} @@ -394,21 +392,23 @@ def runner(observer): idempotency_key="request-log-stream", runner=runner, ) - assert step_started.wait(timeout=1) - with log_file.open("a", encoding="utf-8") as handle: - handle.write("live line one\nlive line two\n") - - step_log = _wait_for_event(events, "step.log") - assert step_log["payload"]["chunk"] == "live line one\nlive line two\n" - assert step_log["payload"]["cursor"] == log_file.stat().st_size - - complete_step.set() - step_complete = _wait_for_event(events, "step.completed") - assert step_complete["payload"]["finalLog"] == ("previous run\nlive line one\nlive line two\n") - assert manager.acknowledge_step_rendered(started["operationId"], step_complete["eventId"])[ + + step_completed = _wait_for_event(events, "step.completed") + # Outlast the historical tail poll interval so a stray publisher would fire. + threading.Event().wait(0.3) + assert not any(event["type"] == "step.log" for event in events) + assert "finalLog" not in step_completed["payload"] + assert step_completed["payload"]["step"] == "Synthesis" + assert step_completed["payload"]["tool"] == "yosys" + assert step_completed["payload"]["state"] == "Success" + assert step_completed["payload"]["stepCommitId"] + assert step_completed["payload"]["workspaceRevision"] == 1 + + assert manager.acknowledge_step_rendered(started["operationId"], step_completed["eventId"])[ "accepted" ] assert _wait_for_terminal(manager, started["operationId"])["state"] == "succeeded" + assert not any(event["type"] == "step.log" for event in events) def _wait_for_event(events: list[dict], event_type: str) -> dict: diff --git a/test/runtime/test_requests.py b/test/runtime/test_requests.py index 16d237dd..2a56057a 100644 --- a/test/runtime/test_requests.py +++ b/test/runtime/test_requests.py @@ -307,23 +307,56 @@ def test_rerun_must_be_boolean(method, params): "operation.start_step", {"workspaceId": "ws-1", "step": "Synthesis", "resetDependents": 1}, ), + ( + "flow.run_step", + {"workspaceId": "ws-1", "step": "Synthesis", "resetDependents": 1}, + ), + ( + "flow.run_step", + {"workspaceId": "ws-1", "step": "Synthesis", "invalidateDependents": 1}, + ), ], ) def test_reset_dependents_must_be_boolean(method, params): with pytest.raises(RequestValidationError) as exc_info: _parse_runtime_request(method, params) - assert exc_info.value.reason == "reset_dependents must be a boolean" + assert "must be a boolean" in exc_info.value.reason -def test_direct_flow_run_step_rejects_gui_only_reset_dependents_field(): - with pytest.raises(RequestValidationError) as exc_info: - _parse_runtime_request( - "flow.run_step", - {"workspaceId": "ws-1", "step": "Synthesis", "resetDependents": True}, - ) +def test_flow_run_step_parses_reset_dependents(): + request = _parse_runtime_request( + "flow.run_step", + {"workspaceId": "ws-1", "step": "Synthesis", "resetDependents": True}, + ) + + assert request == FlowRunStepRequest( + workspace_id="ws-1", + step="Synthesis", + reset_dependents=True, + ) + + +def test_flow_run_step_parses_invalidate_dependents(): + request = _parse_runtime_request( + "flow.run_step", + {"workspaceId": "ws-1", "step": "Synthesis", "invalidateDependents": True}, + ) + + assert request == FlowRunStepRequest( + workspace_id="ws-1", + step="Synthesis", + invalidate_dependents=True, + ) + + +def test_flow_run_step_reset_dependents_defaults_to_false(): + request = _parse_runtime_request( + "flow.run_step", + {"workspaceId": "ws-1", "step": "Synthesis"}, + ) - assert exc_info.value.reason == "unknown field: reset_dependents" + assert request == FlowRunStepRequest(workspace_id="ws-1", step="Synthesis") def test_unknown_runtime_method_has_no_request_model(): diff --git a/test/runtime/test_stdio_isolation.py b/test/runtime/test_stdio_isolation.py new file mode 100644 index 00000000..f5bb8ac4 --- /dev/null +++ b/test/runtime/test_stdio_isolation.py @@ -0,0 +1,84 @@ +"""Tests for chipcompiler.runtime.stdio_isolation.""" + +import os +import sys + +import pytest + +from chipcompiler.runtime.stdio_isolation import StdioIsolation + + +class TestStdioIsolation: + def test_not_installed_raises(self): + iso = StdioIsolation() + with pytest.raises(RuntimeError, match="not installed"): + _ = iso.protocol_stream + + def test_install_returns_writable_stream(self): + iso = StdioIsolation() + saved_stdout_fd = os.dup(1) + saved_stderr_fd = os.dup(2) + saved_stdout = sys.stdout + try: + r_fd, w_fd = os.pipe() + os.dup2(w_fd, 1) + os.close(w_fd) + + r2_fd, w2_fd = os.pipe() + os.dup2(w2_fd, 2) + os.close(w2_fd) + + stream = iso.install() + assert iso.installed + + stream.write(b"protocol data") + stream.flush() + + os.write(1, b"tool output on fd1") + + os.dup2(saved_stdout_fd, 1) + os.dup2(saved_stderr_fd, 2) + sys.stdout = saved_stdout + + protocol_data = os.read(r_fd, 4096) + stderr_data = os.read(r2_fd, 4096) + + assert protocol_data == b"protocol data" + assert b"tool output on fd1" in stderr_data + + iso.close() + os.close(r_fd) + os.close(r2_fd) + finally: + os.dup2(saved_stdout_fd, 1) + os.dup2(saved_stderr_fd, 2) + sys.stdout = saved_stdout + os.close(saved_stdout_fd) + os.close(saved_stderr_fd) + + def test_double_install_is_idempotent(self): + iso = StdioIsolation() + saved_stdout_fd = os.dup(1) + saved_stderr_fd = os.dup(2) + saved_stdout = sys.stdout + try: + r_fd, w_fd = os.pipe() + os.dup2(w_fd, 1) + os.close(w_fd) + r2_fd, w2_fd = os.pipe() + os.dup2(w2_fd, 2) + os.close(w2_fd) + + s1 = iso.install() + s2 = iso.install() + assert s1 is s2 + + iso.close() + os.close(r_fd) + os.close(r2_fd) + finally: + os.dup2(saved_stdout_fd, 1) + os.dup2(saved_stderr_fd, 2) + sys.stdout = saved_stdout + os.close(saved_stdout_fd) + os.close(saved_stderr_fd) diff --git a/test/runtime/test_stdio_server.py b/test/runtime/test_stdio_server.py index 4fb400f3..bd8b5a68 100644 --- a/test/runtime/test_stdio_server.py +++ b/test/runtime/test_stdio_server.py @@ -112,39 +112,34 @@ def test_stdio_server_stops_after_shutdown_notification_in_buffer(): assert stdout.getvalue() == b"" -def test_stdio_server_redirects_print_noise_away_from_protocol_stdout(capfd): - server = RuntimeServer() - server.dispatcher.add_method("test.noisyPrint", lambda: print("tool output") or {"ok": True}) - stdin = io.BytesIO(_request("test.noisyPrint", 1)) - stdout = io.BytesIO() - - rc = run_stdio_server(stdin, stdout, server=server) - - captured = capfd.readouterr() - assert rc == 0 - assert captured.out == "" - assert "tool output" in captured.err - assert _decode_output(stdout.getvalue())[0]["result"] == {"ok": True} - - -def test_stdio_server_redirects_fd_stdout_noise_away_from_protocol_stdout(capfd): - server = RuntimeServer() - - def noisy_fd(): - os.write(1, b"tool output\n") - return {"ok": True} - - server.dispatcher.add_method("test.noisyFd", noisy_fd) - stdin = io.BytesIO(_request("test.noisyFd", 1)) - stdout = io.BytesIO() - - rc = run_stdio_server(stdin, stdout, server=server) +def test_rpc_stdio_subprocess_print_noise_stays_on_stderr(): + """Production path (main) installs StdioIsolation; handler print() goes to stderr.""" + stdin_data = _request("test.noisy", 1) + _request("rpc.shutdown", 2) + completed = subprocess.run( + [ + sys.executable, + "-c", + "import sys;" + "from chipcompiler.runtime.stdio_isolation import StdioIsolation;" + "iso = StdioIsolation(); ps = iso.install();" + "from chipcompiler.runtime.server import RuntimeServer;" + "from chipcompiler.runtime.stdio_server import run_stdio_server;" + "s = RuntimeServer();" + "s.dispatcher.add_method('test.noisy', lambda: print('tool output') or {'ok': True});" + "rc = run_stdio_server(sys.stdin.buffer, ps, server=s);" + "iso.close(); sys.exit(rc)", + ], + input=stdin_data, + capture_output=True, + check=False, + ) - captured = capfd.readouterr() - assert rc == 0 - assert captured.out == "" - assert "tool output" in captured.err - assert _decode_output(stdout.getvalue())[0]["result"] == {"ok": True} + assert completed.returncode == 0 + responses = _decode_output(completed.stdout) + assert len(responses) == 2 + assert responses[0]["result"] == {"ok": True} + assert b"tool output" not in completed.stdout + assert b"tool output" in completed.stderr def test_rpc_stdio_subprocess_smoke(): diff --git a/test/runtime/test_worker.py b/test/runtime/test_worker.py new file mode 100644 index 00000000..57811348 --- /dev/null +++ b/test/runtime/test_worker.py @@ -0,0 +1,552 @@ +"""Tests for chipcompiler.runtime.worker — lifecycle, signal handling, and state repair.""" + +import json +import os +import signal +import sys +import textwrap +import time +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from chipcompiler.runtime.worker import ( + WorkerClient, + WorkerProcessError, + WorkerResult, + classify_worker_exit, + repair_flow_state, +) + + +def _pid_alive(pid: int) -> bool: + try: + os.kill(pid, 0) + except OSError: + return False + # A container PID 1 that does not reap leaves killed children as zombies, + # and os.kill(pid, 0) still succeeds for zombies. + try: + stat = Path(f"/proc/{pid}/stat").read_text() + except OSError: + return False + return stat.rpartition(")")[2].split()[0] != "Z" + + +class TestWorkerResult: + def test_success_result(self): + r = WorkerResult(success=True, response={"result": {"ok": True}}) + assert r.success is True + assert r.response == {"result": {"ok": True}} + + def test_failure_result(self): + r = WorkerResult(success=False, error="timeout", exit_code=1) + assert r.success is False + assert r.error == "timeout" + assert r.exit_code == 1 + + +class TestClassifyWorkerExit: + def test_normal_exit(self): + proc = MagicMock() + proc.returncode = 0 + result = classify_worker_exit(proc) + assert result.success is True + assert result.exit_code == 0 + + def test_nonzero_exit(self): + proc = MagicMock() + proc.returncode = 1 + result = classify_worker_exit(proc) + assert result.success is False + assert result.exit_code == 1 + + def test_signal_kill(self): + proc = MagicMock() + proc.returncode = -signal.SIGKILL + result = classify_worker_exit(proc) + assert result.success is False + assert result.signal_number == signal.SIGKILL + assert "SIGKILL" in result.error + + def test_signal_abort(self): + proc = MagicMock() + proc.returncode = -signal.SIGABRT + result = classify_worker_exit(proc) + assert result.success is False + assert result.signal_number == signal.SIGABRT + + def test_still_running(self): + proc = MagicMock() + proc.returncode = None + result = classify_worker_exit(proc) + assert result.success is False + assert "still running" in result.error + + +class TestRepairFlowState: + def test_repairs_ongoing_to_incomplete(self, tmp_path): + flow_json = tmp_path / "flow.json" + data = { + "steps": [ + {"name": "Synthesis", "tool": "yosys", "state": "Success"}, + {"name": "Placement", "tool": "ecc", "state": "Ongoing"}, + {"name": "Routing", "tool": "ecc", "state": "Unstart"}, + ] + } + flow_json.write_text(json.dumps(data)) + repaired = repair_flow_state(flow_json, active_step="Placement") + assert repaired == ["Placement"] + result = json.loads(flow_json.read_text()) + assert result["steps"][1]["state"] == "Incomplete" + assert result["steps"][0]["state"] == "Success" + assert result["steps"][2]["state"] == "Unstart" + + def test_repairs_only_the_matching_tool_for_duplicate_names(self, tmp_path): + """A flow may carry the same step name under two tools; repair must + match the (name, tool) pair and never touch the other record.""" + flow_json = tmp_path / "flow.json" + data = { + "steps": [ + {"name": "place", "tool": "yosys", "state": "Success"}, + {"name": "place", "tool": "ecc", "state": "Ongoing"}, + ] + } + flow_json.write_text(json.dumps(data)) + repaired = repair_flow_state(flow_json, active_step="place", active_tool="ecc") + assert repaired == ["place"] + result = json.loads(flow_json.read_text()) + assert result["steps"][0]["state"] == "Success" + assert result["steps"][1]["state"] == "Incomplete" + + def test_repairs_active_success_interrupted_after_persisting(self, tmp_path): + """A Success without a completed end marker crashed in post-processing.""" + flow_json = tmp_path / "flow.json" + data = { + "steps": [ + {"name": "Synthesis", "tool": "yosys", "state": "Success"}, + {"name": "Routing", "tool": "ecc", "state": "Success"}, + ] + } + flow_json.write_text(json.dumps(data)) + repaired = repair_flow_state(flow_json, active_step="Synthesis") + assert repaired == ["Synthesis"] + result = json.loads(flow_json.read_text()) + assert result["steps"][0]["state"] == "Incomplete" + # A step that was not active when the worker died keeps its state. + assert result["steps"][1]["state"] == "Success" + + def test_missing_file(self, tmp_path): + flow_json = tmp_path / "nonexistent.json" + repaired = repair_flow_state(flow_json, active_step="X") + assert repaired == [] + + def test_empty_file(self, tmp_path): + flow_json = tmp_path / "flow.json" + flow_json.write_text("{}") + repaired = repair_flow_state(flow_json, active_step="X") + assert repaired == [] + + def test_scoped_repair_only_active_step(self, tmp_path): + flow_json = tmp_path / "flow.json" + data = { + "steps": [ + {"name": "A", "tool": "t1", "state": "Ongoing"}, + {"name": "B", "tool": "t2", "state": "Ongoing"}, + ] + } + flow_json.write_text(json.dumps(data)) + repaired = repair_flow_state(flow_json, active_step="B") + assert repaired == ["B"] + result = json.loads(flow_json.read_text()) + assert result["steps"][0]["state"] == "Ongoing" + assert result["steps"][1]["state"] == "Incomplete" + + def test_scoped_repair_ignores_terminal_steps_not_active(self, tmp_path): + flow_json = tmp_path / "flow.json" + data = { + "steps": [ + {"name": "Synthesis", "tool": "yosys", "state": "Success"}, + ] + } + flow_json.write_text(json.dumps(data)) + repaired = repair_flow_state(flow_json, active_step="Other") + assert repaired == [] + result = json.loads(flow_json.read_text()) + assert result["steps"][0]["state"] == "Success" + + def test_write_failure_raises_oserror(self, tmp_path, monkeypatch): + flow_json = tmp_path / "flow.json" + data = { + "steps": [ + {"name": "A", "tool": "t", "state": "Ongoing"}, + ] + } + flow_json.write_text(json.dumps(data)) + # chmod-based read-only setups are bypassed when tests run as root, + # so simulate the persist failure directly. + monkeypatch.setattr("chipcompiler.runtime.worker.json_write", lambda *a, **k: False) + with pytest.raises(OSError, match="failed to persist"): + repair_flow_state(flow_json, active_step="A") + + +class TestWorkerClientSubprocess: + """Integration test using a real subprocess.""" + + def test_start_and_terminate(self): + client = WorkerClient([sys.executable, "-c", "import time; time.sleep(60)"]) + client.start() + assert client.is_alive() + client.terminate() + assert not client.is_alive() + + def test_rpc_round_trip(self): + script = textwrap.dedent("""\ + import sys, json + data = b"" + while True: + chunk = sys.stdin.buffer.read(1) + if not chunk: + break + data += chunk + if b"\\r\\n\\r\\n" in data: + header, _, body_start = data.partition(b"\\r\\n\\r\\n") + length = int(header.split(b":")[1]) + while len(body_start) < length: + body_start += sys.stdin.buffer.read(1) + request = json.loads(body_start[:length]) + resp = {"jsonrpc": "2.0", "result": {"echo": True}, "id": request["id"]} + response = json.dumps(resp) + frame = f"Content-Length: {len(response)}\\r\\n\\r\\n{response}" + sys.stdout.buffer.write(frame.encode()) + sys.stdout.buffer.flush() + break + """) + client = WorkerClient([sys.executable, "-c", script]) + client.start() + try: + result = client.request("test.echo", {}) + assert result.success is True + assert result.response["result"]["echo"] is True + finally: + client.terminate() + + def test_response_correlation_skips_notification(self): + """A notification before the response must not steal the response slot.""" + script = textwrap.dedent("""\ + import sys, json + data = b"" + while True: + chunk = sys.stdin.buffer.read(1) + if not chunk: + break + data += chunk + if b"\\r\\n\\r\\n" in data: + header, _, body_start = data.partition(b"\\r\\n\\r\\n") + length = int(header.split(b":")[1]) + while len(body_start) < length: + body_start += sys.stdin.buffer.read(1) + request = json.loads(body_start[:length]) + # Send a notification first (no "id" field) + notif = json.dumps( + {"jsonrpc": "2.0", "method": "progress", "params": {"pct": 50}} + ) + frame_n = f"Content-Length: {len(notif)}\\r\\n\\r\\n{notif}" + sys.stdout.buffer.write(frame_n.encode()) + # Then the actual response + resp = json.dumps( + {"jsonrpc": "2.0", "result": {"done": True}, "id": request["id"]} + ) + frame_r = f"Content-Length: {len(resp)}\\r\\n\\r\\n{resp}" + sys.stdout.buffer.write(frame_r.encode()) + sys.stdout.buffer.flush() + break + """) + client = WorkerClient([sys.executable, "-c", script]) + client.start() + try: + result = client.request("test.work", {}, request_id=7) + assert result.success is True + assert result.response["result"]["done"] is True + notif = client.pop_notification() + assert notif is not None + assert notif["method"] == "progress" + finally: + client.terminate() + + def test_malformed_json_raises_protocol_error(self): + script = textwrap.dedent("""\ + import sys + garbage = b"not json at all" + frame = f"Content-Length: {len(garbage)}\\r\\n\\r\\n".encode() + garbage + sys.stdout.buffer.write(frame) + sys.stdout.buffer.flush() + import time; time.sleep(1) + """) + client = WorkerClient([sys.executable, "-c", script]) + client.start() + try: + with pytest.raises(WorkerProcessError, match="malformed JSON"): + client.read_response(request_id=1) + finally: + client.terminate() + + def test_invalid_envelope_missing_jsonrpc_raises(self): + """A response with no 'jsonrpc' field must be rejected.""" + script = textwrap.dedent("""\ + import sys, json + resp = json.dumps({"id": 1, "result": {}}).encode() + frame = f"Content-Length: {len(resp)}\\r\\n\\r\\n".encode() + resp + sys.stdout.buffer.write(frame) + sys.stdout.buffer.flush() + import time; time.sleep(1) + """) + client = WorkerClient([sys.executable, "-c", script]) + client.start() + try: + with pytest.raises(WorkerProcessError, match="jsonrpc"): + client.read_response(request_id=1) + finally: + client.terminate() + + def test_invalid_envelope_both_result_and_error_raises(self): + """A response with both 'result' and 'error' must be rejected.""" + script = textwrap.dedent("""\ + import sys, json + resp = json.dumps({ + "jsonrpc": "2.0", "id": 1, + "result": {}, "error": {"code": -1, "message": "x"} + }).encode() + frame = f"Content-Length: {len(resp)}\\r\\n\\r\\n".encode() + resp + sys.stdout.buffer.write(frame) + sys.stdout.buffer.flush() + import time; time.sleep(1) + """) + client = WorkerClient([sys.executable, "-c", script]) + client.start() + try: + with pytest.raises(WorkerProcessError, match="exactly one"): + client.read_response(request_id=1) + finally: + client.terminate() + + def test_invalid_envelope_neither_result_nor_error_raises(self): + """A response with neither 'result' nor 'error' must be rejected.""" + script = textwrap.dedent("""\ + import sys, json + resp = json.dumps({"jsonrpc": "2.0", "id": 1}).encode() + frame = f"Content-Length: {len(resp)}\\r\\n\\r\\n".encode() + resp + sys.stdout.buffer.write(frame) + sys.stdout.buffer.flush() + import time; time.sleep(1) + """) + client = WorkerClient([sys.executable, "-c", script]) + client.start() + try: + with pytest.raises(WorkerProcessError, match="exactly one"): + client.read_response(request_id=1) + finally: + client.terminate() + + def test_invalid_response_id_boolean_raises(self): + """A response with id=true (boolean) must be rejected, not confused with int 1.""" + script = textwrap.dedent("""\ + import sys, json + resp = json.dumps({"jsonrpc": "2.0", "id": True, "result": {}}).encode() + frame = f"Content-Length: {len(resp)}\\r\\n\\r\\n".encode() + resp + sys.stdout.buffer.write(frame) + sys.stdout.buffer.flush() + import time; time.sleep(1) + """) + client = WorkerClient([sys.executable, "-c", script]) + client.start() + try: + with pytest.raises(WorkerProcessError, match="'id' must be int, str, or null"): + client.read_response(request_id=1) + finally: + client.terminate() + + def test_invalid_response_id_array_raises(self): + """A response with id=[1] (array) must be rejected.""" + script = textwrap.dedent("""\ + import sys, json + resp = json.dumps({"jsonrpc": "2.0", "id": [1], "result": {}}).encode() + frame = f"Content-Length: {len(resp)}\\r\\n\\r\\n".encode() + resp + sys.stdout.buffer.write(frame) + sys.stdout.buffer.flush() + import time; time.sleep(1) + """) + client = WorkerClient([sys.executable, "-c", script]) + client.start() + try: + with pytest.raises(WorkerProcessError, match="'id' must be int, str, or null"): + client.read_response(request_id=1) + finally: + client.terminate() + + def test_terminate_kills_orphaned_child(self): + """After the leader exits, terminate must still signal the process group.""" + script = textwrap.dedent("""\ + import os, sys, time + pid = os.fork() + if pid == 0: + # child: sleep indefinitely + time.sleep(60) + os._exit(0) + else: + # leader: print child pid and exit + sys.stdout.buffer.write(f"{pid}\\n".encode()) + sys.stdout.buffer.flush() + os._exit(0) + """) + client = WorkerClient([sys.executable, "-c", script]) + proc = client.start() + + time.sleep(0.3) + child_pid_line = proc.stdout.readline() + child_pid = int(child_pid_line.strip()) + client.terminate() + time.sleep(0.2) + + assert not _pid_alive(child_pid), ( + "orphaned child should have been killed by process-group signal" + ) + + def test_out_of_order_responses_preserved(self): + """Responses arriving in reverse order must all be retrievable.""" + script = textwrap.dedent("""\ + import sys, json + data = b"" + while True: + chunk = sys.stdin.buffer.read(1) + if not chunk: + break + data += chunk + if b"\\r\\n\\r\\n" in data: + header, _, body_start = data.partition(b"\\r\\n\\r\\n") + length = int(header.split(b":")[1]) + while len(body_start) < length: + body_start += sys.stdin.buffer.read(1) + json.loads(body_start[:length]) + # Send responses in reverse order: id=2 then id=1 + r2 = json.dumps( + {"jsonrpc": "2.0", "result": {"v": 2}, "id": 2} + ) + r1 = json.dumps( + {"jsonrpc": "2.0", "result": {"v": 1}, "id": 1} + ) + for r in [r2, r1]: + frame = f"Content-Length: {len(r)}\\r\\n\\r\\n{r}" + sys.stdout.buffer.write(frame.encode()) + sys.stdout.buffer.flush() + break + """) + client = WorkerClient([sys.executable, "-c", script]) + client.start() + try: + client.send_request("test", {}, request_id=1) + resp1 = client.read_response(request_id=1) + assert resp1["result"]["v"] == 1 + resp2 = client.read_response(request_id=2) + assert resp2["result"]["v"] == 2 + finally: + client.terminate() + + def test_leader_exits_during_sigint_descendant_killed(self): + """Leader exits on SIGINT but descendant ignores it; must still be killed.""" + script = textwrap.dedent("""\ + import os, sys, signal, time + pid = os.fork() + if pid == 0: + signal.signal(signal.SIGINT, signal.SIG_IGN) + time.sleep(60) + os._exit(0) + else: + sys.stdout.buffer.write(f"{pid}\\n".encode()) + sys.stdout.buffer.flush() + # Leader exits immediately on SIGINT + signal.signal(signal.SIGINT, lambda *a: os._exit(0)) + time.sleep(60) + """) + client = WorkerClient([sys.executable, "-c", script]) + proc = client.start() + + time.sleep(0.3) + child_pid_line = proc.stdout.readline() + child_pid = int(child_pid_line.strip()) + client.terminate() + time.sleep(0.5) + + assert not _pid_alive(child_pid), ( + "descendant ignoring SIGINT should still be killed by SIGTERM/SIGKILL" + ) + + def test_descendant_graceful_sigterm_exit(self): + """Descendant handles SIGTERM and exits within grace window — no SIGKILL needed.""" + script = textwrap.dedent("""\ + import os, sys, signal, time + pid = os.fork() + if pid == 0: + # Child: ignore SIGINT, handle SIGTERM with brief cleanup + signal.signal(signal.SIGINT, signal.SIG_IGN) + marker = f"/tmp/ecc-test-grace-{os.getpid()}" + def handle_term(*a): + open(marker, "w").close() + os._exit(0) + signal.signal(signal.SIGTERM, handle_term) + time.sleep(60) + os._exit(0) + else: + sys.stdout.buffer.write(f"{pid}\\n".encode()) + sys.stdout.buffer.flush() + signal.signal(signal.SIGINT, lambda *a: os._exit(0)) + time.sleep(60) + """) + client = WorkerClient([sys.executable, "-c", script]) + proc = client.start() + + time.sleep(0.3) + child_pid_line = proc.stdout.readline() + child_pid = int(child_pid_line.strip()) + client.terminate() + time.sleep(0.5) + + marker = f"/tmp/ecc-test-grace-{child_pid}" + assert not _pid_alive(child_pid), "descendant should have exited on SIGTERM" + assert os.path.exists(marker), "descendant SIGTERM handler should have run (not SIGKILL'd)" + os.unlink(marker) + + def test_graceful_terminate_completes_before_forceful_deadline(self): + """Graceful shutdown must complete well before the SIGKILL deadline (< 5s total).""" + script = textwrap.dedent("""\ + import os, sys, signal, time + pid = os.fork() + if pid == 0: + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, lambda *a: os._exit(0)) + time.sleep(60) + os._exit(0) + else: + sys.stdout.buffer.write(f"{pid}\\n".encode()) + sys.stdout.buffer.flush() + signal.signal(signal.SIGINT, signal.SIG_IGN) + # Reap the child before exiting so no zombie keeps the process + # group visible to killpg under a non-reaping PID 1. + def handle_term(*a): + os.waitpid(pid, 0) + os._exit(0) + signal.signal(signal.SIGTERM, handle_term) + time.sleep(60) + """) + client = WorkerClient([sys.executable, "-c", script]) + proc = client.start() + + time.sleep(0.3) + proc.stdout.readline() + t0 = time.monotonic() + client.terminate() + elapsed = time.monotonic() - t0 + assert elapsed < 5.0, ( + f"graceful terminate took {elapsed:.2f}s — should complete before forceful deadline" + ) diff --git a/test/runtime/test_worker_operation.py b/test/runtime/test_worker_operation.py new file mode 100644 index 00000000..fb5d77ce --- /dev/null +++ b/test/runtime/test_worker_operation.py @@ -0,0 +1,808 @@ +"""Tests for chipcompiler.runtime.worker_operation — typed operation orchestrator.""" + +import json +import os +import sys +import textwrap + +import pytest + +from chipcompiler.runtime.worker import WorkerClient +from chipcompiler.runtime.worker_operation import RunOperation + +# Common RPC helpers injected into subprocess scripts. +_RPC_HELPERS = textwrap.dedent("""\ + import sys, os, json + + def read_request(): + data = b"" + while True: + chunk = sys.stdin.buffer.read(1) + if not chunk: + return None + data += chunk + if b"\\r\\n\\r\\n" in data: + header, _, body_start = data.partition(b"\\r\\n\\r\\n") + length = int(header.split(b":")[1]) + while len(body_start) < length: + body_start += sys.stdin.buffer.read(1) + return json.loads(body_start[:length]) + + def send_response(resp): + payload = json.dumps(resp) + frame = f"Content-Length: {len(payload)}\\r\\n\\r\\n{payload}" + sys.stdout.buffer.write(frame.encode()) + sys.stdout.buffer.flush() + + def make_marker(event, step, tool): + p = json.dumps({"v": 1, "event": event, "step": step, "tool": tool}) + return chr(0x1e).encode() + b"ECC-STEP " + p.encode() + b"\\n" +""") + +LIFECYCLE_SERVER = _RPC_HELPERS + textwrap.dedent("""\ + while True: + req = read_request() + if req is None: + break + method = req.get("method", "") + req_id = req.get("id") + + if method == "rpc.hello": + resp = {"jsonrpc": "2.0", "result": {"version": 1, "capabilities": []}, "id": req_id} + send_response(resp) + elif method == "workspace.open": + send_response({"jsonrpc": "2.0", "result": {"workspaceId": "test"}, "id": req_id}) + elif method == "rpc.shutdown": + send_response({"jsonrpc": "2.0", "result": {"ok": True}, "id": req_id}) + break + elif method == "flow.run": + send_response({"jsonrpc": "2.0", "result": {"steps": ["syn"]}, "id": req_id}) + else: + err = {"code": -32601, "message": "unknown method"} + resp = {"jsonrpc": "2.0", "error": err, "id": req_id} + send_response(resp) +""") + + +class TestRunOperationLifecycle: + def test_successful_lifecycle(self, tmp_path): + script = tmp_path / "server.py" + script.write_text(LIFECYCLE_SERVER) + flow_json = tmp_path / "flow.json" + flow_json.write_text("{}") + op = RunOperation( + workspace_dir=tmp_path, + flow_json_path=flow_json, + worker_argv=[sys.executable, str(script)], + ) + result = op.run("flow.run", {"workspace_id": "test"}) + assert result.success is True + assert result.rpc_result["result"] == {"steps": ["syn"]} + assert result.exit_code == 0 + assert result.archive_error is None + + def test_rpc_error_returns_failure(self, tmp_path): + script = tmp_path / "server.py" + script.write_text(LIFECYCLE_SERVER) + flow_json = tmp_path / "flow.json" + flow_json.write_text("{}") + op = RunOperation( + workspace_dir=tmp_path, + flow_json_path=flow_json, + worker_argv=[sys.executable, str(script)], + ) + result = op.run("unknown.method", {"workspace_id": "test"}) + assert result.success is False + assert "unknown method" in result.error + + +class TestRunOperationSequence: + @staticmethod + def _write_server(tmp_path): + script = tmp_path / "sequence_server.py" + script.write_text( + _RPC_HELPERS + + textwrap.dedent("""\ + received = [] + fail_first = os.environ.get("SEQ_FAIL_FIRST") == "1" + while True: + req = read_request() + if req is None: + break + method = req.get("method", "") + req_id = req.get("id") + received.append(method) + if method == "rpc.hello": + send_response({"jsonrpc": "2.0", "result": {"version": 1}, "id": req_id}) + elif method == "workspace.open": + result = {"workspaceId": "test"} + send_response({"jsonrpc": "2.0", "result": result, "id": req_id}) + elif method == "flow.run_step": + if fail_first: + err = {"code": -32000, "message": "step failed"} + send_response({"jsonrpc": "2.0", "error": err, "id": req_id}) + else: + result = {"step": "ok"} + send_response({"jsonrpc": "2.0", "result": result, "id": req_id}) + elif method == "flow.run": + send_response({"jsonrpc": "2.0", "result": {"ran": True}, "id": req_id}) + elif method == "rpc.shutdown": + with open(os.environ["SEQ_LOG"], "w") as fh: + fh.write("\\n".join(received)) + send_response({"jsonrpc": "2.0", "result": {"ok": True}, "id": req_id}) + break + else: + err = {"code": -32601, "message": "unknown method"} + send_response({"jsonrpc": "2.0", "error": err, "id": req_id}) + """) + ) + return script + + def _make_operation(self, tmp_path, monkeypatch, script, *, fail_first): + log_file = tmp_path / "received.txt" + monkeypatch.setenv("SEQ_LOG", str(log_file)) + if fail_first: + monkeypatch.setenv("SEQ_FAIL_FIRST", "1") + else: + monkeypatch.delenv("SEQ_FAIL_FIRST", raising=False) + flow_json = tmp_path / "flow.json" + flow_json.write_text("{}") + op = RunOperation( + workspace_dir=tmp_path, + flow_json_path=flow_json, + worker_argv=[sys.executable, str(script)], + ) + return op, log_file + + CALLS = [ + ("flow.run_step", {"step": "Synthesis", "rerun": True, "reset_dependents": True}), + ("flow.run", {"rerun": False}), + ] + + def test_successful_sequence_returns_last_call_result(self, tmp_path, monkeypatch): + script = self._write_server(tmp_path) + op, log_file = self._make_operation(tmp_path, monkeypatch, script, fail_first=False) + result = op.run_sequence(self.CALLS) + assert result.success is True + assert result.rpc_result["result"] == {"ran": True} + received = log_file.read_text().splitlines() + assert received == [ + "rpc.hello", + "workspace.open", + "flow.run_step", + "flow.run", + "rpc.shutdown", + ] + + def test_first_failure_skips_follow_up_and_still_shuts_down(self, tmp_path, monkeypatch): + script = self._write_server(tmp_path) + op, log_file = self._make_operation(tmp_path, monkeypatch, script, fail_first=True) + result = op.run_sequence(self.CALLS) + assert result.success is False + assert "step failed" in result.error + received = log_file.read_text().splitlines() + assert "flow.run" not in received + assert received[-1] == "rpc.shutdown" + + +class TestRunOperationRpcErrorRepair: + def test_rpc_error_with_unmatched_step_repairs_flow_state(self, tmp_path): + """A live worker's RPC error still repairs a step left Ongoing.""" + script = tmp_path / "error_after_begin.py" + script.write_text( + _RPC_HELPERS + + textwrap.dedent("""\ + while True: + req = read_request() + if req is None: + break + method = req.get("method", "") + req_id = req.get("id") + if method == "rpc.hello": + send_response({"jsonrpc": "2.0", "result": {"version": 1}, "id": req_id}) + elif method == "workspace.open": + result = {"workspaceId": "x"} + send_response({"jsonrpc": "2.0", "result": result, "id": req_id}) + elif method == "flow.run_step": + os.write(2, make_marker("begin", "Synthesis", "yosys")) + os.write(2, b"partial output\\n") + err = {"code": -32000, "message": "run step Synthesis failed"} + send_response({"jsonrpc": "2.0", "error": err, "id": req_id}) + elif method == "rpc.shutdown": + send_response({"jsonrpc": "2.0", "result": {"ok": True}, "id": req_id}) + break + else: + err = {"code": -32601, "message": "unknown method"} + send_response({"jsonrpc": "2.0", "error": err, "id": req_id}) + """) + ) + flow_json = tmp_path / "flow.json" + data = {"steps": [{"name": "Synthesis", "tool": "yosys", "state": "Ongoing"}]} + flow_json.write_text(json.dumps(data)) + op = RunOperation( + workspace_dir=tmp_path, + flow_json_path=flow_json, + worker_argv=[sys.executable, str(script)], + ) + result = op.run_sequence([("flow.run_step", {"step": "Synthesis", "rerun": True})]) + assert result.success is False + assert "run step Synthesis failed" in result.error + assert result.repaired_steps == ["Synthesis"] + repaired_data = json.loads(flow_json.read_text()) + assert repaired_data["steps"][0]["state"] == "Incomplete" + + +class TestRunOperationCrash: + def test_worker_crash_triggers_repair(self, tmp_path): + crash_script = tmp_path / "crash_after_open.py" + crash_script.write_text( + _RPC_HELPERS + + textwrap.dedent("""\ + req = read_request() # hello + send_response({"jsonrpc": "2.0", "result": {"version": 1}, "id": req["id"]}) + req = read_request() # workspace.open + send_response({"jsonrpc": "2.0", "result": {"workspaceId": "x"}, "id": req["id"]}) + # Emit begin marker on stderr, then crash + os.write(2, make_marker("begin", "Synthesis", "yosys")) + os._exit(1) + """) + ) + flow_json = tmp_path / "flow.json" + data = {"steps": [{"name": "Synthesis", "tool": "yosys", "state": "Ongoing"}]} + flow_json.write_text(json.dumps(data)) + op = RunOperation( + workspace_dir=tmp_path, + flow_json_path=flow_json, + worker_argv=[sys.executable, str(crash_script)], + ) + result = op.run("flow.run", {"workspace_id": "test"}) + assert result.success is False + assert result.repaired_steps == ["Synthesis"] + repaired_data = json.loads(flow_json.read_text()) + assert repaired_data["steps"][0]["state"] == "Incomplete" + + def test_crash_repair_failure_is_reported(self, tmp_path, monkeypatch): + """A repair that cannot persist must surface in the result error.""" + crash_script = tmp_path / "crash_after_open.py" + crash_script.write_text( + _RPC_HELPERS + + textwrap.dedent("""\ + req = read_request() # hello + send_response({"jsonrpc": "2.0", "result": {"version": 1}, "id": req["id"]}) + req = read_request() # workspace.open + send_response({"jsonrpc": "2.0", "result": {"workspaceId": "x"}, "id": req["id"]}) + os.write(2, make_marker("begin", "Synthesis", "yosys")) + os._exit(1) + """) + ) + flow_json = tmp_path / "flow.json" + data = {"steps": [{"name": "Synthesis", "tool": "yosys", "state": "Ongoing"}]} + flow_json.write_text(json.dumps(data)) + + def failing_repair(*args, **kwargs): + raise OSError("disk full") + + monkeypatch.setattr( + "chipcompiler.runtime.worker_operation.repair_flow_state", failing_repair + ) + op = RunOperation( + workspace_dir=tmp_path, + flow_json_path=flow_json, + worker_argv=[sys.executable, str(crash_script)], + ) + result = op.run("flow.run", {"workspace_id": "test"}) + assert result.success is False + assert result.repaired_steps == [] + assert "state repair failed" in result.error + # The record is left as it was — the failure is reported, not hidden. + assert json.loads(flow_json.read_text())["steps"][0]["state"] == "Ongoing" + + def test_parent_sigterm_terminates_the_worker_group(self, tmp_path): + """SIGTERM to the CLI must route through crash recovery and reap the + worker process group instead of leaving EDA descendants running.""" + import signal as signal_module + import threading + import time + + script = _RPC_HELPERS + textwrap.dedent("""\ + req = read_request() # hello + send_response({"jsonrpc": "2.0", "result": {"version": 1}, "id": req["id"]}) + req = read_request() # workspace.open + send_response({"jsonrpc": "2.0", "result": {"workspaceId": "x"}, "id": req["id"]}) + # Hang forever, simulating a long EDA step. + while True: + time.sleep(1) + """) + script = "import time\n" + script + flow_json = tmp_path / "flow.json" + flow_json.write_text(json.dumps({"steps": []})) + op = RunOperation( + workspace_dir=tmp_path, + flow_json_path=flow_json, + worker_argv=[sys.executable, "-c", script], + ) + + def send_sigterm(): + time.sleep(0.5) + os.kill(os.getpid(), signal_module.SIGTERM) + + killer = threading.Thread(target=send_sigterm, daemon=True) + killer.start() + result = op.run("flow.run", {"workspace_id": "test"}) + + assert result.success is False + assert "interrupted" in (result.error or "") + assert result.signal_number == -signal_module.SIGKILL or result.exit_code is not None + + def test_crash_before_any_marker_repairs_persisted_ongoing(self, tmp_path): + """Killed between the Ongoing save and the begin marker: no stream + evidence exists, so recovery falls back to the persisted Ongoing.""" + crash_script = tmp_path / "crash_before_marker.py" + crash_script.write_text( + _RPC_HELPERS + + textwrap.dedent("""\ + req = read_request() # hello + send_response({"jsonrpc": "2.0", "result": {"version": 1}, "id": req["id"]}) + req = read_request() # workspace.open + send_response({"jsonrpc": "2.0", "result": {"workspaceId": "x"}, "id": req["id"]}) + # Die before any marker — as if killed between the Ongoing save + # and the begin write. + os._exit(1) + """) + ) + flow_json = tmp_path / "flow.json" + flow_json.write_text( + json.dumps({"steps": [{"name": "Synthesis", "tool": "yosys", "state": "Ongoing"}]}) + ) + op = RunOperation( + workspace_dir=tmp_path, + flow_json_path=flow_json, + worker_argv=[sys.executable, str(crash_script)], + ) + result = op.run("flow.run", {"workspace_id": "test"}) + assert result.success is False + assert result.repaired_steps == ["Synthesis"] + repaired = json.loads(flow_json.read_text()) + assert repaired["steps"][0]["state"] == "Incomplete" + + def test_worker_crash_no_flow_json_no_repair(self, tmp_path): + script = _RPC_HELPERS + textwrap.dedent("""\ + req = read_request() # hello + send_response({"jsonrpc": "2.0", "result": {"version": 1}, "id": req["id"]}) + req = read_request() # workspace.open + send_response({"jsonrpc": "2.0", "result": {"workspaceId": "x"}, "id": req["id"]}) + sys.exit(1) + """) + flow_json = tmp_path / "flow.json" + op = RunOperation( + workspace_dir=tmp_path, + flow_json_path=flow_json, + worker_argv=[sys.executable, "-c", script], + ) + result = op.run("flow.run", {"workspace_id": "test"}) + assert result.success is False + assert result.repaired_steps == [] + + def test_protocol_failure_routes_to_recovery(self, tmp_path): + """A worker that returns invalid JSON triggers crash recovery.""" + script = _RPC_HELPERS + textwrap.dedent("""\ + req = read_request() # hello + send_response({"jsonrpc": "2.0", "result": {"version": 1}, "id": req["id"]}) + req = read_request() # workspace.open + send_response({"jsonrpc": "2.0", "result": {"workspaceId": "x"}, "id": req["id"]}) + # Emit begin marker, then send garbage on stdout + os.write(2, make_marker("begin", "Place", "ecc")) + req = read_request() # flow.run + # Send invalid frame (bad Content-Length header) + sys.stdout.buffer.write(b"Content-Length: 5\\r\\n\\r\\n{}") + sys.stdout.buffer.flush() + sys.stdout.buffer.close() + os._exit(1) + """) + flow_json = tmp_path / "flow.json" + data = {"steps": [{"name": "Place", "tool": "ecc", "state": "Ongoing"}]} + flow_json.write_text(json.dumps(data)) + op = RunOperation( + workspace_dir=tmp_path, + flow_json_path=flow_json, + worker_argv=[sys.executable, "-c", script], + ) + result = op.run("flow.run", {"workspace_id": "test"}) + assert result.success is False + assert result.repaired_steps == ["Place"] + + +class TestRunOperationArchive: + def test_archive_error_forces_failure(self, tmp_path): + """Success is False when archive path cannot be opened.""" + script = tmp_path / "server_with_markers.py" + script.write_text( + _RPC_HELPERS + + textwrap.dedent("""\ + req = read_request() # hello + send_response({"jsonrpc": "2.0", "result": {"version": 1}, "id": req["id"]}) + req = read_request() # workspace.open + send_response({"jsonrpc": "2.0", "result": {"workspaceId": "ws1"}, "id": req["id"]}) + os.write(2, make_marker("begin", "Synthesis", "yosys")) + os.write(2, b'Synthesizing...\\n') + os.write(2, make_marker("end", "Synthesis", "yosys")) + req = read_request() # flow.run + send_response({"jsonrpc": "2.0", "result": {"steps": ["syn"]}, "id": req["id"]}) + req = read_request() # rpc.shutdown + send_response({"jsonrpc": "2.0", "result": {"ok": True}, "id": req["id"]}) + """) + ) + + # A regular file as path component makes mkdir fail with ENOTDIR even + # when tests run as root (chmod-based read-only setups are bypassed). + blocker = tmp_path / "not_a_dir" + blocker.write_text("regular file") + + def bad_resolver(step: str, tool: str): + return blocker / "sub" / f"{step}.log" + + flow_json = tmp_path / "flow.json" + flow_json.write_text("{}") + op = RunOperation( + workspace_dir=tmp_path, + flow_json_path=flow_json, + worker_argv=[sys.executable, str(script)], + log_path_resolver=bad_resolver, + ) + result = op.run("flow.run", {"workspace_id": "test"}) + assert result.success is False + assert result.archive_error is not None + assert "archive error" in result.error + + def test_display_callback_failure_is_not_an_archive_failure(self, tmp_path): + """A broken on_output renderer must not downgrade the step or fail + the operation when the archive itself is fine.""" + script = tmp_path / "server_with_markers.py" + script.write_text( + _RPC_HELPERS + + textwrap.dedent("""\ + req = read_request() # hello + send_response({"jsonrpc": "2.0", "result": {"version": 1}, "id": req["id"]}) + req = read_request() # workspace.open + send_response({"jsonrpc": "2.0", "result": {"workspaceId": "ws1"}, "id": req["id"]}) + os.write(2, make_marker("begin", "Synthesis", "yosys")) + os.write(2, b'Synthesizing...\\n') + os.write(2, make_marker("end", "Synthesis", "yosys")) + req = read_request() # flow.run + send_response({"jsonrpc": "2.0", "result": {"steps": ["syn"]}, "id": req["id"]}) + req = read_request() # rpc.shutdown + send_response({"jsonrpc": "2.0", "result": {"ok": True}, "id": req["id"]}) + """) + ) + logs_dir = tmp_path / "logs" + + def resolver(step: str, tool: str): + return logs_dir / f"{step}.log" + + def broken_display(data: bytes): + raise RuntimeError("renderer blew up") + + flow_json = tmp_path / "flow.json" + flow_json.write_text( + json.dumps({"steps": [{"name": "Synthesis", "tool": "yosys", "state": "Success"}]}) + ) + op = RunOperation( + workspace_dir=tmp_path, + flow_json_path=flow_json, + worker_argv=[sys.executable, str(script)], + log_path_resolver=resolver, + on_output=broken_display, + ) + result = op.run("flow.run", {"workspace_id": "test"}) + assert result.success is True + assert result.repaired_steps == [] + assert result.log_state is not None + assert result.log_state.display_error is not None + assert result.log_state.error is None + assert json.loads(flow_json.read_text())["steps"][0]["state"] == "Success" + + def test_archive_error_reconciles_the_success_record(self, tmp_path): + """An archive failure must not leave flow.json claiming Success.""" + script = tmp_path / "server_with_markers.py" + script.write_text( + _RPC_HELPERS + + textwrap.dedent("""\ + req = read_request() # hello + send_response({"jsonrpc": "2.0", "result": {"version": 1}, "id": req["id"]}) + req = read_request() # workspace.open + send_response({"jsonrpc": "2.0", "result": {"workspaceId": "ws1"}, "id": req["id"]}) + os.write(2, make_marker("begin", "Synthesis", "yosys")) + os.write(2, b'Synthesizing...\\n') + os.write(2, make_marker("end", "Synthesis", "yosys")) + req = read_request() # flow.run + send_response({"jsonrpc": "2.0", "result": {"steps": ["syn"]}, "id": req["id"]}) + req = read_request() # rpc.shutdown + send_response({"jsonrpc": "2.0", "result": {"ok": True}, "id": req["id"]}) + """) + ) + + blocker = tmp_path / "not_a_dir" + blocker.write_text("regular file") + + def bad_resolver(step: str, tool: str): + return blocker / "sub" / f"{step}.log" + + flow_json = tmp_path / "flow.json" + flow_json.write_text( + json.dumps({"steps": [{"name": "Synthesis", "tool": "yosys", "state": "Success"}]}) + ) + op = RunOperation( + workspace_dir=tmp_path, + flow_json_path=flow_json, + worker_argv=[sys.executable, str(script)], + log_path_resolver=bad_resolver, + ) + result = op.run("flow.run", {"workspace_id": "test"}) + assert result.success is False + assert result.archive_error is not None + # The persisted record is downgraded so a later resume reruns the step + # and recreates the missing log instead of skipping it. + assert result.repaired_steps == ["Synthesis"] + repaired = json.loads(flow_json.read_text()) + assert repaired["steps"][0]["state"] == "Incomplete" + + def test_stderr_archived_to_step_log(self, tmp_path): + script = tmp_path / "server_with_markers.py" + script.write_text( + _RPC_HELPERS + + textwrap.dedent("""\ + req = read_request() # hello + send_response({"jsonrpc": "2.0", "result": {"version": 1}, "id": req["id"]}) + req = read_request() # workspace.open + send_response({"jsonrpc": "2.0", "result": {"workspaceId": "ws1"}, "id": req["id"]}) + os.write(2, make_marker("begin", "Synthesis", "yosys")) + os.write(2, b'Synthesizing module top...\\n') + os.write(2, make_marker("end", "Synthesis", "yosys")) + req = read_request() # flow.run + send_response({"jsonrpc": "2.0", "result": {}, "id": req["id"]}) + req = read_request() # rpc.shutdown + send_response({"jsonrpc": "2.0", "result": {"ok": True}, "id": req["id"]}) + """) + ) + logs_dir = tmp_path / "logs" + + def resolver(step: str, tool: str): + return logs_dir / f"{step}.log" + + flow_json = tmp_path / "flow.json" + flow_json.write_text("{}") + op = RunOperation( + workspace_dir=tmp_path, + flow_json_path=flow_json, + worker_argv=[sys.executable, str(script)], + log_path_resolver=resolver, + ) + result = op.run("flow.run", {"workspace_id": "test"}) + assert result.success is True + log_file = logs_dir / "Synthesis.log" + assert log_file.exists() + assert b"Synthesizing module top..." in log_file.read_bytes() + + +class TestWorkspaceIdInjection: + def test_workspace_id_injected_into_flow_params(self, tmp_path): + """workspace_id from workspace.open is injected into the flow request params.""" + script = tmp_path / "echo_server.py" + script.write_text( + _RPC_HELPERS + + textwrap.dedent("""\ + req = read_request() # hello + send_response({"jsonrpc": "2.0", "result": {"version": 1}, "id": req["id"]}) + req = read_request() # workspace.open + ws_resp = {"workspaceId": "injected-ws"} + send_response({"jsonrpc": "2.0", "result": ws_resp, "id": req["id"]}) + req = read_request() # flow.run — echo params back + send_response({"jsonrpc": "2.0", "result": req.get("params", {}), "id": req["id"]}) + req = read_request() # rpc.shutdown + send_response({"jsonrpc": "2.0", "result": {"ok": True}, "id": req["id"]}) + """) + ) + flow_json = tmp_path / "flow.json" + flow_json.write_text("{}") + op = RunOperation( + workspace_dir=tmp_path, + flow_json_path=flow_json, + worker_argv=[sys.executable, str(script)], + ) + result = op.run("flow.run", {"rerun": True}) + assert result.success is True + assert result.rpc_result["result"]["workspace_id"] == "injected-ws" + assert result.rpc_result["result"]["rerun"] is True + + def test_workspace_open_sends_directory_field(self, tmp_path): + """workspace.open sends 'directory' not 'path'.""" + script = tmp_path / "check_open_server.py" + script.write_text( + _RPC_HELPERS + + textwrap.dedent("""\ + req = read_request() # hello + send_response({"jsonrpc": "2.0", "result": {"version": 1}, "id": req["id"]}) + req = read_request() # workspace.open + params = req.get("params", {}) + assert "directory" in params, f"expected 'directory' in params, got {params}" + assert "path" not in params, f"unexpected 'path' in params: {params}" + send_response({"jsonrpc": "2.0", "result": {"workspaceId": "ok"}, "id": req["id"]}) + req = read_request() # flow.run + send_response({"jsonrpc": "2.0", "result": {}, "id": req["id"]}) + req = read_request() # rpc.shutdown + send_response({"jsonrpc": "2.0", "result": {"ok": True}, "id": req["id"]}) + """) + ) + flow_json = tmp_path / "flow.json" + flow_json.write_text("{}") + op = RunOperation( + workspace_dir=tmp_path, + flow_json_path=flow_json, + worker_argv=[sys.executable, str(script)], + ) + result = op.run("flow.run", {}) + assert result.success is True + + +class TestShutdownValidation: + """Strict ok is True validation in _graceful_shutdown.""" + + @pytest.mark.parametrize( + "ok_value", + [1, "yes", "true", [], {}], + ids=["int", "string", "string-true", "list", "dict"], + ) + def test_truthy_non_boolean_ok_fails_shutdown(self, tmp_path, ok_value): + """Non-True truthy values must not be accepted as successful shutdown.""" + script = tmp_path / "bad_shutdown.py" + script.write_text( + _RPC_HELPERS + + textwrap.dedent(f"""\ + req = read_request() # hello + send_response({{"jsonrpc": "2.0", "result": {{"version": 1}}, "id": req["id"]}}) + req = read_request() # workspace.open + send_response({{"jsonrpc": "2.0", "result": {{"workspaceId": "w"}}, "id": req["id"]}}) + req = read_request() # flow.run + send_response({{"jsonrpc": "2.0", "result": {{}}, "id": req["id"]}}) + req = read_request() # rpc.shutdown + ok_val = {repr(ok_value)} + resp = {{"jsonrpc": "2.0", "result": {{"ok": ok_val}}, "id": req["id"]}} + send_response(resp) + """) + ) + flow_json = tmp_path / "flow.json" + flow_json.write_text("{}") + op = RunOperation( + workspace_dir=tmp_path, + flow_json_path=flow_json, + worker_argv=[sys.executable, str(script)], + ) + result = op.run("flow.run", {}) + assert result.success is False + assert "worker did not exit cleanly" in result.error + + def test_false_ok_fails_shutdown(self, tmp_path): + """ok: false must not be accepted.""" + script = tmp_path / "false_shutdown.py" + script.write_text( + _RPC_HELPERS + + textwrap.dedent("""\ + req = read_request() # hello + send_response({"jsonrpc": "2.0", "result": {"version": 1}, "id": req["id"]}) + req = read_request() # workspace.open + send_response({"jsonrpc": "2.0", "result": {"workspaceId": "w"}, "id": req["id"]}) + req = read_request() # flow.run + send_response({"jsonrpc": "2.0", "result": {}, "id": req["id"]}) + req = read_request() # rpc.shutdown + send_response({"jsonrpc": "2.0", "result": {"ok": False}, "id": req["id"]}) + """) + ) + flow_json = tmp_path / "flow.json" + flow_json.write_text("{}") + op = RunOperation( + workspace_dir=tmp_path, + flow_json_path=flow_json, + worker_argv=[sys.executable, str(script)], + ) + result = op.run("flow.run", {}) + assert result.success is False + assert "worker did not exit cleanly" in result.error + + def test_missing_ok_fails_shutdown(self, tmp_path): + """Missing ok field must not be accepted.""" + script = tmp_path / "no_ok_shutdown.py" + script.write_text( + _RPC_HELPERS + + textwrap.dedent("""\ + req = read_request() # hello + send_response({"jsonrpc": "2.0", "result": {"version": 1}, "id": req["id"]}) + req = read_request() # workspace.open + send_response({"jsonrpc": "2.0", "result": {"workspaceId": "w"}, "id": req["id"]}) + req = read_request() # flow.run + send_response({"jsonrpc": "2.0", "result": {}, "id": req["id"]}) + req = read_request() # rpc.shutdown + send_response({"jsonrpc": "2.0", "result": {}, "id": req["id"]}) + """) + ) + flow_json = tmp_path / "flow.json" + flow_json.write_text("{}") + op = RunOperation( + workspace_dir=tmp_path, + flow_json_path=flow_json, + worker_argv=[sys.executable, str(script)], + ) + result = op.run("flow.run", {}) + assert result.success is False + assert "worker did not exit cleanly" in result.error + + +class TestParseMarkerNonObject: + def test_non_object_json_treated_as_raw(self): + """Non-dict JSON markers don't crash the reader.""" + from chipcompiler.runtime.log_stream import parse_marker + + assert parse_marker(b"\x1eECC-STEP []\n") is None + assert parse_marker(b"\x1eECC-STEP 42\n") is None + assert parse_marker(b'\x1eECC-STEP "hello"\n') is None + assert parse_marker(b"\x1eECC-STEP true\n") is None + assert parse_marker(b"\x1eECC-STEP null\n") is None + + +_ECC_BIN = os.path.join(os.path.dirname(sys.executable), "ecc") + + +@pytest.mark.skipif(not os.path.isfile(_ECC_BIN), reason="ecc binary not installed") +class TestRealServerLifecycle: + """Prove the RPC protocol works against the real ecc rpc serve --stdio --persistent-db.""" + + def test_hello_and_shutdown(self): + """rpc.hello + rpc.shutdown succeed against the real server.""" + client = WorkerClient([_ECC_BIN, "rpc", "serve", "--stdio", "--persistent-db"]) + proc = client.start() + try: + hello = client.request("rpc.hello", {"version": 1}, request_id=0) + assert hello.success is True + assert hello.response["result"]["version"] == 1 + + shutdown = client.request("rpc.shutdown", {}, request_id=0) + assert shutdown.success is True + assert shutdown.response["result"]["ok"] is True + + proc.wait(timeout=5.0) + assert proc.returncode == 0 + finally: + if proc.poll() is None: + client.terminate() + + def test_run_operation_with_real_workspace(self, tmp_path, minimal_ics55_pdk_factory): + """RunOperation lifecycle succeeds against the real server with a valid workspace.""" + from chipcompiler.data import create_workspace + + pdk_root = minimal_ics55_pdk_factory(tmp_path / "ics55") + rtl_path = tmp_path / "gcd.v" + rtl_path.write_text("module gcd(input clk, output y); assign y = clk; endmodule\n") + workspace_dir = tmp_path / "workspace" + create_workspace( + directory=workspace_dir, + origin_def="", + origin_verilog=rtl_path, + pdk="ics55", + pdk_root=pdk_root, + parameters={ + "PDK": "ics55", + "Design": "gcd", + "Top module": "gcd", + "Clock": "clk", + "Frequency max [MHz]": 100, + }, + ) + flow_json = workspace_dir / "home" / "flow.json" + op = RunOperation( + workspace_dir=workspace_dir, + flow_json_path=flow_json, + worker_argv=[_ECC_BIN, "rpc", "serve", "--stdio", "--persistent-db"], + ) + result = op.run("workspace.home", {}) + assert result.success is True + assert result.exit_code == 0 + assert "path" in result.rpc_result["result"] diff --git a/test/runtime/test_workspace_api.py b/test/runtime/test_workspace_api.py index 53960de0..7efb1b7b 100644 --- a/test/runtime/test_workspace_api.py +++ b/test/runtime/test_workspace_api.py @@ -89,8 +89,9 @@ def add_step(self, step, tool, state): {"name": step, "tool": tool, "state": state} ) - def create_step_workspaces(self): + def create_step_workspaces(self, *, executable_steps=None): self.created = True + self.executable_steps = executable_steps def run_steps(self, *, rerun=False): self.run_steps_calls.append(rerun) @@ -1397,6 +1398,405 @@ def step_spec(name, tool): } +def test_flow_run_step_direct_rerun_applies_reset_dependents_from_request(monkeypatch, tmp_path): + _capture, ws = _install_runtime_mocks(monkeypatch, tmp_path) + + def step_spec(name, tool): + step_dir = ws / f"{name}_{tool}" + artifact_dir = step_dir / "output" + artifact_dir.mkdir(parents=True) + (artifact_dir / "stale").write_text(name) + subflow_path = step_dir / "subflow.json" + subflow_path.write_text(json.dumps({"path": str(subflow_path), "steps": []})) + checklist_path = step_dir / "checklist.json" + checklist_path.write_text(json.dumps({"checklist": []})) + return { + "name": name, + "tool": tool, + "output": {"dir": artifact_dir}, + "subflow": SimpleNamespace(path=subflow_path, steps=[]), + "checklist": SimpleNamespace(path=checklist_path, checklist=[]), + } + + synthesis = step_spec("Synthesis", "yosys") + floorplan = step_spec("Floorplan", "ecc") + route = step_spec("route", "ecc") + DummyFlow.workspace_step_specs = (synthesis, floorplan, route) + api = WorkspaceRuntimeApi() + workspace_id = api.open_workspace(WorkspaceOpenRequest(directory=str(ws)))["workspaceId"] + session = api.sessions.get_session(workspace_id) + session.workspace.flow.data = { + "steps": [ + {"name": spec["name"], "tool": spec["tool"], "state": "Success"} + for spec in (synthesis, floorplan, route) + ] + } + + result = api.flow_run_step( + FlowRunStepRequest( + workspace_id=workspace_id, + step="Floorplan", + rerun=True, + reset_dependents=True, + ) + ) + + assert result == {"step": "Floorplan", "state": "Success"} + assert (synthesis["output"]["dir"] / "stale").read_text() == "Synthesis" + for spec in (floorplan, route): + assert list(spec["output"]["dir"].iterdir()) == [] + records = {record["name"]: record for record in session.workspace.flow.data["steps"]} + assert any( + record["name"] == "Synthesis" and record["state"] == "Success" + for record in session.workspace.flow.data["steps"] + ) + assert records["Floorplan"]["state"] == "Unstart" + assert records["route"]["state"] == "Unstart" + + +def test_flow_run_step_direct_rerun_invalidate_dependents_keeps_downstream_outputs( + monkeypatch, tmp_path +): + _capture, ws = _install_runtime_mocks(monkeypatch, tmp_path) + + def step_spec(name, tool): + step_dir = ws / f"{name}_{tool}" + artifact_dir = step_dir / "output" + artifact_dir.mkdir(parents=True) + (artifact_dir / "stale").write_text(name) + subflow_path = step_dir / "subflow.json" + subflow_path.write_text(json.dumps({"path": str(subflow_path), "steps": []})) + checklist_path = step_dir / "checklist.json" + checklist_path.write_text(json.dumps({"checklist": []})) + return { + "name": name, + "tool": tool, + "output": {"dir": artifact_dir}, + "subflow": SimpleNamespace(path=subflow_path, steps=[]), + "checklist": SimpleNamespace(path=checklist_path, checklist=[]), + } + + synthesis = step_spec("Synthesis", "yosys") + floorplan = step_spec("Floorplan", "ecc") + route = step_spec("route", "ecc") + DummyFlow.workspace_step_specs = (synthesis, floorplan, route) + api = WorkspaceRuntimeApi() + workspace_id = api.open_workspace(WorkspaceOpenRequest(directory=str(ws)))["workspaceId"] + session = api.sessions.get_session(workspace_id) + session.workspace.flow.data = { + "steps": [ + {"name": spec["name"], "tool": spec["tool"], "state": "Success"} + for spec in (synthesis, floorplan, route) + ] + } + + result = api.flow_run_step( + FlowRunStepRequest( + workspace_id=workspace_id, + step="Floorplan", + rerun=True, + invalidate_dependents=True, + ) + ) + + assert result == {"step": "Floorplan", "state": "Success"} + assert list(floorplan["output"]["dir"].iterdir()) == [] + assert (route["output"]["dir"] / "stale").read_text() == "route" + records = {record["name"]: record for record in session.workspace.flow.data["steps"]} + assert any( + record["name"] == "Synthesis" and record["state"] == "Success" + for record in session.workspace.flow.data["steps"] + ) + assert records["Floorplan"]["state"] == "Unstart" + assert records["route"]["state"] == "Unstart" + + +def test_flow_run_verifies_only_steps_that_will_execute(monkeypatch, tmp_path): + _capture, ws = _install_runtime_mocks(monkeypatch, tmp_path) + api = WorkspaceRuntimeApi() + workspace_id = api.open_workspace(WorkspaceOpenRequest(directory=str(ws)))["workspaceId"] + session = api.sessions.get_session(workspace_id) + session.workspace.flow.data = { + "steps": [ + {"name": "Synthesis", "tool": "yosys", "state": "Success"}, + {"name": "Floorplan", "tool": "ecc", "state": "Unstart"}, + ] + } + + api.flow_run(FlowRunRequest(workspace_id=workspace_id)) + + assert DummyFlow.instances[-1].executable_steps == {"Floorplan"} + + +def test_flow_run_step_verifies_only_the_target_step(monkeypatch, tmp_path): + _capture, ws = _install_runtime_mocks(monkeypatch, tmp_path) + api = WorkspaceRuntimeApi() + workspace_id = api.open_workspace(WorkspaceOpenRequest(directory=str(ws)))["workspaceId"] + + api.flow_run_step(FlowRunStepRequest(workspace_id=workspace_id, step="Floorplan", rerun=True)) + + assert DummyFlow.instances[-1].executable_steps == {"Floorplan"} + + +def test_rerun_prepare_refuses_to_modify_outputs_when_invalidation_save_fails( + monkeypatch, tmp_path +): + _capture, ws = _install_runtime_mocks(monkeypatch, tmp_path) + + step_dir = ws / "Floorplan_ecc" + artifact_dir = step_dir / "output" + artifact_dir.mkdir(parents=True) + (artifact_dir / "keep.txt").write_text("keep") + subflow_path = step_dir / "subflow.json" + subflow_path.write_text(json.dumps({"path": str(subflow_path), "steps": []})) + checklist_path = step_dir / "checklist.json" + checklist_path.write_text(json.dumps({"checklist": []})) + DummyFlow.workspace_step_specs = ( + { + "name": "Floorplan", + "tool": "ecc", + "output": {"dir": artifact_dir}, + "subflow": SimpleNamespace(path=subflow_path, steps=[]), + "checklist": SimpleNamespace(path=checklist_path, checklist=[]), + }, + ) + api = WorkspaceRuntimeApi() + workspace_id = api.open_workspace(WorkspaceOpenRequest(directory=str(ws)))["workspaceId"] + session = api.sessions.get_session(workspace_id) + session.workspace.flow.data = { + "steps": [{"name": "Floorplan", "tool": "ecc", "state": "Success"}] + } + monkeypatch.setattr(DummyFlow, "save", lambda self: False) + + with pytest.raises(RuntimeApiError, match="failed to persist step invalidation"): + api.flow_run_step( + FlowRunStepRequest(workspace_id=workspace_id, step="Floorplan", rerun=True) + ) + + # The artifact directory survived because the invalidation never persisted. + assert (artifact_dir / "keep.txt").read_text() == "keep" + + +def test_rerun_invalidate_dependents_save_failure_restores_all_records(monkeypatch, tmp_path): + """The combined target+downstream invalidation is one state transition: + when its single save fails, every record is restored and no artifact, + subflow, or checklist is touched.""" + _capture, ws = _install_runtime_mocks(monkeypatch, tmp_path) + + def step_spec(name, tool): + step_dir = ws / f"{name}_{tool}" + artifact_dir = step_dir / "output" + artifact_dir.mkdir(parents=True) + (artifact_dir / "stale").write_text(name) + subflow_path = step_dir / "subflow.json" + subflow_path.write_text(json.dumps({"path": str(subflow_path), "steps": []})) + checklist_path = step_dir / "checklist.json" + checklist_path.write_text(json.dumps({"checklist": []})) + return { + "name": name, + "tool": tool, + "output": {"dir": artifact_dir}, + "subflow": SimpleNamespace(path=subflow_path, steps=[]), + "checklist": SimpleNamespace(path=checklist_path, checklist=[]), + } + + synthesis = step_spec("Synthesis", "yosys") + floorplan = step_spec("Floorplan", "ecc") + route = step_spec("route", "ecc") + DummyFlow.workspace_step_specs = (synthesis, floorplan, route) + api = WorkspaceRuntimeApi() + workspace_id = api.open_workspace(WorkspaceOpenRequest(directory=str(ws)))["workspaceId"] + session = api.sessions.get_session(workspace_id) + session.workspace.flow.data = { + "steps": [ + { + "name": spec["name"], + "tool": spec["tool"], + "state": "Success", + "runtime": "0:01:00", + "peak memory (mb)": 123.0, + "info": {"cached": spec["name"]}, + } + for spec in (synthesis, floorplan, route) + ] + } + # Capture the original record objects and their full contents: rollback + # must restore the same dicts in place, not substitute new ones. + original_records = list(session.workspace.flow.data["steps"]) + original_snapshots = [dict(record) for record in original_records] + + save_calls = [] + + def failing_save(self): + save_calls.append(self) + return False + + monkeypatch.setattr(DummyFlow, "save", failing_save) + + with pytest.raises(RuntimeApiError, match="failed to persist step invalidation"): + api.flow_run_step( + FlowRunStepRequest( + workspace_id=workspace_id, + step="Floorplan", + rerun=True, + invalidate_dependents=True, + ) + ) + + # Exactly one save attempted for the combined invalidation; its failure + # left both memory and disk untouched, and no output was deleted. (The + # session flow build may append a fresh Synthesis entry after the three + # original records; the originals are what the invalidation touches.) + assert len(save_calls) == 1 + steps = session.workspace.flow.data["steps"] + assert steps[:3] == original_snapshots + assert all(step is original for step, original in zip(steps[:3], original_records, strict=True)) + assert (ws / "home" / "flow.json").read_text() == json.dumps({"steps": []}) + assert (floorplan["output"]["dir"] / "stale").read_text() == "Floorplan" + assert (route["output"]["dir"] / "stale").read_text() == "route" + assert json.loads(floorplan["subflow"].path.read_text()) == { + "path": str(floorplan["subflow"].path), + "steps": [], + } + + +def test_rerun_cleanup_failure_restores_persisted_records(monkeypatch, tmp_path): + """If post-save cleanup (artifact/subflow/checklist) fails midway, the + persisted Unstart records are restored so the workspace is not left + half-prepared.""" + _capture, ws = _install_runtime_mocks(monkeypatch, tmp_path) + + def step_spec(name, tool): + step_dir = ws / f"{name}_{tool}" + artifact_dir = step_dir / "output" + artifact_dir.mkdir(parents=True) + (artifact_dir / "stale").write_text(name) + subflow_path = step_dir / "subflow.json" + subflow_path.write_text(json.dumps({"path": str(subflow_path), "steps": []})) + checklist_path = step_dir / "checklist.json" + checklist_path.write_text(json.dumps({"checklist": []})) + return { + "name": name, + "tool": tool, + "output": {"dir": artifact_dir}, + "subflow": SimpleNamespace(path=subflow_path, steps=[]), + "checklist": SimpleNamespace(path=checklist_path, checklist=[]), + } + + synthesis = step_spec("Synthesis", "yosys") + floorplan = step_spec("Floorplan", "ecc") + route = step_spec("route", "ecc") + DummyFlow.workspace_step_specs = (synthesis, floorplan, route) + api = WorkspaceRuntimeApi() + workspace_id = api.open_workspace(WorkspaceOpenRequest(directory=str(ws)))["workspaceId"] + session = api.sessions.get_session(workspace_id) + session.workspace.flow.data = { + "steps": [ + {"name": spec["name"], "tool": spec["tool"], "state": "Success"} + for spec in (synthesis, floorplan, route) + ] + } + original_snapshots = [dict(r) for r in session.workspace.flow.data["steps"]] + + def exploding_reset(workspace_step): + raise OSError("checklist write failed") + + monkeypatch.setattr("chipcompiler.runtime.rerun_prepare._reset_step_checklist", exploding_reset) + + with pytest.raises(OSError, match="checklist write failed"): + api.flow_run_step( + FlowRunStepRequest( + workspace_id=workspace_id, + step="Floorplan", + rerun=True, + invalidate_dependents=True, + ) + ) + + # The failing step was already cleaned (its artifacts are gone), so it + # keeps the persisted Unstart; the untouched steps roll back. + steps = session.workspace.flow.data["steps"] + assert steps[0] == original_snapshots[0] # Synthesis untouched + assert steps[1]["state"] == "Unstart" # Floorplan cleaned before the failure + assert steps[2] == original_snapshots[2] # route rolled back + + +def test_flow_run_step_final_save_failure_leaves_no_success_record(monkeypatch, tmp_path): + """In-process execution path: a failed final save must not leave the + session record at Success, and the next non-rerun call must not skip the + step.""" + from chipcompiler.data import EccStep + from chipcompiler.engine.flow import EngineFlow + from chipcompiler.utility import json_read + + _capture, ws = _install_runtime_mocks(monkeypatch, tmp_path) + + class FinalSaveFailsOnceFlow(EngineFlow): + # Class-level so the flow rebuilt for the next call shares the budget. + failures_left = 1 + + def __init__(self, workspace, engine_db=None): + super().__init__(workspace, engine_db) + self.engine_db = SimpleNamespace(has_init=lambda: True, engine=None) + + def save(self): + if type(self).failures_left > 0 and any( + step.get("state") == "Success" for step in self.workspace.flow.data.get("steps", []) + ): + type(self).failures_left -= 1 + return False + return super().save() + + FinalSaveFailsOnceFlow.failures_left = 1 + monkeypatch.setattr("chipcompiler.engine.EngineFlow", FinalSaveFailsOnceFlow) + monkeypatch.setattr( + "chipcompiler.tools.create_step", + lambda workspace, step, eda, **kwargs: EccStep(name=step, tool=eda), + ) + run_calls = [] + monkeypatch.setattr( + "chipcompiler.tools.run_step", lambda **kwargs: run_calls.append(kwargs) or True + ) + monkeypatch.setattr("chipcompiler.tools.save_layout_image", lambda **kwargs: True) + monkeypatch.setattr(EngineFlow, "check_step_result", lambda self, workspace_step: True) + markers = [] + monkeypatch.setattr( + "chipcompiler.runtime.log_stream.emit_step_marker", + lambda event, *, step, tool: markers.append(event), + ) + + api = WorkspaceRuntimeApi() + workspace_id = api.open_workspace(WorkspaceOpenRequest(directory=str(ws)))["workspaceId"] + session = api.sessions.get_session(workspace_id) + session.workspace.logger = SimpleNamespace( + info=lambda *a, **k: None, + error=lambda *a, **k: None, + warning=lambda *a, **k: None, + exception=lambda *a, **k: None, + log_section=lambda *a, **k: None, + ) + session.workspace.pdk = SimpleNamespace(sdc=None) + + with pytest.raises(RuntimeApiError, match="failed with state Incomplete"): + api.flow_run_step(FlowRunStepRequest(workspace_id=workspace_id, step="Synthesis")) + + # The failed final save downgraded the live record; disk only ever saw + # the Ongoing state, and no end marker fired. + record = session.workspace.flow.data["steps"][0] + assert record["state"] == StateEnum.Imcomplete.value + flow_path = session.workspace.flow.path + assert json_read(flow_path)["steps"][0]["state"] == "Ongoing" + assert markers == ["begin"] + + # The next non-rerun call does not skip the step: it executes again and + # persists Success once the save works. + result = api.flow_run_step(FlowRunStepRequest(workspace_id=workspace_id, step="Synthesis")) + + assert result == {"step": "Synthesis", "state": "Success"} + assert len(run_calls) == 2 + assert json_read(flow_path)["steps"][0]["state"] == "Success" + + def test_flow_run_step_rerun_rejects_an_open_layout_edit(monkeypatch, tmp_path): _capture, ws = _install_runtime_mocks(monkeypatch, tmp_path) api = WorkspaceRuntimeApi() diff --git a/test/test_engine_flow.py b/test/test_engine_flow.py index 3300738f..d816d35a 100644 --- a/test/test_engine_flow.py +++ b/test/test_engine_flow.py @@ -4,7 +4,7 @@ import pytest -import chipcompiler.engine.flow as flow_module +import chipcompiler.engine.runner as runner_module from chipcompiler import tools from chipcompiler.data import ( EccFeature, @@ -17,6 +17,7 @@ YosysOutput, YosysStep, ) +from chipcompiler.data.workspace import Flow from chipcompiler.engine.flow import EngineFlow @@ -30,10 +31,8 @@ def test_engine_flow_persists_run_facts_before_refreshing_qor_analysis( monkeypatch, tmp_path, ): - workspace = Workspace() - workspace.flow.data = { - "steps": [{"name": "route", "tool": "ecc", "state": "Unstart"}], - } + (tmp_path / "home").mkdir(exist_ok=True) + workspace = Workspace(directory=tmp_path, flow=Flow(path=tmp_path / "home" / "flow.json")) step_feature = tmp_path / "feature" / "route.step.json" sdc_path = tmp_path / "gcd.sdc" sdc_contents = "create_clock -name clk -period 2 [get_ports clk]\n" @@ -48,6 +47,9 @@ def test_engine_flow_persists_run_facts_before_refreshing_qor_analysis( feature=EccFeature(step=step_feature), ) engine_flow = EngineFlow(workspace) + engine_flow.workspace.flow.data = { + "steps": [{"name": "route", "tool": "ecc", "state": "Unstart"}], + } engine_flow.workspace_steps = [workspace_step] engine_flow.engine_db = SimpleNamespace(engine=None) refreshed = [] @@ -90,12 +92,238 @@ def test_engine_flow_does_not_delay_short_step_before_return(monkeypatch, tmp_pa sleep_calls = [] monkeypatch.setattr(tools, "run_step", lambda **_kwargs: False) - monkeypatch.setattr(flow_module.time, "sleep", sleep_calls.append) + monkeypatch.setattr(runner_module.time, "sleep", sleep_calls.append) assert engine_flow.run_step(workspace_step) is StateEnum.Imcomplete assert sleep_calls == [] +def test_end_marker_follows_step_writes_and_precedes_completion(monkeypatch, tmp_path): + """The end marker fires after all step-scoped writes and before completion notify.""" + import chipcompiler.runtime.log_stream as log_stream_module + + (tmp_path / "home").mkdir(exist_ok=True) + workspace = Workspace(directory=tmp_path, flow=Flow(path=tmp_path / "home" / "flow.json")) + workspace_step = EccStep( + name="route", + directory=tmp_path, + tool="ecc", + feature=EccFeature(step=tmp_path / "route.feature.json"), + ) + engine_flow = EngineFlow(workspace) + engine_flow.workspace.flow.data = { + "steps": [{"name": "route", "tool": "ecc", "state": "Unstart"}], + } + engine_flow.workspace_steps = [workspace_step] + engine_flow.engine_db = SimpleNamespace(engine=None) + + events: list[tuple[str, object]] = [] + + monkeypatch.setattr(tools, "run_step", lambda **_kwargs: True) + monkeypatch.setattr(engine_flow, "check_step_result", lambda **_kwargs: True) + monkeypatch.setattr( + tools, + "build_step_metrics", + lambda **_kwargs: events.append(("qor", None)) or {}, + ) + monkeypatch.setattr( + tools, + "save_layout_image", + lambda **_kwargs: events.append(("layout", None)), + ) + + original_set_state = engine_flow.set_state + + def recording_set_state(**kwargs): + events.append(("set_state", kwargs.get("state"))) + return original_set_state(**kwargs) + + monkeypatch.setattr(engine_flow, "set_state", recording_set_state) + monkeypatch.setattr( + engine_flow, + "clear_db_engine_after_step", + lambda step, state: events.append(("db_cleanup", state)), + ) + monkeypatch.setattr( + log_stream_module, + "emit_step_marker", + lambda event, *, step, tool: events.append(("marker", event)), + ) + + class CompletionObserver: + def on_step_completed(self, step, state): + # The end marker must already have fired when completion is notified. + assert ("marker", "end") in events + events.append(("observer", "completed")) + + result = engine_flow.run_step(workspace_step, observer=CompletionObserver()) + + assert result == StateEnum.Success + end_index = events.index(("marker", "end")) + assert events.index(("set_state", StateEnum.Success)) < end_index + assert events.index(("qor", None)) < end_index + assert events.index(("layout", None)) < end_index + assert events.index(("db_cleanup", StateEnum.Success)) < end_index + assert end_index < events.index(("observer", "completed")) + + +def test_end_marker_suppressed_when_final_state_persistence_fails(monkeypatch, tmp_path): + """A failed final save downgrades the step and suppresses the end marker. + + Uses a real Flow so the failing save is the one that would have persisted + the record: the Ongoing save succeeds, the one final save fails, and the + canonical record must end Imcomplete in memory and non-Success on disk. + """ + import chipcompiler.runtime.log_stream as log_stream_module + from chipcompiler.utility import json_read + + (tmp_path / "home").mkdir(exist_ok=True) + flow_path = tmp_path / "home" / "flow.json" + workspace = Workspace(directory=tmp_path, flow=Flow(path=flow_path)) + workspace_step = EccStep(name="route", directory=tmp_path, tool="ecc") + engine_flow = EngineFlow(workspace) + engine_flow.workspace.flow.data = { + "steps": [{"name": "route", "tool": "ecc", "state": "Unstart"}], + } + engine_flow.workspace_steps = [workspace_step] + engine_flow.engine_db = SimpleNamespace(engine=None) + + events = [] + monkeypatch.setattr(tools, "run_step", lambda **_kwargs: True) + monkeypatch.setattr(engine_flow, "check_step_result", lambda **_kwargs: True) + monkeypatch.setattr( + log_stream_module, + "emit_step_marker", + lambda event, *, step, tool: events.append(("marker", event)), + ) + + real_save = engine_flow.save + save_calls = [] + + def save_failing_on_final(): + save_calls.append(len(save_calls) + 1) + if len(save_calls) == 1: + return real_save() # the Ongoing save persists + return False # the one final save fails + + monkeypatch.setattr(engine_flow, "save", save_failing_on_final) + + completed_states = [] + + class CompletionObserver: + def on_step_completed(self, step, state): + completed_states.append(state) + + result = engine_flow.run_step(workspace_step, observer=CompletionObserver()) + + assert save_calls == [1, 2] + assert ("marker", "begin") in events + assert ("marker", "end") not in events + assert result == StateEnum.Imcomplete + assert completed_states == [StateEnum.Imcomplete] + # The canonical record is downgraded in memory and never reached disk as + # Success: the only persisted state is the Ongoing from the first save. + record = engine_flow.get_step("route", "ecc") + assert record["state"] == StateEnum.Imcomplete.value + assert json_read(flow_path)["steps"][0]["state"] == StateEnum.Ongoing.value + + +def test_direct_run_step_archive_failure_downgrades(monkeypatch, tmp_path): + """A bare run_step (no run_steps wrapper) with a broken archive path + returns Imcomplete and downgrades the record after the reader drains.""" + workspace = Workspace( + directory=tmp_path, + flow=Flow(path=tmp_path / "home" / "flow.json"), + ) + engine_flow = EngineFlow(workspace) + engine_flow.workspace.flow.data = { + "steps": [{"name": "route", "tool": "ecc", "state": "Unstart"}] + } + step_dir = tmp_path / "route_ecc" + step_dir.mkdir(parents=True) + (step_dir / "log").write_text("regular file") + workspace_step = EccStep(name="route", directory=step_dir, tool="ecc") + engine_flow.workspace_steps = [workspace_step] + engine_flow.engine_db = SimpleNamespace(engine=None) + + monkeypatch.setattr(tools, "run_step", lambda **_kwargs: True) + monkeypatch.setattr(engine_flow, "check_step_result", lambda **_kwargs: True) + + result = engine_flow.run_step(workspace_step) + + assert result == StateEnum.Imcomplete + record = engine_flow.get_step("route", "ecc") + assert record["state"] == StateEnum.Imcomplete.value + + +def test_run_steps_archive_failure_returns_false_and_downgrades(monkeypatch, tmp_path): + """A direct run_steps with a broken archive path must report failure and + downgrade the record instead of returning True over a missing log.""" + import os + + from chipcompiler.runtime.log_stream import emit_step_marker + + workspace = Workspace( + directory=tmp_path, + flow=Flow(path=tmp_path / "home" / "flow.json"), + ) + engine_flow = EngineFlow(workspace) + engine_flow.workspace.flow.data = { + "steps": [{"name": "route", "tool": "ecc", "state": "Unstart"}] + } + step_dir = tmp_path / "route_ecc" + workspace_step = EccStep(name="route", directory=step_dir, tool="ecc") + engine_flow.workspace_steps = [workspace_step] + engine_flow.engine_db = SimpleNamespace(engine=None) + + def run_step_with_markers(ws_step, *, rerun=False, observer=None): + emit_step_marker("begin", step=ws_step.name, tool=ws_step.tool) + os.write(2, b"bytes\n") + emit_step_marker("end", step=ws_step.name, tool=ws_step.tool) + engine_flow.set_state(ws_step.name, ws_step.tool, StateEnum.Success) + return StateEnum.Success + + monkeypatch.setattr(engine_flow, "run_step", run_step_with_markers) + monkeypatch.setattr(engine_flow, "init_db_engine", lambda: True) + + # The step's log directory is a regular file: the archive cannot open. + step_dir.mkdir(parents=True) + (step_dir / "log").write_text("regular file") + + assert engine_flow.run_steps() is False + record = engine_flow.get_step("route", "ecc") + assert record["state"] == StateEnum.Imcomplete.value + + +def test_begin_marker_failure_downgrades_ongoing(monkeypatch, tmp_path): + """If the begin marker cannot reach fd 2, no reader ever sees the step: + downgrade the persisted Ongoing instead of leaving an unfindable record.""" + import chipcompiler.runtime.log_stream as log_stream_module + + (tmp_path / "home").mkdir(exist_ok=True) + workspace = Workspace(directory=tmp_path, flow=Flow(path=tmp_path / "home" / "flow.json")) + engine_flow = EngineFlow(workspace) + engine_flow.workspace.flow.data = { + "steps": [{"name": "route", "tool": "ecc", "state": "Unstart"}] + } + workspace_step = EccStep(name="route", directory=tmp_path, tool="ecc") + engine_flow.workspace_steps = [workspace_step] + engine_flow.engine_db = SimpleNamespace(engine=None) + + def broken_emit(event, *, step, tool): + raise OSError("fd 2 closed") + + monkeypatch.setattr(log_stream_module, "emit_step_marker", broken_emit) + + import pytest as _pytest + + with _pytest.raises(OSError, match="fd 2 closed"): + engine_flow.run_step(workspace_step) + + record = engine_flow.get_step("route", "ecc") + assert record["state"] == StateEnum.Imcomplete.value + + def test_check_step_result_synthesis_uses_common_verilog(tmp_path): verilog = tmp_path / "gcd.v" verilog.write_text("module gcd; endmodule\n") @@ -193,6 +421,59 @@ def fake_create_step(workspace, step, eda, **kwargs): assert sta_step.output.spef is rcx_output.spef # same object, per legacy contract +def test_executable_steps_filter_chains_success_predecessor(monkeypatch, tmp_path): + # create_step_workspaces(executable_steps=...) builds non-executing steps + # without a dependency check, so a Success predecessor whose tool is missing + # still chains its outputs to the executing successor and marks nothing + # Incomplete. + workspace = Workspace( + directory=tmp_path, + flow=Flow(path=tmp_path / "home" / "flow.json"), + ) + flow = EngineFlow(workspace) + # EngineFlow construction loads (and resets) flow.data; set steps after. + flow.workspace.flow.data = { + "steps": [ + {"name": "syn", "tool": "missing-tool", "state": StateEnum.Success.value}, + {"name": "floorplan", "tool": "ecc", "state": StateEnum.Unstart.value}, + ] + } + + predecessor_output = EccOutput( + def_=tmp_path / "syn.def", + verilog=tmp_path / "syn.v", + db=tmp_path / "syn.db", + ) + prebuilt = { + "syn": EccStep(name="syn", tool="missing-tool", output=predecessor_output), + "floorplan": EccStep(name="floorplan", tool="ecc"), + } + calls = [] + + def fake_create_step(workspace, step, eda, *, check_dependency, **kwargs): + calls.append({"step": step, "check_dependency": check_dependency, "inputs": kwargs}) + # Mirror the load_eda_module contract: a missing tool fails the build + # only when the dependency check actually runs. + if check_dependency and eda == "missing-tool": + return None + return prebuilt[step] + + monkeypatch.setattr(tools, "create_step", fake_create_step) + + flow.create_step_workspaces(executable_steps={"floorplan"}) + + assert [call["check_dependency"] for call in calls] == [False, True] + successor_inputs = calls[1]["inputs"] + assert successor_inputs["input_def"] == predecessor_output.def_ + assert successor_inputs["input_verilog"] == predecessor_output.verilog + assert successor_inputs["input_db"] == predecessor_output.db + assert [step.name for step in flow.workspace_steps] == ["syn", "floorplan"] + assert all( + step.get("state") != StateEnum.Imcomplete.value + for step in flow.workspace.flow.data["steps"] + ) + + # --- Phase 2: Silent failure regression tests --- @@ -249,11 +530,12 @@ def raise_on_run(**_kwargs): assert state == StateEnum.Imcomplete def test_no_exception_uses_file_check(self, monkeypatch, tmp_path): - workspace = Workspace() - workspace.flow.data = { + (tmp_path / "home").mkdir(exist_ok=True) + workspace = Workspace(directory=tmp_path, flow=Flow(path=tmp_path / "home" / "flow.json")) + engine_flow = EngineFlow(workspace) + engine_flow.workspace.flow.data = { "steps": [{"name": "route", "tool": "ecc", "state": "Unstart"}], } - engine_flow = EngineFlow(workspace) workspace_step = EccStep(name="route", directory=tmp_path, tool="ecc") engine_flow.workspace_steps = [workspace_step] engine_flow.engine_db = SimpleNamespace(engine=None) diff --git a/test/test_engine_rerun.py b/test/test_engine_rerun.py index bf78c4e7..298bc2ee 100644 --- a/test/test_engine_rerun.py +++ b/test/test_engine_rerun.py @@ -128,6 +128,135 @@ def test_reexecutes_suffix_and_clears_only_executed_outputs(self, monkeypatch, t assert not stale_place.exists() assert not stale_cts.exists() + def test_direct_run_self_archives_step_bytes(self, monkeypatch, tmp_path, capfd): + """In-process rerun routes fd 1/2 through the archiver: step bytes land + in the step log and markers never reach the caller's terminal.""" + import os + + from chipcompiler.runtime.log_stream import emit_step_marker + + flow = _make_run_flow(tmp_path, [("place", "Success"), ("CTS", "Unstart")]) + _write_output(flow, "place") + + def run_step_with_bytes(workspace_step, *, rerun=False): + emit_step_marker("begin", step=workspace_step.name, tool=workspace_step.tool) + os.write(2, f"{workspace_step.name} bytes\n".encode()) + emit_step_marker("end", step=workspace_step.name, tool=workspace_step.tool) + flow.set_state(workspace_step.name, workspace_step.tool, StateEnum.Success) + return StateEnum.Success + + monkeypatch.setattr(flow, "run_step", run_step_with_bytes) + monkeypatch.setattr(flow, "init_db_engine_for_step", lambda step: True) + + result = rerun.run_from(flow, "place") + + assert result.ok + archive = tmp_path / "place_ecc" / "log" / "place.log" + assert archive.read_bytes() == b"place bytes\n" + assert "ECC-STEP" not in capfd.readouterr().err + + def test_archive_failure_fails_and_downgrades_the_record(self, monkeypatch, tmp_path, capfd): + """An in-process archive failure must not leave ok=True over a Success + record with a missing log.""" + import os + + from chipcompiler.runtime.log_stream import emit_step_marker + + flow = _make_run_flow(tmp_path, [("place", "Success")]) + _write_output(flow, "place") + + def run_step_with_markers(workspace_step, *, rerun=False): + emit_step_marker("begin", step=workspace_step.name, tool=workspace_step.tool) + os.write(2, b"bytes\n") + emit_step_marker("end", step=workspace_step.name, tool=workspace_step.tool) + flow.set_state(workspace_step.name, workspace_step.tool, StateEnum.Success) + return StateEnum.Success + + monkeypatch.setattr(flow, "run_step", run_step_with_markers) + monkeypatch.setattr(flow, "init_db_engine_for_step", lambda step: True) + + # Make the archive path unopenable: a regular file where the step's + # log directory must be created (the output dir stays intact). + (tmp_path / "place_ecc" / "log").write_text("regular file") + + result = rerun.run_from(flow, "place") + + assert result.ok is False + assert result.failed == "place" + assert _flow_states(flow) == [StateEnum.Imcomplete.value] + assert "ECC-STEP" not in capfd.readouterr().err + + def test_archive_downgrade_save_failure_does_not_pretend(self, monkeypatch, tmp_path, capfd): + """When the downgrade cannot persist, the disk record honestly keeps + Success while the operation reports failure — no fake repair.""" + import os + + from chipcompiler.runtime.log_stream import emit_step_marker + + flow = _make_run_flow(tmp_path, [("place", "Success")]) + _write_output(flow, "place") + + def run_step_with_markers(workspace_step, *, rerun=False): + emit_step_marker("begin", step=workspace_step.name, tool=workspace_step.tool) + os.write(2, b"bytes\n") + emit_step_marker("end", step=workspace_step.name, tool=workspace_step.tool) + flow.set_state(workspace_step.name, workspace_step.tool, StateEnum.Success) + return StateEnum.Success + + monkeypatch.setattr(flow, "run_step", run_step_with_markers) + monkeypatch.setattr(flow, "init_db_engine_for_step", lambda step: True) + + # The archive cannot open (regular file at the log path), and the + # downgrade save fails too — but the invalidation save must succeed. + (tmp_path / "place_ecc" / "log").write_text("regular file") + real_save = flow.save + + def save_fails_on_downgrade(): + states = [s.get("state") for s in flow.workspace.flow.data.get("steps", [])] + if StateEnum.Imcomplete.value in states: + return False + return real_save() + + monkeypatch.setattr(flow, "save", save_fails_on_downgrade) + + result = rerun.run_from(flow, "place") + + assert result.ok is False + assert result.failed == "place" + # In memory the downgrade happened; on disk the record honestly keeps + # Success (the failed save is logged, not hidden). + assert _flow_states(flow) == [StateEnum.Imcomplete.value] + persisted = json.loads((tmp_path / "home" / "flow.json").read_text()) + assert persisted["steps"][0]["state"] == "Success" + + def test_step_exception_reconciles_archive_before_propagating( + self, monkeypatch, tmp_path, capfd + ): + """A run_step that raises after its begin marker still reconciles the + reader state before the exception propagates.""" + import os + + from chipcompiler.runtime.log_stream import emit_step_marker + + flow = _make_run_flow(tmp_path, [("place", "Success")]) + _write_output(flow, "place") + + def raising_run_step(workspace_step, *, rerun=False): + emit_step_marker("begin", step=workspace_step.name, tool=workspace_step.tool) + os.write(2, b"partial output\n") + raise RuntimeError("post-processing blew up") + + monkeypatch.setattr(flow, "run_step", raising_run_step) + monkeypatch.setattr(flow, "init_db_engine_for_step", lambda step: True) + + with pytest.raises(RuntimeError, match="post-processing"): + rerun.run_from(flow, "place") + + # The unmatched begin downgraded the record instead of leaving a + # stale Success over a partial archive. + assert _flow_states(flow) == [StateEnum.Imcomplete.value] + assert "ECC-STEP" not in capfd.readouterr().err + def test_failure_stops_suffix_and_keeps_downstream_output(self, monkeypatch, tmp_path): flow = _make_run_flow( tmp_path, diff --git a/test/tools/ecc/test_runner.py b/test/tools/ecc/test_runner.py index 3e399a0c..6e002396 100644 --- a/test/tools/ecc/test_runner.py +++ b/test/tools/ecc/test_runner.py @@ -1,6 +1,8 @@ import json from pathlib import Path +import pytest + from chipcompiler.data import ( PDK, EccData, @@ -566,9 +568,10 @@ def test_rcx_checklist_uses_top_module_for_spef_design_token(tmp_path): assert checklist.check_spef_file(str(spef)) is True -def test_save_data_writes_geometry_snapshot_for_physical_step(tmp_path): +@pytest.mark.parametrize("step_name", (StepEnum.ROUTING.value, StepEnum.LVS.value)) +def test_save_data_writes_geometry_snapshot_for_physical_step(tmp_path, step_name): workspace = Workspace(directory=tmp_path, design=OriginDesign(name="gcd", top_module="gcd")) - step = build_step(workspace, StepEnum.ROUTING.value, None, None) + step = build_step(workspace, step_name, None, None) module = SnapshotSaveEccModule(write_snapshot=True) assert ecc_runner.save_data(workspace, step, module, feature_step=False) is True @@ -626,11 +629,18 @@ def test_save_data_fails_when_geometry_snapshot_cannot_be_written(tmp_path): ) -def test_engine_flow_requires_geometry_manifest_for_physical_steps(tmp_path): +@pytest.mark.parametrize("step_name", (StepEnum.ROUTING.value, StepEnum.LVS.value)) +def test_engine_flow_requires_geometry_manifest_for_physical_steps(tmp_path, step_name): workspace = Workspace(directory=tmp_path, design=OriginDesign(name="gcd", top_module="gcd")) - step = build_step(workspace, StepEnum.ROUTING.value, None, None) + step = build_step(workspace, step_name, None, None) build_step_space(step) - for output_path in (step.output.def_, step.output.verilog, step.output.gds): + for output_path in ( + step.output.def_, + step.output.verilog, + step.output.gds, + step.report.step, + step.feature.step, + ): assert output_path is not None output_path.write_text("", encoding="utf-8") assert step.output.geometry_manifest is not None diff --git a/test/tools/ecc_sizer/test_runner.py b/test/tools/ecc_sizer/test_runner.py index 1823b478..cea3a4e2 100644 --- a/test/tools/ecc_sizer/test_runner.py +++ b/test/tools/ecc_sizer/test_runner.py @@ -28,7 +28,7 @@ def __getattribute__(self, name): calls = [] - def fake_run(command, cwd, stdout, stderr, check): + def fake_run(command, cwd, stderr, check): calls.append((command, cwd, stderr, check)) os.makedirs(os.path.dirname(str(step.output.def_)), exist_ok=True) with open(str(step.output.def_), "w", encoding="utf-8") as file: @@ -118,7 +118,7 @@ def test_sizer_runner_marks_subflow_incomplete_when_outputs_are_missing( monkeypatch.setattr( subprocess, "run", - lambda command, cwd, stdout, stderr, check: SimpleNamespace(returncode=0), + lambda command, cwd, stderr, check: SimpleNamespace(returncode=0), ) assert sizer_runner.run_step(workspace, step) == StateEnum.Imcomplete @@ -197,25 +197,11 @@ def test_timing_opt_step_result_does_not_require_gds(tmp_path): def test_engine_flow_clears_cached_db_after_successful_sizer_step(tmp_path, monkeypatch): import chipcompiler.tools as tools_api - from chipcompiler.engine import flow as flow_module + from chipcompiler.engine import runner as flow_module from chipcompiler.engine.flow import EngineFlow workspace = _workspace(tmp_path) workspace.flow.path = tmp_path / "flow.json" - workspace.flow.data = { - "steps": [ - { - "name": StepEnum.TIMING_OPT.value, - "tool": "sizer", - "state": StateEnum.Unstart.value, - }, - { - "name": StepEnum.LEGALIZATION.value, - "tool": "ecc", - "state": StateEnum.Unstart.value, - }, - ] - } sizer_step = EccStep( name=StepEnum.TIMING_OPT.value, @@ -246,6 +232,21 @@ def close(self): pre_sizer_db_closed.append(True) engine_flow = EngineFlow(workspace) + engine_flow.workspace.flow.data = { + "steps": [ + { + "name": StepEnum.TIMING_OPT.value, + "tool": "sizer", + "state": StateEnum.Unstart.value, + }, + { + "name": StepEnum.LEGALIZATION.value, + "tool": "ecc", + "state": StateEnum.Unstart.value, + }, + ] + } + engine_flow.save() engine_flow.workspace_steps = [sizer_step, post_sizer_step] monkeypatch.setattr(engine_flow, "engine_db", CloseableDb()) diff --git a/test/tools/yosys/test_runner.py b/test/tools/yosys/test_runner.py index 77e163e2..34078ae7 100644 --- a/test/tools/yosys/test_runner.py +++ b/test/tools/yosys/test_runner.py @@ -48,7 +48,7 @@ def __init__(self, workspace, workspace_step): def check(self): return None - def fake_check_slang_support(yosys_cmd, cwd_dir, yosys_env, log_file): + def fake_check_slang_support(yosys_cmd, cwd_dir, yosys_env): check_calls.append( { "yosys_cmd": list(yosys_cmd), @@ -58,7 +58,7 @@ def fake_check_slang_support(yosys_cmd, cwd_dir, yosys_env, log_file): ) return True - def fake_run(cmd, cwd, env, stdout, stderr): + def fake_run(cmd, cwd, env, stderr): run_calls.append( { "cmd": list(cmd), @@ -118,7 +118,7 @@ def __init__(self, workspace, workspace_step): def check(self): return None - def fake_run(cmd, cwd, env, stdout, stderr): + def fake_run(cmd, cwd, env, stderr): output_file.write_text("module top(); endmodule\n") return SimpleNamespace(returncode=0) @@ -147,11 +147,10 @@ def __init__(self, workspace, workspace_step): def update_step(self, step_name, state, runtime="", memory=0, info=None): updates.append((step_name, state)) - def fake_check_slang_support(yosys_cmd, cwd_dir, yosys_env, log_file): - log_file.write("Error: yosys slang frontend check failed.\n") + def fake_check_slang_support(yosys_cmd, cwd_dir, yosys_env): return False - def fake_run(cmd, cwd, env, stdout, stderr): + def fake_run(cmd, cwd, env, stderr): run_calls.append(list(cmd)) raise AssertionError("Synthesis should not run when slang check fails") @@ -166,7 +165,6 @@ def fake_run(cmd, cwd, env, stdout, stderr): assert result is False assert run_calls == [] assert ("run yosys", StateEnum.Invalid) in updates - assert "slang frontend check failed" in log_file.read_text() def test_run_step_skips_slang_check_for_native_verilog(tmp_path, monkeypatch): @@ -191,7 +189,7 @@ def check(self): def fail_slang_check(*args, **kwargs): raise AssertionError("native Verilog must not probe the Slang frontend") - def fake_run(cmd, cwd, env, stdout, stderr): + def fake_run(cmd, cwd, env, stderr): output_file.write_text("module top(); endmodule\n") return SimpleNamespace(returncode=0) @@ -225,4 +223,3 @@ def update_step(self, step_name, state, runtime="", memory=0, info=None): assert result is False assert ("run yosys", StateEnum.Invalid) in updates - assert "yosys is not available" in log_file.read_text() diff --git a/test/tools/yosys/test_utility.py b/test/tools/yosys/test_utility.py index 3ff7c327..d5c119ad 100644 --- a/test/tools/yosys/test_utility.py +++ b/test/tools/yosys/test_utility.py @@ -66,11 +66,10 @@ def test_get_yosys_runtime_builds_local_env_without_mutating_global_env(tmp_path assert before == after -def test_check_slang_support_accepts_builtin_frontend(tmp_path, monkeypatch): - log_path = tmp_path / "check.log" +def test_check_slang_support_accepts_builtin_frontend(monkeypatch): calls = [] - def fake_run(cmd, cwd, env, stdout, stderr, timeout): + def fake_run(cmd, cwd, env, stderr, timeout, stdout=None): calls.append( { "cmd": list(cmd), @@ -82,13 +81,11 @@ def fake_run(cmd, cwd, env, stdout, stderr, timeout): return SimpleNamespace(returncode=0, stdout=b"read_slang [options] [filename]") monkeypatch.setattr(utility.subprocess, "run", fake_run) - with open(log_path, "w") as log_file: - ok = utility.check_slang_support( - yosys_cmd=["yosys"], - cwd_dir="/tmp/test", - yosys_env={"PATH": "/tmp/bin"}, - log_file=log_file, - ) + ok = utility.check_slang_support( + yosys_cmd=["yosys"], + cwd_dir="/tmp/test", + yosys_env={"PATH": "/tmp/bin"}, + ) assert ok is True assert len(calls) == 1 @@ -98,24 +95,21 @@ def fake_run(cmd, cwd, env, stdout, stderr, timeout): assert calls[0]["timeout"] == 60 -def test_check_slang_support_falls_back_to_plugin(tmp_path, monkeypatch): - log_path = tmp_path / "check.log" +def test_check_slang_support_falls_back_to_plugin(monkeypatch): calls = [] - def fake_run(cmd, cwd, env, stdout, stderr, timeout): + def fake_run(cmd, cwd, env, stderr, timeout, stdout=None): calls.append(list(cmd)) if "help read_slang" in cmd: return SimpleNamespace(returncode=0, stdout=b"No such command or cell type: read_slang") return SimpleNamespace(returncode=0, stdout=b"") monkeypatch.setattr(utility.subprocess, "run", fake_run) - with open(log_path, "w") as log_file: - ok = utility.check_slang_support( - yosys_cmd=["yosys"], - cwd_dir="/tmp/test", - yosys_env={"PATH": "/tmp/bin"}, - log_file=log_file, - ) + ok = utility.check_slang_support( + yosys_cmd=["yosys"], + cwd_dir="/tmp/test", + yosys_env={"PATH": "/tmp/bin"}, + ) assert ok is True assert calls == [ @@ -124,22 +118,18 @@ def fake_run(cmd, cwd, env, stdout, stderr, timeout): ] -def test_check_slang_support_writes_error_on_failure(tmp_path, monkeypatch): - log_path = tmp_path / "check_fail.log" - - def fake_run(cmd, cwd, env, stdout, stderr, timeout): +def test_check_slang_support_prints_error_on_failure(monkeypatch, capsys): + def fake_run(cmd, cwd, env, stderr, timeout, stdout=None): if "help read_slang" in cmd: return SimpleNamespace(returncode=0, stdout=b"No such command or cell type: read_slang") return SimpleNamespace(returncode=1, stdout=b"") monkeypatch.setattr(utility.subprocess, "run", fake_run) - with open(log_path, "w") as log_file: - ok = utility.check_slang_support( - yosys_cmd=["yosys"], - cwd_dir="/tmp/test", - yosys_env={"PATH": "/tmp/bin"}, - log_file=log_file, - ) + ok = utility.check_slang_support( + yosys_cmd=["yosys"], + cwd_dir="/tmp/test", + yosys_env={"PATH": "/tmp/bin"}, + ) assert ok is False - assert "slang frontend check failed" in log_path.read_text() + assert "slang frontend check failed" in capsys.readouterr().out diff --git a/test/utility/test_json.py b/test/utility/test_json.py index f79cf73c..26455ce0 100644 --- a/test/utility/test_json.py +++ b/test/utility/test_json.py @@ -191,8 +191,9 @@ def test_set_state_updates_in_memory_when_save_fails(self, tmp_path, monkeypatch monkeypatch.setattr("chipcompiler.utility.json_write", lambda *a, **kw: False) result = flow.set_state("SYNTHESIS", "yosys", StateEnum.Success) - assert result is True - # In-memory state is updated + # The record is mutated in memory, but the result reports the actual + # persistence status so callers can distrust an unpersisted state. + assert result is False assert workspace.flow.data["steps"][0]["state"] == StateEnum.Success.value def test_stale_file_causes_rerun_on_resume(self, tmp_path, monkeypatch):