fix(engine): stop a failed child-agent call from killing the whole run

docs/RUNTIME_HOOKS.md

@@ -109,8 +109,9 @@ hooks:
   The id is resolved through `plugins/plugins.toml` `[hooks.plugins]`.
 - **`ref`** *(advanced / backwards compatible)* — explicit import path to the
   hook callable. Use either `ref` or `plugin` + `method`, never both.
-- **`failure_policy`** *(optional, `fail` | `warn`, default `fail`)* — `fail`
-  aborts the operation (fail-closed); `warn` logs and continues.
+- **`failure_policy`** *(optional, `fail` | `warn`, default `fail` — except
+  `on_tool_error`, which defaults to `warn`)* — `fail` aborts the operation
+  (fail-closed); `warn` logs and continues.
 
 ### Validation
 
@@ -443,7 +444,8 @@ that and rely on env/config-based service credentials for the discovery phase.
 
 ## Error policy
 
-Fail-closed by default — security hooks must not be silently skipped:
+Fail-closed by default, except `on_tool_error` (see below) — security hooks
+must not be silently skipped:
 
 | Point | On hook failure (`failure_policy: fail`) |
 |---|---|
@@ -454,15 +456,16 @@ Fail-closed by default — security hooks must not be silently skipped:
 | `before_tool_call` | the run fails (a policy gate can block the call) |
 | `after_tool_call` | the run fails |
 | `transform_tool_result` | the run fails (use `warn` to pass the original result through) |
-| `on_tool_error` | the run fails |
+| `on_tool_error` | the run fails — but **defaults to `warn`** (see below) |
 | `before_mcp_request` | the MCP request fails |
 | `after_mcp_response` | the MCP request fails |
 | `on_run_error` | logged; **the original run error is preserved** |
 
 Use `failure_policy: warn` for best-effort hooks (e.g. audit) that should log
-and continue instead of aborting. Hook errors are never silently swallowed:
-they are logged with their point and `ref`, and (except `on_engine_stop` and
-`on_run_error`, which are best-effort) raised.
+and continue instead of aborting — `on_tool_error` already defaults to this.
+Hook errors are never silently swallowed: they are logged with their point and
+`ref`, and (except `on_engine_stop` and `on_run_error`, which are best-effort)
+raised.
 
 ---
 

docs/runtime-hooks.mdx

@@ -170,11 +170,11 @@ The YAML never contains an import path, a config block, or a secret — it only
 </ParamField>
 
 <ParamField path="failure_policy" type="string" default="fail">
-  What happens if the hook itself raises.
+  What happens if the hook itself raises. Defaults to `fail` everywhere except `on_tool_error`, which defaults to `warn`.
 
   <Expandable title="possible values">
     <ResponseField name="fail" type="string">
-      **Default (fail-closed).** The operation is aborted and the request fails. Use for anything security-critical, like auth — a broken auth hook must never fail open.
+      **Fail-closed.** The operation is aborted and the request fails. Use for anything security-critical, like auth — a broken auth hook must never fail open.
     </ResponseField>
     <ResponseField name="warn" type="string">
       The error is logged and the operation continues as if the hook had succeeded. Use for best-effort work like audit logging, where a logging failure shouldn't block the user.

examples/config.schema.json

@@ -195,7 +195,8 @@
         "failure_policy": {
           "type": "string",
           "enum": ["fail", "warn"],
-          "default": "fail"
+          "default": "fail",
+          "description": "Defaults to 'fail' (fail-closed) at every hook point except on_tool_error, which defaults to 'warn'."
         }
       },
       "oneOf": [

src/agent_engine/core/spec.py

@@ -2,10 +2,16 @@
 
 from abc import ABC, abstractmethod
 from dataclasses import dataclass, field
+from enum import StrEnum
 
 from agent_engine.core.execution import ExecutionPolicy
 
 
+class FailurePolicy(StrEnum):
+    FAIL = "fail"
+    WARN = "warn"
+
+
 @dataclass(frozen=True)
 class BaseModelConfig:
     provider: str
@@ -117,16 +123,14 @@ class GraphNode:
 class HookSpec:
     """One declared runtime hook: where it runs, what it is, how it behaves.
 
-    ``point`` is a hook lifecycle point (e.g. "before_mcp_request"). A hook is
-    declared either by explicit Python ``ref`` or by managed ``plugin`` +
-    ``method`` resolved through plugins.toml. ``failure_policy`` is "fail"
-    (default, fail-closed) or "warn" (best-effort: log and continue on hook
-    error).
+    ``failure_policy``'s ``FAIL`` default applies only when constructed
+    directly; the YAML parser picks its own default per point instead
+    (``agent_engine.runtime.hooks.models.DEFAULT_FAILURE_POLICY``).
     """
 
     point: str
     ref: str | None = None
-    failure_policy: str = "fail"
+    failure_policy: FailurePolicy = FailurePolicy.FAIL
     plugin: str | None = None
     method: str | None = None
 

src/agent_engine/engine/langgraph/engine.py

@@ -383,6 +383,7 @@ def accumulate_tokens(inp: int, out: int) -> None:
                 run_id=ctx.run_id,
                 system=self._system_name,
                 error=type(exc).__name__,
+                message=exc,
             )
             await self._fail_run(ctx.run_id)
             await hook_manager.run_run_error(ctx, exc)
@@ -645,6 +646,7 @@ async def run_graph() -> None:
                     run_id=ctx.run_id,
                     system=self._system_name,
                     error=type(exc).__name__,
+                    message=exc,
                 )
                 await self._fail_run(ctx.run_id)
                 await hook_manager.run_run_error(ctx, exc)

src/agent_engine/engine/langgraph/nodes.py

@@ -33,14 +33,15 @@
     run_tool_loop,
 )
 from agent_engine.loaders.resolver_loader import ResolverLoader
-from agent_engine.logging_config import log
+from agent_engine.logging_config import log, safe_error
 from agent_engine.runtime.execution import (
     ExecutionLimitExceeded,
     blocked_message,
     current_execution,
     log_limit,
 )
 from agent_engine.runtime.hooks import (
+    HookExecutionError,
     HookManager,
     ToolCallContext,
     ToolRequestContext,
@@ -345,7 +346,7 @@ async def _record_error(
 
         The failure is returned (not raised) so the model can read it and recover.
         """
-        error = str(exc)[:200]
+        error = safe_error(exc)
         used_tools.append(self._usage(call, "failed", error=error))
         self._log_call(logging.WARNING, "tool call failed", call, ms=latency_ms, error=error)
 
@@ -582,8 +583,24 @@ async def invoke(message: str) -> str:
                 # to the LangGraph runtime so the checkpoint is taken — it is
                 # control flow, not a child failure. Never swallow it.
                 raise
+            except HookExecutionError:
+                # A fail-closed hook (e.g. before_tool_call) rejected this child's
+                # own call. That must abort the run like any other fail-closed
+                # hook, not be reported as a recoverable child-agent failure.
+                raise
             except Exception as exc:
-                return f"Agent error: {exc}"
+                log(
+                    logger,
+                    logging.WARNING,
+                    "child agent call failed",
+                    agent=entry.id,
+                    error=type(exc).__name__,
+                    message=exc,
+                )
+                return (
+                    f"Agent '{entry.id}' failed to complete this request. "
+                    "This is an internal error, not a problem with the arguments."
+                )
             finally:
                 current_streams.reset(sink_token)
 
@@ -645,7 +662,28 @@ async def invoke_tool(tc: dict[str, Any]) -> str:
                 except ExecutionLimitExceeded as exc:
                     log_limit(exc)
                     return blocked_message(exc)
-            return cast(str, await tool.ainvoke(tc["args"]))
+            try:
+                return cast(str, await tool.ainvoke(tc["args"]))
+            except GraphInterrupt:
+                raise
+            except HookExecutionError:
+                # A fail-closed hook (e.g. before_tool_call) rejected this call.
+                # That must abort the run like any other fail-closed hook, not be
+                # reported as a recoverable child-agent failure.
+                raise
+            except Exception as exc:
+                log(
+                    logger,
+                    logging.WARNING,
+                    "child agent call failed",
+                    agent=tc["name"],
+                    error=type(exc).__name__,
+                    message=exc,
+                )
+                return (
+                    f"Agent '{tc['name']}' failed to complete this request. "
+                    "This is an internal error, not a problem with the arguments."
+                )
 
         response = await run_tool_loop(bound_model, messages, state, self._node_path, invoke_tool)
         answer = as_text(response.content)

src/agent_engine/logging_config.py

@@ -4,9 +4,31 @@
 import logging
 import os
 
+from pydantic import ValidationError
+
 request_id_var: contextvars.ContextVar[str] = contextvars.ContextVar("request_id", default="")
 
 _NOISY = ("httpx", "httpcore", "anthropic", "openai", "urllib3")
+_ERROR_LIMIT = 200
+
+
+def validation_problems(exc: ValidationError) -> str:
+    """Which fields were rejected and why, without the values that were sent."""
+    return "; ".join(
+        f"{'.'.join(str(part) for part in err['loc']) or 'input'}: {err['msg']}"
+        for err in exc.errors()
+    )
+
+
+def safe_error(exc: BaseException, limit: int = _ERROR_LIMIT) -> str:
+    """An exception rendered for a log field: bounded, single-line, no payload.
+
+    ``str()`` on a pydantic error appends the ``input_value`` it rejected — the
+    caller's own arguments — so validation failures are reduced to their field
+    reasons instead.
+    """
+    detail = validation_problems(exc) if isinstance(exc, ValidationError) else str(exc)
+    return " ".join(detail.split())[:limit]
 
 
 def _kv(key: str, value: object) -> str:
@@ -29,7 +51,11 @@ def format(self, record: logging.LogRecord) -> str:
 
 
 def log(logger: logging.Logger, level: int, event: str, **fields: object) -> None:
-    logger.log(level, event, extra={"fields": fields})
+    """Emit one structured line. Exception values are rendered by ``safe_error``,
+    so no call site can leak a rejected payload into a log field by passing the
+    exception straight through."""
+    safe = {k: safe_error(v) if isinstance(v, BaseException) else v for k, v in fields.items()}
+    logger.log(level, event, extra={"fields": safe})
 
 
 def configure_logging(level: str | None = None) -> None:

src/agent_engine/observability/providers/logging/provider.py

@@ -26,7 +26,7 @@ def on_llm_end(self, response: Any, **kwargs: Any) -> None:
         log(logger, logging.INFO, "llm end", tokens=usage)
 
     def on_llm_error(self, error: BaseException, **kwargs: Any) -> None:
-        log(logger, logging.ERROR, "llm error", error=str(error))
+        log(logger, logging.ERROR, "llm error", error=error)
 
     def on_tool_start(self, serialized: dict[str, Any], input_str: str, **kwargs: Any) -> None:
         log(logger, logging.INFO, "tool start", name=(serialized or {}).get("name", "?"))
@@ -37,7 +37,7 @@ def on_tool_end(self, output: Any, **kwargs: Any) -> None:
         log(logger, logging.DEBUG, "tool output", value=str(output)[:300])
 
     def on_tool_error(self, error: BaseException, **kwargs: Any) -> None:
-        log(logger, logging.WARNING, "tool end", status="error", error=str(error))
+        log(logger, logging.WARNING, "tool end", status="error", error=error)
 
     def on_chain_start(
         self, serialized: dict[str, Any], inputs: dict[str, Any], **kwargs: Any

src/agent_engine/parsers/yaml/parser.py

@@ -14,6 +14,7 @@
     BaseModelConfig,
     BasePromptSet,
     DefaultsConfig,
+    FailurePolicy,
     GraphNode,
     HooksConfig,
     HookSpec,
@@ -30,7 +31,7 @@
 )
 from agent_engine.parsers.errors import ParseError
 from agent_engine.parsers.parser import Parser
-from agent_engine.runtime.hooks.models import HOOK_POINTS
+from agent_engine.runtime.hooks.models import DEFAULT_FAILURE_POLICY, HOOK_POINTS
 
 _SECRET_MARKERS = ("api_key", "apikey", "secret", "token", "password", "private_key")
 _SECRET_KEY_EXEMPTIONS = {"max_tokens"}
@@ -122,7 +123,6 @@ def _validate_unknown_keys(
         )
 
 
-
 def _validate_plugins(plugins: Any, errors: list[ValidationError]) -> None:
     if plugins is None:
         return
@@ -218,7 +218,9 @@ def _build_hooks(raw: Any) -> HooksConfig:
                 HookSpec(
                     point=point,
                     ref=entry.get("ref"),
-                    failure_policy=entry.get("failure_policy", "fail"),
+                    failure_policy=FailurePolicy(
+                        entry.get("failure_policy", DEFAULT_FAILURE_POLICY[point])
+                    ),
                     plugin=entry.get("plugin"),
                     method=entry.get("method"),
                 )
@@ -320,9 +322,10 @@ def _validate_hook_entry(point: str, index: int, entry: Any, errors: list[Valida
                 "Removed field; hook configuration belongs in hook/plugin code",
             )
         )
-    policy = entry.get("failure_policy", "fail")
-    if policy not in ("fail", "warn"):
-        errors.append(ValidationError(f"{base}.failure_policy", "Must be 'fail' or 'warn'"))
+    policy = entry.get("failure_policy", FailurePolicy.FAIL.value)
+    options = [p.value for p in FailurePolicy]
+    if policy not in options:
+        errors.append(ValidationError(f"{base}.failure_policy", f"Must be one of {options}"))
 
 
 def _validate_hooks(hooks: Any, errors: list[ValidationError]) -> None:
@@ -415,7 +418,6 @@ def _validate_mcps(mcps: dict[str, Any], errors: list[ValidationError]) -> None:
         _validate_mcp_tool_tags(mcp_id, raw, errors)
 
 
-
 def _validate_model(
     path: str, raw: Any, errors: list[ValidationError], is_fallback: bool = False
 ) -> None:
@@ -424,10 +426,7 @@ def _validate_model(
     if not isinstance(raw, dict):
         errors.append(ValidationError(path, "Must be a mapping"))
         return
-    _validate_unknown_keys(
-        path, raw, _FALLBACK_MODEL_KEYS if is_fallback else _MODEL_KEYS, errors
-    )
-
+    _validate_unknown_keys(path, raw, _FALLBACK_MODEL_KEYS if is_fallback else _MODEL_KEYS, errors)
 
     provider = raw.get("provider")
     if not isinstance(provider, str) or not provider.strip():
@@ -588,7 +587,6 @@ def _validate(self, data: dict[str, Any]) -> list[ValidationError]:
 
         return errors
 
-
     def _validate_graph(
         self,
         graph: Any,

src/agent_engine/runtime/hooks/errors.py

@@ -27,10 +27,15 @@ def __init__(self, point: str, ref: str, reason: str) -> None:
 
 
 class HookExecutionError(HookError):
-    """A hook raised while running. Wraps the original exception as ``__cause__``."""
+    """A hook raised while running. Wraps the original exception as ``__cause__``.
+
+    The message names only the failing hook's point/ref and the cause's type —
+    never the cause's own message, which may carry payload or credential
+    material the hook was handling. ``str(exc)`` is safe to log anywhere.
+    """
 
     def __init__(self, point: str, ref: str, cause: BaseException) -> None:
         self.point = point
         self.ref = ref
         self.cause = cause
-        super().__init__(f"Hook for '{point}' (ref='{ref}') failed: {cause}")
+        super().__init__(f"Hook for '{point}' (ref='{ref}') failed: {type(cause).__name__}")

src/agent_engine/runtime/hooks/manager.py

@@ -18,7 +18,7 @@
     for best-effort audit hooks).
   * ``transform_tool_result`` fail-> the run fails (use ``failure_policy: warn``
     to pass the original, untransformed result through instead).
-  * ``on_tool_error`` failure     -> the run fails (use ``failure_policy: warn``).
+  * ``on_tool_error`` failure     -> logged and skipped (default ``warn`` here).
   * ``before_mcp_request`` fail   -> the MCP request fails.
   * ``after_mcp_response`` fail   -> the MCP request fails (observe-only payload).
   * ``on_run_error`` failure      -> logged; the original run error is preserved.
@@ -38,6 +38,7 @@
 from pathlib import Path
 from typing import Any
 
+from agent_engine.core.spec import FailurePolicy
 from agent_engine.runtime.hooks.errors import HookExecutionError, HookLoadError
 from agent_engine.runtime.hooks.loader import HookLoader
 from agent_engine.runtime.hooks.models import (
@@ -65,7 +66,7 @@ class LoadedHook:
     point: HookPoint
     ref: str
     func: Any
-    failure_policy: str = "fail"
+    failure_policy: FailurePolicy = FailurePolicy.FAIL
     plugin: str | None = None
     method: str | None = None
     event_mode: bool = False
@@ -354,7 +355,7 @@ async def _invoke(
                 hook.failure_policy,
                 type(exc).__name__,
             )
-            if hook.failure_policy == "warn":
+            if hook.failure_policy == FailurePolicy.WARN:
                 return None
             raise HookExecutionError(hook.point, hook.ref, exc) from exc
         return result