From 06ed288a7760dc772c55817034713c9dd0234080 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 6 Aug 2026 11:09:35 -0700 Subject: [PATCH 1/5] feat(observability): add MCP shadow timeout diagnostics Signed-off-by: Apurv Kumaria --- Dockerfile | 6 +- docs/reference/commands.mdx | 1 + docs/reference/troubleshoot-mcp-servers.mdx | 72 ++++- .../openclaw-2026.7.1-dependency-review.md | 25 +- ...openclaw-managed-transport-diagnostics.mts | 144 ++++++++-- src/lib/onboard/sandbox-create-launch.test.ts | 33 +++ src/lib/onboard/sandbox-create-launch.ts | 17 ++ test/openclaw-dependency-review.test.ts | 9 +- ...anaged-transport-diagnostics-patch.test.ts | 270 +++++++++++++++++- 9 files changed, 533 insertions(+), 44 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7b3a1edf092..d5f59d930e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1036,8 +1036,10 @@ RUN node --experimental-strip-types /usr/local/lib/nemoclaw/patch-openclaw-mcp-r # request fails. OpenClaw 2026.7.1 surfaces only the transport error text, which # does not say whether policy, CONNECT, TLS, the upstream connection, the # request, or response headers failed. The fetch-boundary wrapper is -# failure-only, never retries, never alters the request, and never reads a 2xx -# body, so streaming responses stay behaviorally unchanged. It is inert unless +# failure-only by default, never retries, never alters the request, and never +# reads a 2xx body, so streaming responses stay behaviorally unchanged. Successful request +# timing is silent unless NEMOCLAW_MCP_SHADOW_DIAGNOSTICS=1 is explicitly +# forwarded into an OpenClaw sandbox. The wrapper is inert unless # OPENSHELL_SANDBOX=1. # # Removal criterion: drop when upstream OpenClaw emits phase-classified, diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index c22d4eef9a7..2611ec8b4f8 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -4139,6 +4139,7 @@ OpenClaw-specific onboarding configuration: | `BRAVE_API_KEY` | Brave Search API key | Supplies and implicitly selects Brave Search when no web search provider is set. NemoClaw validates the key and stores it in OpenShell rather than the sandbox. | | `TAVILY_API_KEY` | Tavily Search API key | Supplies and implicitly selects Tavily Search when no provider is set and no Brave key is available. NemoClaw validates the key and stores it in OpenShell rather than the sandbox. | | `NEMOCLAW_AGENT_TIMEOUT` | positive integer (seconds) | Overrides `agents.defaults.timeoutSeconds` in the built OpenClaw config. Raise for slow inference. | +| `NEMOCLAW_MCP_SHADOW_DIAGNOSTICS` | literal `1` to enable | Forwards opt-in successful Streamable HTTP MCP timing diagnostics to a newly created or rebuilt OpenClaw sandbox. It does not change timeouts, retries, requests, or responses. Unset it and rebuild after evidence collection to restore failure-only logging. Other values are ignored. | | `NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS` | positive number of seconds | Sets the post-pairing poll cadence for the in-sandbox OpenClaw auto-pair watcher. Defaults to `5` so late allowlisted CLI and browser scope upgrades are approved before clients time out. Raise only on load-sensitive gateways. | | `NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS` | positive integer | Sets how many fast polls run after the watcher observes a fresh allowlisted scope-upgrade request. Defaults to `5`; set lower only when you need to reduce gateway polling. | | `NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS` | positive number of seconds | Sets the fast-reentry interval after a fresh allowlisted scope-upgrade request. Defaults to `1`. | diff --git a/docs/reference/troubleshoot-mcp-servers.mdx b/docs/reference/troubleshoot-mcp-servers.mdx index 2ca2faca53e..b6e57c42e18 100644 --- a/docs/reference/troubleshoot-mcp-servers.mdx +++ b/docs/reference/troubleshoot-mcp-servers.mdx @@ -3,9 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 title: "Troubleshoot MCP Servers" sidebar-title: "Troubleshoot MCP Servers" -description: "Diagnose managed MCP credential resolution, DNS pin drift, incomplete transactions, capability gaps, policy drift, and lifecycle-lock failures." -description-agent: "Provides symptom-based remediation for NemoClaw-managed MCP servers, including trusted private endpoints. Use when mcp add, status, restart, remove, rebuild, or destroy does not converge." -keywords: ["troubleshoot nemoclaw mcp", "mcp credential resolution", "mcp dns pin drift", "mcp policy drift", "mcp transaction"] +description: "Diagnose managed MCP transport failures, timeout evidence, credential resolution, DNS pin drift, incomplete transactions, capability gaps, policy drift, and lifecycle-lock failures." +description-agent: "Provides symptom-based remediation for NemoClaw-managed MCP servers, including transport diagnostics and trusted private endpoints. Use when MCP requests, add, status, restart, remove, rebuild, or destroy do not converge." +keywords: ["troubleshoot nemoclaw mcp", "mcp transport diagnostics", "mcp timeout evidence", "mcp credential resolution", "mcp dns pin drift", "mcp policy drift", "mcp transaction"] content: type: "troubleshooting" --- @@ -102,12 +102,20 @@ $$nemoclaw logs --tail 100 ```text [nemoclaw] managed_transport_failure [nemoclaw] consumer=mcp +[nemoclaw] mcp_server=gitlab +[nemoclaw] operation=tools/list +[nemoclaw] transport_generation=1 +[nemoclaw] request_sequence=3 [nemoclaw] route=proxy_configured [nemoclaw] proxy=127.0.0.1:3128 [nemoclaw] target=mcp.example.com:443 +[nemoclaw] connection_timeout_ms=30000 +[nemoclaw] request_timeout_ms=60000 +[nemoclaw] catalog_list_timeout_ms=1500 +[nemoclaw] effective_timeout_ms=1500 [nemoclaw] transport_phase=response_headers [nemoclaw] http_status=503 -[nemoclaw] elapsed_ms=1512 +[nemoclaw] elapsed_ms=412 [nemoclaw] server=envoy [nemoclaw] x_envoy_response_flags=UF,URX [nemoclaw] session_present=true @@ -130,11 +138,18 @@ It states which transport step failed: `route=unknown` states that the diagnostic did not observe one. These values report configuration evidence and do not prove whether the failed request used a proxy. When present, `proxy` contains the configured proxy endpoint without credentials. +`mcp_server` contains the configured MCP server name when it passes validation and redaction. +`operation` contains a validated JSON-RPC method, such as `initialize`, `tools/list`, or `tools/call`. +For other request shapes, it contains `rpc/unknown`, `transport/listen`, or `transport/close` when that transport action is known. +The diagnostic never records JSON-RPC parameters, tool names, tool arguments, or successful response bodies. +`transport_generation` identifies one constructed transport for the configured server. +`request_sequence` counts requests through that transport. +The connection, request, catalog-list, and effective timeout fields report the resolved millisecond budgets for the request operation. `server`, `via`, `x_request_id`, and the `x_envoy_*` fields are the proxy's own account of the failure and appear only when the proxy sends them. A proxy-generated response does not by itself prove whether the proxy or its upstream produced the failure; keep both the status and the cause chain when reporting one. -The line group is written only for a failure, so normal traffic adds no logging. -A 2xx response does not emit this diagnostic or read its body. +By default, the line group is written only for a failure, so normal traffic adds no logging. +A 2xx response does not emit this failure diagnostic or read its body. This fetch boundary cannot report a failure that occurs while the caller later reads a 2xx response body. For a non-2xx response, OpenClaw does not wait for asynchronous body sampling before it receives the response. Non-2xx response diagnostics are best-effort. @@ -143,7 +158,49 @@ The line group can also be absent if OpenClaw exits before collection completes. The sampler waits at most 250 ms and retains at most 2,048 response bytes. The redacted `error_body` value contains at most 2,048 UTF-8 bytes before JSON encoding. It redacts session identifiers, bearer tokens, structured credentials such as `access_token`, `refresh_token`, and `client_secret`, and known token prefixes. -The diagnostic does not include a JSON-RPC operation because this boundary sees only the endpoint. + +### Collect Shadow Timeout Evidence + +Set `NEMOCLAW_MCP_SHADOW_DIAGNOSTICS=1` before you create or rebuild an OpenClaw sandbox to collect successful request timing without applying a timeout change. +NemoClaw accepts only the literal value `1` and forwards it only to OpenClaw sandboxes. +The setting is disabled by default and affects only remote Streamable HTTP MCP requests inside an OpenShell sandbox. + + +Enabling shadow diagnostics records one timing event for every successful remote Streamable HTTP MCP request and increases log volume. +Each event includes the endpoint, configured server name, operation, and timeout metadata, but excludes tool arguments and successful response bodies. +Rebuilding also interrupts the agent while NemoClaw replaces the sandbox and restores its managed state. + + +For an existing sandbox, export the setting and rebuild: + +```bash +export NEMOCLAW_MCP_SHADOW_DIAGNOSTICS=1 +$$nemoclaw rebuild +``` + +For a new sandbox, export the setting before `$$nemoclaw onboard` creates it. + +When enabled, each successful request emits a `managed_transport_shadow` line group with the same safe operation, transport, timeout, and elapsed-time fields. +The diagnostic does not read the successful response body or change the returned response. + +For `tools/list`, NemoClaw retains up to 64 successful elapsed-time samples per endpoint in the OpenClaw process. +After five samples, the event reports `shadow_p95_ms` and `shadow_recommended_timeout_ms`. +The proposed value is 1.5 times the observed p95 and is rounded up to 100 ms. +It cannot be less than the active catalog-list budget or 1,500 ms, and it cannot exceed 10,000 ms. +If the active budget already exceeds 10,000 ms, NemoClaw does not emit a recommendation. +An abort observed near the effective `tools/list` budget proposes twice that budget under the same constraints. +An explicit HTTP 503 does not produce a timeout recommendation because a longer timeout does not repair a returned upstream failure. + +The shadow recommendation is evidence only. +It does not change a timeout, retry a request, or persist samples across an OpenClaw process restart. +After you collect the evidence, remove the setting and rebuild to restore failure-only logging: + +```bash +unset NEMOCLAW_MCP_SHADOW_DIAGNOSTICS +$$nemoclaw rebuild +``` + +Verify that later successful MCP requests emit no `managed_transport_shadow` line groups. To find the matching sandbox audit record, read the sandbox log for the same endpoint around the failure time. The `logs` command reads both OpenClaw gateway output and OpenShell audit events, so it needs no separate audit flag. @@ -155,6 +212,7 @@ $$nemoclaw logs --tail 200 Match on `target` and the timestamp. If NemoClaw cannot enable OpenShell audit logs, it prints a warning and the policy denial events can be missing from the output. `diagnostic_id` is a local identifier created for one line group. +NemoClaw creates it before the request starts, but does not add it to the request. It is not a distributed trace identifier and does not appear in OpenShell audit events. Correlate with OpenShell by endpoint and time until OpenShell records a shared identifier. diff --git a/docs/security/openclaw-2026.7.1-dependency-review.md b/docs/security/openclaw-2026.7.1-dependency-review.md index 076f70ecfb0..0fb1dc2420f 100644 --- a/docs/security/openclaw-2026.7.1-dependency-review.md +++ b/docs/security/openclaw-2026.7.1-dependency-review.md @@ -448,8 +448,11 @@ An unrecognized compiled shape fails the image build instead of silently skippin Reviewed behavior: -- Failure-only. - A 2xx response returns untouched and emits nothing, so normal traffic produces no per-request logging. +- Failure-only by default. + A 2xx response returns untouched and emits nothing unless the OpenClaw gateway process has `NEMOCLAW_MCP_SHADOW_DIAGNOSTICS=1`. +- The opt-in shadow mode emits timing-only `managed_transport_shadow` events for successful requests without reading their response bodies. + It does not change a timeout, retry a request, alter a response, or persist samples across an OpenClaw process restart. + Sandbox creation forwards only the literal value `1`, and only for OpenClaw. - The wrapper never retries, never alters the request, never changes proxy selection, and never weakens TLS verification. It rethrows a transport error unchanged. - `route=proxy_configured` means that `HTTPS_PROXY`, `https_proxy`, `HTTP_PROXY`, or `http_proxy` was configured. @@ -478,8 +481,20 @@ Reviewed behavior: That diagnostic has no response headers or `http_status` because `fetch` did not return a response. - A returned non-2xx response sets `transport_phase=response_headers`. It carries `http_status` and any allowlisted response headers that are present. -- The fetch boundary does not expose the JSON-RPC operation, so the diagnostic records the endpoint without an `operation` field. -- Each emitted diagnostic receives a local 32-character hexadecimal `diagnostic_id`. +- The wrapper parses only a string request body of at most 16,384 characters to report a validated JSON-RPC method. + It never reports JSON-RPC parameters, tool names, tool arguments, or successful response bodies. + An unsupported request shape reports `rpc/unknown`, and known GET and DELETE transport actions report `transport/listen` and `transport/close`. +- Each event reports the configured server name as `mcp_server` when validation and redaction retain it. + It also reports the transport generation, the request sequence, and the resolved connection, request, catalog-list, and effective timeout budgets. +- Shadow recommendations apply only to `tools/list`. + The wrapper retains up to 64 successful elapsed-time samples per endpoint and reports p95 after five samples. + It proposes p95 times 1.5, rounded up to 100 ms, with a 1,500 ms floor and a 10,000 ms ceiling. + A proposal cannot be less than the active catalog-list budget. + The wrapper emits no recommendation when that active budget already exceeds 10,000 ms. + A near-budget abort proposes twice the effective budget under the same constraints. + An explicit HTTP 503 produces no timeout recommendation. +- Each request receives a local 32-character hexadecimal `diagnostic_id` before `fetch` starts. + The wrapper does not add that identifier to the request. - The wrapper is inert unless `OPENSHELL_SANDBOX=1`, so it does not change host-side behavior. `diagnostic_id` is not a distributed trace identifier and does not correlate with an OpenShell audit event. @@ -494,7 +509,7 @@ The two patches compose independently. The injected helper in `scripts/patch-openclaw-managed-transport-diagnostics.mts` is the shipped runtime source of truth. `test/openclaw-managed-transport-diagnostics-patch.test.ts` executes that exact helper. It pins the compiled preimage, patch idempotence, fail-closed rejection of an unrecognized shape, and the untouched SSE boundary. -It also covers failure-only emission, no-retry and unchanged-response contracts, asynchronous body sampling, byte and time bounds, redaction, the header allowlist, local diagnostic identifiers, session-presence reporting, transport-phase classification, route evidence, and sandbox gating. +It also covers default failure-only emission, opt-in successful timing events, bounded shadow recommendations, explicit 503 exclusion, no-retry and unchanged-response contracts, safe operation reporting, asynchronous body sampling, byte and time bounds, redaction, the header allowlist, local diagnostic identifiers, session-presence reporting, transport-phase classification, route evidence, and sandbox gating. A reusable source schema is deferred until a production consumer requires one. Removal criterion: drop this patch when the reviewed OpenClaw release emits redacted diagnostics classified by transport phase for remote MCP fetch failures. diff --git a/scripts/patch-openclaw-managed-transport-diagnostics.mts b/scripts/patch-openclaw-managed-transport-diagnostics.mts index 87f0171a522..37977410485 100755 --- a/scripts/patch-openclaw-managed-transport-diagnostics.mts +++ b/scripts/patch-openclaw-managed-transport-diagnostics.mts @@ -24,7 +24,13 @@ const STREAMABLE_TRANSPORT_REPLACEMENT = [ '\tif (resolved.transportType === "streamable-http") return {', "\t\ttransport: new StreamableHTTPClientTransport(new URL(resolved.url), {", '\t\t\trequestInit: resolved.auth === "oauth" || !headers ? void 0 : { headers },', - "\t\t\tfetch: nemoClawManagedTransportFetch(httpFetch, resolved.url),", + "\t\t\tfetch: nemoClawManagedTransportFetch(httpFetch, {", + "\t\t\t\tserverName,", + "\t\t\t\tserverUrl: resolved.url,", + "\t\t\t\tconnectionTimeoutMs: resolved.connectionTimeoutMs,", + "\t\t\t\trequestTimeoutMs: resolved.requestTimeoutMs,", + "\t\t\t\tcatalogListTimeoutMs: getCatalogListTimeoutMs(rawServer, resolved.requestTimeoutMs)", + "\t\t\t}),", ].join("\n"); const UNPATCHED_TARGET_PATTERNS = [STREAMABLE_TRANSPORT_PATTERN]; @@ -32,14 +38,24 @@ const REQUIRED_PATTERNS = [...UNPATCHED_TARGET_PATTERNS]; const PATCHED_REQUIRED_PATTERNS = [MARKER, STREAMABLE_TRANSPORT_REPLACEMENT]; /** - * Failure-only managed-transport diagnostics for the remote Streamable HTTP MCP - * fetch boundary. The wrapper never retries, never alters the request, and never - * reads a 2xx body, so streaming responses stay behaviorally unchanged. + * Managed-transport failure diagnostics plus opt-in, timing-only success + * sampling for the remote Streamable HTTP MCP fetch boundary. The wrapper + * never retries, never alters the request, and never reads a 2xx body, so + * streaming responses stay behaviorally unchanged. */ export const INJECTED_DIAGNOSTIC_HELPER = [ "", MARKER, 'const NEMOCLAW_MTD_EVENT = "managed_transport_failure";', + 'const NEMOCLAW_MTD_SHADOW_EVENT = "managed_transport_shadow";', + 'const NEMOCLAW_MTD_SHADOW_ENV = "NEMOCLAW_MCP_SHADOW_DIAGNOSTICS";', + "const NEMOCLAW_MTD_SHADOW_MIN_SAMPLES = 5;", + "const NEMOCLAW_MTD_SHADOW_SAMPLE_LIMIT = 64;", + "const NEMOCLAW_MTD_SHADOW_MIN_TIMEOUT_MS = 1500;", + "const NEMOCLAW_MTD_SHADOW_MAX_TIMEOUT_MS = 10000;", + "const NEMOCLAW_MTD_SHADOW_SAFETY_FACTOR = 1.5;", + "const NEMOCLAW_MTD_SHADOW_SAMPLES = new Map();", + "const NEMOCLAW_MTD_TRANSPORT_GENERATIONS = new Map();", "const NEMOCLAW_MTD_BODY_LIMIT = 2048;", "const NEMOCLAW_MTD_BODY_TIMEOUT_MS = 250;", "const NEMOCLAW_MTD_SAFE_HEADERS = [", @@ -170,10 +186,58 @@ export const INJECTED_DIAGNOSTIC_HELPER = [ "function nemoClawMtdDiagnosticId() {", '\treturn globalThis.crypto.randomUUID().replaceAll("-", "");', "}", + "function nemoClawMtdOperation(init) {", + '\tif (!init || typeof init !== "object") return undefined;', + '\tconst method = typeof init.method === "string" ? init.method.toUpperCase() : "";', + '\tif (method === "DELETE") return "transport/close";', + '\tif (method === "GET") return "transport/listen";', + '\tif (typeof init.body !== "string" || init.body.length > 16384) return method === "POST" ? "rpc/unknown" : undefined;', + "\ttry {", + "\t\tconst payload = JSON.parse(init.body);", + '\t\tif (!payload || typeof payload !== "object" || Array.isArray(payload)) return "rpc/unknown";', + "\t\tconst rpcMethod = payload.method;", + '\t\tif (typeof rpcMethod !== "string" || rpcMethod.length === 0 || rpcMethod.length > 64 || !/^[A-Za-z0-9_./-]+$/.test(rpcMethod)) return "rpc/unknown";', + "\t\treturn nemoClawMtdRedact(rpcMethod, 64);", + "\t} catch {", + '\t\treturn "rpc/unknown";', + "\t}", + "}", + "function nemoClawMtdEffectiveTimeout(operation, options) {", + '\tif (operation === "initialize") return options.connectionTimeoutMs;', + '\tif (operation === "tools/list") return options.catalogListTimeoutMs;', + '\tif (operation === "tools/call" || operation === "resources/list" || operation === "resources/read" || operation === "prompts/list" || operation === "prompts/get") return options.requestTimeoutMs;', + "\treturn undefined;", + "}", + "function nemoClawMtdTransportGeneration(serverName) {", + '\tconst key = typeof serverName === "string" ? serverName : "";', + "\tconst generation = (NEMOCLAW_MTD_TRANSPORT_GENERATIONS.get(key) || 0) + 1;", + "\tNEMOCLAW_MTD_TRANSPORT_GENERATIONS.set(key, generation);", + "\treturn generation;", + "}", + "function nemoClawMtdShadowRecommendation(key, elapsedMs, effectiveTimeoutMs, timedOut) {", + "\tconst samples = NEMOCLAW_MTD_SHADOW_SAMPLES.get(key) || [];", + "\tif (!timedOut) {", + "\t\tsamples.push(elapsedMs);", + "\t\tif (samples.length > NEMOCLAW_MTD_SHADOW_SAMPLE_LIMIT) samples.shift();", + "\t\tNEMOCLAW_MTD_SHADOW_SAMPLES.set(key, samples);", + "\t}", + "\tconst sorted = [...samples].sort((left, right) => left - right);", + "\tconst percentile = sorted.length > 0 ? sorted[Math.max(0, Math.ceil(sorted.length * 0.95) - 1)] : undefined;", + "\tlet recommended;", + '\tif (timedOut && typeof effectiveTimeoutMs === "number") recommended = effectiveTimeoutMs * 2;', + "\telse if (sorted.length >= NEMOCLAW_MTD_SHADOW_MIN_SAMPLES) recommended = percentile * NEMOCLAW_MTD_SHADOW_SAFETY_FACTOR;", + '\tif (typeof recommended === "number" && typeof effectiveTimeoutMs === "number" && effectiveTimeoutMs > NEMOCLAW_MTD_SHADOW_MAX_TIMEOUT_MS) recommended = undefined;', + '\tif (typeof recommended === "number") recommended = Math.min(NEMOCLAW_MTD_SHADOW_MAX_TIMEOUT_MS, Math.max(NEMOCLAW_MTD_SHADOW_MIN_TIMEOUT_MS, effectiveTimeoutMs || 0, Math.ceil(recommended / 100) * 100));', + "\treturn { sampleCount: sorted.length, percentile, recommended };", + "}", + "function nemoClawMtdTimedOut(chain, elapsedMs, effectiveTimeoutMs) {", + '\tif (typeof effectiveTimeoutMs !== "number" || elapsedMs < effectiveTimeoutMs * 0.8) return false;', + '\treturn chain.some((cause) => cause.name === "AbortError" || cause.code === "UND_ERR_ABORTED" || cause.code === "UND_ERR_HEADERS_TIMEOUT");', + "}", "function nemoClawMtdEmit(fields) {", - "\tconst lines = [NEMOCLAW_MTD_EVENT];", + "\tconst lines = [fields.event || NEMOCLAW_MTD_EVENT];", "\tfor (const [key, value] of Object.entries(fields)) {", - "\t\tif (value === undefined) continue;", + '\t\tif (key === "event" || value === undefined) continue;', '\t\tconst encoded = typeof value === "object" || key === "error_body" ? JSON.stringify(value) : String(value);', '\t\tlines.push(key + "=" + encoded);', "\t}", @@ -184,41 +248,79 @@ export const INJECTED_DIAGNOSTIC_HELPER = [ "\tconst errorBody = await nemoClawMtdErrorBody(response, headers.content_type);", "\tnemoClawMtdEmit({ ...fields, ...headers, error_body: errorBody });", "}", - "function nemoClawManagedTransportFetch(inner, serverUrl) {", + "function nemoClawManagedTransportFetch(inner, options) {", '\tif (process.env.OPENSHELL_SANDBOX !== "1") return inner;', "\tconst proxy = process.env.HTTPS_PROXY || process.env.https_proxy || process.env.HTTP_PROXY || process.env.http_proxy;", + "\tconst safeServerName = nemoClawMtdSafeCode(options.serverName);", + "\tconst serverName = safeServerName ? nemoClawMtdRedact(safeServerName, 64) : undefined;", + "\tconst target = nemoClawMtdEndpoint(options.serverUrl);", + "\tconst transportGeneration = nemoClawMtdTransportGeneration(options.serverName);", + "\tlet requestSequence = 0;", "\treturn async (input, init) => {", "\t\tconst startedAt = Date.now();", + "\t\tconst diagnosticId = nemoClawMtdDiagnosticId();", + "\t\tconst operation = nemoClawMtdOperation(init);", + "\t\tconst effectiveTimeoutMs = nemoClawMtdEffectiveTimeout(operation, options);", + "\t\trequestSequence += 1;", + "\t\tconst common = {", + '\t\t\tconsumer: "mcp",', + "\t\t\tmcp_server: serverName,", + "\t\t\toperation,", + "\t\t\ttransport_generation: transportGeneration,", + "\t\t\trequest_sequence: requestSequence,", + '\t\t\troute: proxy ? "proxy_configured" : "unknown",', + "\t\t\tproxy: proxy ? nemoClawMtdEndpoint(proxy) : undefined,", + "\t\t\ttarget,", + "\t\t\tconnection_timeout_ms: options.connectionTimeoutMs,", + "\t\t\trequest_timeout_ms: options.requestTimeoutMs,", + "\t\t\tcatalog_list_timeout_ms: options.catalogListTimeoutMs,", + "\t\t\teffective_timeout_ms: effectiveTimeoutMs,", + "\t\t\tdiagnostic_id: diagnosticId", + "\t\t};", '\t\tconst sessionPresent = Boolean(init && init.headers && new Headers(init.headers).get("mcp-session-id"));', "\t\ttry {", "\t\t\tconst response = await inner(input, init);", - "\t\t\tif (response && response.ok) return response;", + "\t\t\tconst elapsedMs = Date.now() - startedAt;", + "\t\t\tif (response && response.ok) {", + '\t\t\t\tif (process.env[NEMOCLAW_MTD_SHADOW_ENV] === "1") {', + '\t\t\t\t\tconst shadow = operation === "tools/list" ? nemoClawMtdShadowRecommendation((target || "unknown") + "\\0" + operation, elapsedMs, effectiveTimeoutMs, false) : undefined;', + "\t\t\t\t\tnemoClawMtdEmit({", + "\t\t\t\t\t\tevent: NEMOCLAW_MTD_SHADOW_EVENT,", + "\t\t\t\t\t\t...common,", + "\t\t\t\t\t\thttp_status: response.status,", + "\t\t\t\t\t\telapsed_ms: elapsedMs,", + "\t\t\t\t\t\tsession_present: sessionPresent,", + "\t\t\t\t\t\tshadow_sample_count: shadow && shadow.sampleCount,", + "\t\t\t\t\t\tshadow_p95_ms: shadow && shadow.percentile,", + "\t\t\t\t\t\tshadow_recommended_timeout_ms: shadow && shadow.recommended", + "\t\t\t\t\t});", + "\t\t\t\t}", + "\t\t\t\treturn response;", + "\t\t\t}", "\t\t\tvoid nemoClawMtdEmitResponseFailure(response, {", - '\t\t\t\tconsumer: "mcp",', - '\t\t\t\troute: proxy ? "proxy_configured" : "unknown",', - "\t\t\t\tproxy: proxy ? nemoClawMtdEndpoint(proxy) : undefined,", - "\t\t\t\ttarget: nemoClawMtdEndpoint(serverUrl),", + "\t\t\t\t...common,", '\t\t\t\ttransport_phase: "response_headers",', "\t\t\t\thttp_status: response ? response.status : undefined,", - "\t\t\t\telapsed_ms: Date.now() - startedAt,", - "\t\t\t\tsession_present: sessionPresent,", - "\t\t\t\tdiagnostic_id: nemoClawMtdDiagnosticId()", + "\t\t\t\telapsed_ms: elapsedMs,", + "\t\t\t\tsession_present: sessionPresent", "\t\t\t}).catch(() => {});", "\t\t\treturn response;", "\t\t} catch (error) {", "\t\t\ttry {", "\t\t\t\tconst chain = nemoClawMtdCauseChain(error);", + "\t\t\t\tconst elapsedMs = Date.now() - startedAt;", + '\t\t\t\tconst timedOut = operation === "tools/list" && nemoClawMtdTimedOut(chain, elapsedMs, effectiveTimeoutMs);', + '\t\t\t\tconst shadow = process.env[NEMOCLAW_MTD_SHADOW_ENV] === "1" && operation === "tools/list" && timedOut ? nemoClawMtdShadowRecommendation((target || "unknown") + "\\0" + operation, elapsedMs, effectiveTimeoutMs, true) : undefined;', "\t\t\t\tnemoClawMtdEmit({", - '\t\t\t\t\tconsumer: "mcp",', - '\t\t\t\t\troute: proxy ? "proxy_configured" : "unknown",', - "\t\t\t\t\tproxy: proxy ? nemoClawMtdEndpoint(proxy) : undefined,", - "\t\t\t\t\ttarget: nemoClawMtdEndpoint(serverUrl),", + "\t\t\t\t\t...common,", "\t\t\t\t\ttransport_phase: nemoClawMtdPhase(chain),", - "\t\t\t\t\telapsed_ms: Date.now() - startedAt,", + "\t\t\t\t\telapsed_ms: elapsedMs,", "\t\t\t\t\tcause_code: chain[0] && chain[0].code,", "\t\t\t\t\tsession_present: sessionPresent,", "\t\t\t\t\tcause_chain: chain,", - "\t\t\t\t\tdiagnostic_id: nemoClawMtdDiagnosticId()", + "\t\t\t\t\tshadow_sample_count: shadow && shadow.sampleCount,", + "\t\t\t\t\tshadow_p95_ms: shadow && shadow.percentile,", + "\t\t\t\t\tshadow_recommended_timeout_ms: shadow && shadow.recommended", "\t\t\t\t});", "\t\t\t} catch {}", "\t\t\tthrow error;", diff --git a/src/lib/onboard/sandbox-create-launch.test.ts b/src/lib/onboard/sandbox-create-launch.test.ts index fdb3313f68a..c4206b8a959 100644 --- a/src/lib/onboard/sandbox-create-launch.test.ts +++ b/src/lib/onboard/sandbox-create-launch.test.ts @@ -105,6 +105,39 @@ describe("buildSandboxRuntimeEnvArgs", () => { }).envArgs; expect(envArgs.some((arg) => arg.startsWith("NEMOCLAW_SANDBOX_NAME="))).toBe(false); }); + + it("forwards only the literal OpenClaw MCP shadow diagnostic opt-in", () => { + const base = { + chatUiUrl: "", + manageDashboard: false, + getDashboardForwardPort: () => "0", + hermesDashboardState: disabledHermesDashboardState, + extraPlaceholderKeys: [], + }; + const openclaw = { name: "openclaw", configPaths: { dir: "/sandbox/.openclaw" } } as any; + + expect( + buildSandboxRuntimeEnvArgs({ + ...base, + agent: openclaw, + env: { NEMOCLAW_MCP_SHADOW_DIAGNOSTICS: " 1 " }, + }).envArgs, + ).toContain("NEMOCLAW_MCP_SHADOW_DIAGNOSTICS=1"); + expect( + buildSandboxRuntimeEnvArgs({ + ...base, + agent: openclaw, + env: { NEMOCLAW_MCP_SHADOW_DIAGNOSTICS: "true" }, + }).envArgs.some((entry) => entry.startsWith("NEMOCLAW_MCP_SHADOW_DIAGNOSTICS=")), + ).toBe(false); + expect( + buildSandboxRuntimeEnvArgs({ + ...base, + agent: loadAgent("hermes"), + env: { NEMOCLAW_MCP_SHADOW_DIAGNOSTICS: "1" }, + }).envArgs, + ).not.toContain("NEMOCLAW_MCP_SHADOW_DIAGNOSTICS=1"); + }); }); describe("prepareSandboxCreateLaunch", () => { diff --git a/src/lib/onboard/sandbox-create-launch.ts b/src/lib/onboard/sandbox-create-launch.ts index cd20e849589..15a5eb36cfe 100644 --- a/src/lib/onboard/sandbox-create-launch.ts +++ b/src/lib/onboard/sandbox-create-launch.ts @@ -38,6 +38,11 @@ const OPENCLAW_AUTO_PAIR_RUNTIME_ENV_KEYS = [ "NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS", ] as const; +// This opt-in emits timing-only MCP success events from the reviewed OpenClaw +// dist patch. Accept only the literal enabled value, and only for OpenClaw, so +// the broader host environment never becomes sandbox runtime input. +const OPENCLAW_DIAGNOSTIC_RUNTIME_ENV_KEYS = ["NEMOCLAW_MCP_SHADOW_DIAGNOSTICS"] as const; + function appendOpenClawAutoPairRuntimeEnvArgs( envArgs: string[], agent: AgentDefinition | null, @@ -50,6 +55,17 @@ function appendOpenClawAutoPairRuntimeEnvArgs( } } +function appendOpenClawDiagnosticRuntimeEnvArgs( + envArgs: string[], + agent: AgentDefinition | null, + env: NodeJS.ProcessEnv, +): void { + if (agent && agent.name !== "openclaw") return; + for (const key of OPENCLAW_DIAGNOSTIC_RUNTIME_ENV_KEYS) { + if (env[key]?.trim() === "1") envArgs.push(formatEnvAssignment(key, "1")); + } +} + export interface SandboxCreateLaunchInput { agent: AgentDefinition | null | undefined; observabilityEnabled?: boolean; @@ -170,6 +186,7 @@ export function buildSandboxRuntimeEnvArgs(input: SandboxRuntimeEnvArgsInput): { appendOpenClawRuntimeEnvArgs(envArgs, agent); appendOpenClawAutoPairRuntimeEnvArgs(envArgs, agent, env); + appendOpenClawDiagnosticRuntimeEnvArgs(envArgs, agent, env); appendHermesDashboardEnvArgs(envArgs, input.hermesDashboardState, formatEnvAssignment); appendHostProxyEnvArgs(envArgs, env, { dropCredentialBearingProxyUrls: diff --git a/test/openclaw-dependency-review.test.ts b/test/openclaw-dependency-review.test.ts index 70df8909256..9e97fcb82bb 100644 --- a/test/openclaw-dependency-review.test.ts +++ b/test/openclaw-dependency-review.test.ts @@ -195,7 +195,9 @@ describe("OpenClaw 2026.6.10 dependency review contract", () => { expect(review).toContain("## Managed Outbound Transport Diagnostics"); expect(review).toContain("scripts/patch-openclaw-managed-transport-diagnostics.mts"); expect(review).toContain("The sibling SSE transport boundary is deliberately left unwrapped."); - expect(review).toContain("Failure-only."); + expect(review).toContain("Failure-only by default."); + expect(review).toContain("`NEMOCLAW_MCP_SHADOW_DIAGNOSTICS=1`"); + expect(review).toContain("timing-only `managed_transport_shadow` events"); expect(review).toContain("never retries, never alters the request"); expect(review).toContain("`route=proxy_configured` means that `HTTPS_PROXY`"); expect(review).toContain("report configuration evidence"); @@ -213,7 +215,10 @@ describe("OpenClaw 2026.6.10 dependency review contract", () => { expect(review).toContain("A returned non-2xx response sets `transport_phase=response_headers`"); expect(review).toContain("transport-phase signal"); expect(review).toContain("does not inspect a 2xx response body"); - expect(review).toContain("without an `operation` field"); + expect(review).toContain("report a validated JSON-RPC method"); + expect(review).toContain("reports the configured server name as `mcp_server`"); + expect(review).toContain("Shadow recommendations apply only to `tools/list`"); + expect(review).toContain("An explicit HTTP 503 produces no timeout recommendation."); expect(review).toContain("structured credentials such as `access_token`"); expect(review).toContain("The peer address is not recorded."); expect(review).toContain("The `mcp-session-id` value is never emitted."); diff --git a/test/openclaw-managed-transport-diagnostics-patch.test.ts b/test/openclaw-managed-transport-diagnostics-patch.test.ts index b2e77b7075f..15c69dac59c 100644 --- a/test/openclaw-managed-transport-diagnostics-patch.test.ts +++ b/test/openclaw-managed-transport-diagnostics-patch.test.ts @@ -51,17 +51,28 @@ function bundleMcpRuntimeFixture(): string { interface HelperHarness { wrap: ( inner: typeof fetch, - serverUrl: string, + serverUrl: + | string + | { + serverName: string; + serverUrl: string; + connectionTimeoutMs: number; + requestTimeoutMs: number; + catalogListTimeoutMs: number; + }, ) => (input: unknown, init?: RequestInit) => Promise; stderr: string[]; } -function loadHelper(env: Record = { OPENSHELL_SANDBOX: "1" }): HelperHarness { +function loadHelper( + env: Record = { OPENSHELL_SANDBOX: "1" }, + now: () => number = () => Date.now(), +): HelperHarness { const stderr: string[] = []; const context = vm.createContext({ Headers, URL, - Date, + Date: { now }, TextDecoder, TextEncoder, Object, @@ -75,10 +86,23 @@ function loadHelper(env: Record = { OPENSHELL_SANDBOX: "1" }): H process: { env, stderr: { write: (chunk: string) => stderr.push(chunk) } }, setTimeout, }); - const wrap = vm.runInContext( + const injectedWrap = vm.runInContext( `${INJECTED_DIAGNOSTIC_HELPER}\nnemoClawManagedTransportFetch;`, context, ); + const wrap: HelperHarness["wrap"] = (inner, serverUrl) => + injectedWrap( + inner, + typeof serverUrl === "string" + ? { + serverName: "remotedocs", + serverUrl, + connectionTimeoutMs: 30_000, + requestTimeoutMs: 60_000, + catalogListTimeoutMs: 1_500, + } + : serverUrl, + ); return { wrap, stderr }; } @@ -93,6 +117,24 @@ function emittedEvent(stderr: string[]): Record { ); } +function emittedEvents(stderr: string[]): Array> { + return stderr.map((chunk) => { + const lines = chunk + .split("\n") + .map((line) => line.replace(/^\[nemoclaw\] /, "")) + .filter(Boolean); + return { + event: lines[0], + ...Object.fromEntries( + lines + .slice(1) + .filter((line) => line.includes("=")) + .map((line) => [line.slice(0, line.indexOf("=")), line.slice(line.indexOf("=") + 1)]), + ), + }; + }); +} + async function waitForDiagnostic(stderr: string[]): Promise { await expect.poll(() => stderr.length, { interval: 5, timeout: 1000 }).toBeGreaterThan(0); } @@ -103,9 +145,8 @@ describe("patchManagedTransportDiagnosticsText", () => { expect(result.status).toBe("patched"); expect(result.text).toContain(MARKER); - expect(result.text).toContain( - "\t\t\tfetch: nemoClawManagedTransportFetch(httpFetch, resolved.url),", - ); + expect(result.text).toContain("\t\t\tfetch: nemoClawManagedTransportFetch(httpFetch, {"); + expect(result.text).toContain("\t\t\t\tcatalogListTimeoutMs: getCatalogListTimeoutMs("); }); it("leaves the SSE transport boundary untouched (#7957)", () => { @@ -158,6 +199,220 @@ describe("injected managed transport wrapper", () => { expect(stderr).toEqual([]); }); + it("emits opt-in operation timing and a bounded shadow recommendation without changing responses (#7957)", async () => { + let now = 0; + const { wrap, stderr } = loadHelper( + { + OPENSHELL_SANDBOX: "1", + NEMOCLAW_MCP_SHADOW_DIAGNOSTICS: "1", + }, + () => now, + ); + const delays = [1_000, 1_600, 3_000, 6_000, 1_200]; + let call = 0; + const inner = async () => { + now += delays[call]; + call += 1; + return new Response("ok", { status: 200 }); + }; + const wrapped = wrap(inner as unknown as typeof fetch, { + serverName: "gitlab", + serverUrl: "https://mcp.test/rpc", + connectionTimeoutMs: 30_000, + requestTimeoutMs: 60_000, + catalogListTimeoutMs: 1_500, + }); + + for (let index = 0; index < delays.length; index += 1) { + const response = await wrapped("https://mcp.test/rpc", { + method: "POST", + body: JSON.stringify({ jsonrpc: "2.0", id: index, method: "tools/list" }), + }); + expect(response.status).toBe(200); + } + + const events = emittedEvents(stderr); + expect(events).toHaveLength(5); + expect(events[0]).toMatchObject({ + event: "managed_transport_shadow", + mcp_server: "gitlab", + operation: "tools/list", + transport_generation: "1", + request_sequence: "1", + connection_timeout_ms: "30000", + request_timeout_ms: "60000", + catalog_list_timeout_ms: "1500", + effective_timeout_ms: "1500", + elapsed_ms: "1000", + shadow_sample_count: "1", + }); + expect(events[4]).toMatchObject({ + request_sequence: "5", + shadow_sample_count: "5", + shadow_p95_ms: "6000", + shadow_recommended_timeout_ms: "9000", + }); + expect(events[0].shadow_recommended_timeout_ms).toBeUndefined(); + }); + + it("recommends a bounded larger catalog budget for an observed tools/list abort (#7957)", async () => { + let now = 0; + const { wrap, stderr } = loadHelper( + { + OPENSHELL_SANDBOX: "1", + NEMOCLAW_MCP_SHADOW_DIAGNOSTICS: "1", + }, + () => now, + ); + const error = Object.assign(new Error("This operation was aborted"), { + name: "AbortError", + code: "UND_ERR_ABORTED", + }); + const inner = async () => { + now += 1_500; + throw error; + }; + + await expect( + wrap(inner as unknown as typeof fetch, "https://mcp.test/rpc")("https://mcp.test/rpc", { + method: "POST", + body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list" }), + }), + ).rejects.toBe(error); + + expect(emittedEvents(stderr)[0]).toMatchObject({ + event: "managed_transport_failure", + operation: "tools/list", + elapsed_ms: "1500", + effective_timeout_ms: "1500", + shadow_sample_count: "0", + shadow_recommended_timeout_ms: "3000", + }); + }); + + it("never recommends a timeout below the active catalog budget (#7957)", async () => { + let now = 0; + const { wrap, stderr } = loadHelper( + { + OPENSHELL_SANDBOX: "1", + NEMOCLAW_MCP_SHADOW_DIAGNOSTICS: "1", + }, + () => now, + ); + const inner = async () => { + now += 1_000; + return new Response("ok", { status: 200 }); + }; + const wrapped = wrap(inner as unknown as typeof fetch, { + serverName: "jira", + serverUrl: "https://mcp.test/rpc", + connectionTimeoutMs: 30_000, + requestTimeoutMs: 60_000, + catalogListTimeoutMs: 5_000, + }); + + for (let id = 0; id < 5; id += 1) { + await wrapped("https://mcp.test/rpc", { + method: "POST", + body: JSON.stringify({ jsonrpc: "2.0", id, method: "tools/list" }), + }); + } + + expect(emittedEvents(stderr)[4].shadow_recommended_timeout_ms).toBe("5000"); + }); + + it("keeps an explicit 503 separate from timeout recommendations and observes a later success (#7957)", async () => { + let now = 0; + const { wrap, stderr } = loadHelper( + { + OPENSHELL_SANDBOX: "1", + NEMOCLAW_MCP_SHADOW_DIAGNOSTICS: "1", + }, + () => now, + ); + let calls = 0; + const inner = async () => { + calls += 1; + now += 200; + return calls === 1 + ? new Response("upstream reset", { status: 503 }) + : new Response("ok", { status: 200 }); + }; + const wrapped = wrap(inner as unknown as typeof fetch, "https://mcp.test/rpc"); + const init = { + method: "POST", + body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list" }), + }; + + expect((await wrapped("https://mcp.test/rpc", init)).status).toBe(503); + await waitForDiagnostic(stderr); + expect((await wrapped("https://mcp.test/rpc", init)).status).toBe(200); + const events = emittedEvents(stderr); + + expect(calls).toBe(2); + expect(events).toHaveLength(2); + expect(events[0]).toMatchObject({ + event: "managed_transport_failure", + http_status: "503", + request_sequence: "1", + }); + expect(events[0].shadow_recommended_timeout_ms).toBeUndefined(); + expect(events[1]).toMatchObject({ + event: "managed_transport_shadow", + http_status: "200", + request_sequence: "2", + }); + expect(events[0].diagnostic_id).not.toBe(events[1].diagnostic_id); + }); + + it("logs only the safe RPC method and never tool arguments (#7957)", async () => { + const { wrap, stderr } = loadHelper({ + OPENSHELL_SANDBOX: "1", + NEMOCLAW_MCP_SHADOW_DIAGNOSTICS: "1", + }); + const inner = async () => new Response("ok", { status: 200 }); + + await wrap(inner as unknown as typeof fetch, "https://mcp.test/rpc")("https://mcp.test/rpc", { + method: "POST", + body: JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "tools/call", + params: { name: "search", arguments: { token: "secret-tool-argument" } }, + }), + }); + + expect(emittedEvents(stderr)[0]).toMatchObject({ + event: "managed_transport_shadow", + operation: "tools/call", + effective_timeout_ms: "60000", + }); + expect(stderr.join("")).not.toContain("secret-tool-argument"); + expect(stderr.join("")).not.toContain("search"); + }); + + it("redacts a credential-shaped server name before emitting shadow timing (#7957)", async () => { + const { wrap, stderr } = loadHelper({ + OPENSHELL_SANDBOX: "1", + NEMOCLAW_MCP_SHADOW_DIAGNOSTICS: "1", + }); + const inner = async () => new Response("ok", { status: 200 }); + + await wrap(inner as unknown as typeof fetch, { + serverName: "sk-proj-secret-server-name-1234567890", + serverUrl: "https://mcp.test/rpc", + connectionTimeoutMs: 30_000, + requestTimeoutMs: 60_000, + catalogListTimeoutMs: 1_500, + })("https://mcp.test/rpc", { + method: "POST", + body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list" }), + }); + + expect(emittedEvents(stderr)[0].mcp_server).toContain(""); + expect(stderr.join("")).not.toContain("secret-server-name"); + }); + it("records a failed proxy response with the canonical safe headers (#7957)", async () => { const { wrap, stderr } = loadHelper(); const inner = async () => @@ -182,6 +437,7 @@ describe("injected managed transport wrapper", () => { expect(event.transport_phase).toBe("response_headers"); expect(event.http_status).toBe("503"); expect(event.target).toBe("mcp.test:8443"); + expect(event.mcp_server).toBe("remotedocs"); expect(event.server).toContain(""); expect(event.x_request_id).toContain(""); expect(event.x_envoy_response_flags).toBe("UF,URX"); From 3f050143121cfa6bfdbe394dd60f6a4c3ab8a838 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 6 Aug 2026 11:17:30 -0700 Subject: [PATCH 2/5] fix(observability): defer MCP p95 until sample floor Signed-off-by: Apurv Kumaria --- scripts/patch-openclaw-managed-transport-diagnostics.mts | 2 +- test/openclaw-managed-transport-diagnostics-patch.test.ts | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/patch-openclaw-managed-transport-diagnostics.mts b/scripts/patch-openclaw-managed-transport-diagnostics.mts index 37977410485..1cae9c206cf 100755 --- a/scripts/patch-openclaw-managed-transport-diagnostics.mts +++ b/scripts/patch-openclaw-managed-transport-diagnostics.mts @@ -222,7 +222,7 @@ export const INJECTED_DIAGNOSTIC_HELPER = [ "\t\tNEMOCLAW_MTD_SHADOW_SAMPLES.set(key, samples);", "\t}", "\tconst sorted = [...samples].sort((left, right) => left - right);", - "\tconst percentile = sorted.length > 0 ? sorted[Math.max(0, Math.ceil(sorted.length * 0.95) - 1)] : undefined;", + "\tconst percentile = sorted.length >= NEMOCLAW_MTD_SHADOW_MIN_SAMPLES ? sorted[Math.max(0, Math.ceil(sorted.length * 0.95) - 1)] : undefined;", "\tlet recommended;", '\tif (timedOut && typeof effectiveTimeoutMs === "number") recommended = effectiveTimeoutMs * 2;', "\telse if (sorted.length >= NEMOCLAW_MTD_SHADOW_MIN_SAMPLES) recommended = percentile * NEMOCLAW_MTD_SHADOW_SAFETY_FACTOR;", diff --git a/test/openclaw-managed-transport-diagnostics-patch.test.ts b/test/openclaw-managed-transport-diagnostics-patch.test.ts index 15c69dac59c..7927c1a0853 100644 --- a/test/openclaw-managed-transport-diagnostics-patch.test.ts +++ b/test/openclaw-managed-transport-diagnostics-patch.test.ts @@ -252,7 +252,10 @@ describe("injected managed transport wrapper", () => { shadow_p95_ms: "6000", shadow_recommended_timeout_ms: "9000", }); - expect(events[0].shadow_recommended_timeout_ms).toBeUndefined(); + for (const event of events.slice(0, 4)) { + expect(event.shadow_p95_ms).toBeUndefined(); + expect(event.shadow_recommended_timeout_ms).toBeUndefined(); + } }); it("recommends a bounded larger catalog budget for an observed tools/list abort (#7957)", async () => { From 0ce19bc3248da7f9f6ccd7e717a671561009fa1c Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 6 Aug 2026 11:57:08 -0700 Subject: [PATCH 3/5] fix(observability): isolate MCP diagnostic failures Signed-off-by: Apurv Kumaria --- docs/reference/troubleshoot-mcp-servers.mdx | 14 ++-- .../openclaw-2026.7.1-dependency-review.md | 11 +-- ...openclaw-managed-transport-diagnostics.mts | 22 +++--- ...anaged-transport-diagnostics-patch.test.ts | 67 ++++++++++++++++++- 4 files changed, 94 insertions(+), 20 deletions(-) diff --git a/docs/reference/troubleshoot-mcp-servers.mdx b/docs/reference/troubleshoot-mcp-servers.mdx index b6e57c42e18..b34bbc46e54 100644 --- a/docs/reference/troubleshoot-mcp-servers.mdx +++ b/docs/reference/troubleshoot-mcp-servers.mdx @@ -166,8 +166,8 @@ NemoClaw accepts only the literal value `1` and forwards it only to OpenClaw san The setting is disabled by default and affects only remote Streamable HTTP MCP requests inside an OpenShell sandbox. -Enabling shadow diagnostics records one timing event for every successful remote Streamable HTTP MCP request and increases log volume. -Each event includes the endpoint, configured server name, operation, and timeout metadata, but excludes tool arguments and successful response bodies. +Enabling shadow diagnostics attempts to record one timing event for every successful remote Streamable HTTP MCP request and increases log volume. +Each event includes the target host and port, configured server name, operation, and timeout metadata, but excludes URL paths, tool arguments, and successful response bodies. Rebuilding also interrupts the agent while NemoClaw replaces the sandbox and restores its managed state. @@ -180,10 +180,12 @@ $$nemoclaw rebuild For a new sandbox, export the setting before `$$nemoclaw onboard` creates it. -When enabled, each successful request emits a `managed_transport_shadow` line group with the same safe operation, transport, timeout, and elapsed-time fields. +When enabled, each successful request attempts to emit a `managed_transport_shadow` line group with the same safe operation, transport, timeout, and elapsed-time fields. The diagnostic does not read the successful response body or change the returned response. +Identifier generation, serialization, or standard-error output failure can omit the line group without blocking the request or changing its response. -For `tools/list`, NemoClaw retains up to 64 successful elapsed-time samples per endpoint in the OpenClaw process. +For `tools/list`, NemoClaw retains up to 64 successful elapsed-time samples per target host and port in the OpenClaw process. +Different MCP URL paths on the same target host and port share this sample set because the diagnostic does not retain URL paths. After five samples, the event reports `shadow_p95_ms` and `shadow_recommended_timeout_ms`. The proposed value is 1.5 times the observed p95 and is rounded up to 100 ms. It cannot be less than the active catalog-list budget or 1,500 ms, and it cannot exceed 10,000 ms. @@ -211,8 +213,8 @@ $$nemoclaw logs --tail 200 Match on `target` and the timestamp. If NemoClaw cannot enable OpenShell audit logs, it prints a warning and the policy denial events can be missing from the output. -`diagnostic_id` is a local identifier created for one line group. -NemoClaw creates it before the request starts, but does not add it to the request. +`diagnostic_id` is a local identifier created for one line group when identifier generation succeeds. +NemoClaw attempts to create it before the request starts, but does not add it to the request. It is not a distributed trace identifier and does not appear in OpenShell audit events. Correlate with OpenShell by endpoint and time until OpenShell records a shared identifier. diff --git a/docs/security/openclaw-2026.7.1-dependency-review.md b/docs/security/openclaw-2026.7.1-dependency-review.md index 0fb1dc2420f..2ba8da33dfc 100644 --- a/docs/security/openclaw-2026.7.1-dependency-review.md +++ b/docs/security/openclaw-2026.7.1-dependency-review.md @@ -5,7 +5,7 @@ Review date: 2026-07-21 -Last updated: 2026-08-03 +Last updated: 2026-08-06 ## Decision @@ -450,8 +450,9 @@ Reviewed behavior: - Failure-only by default. A 2xx response returns untouched and emits nothing unless the OpenClaw gateway process has `NEMOCLAW_MCP_SHADOW_DIAGNOSTICS=1`. -- The opt-in shadow mode emits timing-only `managed_transport_shadow` events for successful requests without reading their response bodies. +- The opt-in shadow mode attempts to emit timing-only `managed_transport_shadow` events for successful requests without reading their response bodies. It does not change a timeout, retry a request, alter a response, or persist samples across an OpenClaw process restart. + Identifier generation, serialization, or standard-error output failure can omit an event without blocking the request or changing its response. Sandbox creation forwards only the literal value `1`, and only for OpenClaw. - The wrapper never retries, never alters the request, never changes proxy selection, and never weakens TLS verification. It rethrows a transport error unchanged. @@ -487,13 +488,15 @@ Reviewed behavior: - Each event reports the configured server name as `mcp_server` when validation and redaction retain it. It also reports the transport generation, the request sequence, and the resolved connection, request, catalog-list, and effective timeout budgets. - Shadow recommendations apply only to `tools/list`. - The wrapper retains up to 64 successful elapsed-time samples per endpoint and reports p95 after five samples. + The wrapper retains up to 64 successful elapsed-time samples per target host and port and reports p95 after five samples. + Different MCP URL paths on the same target host and port share this sample set because the wrapper does not retain URL paths. It proposes p95 times 1.5, rounded up to 100 ms, with a 1,500 ms floor and a 10,000 ms ceiling. A proposal cannot be less than the active catalog-list budget. The wrapper emits no recommendation when that active budget already exceeds 10,000 ms. A near-budget abort proposes twice the effective budget under the same constraints. An explicit HTTP 503 produces no timeout recommendation. -- Each request receives a local 32-character hexadecimal `diagnostic_id` before `fetch` starts. +- The wrapper attempts to create a local 32-character hexadecimal `diagnostic_id` for each request before `fetch` starts. + If identifier generation fails, the wrapper omits the field and continues the request. The wrapper does not add that identifier to the request. - The wrapper is inert unless `OPENSHELL_SANDBOX=1`, so it does not change host-side behavior. diff --git a/scripts/patch-openclaw-managed-transport-diagnostics.mts b/scripts/patch-openclaw-managed-transport-diagnostics.mts index 1cae9c206cf..95708c9a2ba 100755 --- a/scripts/patch-openclaw-managed-transport-diagnostics.mts +++ b/scripts/patch-openclaw-managed-transport-diagnostics.mts @@ -184,7 +184,11 @@ export const INJECTED_DIAGNOSTIC_HELPER = [ "\t}", "}", "function nemoClawMtdDiagnosticId() {", - '\treturn globalThis.crypto.randomUUID().replaceAll("-", "");', + "\ttry {", + '\t\treturn globalThis.crypto.randomUUID().replaceAll("-", "");', + "\t} catch {", + "\t\treturn undefined;", + "\t}", "}", "function nemoClawMtdOperation(init) {", '\tif (!init || typeof init !== "object") return undefined;', @@ -235,13 +239,15 @@ export const INJECTED_DIAGNOSTIC_HELPER = [ '\treturn chain.some((cause) => cause.name === "AbortError" || cause.code === "UND_ERR_ABORTED" || cause.code === "UND_ERR_HEADERS_TIMEOUT");', "}", "function nemoClawMtdEmit(fields) {", - "\tconst lines = [fields.event || NEMOCLAW_MTD_EVENT];", - "\tfor (const [key, value] of Object.entries(fields)) {", - '\t\tif (key === "event" || value === undefined) continue;', - '\t\tconst encoded = typeof value === "object" || key === "error_body" ? JSON.stringify(value) : String(value);', - '\t\tlines.push(key + "=" + encoded);', - "\t}", - '\tprocess.stderr.write("[nemoclaw] " + lines.join("\\n[nemoclaw] ") + "\\n");', + "\ttry {", + "\t\tconst lines = [fields.event || NEMOCLAW_MTD_EVENT];", + "\t\tfor (const [key, value] of Object.entries(fields)) {", + '\t\t\tif (key === "event" || value === undefined) continue;', + '\t\t\tconst encoded = typeof value === "object" || key === "error_body" ? JSON.stringify(value) : String(value);', + '\t\t\tlines.push(key + "=" + encoded);', + "\t\t}", + '\t\tprocess.stderr.write("[nemoclaw] " + lines.join("\\n[nemoclaw] ") + "\\n");', + "\t} catch {}", "}", "async function nemoClawMtdEmitResponseFailure(response, fields) {", "\tconst headers = nemoClawMtdHeaders(response);", diff --git a/test/openclaw-managed-transport-diagnostics-patch.test.ts b/test/openclaw-managed-transport-diagnostics-patch.test.ts index 7927c1a0853..17164abfe84 100644 --- a/test/openclaw-managed-transport-diagnostics-patch.test.ts +++ b/test/openclaw-managed-transport-diagnostics-patch.test.ts @@ -67,6 +67,10 @@ interface HelperHarness { function loadHelper( env: Record = { OPENSHELL_SANDBOX: "1" }, now: () => number = () => Date.now(), + runtime: { + crypto?: Pick; + writeStderr?: (chunk: string) => void; + } = {}, ): HelperHarness { const stderr: string[] = []; const context = vm.createContext({ @@ -82,8 +86,11 @@ function loadHelper( String, Set, clearTimeout, - crypto: webcrypto, - process: { env, stderr: { write: (chunk: string) => stderr.push(chunk) } }, + crypto: runtime.crypto ?? webcrypto, + process: { + env, + stderr: { write: runtime.writeStderr ?? ((chunk: string) => stderr.push(chunk)) }, + }, setTimeout, }); const injectedWrap = vm.runInContext( @@ -199,6 +206,62 @@ describe("injected managed transport wrapper", () => { expect(stderr).toEqual([]); }); + it("sends the request when diagnostic identifier generation fails (#7957)", async () => { + let innerCalls = 0; + const { wrap, stderr } = loadHelper({ OPENSHELL_SANDBOX: "1" }, () => Date.now(), { + crypto: { + randomUUID: () => { + throw new Error("entropy unavailable"); + }, + }, + }); + const inner = async () => { + innerCalls += 1; + return new Response("ok", { status: 200 }); + }; + + const response = await wrap( + inner as unknown as typeof fetch, + "https://mcp.test/rpc", + )("https://mcp.test/rpc"); + + expect(response.status).toBe(200); + expect(innerCalls).toBe(1); + expect(stderr).toEqual([]); + }); + + it("returns a successful response when shadow diagnostic emission fails (#7957)", async () => { + let innerCalls = 0; + const { wrap } = loadHelper( + { + OPENSHELL_SANDBOX: "1", + NEMOCLAW_MCP_SHADOW_DIAGNOSTICS: "1", + }, + () => Date.now(), + { + writeStderr: () => { + throw new Error("stderr unavailable"); + }, + }, + ); + const expected = new Response("ok", { status: 200 }); + const inner = async () => { + innerCalls += 1; + return expected; + }; + + const response = await wrap(inner as unknown as typeof fetch, "https://mcp.test/rpc")( + "https://mcp.test/rpc", + { + method: "POST", + body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list" }), + }, + ); + + expect(response).toBe(expected); + expect(innerCalls).toBe(1); + }); + it("emits opt-in operation timing and a bounded shadow recommendation without changing responses (#7957)", async () => { let now = 0; const { wrap, stderr } = loadHelper( From 274b3376daf19f9fc91c68ab39ecf13846dd82ee Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Thu, 6 Aug 2026 12:20:28 -0700 Subject: [PATCH 4/5] docs(observability): clarify MCP diagnostic fields Signed-off-by: Senthil Ravichandran --- docs/reference/troubleshoot-mcp-servers.mdx | 2 +- docs/security/openclaw-2026.7.1-dependency-review.md | 4 ++-- scripts/patch-openclaw-managed-transport-diagnostics.mts | 4 ++-- src/lib/onboard/sandbox-create-launch.ts | 2 +- test/openclaw-dependency-review.test.ts | 2 +- test/openclaw-managed-transport-diagnostics-patch.test.ts | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/reference/troubleshoot-mcp-servers.mdx b/docs/reference/troubleshoot-mcp-servers.mdx index b34bbc46e54..f22fb005041 100644 --- a/docs/reference/troubleshoot-mcp-servers.mdx +++ b/docs/reference/troubleshoot-mcp-servers.mdx @@ -180,7 +180,7 @@ $$nemoclaw rebuild For a new sandbox, export the setting before `$$nemoclaw onboard` creates it. -When enabled, each successful request attempts to emit a `managed_transport_shadow` line group with the same safe operation, transport, timeout, and elapsed-time fields. +When enabled, each successful request attempts to emit a `managed_transport_shadow` line group with the same validated operation, transport, timeout, and elapsed-time fields. The diagnostic does not read the successful response body or change the returned response. Identifier generation, serialization, or standard-error output failure can omit the line group without blocking the request or changing its response. diff --git a/docs/security/openclaw-2026.7.1-dependency-review.md b/docs/security/openclaw-2026.7.1-dependency-review.md index 2ba8da33dfc..2915696dbf1 100644 --- a/docs/security/openclaw-2026.7.1-dependency-review.md +++ b/docs/security/openclaw-2026.7.1-dependency-review.md @@ -450,7 +450,7 @@ Reviewed behavior: - Failure-only by default. A 2xx response returns untouched and emits nothing unless the OpenClaw gateway process has `NEMOCLAW_MCP_SHADOW_DIAGNOSTICS=1`. -- The opt-in shadow mode attempts to emit timing-only `managed_transport_shadow` events for successful requests without reading their response bodies. +- The opt-in shadow mode attempts to emit successful-request `managed_transport_shadow` timing events without reading their response bodies. It does not change a timeout, retry a request, alter a response, or persist samples across an OpenClaw process restart. Identifier generation, serialization, or standard-error output failure can omit an event without blocking the request or changing its response. Sandbox creation forwards only the literal value `1`, and only for OpenClaw. @@ -512,7 +512,7 @@ The two patches compose independently. The injected helper in `scripts/patch-openclaw-managed-transport-diagnostics.mts` is the shipped runtime source of truth. `test/openclaw-managed-transport-diagnostics-patch.test.ts` executes that exact helper. It pins the compiled preimage, patch idempotence, fail-closed rejection of an unrecognized shape, and the untouched SSE boundary. -It also covers default failure-only emission, opt-in successful timing events, bounded shadow recommendations, explicit 503 exclusion, no-retry and unchanged-response contracts, safe operation reporting, asynchronous body sampling, byte and time bounds, redaction, the header allowlist, local diagnostic identifiers, session-presence reporting, transport-phase classification, route evidence, and sandbox gating. +It also covers default failure-only emission, opt-in successful timing events, bounded shadow recommendations, explicit 503 exclusion, no-retry and unchanged-response contracts, validated operation reporting, asynchronous body sampling, byte and time bounds, redaction, the header allowlist, local diagnostic identifiers, session-presence reporting, transport-phase classification, route evidence, and sandbox gating. A reusable source schema is deferred until a production consumer requires one. Removal criterion: drop this patch when the reviewed OpenClaw release emits redacted diagnostics classified by transport phase for remote MCP fetch failures. diff --git a/scripts/patch-openclaw-managed-transport-diagnostics.mts b/scripts/patch-openclaw-managed-transport-diagnostics.mts index 95708c9a2ba..e0963bdbaab 100755 --- a/scripts/patch-openclaw-managed-transport-diagnostics.mts +++ b/scripts/patch-openclaw-managed-transport-diagnostics.mts @@ -38,8 +38,8 @@ const REQUIRED_PATTERNS = [...UNPATCHED_TARGET_PATTERNS]; const PATCHED_REQUIRED_PATTERNS = [MARKER, STREAMABLE_TRANSPORT_REPLACEMENT]; /** - * Managed-transport failure diagnostics plus opt-in, timing-only success - * sampling for the remote Streamable HTTP MCP fetch boundary. The wrapper + * Managed-transport failure diagnostics plus opt-in success timing for the + * remote Streamable HTTP MCP fetch boundary. The wrapper * never retries, never alters the request, and never reads a 2xx body, so * streaming responses stay behaviorally unchanged. */ diff --git a/src/lib/onboard/sandbox-create-launch.ts b/src/lib/onboard/sandbox-create-launch.ts index 15a5eb36cfe..b7eeb3d0a5e 100644 --- a/src/lib/onboard/sandbox-create-launch.ts +++ b/src/lib/onboard/sandbox-create-launch.ts @@ -38,7 +38,7 @@ const OPENCLAW_AUTO_PAIR_RUNTIME_ENV_KEYS = [ "NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS", ] as const; -// This opt-in emits timing-only MCP success events from the reviewed OpenClaw +// This opt-in emits MCP success timing events from the reviewed OpenClaw // dist patch. Accept only the literal enabled value, and only for OpenClaw, so // the broader host environment never becomes sandbox runtime input. const OPENCLAW_DIAGNOSTIC_RUNTIME_ENV_KEYS = ["NEMOCLAW_MCP_SHADOW_DIAGNOSTICS"] as const; diff --git a/test/openclaw-dependency-review.test.ts b/test/openclaw-dependency-review.test.ts index 9e97fcb82bb..7638ef5bb9d 100644 --- a/test/openclaw-dependency-review.test.ts +++ b/test/openclaw-dependency-review.test.ts @@ -197,7 +197,7 @@ describe("OpenClaw 2026.6.10 dependency review contract", () => { expect(review).toContain("The sibling SSE transport boundary is deliberately left unwrapped."); expect(review).toContain("Failure-only by default."); expect(review).toContain("`NEMOCLAW_MCP_SHADOW_DIAGNOSTICS=1`"); - expect(review).toContain("timing-only `managed_transport_shadow` events"); + expect(review).toContain("successful-request `managed_transport_shadow` timing events"); expect(review).toContain("never retries, never alters the request"); expect(review).toContain("`route=proxy_configured` means that `HTTPS_PROXY`"); expect(review).toContain("report configuration evidence"); diff --git a/test/openclaw-managed-transport-diagnostics-patch.test.ts b/test/openclaw-managed-transport-diagnostics-patch.test.ts index 17164abfe84..e6eafa82e31 100644 --- a/test/openclaw-managed-transport-diagnostics-patch.test.ts +++ b/test/openclaw-managed-transport-diagnostics-patch.test.ts @@ -431,7 +431,7 @@ describe("injected managed transport wrapper", () => { expect(events[0].diagnostic_id).not.toBe(events[1].diagnostic_id); }); - it("logs only the safe RPC method and never tool arguments (#7957)", async () => { + it("logs only the validated RPC method and omits tool names and arguments (#7957)", async () => { const { wrap, stderr } = loadHelper({ OPENSHELL_SANDBOX: "1", NEMOCLAW_MCP_SHADOW_DIAGNOSTICS: "1", From b8edb24a2f1d70553bd11eb887e85bed61242153 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 6 Aug 2026 12:22:32 -0700 Subject: [PATCH 5/5] test(observability): prove diagnostic failure hooks Signed-off-by: Apurv Kumaria --- test/openclaw-managed-transport-diagnostics-patch.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/openclaw-managed-transport-diagnostics-patch.test.ts b/test/openclaw-managed-transport-diagnostics-patch.test.ts index e6eafa82e31..bacfef5b1fc 100644 --- a/test/openclaw-managed-transport-diagnostics-patch.test.ts +++ b/test/openclaw-managed-transport-diagnostics-patch.test.ts @@ -207,10 +207,12 @@ describe("injected managed transport wrapper", () => { }); it("sends the request when diagnostic identifier generation fails (#7957)", async () => { + let randomUuidCalls = 0; let innerCalls = 0; const { wrap, stderr } = loadHelper({ OPENSHELL_SANDBOX: "1" }, () => Date.now(), { crypto: { randomUUID: () => { + randomUuidCalls += 1; throw new Error("entropy unavailable"); }, }, @@ -227,10 +229,12 @@ describe("injected managed transport wrapper", () => { expect(response.status).toBe(200); expect(innerCalls).toBe(1); + expect(randomUuidCalls).toBeGreaterThan(0); expect(stderr).toEqual([]); }); it("returns a successful response when shadow diagnostic emission fails (#7957)", async () => { + let stderrWrites = 0; let innerCalls = 0; const { wrap } = loadHelper( { @@ -240,6 +244,7 @@ describe("injected managed transport wrapper", () => { () => Date.now(), { writeStderr: () => { + stderrWrites += 1; throw new Error("stderr unavailable"); }, }, @@ -260,6 +265,7 @@ describe("injected managed transport wrapper", () => { expect(response).toBe(expected); expect(innerCalls).toBe(1); + expect(stderrWrites).toBeGreaterThan(0); }); it("emits opt-in operation timing and a bounded shadow recommendation without changing responses (#7957)", async () => {