Improve Executor tool guidance

.changeset/calm-tools-guide.md

@@ -0,0 +1,5 @@
+---
+"executor": patch
+---
+
+Clarify execute skill loading and surface schema validation constraints in tool descriptions.

e2e/scenarios/tool-descriptions.test.ts

@@ -61,7 +61,7 @@ const ordersOpenApiSpec = (baseUrl: string): string =>
               in: "path",
               required: true,
               description: "Unique order identifier (ULID).",
-              schema: { type: "string" },
+              schema: { type: "string", minLength: 26, maxLength: 26 },
             },
             {
               name: "include",
@@ -384,12 +384,65 @@ scenario(
         const openapiTools = yield* snapshotFor(openapiSlug);
         const graphqlTools = yield* snapshotFor(graphqlSlug);
 
+        const session = mcp.session(identity);
+        const advertisedTools = yield* session.describeTools();
+        const executeDescription =
+          advertisedTools.find((tool) => tool.name === "execute")?.description ?? "";
+        expect(
+          executeDescription,
+          "the execute description directs models to the companion MCP tool",
+        ).toContain("companion `skills` MCP tool");
+        expect(
+          executeDescription,
+          "the execute description does not suggest a nonexistent sandbox function",
+        ).not.toContain("skills({");
+
+        const executeSkill = yield* session.call("skills", { name: "execute" });
+        expect(executeSkill.ok, "the execute guide is available through skills").toBe(true);
+        expect(executeSkill.text, "the guide tells models to inspect validation limits").toContain(
+          "inputConstraints",
+        );
+        expect(executeSkill.text, "the guide distinguishes nested MCP domain status").toContain(
+          "data.structuredContent",
+        );
+
+        const describedGetOrderSnapshot = openapiTools.find((tool) =>
+          String(tool.address).endsWith(".getOrder"),
+        );
+        expect(
+          describedGetOrderSnapshot,
+          "the constrained getOrder operation is present",
+        ).toBeDefined();
+        const getOrderPath = String(describedGetOrderSnapshot?.address ?? "").replace(
+          /^tools\./,
+          "",
+        );
+        const describedResult = yield* session.call("execute", {
+          code: `return await tools.describe.tool({ path: ${JSON.stringify(getOrderPath)} });`,
+        });
+        expect(describedResult.ok, `the sandbox describes getOrder: ${describedResult.text}`).toBe(
+          true,
+        );
+        const describedGetOrder = JSON.parse(describedResult.text) as {
+          readonly inputConstraints?: readonly {
+            readonly path: string;
+            readonly rules: readonly string[];
+          }[];
+        };
+        const orderIdConstraints = describedGetOrder.inputConstraints?.find((constraint) =>
+          constraint.path.endsWith("orderId"),
+        );
+        expect(
+          orderIdConstraints?.rules,
+          "describe.tool preserves the ULID length constraint TypeScript cannot express",
+        ).toEqual(["length >= 26", "length <= 26"]);
+
         // The execute tool's description over the real MCP surface — the
         // connected-integration inventory an MCP client (and its model) reads.
         // Only this run's lines: the shared selfhost admin may have other
         // integrations in the inventory.
         const readInventory = () =>
-          Effect.map(mcp.session(identity).describeTools(), (mcpTools) =>
+          Effect.map(session.describeTools(), (mcpTools) =>
             (mcpTools.find((tool) => tool.name === "execute")?.description ?? "")
               .split("## Available integrations")[1]
               ?.split("\n")

packages/core/execution/src/description.test.ts

@@ -99,7 +99,8 @@ describe("buildExecuteDescription", () => {
       expect(description).toContain("Execute TypeScript in a sandboxed runtime");
       // The full how-to now lives behind the `skills` tool, so the description
       // points there rather than inlining the workflow/rules.
-      expect(description).toContain('skills({ name: "execute" })');
+      expect(description).toContain("companion `skills` MCP tool");
+      expect(description).toContain("Do not call `skills` inside this sandbox");
       expect(description).not.toContain("Use `emit(value)` to append user-visible output");
       expect(description).not.toContain("## Workflow");
       expect(description).not.toContain("## Rules");

packages/core/execution/src/description.ts

@@ -29,7 +29,7 @@ export const buildExecuteDescription = (executor: Executor): Effect.Effect<strin
       const lines = [
         "Execute TypeScript in a sandboxed runtime.",
         "",
-        'Before writing code, call `skills({ name: "execute" })` for the workflow on how to use this tool.',
+        "Before writing code, fetch the `execute` guide with the companion `skills` MCP tool (name: `execute`). Do not call `skills` inside this sandbox; it is not available here.",
       ];
       const inventory = formatIntegrationInventory(connections);
       if (inventory.length > 0) {

packages/core/execution/src/schema-constraints.test.ts

@@ -0,0 +1,133 @@
+import { describe, expect, it } from "@effect/vitest";
+
+import { summarizeInputConstraints } from "./schema-constraints";
+
+describe("summarizeInputConstraints", () => {
+  it("surfaces numeric, collection, and per-item string limits", () => {
+    expect(
+      summarizeInputConstraints({
+        type: "object",
+        properties: {
+          max_events: { type: "integer", minimum: 1, maximum: 100 },
+          events: {
+            type: "array",
+            minItems: 1,
+            maxItems: 100,
+            uniqueItems: true,
+            items: { type: "string", minLength: 1, maxLength: 4000 },
+          },
+        },
+      }),
+    ).toEqual([
+      { path: "max_events", rules: ["value >= 1", "value <= 100"] },
+      { path: "events", rules: ["items >= 1", "items <= 100", "items unique"] },
+      { path: "events[]", rules: ["length >= 1", "length <= 4000"] },
+    ]);
+  });
+
+  it("follows local and separately stored definitions without recursing forever", () => {
+    expect(
+      summarizeInputConstraints(
+        {
+          type: "object",
+          properties: {
+            local: { $ref: "#/$defs/Local" },
+            shared: { $ref: "#/$defs/Shared" },
+          },
+          $defs: {
+            Local: { type: "string", pattern: "^[a-z]+$" },
+          },
+        },
+        {
+          Shared: {
+            type: "object",
+            maxProperties: 3,
+            properties: { child: { $ref: "#/$defs/Shared" } },
+          },
+        },
+      ),
+    ).toEqual([
+      { path: "local", rules: ['matches "^[a-z]+$"'] },
+      { path: "shared", rules: ["properties <= 3"] },
+      { path: "shared.child", rules: ["properties <= 3"] },
+    ]);
+  });
+
+  it("handles numeric and OpenAPI 3 boolean exclusive bounds", () => {
+    expect(
+      summarizeInputConstraints({
+        type: "object",
+        properties: {
+          openapi3: { type: "number", minimum: 0, exclusiveMinimum: true },
+          jsonSchema: { type: "number", exclusiveMaximum: 10 },
+          ordinary: { type: "number", minimum: 0, maximum: 10 },
+        },
+      }),
+    ).toEqual([
+      { path: "openapi3", rules: ["value > 0"] },
+      { path: "jsonSchema", rules: ["value < 10"] },
+      { path: "ordinary", rules: ["value >= 0", "value <= 10"] },
+    ]);
+  });
+
+  it("collects allOf rules but does not conjoin anyOf or oneOf branches", () => {
+    expect(
+      summarizeInputConstraints({
+        type: "object",
+        properties: {
+          conjunctive: { allOf: [{ minimum: 1 }, { maximum: 10 }] },
+          alternative: {
+            oneOf: [
+              { minimum: 1, maximum: 10 },
+              { minimum: 100, maximum: 200 },
+            ],
+          },
+          nullable: { anyOf: [{ type: "string", maxLength: 50 }, { type: "null" }] },
+        },
+      }),
+    ).toEqual([{ path: "conjunctive", rules: ["value >= 1", "value <= 10"] }]);
+  });
+
+  it("supports modern and draft-4 tuple item schemas", () => {
+    expect(
+      summarizeInputConstraints({
+        type: "object",
+        properties: {
+          modern: { type: "array", prefixItems: [{ maxLength: 10 }, { maximum: 5 }] },
+          legacy: { type: "array", items: [{ minLength: 2 }, { minimum: 1 }] },
+        },
+      }),
+    ).toEqual([
+      { path: "modern[0]", rules: ["length <= 10"] },
+      { path: "modern[1]", rules: ["value <= 5"] },
+      { path: "legacy[0]", rules: ["length >= 2"] },
+      { path: "legacy[1]", rules: ["value >= 1"] },
+    ]);
+  });
+
+  it("labels root constraints and avoids no-op minimums", () => {
+    expect(
+      summarizeInputConstraints({
+        type: "array",
+        minItems: 0,
+        maxItems: 20,
+        items: { type: "string", minLength: 0, format: "email" },
+      }),
+    ).toEqual([
+      { path: "(root)", rules: ["items <= 20"] },
+      { path: "(root)[]", rules: ["format email"] },
+    ]);
+  });
+
+  it("does not guess a flat definition for a deeper unresolved pointer", () => {
+    expect(
+      summarizeInputConstraints(
+        {
+          type: "object",
+          properties: { name: { $ref: "#/$defs/Pet/properties/name" } },
+        },
+        { name: { type: "string", maxLength: 10 } },
+      ),
+    ).toEqual([]);
+  });
+});

packages/core/execution/src/schema-constraints.ts

@@ -0,0 +1,159 @@
+type JsonObject = Readonly<Record<string, unknown>>;
+
+export type ToolInputConstraint = {
+  readonly path: string;
+  readonly rules: readonly string[];
+};
+
+const isJsonObject = (value: unknown): value is JsonObject =>
+  typeof value === "object" && value !== null && !Array.isArray(value);
+
+const finiteNumber = (value: unknown): number | undefined =>
+  typeof value === "number" && Number.isFinite(value) ? value : undefined;
+
+const nonNegativeInteger = (value: unknown): number | undefined =>
+  typeof value === "number" && Number.isInteger(value) && value >= 0 ? value : undefined;
+
+const stringValue = (value: unknown): string | undefined =>
+  typeof value === "string" && value.length > 0 ? value : undefined;
+
+const rulesForSchema = (schema: JsonObject): readonly string[] => {
+  const rules: string[] = [];
+  const minimum = finiteNumber(schema.minimum);
+  const exclusiveMinimum = finiteNumber(schema.exclusiveMinimum);
+  const maximum = finiteNumber(schema.maximum);
+  const exclusiveMaximum = finiteNumber(schema.exclusiveMaximum);
+  const multipleOf = finiteNumber(schema.multipleOf);
+  const minLength = nonNegativeInteger(schema.minLength);
+  const maxLength = nonNegativeInteger(schema.maxLength);
+  const minItems = nonNegativeInteger(schema.minItems);
+  const maxItems = nonNegativeInteger(schema.maxItems);
+  const minProperties = nonNegativeInteger(schema.minProperties);
+  const maxProperties = nonNegativeInteger(schema.maxProperties);
+  const pattern = stringValue(schema.pattern);
+  const format = stringValue(schema.format);
+
+  if (exclusiveMinimum !== undefined) {
+    rules.push(`value > ${exclusiveMinimum}`);
+  } else if (minimum !== undefined) {
+    rules.push(`${schema.exclusiveMinimum === true ? "value >" : "value >="} ${minimum}`);
+  }
+  if (exclusiveMaximum !== undefined) {
+    rules.push(`value < ${exclusiveMaximum}`);
+  } else if (maximum !== undefined) {
+    rules.push(`${schema.exclusiveMaximum === true ? "value <" : "value <="} ${maximum}`);
+  }
+  if (multipleOf !== undefined) rules.push(`multiple of ${multipleOf}`);
+  if (minLength !== undefined && minLength > 0) rules.push(`length >= ${minLength}`);
+  if (maxLength !== undefined) rules.push(`length <= ${maxLength}`);
+  if (minItems !== undefined && minItems > 0) rules.push(`items >= ${minItems}`);
+  if (maxItems !== undefined) rules.push(`items <= ${maxItems}`);
+  if (schema.uniqueItems === true) rules.push("items unique");
+  if (minProperties !== undefined && minProperties > 0)
+    rules.push(`properties >= ${minProperties}`);
+  if (maxProperties !== undefined) rules.push(`properties <= ${maxProperties}`);
+  if (pattern !== undefined) rules.push(`matches ${JSON.stringify(pattern)}`);
+  if (format !== undefined) rules.push(`format ${format}`);
+
+  return rules;
+};
+
+const decodeJsonPointerSegment = (segment: string): string =>
+  segment.replaceAll("~1", "/").replaceAll("~0", "~");
+
+const resolveReference = (
+  reference: string,
+  root: JsonObject,
+  definitions: Readonly<Record<string, unknown>>,
+): unknown => {
+  if (!reference.startsWith("#/")) return undefined;
+  const segments = reference.slice(2).split("/").map(decodeJsonPointerSegment);
+  let current: unknown = root;
+  for (const segment of segments) {
+    if (!isJsonObject(current) || !(segment in current)) {
+      current = undefined;
+      break;
+    }
+    current = current[segment];
+  }
+  if (current !== undefined) return current;
+
+  // executor.tools.schema() stores referenced definitions separately from the
+  // input root. Only fall back for the exact flat definition shape it exposes;
+  // guessing from the final segment of a deeper pointer can return a different
+  // schema and publish incorrect constraints.
+  const flatDefinition = /^#\/(?:\$defs|definitions)\/([^/]+)$/.exec(reference);
+  return flatDefinition === null
+    ? undefined
+    : definitions[decodeJsonPointerSegment(flatDefinition[1] ?? "")];
+};
+
+/**
+ * Summarize the validation keywords that TypeScript cannot express.
+ *
+ * The result stays intentionally compact: callers still use the TypeScript
+ * preview for shape and only consult this list for numeric, collection, and
+ * string constraints that would otherwise be invisible.
+ */
+export const summarizeInputConstraints = (
+  inputSchema: unknown,
+  schemaDefinitions: Readonly<Record<string, unknown>> = {},
+): readonly ToolInputConstraint[] => {
+  if (!isJsonObject(inputSchema)) return [];
+
+  const byPath = new Map<string, Set<string>>();
+  const activeReferences = new Set<string>();
+
+  const addRules = (path: string, rules: readonly string[]): void => {
+    if (rules.length === 0) return;
+    const existing = byPath.get(path) ?? new Set<string>();
+    for (const rule of rules) existing.add(rule);
+    byPath.set(path, existing);
+  };
+
+  const visit = (value: unknown, path: string): void => {
+    if (!isJsonObject(value)) return;
+
+    const reference = stringValue(value.$ref);
+    if (reference !== undefined && !activeReferences.has(reference)) {
+      const resolved = resolveReference(reference, inputSchema, schemaDefinitions);
+      if (resolved !== undefined) {
+        activeReferences.add(reference);
+        visit(resolved, path);
+        activeReferences.delete(reference);
+      }
+    } else if (reference !== undefined) {
+      const resolved = resolveReference(reference, inputSchema, schemaDefinitions);
+      if (isJsonObject(resolved)) addRules(path, rulesForSchema(resolved));
+    }
+
+    addRules(path, rulesForSchema(value));
+
+    if (isJsonObject(value.properties)) {
+      for (const [name, child] of Object.entries(value.properties)) {
+        visit(child, path === "(root)" ? name : `${path}.${name}`);
+      }
+    }
+
+    if (Array.isArray(value.prefixItems)) {
+      value.prefixItems.forEach((child, index) => visit(child, `${path}[${index}]`));
+    }
+    if (Array.isArray(value.items)) {
+      value.items.forEach((child, index) => visit(child, `${path}[${index}]`));
+    } else if (isJsonObject(value.items)) {
+      visit(value.items, `${path}[]`);
+    }
+    if (isJsonObject(value.additionalProperties)) visit(value.additionalProperties, `${path}.*`);
+
+    // allOf constraints are conjunctive. anyOf/oneOf constraints are not: a
+    // flat list would turn alternatives into an impossible conjunction, so we
+    // intentionally omit branch-local rules until the contract can represent
+    // per-branch groups.
+    if (Array.isArray(value.allOf)) {
+      for (const variant of value.allOf) visit(variant, path);
+    }
+  };
+
+  visit(inputSchema, "(root)");
+  return [...byPath.entries()].map(([path, rules]) => ({ path, rules: [...rules] }));
+};