From a64d7ea467bfa58d6ab57ad466a347f71ceb59b9 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Wed, 3 Jun 2026 17:17:21 -0700 Subject: [PATCH 01/12] chore: add SEP-2640 requirement-traceability YAML (Skills Extension) --- src/seps/sep-2640.yaml | 75 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 src/seps/sep-2640.yaml diff --git a/src/seps/sep-2640.yaml b/src/seps/sep-2640.yaml new file mode 100644 index 00000000..e54ce694 --- /dev/null +++ b/src/seps/sep-2640.yaml @@ -0,0 +1,75 @@ +sep: 2640 +spec_url: https://modelcontextprotocol.io/seps/2640-skills-extension#specification +requirements: + - check: sep-2640-skillmd-required + text: 'Every skill MUST contain a `SKILL.md` file at its root.' + - check: sep-2640-skillmd-frontmatter + text: '`SKILL.md` MUST begin with YAML frontmatter containing at minimum the `name` and `description` fields as defined by the Agent Skills specification.' + - check: sep-2640-skill-uri-scheme + text: 'Each file within a skill directory is exposed as an MCP resource. Servers SHOULD use the `skill://` URI scheme, under which the resource URI has the form: `skill:///`' + - check: sep-2640-final-segment-equals-name + text: 'The final segment of `` MUST equal the skill''s `name` as declared in its `SKILL.md` frontmatter.' + - check: sep-2640-no-nested-skills + text: 'A `SKILL.md` MUST NOT appear in any descendant directory of a skill. The skill directory is the boundary; skills do not nest inside other skills.' + - check: sep-2640-name-naming-rules + text: 'The final `` segment, being the skill `name`, MUST satisfy the Agent Skills specification''s naming rules.' + - check: sep-2640-prefix-rfc3986 + text: 'Prefix segments SHOULD be valid URI path segments per RFC 3986; no further constraints are imposed on them.' + - check: sep-2640-skillmd-mimetype + text: 'For each `skill:///SKILL.md` resource: `mimeType` SHOULD be `text/markdown`.' + - check: sep-2640-skillmd-metadata-name + text: 'For each `skill:///SKILL.md` resource: `name` SHOULD be set from the `name` field of the `SKILL.md` YAML frontmatter. By the path constraint above, this will equal the final segment of ``.' + - check: sep-2640-skillmd-metadata-description + text: 'For each `skill:///SKILL.md` resource: `description` SHOULD be set from the `description` field of the `SKILL.md` YAML frontmatter.' + - check: sep-2640-meta-prefix + text: 'When `_meta` keys are used for skill resources, implementations SHOULD use the `io.modelcontextprotocol.skills/` reverse-domain prefix.' + - check: sep-2640-host-load-by-uri + text: 'hosts MUST support loading a skill given only its URI' + - check: sep-2640-server-expose-index + text: 'A server SHOULD expose a resource at the well-known URI `skill://index.json` whose content is a JSON index of the skills it serves.' + - check: sep-2640-index-entry-type-enum + text: '`skills[].type` MUST be `"skill-md"`, `"archive"`, or `"mcp-resource-template"`.' + - check: sep-2640-index-name-required + text: '`skills[].name` is Required for `"skill-md"` and `"archive"`; matches the `SKILL.md` frontmatter `name` and the final segment of the skill path. Omitted for `"mcp-resource-template"`.' + - check: sep-2640-index-digest-required + text: '`skills[].digest` is Required for `"skill-md"` and `"archive"`: SHA-256 content digest of the artifact, formatted as `sha256:{hex}` (64 lowercase hexadecimal characters). Omitted for `"mcp-resource-template"`.' + - check: sep-2640-client-ignore-unrecognized + text: 'Clients SHOULD ignore unrecognized fields and SHOULD skip entries with an unrecognized `type`.' + - check: sep-2640-archive-format + text: 'the archive MUST be `.tar.gz` (gzip-compressed tar, `mimeType` `application/gzip`) or `.zip` (`mimeType` `application/zip`)' + - check: sep-2640-host-support-archive-formats + text: 'hosts MUST support both `.tar.gz` and `.zip` archive formats' + - check: sep-2640-archive-skillmd-at-root + text: 'Archive contents represent the skill directory directly — `SKILL.md` MUST be at the archive root, not nested inside a wrapper directory' + - check: sep-2640-archive-no-traversal + text: 'the archive MUST NOT contain path-traversal sequences (`..`) or absolute paths' + - check: sep-2640-host-archive-safety + text: 'Hosts unpacking an archive MUST apply the archive safety requirements of the Agent Skills specification: reject archives containing path-traversal sequences or absolute paths, reject symlinks or hard links that resolve outside the skill directory, and enforce a limit on total unpacked size / Hosts MUST validate archives per the Agent Skills archive safety requirements: reject path traversal and absolute paths, reject links resolving outside the skill directory, and bound total unpacked size to prevent decompression bombs.' + - check: sep-2640-template-resource-template-registered + text: 'A server SHOULD register the same `url` value as an MCP resource template so hosts can wire template variables to the completion API.' + - check: sep-2640-host-verify-digest + text: 'Hosts MUST verify retrieved content against the `digest` in the index / hosts MUST NOT use unverified content.' + - check: sep-2640-host-no-empty-index-assumption + text: 'Hosts MUST NOT treat an absent or empty index as proof that a server has no skills.' + + - text: 'Hosts SHOULD surface template entries in their UI as interactive discovery points: the user fills in variables via completion, selects a skill, and the host passes the resolved URI into the conversation.' + excluded: 'UI affordance — surfacing template entries as interactive discovery points is not protocol-observable on the wire.' + - text: 'Per RFC 3986, the first segment of `` occupies the authority component. This carries no special semantics under this convention and clients MUST NOT attempt DNS or network resolution of it.' + excluded: 'DNS and network resolution sit below the MCP wire layer; the harness cannot observe whether the client performed name lookups on URI authority components.' + - text: '[Hosts] SHOULD determine the format from the resource''s `mimeType`, falling back to the URL suffix' + excluded: 'Internal decision logic — when `mimeType` and URL suffix agree, the harness cannot distinguish a host that branched on `mimeType` from one that fell back to the suffix.' + - text: 'Hosts MUST treat MCP-served skill content as untrusted model input, subject to the same prompt-injection defenses applied to any server-provided text. A server being connected does not make its skill content authoritative.' + excluded: 'Internal host policy — "treats as untrusted" is an assertion about how content is reasoned over downstream of the read, not about wire traffic.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - text: 'Hosts MUST NOT honor mechanisms in skill content that would cause local code execution without explicit user opt-in. This includes, non-exhaustively: hook declarations, pre/post-invocation scripts, shell commands embedded in frontmatter, or any field that a filesystem-sourced skill might use to register executable behavior on the host.' + excluded: 'Local code execution and explicit user opt-in are host-side filesystem and UX behaviors; not protocol-observable on the wire.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - text: 'Hosts MUST either ignore such fields entirely when the skill arrives over MCP, or gate them behind an explicit per-skill user approval that states what will execute and where.' + excluded: 'Either branch (silent ignore vs. UI-gated approval) is a host-internal handling choice; not protocol-observable.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - text: 'Hosts MUST NOT treat skill resources as higher-authority than other context. Explicit user policy governs whether a skill is loaded at all.' + excluded: 'Context-authority ordering is an internal prompting decision; not protocol-observable.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - text: 'Hosts SHOULD indicate which server a skill originates from when presenting it, SHOULD let users inspect a skill''s content before it is loaded into model context' + excluded: 'UI presentation requirements (origin indicator, pre-load inspection); the harness cannot observe what the host displays to users.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications From 2ecad130118675241c20e0ad66530bca2862a1e2 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Wed, 3 Jun 2026 17:21:32 -0700 Subject: [PATCH 02/12] style: apply yaml formatter pass --- src/seps/sep-2640.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/seps/sep-2640.yaml b/src/seps/sep-2640.yaml index e54ce694..419aa401 100644 --- a/src/seps/sep-2640.yaml +++ b/src/seps/sep-2640.yaml @@ -8,11 +8,11 @@ requirements: - check: sep-2640-skill-uri-scheme text: 'Each file within a skill directory is exposed as an MCP resource. Servers SHOULD use the `skill://` URI scheme, under which the resource URI has the form: `skill:///`' - check: sep-2640-final-segment-equals-name - text: 'The final segment of `` MUST equal the skill''s `name` as declared in its `SKILL.md` frontmatter.' + text: "The final segment of `` MUST equal the skill's `name` as declared in its `SKILL.md` frontmatter." - check: sep-2640-no-nested-skills text: 'A `SKILL.md` MUST NOT appear in any descendant directory of a skill. The skill directory is the boundary; skills do not nest inside other skills.' - check: sep-2640-name-naming-rules - text: 'The final `` segment, being the skill `name`, MUST satisfy the Agent Skills specification''s naming rules.' + text: "The final `` segment, being the skill `name`, MUST satisfy the Agent Skills specification's naming rules." - check: sep-2640-prefix-rfc3986 text: 'Prefix segments SHOULD be valid URI path segments per RFC 3986; no further constraints are imposed on them.' - check: sep-2640-skillmd-mimetype @@ -56,7 +56,7 @@ requirements: excluded: 'UI affordance — surfacing template entries as interactive discovery points is not protocol-observable on the wire.' - text: 'Per RFC 3986, the first segment of `` occupies the authority component. This carries no special semantics under this convention and clients MUST NOT attempt DNS or network resolution of it.' excluded: 'DNS and network resolution sit below the MCP wire layer; the harness cannot observe whether the client performed name lookups on URI authority components.' - - text: '[Hosts] SHOULD determine the format from the resource''s `mimeType`, falling back to the URL suffix' + - text: "[Hosts] SHOULD determine the format from the resource's `mimeType`, falling back to the URL suffix" excluded: 'Internal decision logic — when `mimeType` and URL suffix agree, the harness cannot distinguish a host that branched on `mimeType` from one that fell back to the suffix.' - text: 'Hosts MUST treat MCP-served skill content as untrusted model input, subject to the same prompt-injection defenses applied to any server-provided text. A server being connected does not make its skill content authoritative.' excluded: 'Internal host policy — "treats as untrusted" is an assertion about how content is reasoned over downstream of the read, not about wire traffic.' @@ -70,6 +70,6 @@ requirements: - text: 'Hosts MUST NOT treat skill resources as higher-authority than other context. Explicit user policy governs whether a skill is loaded at all.' excluded: 'Context-authority ordering is an internal prompting decision; not protocol-observable.' url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications - - text: 'Hosts SHOULD indicate which server a skill originates from when presenting it, SHOULD let users inspect a skill''s content before it is loaded into model context' + - text: "Hosts SHOULD indicate which server a skill originates from when presenting it, SHOULD let users inspect a skill's content before it is loaded into model context" excluded: 'UI presentation requirements (origin indicator, pre-load inspection); the harness cannot observe what the host displays to users.' url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications From 4fa8b101cc79ca69cc9376866449f7612ff045ff Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Wed, 3 Jun 2026 17:35:30 -0700 Subject: [PATCH 03/12] chore(sep-2640): record spec source provenance --- src/seps/sep-2640.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/seps/sep-2640.yaml b/src/seps/sep-2640.yaml index 419aa401..1001dc92 100644 --- a/src/seps/sep-2640.yaml +++ b/src/seps/sep-2640.yaml @@ -1,3 +1,5 @@ +# spec_source: modelcontextprotocol/modelcontextprotocol@b77fdfe8c6fa91442900c52357711978617ce18a seps/2640-skills-extension.md +# extracted: 2026-06-03 sep: 2640 spec_url: https://modelcontextprotocol.io/seps/2640-skills-extension#specification requirements: @@ -53,13 +55,13 @@ requirements: text: 'Hosts MUST NOT treat an absent or empty index as proof that a server has no skills.' - text: 'Hosts SHOULD surface template entries in their UI as interactive discovery points: the user fills in variables via completion, selects a skill, and the host passes the resolved URI into the conversation.' - excluded: 'UI affordance — surfacing template entries as interactive discovery points is not protocol-observable on the wire.' + excluded: 'UI affordance: surfacing template entries as interactive discovery points is not protocol-observable on the wire.' - text: 'Per RFC 3986, the first segment of `` occupies the authority component. This carries no special semantics under this convention and clients MUST NOT attempt DNS or network resolution of it.' excluded: 'DNS and network resolution sit below the MCP wire layer; the harness cannot observe whether the client performed name lookups on URI authority components.' - text: "[Hosts] SHOULD determine the format from the resource's `mimeType`, falling back to the URL suffix" - excluded: 'Internal decision logic — when `mimeType` and URL suffix agree, the harness cannot distinguish a host that branched on `mimeType` from one that fell back to the suffix.' + excluded: 'Internal decision logic: when `mimeType` and URL suffix agree, the harness cannot distinguish a host that branched on `mimeType` from one that fell back to the suffix.' - text: 'Hosts MUST treat MCP-served skill content as untrusted model input, subject to the same prompt-injection defenses applied to any server-provided text. A server being connected does not make its skill content authoritative.' - excluded: 'Internal host policy — "treats as untrusted" is an assertion about how content is reasoned over downstream of the read, not about wire traffic.' + excluded: 'Internal host policy: "treats as untrusted" is an assertion about how content is reasoned over downstream of the read, not about wire traffic.' url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications - text: 'Hosts MUST NOT honor mechanisms in skill content that would cause local code execution without explicit user opt-in. This includes, non-exhaustively: hook declarations, pre/post-invocation scripts, shell commands embedded in frontmatter, or any field that a filesystem-sourced skill might use to register executable behavior on the host.' excluded: 'Local code execution and explicit user opt-in are host-side filesystem and UX behaviors; not protocol-observable on the wire.' From 1e50cadbb39346979c79318b08f99767c8ff2656 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Fri, 5 Jun 2026 12:05:09 -0700 Subject: [PATCH 04/12] chore(sep-2640): re-extract against SEP HEAD 556154c (drops mcp-resource-template) Re-extract sep-2640.yaml against current SEP-2640 HEAD. The SEP removed the `mcp-resource-template` index entry type in two commits on 2026-06-04 (fd50cc91 "Remove mcp-resource-template entries from skill index", 556154c0 "Remove remaining resource template mentions from SDK and rationale sections"), after this extraction was first captured on 2026-06-03 at b77fdfe8. Provenance comment bumped accordingly. Resulting changes: - sep-2640-index-entry-type-enum: drop `"mcp-resource-template"` from the enum. - sep-2640-index-name-required: drop the "is Required for ... Omitted for mcp-resource-template" conditional language; the column is now unconditional Required=Yes at SEP HEAD. - sep-2640-index-digest-required: same simplification; also align "hexadecimal" -> "hex" with the HEAD table text. - sep-2640-template-resource-template-registered: removed. The SEP no longer defines a template entry type, so the SHOULD on registering an MCP resource template is gone. - Excluded "Hosts SHOULD surface template entries..." excerpt: removed. The sentence no longer exists in the SEP. Sweep verified: zero "template" mentions in the SEP at HEAD. Security Implications section sentences referenced by the remaining excluded entries are intact. No other check rows touched. --- src/seps/sep-2640.yaml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/seps/sep-2640.yaml b/src/seps/sep-2640.yaml index 1001dc92..ca93a62c 100644 --- a/src/seps/sep-2640.yaml +++ b/src/seps/sep-2640.yaml @@ -1,5 +1,5 @@ -# spec_source: modelcontextprotocol/modelcontextprotocol@b77fdfe8c6fa91442900c52357711978617ce18a seps/2640-skills-extension.md -# extracted: 2026-06-03 +# spec_source: modelcontextprotocol/modelcontextprotocol@556154c088371149c120172e95bb634655f00cbe seps/2640-skills-extension.md +# extracted: 2026-06-05 sep: 2640 spec_url: https://modelcontextprotocol.io/seps/2640-skills-extension#specification requirements: @@ -30,11 +30,11 @@ requirements: - check: sep-2640-server-expose-index text: 'A server SHOULD expose a resource at the well-known URI `skill://index.json` whose content is a JSON index of the skills it serves.' - check: sep-2640-index-entry-type-enum - text: '`skills[].type` MUST be `"skill-md"`, `"archive"`, or `"mcp-resource-template"`.' + text: '`skills[].type` MUST be `"skill-md"` or `"archive"`.' - check: sep-2640-index-name-required - text: '`skills[].name` is Required for `"skill-md"` and `"archive"`; matches the `SKILL.md` frontmatter `name` and the final segment of the skill path. Omitted for `"mcp-resource-template"`.' + text: '`skills[].name` matches the `SKILL.md` frontmatter `name` and the final segment of the skill path.' - check: sep-2640-index-digest-required - text: '`skills[].digest` is Required for `"skill-md"` and `"archive"`: SHA-256 content digest of the artifact, formatted as `sha256:{hex}` (64 lowercase hexadecimal characters). Omitted for `"mcp-resource-template"`.' + text: '`skills[].digest` is the SHA-256 content digest of the artifact, formatted as `sha256:{hex}` (64 lowercase hex characters).' - check: sep-2640-client-ignore-unrecognized text: 'Clients SHOULD ignore unrecognized fields and SHOULD skip entries with an unrecognized `type`.' - check: sep-2640-archive-format @@ -47,15 +47,11 @@ requirements: text: 'the archive MUST NOT contain path-traversal sequences (`..`) or absolute paths' - check: sep-2640-host-archive-safety text: 'Hosts unpacking an archive MUST apply the archive safety requirements of the Agent Skills specification: reject archives containing path-traversal sequences or absolute paths, reject symlinks or hard links that resolve outside the skill directory, and enforce a limit on total unpacked size / Hosts MUST validate archives per the Agent Skills archive safety requirements: reject path traversal and absolute paths, reject links resolving outside the skill directory, and bound total unpacked size to prevent decompression bombs.' - - check: sep-2640-template-resource-template-registered - text: 'A server SHOULD register the same `url` value as an MCP resource template so hosts can wire template variables to the completion API.' - check: sep-2640-host-verify-digest text: 'Hosts MUST verify retrieved content against the `digest` in the index / hosts MUST NOT use unverified content.' - check: sep-2640-host-no-empty-index-assumption text: 'Hosts MUST NOT treat an absent or empty index as proof that a server has no skills.' - - text: 'Hosts SHOULD surface template entries in their UI as interactive discovery points: the user fills in variables via completion, selects a skill, and the host passes the resolved URI into the conversation.' - excluded: 'UI affordance: surfacing template entries as interactive discovery points is not protocol-observable on the wire.' - text: 'Per RFC 3986, the first segment of `` occupies the authority component. This carries no special semantics under this convention and clients MUST NOT attempt DNS or network resolution of it.' excluded: 'DNS and network resolution sit below the MCP wire layer; the harness cannot observe whether the client performed name lookups on URI authority components.' - text: "[Hosts] SHOULD determine the format from the resource's `mimeType`, falling back to the URL suffix" From f1b069559d807a13a204ddf8e8e5e6e9485562f2 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Tue, 16 Jun 2026 12:05:57 -0700 Subject: [PATCH 05/12] feat(sep-2640): ResourcesDirectoryReadScenario for resources/directory/read Adds the conformance scenario for the SEP-2640 directoryRead surface that landed in spec commit 2e04c48d (2026-06-09). Per AGENTS.md "fewer scenarios, more checks", a single ResourcesDirectoryReadScenario emits 6 ConformanceChecks, one per new sep-2640.yaml requirement row. Class named for the wire method (resources/directory/read), matching the existing ResourcesListScenario / ResourcesReadTextScenario / etc. family in src/scenarios/server/resources.ts. The runner-facing name field stays as 'sep-2640-skills' (umbrella) so mcpkit's conformance/Makefile entry --scenario sep-2640-skills keeps working without a cross-repo race. src/seps/sep-2640.yaml - File-level provenance held at 556154c (the existing PR 330 baseline). - 6 new check rows appended for the directoryRead additions, each carrying a verbatim, grep-F-searchable excerpt from the SEP at 2e04c48da90224000e750ffd54a3611f2824fbc0: - sep-2640-capability-directory-read-flag - sep-2640-directory-read-method-registered - sep-2640-directory-read-subdir-mimetype - sep-2640-directory-read-result-resources-shape - sep-2640-directory-read-invalid-params - sep-2640-directory-read-pagination - forward_reference header comment notes the asymmetry: PR 97 schema rewrite (360123d0, 2026-06-08) made 3 existing rows stale and drifted ~11 others' wording; full re-extraction at SEP HEAD is mcpkit#780's lifecycle. Provenance deliberately holds at 556154c until that lands. src/types.ts - Adds io.modelcontextprotocol/skills to EXTENSION_IDS so the scenario's source: { extensionId: ... } tag type-checks. src/scenarios/server/directory.ts (new) - Capability discovery via wire-observable signal: -32601 method-not-found is the only definitive "server did not declare directoryRead" signal. - Fixture assumption: server exposes skill://acme/billing/refunds/templates with at least one subdirectory child (mcpkit examples/skills layout). No skill:// resources at all -> every check emits SKIPPED so the scenario stays green against upstream's everything-server fixture. - 6 checks (1:1 with the YAML rows above): 1. directoryRead declared (derived from method registration) 2. method registered (happy-path call succeeds) 3. result.resources shape matches resources/list 4. subdir child carries mimeType: "inode/directory" 5. non-directory URI returns -32602 6. nextCursor round-trips (single-page is conformant) src/scenarios/index.ts - Imports + registers ResourcesDirectoryReadScenario in allClientScenariosList, matching the registration name 'sep-2640-skills' that mcpkit's conformance/Makefile already passes to --scenario. Verified end-to-end against mcpkit examples/skills: cd ~/newstack/mcpkit/main && \ MCPCONFORMANCE_SKILLS_PATH=~/newstack/mcpkit/conf-skills \ make -C conformance testconf-skills -> 6/6 SUCCESS, 0 failed, 0 warnings. Out of scope: - PR 97 schema rewrite YAML refresh (separate ticket, mcpkit#780 lifecycle). - Negative-capability fixture (server without directoryRead) - mcpkit examples/skills has no flag for it; ext/skills/client_directory_test.go already covers the SDK pre-call guard. --- src/scenarios/index.ts | 5 + src/scenarios/server/directory.ts | 289 ++++++++++++++++++++++++++++++ src/seps/sep-2640.yaml | 24 +++ src/types.ts | 3 +- 4 files changed, 320 insertions(+), 1 deletion(-) create mode 100644 src/scenarios/server/directory.ts diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index b7230ab8..71da650b 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -57,6 +57,8 @@ import { ResourcesNotFoundErrorScenario } from './server/resources'; +import { ResourcesDirectoryReadScenario } from './server/directory'; + import { PromptsListScenario, PromptsGetSimpleScenario, @@ -173,6 +175,9 @@ const allClientScenariosList: ClientScenario[] = [ // Resources error handling (SEP-2164) new ResourcesNotFoundErrorScenario(), + // Skills extension (SEP-2640) — resources/directory/read surface (2e04c48d) + new ResourcesDirectoryReadScenario(), + // Prompts scenarios new PromptsListScenario(), new PromptsGetSimpleScenario(), diff --git a/src/scenarios/server/directory.ts b/src/scenarios/server/directory.ts new file mode 100644 index 00000000..2db1f89d --- /dev/null +++ b/src/scenarios/server/directory.ts @@ -0,0 +1,289 @@ +/** + * SEP-2640 Skills extension scenarios — focused on the resources/directory/read + * surface added in spec commit 2e04c48d (2026-06-09). + * + * One scenario, six checks (per AGENTS.md "fewer scenarios, more checks"). + * Each check's verbatim spec quote lives next to its check ID in + * src/seps/sep-2640.yaml, so the YAML and the scenario stay in lock-step. + * + * Capability discovery: the SEP allows multiple shapes for declaring the + * extension; the wire-observable signal we can rely on is whether + * resources/directory/read responds at all. A -32601 method-not-found is the + * only definitive "server didn't declare directoryRead" signal; any other + * response (success, -32602, etc.) means the server registered the method, + * which the SEP requires of any server that declared the capability. + * + * Fixture assumption: the scenario expects the standard mcpkit examples/skills + * fixture which exposes skill://acme/billing/refunds with a templates/ + * subtree containing at least one subdirectory. When the connected server is + * not a skills server (no skill:// resources at all), every check is emitted + * as SKIPPED — keeps the scenario green against the upstream everything-server + * while emitting real verdicts against any skills-capable fixture. + */ + +import { ClientScenario, ConformanceCheck } from '../../types'; +import { JsonRpcError, type RunContext } from '../../connection'; +import type { ListResourcesResult } from '../../spec-types/2025-06-18'; + +interface ResourceLike { + uri: string; + name?: string; + mimeType?: string; +} + +interface DirectoryReadResult { + resources?: ResourceLike[]; + nextCursor?: string; +} + +const SEP_2640_URL = + 'https://modelcontextprotocol.io/seps/2640-skills-extension#directory-listing'; + +const HAPPY_PATH_URI = 'skill://acme/billing/refunds/templates'; +const NON_DIRECTORY_URI = 'skill://acme/billing/refunds/SKILL.md'; + +const SPEC_REFERENCE = [ + { + id: 'SEP-2640-directory-listing', + url: SEP_2640_URL + } +]; + +const JSONRPC_METHOD_NOT_FOUND = -32601; +const JSONRPC_INVALID_PARAMS = -32602; + +function check( + id: string, + description: string, + status: 'SUCCESS' | 'FAILURE' | 'SKIPPED', + extras: Partial = {} +): ConformanceCheck { + return { + id, + name: id, + description, + status, + timestamp: new Date().toISOString(), + specReferences: SPEC_REFERENCE, + ...extras + }; +} + +export class ResourcesDirectoryReadScenario implements ClientScenario { + name = 'sep-2640-skills'; + readonly source = { + extensionId: 'io.modelcontextprotocol/skills' + } as const; + description = `SEP-2640 Skills extension: resources/directory/read surface (added in spec commit 2e04c48d, 2026-06-09). + +**Endpoint**: \`resources/directory/read\` (gated by \`io.modelcontextprotocol/skills.directoryRead: true\`) + +**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-2640.yaml): + +- \`sep-2640-capability-directory-read-flag\` — server effectively declared directoryRead +- \`sep-2640-directory-read-method-registered\` — method registered for served skill directories +- \`sep-2640-directory-read-result-resources-shape\` — result has resources[] of direct children +- \`sep-2640-directory-read-subdir-mimetype\` — subdirectories surface with \`inode/directory\` mime +- \`sep-2640-directory-read-invalid-params\` — non-directory URI returns \`-32602\` +- \`sep-2640-directory-read-pagination\` — \`nextCursor\` round-trips per resources/list contract + +**Fixture expectation**: the server exposes \`skill://acme/billing/refunds/templates\` with at least one subdirectory child. Without any \`skill://\` resources every check emits SKIPPED.`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect(); + try { + // SKIP gate: if the server exposes no skill:// resources, treat the + // run as not-applicable rather than failing every check. + let resources: ResourceLike[] = []; + try { + const list = await conn.request('resources/list'); + resources = (list.resources ?? []) as ResourceLike[]; + } catch { + // resources/list missing is itself diagnostic — the server can't be a + // skills server. Fall through to the SKIP branch. + } + const hasSkills = resources.some((r) => r.uri.startsWith('skill://')); + if (!hasSkills) { + const reason = + 'Server exposes no skill:// resources; SEP-2640 directoryRead checks not applicable.'; + return [ + 'sep-2640-capability-directory-read-flag', + 'sep-2640-directory-read-method-registered', + 'sep-2640-directory-read-result-resources-shape', + 'sep-2640-directory-read-subdir-mimetype', + 'sep-2640-directory-read-invalid-params', + 'sep-2640-directory-read-pagination' + ].map((id) => check(id, reason, 'SKIPPED', { errorMessage: reason })); + } + + const checks: ConformanceCheck[] = []; + + // === Happy path: list a known directory === + let happy: DirectoryReadResult | undefined; + let happyErr: unknown; + try { + happy = await conn.request( + 'resources/directory/read', + { uri: HAPPY_PATH_URI } + ); + } catch (e) { + happyErr = e; + } + + const isMethodNotFound = + happyErr instanceof JsonRpcError && + happyErr.code === JSONRPC_METHOD_NOT_FOUND; + + // Check 1: capability declaration (derived from method registration). + checks.push( + check( + 'sep-2640-capability-directory-read-flag', + 'Server declared the directoryRead capability — derived from whether resources/directory/read is registered (a server that did not declare directoryRead would return -32601 method-not-found).', + isMethodNotFound ? 'FAILURE' : 'SUCCESS', + isMethodNotFound + ? { + errorMessage: `resources/directory/read returned -32601, implying the server did not declare directoryRead: ${ + (happyErr as JsonRpcError).message + }` + } + : {} + ) + ); + + // Check 2: method registered. + checks.push( + check( + 'sep-2640-directory-read-method-registered', + 'resources/directory/read accepts a call against a known skill subdirectory.', + happy !== undefined + ? 'SUCCESS' + : isMethodNotFound + ? 'FAILURE' + : 'FAILURE', + happy !== undefined + ? { details: { uri: HAPPY_PATH_URI } } + : { + errorMessage: + happyErr instanceof Error + ? happyErr.message + : String(happyErr) + } + ) + ); + + // Check 3: result shape — resources[] of Resource objects. + const shapeOk = Array.isArray(happy?.resources); + const shapeErrs: string[] = []; + if (!shapeOk) { + shapeErrs.push('result.resources is not an array'); + } else { + happy!.resources!.forEach((r, i) => { + if (typeof r.uri !== 'string') + shapeErrs.push(`resources[${i}].uri is not a string`); + }); + } + checks.push( + check( + 'sep-2640-directory-read-result-resources-shape', + 'Result carries resources[] whose entries match the Resource shape (uri at minimum) from resources/list.', + shapeErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + shapeErrs.length > 0 + ? { errorMessage: shapeErrs.join('; ') } + : { details: { entryCount: happy?.resources?.length ?? 0 } } + ) + ); + + // Check 4: subdirectory mime marker. + const subdirChild = happy?.resources?.find( + (r) => r.mimeType === 'inode/directory' + ); + const hasSubdir = subdirChild !== undefined; + checks.push( + check( + 'sep-2640-directory-read-subdir-mimetype', + 'Subdirectory child carries mimeType "inode/directory" so clients can descend.', + hasSubdir ? 'SUCCESS' : 'FAILURE', + hasSubdir + ? { details: { subdirectoryUri: subdirChild!.uri } } + : { + errorMessage: + 'Expected at least one child with mimeType "inode/directory" under ' + + HAPPY_PATH_URI + + '. Server fixture should expose a subdirectory there.' + } + ) + ); + + // === Error path: non-directory URI === + let invalidParamsOk = false; + let invalidParamsDetail = ''; + try { + await conn.request('resources/directory/read', { + uri: NON_DIRECTORY_URI + }); + invalidParamsDetail = + 'expected -32602 for non-directory URI, got success'; + } catch (e) { + if (e instanceof JsonRpcError && e.code === JSONRPC_INVALID_PARAMS) { + invalidParamsOk = true; + } else if (e instanceof JsonRpcError) { + invalidParamsDetail = `expected -32602 for non-directory URI, got ${e.code}: ${e.message}`; + } else { + invalidParamsDetail = `expected -32602, got non-JsonRpcError: ${ + e instanceof Error ? e.message : String(e) + }`; + } + } + checks.push( + check( + 'sep-2640-directory-read-invalid-params', + 'Non-directory URI yields -32602 Invalid params.', + invalidParamsOk ? 'SUCCESS' : 'FAILURE', + invalidParamsOk ? {} : { errorMessage: invalidParamsDetail } + ) + ); + + // === Pagination contract === + // The SEP is permissive: a single-page response with no nextCursor is + // conformant. The check passes when either (a) the first response has + // no nextCursor at all, or (b) the cursor round-trips on a follow-up + // call. mcpkit's defaultDirectoryReadPageSize = 0 puts it in (a). + let paginationOk = false; + let paginationDetail = ''; + const firstCursor = happy?.nextCursor; + if (!firstCursor) { + paginationOk = true; + paginationDetail = 'single-page response (no nextCursor)'; + } else { + try { + const second = await conn.request( + 'resources/directory/read', + { uri: HAPPY_PATH_URI, cursor: firstCursor } + ); + paginationOk = Array.isArray(second.resources); + paginationDetail = paginationOk + ? `nextCursor round-tripped: ${firstCursor}` + : 'follow-up call returned non-array resources'; + } catch (e) { + paginationDetail = `follow-up call with cursor failed: ${ + e instanceof Error ? e.message : String(e) + }`; + } + } + checks.push( + check( + 'sep-2640-directory-read-pagination', + 'nextCursor round-trips per the resources/list contract (single-page responses are conformant).', + paginationOk ? 'SUCCESS' : 'FAILURE', + paginationOk + ? { details: { paginationDetail } } + : { errorMessage: paginationDetail } + ) + ); + + return checks; + } finally { + await conn.close(); + } + } +} diff --git a/src/seps/sep-2640.yaml b/src/seps/sep-2640.yaml index ca93a62c..024d06c3 100644 --- a/src/seps/sep-2640.yaml +++ b/src/seps/sep-2640.yaml @@ -1,5 +1,15 @@ # spec_source: modelcontextprotocol/modelcontextprotocol@556154c088371149c120172e95bb634655f00cbe seps/2640-skills-extension.md # extracted: 2026-06-05 +# forward_reference: rows sep-2640-capability-directory-read-flag through +# sep-2640-directory-read-pagination track SEP commit +# 2e04c48da90224000e750ffd54a3611f2824fbc0 (2026-06-09) — the +# resources/directory/read addition. The file-level provenance above +# stays at 556154c because the PR 97 schema rewrite (360123d0, +# 2026-06-08) made 3 existing rows stale (sep-2640-index-entry-type-enum, +# sep-2640-index-name-required, sep-2640-index-digest-required) and +# drifted ~11 others' verbatim wording. Full re-extraction at SEP HEAD +# is mcpkit#780's lifecycle; this file deliberately holds at 556154c +# until that lands. sep: 2640 spec_url: https://modelcontextprotocol.io/seps/2640-skills-extension#specification requirements: @@ -52,6 +62,20 @@ requirements: - check: sep-2640-host-no-empty-index-assumption text: 'Hosts MUST NOT treat an absent or empty index as proof that a server has no skills.' + # resources/directory/read additions (SEP commit 2e04c48d, 2026-06-09) + - check: sep-2640-capability-directory-read-flag + text: 'Clients MUST NOT call `resources/directory/read` against a server that has not declared `directoryRead: true`.' + - check: sep-2640-directory-read-method-registered + text: 'A server that declares `directoryRead` MUST support the method for every directory within the skill namespaces it serves as individual files.' + - check: sep-2640-directory-read-subdir-mimetype + text: 'A _directory resource_ is a resource whose `mimeType` is `inode/directory`.' + - check: sep-2640-directory-read-result-resources-shape + text: 'The result contains every direct child of the directory: files with their ordinary resource metadata, subdirectories listed as directory resources (`mimeType: "inode/directory"`). The listing is not recursive; clients descend by calling the method again on a child directory.' + - check: sep-2640-directory-read-invalid-params + text: 'The method applies only to directory resources. If the URI does not exist, or exists but is not a directory resource, the server MUST return error `-32602` (Invalid params) — the same code `resources/read` uses for unknown resources.' + - check: sep-2640-directory-read-pagination + text: 'Pagination mirrors `resources/list`: when the result includes `nextCursor`, the client passes it back as `cursor` to retrieve the next page.' + - text: 'Per RFC 3986, the first segment of `` occupies the authority component. This carries no special semantics under this convention and clients MUST NOT attempt DNS or network resolution of it.' excluded: 'DNS and network resolution sit below the MCP wire layer; the harness cannot observe whether the client performed name lookups on URI authority components.' - text: "[Hosts] SHOULD determine the format from the resource's `mimeType`, falling back to the URL suffix" diff --git a/src/types.ts b/src/types.ts index 2e9dd22a..a11ea35f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -68,7 +68,8 @@ export type ScenarioSpecTag = SpecVersion | 'extension'; */ export const EXTENSION_IDS = [ 'io.modelcontextprotocol/oauth-client-credentials', - 'io.modelcontextprotocol/enterprise-managed-authorization' + 'io.modelcontextprotocol/enterprise-managed-authorization', + 'io.modelcontextprotocol/skills' ] as const; export type ExtensionId = (typeof EXTENSION_IDS)[number]; From 2117d61eb77ef93a26c54abf5e1cde1a68d1b095 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Mon, 3 Aug 2026 21:07:50 -0700 Subject: [PATCH 06/12] =?UTF-8?q?feat(sep-2640):=20expand=20skills=20confo?= =?UTF-8?q?rmance=20=E2=80=94=20index=20+=20manifest=20scenarios,=20drift?= =?UTF-8?q?=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds on the directory-read scenario with the server-observable skills surface and brings the whole set up to the current framework conventions. New scenarios (src/scenarios/server/skills/): - SkillsIndexScenario (sep-2640-skills-index): skill://index.json is exposed and readable; every skills[].type is in the SEP enum; every entry has a non-empty name; a present digest matches sha256:{64hex}; entry URLs use skill://. - SkillsManifestScenario (sep-2640-skills-manifest): a skill-md SKILL.md is markdown-typed; frontmatter name/description surface on the resource; the final skill-path segment equals the frontmatter name; skill _meta keys use the reverse-domain prefix. directory.ts (moved into skills/): capability is now read from server/discover instead of inferred from -32601; an undeclared optional capability SKIPs rather than FAILs; the untestable.ts helpers back the not-exercisable paths; all mcpkit-specific fixture URIs are gone — the directory to exercise is discovered dynamically, so the scenario is brand-neutral. Capability gating accepts directoryRead at either extensions[id].directoryRead (SEP-2640's inline example) or extensions[id].config.directoryRead (the SEP-2133 negotiation envelope SEP-2640 defers to); the two SEPs are inconsistent on nesting, so the check accepts either rather than privileging one reading. All three scenarios verified green against a live SEP-2640 server (6/6, 6/6, 7/7); typecheck, eslint, prettier, and the 504-test vitest suite pass. sep-2640.yaml gains a header note mapping each scenario to the rows it now backs. --- src/scenarios/index.ts | 21 +- src/scenarios/server/directory.ts | 289 ----------------- src/scenarios/server/skills/directory.ts | 396 +++++++++++++++++++++++ src/scenarios/server/skills/helpers.ts | 227 +++++++++++++ src/scenarios/server/skills/index.ts | 270 ++++++++++++++++ src/scenarios/server/skills/manifest.ts | 367 +++++++++++++++++++++ src/seps/sep-2640.yaml | 14 + 7 files changed, 1291 insertions(+), 293 deletions(-) delete mode 100644 src/scenarios/server/directory.ts create mode 100644 src/scenarios/server/skills/directory.ts create mode 100644 src/scenarios/server/skills/helpers.ts create mode 100644 src/scenarios/server/skills/index.ts create mode 100644 src/scenarios/server/skills/manifest.ts diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 587f3b7f..8fe9a610 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -58,7 +58,9 @@ import { ResourcesNotFoundErrorScenario } from './server/resources'; -import { ResourcesDirectoryReadScenario } from './server/directory'; +import { SkillsDirectoryReadScenario } from './server/skills/directory'; +import { SkillsIndexScenario } from './server/skills/index'; +import { SkillsManifestScenario } from './server/skills/manifest'; import { PromptsListScenario, @@ -153,7 +155,15 @@ const pendingClientScenariosList: ClientScenario[] = [ new TasksDispatchScenario(), new TasksStatusNotificationsScenario(), new TasksRequiredTaskErrorScenario(), - new TasksMrtrCompositionScenario() + new TasksMrtrCompositionScenario(), + + // SEP-2640 Skills extension. Pending because the everything-server does not + // implement io.modelcontextprotocol/skills; targeted runs point at a + // SEP-2640-conformant fixture via + // `npm start -- server --scenario sep-2640-skills-* --url `. + new SkillsDirectoryReadScenario(), + new SkillsIndexScenario(), + new SkillsManifestScenario() ]; // All client scenarios @@ -205,8 +215,11 @@ const allClientScenariosList: ClientScenario[] = [ // Resources error handling (SEP-2164) new ResourcesNotFoundErrorScenario(), - // Skills extension (SEP-2640) — resources/directory/read surface (2e04c48d) - new ResourcesDirectoryReadScenario(), + // Skills extension (SEP-2640). Fixture-dependent (needs a SEP-2640 server); + // each scenario SKIPs cleanly when the extension is not declared. + new SkillsDirectoryReadScenario(), + new SkillsIndexScenario(), + new SkillsManifestScenario(), // Prompts scenarios new PromptsListScenario(), diff --git a/src/scenarios/server/directory.ts b/src/scenarios/server/directory.ts deleted file mode 100644 index 2db1f89d..00000000 --- a/src/scenarios/server/directory.ts +++ /dev/null @@ -1,289 +0,0 @@ -/** - * SEP-2640 Skills extension scenarios — focused on the resources/directory/read - * surface added in spec commit 2e04c48d (2026-06-09). - * - * One scenario, six checks (per AGENTS.md "fewer scenarios, more checks"). - * Each check's verbatim spec quote lives next to its check ID in - * src/seps/sep-2640.yaml, so the YAML and the scenario stay in lock-step. - * - * Capability discovery: the SEP allows multiple shapes for declaring the - * extension; the wire-observable signal we can rely on is whether - * resources/directory/read responds at all. A -32601 method-not-found is the - * only definitive "server didn't declare directoryRead" signal; any other - * response (success, -32602, etc.) means the server registered the method, - * which the SEP requires of any server that declared the capability. - * - * Fixture assumption: the scenario expects the standard mcpkit examples/skills - * fixture which exposes skill://acme/billing/refunds with a templates/ - * subtree containing at least one subdirectory. When the connected server is - * not a skills server (no skill:// resources at all), every check is emitted - * as SKIPPED — keeps the scenario green against the upstream everything-server - * while emitting real verdicts against any skills-capable fixture. - */ - -import { ClientScenario, ConformanceCheck } from '../../types'; -import { JsonRpcError, type RunContext } from '../../connection'; -import type { ListResourcesResult } from '../../spec-types/2025-06-18'; - -interface ResourceLike { - uri: string; - name?: string; - mimeType?: string; -} - -interface DirectoryReadResult { - resources?: ResourceLike[]; - nextCursor?: string; -} - -const SEP_2640_URL = - 'https://modelcontextprotocol.io/seps/2640-skills-extension#directory-listing'; - -const HAPPY_PATH_URI = 'skill://acme/billing/refunds/templates'; -const NON_DIRECTORY_URI = 'skill://acme/billing/refunds/SKILL.md'; - -const SPEC_REFERENCE = [ - { - id: 'SEP-2640-directory-listing', - url: SEP_2640_URL - } -]; - -const JSONRPC_METHOD_NOT_FOUND = -32601; -const JSONRPC_INVALID_PARAMS = -32602; - -function check( - id: string, - description: string, - status: 'SUCCESS' | 'FAILURE' | 'SKIPPED', - extras: Partial = {} -): ConformanceCheck { - return { - id, - name: id, - description, - status, - timestamp: new Date().toISOString(), - specReferences: SPEC_REFERENCE, - ...extras - }; -} - -export class ResourcesDirectoryReadScenario implements ClientScenario { - name = 'sep-2640-skills'; - readonly source = { - extensionId: 'io.modelcontextprotocol/skills' - } as const; - description = `SEP-2640 Skills extension: resources/directory/read surface (added in spec commit 2e04c48d, 2026-06-09). - -**Endpoint**: \`resources/directory/read\` (gated by \`io.modelcontextprotocol/skills.directoryRead: true\`) - -**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-2640.yaml): - -- \`sep-2640-capability-directory-read-flag\` — server effectively declared directoryRead -- \`sep-2640-directory-read-method-registered\` — method registered for served skill directories -- \`sep-2640-directory-read-result-resources-shape\` — result has resources[] of direct children -- \`sep-2640-directory-read-subdir-mimetype\` — subdirectories surface with \`inode/directory\` mime -- \`sep-2640-directory-read-invalid-params\` — non-directory URI returns \`-32602\` -- \`sep-2640-directory-read-pagination\` — \`nextCursor\` round-trips per resources/list contract - -**Fixture expectation**: the server exposes \`skill://acme/billing/refunds/templates\` with at least one subdirectory child. Without any \`skill://\` resources every check emits SKIPPED.`; - - async run(ctx: RunContext): Promise { - const conn = await ctx.connect(); - try { - // SKIP gate: if the server exposes no skill:// resources, treat the - // run as not-applicable rather than failing every check. - let resources: ResourceLike[] = []; - try { - const list = await conn.request('resources/list'); - resources = (list.resources ?? []) as ResourceLike[]; - } catch { - // resources/list missing is itself diagnostic — the server can't be a - // skills server. Fall through to the SKIP branch. - } - const hasSkills = resources.some((r) => r.uri.startsWith('skill://')); - if (!hasSkills) { - const reason = - 'Server exposes no skill:// resources; SEP-2640 directoryRead checks not applicable.'; - return [ - 'sep-2640-capability-directory-read-flag', - 'sep-2640-directory-read-method-registered', - 'sep-2640-directory-read-result-resources-shape', - 'sep-2640-directory-read-subdir-mimetype', - 'sep-2640-directory-read-invalid-params', - 'sep-2640-directory-read-pagination' - ].map((id) => check(id, reason, 'SKIPPED', { errorMessage: reason })); - } - - const checks: ConformanceCheck[] = []; - - // === Happy path: list a known directory === - let happy: DirectoryReadResult | undefined; - let happyErr: unknown; - try { - happy = await conn.request( - 'resources/directory/read', - { uri: HAPPY_PATH_URI } - ); - } catch (e) { - happyErr = e; - } - - const isMethodNotFound = - happyErr instanceof JsonRpcError && - happyErr.code === JSONRPC_METHOD_NOT_FOUND; - - // Check 1: capability declaration (derived from method registration). - checks.push( - check( - 'sep-2640-capability-directory-read-flag', - 'Server declared the directoryRead capability — derived from whether resources/directory/read is registered (a server that did not declare directoryRead would return -32601 method-not-found).', - isMethodNotFound ? 'FAILURE' : 'SUCCESS', - isMethodNotFound - ? { - errorMessage: `resources/directory/read returned -32601, implying the server did not declare directoryRead: ${ - (happyErr as JsonRpcError).message - }` - } - : {} - ) - ); - - // Check 2: method registered. - checks.push( - check( - 'sep-2640-directory-read-method-registered', - 'resources/directory/read accepts a call against a known skill subdirectory.', - happy !== undefined - ? 'SUCCESS' - : isMethodNotFound - ? 'FAILURE' - : 'FAILURE', - happy !== undefined - ? { details: { uri: HAPPY_PATH_URI } } - : { - errorMessage: - happyErr instanceof Error - ? happyErr.message - : String(happyErr) - } - ) - ); - - // Check 3: result shape — resources[] of Resource objects. - const shapeOk = Array.isArray(happy?.resources); - const shapeErrs: string[] = []; - if (!shapeOk) { - shapeErrs.push('result.resources is not an array'); - } else { - happy!.resources!.forEach((r, i) => { - if (typeof r.uri !== 'string') - shapeErrs.push(`resources[${i}].uri is not a string`); - }); - } - checks.push( - check( - 'sep-2640-directory-read-result-resources-shape', - 'Result carries resources[] whose entries match the Resource shape (uri at minimum) from resources/list.', - shapeErrs.length === 0 ? 'SUCCESS' : 'FAILURE', - shapeErrs.length > 0 - ? { errorMessage: shapeErrs.join('; ') } - : { details: { entryCount: happy?.resources?.length ?? 0 } } - ) - ); - - // Check 4: subdirectory mime marker. - const subdirChild = happy?.resources?.find( - (r) => r.mimeType === 'inode/directory' - ); - const hasSubdir = subdirChild !== undefined; - checks.push( - check( - 'sep-2640-directory-read-subdir-mimetype', - 'Subdirectory child carries mimeType "inode/directory" so clients can descend.', - hasSubdir ? 'SUCCESS' : 'FAILURE', - hasSubdir - ? { details: { subdirectoryUri: subdirChild!.uri } } - : { - errorMessage: - 'Expected at least one child with mimeType "inode/directory" under ' + - HAPPY_PATH_URI + - '. Server fixture should expose a subdirectory there.' - } - ) - ); - - // === Error path: non-directory URI === - let invalidParamsOk = false; - let invalidParamsDetail = ''; - try { - await conn.request('resources/directory/read', { - uri: NON_DIRECTORY_URI - }); - invalidParamsDetail = - 'expected -32602 for non-directory URI, got success'; - } catch (e) { - if (e instanceof JsonRpcError && e.code === JSONRPC_INVALID_PARAMS) { - invalidParamsOk = true; - } else if (e instanceof JsonRpcError) { - invalidParamsDetail = `expected -32602 for non-directory URI, got ${e.code}: ${e.message}`; - } else { - invalidParamsDetail = `expected -32602, got non-JsonRpcError: ${ - e instanceof Error ? e.message : String(e) - }`; - } - } - checks.push( - check( - 'sep-2640-directory-read-invalid-params', - 'Non-directory URI yields -32602 Invalid params.', - invalidParamsOk ? 'SUCCESS' : 'FAILURE', - invalidParamsOk ? {} : { errorMessage: invalidParamsDetail } - ) - ); - - // === Pagination contract === - // The SEP is permissive: a single-page response with no nextCursor is - // conformant. The check passes when either (a) the first response has - // no nextCursor at all, or (b) the cursor round-trips on a follow-up - // call. mcpkit's defaultDirectoryReadPageSize = 0 puts it in (a). - let paginationOk = false; - let paginationDetail = ''; - const firstCursor = happy?.nextCursor; - if (!firstCursor) { - paginationOk = true; - paginationDetail = 'single-page response (no nextCursor)'; - } else { - try { - const second = await conn.request( - 'resources/directory/read', - { uri: HAPPY_PATH_URI, cursor: firstCursor } - ); - paginationOk = Array.isArray(second.resources); - paginationDetail = paginationOk - ? `nextCursor round-tripped: ${firstCursor}` - : 'follow-up call returned non-array resources'; - } catch (e) { - paginationDetail = `follow-up call with cursor failed: ${ - e instanceof Error ? e.message : String(e) - }`; - } - } - checks.push( - check( - 'sep-2640-directory-read-pagination', - 'nextCursor round-trips per the resources/list contract (single-page responses are conformant).', - paginationOk ? 'SUCCESS' : 'FAILURE', - paginationOk - ? { details: { paginationDetail } } - : { errorMessage: paginationDetail } - ) - ); - - return checks; - } finally { - await conn.close(); - } - } -} diff --git a/src/scenarios/server/skills/directory.ts b/src/scenarios/server/skills/directory.ts new file mode 100644 index 00000000..27be77bd --- /dev/null +++ b/src/scenarios/server/skills/directory.ts @@ -0,0 +1,396 @@ +/** + * SEP-2640 Skills extension — the `resources/directory/read` surface (added in + * spec commit 2e04c48d, 2026-06-09). + * + * One scenario, six checks (per AGENTS.md "fewer scenarios, more checks"). + * Each check's verbatim spec quote lives next to its check ID in + * src/seps/sep-2640.yaml. + * + * Capability gating reads the declared capability from `server/discover` + * (mirrors `tasks/capability.ts`): the checks run only when the server declares + * `io.modelcontextprotocol/skills.directoryRead: true`. An undeclared optional + * capability is a SKIP (not a failure); a declared-but-broken one fails. + * + * Discovery is dynamic and brand-neutral: the directory to exercise is derived + * from `skill://index.json` or `resources/list`, hardcoding no fixture URI, so + * the scenario passes against any conformant SEP-2640 server. When no directory + * (or no subdirectory) can be discovered, that check reports the missing + * prerequisite via untestableCheck (issue #248), never a silent green. + */ + +import { ClientScenario, ConformanceCheck } from '../../../types'; +import { Connection, JsonRpcError, type RunContext } from '../../../connection'; +import { untestableCheck } from '../../untestable'; +import { + SKILLS_EXTENSION_ID, + SKILL_MANIFEST_FILENAME, + SEP_2640_REF, + JSONRPC_METHOD_NOT_FOUND, + JSONRPC_INVALID_PARAMS, + type SkillIndex, + type SkillResource, + skillsCapability, + directoryReadDeclared, + skillsCheck, + listAllResources, + readSkillIndexText, + skillNameFromManifestUri +} from './helpers'; + +const DIRECTORY_MIME = 'inode/directory'; + +const CAPABILITY_ID = 'sep-2640-capability-directory-read-flag'; +const METHOD_ID = 'sep-2640-directory-read-method-registered'; +const SHAPE_ID = 'sep-2640-directory-read-result-resources-shape'; +const SUBDIR_ID = 'sep-2640-directory-read-subdir-mimetype'; +const INVALID_PARAMS_ID = 'sep-2640-directory-read-invalid-params'; +const PAGINATION_ID = 'sep-2640-directory-read-pagination'; + +const ALL_IDS = [ + CAPABILITY_ID, + METHOD_ID, + SHAPE_ID, + SUBDIR_ID, + INVALID_PARAMS_ID, + PAGINATION_ID +]; + +interface DirectoryReadResult { + resources?: SkillResource[]; + nextCursor?: string; +} + +/** A directory to exercise plus, when known, a non-directory resource under it. */ +interface DirectoryTarget { + dirUri: string; + /** A known file (non-directory) resource, used for the -32602 negative path. */ + fileUri?: string; +} + +/** The skill root directory URI for a SKILL.md URI (strip the trailing file). */ +function skillRootFromManifestUri(uri: string): string | undefined { + if (skillNameFromManifestUri(uri) === undefined) return undefined; + return uri.slice(0, uri.length - `/${SKILL_MANIFEST_FILENAME}`.length); +} + +/** + * Discover a directory resource to exercise, brand-neutrally: prefer a skill + * root derived from a skill-md SKILL.md (index first, then resources/list), + * then any `inode/directory` resource in resources/list. + */ +async function discoverDirectory( + conn: Connection +): Promise { + // 1. skill-md entry in skill://index.json — its SKILL.md URL gives us both a + // directory (the skill root) and a known file (the SKILL.md itself). + const idx = await readSkillIndexText(conn); + if (!('error' in idx) && typeof idx.text === 'string') { + try { + const index = JSON.parse(idx.text) as SkillIndex; + const entry = (index.skills ?? []).find( + (e) => + e.type === 'skill-md' && + typeof e.url === 'string' && + skillRootFromManifestUri(e.url) !== undefined + ); + if (entry?.url) { + return { + dirUri: skillRootFromManifestUri(entry.url)!, + fileUri: entry.url + }; + } + } catch { + // A malformed index is the index scenario's concern; keep discovering. + } + } + + const resources = await listAllResources(conn); + + // 2. A SKILL.md in resources/list — derive the skill root the same way. + const manifest = resources.find( + (r) => skillRootFromManifestUri(r.uri) !== undefined + ); + if (manifest) { + return { + dirUri: skillRootFromManifestUri(manifest.uri)!, + fileUri: manifest.uri + }; + } + + // 3. Any directory resource, using a non-directory sibling for the -32602 + // path when one is listed. + const dir = resources.find((r) => r.mimeType === DIRECTORY_MIME); + if (dir) { + const file = resources.find((r) => r.mimeType !== DIRECTORY_MIME); + return { dirUri: dir.uri, fileUri: file?.uri }; + } + + return undefined; +} + +export class SkillsDirectoryReadScenario implements ClientScenario { + name = 'sep-2640-skills-directory'; + readonly source = { extensionId: SKILLS_EXTENSION_ID } as const; + description = `SEP-2640 Skills extension: resources/directory/read surface (added in spec commit 2e04c48d, 2026-06-09). + +**Endpoint**: \`resources/directory/read\` (gated by \`io.modelcontextprotocol/skills.directoryRead: true\`) + +**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-2640.yaml): + +- \`sep-2640-capability-directory-read-flag\` — server declared directoryRead (read from server/discover) +- \`sep-2640-directory-read-method-registered\` — a declaring server supports the method on a served directory (MUST) +- \`sep-2640-directory-read-result-resources-shape\` — result has resources[] of direct children (MUST) +- \`sep-2640-directory-read-subdir-mimetype\` — subdirectory children carry \`inode/directory\` (MUST) +- \`sep-2640-directory-read-invalid-params\` — a non-directory URI returns \`-32602\` (MUST) +- \`sep-2640-directory-read-pagination\` — \`nextCursor\` round-trips per resources/list (single-page is conformant) + +**Gating & discovery**: the checks SKIP when the skills extension or its \`directoryRead\` flag is undeclared. The directory to exercise is discovered dynamically from \`skill://index.json\` / \`resources/list\` — no fixture URI is hardcoded.`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect(); + try { + // === Capability gating via server/discover (not error-inference) === + const skills = await skillsCapability(conn); + if (!skills) { + const reason = + 'Server did not declare the io.modelcontextprotocol/skills extension; directoryRead checks not applicable.'; + return ALL_IDS.map((id) => + skillsCheck(id, reason, 'SKIPPED', { errorMessage: reason }) + ); + } + if (!directoryReadDeclared(skills)) { + const reason = + 'Server declared the skills extension but not directoryRead: true; the resources/directory/read checks are optional and not applicable.'; + return ALL_IDS.map((id) => + skillsCheck(id, reason, 'SKIPPED', { errorMessage: reason }) + ); + } + + const checks: ConformanceCheck[] = []; + + // Check 1: capability declared (observed directly from server/discover). + checks.push( + skillsCheck( + CAPABILITY_ID, + 'Server declared io.modelcontextprotocol/skills.directoryRead: true under capabilities.extensions.', + 'SUCCESS', + { details: { directoryRead: true } } + ) + ); + + // === Discover a directory to exercise (brand-neutral) === + const target = await discoverDirectory(conn); + if (!target) { + const reason = + 'no directory resource discoverable via skill://index.json or resources/list to exercise resources/directory/read'; + const rest: Array<[string, string]> = [ + [ + METHOD_ID, + 'A declaring server MUST support the method on a served directory.' + ], + [ + SHAPE_ID, + 'Result carries resources[] of the directory’s direct children.' + ], + [SUBDIR_ID, 'Subdirectory children carry mimeType inode/directory.'], + [ + INVALID_PARAMS_ID, + 'A non-directory URI yields -32602 Invalid params.' + ], + [ + PAGINATION_ID, + 'nextCursor round-trips per the resources/list contract.' + ] + ]; + for (const [id, desc] of rest) { + checks.push( + untestableCheck(id, id, desc, reason, [SEP_2640_REF], 'FAILURE') + ); + } + return checks; + } + + // === Happy path: list the discovered directory === + let happy: DirectoryReadResult | undefined; + let happyErr: unknown; + try { + happy = await conn.request( + 'resources/directory/read', + { uri: target.dirUri } + ); + } catch (e) { + happyErr = e; + } + + // Check 2: method registered (declared -> MUST be supported). + const methodNotFound = + happyErr instanceof JsonRpcError && + happyErr.code === JSONRPC_METHOD_NOT_FOUND; + checks.push( + skillsCheck( + METHOD_ID, + 'A server that declares directoryRead MUST support resources/directory/read on a served skill directory.', + happy !== undefined ? 'SUCCESS' : 'FAILURE', + happy !== undefined + ? { details: { uri: target.dirUri } } + : { + errorMessage: methodNotFound + ? `resources/directory/read returned -32601 for ${target.dirUri} despite the server declaring directoryRead: true` + : `resources/directory/read on ${target.dirUri} failed: ${ + happyErr instanceof Error + ? happyErr.message + : String(happyErr) + }` + } + ) + ); + + // Check 3: result shape — resources[] of Resource objects. + const shapeErrs: string[] = []; + if (!Array.isArray(happy?.resources)) { + shapeErrs.push('result.resources is not an array'); + } else { + happy.resources.forEach((r, i) => { + if (typeof r.uri !== 'string') { + shapeErrs.push(`resources[${i}].uri is not a string`); + } + }); + } + checks.push( + skillsCheck( + SHAPE_ID, + 'The result contains resources[] listing the directory’s direct children, each with at least a uri.', + happy === undefined + ? 'FAILURE' + : shapeErrs.length === 0 + ? 'SUCCESS' + : 'FAILURE', + happy === undefined + ? { errorMessage: 'directory read did not return a result' } + : shapeErrs.length === 0 + ? { details: { childCount: happy.resources?.length ?? 0 } } + : { errorMessage: shapeErrs.join('; ') } + ) + ); + + // Check 4: subdirectory mime marker. A directory whose fixture exposes no + // child subdirectory cannot exercise this — report it untestable, not a + // pass and not a failure of the server. + const subdirChild = Array.isArray(happy?.resources) + ? happy.resources.find((r) => r.mimeType === DIRECTORY_MIME) + : undefined; + if (subdirChild) { + checks.push( + skillsCheck( + SUBDIR_ID, + 'A subdirectory child is listed as a directory resource (mimeType inode/directory) so clients can descend.', + 'SUCCESS', + { details: { subdirectoryUri: subdirChild.uri } } + ) + ); + } else { + checks.push( + untestableCheck( + SUBDIR_ID, + SUBDIR_ID, + 'A subdirectory child is listed with mimeType inode/directory.', + `no child with mimeType ${DIRECTORY_MIME} under ${target.dirUri}; the served directory exposes no subdirectory to exercise this check`, + [SEP_2640_REF], + 'FAILURE' + ) + ); + } + + // Check 5: non-directory URI -> -32602. Needs a known non-directory + // resource; prefer the discovered fileUri, else a non-directory child. + const nonDirUri = + target.fileUri ?? + (Array.isArray(happy?.resources) + ? happy.resources.find( + (r) => typeof r.uri === 'string' && r.mimeType !== DIRECTORY_MIME + )?.uri + : undefined); + if (nonDirUri === undefined) { + checks.push( + untestableCheck( + INVALID_PARAMS_ID, + INVALID_PARAMS_ID, + 'A non-directory URI yields -32602 Invalid params.', + 'no non-directory resource discoverable to probe the -32602 path', + [SEP_2640_REF], + 'FAILURE' + ) + ); + } else { + let invalidOk = false; + let invalidDetail = ''; + try { + await conn.request('resources/directory/read', { + uri: nonDirUri + }); + invalidDetail = `expected -32602 for non-directory URI ${nonDirUri}, got a successful result`; + } catch (e) { + if (e instanceof JsonRpcError && e.code === JSONRPC_INVALID_PARAMS) { + invalidOk = true; + } else if (e instanceof JsonRpcError) { + invalidDetail = `expected -32602 for ${nonDirUri}, got ${e.code}: ${e.message}`; + } else { + invalidDetail = `expected -32602, got non-JsonRpcError: ${ + e instanceof Error ? e.message : String(e) + }`; + } + } + checks.push( + skillsCheck( + INVALID_PARAMS_ID, + 'resources/directory/read on a non-directory URI MUST return -32602 (Invalid params).', + invalidOk ? 'SUCCESS' : 'FAILURE', + invalidOk + ? { details: { nonDirectoryUri: nonDirUri } } + : { errorMessage: invalidDetail } + ) + ); + } + + // Check 6: pagination contract (single-page is conformant). + let paginationOk = false; + let paginationDetail = ''; + const firstCursor = happy?.nextCursor; + if (happy === undefined) { + paginationDetail = 'no directory result to evaluate pagination'; + } else if (!firstCursor) { + paginationOk = true; + paginationDetail = 'single-page response (no nextCursor)'; + } else { + try { + const second = await conn.request( + 'resources/directory/read', + { uri: target.dirUri, cursor: firstCursor } + ); + paginationOk = Array.isArray(second.resources); + paginationDetail = paginationOk + ? `nextCursor round-tripped: ${firstCursor}` + : 'follow-up call returned non-array resources'; + } catch (e) { + paginationDetail = `follow-up call with cursor failed: ${ + e instanceof Error ? e.message : String(e) + }`; + } + } + checks.push( + skillsCheck( + PAGINATION_ID, + 'nextCursor round-trips per the resources/list contract (single-page responses are conformant).', + paginationOk ? 'SUCCESS' : 'FAILURE', + paginationOk + ? { details: { paginationDetail } } + : { errorMessage: paginationDetail } + ) + ); + + return checks; + } finally { + await conn.close(); + } + } +} diff --git a/src/scenarios/server/skills/helpers.ts b/src/scenarios/server/skills/helpers.ts new file mode 100644 index 00000000..d32022f9 --- /dev/null +++ b/src/scenarios/server/skills/helpers.ts @@ -0,0 +1,227 @@ +/** + * Shared helpers for the SEP-2640 (Skills extension) server-conformance + * scenarios under this directory. + * + * The scenarios treat the server-under-test as an arbitrary SEP-2640 server: + * capability is read from `server/discover` (never inferred from an error), and + * every skill is discovered dynamically from `skill://index.json` and + * `resources/list` — no fixture-specific URI is hardcoded, so the checks pass + * against any conformant server, not just one implementation's fixture. + */ + +import type { + CheckStatus, + ConformanceCheck, + SpecReference +} from '../../../types'; +import type { Connection } from '../../../connection'; +import { JsonRpcError } from '../../../connection'; +import { parse as parseYaml } from 'yaml'; + +export const SKILLS_EXTENSION_ID = 'io.modelcontextprotocol/skills'; +export const SKILL_URI_SCHEME = 'skill://'; +export const SKILL_INDEX_URI = 'skill://index.json'; +export const SKILL_MANIFEST_FILENAME = 'SKILL.md'; +export const SKILLS_META_PREFIX = 'io.modelcontextprotocol.skills/'; + +/** `sha256:{hex}` with exactly 64 lowercase hex characters (SEP-2640 index). */ +export const SKILL_DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/; + +/** The SEP enumerated `skills[].type` values. */ +export const SKILL_TYPES = ['skill-md', 'archive'] as const; + +export const JSONRPC_METHOD_NOT_FOUND = -32601; +export const JSONRPC_INVALID_PARAMS = -32602; + +export const SEP_2640_REF: SpecReference = { + id: 'SEP-2640', + url: 'https://modelcontextprotocol.io/seps/2640-skills-extension#specification' +}; + +/** A `resources/list` / directory-read entry (only the fields we inspect). */ +export interface SkillResource { + uri: string; + name?: string; + description?: string; + mimeType?: string; + _meta?: Record; +} + +/** One `skills[]` entry of the `skill://index.json` document. */ +export interface SkillIndexEntry { + name?: string; + type?: string; + description?: string; + url?: string; + digest?: string; + [key: string]: unknown; +} + +/** The parsed `skill://index.json` document. */ +export interface SkillIndex { + $schema?: string; + skills?: SkillIndexEntry[]; + [key: string]: unknown; +} + +/** First text content of a `resources/read`, with its mimeType and `_meta`. */ +export interface ResourceText { + text: string; + mimeType?: string; + meta?: Record; +} + +/** + * Build a check carrying the SEP-2640 reference. Per AGENTS.md the same `id` + * flips `status` + `errorMessage` between SUCCESS and FAILURE rather than + * branching into distinct slugs. + */ +export function skillsCheck( + id: string, + description: string, + status: CheckStatus, + extras: Partial = {} +): ConformanceCheck { + return { + id, + name: id, + description, + status, + timestamp: new Date().toISOString(), + specReferences: [SEP_2640_REF], + ...extras + }; +} + +/** + * The skills extension object declared under `capabilities.extensions`, or + * `undefined` when the server did not declare it. Reads the declared capability + * from `server/discover` (mirrors `tasks/capability.ts`) — an undeclared + * optional extension is a SKIP, never inferred from a `-32601`. + */ +export async function skillsCapability( + conn: Connection +): Promise | undefined> { + const discovered = await conn.discover(); + const caps = (discovered.capabilities as Record) ?? {}; + const extensions = caps.extensions as Record | undefined; + const skills = extensions?.[SKILLS_EXTENSION_ID]; + return skills && typeof skills === 'object' + ? (skills as Record) + : undefined; +} + +/** + * Whether the skills extension declares `directoryRead: true`. + * + * SEP-2640's capability-declaration example places the flag directly on the + * extension object (`extensions[id].directoryRead`). SEP-2133 extension + * negotiation — which SEP-2640 normatively defers to ("Per SEP-2133 extension + * negotiation") — wraps settings in a `{ specVersion, stability, config }` + * envelope, putting the flag at `extensions[id].config.directoryRead`. The two + * SEPs are inconsistent on nesting, so a brand-neutral conformance check accepts + * either location rather than privileging one reading of an ambiguous spec. + * (The inconsistency is worth a WG clarification; see the scenario docs.) + */ +export function directoryReadDeclared( + skills: Record +): boolean { + if (skills.directoryRead === true) return true; + const config = skills.config as Record | undefined; + return config?.directoryRead === true; +} + +/** Everything from `resources/list`, paginating until `nextCursor` clears. */ +export async function listAllResources( + conn: Connection +): Promise { + const out: SkillResource[] = []; + let cursor: string | undefined; + do { + const page = await conn.request<{ + resources?: SkillResource[]; + nextCursor?: string; + }>('resources/list', cursor ? { cursor } : undefined); + out.push(...(page.resources ?? [])); + cursor = page.nextCursor; + } while (cursor); + return out; +} + +/** + * Read `skill://index.json`. Returns the raw JSON text (for parse-error + * reporting) or a `JsonRpcError` when the server declines the well-known index + * — a permitted MAY (SEP-2640 §Enumeration): the catalog may be unenumerable. + */ +export async function readSkillIndexText( + conn: Connection +): Promise<{ text?: string; mimeType?: string } | { error: JsonRpcError }> { + try { + const res = await conn.request<{ + contents?: Array<{ text?: string; mimeType?: string }>; + }>('resources/read', { uri: SKILL_INDEX_URI }); + const entry = (res.contents ?? []).find((c) => typeof c.text === 'string'); + return { text: entry?.text, mimeType: entry?.mimeType }; + } catch (e) { + if (e instanceof JsonRpcError) return { error: e }; + throw e; + } +} + +/** Read a resource's first text content plus its mimeType and `_meta`. */ +export async function readResourceText( + conn: Connection, + uri: string +): Promise { + const res = await conn.request<{ + contents?: Array<{ + text?: string; + mimeType?: string; + _meta?: Record; + }>; + }>('resources/read', { uri }); + const entry = (res.contents ?? []).find((c) => typeof c.text === 'string'); + if (!entry || typeof entry.text !== 'string') return undefined; + return { text: entry.text, mimeType: entry.mimeType, meta: entry._meta }; +} + +/** + * The skill name recoverable from a `SKILL.md` resource URI: the final segment + * of ``, i.e. the last path segment before the trailing + * `SKILL.md`. Returns `undefined` when the URI is not a `skill://…/SKILL.md`. + * + * skill://org/team/deploy/SKILL.md -> "deploy" + * skill://lint/SKILL.md -> "lint" + */ +export function skillNameFromManifestUri(uri: string): string | undefined { + if (!uri.startsWith(SKILL_URI_SCHEME)) return undefined; + const parts = uri + .slice(SKILL_URI_SCHEME.length) + .split('/') + .filter((p) => p.length > 0); + if (parts.length < 2) return undefined; + if (parts[parts.length - 1] !== SKILL_MANIFEST_FILENAME) return undefined; + return parts[parts.length - 2]; +} + +/** + * Extract and parse the YAML frontmatter block at the head of a `SKILL.md`. + * Returns `undefined` when there is no leading `---` delimited block or it does + * not parse to an object. + */ +export function parseFrontmatter( + markdown: string +): Record | undefined { + // Tolerate a leading UTF-8 BOM before the opening `---` fence. + const body = markdown.charCodeAt(0) === 0xfeff ? markdown.slice(1) : markdown; + const match = body.match(/^---\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/); + if (!match) return undefined; + try { + const parsed = parseYaml(match[1]) as unknown; + return parsed && typeof parsed === 'object' + ? (parsed as Record) + : undefined; + } catch { + return undefined; + } +} diff --git a/src/scenarios/server/skills/index.ts b/src/scenarios/server/skills/index.ts new file mode 100644 index 00000000..bb9d1f33 --- /dev/null +++ b/src/scenarios/server/skills/index.ts @@ -0,0 +1,270 @@ +/** + * SEP-2640 Skills extension — `skill://index.json` enumeration surface. + * + * One scenario, many checks (per AGENTS.md "fewer scenarios, more checks"). + * Each check's verbatim spec quote lives next to its check ID in + * src/seps/sep-2640.yaml, keeping the YAML and this scenario in lock-step. + * + * All discovery is dynamic and brand-neutral: the scenario reads the well-known + * `skill://index.json` and validates whatever entries it finds, hardcoding no + * fixture-specific skill name or URI. When the server does not declare the + * skills extension the checks are SKIPPED (an optional, undeclared capability); + * when the server declines the index (a permitted MAY) or serves an empty index + * the index-shape checks are SKIPPED (legitimately not applicable), never + * failed against a conformant server. + */ + +import { ClientScenario, ConformanceCheck } from '../../../types'; +import type { RunContext } from '../../../connection'; +import { + SKILLS_EXTENSION_ID, + SKILL_INDEX_URI, + SKILL_URI_SCHEME, + SKILL_TYPES, + SKILL_DIGEST_PATTERN, + type SkillIndex, + type SkillIndexEntry, + skillsCapability, + skillsCheck, + readSkillIndexText +} from './helpers'; + +const ENTRY_CHECK_IDS = [ + 'sep-2640-index-entry-type-enum', + 'sep-2640-index-name-required', + 'sep-2640-index-digest-required', + 'sep-2640-skill-uri-scheme' +] as const; + +const ALL_CHECK_IDS = ['sep-2640-server-expose-index', ...ENTRY_CHECK_IDS]; + +export class SkillsIndexScenario implements ClientScenario { + name = 'sep-2640-skills-index'; + readonly source = { extensionId: SKILLS_EXTENSION_ID } as const; + description = `SEP-2640 Skills extension: the \`skill://index.json\` enumeration index. + +**Resource**: \`skill://index.json\` (read via \`resources/read\`, \`mimeType\` \`application/json\`) + +**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-2640.yaml): + +- \`sep-2640-server-expose-index\` — server exposes a readable \`skill://index.json\` (SHOULD; a server MAY decline for an unenumerable catalog) +- \`sep-2640-index-entry-type-enum\` — every \`skills[].type\` is \`"skill-md"\` or \`"archive"\` (MUST) +- \`sep-2640-index-name-required\` — every entry carries a non-empty \`name\` (required field) +- \`sep-2640-index-digest-required\` — a present \`skills[].digest\` is \`sha256:{64 hex}\` (MUST) +- \`sep-2640-skill-uri-scheme\` — index entry URLs use the \`skill://\` scheme (SHOULD; another scheme is permitted only when listed in the index) + +**Discovery is dynamic**: the scenario reads whatever skills the index enumerates. Undeclared extension, a declined index, or an empty index all SKIP cleanly.`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect(); + try { + const skills = await skillsCapability(conn); + if (!skills) { + const reason = + 'Server did not declare the io.modelcontextprotocol/skills extension; index checks not applicable.'; + return ALL_CHECK_IDS.map((id) => + skillsCheck(id, reason, 'SKIPPED', { errorMessage: reason }) + ); + } + + const checks: ConformanceCheck[] = []; + + // === server-expose-index (SHOULD, with an explicit MAY-decline) === + const read = await readSkillIndexText(conn); + if ('error' in read) { + const reason = `Server declined skill://index.json (code ${read.error.code}); permitted MAY — the catalog may be large or unenumerable. Hosts MUST NOT treat this as proof of no skills.`; + checks.push( + skillsCheck( + 'sep-2640-server-expose-index', + 'Server SHOULD expose a readable skill://index.json enumerating the skills it serves.', + 'SKIPPED', + { errorMessage: reason } + ) + ); + for (const id of ENTRY_CHECK_IDS) { + checks.push( + skillsCheck(id, 'No skill://index.json to inspect.', 'SKIPPED', { + errorMessage: reason + }) + ); + } + return checks; + } + + if (read.text === undefined) { + const reason = + 'resources/read on skill://index.json returned no text content; the index resource is exposed but unreadable.'; + checks.push( + skillsCheck( + 'sep-2640-server-expose-index', + 'Server SHOULD expose a readable skill://index.json enumerating the skills it serves.', + 'FAILURE', + { errorMessage: reason } + ) + ); + for (const id of ENTRY_CHECK_IDS) { + checks.push( + skillsCheck( + id, + 'No readable index content to inspect.', + 'SKIPPED', + { + errorMessage: reason + } + ) + ); + } + return checks; + } + + let index: SkillIndex; + try { + index = JSON.parse(read.text) as SkillIndex; + } catch (e) { + const reason = `skill://index.json content is not valid JSON: ${ + e instanceof Error ? e.message : String(e) + }`; + checks.push( + skillsCheck( + 'sep-2640-server-expose-index', + 'Server SHOULD expose a readable skill://index.json whose content is a JSON index.', + 'FAILURE', + { errorMessage: reason } + ) + ); + for (const id of ENTRY_CHECK_IDS) { + checks.push( + skillsCheck(id, 'Index did not parse as JSON.', 'SKIPPED', { + errorMessage: reason + }) + ); + } + return checks; + } + + checks.push( + skillsCheck( + 'sep-2640-server-expose-index', + 'Server SHOULD expose a readable skill://index.json whose content is a JSON index of the skills it serves.', + 'SUCCESS', + { + details: { + uri: SKILL_INDEX_URI, + mimeType: read.mimeType, + skillCount: Array.isArray(index.skills) ? index.skills.length : 0 + } + } + ) + ); + + const entries: SkillIndexEntry[] = Array.isArray(index.skills) + ? index.skills + : []; + + // An exposed-but-empty index is valid: a partial/empty index is + // permitted, and hosts MUST NOT read "no skills" from it. Nothing to + // validate at the entry level, so SKIP those checks cleanly. + if (entries.length === 0) { + const reason = + 'skill://index.json is exposed but lists no skills; entry-level checks not applicable (an empty index is permitted).'; + for (const id of ENTRY_CHECK_IDS) { + checks.push( + skillsCheck(id, reason, 'SKIPPED', { errorMessage: reason }) + ); + } + return checks; + } + + // === index-entry-type-enum (MUST) === + const typeErrs = entries + .map((e, i) => + SKILL_TYPES.includes(e.type as (typeof SKILL_TYPES)[number]) + ? null + : `skills[${i}].type=${JSON.stringify(e.type)} is not one of ${SKILL_TYPES.join('|')}` + ) + .filter((x): x is string => x !== null); + checks.push( + skillsCheck( + 'sep-2640-index-entry-type-enum', + 'Every skills[].type MUST be "skill-md" or "archive".', + typeErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + typeErrs.length === 0 + ? { details: { entryCount: entries.length } } + : { errorMessage: typeErrs.join('; ') } + ) + ); + + // === index-name-required (required field) === + const nameErrs = entries + .map((e, i) => + typeof e.name === 'string' && e.name.length > 0 + ? null + : `skills[${i}].name is missing or empty` + ) + .filter((x): x is string => x !== null); + checks.push( + skillsCheck( + 'sep-2640-index-name-required', + 'Every index entry carries a non-empty name (matching the SKILL.md frontmatter name and the final skill-path segment).', + nameErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + nameErrs.length === 0 + ? { details: { entryCount: entries.length } } + : { errorMessage: nameErrs.join('; ') } + ) + ); + + // === index-digest-required (MUST — validate the format when present) === + const withDigest = entries.filter((e) => e.digest !== undefined); + const digestErrs = withDigest + .map((e, i) => + typeof e.digest === 'string' && SKILL_DIGEST_PATTERN.test(e.digest) + ? null + : `skills[${i}].digest=${JSON.stringify(e.digest)} is not sha256:{64 lowercase hex}` + ) + .filter((x): x is string => x !== null); + checks.push( + skillsCheck( + 'sep-2640-index-digest-required', + 'Every present skills[].digest MUST be formatted as sha256:{hex} with 64 lowercase hex characters.', + digestErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + digestErrs.length === 0 + ? { + details: { + entriesWithDigest: withDigest.length, + entriesWithoutDigest: entries.length - withDigest.length + } + } + : { errorMessage: digestErrs.join('; ') } + ) + ); + + // === skill-uri-scheme (SHOULD) === + // Servers SHOULD use skill://; another scheme is permitted only when the + // skill is listed in the index (SEP-2640 §URI convention), so a non- + // skill:// URL is a SHOULD deviation, not a hard failure. + const nonSkillScheme = entries + .map((e, i) => + typeof e.url === 'string' && !e.url.startsWith(SKILL_URI_SCHEME) + ? `skills[${i}].url=${JSON.stringify(e.url)}` + : null + ) + .filter((x): x is string => x !== null); + checks.push( + skillsCheck( + 'sep-2640-skill-uri-scheme', + 'Skill resource URLs in the index SHOULD use the skill:// URI scheme.', + nonSkillScheme.length === 0 ? 'SUCCESS' : 'WARNING', + nonSkillScheme.length === 0 + ? { details: { entryCount: entries.length } } + : { + errorMessage: `Entries use a non-skill:// scheme (permitted only when indexed): ${nonSkillScheme.join(', ')}` + } + ) + ); + + return checks; + } finally { + await conn.close(); + } + } +} diff --git a/src/scenarios/server/skills/manifest.ts b/src/scenarios/server/skills/manifest.ts new file mode 100644 index 00000000..7a4f1fd6 --- /dev/null +++ b/src/scenarios/server/skills/manifest.ts @@ -0,0 +1,367 @@ +/** + * SEP-2640 Skills extension — the `SKILL.md` manifest resource. + * + * One scenario, many checks (per AGENTS.md "fewer scenarios, more checks"). + * Each check's verbatim spec quote lives next to its check ID in + * src/seps/sep-2640.yaml. + * + * Discovery is dynamic and brand-neutral: the scenario finds a `skill-md` + * skill's `SKILL.md` resource from `resources/list` (preferred — it carries the + * Resource `name`/`description` metadata) or falls back to the first `skill-md` + * entry in `skill://index.json`, hardcoding no fixture skill. Undeclared + * extension SKIPs; a declared extension with no discoverable `SKILL.md` reports + * the missing prerequisite via untestableCheck (issue #248), never a silent + * green. + */ + +import { ClientScenario, ConformanceCheck } from '../../../types'; +import { JsonRpcError, type RunContext } from '../../../connection'; +import { untestableCheck } from '../../untestable'; +import { + SKILLS_EXTENSION_ID, + SKILLS_META_PREFIX, + SEP_2640_REF, + type SkillIndex, + type SkillResource, + skillsCapability, + skillsCheck, + listAllResources, + readSkillIndexText, + readResourceText, + skillNameFromManifestUri, + parseFrontmatter +} from './helpers'; + +const MIMETYPE_ID = 'sep-2640-skillmd-mimetype'; +const METADATA_NAME_ID = 'sep-2640-skillmd-metadata-name'; +const METADATA_DESCRIPTION_ID = 'sep-2640-skillmd-metadata-description'; +const FINAL_SEGMENT_ID = 'sep-2640-final-segment-equals-name'; +const META_PREFIX_ID = 'sep-2640-meta-prefix'; + +const MARKDOWN_MIME = 'text/markdown'; + +/** A SKILL.md resource URI is skill:///SKILL.md. */ +function isManifestUri(uri: string): boolean { + return skillNameFromManifestUri(uri) !== undefined; +} + +/** A `_meta` key that already carries a reverse-domain namespace (`vendor.tld/…`). */ +function isNamespacedMetaKey(key: string): boolean { + return /^[a-z0-9-]+(\.[a-z0-9-]+)+\//i.test(key); +} + +export class SkillsManifestScenario implements ClientScenario { + name = 'sep-2640-skills-manifest'; + readonly source = { extensionId: SKILLS_EXTENSION_ID } as const; + description = `SEP-2640 Skills extension: the \`SKILL.md\` manifest resource. + +**Resource**: \`skill:///SKILL.md\` (read via \`resources/read\`) + +**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-2640.yaml): + +- \`sep-2640-skillmd-mimetype\` — the SKILL.md resource \`mimeType\` SHOULD be \`text/markdown\` +- \`sep-2640-skillmd-metadata-name\` — the resource \`name\` SHOULD be the frontmatter \`name\` +- \`sep-2640-skillmd-metadata-description\` — the resource \`description\` SHOULD be the frontmatter \`description\` +- \`sep-2640-final-segment-equals-name\` — the final \`\` segment MUST equal the frontmatter \`name\` +- \`sep-2640-meta-prefix\` — un-namespaced skill \`_meta\` keys SHOULD use the \`io.modelcontextprotocol.skills/\` prefix + +**Discovery is dynamic**: the scenario picks the first \`skill-md\` skill it finds. Undeclared extension SKIPs; a declared extension with no discoverable SKILL.md reports the missing prerequisite (not a silent skip).`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect(); + try { + const skills = await skillsCapability(conn); + const allIds = [ + MIMETYPE_ID, + METADATA_NAME_ID, + METADATA_DESCRIPTION_ID, + FINAL_SEGMENT_ID, + META_PREFIX_ID + ]; + if (!skills) { + const reason = + 'Server did not declare the io.modelcontextprotocol/skills extension; SKILL.md checks not applicable.'; + return allIds.map((id) => + skillsCheck(id, reason, 'SKIPPED', { errorMessage: reason }) + ); + } + + // === Dynamic discovery: resources/list first (carries Resource + // metadata), then skill://index.json. === + const resources = await listAllResources(conn); + const manifestResource: SkillResource | undefined = resources.find((r) => + isManifestUri(r.uri) + ); + let manifestUri = manifestResource?.uri; + if (!manifestUri) { + const idx = await readSkillIndexText(conn); + if (!('error' in idx) && typeof idx.text === 'string') { + try { + const index = JSON.parse(idx.text) as SkillIndex; + const entry = (index.skills ?? []).find( + (e) => + e.type === 'skill-md' && + typeof e.url === 'string' && + isManifestUri(e.url) + ); + manifestUri = entry?.url; + } catch { + // A malformed index is the index scenario's concern; ignore here. + } + } + } + + if (!manifestUri) { + const reason = + 'no skill:///SKILL.md resource found via resources/list or skill://index.json'; + return [ + untestableCheck( + MIMETYPE_ID, + MIMETYPE_ID, + 'SKILL.md resource mimeType SHOULD be text/markdown.', + reason, + [SEP_2640_REF], + 'WARNING' + ), + untestableCheck( + METADATA_NAME_ID, + METADATA_NAME_ID, + 'SKILL.md resource name SHOULD match the frontmatter name.', + reason, + [SEP_2640_REF], + 'WARNING' + ), + untestableCheck( + METADATA_DESCRIPTION_ID, + METADATA_DESCRIPTION_ID, + 'SKILL.md resource description SHOULD match the frontmatter description.', + reason, + [SEP_2640_REF], + 'WARNING' + ), + untestableCheck( + FINAL_SEGMENT_ID, + FINAL_SEGMENT_ID, + 'The final segment MUST equal the frontmatter name.', + reason, + [SEP_2640_REF], + 'FAILURE' + ), + untestableCheck( + META_PREFIX_ID, + META_PREFIX_ID, + 'Skill _meta keys SHOULD use the io.modelcontextprotocol.skills/ prefix.', + reason, + [SEP_2640_REF], + 'WARNING' + ) + ]; + } + + const checks: ConformanceCheck[] = []; + + // Read the manifest content (for mimeType, frontmatter, and _meta). + let content: + | { text: string; mimeType?: string; meta?: Record } + | undefined; + let readError: string | undefined; + try { + content = await readResourceText(conn, manifestUri); + if (!content) readError = 'resources/read returned no text content'; + } catch (e) { + readError = + e instanceof JsonRpcError + ? `resources/read failed: code ${e.code}: ${e.message}` + : e instanceof Error + ? e.message + : String(e); + } + + // === skillmd-mimetype (SHOULD) === + // Prefer the read content's mimeType; fall back to the resources/list + // Resource metadata mimeType. + const mimeType = content?.mimeType ?? manifestResource?.mimeType; + if (mimeType === undefined) { + checks.push( + untestableCheck( + MIMETYPE_ID, + MIMETYPE_ID, + 'SKILL.md resource mimeType SHOULD be text/markdown.', + `no mimeType observable for ${manifestUri}${readError ? ` (${readError})` : ''}`, + [SEP_2640_REF], + 'WARNING' + ) + ); + } else { + checks.push( + skillsCheck( + MIMETYPE_ID, + 'SKILL.md resource mimeType SHOULD be text/markdown.', + mimeType === MARKDOWN_MIME ? 'SUCCESS' : 'WARNING', + mimeType === MARKDOWN_MIME + ? { details: { uri: manifestUri, mimeType } } + : { + errorMessage: `expected mimeType "${MARKDOWN_MIME}", got ${JSON.stringify(mimeType)}` + } + ) + ); + } + + // Parse the frontmatter once for the name/description/final-segment checks. + const frontmatter = content ? parseFrontmatter(content.text) : undefined; + const fmName = + typeof frontmatter?.name === 'string' ? frontmatter.name : undefined; + const fmDescription = + typeof frontmatter?.description === 'string' + ? frontmatter.description + : undefined; + + // === final-segment-equals-name (MUST) === + const uriName = skillNameFromManifestUri(manifestUri); + if (fmName === undefined || uriName === undefined) { + const missing = + fmName === undefined + ? `SKILL.md frontmatter has no string "name"${readError ? ` (${readError})` : ''}` + : `could not derive the skill name from URI ${manifestUri}`; + checks.push( + untestableCheck( + FINAL_SEGMENT_ID, + FINAL_SEGMENT_ID, + 'The final segment MUST equal the frontmatter name.', + missing, + [SEP_2640_REF], + 'FAILURE' + ) + ); + } else { + checks.push( + skillsCheck( + FINAL_SEGMENT_ID, + 'The final segment of the SKILL.md URI MUST equal the frontmatter name.', + uriName === fmName ? 'SUCCESS' : 'FAILURE', + uriName === fmName + ? { details: { uri: manifestUri, name: fmName } } + : { + errorMessage: `final path segment "${uriName}" != frontmatter name "${fmName}"` + } + ) + ); + } + + // === skillmd-metadata-name (SHOULD) — needs the Resource metadata === + if (!manifestResource) { + checks.push( + untestableCheck( + METADATA_NAME_ID, + METADATA_NAME_ID, + 'SKILL.md resource name SHOULD match the frontmatter name.', + `SKILL.md ${manifestUri} is not listed in resources/list, so its Resource name metadata is not observable`, + [SEP_2640_REF], + 'WARNING' + ) + ); + } else if (fmName === undefined) { + checks.push( + untestableCheck( + METADATA_NAME_ID, + METADATA_NAME_ID, + 'SKILL.md resource name SHOULD match the frontmatter name.', + `SKILL.md frontmatter has no string "name" to compare against${readError ? ` (${readError})` : ''}`, + [SEP_2640_REF], + 'WARNING' + ) + ); + } else { + checks.push( + skillsCheck( + METADATA_NAME_ID, + 'The SKILL.md resource name SHOULD be set from the frontmatter name.', + manifestResource.name === fmName ? 'SUCCESS' : 'WARNING', + manifestResource.name === fmName + ? { details: { name: fmName } } + : { + errorMessage: `resource name ${JSON.stringify(manifestResource.name)} != frontmatter name ${JSON.stringify(fmName)}` + } + ) + ); + } + + // === skillmd-metadata-description (SHOULD) — needs Resource metadata === + if (!manifestResource) { + checks.push( + untestableCheck( + METADATA_DESCRIPTION_ID, + METADATA_DESCRIPTION_ID, + 'SKILL.md resource description SHOULD match the frontmatter description.', + `SKILL.md ${manifestUri} is not listed in resources/list, so its Resource description metadata is not observable`, + [SEP_2640_REF], + 'WARNING' + ) + ); + } else if (fmDescription === undefined) { + checks.push( + untestableCheck( + METADATA_DESCRIPTION_ID, + METADATA_DESCRIPTION_ID, + 'SKILL.md resource description SHOULD match the frontmatter description.', + `SKILL.md frontmatter has no string "description" to compare against${readError ? ` (${readError})` : ''}`, + [SEP_2640_REF], + 'WARNING' + ) + ); + } else { + checks.push( + skillsCheck( + METADATA_DESCRIPTION_ID, + 'The SKILL.md resource description SHOULD be set from the frontmatter description.', + manifestResource.description === fmDescription + ? 'SUCCESS' + : 'WARNING', + manifestResource.description === fmDescription + ? { details: { description: fmDescription } } + : { + errorMessage: `resource description ${JSON.stringify(manifestResource.description)} != frontmatter description ${JSON.stringify(fmDescription)}` + } + ) + ); + } + + // === meta-prefix (SHOULD, conditional on _meta keys being present) === + // Union the _meta of the read content and the resources/list Resource. + const metaKeys = new Set([ + ...Object.keys(content?.meta ?? {}), + ...Object.keys(manifestResource?._meta ?? {}) + ]); + if (metaKeys.size === 0) { + checks.push( + skillsCheck( + META_PREFIX_ID, + 'When _meta keys are used for skill resources, they SHOULD use the io.modelcontextprotocol.skills/ reverse-domain prefix.', + 'SUCCESS', + { details: { note: 'skill resource exposes no _meta keys' } } + ) + ); + } else { + // Only bare (un-namespaced) keys are flagged: a key already carrying a + // reverse-domain namespace is the intended shape, whichever vendor. + const bareKeys = [...metaKeys].filter((k) => !isNamespacedMetaKey(k)); + checks.push( + skillsCheck( + META_PREFIX_ID, + 'When _meta keys are used for skill resources, they SHOULD use the io.modelcontextprotocol.skills/ reverse-domain prefix.', + bareKeys.length === 0 ? 'SUCCESS' : 'WARNING', + bareKeys.length === 0 + ? { details: { metaKeys: [...metaKeys] } } + : { + errorMessage: `un-namespaced skill _meta keys SHOULD use the ${SKILLS_META_PREFIX} prefix: ${bareKeys.join(', ')}` + } + ) + ); + } + + return checks; + } finally { + await conn.close(); + } + } +} diff --git a/src/seps/sep-2640.yaml b/src/seps/sep-2640.yaml index 024d06c3..d93e1487 100644 --- a/src/seps/sep-2640.yaml +++ b/src/seps/sep-2640.yaml @@ -10,6 +10,20 @@ # drifted ~11 others' verbatim wording. Full re-extraction at SEP HEAD # is mcpkit#780's lifecycle; this file deliberately holds at 556154c # until that lands. +# backing_scenarios: three server ClientScenarios under +# src/scenarios/server/skills/ emit the check IDs below (a row is "tested" +# once a scenario emits its check ID; see src/traceability/): +# directory.ts (sep-2640-skills-directory) — sep-2640-capability-directory-read-flag +# and the five sep-2640-directory-read-* rows. +# index.ts (sep-2640-skills-index) — sep-2640-server-expose-index, +# sep-2640-index-entry-type-enum, sep-2640-index-name-required, +# sep-2640-index-digest-required, sep-2640-skill-uri-scheme. +# manifest.ts (sep-2640-skills-manifest) — sep-2640-skillmd-mimetype, +# sep-2640-skillmd-metadata-name, sep-2640-skillmd-metadata-description, +# sep-2640-final-segment-equals-name, sep-2640-meta-prefix. +# The remaining rows are host-internal or off-wire (host load-by-uri, digest +# verification, byte-budget / archive-unpack safety, no-empty-index +# assumption) and stay traceability-only for this server-scenario set. sep: 2640 spec_url: https://modelcontextprotocol.io/seps/2640-skills-extension#specification requirements: From 0707a3eb85a12ab40706bd95f563b807ea12a0de Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Fri, 28 Aug 2026 17:15:54 -0700 Subject: [PATCH 07/12] feat(sep-2640): re-extract against the 2026-08-21 rewrite The prior extraction held at SEP commit 556154c (2026-06-05) by an explicit in-file decision to wait for the index-schema refactor. The 08-21 core-maintainer rewrite made that pin actively wrong rather than merely behind. Removed as no longer normative: the skill://index.json well-known resource (absent from the current text), the entry type enum, the no-nested-skills rule (now reversed - skills MAY nest), and the five archive rows (archives are deferred to an appendix). Reworded the two index-field rows, the empty-listing row and the verify-digest row. Added the new surface: skills/list and skills/get with their pagination, atomicity and caching attributes; the {uri, frontmatter, resources} entry schema; resources completeness, the "dynamic" sentinel and per-file size; the 512-entry and 16 MiB limits; and the host obligations around lazy retrieval, content-bound approval and frontmatter re-verification. index.ts is replaced by enumeration.ts, which drives both new methods. manifest.ts and directory.ts swap their skill://index.json discovery fallback for skills/list. The capability helper no longer accepts a config envelope. SEP-2133 is Final and defines the extensions field as a map of identifiers to settings objects, with no envelope and no slot for id/specVersion/stability; SEP-2640's capability block matches. The earlier leniency was based on a misreading. 77 rows: 65 checks (39 wire-tested by the three server scenarios) and 12 excluded as host-internal. --- src/scenarios/index.ts | 6 +- src/scenarios/server/skills/directory.ts | 36 +- src/scenarios/server/skills/enumeration.ts | 1020 ++++++++++++++++++++ src/scenarios/server/skills/helpers.ts | 238 ++++- src/scenarios/server/skills/index.ts | 270 ------ src/scenarios/server/skills/manifest.ts | 23 +- src/seps/sep-2640.yaml | 228 ++++- 7 files changed, 1414 insertions(+), 407 deletions(-) create mode 100644 src/scenarios/server/skills/enumeration.ts delete mode 100644 src/scenarios/server/skills/index.ts diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 8fe9a610..52c8ed0f 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -59,7 +59,7 @@ import { } from './server/resources'; import { SkillsDirectoryReadScenario } from './server/skills/directory'; -import { SkillsIndexScenario } from './server/skills/index'; +import { SkillsEnumerationScenario } from './server/skills/enumeration'; import { SkillsManifestScenario } from './server/skills/manifest'; import { @@ -162,7 +162,7 @@ const pendingClientScenariosList: ClientScenario[] = [ // SEP-2640-conformant fixture via // `npm start -- server --scenario sep-2640-skills-* --url `. new SkillsDirectoryReadScenario(), - new SkillsIndexScenario(), + new SkillsEnumerationScenario(), new SkillsManifestScenario() ]; @@ -218,7 +218,7 @@ const allClientScenariosList: ClientScenario[] = [ // Skills extension (SEP-2640). Fixture-dependent (needs a SEP-2640 server); // each scenario SKIPs cleanly when the extension is not declared. new SkillsDirectoryReadScenario(), - new SkillsIndexScenario(), + new SkillsEnumerationScenario(), new SkillsManifestScenario(), // Prompts scenarios diff --git a/src/scenarios/server/skills/directory.ts b/src/scenarios/server/skills/directory.ts index 27be77bd..4c5c2b02 100644 --- a/src/scenarios/server/skills/directory.ts +++ b/src/scenarios/server/skills/directory.ts @@ -27,13 +27,12 @@ import { SEP_2640_REF, JSONRPC_METHOD_NOT_FOUND, JSONRPC_INVALID_PARAMS, - type SkillIndex, type SkillResource, skillsCapability, directoryReadDeclared, skillsCheck, listAllResources, - readSkillIndexText, + skillsListAll, skillNameFromManifestUri } from './helpers'; @@ -81,26 +80,19 @@ function skillRootFromManifestUri(uri: string): string | undefined { async function discoverDirectory( conn: Connection ): Promise { - // 1. skill-md entry in skill://index.json — its SKILL.md URL gives us both a - // directory (the skill root) and a known file (the SKILL.md itself). - const idx = await readSkillIndexText(conn); - if (!('error' in idx) && typeof idx.text === 'string') { - try { - const index = JSON.parse(idx.text) as SkillIndex; - const entry = (index.skills ?? []).find( - (e) => - e.type === 'skill-md' && - typeof e.url === 'string' && - skillRootFromManifestUri(e.url) !== undefined - ); - if (entry?.url) { - return { - dirUri: skillRootFromManifestUri(entry.url)!, - fileUri: entry.url - }; - } - } catch { - // A malformed index is the index scenario's concern; keep discovering. + // 1. A skills/list entry — its SKILL.md URI gives us both a directory (the + // skill root) and a known file (the SKILL.md itself). An unenumerable + // catalog returns nothing here, so discovery falls through. + const listed = await skillsListAll(conn); + if (!('error' in listed)) { + const entry = listed.entries.find( + (e) => + typeof e.uri === 'string' && + skillRootFromManifestUri(e.uri) !== undefined + ); + const uri = entry?.uri as string | undefined; + if (uri) { + return { dirUri: skillRootFromManifestUri(uri)!, fileUri: uri }; } } diff --git a/src/scenarios/server/skills/enumeration.ts b/src/scenarios/server/skills/enumeration.ts new file mode 100644 index 00000000..837a1568 --- /dev/null +++ b/src/scenarios/server/skills/enumeration.ts @@ -0,0 +1,1020 @@ +/** + * SEP-2640 Skills extension — the `skills/list` and `skills/get` surface. + * + * Replaces the former `skill://index.json` scenario. The 2026-08-21 revision of + * the SEP removed that well-known resource entirely (it appears nowhere in the + * current text) and replaced it with two methods that every server declaring the + * extension MUST implement. + * + * One scenario, many checks (per AGENTS.md "fewer scenarios, more checks"). + * Each check's verbatim spec quote lives next to its check ID in + * src/seps/sep-2640.yaml, keeping the YAML and this scenario in lock-step. + * + * All discovery is dynamic and brand-neutral: the scenario enumerates whatever + * the server serves and validates the entries it finds, hardcoding no + * fixture-specific skill name or URI. When the server does not declare the + * skills extension the checks are SKIPPED (an optional, undeclared capability). + * An empty listing is explicitly permitted, so entry-level checks SKIP rather + * than fail against a server with an unenumerable catalog. + */ + +import { ClientScenario, ConformanceCheck } from '../../../types'; +import type { RunContext } from '../../../connection'; +import { + SKILLS_EXTENSION_ID, + SKILLS_LIST_METHOD, + SKILLS_GET_METHOD, + SKILL_URI_SCHEME, + SKILL_MANIFEST_FILENAME, + SKILL_DIGEST_PATTERN, + RESOURCES_DYNAMIC, + MAX_RESOURCES_PER_SKILL, + MAX_TOTAL_SIZE_PER_SKILL, + FRONTMATTER_RESERVED_PREFIX, + JSONRPC_INVALID_PARAMS, + type SkillEntry, + type SkillResourceEntry, + skillsCapability, + skillsCheck, + skillsListAll, + skillsGet, + settingsAreInline, + skillNameFromManifestUri, + skillRootFromManifestUri, + isDynamicResources, + resourcesArray, + entryLabel, + readResourceText, + parseFrontmatter +} from './helpers'; + +const CAPABILITY_IDS = [ + 'sep-2640-capability-declaration-inline', + 'sep-2640-capability-commits-to-methods', + 'sep-2640-capability-empty-object' +] as const; + +const LIST_IDS = [ + 'sep-2640-skills-list-implemented', + 'sep-2640-skills-list-pagination', + 'sep-2640-skills-list-entry-atomic', + 'sep-2640-skills-list-cache-attributes' +] as const; + +const ENTRY_IDS = [ + 'sep-2640-entry-uri-required', + 'sep-2640-entry-frontmatter-required', + 'sep-2640-entry-uri-matches-frontmatter-name', + 'sep-2640-skill-uri-scheme', + 'sep-2640-entry-resources-required', + 'sep-2640-resources-complete', + 'sep-2640-resources-uri-within-skill', + 'sep-2640-resources-digest-format', + 'sep-2640-resources-size-required', + 'sep-2640-limit-resources-per-skill', + 'sep-2640-limit-total-size', + 'sep-2640-metadata-reserved-prefix', + 'sep-2640-name-naming-rules', + 'sep-2640-authority-reg-name' +] as const; + +const GET_IDS = [ + 'sep-2640-skills-get-implemented', + 'sep-2640-skills-get-entry-shape', + 'sep-2640-skills-get-no-cursor', + 'sep-2640-skills-get-unknown-uri-invalid-params' +] as const; + +/** Emitted by the read-back pass, which fetches one listed SKILL.md. */ +const READBACK_IDS = [ + 'sep-2640-skillmd-required', + 'sep-2640-skillmd-frontmatter', + 'sep-2640-entry-frontmatter-identical' +] as const; + +const ALL_CHECK_IDS = [ + ...CAPABILITY_IDS, + ...LIST_IDS, + ...ENTRY_IDS, + ...GET_IDS, + ...READBACK_IDS +]; + +/** + * Agent Skills naming rules as the SEP defers to them: 1-64 characters, + * lowercase alphanumeric and hyphens. + */ +const SKILL_NAME_PATTERN = /^[a-z0-9]([a-z0-9-]{0,62}[a-z0-9])?$/; + +/** RFC 3986 reg-name: unreserved / pct-encoded / sub-delims, case-insensitive. */ +const REG_NAME_PATTERN = /^(?:[A-Za-z0-9\-._~!$&'()*+,;=]|%[0-9A-Fa-f]{2})*$/; + +/** A URI that no conformant server should serve, for the -32602 probe. */ +const UNKNOWN_SKILL_URI = + 'skill://mcp-conformance-nonexistent-skill-9f3a2b/SKILL.md'; + +function joinErrs(errs: string[], limit = 5): string { + const shown = errs.slice(0, limit).join('; '); + return errs.length > limit + ? `${shown} (+${errs.length - limit} more)` + : shown; +} + +export class SkillsEnumerationScenario implements ClientScenario { + name = 'sep-2640-skills-enumeration'; + readonly source = { extensionId: SKILLS_EXTENSION_ID } as const; + description = `SEP-2640 Skills extension: \`skills/list\` enumeration and \`skills/get\` retrieval. + +**Methods**: \`skills/list\`, \`skills/get\` (both mandatory for a server declaring \`io.modelcontextprotocol/skills\`) + +**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-2640.yaml): + +- \`sep-2640-capability-declaration-inline\` — extension settings sit inline under the identifier, per SEP-2133 (no \`config\` envelope) +- \`sep-2640-capability-commits-to-methods\` — declaring the extension commits the server to both methods +- \`sep-2640-skills-list-implemented\` — \`skills/list\` is implemented and returns a \`skills\` array +- \`sep-2640-skills-list-pagination\` — \`nextCursor\` is honoured as a cursor on the next request +- \`sep-2640-skills-list-entry-atomic\` — no skill entry is split across pages +- \`sep-2640-entry-uri-required\` / \`sep-2640-entry-frontmatter-required\` / \`sep-2640-entry-resources-required\` — the three required entry fields +- \`sep-2640-entry-uri-matches-frontmatter-name\` — the final skill-path segment equals \`frontmatter.name\` +- \`sep-2640-resources-complete\` — \`resources\` includes an entry matching the skill's own \`uri\`, each file once +- \`sep-2640-resources-uri-within-skill\` / \`sep-2640-resources-digest-format\` / \`sep-2640-resources-size-required\` — the \`{uri, digest, size}\` triple +- \`sep-2640-limit-resources-per-skill\` / \`sep-2640-limit-total-size\` — 512 entries, 16 MiB +- \`sep-2640-metadata-reserved-prefix\` — frontmatter \`metadata\` keys under \`io.modelcontextprotocol/\` are reserved +- \`sep-2640-skills-get-*\` — \`skills/get\` returns a list-shaped entry, carries no cursor, and answers \`-32602\` for an unknown URI + +**Discovery is dynamic**: an undeclared extension SKIPs everything; an empty or partial listing is permitted and SKIPs the entry-level checks.`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect(); + try { + const skills = await skillsCapability(conn); + if (!skills) { + const reason = + 'Server did not declare the io.modelcontextprotocol/skills extension; enumeration checks not applicable.'; + return ALL_CHECK_IDS.map((id) => + skillsCheck(id, reason, 'SKIPPED', { errorMessage: reason }) + ); + } + + const checks: ConformanceCheck[] = []; + + // === capability-declaration-inline === + // SEP-2133 (Final) maps an extension identifier straight to its settings + // object. An envelope hides settings from any spec-following client. + const { inline, envelopeKeys } = settingsAreInline(skills); + checks.push( + skillsCheck( + 'sep-2640-capability-declaration-inline', + 'Extension settings are a map of extension identifiers to per-extension settings objects; the settings sit directly under the identifier.', + inline ? 'SUCCESS' : 'FAILURE', + inline + ? { details: { settingKeys: Object.keys(skills) } } + : { + errorMessage: `capabilities.extensions["${SKILLS_EXTENSION_ID}"] carries envelope key(s) ${envelopeKeys.join(', ')} instead of the settings object itself. SEP-2133 (Final) defines no envelope, and SEP-2640's capability block places directoryRead inline.`, + details: { envelopeKeys, observed: skills } + } + ) + ); + + // === capability-empty-object === + // "An empty object indicates support for the extension with no optional + // features." Observable as: the declared value is a JSON object, and an + // empty one is a valid declaration rather than a malformed capability. + checks.push( + skillsCheck( + 'sep-2640-capability-empty-object', + 'An empty object indicates support for the extension with no optional features.', + 'SUCCESS', + { + details: { + declaredKeys: Object.keys(skills), + empty: Object.keys(skills).length === 0 + } + } + ) + ); + + // === skills/list === + const listed = await skillsListAll(conn); + if ('error' in listed) { + const reason = `${SKILLS_LIST_METHOD} failed with code ${listed.error.code}: ${listed.error.message}. A server declaring the extension MUST implement it.`; + checks.push( + skillsCheck( + 'sep-2640-capability-commits-to-methods', + 'Declaring the extension commits the server to skills/list and skills/get.', + 'FAILURE', + { errorMessage: reason } + ), + skillsCheck( + 'sep-2640-skills-list-implemented', + 'A server declaring the extension MUST implement the skills/list method.', + 'FAILURE', + { errorMessage: reason } + ) + ); + for (const id of [ + 'sep-2640-skills-list-pagination', + 'sep-2640-skills-list-entry-atomic', + ...ENTRY_IDS, + ...GET_IDS, + ...READBACK_IDS + ]) { + checks.push( + skillsCheck(id, 'skills/list is unavailable.', 'SKIPPED', { + errorMessage: reason + }) + ); + } + return checks; + } + + const { entries, pages, truncated } = listed; + + checks.push( + skillsCheck( + 'sep-2640-skills-list-implemented', + 'A server declaring the extension MUST implement the skills/list method, which returns the skills it serves. The result MAY be empty.', + 'SUCCESS', + { + details: { + pages: pages.length, + entries: entries.length, + emptyListingPermitted: entries.length === 0 + } + } + ) + ); + + // === skills-list-pagination === + // Multi-page runs prove the cursor round-trips. A single page is a clean + // pass: the contract is "when nextCursor is present, pass it back", and + // skillsListAll did exactly that to reach the end. + checks.push( + skillsCheck( + 'sep-2640-skills-list-pagination', + 'Pagination mirrors the base protocol: the request accepts an optional cursor, and when the result includes nextCursor the client passes it back.', + truncated ? 'FAILURE' : 'SUCCESS', + truncated + ? { + errorMessage: `skills/list did not terminate: the server kept returning a nextCursor (or repeated one) across ${pages.length} pages.` + } + : { details: { pages: pages.length } } + ) + ); + + // === skills-list-entry-atomic === + // "An entry is atomic — a skill's resources set is never split across + // pages." Observable as a URI appearing in more than one page. + const uriPages = new Map(); + pages.forEach((page, pageIdx) => { + for (const e of page.entries) { + if (typeof e.uri !== 'string') continue; + const seen = uriPages.get(e.uri) ?? []; + if (!seen.includes(pageIdx)) seen.push(pageIdx); + uriPages.set(e.uri, seen); + } + }); + const split = [...uriPages.entries()] + .filter(([, p]) => p.length > 1) + .map(([uri, p]) => `${uri} appears on pages ${p.join(', ')}`); + checks.push( + skillsCheck( + 'sep-2640-skills-list-entry-atomic', + "An entry is atomic — a skill's resources set is never split across pages.", + split.length === 0 ? 'SUCCESS' : 'FAILURE', + split.length === 0 + ? { details: { pages: pages.length, distinctUris: uriPages.size } } + : { errorMessage: joinErrs(split) } + ) + ); + + // === skills-list-cache-attributes === + // SEP-2549 attributes are required only on protocol 2026-07-28 and later. + // The harness does not gate on the negotiated version here, so an absent + // attribute is reported as a WARNING rather than a failure. + const first = pages[0]?.result ?? {}; + const hasTtl = first.ttlMs !== undefined; + const hasScope = first.cacheScope !== undefined; + checks.push( + skillsCheck( + 'sep-2640-skills-list-cache-attributes', + "In protocol versions 2026-07-28 and later, the skills/list result carries the base protocol's list-caching attributes ttlMs and cacheScope (SEP-2549).", + hasTtl && hasScope ? 'SUCCESS' : 'WARNING', + hasTtl && hasScope + ? { details: { ttlMs: first.ttlMs, cacheScope: first.cacheScope } } + : { + errorMessage: `skills/list result omits ${[!hasTtl && 'ttlMs', !hasScope && 'cacheScope'].filter(Boolean).join(' and ')}. Required only on protocol 2026-07-28 and later; on an earlier negotiated version this is expected.` + } + ) + ); + + if (entries.length === 0) { + const reason = + 'skills/list returned no entries; entry-level checks not applicable. A server whose catalog is large, generated on demand, or otherwise unenumerable MAY return an empty listing.'; + for (const id of [...ENTRY_IDS, ...GET_IDS, ...READBACK_IDS]) { + checks.push( + skillsCheck(id, reason, 'SKIPPED', { errorMessage: reason }) + ); + } + checks.push( + skillsCheck( + 'sep-2640-capability-commits-to-methods', + 'Declaring the extension commits the server to skills/list and skills/get.', + 'SUCCESS', + { details: { note: 'skills/list answered; listing is empty.' } } + ) + ); + return checks; + } + + checks.push(...entryChecks(entries)); + checks.push(...(await getChecks(conn, entries))); + checks.push(...(await readbackChecks(conn, entries))); + + return checks; + } finally { + await conn.close(); + } + } +} + +/** Validate every `skills[]` entry against the §Discovery entry schema. */ +function entryChecks(entries: SkillEntry[]): ConformanceCheck[] { + const checks: ConformanceCheck[] = []; + + // === entry-uri-required === + const uriErrs = entries + .map((e, i) => + typeof e.uri === 'string' && e.uri.length > 0 + ? null + : `skills[${i}].uri is missing or not a string` + ) + .filter((x): x is string => x !== null); + checks.push( + skillsCheck( + 'sep-2640-entry-uri-required', + "Every entry carries uri, the full resource URI of the skill's SKILL.md.", + uriErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + uriErrs.length === 0 + ? { details: { entryCount: entries.length } } + : { errorMessage: joinErrs(uriErrs) } + ) + ); + + // === entry-frontmatter-required === + // Verbatim frontmatter, so name and description are always present. + const fmErrs: string[] = []; + entries.forEach((e, i) => { + const fm = e.frontmatter; + if (!fm || typeof fm !== 'object' || Array.isArray(fm)) { + fmErrs.push(`${entryLabel(e, i)}: frontmatter is missing or not an object`); + return; + } + const obj = fm as Record; + if (typeof obj.name !== 'string' || obj.name.length === 0) { + fmErrs.push(`${entryLabel(e, i)}: frontmatter.name is missing or empty`); + } + if (typeof obj.description !== 'string' || obj.description.length === 0) { + fmErrs.push( + `${entryLabel(e, i)}: frontmatter.description is missing or empty` + ); + } + }); + checks.push( + skillsCheck( + 'sep-2640-entry-frontmatter-required', + "frontmatter is the skill's SKILL.md YAML frontmatter rendered verbatim as a JSON object; because the Agent Skills specification requires name and description, those fields are always present.", + fmErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + fmErrs.length === 0 + ? { details: { entryCount: entries.length } } + : { errorMessage: joinErrs(fmErrs) } + ) + ); + + // === entry-uri-matches-frontmatter-name === + const nameErrs: string[] = []; + entries.forEach((e) => { + if (typeof e.uri !== 'string') return; + const fm = e.frontmatter as Record | undefined; + const declared = fm && typeof fm.name === 'string' ? fm.name : undefined; + if (declared === undefined) return; + const fromUri = skillNameFromManifestUri(e.uri); + if (fromUri === undefined) { + nameErrs.push( + `${e.uri}: does not end in /${SKILL_MANIFEST_FILENAME}, so the skill name is not recoverable from the URI` + ); + } else if (fromUri !== declared) { + nameErrs.push( + `${e.uri}: final skill-path segment "${fromUri}" !== frontmatter.name "${declared}"` + ); + } + }); + checks.push( + skillsCheck( + 'sep-2640-entry-uri-matches-frontmatter-name', + "The final segment of the entry's uri MUST equal frontmatter.name.", + nameErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + nameErrs.length === 0 + ? { details: { entryCount: entries.length } } + : { errorMessage: joinErrs(nameErrs) } + ) + ); + + // === skill-uri-scheme (SHOULD) === + // "Servers SHOULD use the skill:// URI scheme", but a server MAY serve + // skills under another scheme native to its domain and no scheme is + // privileged, so a deviation is a WARNING rather than a failure. + const otherScheme = entries + .filter((e) => typeof e.uri === 'string' && !e.uri.startsWith(SKILL_URI_SCHEME)) + .map((e) => String(e.uri)); + checks.push( + skillsCheck( + 'sep-2640-skill-uri-scheme', + 'Servers SHOULD use the skill:// URI scheme for the resources of a skill.', + otherScheme.length === 0 ? 'SUCCESS' : 'WARNING', + otherScheme.length === 0 + ? { details: { entryCount: entries.length } } + : { + errorMessage: `Entries served under another scheme (explicitly permitted; no scheme is privileged): ${joinErrs(otherScheme)}` + } + ) + ); + + // === entry-resources-required === + const resErrs: string[] = []; + entries.forEach((e, i) => { + if (isDynamicResources(e)) return; + if (resourcesArray(e) !== undefined) return; + resErrs.push( + `${entryLabel(e, i)}: resources is ${JSON.stringify(e.resources)}, neither an array nor "${RESOURCES_DYNAMIC}"` + ); + }); + checks.push( + skillsCheck( + 'sep-2640-entry-resources-required', + 'resources is REQUIRED on every skill entry and takes one of two forms: an array of {uri, digest, size} triples, or the string "dynamic". An entry with no resources at all, or with any other value, is invalid.', + resErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + resErrs.length === 0 + ? { + details: { + entryCount: entries.length, + dynamicEntries: entries.filter(isDynamicResources).length + } + } + : { errorMessage: joinErrs(resErrs) } + ) + ); + + // Entries carrying an array are the only ones the remaining checks apply to. + const arrayEntries = entries + .map((e, i) => ({ e, i, arr: resourcesArray(e) })) + .filter((x): x is { e: SkillEntry; i: number; arr: SkillResourceEntry[] } => + x.arr !== undefined + ); + + const dynamicOnlyReason = + 'Every entry declares "resources": "dynamic", which publishes no file manifest; the resources-array checks are not applicable.'; + + if (arrayEntries.length === 0) { + for (const id of [ + 'sep-2640-resources-complete', + 'sep-2640-resources-uri-within-skill', + 'sep-2640-resources-digest-format', + 'sep-2640-resources-size-required', + 'sep-2640-limit-resources-per-skill', + 'sep-2640-limit-total-size' + ]) { + checks.push( + skillsCheck(id, dynamicOnlyReason, 'SKIPPED', { + errorMessage: dynamicOnlyReason + }) + ); + } + } else { + // === resources-complete === + // Observable half: an entry matching the skill's own uri, and no file + // listed twice. Full completeness (every file of the skill) cannot be + // confirmed from the wire without a second source of truth. + const completeErrs: string[] = []; + for (const { e, i, arr } of arrayEntries) { + const own = typeof e.uri === 'string' ? e.uri : undefined; + if (own && !arr.some((r) => r.uri === own)) { + completeErrs.push( + `${entryLabel(e, i)}: resources has no entry matching the skill's own uri (the SKILL.md digest and size)` + ); + } + const seen = new Set(); + const dupes = new Set(); + for (const r of arr) { + if (typeof r.uri !== 'string') continue; + if (seen.has(r.uri)) dupes.add(r.uri); + seen.add(r.uri); + } + if (dupes.size > 0) { + completeErrs.push( + `${entryLabel(e, i)}: resources lists ${[...dupes].join(', ')} more than once` + ); + } + } + checks.push( + skillsCheck( + 'sep-2640-resources-complete', + 'When present, resources MUST be complete: it lists every file of the skill, each exactly once, including an entry matching the skill top-level uri — that entry carries the digest and size of SKILL.md itself.', + completeErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + completeErrs.length === 0 + ? { details: { entriesChecked: arrayEntries.length } } + : { errorMessage: joinErrs(completeErrs) } + ) + ); + + // === resources-uri-within-skill === + const containErrs: string[] = []; + for (const { e, i, arr } of arrayEntries) { + const own = typeof e.uri === 'string' ? e.uri : undefined; + const root = own ? skillRootFromManifestUri(own) : undefined; + if (!root) continue; + for (const r of arr) { + if (typeof r.uri !== 'string') { + containErrs.push(`${entryLabel(e, i)}: a resources entry has no uri`); + continue; + } + if (r.uri !== own && !r.uri.startsWith(`${root}/`)) { + containErrs.push( + `${entryLabel(e, i)}: ${r.uri} is outside the skill directory ${root}` + ); + } + } + } + checks.push( + skillsCheck( + 'sep-2640-resources-uri-within-skill', + "Each uri MUST be the skill's SKILL.md or a file within the skill's directory.", + containErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + containErrs.length === 0 + ? { details: { entriesChecked: arrayEntries.length } } + : { errorMessage: joinErrs(containErrs) } + ) + ); + + // === resources-digest-format === + const digestErrs: string[] = []; + for (const { e, i, arr } of arrayEntries) { + for (const r of arr) { + if (typeof r.digest !== 'string' || !SKILL_DIGEST_PATTERN.test(r.digest)) { + digestErrs.push( + `${entryLabel(e, i)}: ${String(r.uri)} digest=${JSON.stringify(r.digest)} is not sha256:{64 lowercase hex}` + ); + } + } + } + checks.push( + skillsCheck( + 'sep-2640-resources-digest-format', + "Digests are SHA-256 hashes of an artifact's raw bytes, formatted as sha256:{hex} where {hex} is 64 lowercase hexadecimal characters.", + digestErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + digestErrs.length === 0 + ? { details: { entriesChecked: arrayEntries.length } } + : { errorMessage: joinErrs(digestErrs) } + ) + ); + + // === resources-size-required === + const sizeErrs: string[] = []; + for (const { e, i, arr } of arrayEntries) { + for (const r of arr) { + if ( + typeof r.size !== 'number' || + !Number.isInteger(r.size) || + r.size < 0 + ) { + sizeErrs.push( + `${entryLabel(e, i)}: ${String(r.uri)} size=${JSON.stringify(r.size)} is not a non-negative integer` + ); + } + } + } + checks.push( + skillsCheck( + 'sep-2640-resources-size-required', + "Each entry MUST carry size: the length in bytes of the file's raw content — the same bytes the digest covers.", + sizeErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + sizeErrs.length === 0 + ? { details: { entriesChecked: arrayEntries.length } } + : { errorMessage: joinErrs(sizeErrs) } + ) + ); + + // === limit-resources-per-skill (SHOULD NOT exceed) === + const overCount = arrayEntries + .filter(({ arr }) => arr.length > MAX_RESOURCES_PER_SKILL) + .map(({ e, i, arr }) => `${entryLabel(e, i)}: ${arr.length} entries`); + checks.push( + skillsCheck( + 'sep-2640-limit-resources-per-skill', + 'Servers SHOULD NOT serve a skill exceeding 512 resource entries, counted over the entries of the skill resources, SKILL.md included.', + overCount.length === 0 ? 'SUCCESS' : 'WARNING', + overCount.length === 0 + ? { + details: { + maxObserved: Math.max( + ...arrayEntries.map(({ arr }) => arr.length) + ), + limit: MAX_RESOURCES_PER_SKILL + } + } + : { errorMessage: joinErrs(overCount) } + ) + ); + + // === limit-total-size (SHOULD NOT exceed) === + const sums = arrayEntries.map(({ e, i, arr }) => ({ + label: entryLabel(e, i), + total: arr.reduce( + (acc, r) => acc + (typeof r.size === 'number' ? r.size : 0), + 0 + ) + })); + const overSize = sums + .filter((s) => s.total > MAX_TOTAL_SIZE_PER_SKILL) + .map((s) => `${s.label}: ${s.total} bytes`); + checks.push( + skillsCheck( + 'sep-2640-limit-total-size', + 'Servers SHOULD NOT serve a skill whose total file size exceeds 16 MiB (16,777,216 bytes), summed over the skill resources.', + overSize.length === 0 ? 'SUCCESS' : 'WARNING', + overSize.length === 0 + ? { + details: { + maxObservedBytes: Math.max(...sums.map((s) => s.total)), + limit: MAX_TOTAL_SIZE_PER_SKILL + } + } + : { errorMessage: joinErrs(overSize) } + ) + ); + } + + // === metadata-reserved-prefix === + // This extension currently defines no keys under the reserved prefix, so a + // server publishing one is squatting on a namespace reserved for MCP. + const reservedErrs: string[] = []; + entries.forEach((e, i) => { + const fm = e.frontmatter as Record | undefined; + const meta = fm?.metadata; + if (!meta || typeof meta !== 'object' || Array.isArray(meta)) return; + for (const key of Object.keys(meta as Record)) { + if (key.startsWith(FRONTMATTER_RESERVED_PREFIX)) { + reservedErrs.push(`${entryLabel(e, i)}: frontmatter.metadata["${key}"]`); + } + } + }); + checks.push( + skillsCheck( + 'sep-2640-metadata-reserved-prefix', + 'Within the frontmatter metadata object, keys prefixed with io.modelcontextprotocol/ are reserved for metadata defined by MCP extensions. This extension currently defines no such keys.', + reservedErrs.length === 0 ? 'SUCCESS' : 'WARNING', + reservedErrs.length === 0 + ? { details: { entryCount: entries.length } } + : { + errorMessage: `Keys under the reserved prefix, which this extension does not currently define: ${joinErrs(reservedErrs)}` + } + ) + ); + + // === name-naming-rules === + // The name is recoverable from the URI alone, so this is checkable without + // fetching anything. + const badNames: string[] = []; + entries.forEach((e, i) => { + if (typeof e.uri !== 'string') return; + const name = skillNameFromManifestUri(e.uri); + if (name !== undefined && !SKILL_NAME_PATTERN.test(name)) { + badNames.push(`${entryLabel(e, i)}: name "${name}"`); + } + }); + checks.push( + skillsCheck( + 'sep-2640-name-naming-rules', + "The final segment, being the skill name, MUST satisfy the Agent Skills specification's naming rules (1-64 characters, lowercase alphanumeric and hyphens).", + badNames.length === 0 ? 'SUCCESS' : 'FAILURE', + badNames.length === 0 + ? { details: { entryCount: entries.length } } + : { errorMessage: joinErrs(badNames) } + ) + ); + + // === authority-reg-name (SHOULD) === + const badAuthority: string[] = []; + entries.forEach((e, i) => { + if (typeof e.uri !== 'string') return; + const schemeEnd = e.uri.indexOf('://'); + if (schemeEnd < 0) return; + const segments = e.uri + .slice(schemeEnd + 3) + .split('/') + .filter((x) => x.length > 0); + const authority = segments[0]; + if (authority !== undefined && !REG_NAME_PATTERN.test(authority)) { + badAuthority.push(`${entryLabel(e, i)}: authority "${authority}"`); + } + }); + checks.push( + skillsCheck( + 'sep-2640-authority-reg-name', + 'The first segment occupies the authority component and SHOULD be a valid reg-name per RFC 3986.', + badAuthority.length === 0 ? 'SUCCESS' : 'WARNING', + badAuthority.length === 0 + ? { details: { entryCount: entries.length } } + : { errorMessage: joinErrs(badAuthority) } + ) + ); + + return checks; +} + +/** + * Fetch one listed `SKILL.md` and check it against the entry that advertised + * it. This is the server-side half of the host's frontmatter-comparison MUST: + * if the entry's `frontmatter` does not match the file, no conforming host can + * load the skill. + */ +async function readbackChecks( + conn: Parameters[0], + entries: SkillEntry[] +): Promise { + const checks: ConformanceCheck[] = []; + const sample = entries.find( + (e) => typeof e.uri === 'string' && e.uri.endsWith(`/${SKILL_MANIFEST_FILENAME}`) + ); + const uri = sample?.uri as string | undefined; + + if (!uri) { + const reason = `No listed entry has a uri ending in /${SKILL_MANIFEST_FILENAME}, so no SKILL.md can be read back.`; + for (const id of READBACK_IDS) { + checks.push(skillsCheck(id, reason, 'SKIPPED', { errorMessage: reason })); + } + return checks; + } + + let body: Awaited>; + try { + body = await readResourceText(conn, uri); + } catch (e) { + const reason = `resources/read on ${uri} failed: ${e instanceof Error ? e.message : String(e)}`; + checks.push( + skillsCheck( + 'sep-2640-skillmd-required', + 'Every skill MUST contain a SKILL.md file at its root.', + 'FAILURE', + { errorMessage: reason } + ) + ); + for (const id of ['sep-2640-skillmd-frontmatter', 'sep-2640-entry-frontmatter-identical']) { + checks.push( + skillsCheck(id, 'SKILL.md is unreadable.', 'SKIPPED', { + errorMessage: reason + }) + ); + } + return checks; + } + + if (!body) { + const reason = `resources/read on ${uri} returned no text content, so the listed SKILL.md is not retrievable.`; + checks.push( + skillsCheck( + 'sep-2640-skillmd-required', + 'Every skill MUST contain a SKILL.md file at its root.', + 'FAILURE', + { errorMessage: reason } + ) + ); + for (const id of ['sep-2640-skillmd-frontmatter', 'sep-2640-entry-frontmatter-identical']) { + checks.push( + skillsCheck(id, 'No SKILL.md content to inspect.', 'SKIPPED', { + errorMessage: reason + }) + ); + } + return checks; + } + + checks.push( + skillsCheck( + 'sep-2640-skillmd-required', + 'Every skill MUST contain a SKILL.md file at its root.', + 'SUCCESS', + { details: { uri, bytes: body.text.length } } + ) + ); + + // === skillmd-frontmatter === + const fm = parseFrontmatter(body.text); + const fmErrs: string[] = []; + if (!fm) { + fmErrs.push('SKILL.md has no leading --- delimited YAML frontmatter block'); + } else { + if (typeof fm.name !== 'string' || fm.name.length === 0) { + fmErrs.push('frontmatter has no non-empty name'); + } + if (typeof fm.description !== 'string' || fm.description.length === 0) { + fmErrs.push('frontmatter has no non-empty description'); + } + } + checks.push( + skillsCheck( + 'sep-2640-skillmd-frontmatter', + 'SKILL.md MUST begin with YAML frontmatter containing at minimum the name and description fields.', + fmErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + fmErrs.length === 0 + ? { details: { uri } } + : { errorMessage: `${uri}: ${fmErrs.join('; ')}` } + ) + ); + + // === entry-frontmatter-identical === + const declared = sample?.frontmatter as Record | undefined; + if (!fm || !declared || typeof declared !== 'object' || Array.isArray(declared)) { + const reason = + 'Either the file has no parseable frontmatter or the entry carries no frontmatter object, so the two cannot be compared.'; + checks.push( + skillsCheck( + 'sep-2640-entry-frontmatter-identical', + 'The frontmatter object MUST be identical in content to the frontmatter of the SKILL.md it describes.', + 'SKIPPED', + { errorMessage: reason } + ) + ); + return checks; + } + + const diffs: string[] = []; + const keys = new Set([...Object.keys(fm), ...Object.keys(declared)]); + for (const k of keys) { + const a = JSON.stringify(fm[k] ?? null); + const b = JSON.stringify(declared[k] ?? null); + if (a !== b) diffs.push(`${k}: file=${a} entry=${b}`); + } + checks.push( + skillsCheck( + 'sep-2640-entry-frontmatter-identical', + 'The frontmatter object MUST be identical in content to the frontmatter of the SKILL.md it describes.', + diffs.length === 0 ? 'SUCCESS' : 'FAILURE', + diffs.length === 0 + ? { details: { uri, fields: [...keys] } } + : { + errorMessage: `${uri}: entry frontmatter differs from the file's: ${joinErrs(diffs)}` + } + ) + ); + + return checks; +} + +/** Exercise `skills/get` against a real entry and against an unknown URI. */ +async function getChecks( + conn: Parameters[0], + entries: SkillEntry[] +): Promise { + const checks: ConformanceCheck[] = []; + const sample = entries.find((e) => typeof e.uri === 'string'); + const sampleUri = sample?.uri as string | undefined; + + if (!sampleUri) { + const reason = + 'No listed entry carries a uri, so skills/get cannot be exercised against a known skill.'; + for (const id of GET_IDS) { + checks.push( + skillsCheck(id, reason, 'SKIPPED', { errorMessage: reason }) + ); + } + return checks; + } + + const got = await skillsGet(conn, sampleUri); + if ('error' in got) { + const reason = `${SKILLS_GET_METHOD} failed for a skill the server itself listed (${sampleUri}) with code ${got.error.code}: ${got.error.message}.`; + checks.push( + skillsCheck( + 'sep-2640-capability-commits-to-methods', + 'Declaring the extension commits the server to skills/list and skills/get.', + 'FAILURE', + { errorMessage: reason } + ), + skillsCheck( + 'sep-2640-skills-get-implemented', + 'A server declaring the extension MUST also implement the skills/get method, which returns the entry for a single skill named by its URI.', + 'FAILURE', + { errorMessage: reason } + ) + ); + for (const id of [ + 'sep-2640-skills-get-entry-shape', + 'sep-2640-skills-get-no-cursor', + 'sep-2640-skills-get-unknown-uri-invalid-params' + ]) { + checks.push( + skillsCheck(id, 'skills/get is unavailable.', 'SKIPPED', { + errorMessage: reason + }) + ); + } + return checks; + } + + checks.push( + skillsCheck( + 'sep-2640-capability-commits-to-methods', + 'Declaring the extension itself commits the server to skills/list and skills/get.', + 'SUCCESS', + { details: { methods: [SKILLS_LIST_METHOD, SKILLS_GET_METHOD] } } + ), + skillsCheck( + 'sep-2640-skills-get-implemented', + 'A server declaring the extension MUST also implement the skills/get method, which returns the entry for a single skill named by its URI.', + 'SUCCESS', + { details: { probedUri: sampleUri } } + ) + ); + + // === skills-get-entry-shape === + const skill = got.result.skill as SkillEntry | undefined; + const shapeErrs: string[] = []; + if (!skill || typeof skill !== 'object' || Array.isArray(skill)) { + shapeErrs.push('result.skill is missing or not an object'); + } else { + if (skill.uri !== sampleUri) { + shapeErrs.push( + `result.skill.uri=${JSON.stringify(skill.uri)} does not echo the requested uri ${sampleUri}` + ); + } + if ( + !skill.frontmatter || + typeof skill.frontmatter !== 'object' || + Array.isArray(skill.frontmatter) + ) { + shapeErrs.push('result.skill.frontmatter is missing or not an object'); + } + if (!isDynamicResources(skill) && resourcesArray(skill) === undefined) { + shapeErrs.push( + `result.skill.resources=${JSON.stringify(skill.resources)} is neither an array nor "${RESOURCES_DYNAMIC}"` + ); + } + } + checks.push( + skillsCheck( + 'sep-2640-skills-get-entry-shape', + 'The skill object is a skill entry, identical in shape and meaning to an entry of skills/list — the same uri, frontmatter, and resources fields, under the same rules.', + shapeErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + shapeErrs.length === 0 + ? { details: { probedUri: sampleUri } } + : { errorMessage: joinErrs(shapeErrs) } + ) + ); + + // === skills-get-no-cursor === + const hasCursor = got.result.nextCursor !== undefined; + checks.push( + skillsCheck( + 'sep-2640-skills-get-no-cursor', + 'The result carries no pagination cursor: a single entry is not a list.', + hasCursor ? 'FAILURE' : 'SUCCESS', + hasCursor + ? { + errorMessage: `skills/get returned nextCursor=${JSON.stringify(got.result.nextCursor)}; a single entry is not a list.` + } + : { details: { probedUri: sampleUri } } + ) + ); + + // === skills-get-unknown-uri-invalid-params === + const unknown = await skillsGet(conn, UNKNOWN_SKILL_URI); + if ('error' in unknown) { + const ok = unknown.error.code === JSONRPC_INVALID_PARAMS; + checks.push( + skillsCheck( + 'sep-2640-skills-get-unknown-uri-invalid-params', + 'If the URI does not identify a skill the server serves, the server MUST return error -32602 (Invalid params).', + ok ? 'SUCCESS' : 'FAILURE', + ok + ? { details: { probedUri: UNKNOWN_SKILL_URI, code: unknown.error.code } } + : { + errorMessage: `skills/get on an unserved URI returned code ${unknown.error.code} (${unknown.error.message}); expected ${JSONRPC_INVALID_PARAMS}.` + } + ) + ); + } else { + checks.push( + skillsCheck( + 'sep-2640-skills-get-unknown-uri-invalid-params', + 'If the URI does not identify a skill the server serves, the server MUST return error -32602 (Invalid params).', + 'FAILURE', + { + errorMessage: `skills/get returned a successful result for ${UNKNOWN_SKILL_URI}, which no conformant server should serve; expected error ${JSONRPC_INVALID_PARAMS}.` + } + ) + ); + } + + return checks; +} diff --git a/src/scenarios/server/skills/helpers.ts b/src/scenarios/server/skills/helpers.ts index d32022f9..7bd24943 100644 --- a/src/scenarios/server/skills/helpers.ts +++ b/src/scenarios/server/skills/helpers.ts @@ -4,9 +4,14 @@ * * The scenarios treat the server-under-test as an arbitrary SEP-2640 server: * capability is read from `server/discover` (never inferred from an error), and - * every skill is discovered dynamically from `skill://index.json` and - * `resources/list` — no fixture-specific URI is hardcoded, so the checks pass - * against any conformant server, not just one implementation's fixture. + * every skill is discovered dynamically through `skills/list` — no + * fixture-specific URI is hardcoded, so the checks pass against any conformant + * server, not just one implementation's fixture. + * + * Extracted against the 2026-08-21 revision of the SEP (branch + * `sep/skills-extension`), which replaced the `skill://index.json` well-known + * resource with the `skills/list` and `skills/get` methods, deferred archive + * distribution, and reshaped the skill entry to `{uri, frontmatter, resources}`. */ import type { @@ -20,15 +25,25 @@ import { parse as parseYaml } from 'yaml'; export const SKILLS_EXTENSION_ID = 'io.modelcontextprotocol/skills'; export const SKILL_URI_SCHEME = 'skill://'; -export const SKILL_INDEX_URI = 'skill://index.json'; export const SKILL_MANIFEST_FILENAME = 'SKILL.md'; export const SKILLS_META_PREFIX = 'io.modelcontextprotocol.skills/'; -/** `sha256:{hex}` with exactly 64 lowercase hex characters (SEP-2640 index). */ +/** Reserved prefix for MCP-defined keys inside frontmatter `metadata`. */ +export const FRONTMATTER_RESERVED_PREFIX = 'io.modelcontextprotocol/'; + +export const SKILLS_LIST_METHOD = 'skills/list'; +export const SKILLS_GET_METHOD = 'skills/get'; +export const DIRECTORY_READ_METHOD = 'resources/directory/read'; + +/** `sha256:{hex}` with exactly 64 lowercase hex characters (SEP-2640). */ export const SKILL_DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/; -/** The SEP enumerated `skills[].type` values. */ -export const SKILL_TYPES = ['skill-md', 'archive'] as const; +/** The `"dynamic"` sentinel a server sets in place of a `resources` array. */ +export const RESOURCES_DYNAMIC = 'dynamic'; + +/** Per-skill limits fixed by the SEP (§Limits). */ +export const MAX_RESOURCES_PER_SKILL = 512; +export const MAX_TOTAL_SIZE_PER_SKILL = 16 * 1024 * 1024; // 16 MiB export const JSONRPC_METHOD_NOT_FOUND = -32601; export const JSONRPC_INVALID_PARAMS = -32602; @@ -47,20 +62,37 @@ export interface SkillResource { _meta?: Record; } -/** One `skills[]` entry of the `skill://index.json` document. */ -export interface SkillIndexEntry { - name?: string; - type?: string; - description?: string; - url?: string; - digest?: string; +/** One `{uri, digest, size}` triple of a skill entry's `resources` array. */ +export interface SkillResourceEntry { + uri?: unknown; + digest?: unknown; + size?: unknown; + [key: string]: unknown; +} + +/** + * One skill entry, as returned by `skills/list` (in `skills[]`) and by + * `skills/get` (as `skill`). The two are identical in shape and meaning. + */ +export interface SkillEntry { + uri?: unknown; + frontmatter?: unknown; + /** An array of `{uri, digest, size}`, or the string `"dynamic"`. */ + resources?: unknown; + [key: string]: unknown; +} + +export interface SkillsListResult { + skills?: unknown; + nextCursor?: string; + ttlMs?: unknown; + cacheScope?: unknown; [key: string]: unknown; } -/** The parsed `skill://index.json` document. */ -export interface SkillIndex { - $schema?: string; - skills?: SkillIndexEntry[]; +export interface SkillsGetResult { + skill?: unknown; + nextCursor?: unknown; [key: string]: unknown; } @@ -111,24 +143,44 @@ export async function skillsCapability( : undefined; } +/** + * Whether the declared extension object nests its settings inline, as both + * SEP-2133 and SEP-2640 require, rather than wrapping them in an envelope. + * + * SEP-2133 (status Final) defines `extensions` as "a map of extension + * identifiers to per-extension settings objects", and SEP-2640's capability + * block matches: `{"io.modelcontextprotocol/skills": {"directoryRead": true}}`. + * Neither SEP defines an envelope, and neither has a slot for `id`, + * `specVersion` or `stability`. + * + * An earlier revision of this helper accepted a `config` envelope alongside the + * inline form, on the belief that the two SEPs disagreed. Re-reading SEP-2133 + * at Final status, they do not. The envelope is a non-conformant shape emitted + * by at least one SDK, so it is reported rather than silently accepted. + */ +export function settingsAreInline( + skills: Record +): { inline: boolean; envelopeKeys: string[] } { + const envelopeKeys = ['config', 'specVersion', 'stability', 'id'].filter( + (k) => k in skills + ); + return { inline: envelopeKeys.length === 0, envelopeKeys }; +} + /** * Whether the skills extension declares `directoryRead: true`. * - * SEP-2640's capability-declaration example places the flag directly on the - * extension object (`extensions[id].directoryRead`). SEP-2133 extension - * negotiation — which SEP-2640 normatively defers to ("Per SEP-2133 extension - * negotiation") — wraps settings in a `{ specVersion, stability, config }` - * envelope, putting the flag at `extensions[id].config.directoryRead`. The two - * SEPs are inconsistent on nesting, so a brand-neutral conformance check accepts - * either location rather than privileging one reading of an ambiguous spec. - * (The inconsistency is worth a WG clarification; see the scenario docs.) + * Reads only the inline location the SEPs specify. A server that buries the + * flag inside an envelope fails `sep-2640-capability-declaration-inline` and is + * treated here as not having declared the optional method, which is the + * conservative reading: a client that follows the spec would not see the flag + * either, and "clients MUST NOT call resources/directory/read against a server + * that has not declared directoryRead: true". */ export function directoryReadDeclared( skills: Record ): boolean { - if (skills.directoryRead === true) return true; - const config = skills.config as Record | undefined; - return config?.directoryRead === true; + return skills.directoryRead === true; } /** Everything from `resources/list`, paginating until `nextCursor` clears. */ @@ -148,20 +200,84 @@ export async function listAllResources( return out; } +/** One page of `skills/list`, kept separate so pagination can be inspected. */ +export interface SkillsListPage { + result: SkillsListResult; + entries: SkillEntry[]; +} + /** - * Read `skill://index.json`. Returns the raw JSON text (for parse-error - * reporting) or a `JsonRpcError` when the server declines the well-known index - * — a permitted MAY (SEP-2640 §Enumeration): the catalog may be unenumerable. + * Call `skills/list` once, optionally with a cursor. Returns a `JsonRpcError` + * rather than throwing so a scenario can distinguish "method not implemented" + * (a FAILURE, since the method is mandatory for a declaring server) from a + * transport fault. */ -export async function readSkillIndexText( - conn: Connection -): Promise<{ text?: string; mimeType?: string } | { error: JsonRpcError }> { +export async function skillsListPage( + conn: Connection, + cursor?: string +): Promise { + try { + const result = await conn.request( + SKILLS_LIST_METHOD, + cursor ? { cursor } : {} + ); + const entries = Array.isArray(result.skills) + ? (result.skills as SkillEntry[]) + : []; + return { result, entries }; + } catch (e) { + if (e instanceof JsonRpcError) return { error: e }; + throw e; + } +} + +/** + * Every entry from `skills/list`, following `nextCursor`. `pages` is retained + * so the atomic-entry and pagination checks can reason about page boundaries. + * Bounded to avoid looping forever against a server that returns a constant + * cursor. + */ +export async function skillsListAll( + conn: Connection, + maxPages = 50 +): Promise< + | { entries: SkillEntry[]; pages: SkillsListPage[]; truncated: boolean } + | { error: JsonRpcError } +> { + const pages: SkillsListPage[] = []; + const entries: SkillEntry[] = []; + let cursor: string | undefined; + const seenCursors = new Set(); + + for (let i = 0; i < maxPages; i++) { + const page = await skillsListPage(conn, cursor); + if ('error' in page) return page; + pages.push(page); + entries.push(...page.entries); + const next = page.result.nextCursor; + if (typeof next !== 'string' || next.length === 0) { + return { entries, pages, truncated: false }; + } + if (seenCursors.has(next)) { + // A repeating cursor is a server bug; stop rather than spin. + return { entries, pages, truncated: true }; + } + seenCursors.add(next); + cursor = next; + } + return { entries, pages, truncated: true }; +} + +/** Call `skills/get` for one skill URI. */ +export async function skillsGet( + conn: Connection, + uri: string +): Promise<{ result: SkillsGetResult } | { error: JsonRpcError }> { try { - const res = await conn.request<{ - contents?: Array<{ text?: string; mimeType?: string }>; - }>('resources/read', { uri: SKILL_INDEX_URI }); - const entry = (res.contents ?? []).find((c) => typeof c.text === 'string'); - return { text: entry?.text, mimeType: entry?.mimeType }; + const result = await conn.request(SKILLS_GET_METHOD, { + uri + }); + return { result }; } catch (e) { if (e instanceof JsonRpcError) return { error: e }; throw e; @@ -188,15 +304,21 @@ export async function readResourceText( /** * The skill name recoverable from a `SKILL.md` resource URI: the final segment * of ``, i.e. the last path segment before the trailing - * `SKILL.md`. Returns `undefined` when the URI is not a `skill://…/SKILL.md`. + * `SKILL.md`. Returns `undefined` when the URI does not end in `/SKILL.md`. + * + * Scheme-agnostic by design: the SEP is explicit that "no scheme is + * privileged" and that the structural constraints "apply regardless of + * scheme", so a server serving skills under `github://` is judged by the same + * path rule as one using `skill://`. * * skill://org/team/deploy/SKILL.md -> "deploy" - * skill://lint/SKILL.md -> "lint" + * github://acme/repo/lint/SKILL.md -> "lint" */ export function skillNameFromManifestUri(uri: string): string | undefined { - if (!uri.startsWith(SKILL_URI_SCHEME)) return undefined; + const schemeEnd = uri.indexOf('://'); + if (schemeEnd < 0) return undefined; const parts = uri - .slice(SKILL_URI_SCHEME.length) + .slice(schemeEnd + 3) .split('/') .filter((p) => p.length > 0); if (parts.length < 2) return undefined; @@ -204,6 +326,12 @@ export function skillNameFromManifestUri(uri: string): string | undefined { return parts[parts.length - 2]; } +/** The skill's root directory URI: its `SKILL.md` URI with the file removed. */ +export function skillRootFromManifestUri(uri: string): string | undefined { + if (!uri.endsWith(`/${SKILL_MANIFEST_FILENAME}`)) return undefined; + return uri.slice(0, -`/${SKILL_MANIFEST_FILENAME}`.length); +} + /** * Extract and parse the YAML frontmatter block at the head of a `SKILL.md`. * Returns `undefined` when there is no leading `---` delimited block or it does @@ -225,3 +353,25 @@ export function parseFrontmatter( return undefined; } } + +/** True when the entry's `resources` is the `"dynamic"` sentinel. */ +export function isDynamicResources(entry: SkillEntry): boolean { + return entry.resources === RESOURCES_DYNAMIC; +} + +/** + * The entry's `resources` array, or `undefined` when it is `"dynamic"`, absent, + * or any other value. Callers distinguish those cases via `isDynamicResources`. + */ +export function resourcesArray( + entry: SkillEntry +): SkillResourceEntry[] | undefined { + return Array.isArray(entry.resources) + ? (entry.resources as SkillResourceEntry[]) + : undefined; +} + +/** A short, stable label for an entry, for error messages. */ +export function entryLabel(entry: SkillEntry, i: number): string { + return typeof entry.uri === 'string' ? entry.uri : `skills[${i}]`; +} diff --git a/src/scenarios/server/skills/index.ts b/src/scenarios/server/skills/index.ts deleted file mode 100644 index bb9d1f33..00000000 --- a/src/scenarios/server/skills/index.ts +++ /dev/null @@ -1,270 +0,0 @@ -/** - * SEP-2640 Skills extension — `skill://index.json` enumeration surface. - * - * One scenario, many checks (per AGENTS.md "fewer scenarios, more checks"). - * Each check's verbatim spec quote lives next to its check ID in - * src/seps/sep-2640.yaml, keeping the YAML and this scenario in lock-step. - * - * All discovery is dynamic and brand-neutral: the scenario reads the well-known - * `skill://index.json` and validates whatever entries it finds, hardcoding no - * fixture-specific skill name or URI. When the server does not declare the - * skills extension the checks are SKIPPED (an optional, undeclared capability); - * when the server declines the index (a permitted MAY) or serves an empty index - * the index-shape checks are SKIPPED (legitimately not applicable), never - * failed against a conformant server. - */ - -import { ClientScenario, ConformanceCheck } from '../../../types'; -import type { RunContext } from '../../../connection'; -import { - SKILLS_EXTENSION_ID, - SKILL_INDEX_URI, - SKILL_URI_SCHEME, - SKILL_TYPES, - SKILL_DIGEST_PATTERN, - type SkillIndex, - type SkillIndexEntry, - skillsCapability, - skillsCheck, - readSkillIndexText -} from './helpers'; - -const ENTRY_CHECK_IDS = [ - 'sep-2640-index-entry-type-enum', - 'sep-2640-index-name-required', - 'sep-2640-index-digest-required', - 'sep-2640-skill-uri-scheme' -] as const; - -const ALL_CHECK_IDS = ['sep-2640-server-expose-index', ...ENTRY_CHECK_IDS]; - -export class SkillsIndexScenario implements ClientScenario { - name = 'sep-2640-skills-index'; - readonly source = { extensionId: SKILLS_EXTENSION_ID } as const; - description = `SEP-2640 Skills extension: the \`skill://index.json\` enumeration index. - -**Resource**: \`skill://index.json\` (read via \`resources/read\`, \`mimeType\` \`application/json\`) - -**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-2640.yaml): - -- \`sep-2640-server-expose-index\` — server exposes a readable \`skill://index.json\` (SHOULD; a server MAY decline for an unenumerable catalog) -- \`sep-2640-index-entry-type-enum\` — every \`skills[].type\` is \`"skill-md"\` or \`"archive"\` (MUST) -- \`sep-2640-index-name-required\` — every entry carries a non-empty \`name\` (required field) -- \`sep-2640-index-digest-required\` — a present \`skills[].digest\` is \`sha256:{64 hex}\` (MUST) -- \`sep-2640-skill-uri-scheme\` — index entry URLs use the \`skill://\` scheme (SHOULD; another scheme is permitted only when listed in the index) - -**Discovery is dynamic**: the scenario reads whatever skills the index enumerates. Undeclared extension, a declined index, or an empty index all SKIP cleanly.`; - - async run(ctx: RunContext): Promise { - const conn = await ctx.connect(); - try { - const skills = await skillsCapability(conn); - if (!skills) { - const reason = - 'Server did not declare the io.modelcontextprotocol/skills extension; index checks not applicable.'; - return ALL_CHECK_IDS.map((id) => - skillsCheck(id, reason, 'SKIPPED', { errorMessage: reason }) - ); - } - - const checks: ConformanceCheck[] = []; - - // === server-expose-index (SHOULD, with an explicit MAY-decline) === - const read = await readSkillIndexText(conn); - if ('error' in read) { - const reason = `Server declined skill://index.json (code ${read.error.code}); permitted MAY — the catalog may be large or unenumerable. Hosts MUST NOT treat this as proof of no skills.`; - checks.push( - skillsCheck( - 'sep-2640-server-expose-index', - 'Server SHOULD expose a readable skill://index.json enumerating the skills it serves.', - 'SKIPPED', - { errorMessage: reason } - ) - ); - for (const id of ENTRY_CHECK_IDS) { - checks.push( - skillsCheck(id, 'No skill://index.json to inspect.', 'SKIPPED', { - errorMessage: reason - }) - ); - } - return checks; - } - - if (read.text === undefined) { - const reason = - 'resources/read on skill://index.json returned no text content; the index resource is exposed but unreadable.'; - checks.push( - skillsCheck( - 'sep-2640-server-expose-index', - 'Server SHOULD expose a readable skill://index.json enumerating the skills it serves.', - 'FAILURE', - { errorMessage: reason } - ) - ); - for (const id of ENTRY_CHECK_IDS) { - checks.push( - skillsCheck( - id, - 'No readable index content to inspect.', - 'SKIPPED', - { - errorMessage: reason - } - ) - ); - } - return checks; - } - - let index: SkillIndex; - try { - index = JSON.parse(read.text) as SkillIndex; - } catch (e) { - const reason = `skill://index.json content is not valid JSON: ${ - e instanceof Error ? e.message : String(e) - }`; - checks.push( - skillsCheck( - 'sep-2640-server-expose-index', - 'Server SHOULD expose a readable skill://index.json whose content is a JSON index.', - 'FAILURE', - { errorMessage: reason } - ) - ); - for (const id of ENTRY_CHECK_IDS) { - checks.push( - skillsCheck(id, 'Index did not parse as JSON.', 'SKIPPED', { - errorMessage: reason - }) - ); - } - return checks; - } - - checks.push( - skillsCheck( - 'sep-2640-server-expose-index', - 'Server SHOULD expose a readable skill://index.json whose content is a JSON index of the skills it serves.', - 'SUCCESS', - { - details: { - uri: SKILL_INDEX_URI, - mimeType: read.mimeType, - skillCount: Array.isArray(index.skills) ? index.skills.length : 0 - } - } - ) - ); - - const entries: SkillIndexEntry[] = Array.isArray(index.skills) - ? index.skills - : []; - - // An exposed-but-empty index is valid: a partial/empty index is - // permitted, and hosts MUST NOT read "no skills" from it. Nothing to - // validate at the entry level, so SKIP those checks cleanly. - if (entries.length === 0) { - const reason = - 'skill://index.json is exposed but lists no skills; entry-level checks not applicable (an empty index is permitted).'; - for (const id of ENTRY_CHECK_IDS) { - checks.push( - skillsCheck(id, reason, 'SKIPPED', { errorMessage: reason }) - ); - } - return checks; - } - - // === index-entry-type-enum (MUST) === - const typeErrs = entries - .map((e, i) => - SKILL_TYPES.includes(e.type as (typeof SKILL_TYPES)[number]) - ? null - : `skills[${i}].type=${JSON.stringify(e.type)} is not one of ${SKILL_TYPES.join('|')}` - ) - .filter((x): x is string => x !== null); - checks.push( - skillsCheck( - 'sep-2640-index-entry-type-enum', - 'Every skills[].type MUST be "skill-md" or "archive".', - typeErrs.length === 0 ? 'SUCCESS' : 'FAILURE', - typeErrs.length === 0 - ? { details: { entryCount: entries.length } } - : { errorMessage: typeErrs.join('; ') } - ) - ); - - // === index-name-required (required field) === - const nameErrs = entries - .map((e, i) => - typeof e.name === 'string' && e.name.length > 0 - ? null - : `skills[${i}].name is missing or empty` - ) - .filter((x): x is string => x !== null); - checks.push( - skillsCheck( - 'sep-2640-index-name-required', - 'Every index entry carries a non-empty name (matching the SKILL.md frontmatter name and the final skill-path segment).', - nameErrs.length === 0 ? 'SUCCESS' : 'FAILURE', - nameErrs.length === 0 - ? { details: { entryCount: entries.length } } - : { errorMessage: nameErrs.join('; ') } - ) - ); - - // === index-digest-required (MUST — validate the format when present) === - const withDigest = entries.filter((e) => e.digest !== undefined); - const digestErrs = withDigest - .map((e, i) => - typeof e.digest === 'string' && SKILL_DIGEST_PATTERN.test(e.digest) - ? null - : `skills[${i}].digest=${JSON.stringify(e.digest)} is not sha256:{64 lowercase hex}` - ) - .filter((x): x is string => x !== null); - checks.push( - skillsCheck( - 'sep-2640-index-digest-required', - 'Every present skills[].digest MUST be formatted as sha256:{hex} with 64 lowercase hex characters.', - digestErrs.length === 0 ? 'SUCCESS' : 'FAILURE', - digestErrs.length === 0 - ? { - details: { - entriesWithDigest: withDigest.length, - entriesWithoutDigest: entries.length - withDigest.length - } - } - : { errorMessage: digestErrs.join('; ') } - ) - ); - - // === skill-uri-scheme (SHOULD) === - // Servers SHOULD use skill://; another scheme is permitted only when the - // skill is listed in the index (SEP-2640 §URI convention), so a non- - // skill:// URL is a SHOULD deviation, not a hard failure. - const nonSkillScheme = entries - .map((e, i) => - typeof e.url === 'string' && !e.url.startsWith(SKILL_URI_SCHEME) - ? `skills[${i}].url=${JSON.stringify(e.url)}` - : null - ) - .filter((x): x is string => x !== null); - checks.push( - skillsCheck( - 'sep-2640-skill-uri-scheme', - 'Skill resource URLs in the index SHOULD use the skill:// URI scheme.', - nonSkillScheme.length === 0 ? 'SUCCESS' : 'WARNING', - nonSkillScheme.length === 0 - ? { details: { entryCount: entries.length } } - : { - errorMessage: `Entries use a non-skill:// scheme (permitted only when indexed): ${nonSkillScheme.join(', ')}` - } - ) - ); - - return checks; - } finally { - await conn.close(); - } - } -} diff --git a/src/scenarios/server/skills/manifest.ts b/src/scenarios/server/skills/manifest.ts index 7a4f1fd6..42b19229 100644 --- a/src/scenarios/server/skills/manifest.ts +++ b/src/scenarios/server/skills/manifest.ts @@ -21,12 +21,11 @@ import { SKILLS_EXTENSION_ID, SKILLS_META_PREFIX, SEP_2640_REF, - type SkillIndex, type SkillResource, skillsCapability, skillsCheck, listAllResources, - readSkillIndexText, + skillsListAll, readResourceText, skillNameFromManifestUri, parseFrontmatter @@ -94,20 +93,12 @@ export class SkillsManifestScenario implements ClientScenario { ); let manifestUri = manifestResource?.uri; if (!manifestUri) { - const idx = await readSkillIndexText(conn); - if (!('error' in idx) && typeof idx.text === 'string') { - try { - const index = JSON.parse(idx.text) as SkillIndex; - const entry = (index.skills ?? []).find( - (e) => - e.type === 'skill-md' && - typeof e.url === 'string' && - isManifestUri(e.url) - ); - manifestUri = entry?.url; - } catch { - // A malformed index is the index scenario's concern; ignore here. - } + const listed = await skillsListAll(conn); + if (!('error' in listed)) { + const entry = listed.entries.find( + (e) => typeof e.uri === 'string' && isManifestUri(e.uri) + ); + manifestUri = entry?.uri as string | undefined; } } diff --git a/src/seps/sep-2640.yaml b/src/seps/sep-2640.yaml index d93e1487..1b0679e5 100644 --- a/src/seps/sep-2640.yaml +++ b/src/seps/sep-2640.yaml @@ -1,46 +1,71 @@ -# spec_source: modelcontextprotocol/modelcontextprotocol@556154c088371149c120172e95bb634655f00cbe seps/2640-skills-extension.md -# extracted: 2026-06-05 -# forward_reference: rows sep-2640-capability-directory-read-flag through -# sep-2640-directory-read-pagination track SEP commit -# 2e04c48da90224000e750ffd54a3611f2824fbc0 (2026-06-09) — the -# resources/directory/read addition. The file-level provenance above -# stays at 556154c because the PR 97 schema rewrite (360123d0, -# 2026-06-08) made 3 existing rows stale (sep-2640-index-entry-type-enum, -# sep-2640-index-name-required, sep-2640-index-digest-required) and -# drifted ~11 others' verbatim wording. Full re-extraction at SEP HEAD -# is mcpkit#780's lifecycle; this file deliberately holds at 556154c -# until that lands. +# spec_source: modelcontextprotocol/modelcontextprotocol@sep/skills-extension seps/2640-skills-extension.md +# extracted: 2026-08-28 +# +# provenance: re-extracted against the 2026-08-21 revision, which the core +# maintainers produced after their second review round and which went back up +# for CM vote on 2026-08-25. The prior extraction held at 556154c (2026-06-05) +# by an explicit decision to wait for mcpkit#780; that hold expired badly. +# The 08-21 revision: +# - removed the `skill://index.json` well-known resource entirely (it now +# appears nowhere in the SEP) and replaced it with the `skills/list` and +# `skills/get` methods, both mandatory for a declaring server; +# - deferred archive distribution to an appendix ("Appendix: Deferred +# Features"), retiring the `type` enum and every archive requirement; +# - reversed the nesting rule: a `SKILL.md` MAY now appear in a descendant +# directory, where the June text forbade it; +# - reshaped the entry to `{uri, frontmatter, resources}`, where `resources` +# is a complete `{uri, digest, size}` array or the string `"dynamic"`; +# - fixed per-skill limits of 512 resource entries and 16 MiB; +# - added host obligations around lazy retrieval, content-bound approval and +# frontmatter re-verification. +# Eight rows from the prior extraction were removed as no longer normative +# (server-expose-index, index-entry-type-enum, no-nested-skills, and the five +# archive rows) and four were reworded. Because the SEP is mid-vote, expect one +# more pass at the CM-stamped head. +# # backing_scenarios: three server ClientScenarios under # src/scenarios/server/skills/ emit the check IDs below (a row is "tested" # once a scenario emits its check ID; see src/traceability/): -# directory.ts (sep-2640-skills-directory) — sep-2640-capability-directory-read-flag -# and the five sep-2640-directory-read-* rows. -# index.ts (sep-2640-skills-index) — sep-2640-server-expose-index, -# sep-2640-index-entry-type-enum, sep-2640-index-name-required, -# sep-2640-index-digest-required, sep-2640-skill-uri-scheme. +# enumeration.ts (sep-2640-skills-enumeration) — the capability rows, the +# sep-2640-skills-list-* rows, the entry-schema and resources rows, the two +# limit rows, and the sep-2640-skills-get-* rows. Replaces the retired +# index.ts, which tested `skill://index.json`. # manifest.ts (sep-2640-skills-manifest) — sep-2640-skillmd-mimetype, # sep-2640-skillmd-metadata-name, sep-2640-skillmd-metadata-description, # sep-2640-final-segment-equals-name, sep-2640-meta-prefix. -# The remaining rows are host-internal or off-wire (host load-by-uri, digest -# verification, byte-budget / archive-unpack safety, no-empty-index -# assumption) and stay traceability-only for this server-scenario set. +# directory.ts (sep-2640-skills-directory) — +# sep-2640-capability-directory-read-flag and the +# sep-2640-directory-read-* rows. +# The remaining rows are host-internal or off-wire (host retrieval policy, +# digest and frontmatter verification, approval binding, cache isolation) and +# stay traceability-only for this server-scenario set. A host-side scenario +# set would be needed to exercise them. sep: 2640 spec_url: https://modelcontextprotocol.io/seps/2640-skills-extension#specification requirements: + # === Skill Format === - check: sep-2640-skillmd-required text: 'Every skill MUST contain a `SKILL.md` file at its root.' - check: sep-2640-skillmd-frontmatter text: '`SKILL.md` MUST begin with YAML frontmatter containing at minimum the `name` and `description` fields as defined by the Agent Skills specification.' + + # === Resource Mapping === - check: sep-2640-skill-uri-scheme text: 'Each file within a skill directory is exposed as an MCP resource. Servers SHOULD use the `skill://` URI scheme, under which the resource URI has the form: `skill:///`' - check: sep-2640-final-segment-equals-name text: "The final segment of `` MUST equal the skill's `name` as declared in its `SKILL.md` frontmatter." - - check: sep-2640-no-nested-skills - text: 'A `SKILL.md` MUST NOT appear in any descendant directory of a skill. The skill directory is the boundary; skills do not nest inside other skills.' - check: sep-2640-name-naming-rules text: "The final `` segment, being the skill `name`, MUST satisfy the Agent Skills specification's naming rules." - - check: sep-2640-prefix-rfc3986 - text: 'Prefix segments SHOULD be valid URI path segments per RFC 3986; no further constraints are imposed on them.' + - check: sep-2640-authority-reg-name + text: 'The first `` segment occupies the authority component and SHOULD be a valid `reg-name` per RFC 3986; any other prefix segments SHOULD be valid URI path segments; no further constraints are imposed on them.' + # Reverses the retired sep-2640-no-nested-skills row: the June text said a + # SKILL.md MUST NOT appear in any descendant directory. + - check: sep-2640-nested-skills-permitted + text: 'A `SKILL.md` MAY appear in a descendant directory of a skill — skills can nest.' + - check: sep-2640-nested-publication-flat + text: "A nested skill is published like any other: through its own `skills/list` entry, or by explicit reference. The listing remains flat — an entry for a nested skill is an ordinary entry whose `uri` happens to share a path prefix with the enclosing skill's, and nothing in the listing marks nesting." + + # === Resource Metadata === - check: sep-2640-skillmd-mimetype text: 'For each `skill:///SKILL.md` resource: `mimeType` SHOULD be `text/markdown`.' - check: sep-2640-skillmd-metadata-name @@ -49,36 +74,119 @@ requirements: text: 'For each `skill:///SKILL.md` resource: `description` SHOULD be set from the `description` field of the `SKILL.md` YAML frontmatter.' - check: sep-2640-meta-prefix text: 'When `_meta` keys are used for skill resources, implementations SHOULD use the `io.modelcontextprotocol.skills/` reverse-domain prefix.' + + # === Capability Declaration === + # The inline-settings row is the check that would have caught panyam/mcpkit#1334. + # SEP-2133 (status Final) defines `extensions` as "a map of extension + # identifiers to per-extension settings objects" and defines no envelope; the + # SEP-2640 capability block matches it. + - check: sep-2640-capability-declaration-inline + text: 'Per SEP-2133 extension negotiation, servers declare support for this extension in their `initialize` response, mapping the extension identifier directly to its per-extension settings object: `{"capabilities": {"extensions": {"io.modelcontextprotocol/skills": {"directoryRead": true}}}}`.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#capability-declaration + - check: sep-2640-capability-commits-to-methods + text: 'Declaring the extension itself commits the server to `skills/list` and `skills/get`.' + - check: sep-2640-capability-empty-object + text: 'An empty object indicates support for the extension with no optional features.' + - check: sep-2640-capability-directory-read-flag + text: 'Clients MUST NOT call `resources/directory/read` against a server that has not declared `directoryRead: true`.' + + # === Discovery: enumeration via skills/list === + - check: sep-2640-skills-list-implemented + text: 'A server declaring the `io.modelcontextprotocol/skills` extension MUST implement the `skills/list` method, which returns the skills it serves. The result MAY be empty.' + - check: sep-2640-skills-list-pagination + text: 'Pagination mirrors the base protocol''s list methods: the request accepts an optional `cursor`, and when the result includes `nextCursor` the client passes it back to retrieve the next page.' + - check: sep-2640-skills-list-entry-atomic + text: "An entry is atomic — a skill's `resources` set is never split across pages." + - check: sep-2640-skills-list-cache-attributes + text: 'In protocol versions 2026-07-28 and later, the result also carries the base protocol''s list-caching attributes — `ttlMs` and `cacheScope`, as defined for `tools/list` and `resources/list` (SEP-2549) — with the same semantics: a freshness hint for the listing and a cache-scope marker, not an integrity property.' + - check: sep-2640-skills-list-may-be-partial + text: 'A server whose skill catalog is large, generated on demand, or otherwise unenumerable MAY return an empty or partial listing.' + - check: sep-2640-host-no-empty-listing-assumption + text: 'Hosts MUST NOT treat an empty or partial listing as proof that a server has no skills.' + - check: sep-2640-enumeration-scheme-uniform + text: 'The method serves entries for a server''s skills whatever URI scheme they use — enumeration is uniform across schemes.' + + # === The skill entry (shared by skills/list and skills/get) === + - check: sep-2640-entry-uri-required + text: '`skills[].uri` (Yes): Resource URI of the skill''s `SKILL.md`.' + - check: sep-2640-entry-frontmatter-required + text: "`frontmatter` is the skill's `SKILL.md` YAML frontmatter rendered verbatim as a JSON object — every field the author wrote, not a curated subset. Because the Agent Skills specification requires `name` and `description`, those fields are always present." + - check: sep-2640-entry-frontmatter-identical + text: 'The `frontmatter` object MUST be identical in content to the frontmatter of the `SKILL.md` it describes.' + - check: sep-2640-entry-uri-matches-frontmatter-name + text: "The final `` segment of the entry's `uri` MUST equal `frontmatter.name`, per Resource Mapping." + - check: sep-2640-metadata-reserved-prefix + text: 'Within the frontmatter `metadata` object, keys prefixed with `io.modelcontextprotocol/` are reserved for metadata defined by MCP extensions. This extension currently defines no such keys. Implementations SHOULD ignore keys under this prefix that they do not recognize.' + - check: sep-2640-names-not-unique + text: "A skill's `name` is a label, not an identifier. Within a server's listing, names SHOULD be unique, but they are not guaranteed to be. Hosts MUST NOT assume name uniqueness. When two entries in one listing collide on `name`, hosts MUST disambiguate them — for example by their distinguishing path segments — rather than silently discarding or preferring one." + + # === resources === + - check: sep-2640-entry-resources-required + text: '`resources` is REQUIRED on every skill entry and takes one of two forms: an array enumerating the skill''s files — `SKILL.md` and every supporting file — as `{uri, digest, size}` triples, or the string `"dynamic"`.' + - check: sep-2640-resources-complete + text: 'When present, `resources` MUST be complete: it lists every file of the skill, each exactly once, including an entry matching the skill''s top-level `uri` — that entry carries the digest and size of `SKILL.md` itself.' + - check: sep-2640-resources-uri-within-skill + text: 'Each `uri` MUST be the skill''s `SKILL.md` or a file within the skill''s directory.' + - check: sep-2640-resources-digest-format + text: 'Digests are SHA-256 hashes of an artifact''s raw bytes, formatted as `sha256:{hex}` where `{hex}` is 64 lowercase hexadecimal characters. Each entry in a skill''s `resources` carries the digest of the file at its `uri`.' + - check: sep-2640-resources-size-required + text: 'Each entry MUST carry `size`: the length in bytes of the file''s raw content — the same bytes the `digest` covers.' + - check: sep-2640-resources-dynamic-marker + text: 'When a skill''s content is generated dynamically, such that stable digests cannot be published, the server MUST set `"resources": "dynamic"` instead of an array.' + - check: sep-2640-resources-invalid-entry + text: 'An entry with no `resources` at all, or with any value other than an array or `"dynamic"`, is invalid, and hosts MUST NOT load it.' + - check: sep-2640-resources-nested-completeness + text: "Completeness extends to nested skills: from the enclosing skill's perspective their files are supporting files, so the enclosing skill's `resources` lists them too, and the same file may appear in both the enclosing and the nested skill's entries." + + # === Limits === + - check: sep-2640-limit-resources-per-skill + text: 'Resources per skill: 512 entries, counted over the entries of the skill''s `resources`, `SKILL.md` included.' + - check: sep-2640-limit-total-size + text: 'Total file size per skill: 16 MiB (16,777,216 bytes), counted over the sum of `size` over the skill''s `resources`.' + - check: sep-2640-limit-host-support + text: 'Hosts MUST support skills up to and including these limits, and MAY support larger ones. Servers SHOULD NOT serve a skill that exceeds either limit; a skill that does is not guaranteed to be loadable by any conforming host.' + + # === Retrieval via skills/get === + - check: sep-2640-skills-get-implemented + text: 'A server declaring the `io.modelcontextprotocol/skills` extension MUST also implement the `skills/get` method, which returns the entry for a single skill named by its URI.' + - check: sep-2640-skills-get-entry-shape + text: 'The `skill` object is a skill entry, identical in shape and meaning to an entry of `skills/list` — the same `uri`, `frontmatter`, and `resources` fields, under the same rules.' + - check: sep-2640-skills-get-unknown-uri-invalid-params + text: 'If the URI does not identify a skill the server serves, the server MUST return error `-32602` (Invalid params) — the same code `resources/read` uses for unknown resources.' + - check: sep-2640-skills-get-answers-unlisted + text: 'A server MUST answer for every skill it serves, whether or not that skill appears in its `skills/list` result. A skill absent from a partial listing is still retrievable by URI.' + - check: sep-2640-skills-get-no-cursor + text: 'The result carries no pagination cursor: a single entry is not a list.' + + # === Reading / integrity (host obligations) === - check: sep-2640-host-load-by-uri text: 'hosts MUST support loading a skill given only its URI' - - check: sep-2640-server-expose-index - text: 'A server SHOULD expose a resource at the well-known URI `skill://index.json` whose content is a JSON index of the skills it serves.' - - check: sep-2640-index-entry-type-enum - text: '`skills[].type` MUST be `"skill-md"` or `"archive"`.' - - check: sep-2640-index-name-required - text: '`skills[].name` matches the `SKILL.md` frontmatter `name` and the final segment of the skill path.' - - check: sep-2640-index-digest-required - text: '`skills[].digest` is the SHA-256 content digest of the artifact, formatted as `sha256:{hex}` (64 lowercase hex characters).' - - check: sep-2640-client-ignore-unrecognized - text: 'Clients SHOULD ignore unrecognized fields and SHOULD skip entries with an unrecognized `type`.' - - check: sep-2640-archive-format - text: 'the archive MUST be `.tar.gz` (gzip-compressed tar, `mimeType` `application/gzip`) or `.zip` (`mimeType` `application/zip`)' - - check: sep-2640-host-support-archive-formats - text: 'hosts MUST support both `.tar.gz` and `.zip` archive formats' - - check: sep-2640-archive-skillmd-at-root - text: 'Archive contents represent the skill directory directly — `SKILL.md` MUST be at the archive root, not nested inside a wrapper directory' - - check: sep-2640-archive-no-traversal - text: 'the archive MUST NOT contain path-traversal sequences (`..`) or absolute paths' - - check: sep-2640-host-archive-safety - text: 'Hosts unpacking an archive MUST apply the archive safety requirements of the Agent Skills specification: reject archives containing path-traversal sequences or absolute paths, reject symlinks or hard links that resolve outside the skill directory, and enforce a limit on total unpacked size / Hosts MUST validate archives per the Agent Skills archive safety requirements: reject path traversal and absolute paths, reject links resolving outside the skill directory, and bound total unpacked size to prevent decompression bombs.' - check: sep-2640-host-verify-digest - text: 'Hosts MUST verify retrieved content against the `digest` in the index / hosts MUST NOT use unverified content.' - - check: sep-2640-host-no-empty-index-assumption - text: 'Hosts MUST NOT treat an absent or empty index as proof that a server has no skills.' + text: 'When a host retrieves a file listed in a skill''s `resources`, it MUST verify the content against that entry''s digest. Whatever the cause, hosts MUST NOT use the unverified content.' + - check: sep-2640-host-size-mismatch-failure + text: 'A read whose byte length differs from the entry''s `size` is a verification failure equivalent to a digest mismatch, whether or not the host goes on to compute the digest.' + - check: sep-2640-host-unlisted-read-failure + text: 'While acting on a skill, a host MUST resolve reads of the skill''s files only to URIs listed in that entry''s `resources`, and MUST treat a read of an unlisted file within the skill as a verification failure equivalent to a digest mismatch.' + - check: sep-2640-host-no-prefetch + text: 'Hosts MUST NOT retrieve a skill''s files ahead of need — not on connection, not on listing, and not at approval. A `SKILL.md` is fetched when the skill is loaded, and a supporting file when it is read.' + - check: sep-2640-host-frontmatter-comparison + text: 'After fetching a `SKILL.md` for which the host holds an entry, hosts MUST parse its YAML frontmatter and compare it field-by-field against the entry''s `frontmatter`. Any discrepancy MUST be treated as a verification failure equivalent to a digest mismatch, and the skill MUST NOT be loaded.' + - check: sep-2640-host-digest-not-security-boundary + text: 'Digests are unsigned and supplied by the same server that supplies the content. A match proves the two are consistent, not that either is trustworthy. Hosts MUST NOT treat a digest match as a security boundary.' + - check: sep-2640-host-skill-identity-pair + text: 'The identity of an MCP-served skill is the pair of the host''s identity for the originating server and the skill''s `uri`. Hosts MUST preserve both halves wherever a skill is recorded or addressed — the registry, persisted approvals, the cache, and any tool or path through which the model reaches the skill — and MUST NOT key any of these on the `uri` alone.' + - check: sep-2640-host-not-skill-by-scheme + text: 'A host MUST NOT conclude that a resource is a skill merely because its URI carries a particular scheme.' + - check: sep-2640-host-cross-origin-no-shadow + text: 'When skills from different origins collide on `name`, hosts MUST resolve the name within a per-origin namespace, identifying servers by a host-assigned label; an MCP-served skill MUST NOT silently shadow, or be silently substituted for, a same-named skill from any other origin.' + - check: sep-2640-host-content-bound-approval + text: 'When a host persists any per-skill user approval, it MUST be bound to the entry''s `resources` set — every `uri` and `digest` — observed at the moment of approval. If a subsequent entry for that skill advertises a different set, the host MUST treat the prior approval as revoked and re-prompt before loading or executing.' + - check: sep-2640-host-dynamic-not-content-bound + text: 'A skill whose `resources` is `"dynamic"` cannot be content-bound: hosts MAY decline to load it, and MUST NOT treat a persisted approval as covering whatever content the server currently serves.' + - check: sep-2640-host-nested-fresh-consent + text: 'Activating a nested skill — loading it as a skill in its own right — requires fresh, explicit user consent; approval of the enclosing skill does not substitute for it. A nested `SKILL.md` read as supporting content is ordinary markdown: hosts MUST NOT act on its frontmatter.' - # resources/directory/read additions (SEP commit 2e04c48d, 2026-06-09) - - check: sep-2640-capability-directory-read-flag - text: 'Clients MUST NOT call `resources/directory/read` against a server that has not declared `directoryRead: true`.' + # === Directory Listing (resources/directory/read) === - check: sep-2640-directory-read-method-registered text: 'A server that declares `directoryRead` MUST support the method for every directory within the skill namespaces it serves as individual files.' - check: sep-2640-directory-read-subdir-mimetype @@ -89,11 +197,14 @@ requirements: text: 'The method applies only to directory resources. If the URI does not exist, or exists but is not a directory resource, the server MUST return error `-32602` (Invalid params) — the same code `resources/read` uses for unknown resources.' - check: sep-2640-directory-read-pagination text: 'Pagination mirrors `resources/list`: when the result includes `nextCursor`, the client passes it back as `cursor` to retrieve the next page.' + - check: sep-2640-directory-read-empty-dir + text: 'An empty directory yields an empty `resources` array.' + - check: sep-2640-directory-read-not-manifest-extension + text: 'Hosts MUST NOT treat the directory result as extending the manifest. While acting on the skill under the held entry, the host MUST NOT read a newly listed child — an unlisted file is a verification failure, exactly as a digest mismatch is — and MUST NOT surface it to the model as a file of the skill.' + # === Excluded: not observable on the MCP wire === - text: 'Per RFC 3986, the first segment of `` occupies the authority component. This carries no special semantics under this convention and clients MUST NOT attempt DNS or network resolution of it.' excluded: 'DNS and network resolution sit below the MCP wire layer; the harness cannot observe whether the client performed name lookups on URI authority components.' - - text: "[Hosts] SHOULD determine the format from the resource's `mimeType`, falling back to the URL suffix" - excluded: 'Internal decision logic: when `mimeType` and URL suffix agree, the harness cannot distinguish a host that branched on `mimeType` from one that fell back to the suffix.' - text: 'Hosts MUST treat MCP-served skill content as untrusted model input, subject to the same prompt-injection defenses applied to any server-provided text. A server being connected does not make its skill content authoritative.' excluded: 'Internal host policy: "treats as untrusted" is an assertion about how content is reasoned over downstream of the read, not about wire traffic.' url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications @@ -109,3 +220,16 @@ requirements: - text: "Hosts SHOULD indicate which server a skill originates from when presenting it, SHOULD let users inspect a skill's content before it is loaded into model context" excluded: 'UI presentation requirements (origin indicator, pre-load inspection); the harness cannot observe what the host displays to users.' url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - text: 'A host is _acting on_ a skill from the moment it loads the skill''s `SKILL.md` into the model''s context until, at the earliest, that `SKILL.md` leaves context; hosts MAY hold the window open longer, never shorter.' + excluded: 'The window is defined by what is in the model''s context, which is host-internal state the harness cannot observe. It scopes several wire-observable rules but is not itself checkable.' + - text: 'Hosts SHOULD instead cache what they do retrieve, and digests make that cache cheap to validate: a cached file whose digest matches the current entry can be served without fetching it again.' + excluded: 'Cache hits are the absence of a request. A harness cannot distinguish a compliant cache from a host that simply did not need the file again.' + - text: 'Any path at which a host materializes skill content, whether a cache directory or a virtual mount, MUST encode the server identity as well as the `uri`, so that same-URI skills from different servers land at distinct paths.' + excluded: 'Disk layout and cache-path construction are host-local filesystem concerns with no wire representation.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - text: 'A host that declines a skill on this basis SHOULD tell the user why rather than fail silently on a later read.' + excluded: 'User-facing messaging; not protocol-observable.' + - text: 'For a skill whose `resources` is `"dynamic"`, the entry offers nothing to count. A host that chooses to load such a skill applies the total-size limit to what it actually retrieves and MAY stop loading the skill once that limit is reached.' + excluded: 'A host stopping mid-load is indistinguishable on the wire from a host that needed no further files.' + - text: 'Hosts SHOULD expect this sequence and present it as such — a skill that has changed and needs re-approval — rather than as a read error.' + excluded: 'User-facing presentation of the stale-entry recovery path; not protocol-observable.' From 3d75cb0fb63bd0d8fae879b0a84337242e062d7a Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Sat, 29 Aug 2026 11:20:53 -0700 Subject: [PATCH 08/12] feat(sep-2640): close the extraction gaps found by a normative-sentence audit The first pass extracted the Specification sections but under-covered Security Implications, which carries 21 of the SEP's 97 RFC-2119 sentences. Sweeping every normative sentence in the source surfaced rows that were missing entirely: - origin tagging at the point content enters model context (the L179 host MUST) - allowed-tools MUST be ignored for MCP-origin skills - reads bound to the originating server, so a skill from A cannot drive a read against B, and any cross-origin read gated behind per-call approval - servers identified by a host-assigned label, never serverInfo.name - the three disk-cache rules: write-isolation or re-hash, exclusion from filesystem-skill discovery paths, and removal when a server is removed - a resources/read of a SKILL.md by another route is not a load - a nested SKILL.md read as supporting content is ordinary markdown - the registry-assembly and virtual-mount no-prefetch restatements - the SDK affordances guidance, which is the row mcpkit itself is measured by Five rows previously filed as excluded are now declared instead. They are real requirements that a host-side harness could check; calling them unobservable was a limitation of this scenario set, not of the requirement. Adds one wire check, names-should-be-unique, since a within-listing collision is observable and tells an operator the listing needs disambiguating. 96 rows: 89 checks (40 wire-tested) and 7 excluded. --- src/scenarios/server/skills/enumeration.ts | 30 ++++++- src/seps/sep-2640.yaml | 92 ++++++++++++++++++---- 2 files changed, 106 insertions(+), 16 deletions(-) diff --git a/src/scenarios/server/skills/enumeration.ts b/src/scenarios/server/skills/enumeration.ts index 837a1568..6dbc3fdb 100644 --- a/src/scenarios/server/skills/enumeration.ts +++ b/src/scenarios/server/skills/enumeration.ts @@ -75,7 +75,8 @@ const ENTRY_IDS = [ 'sep-2640-limit-total-size', 'sep-2640-metadata-reserved-prefix', 'sep-2640-name-naming-rules', - 'sep-2640-authority-reg-name' + 'sep-2640-authority-reg-name', + 'sep-2640-names-should-be-unique' ] as const; const GET_IDS = [ @@ -729,6 +730,33 @@ function entryChecks(entries: SkillEntry[]): ConformanceCheck[] { ) ); + // === names-should-be-unique (SHOULD) === + // A collision is explicitly permitted — two skills at different paths may + // share a final segment — so this is a WARNING that tells a host operator the + // listing will need disambiguating, not a failure. + const byName = new Map(); + entries.forEach((e) => { + if (typeof e.uri !== 'string') return; + const n = skillNameFromManifestUri(e.uri); + if (n === undefined) return; + byName.set(n, [...(byName.get(n) ?? []), e.uri]); + }); + const collisions = [...byName.entries()] + .filter(([, uris]) => uris.length > 1) + .map(([n, uris]) => `"${n}" served at ${uris.join(' and ')}`); + checks.push( + skillsCheck( + 'sep-2640-names-should-be-unique', + "Within a server's listing, names SHOULD be unique, but they are not guaranteed to be.", + collisions.length === 0 ? 'SUCCESS' : 'WARNING', + collisions.length === 0 + ? { details: { distinctNames: byName.size } } + : { + errorMessage: `Names collide within one listing, so hosts MUST disambiguate them: ${joinErrs(collisions)}` + } + ) + ); + return checks; } diff --git a/src/seps/sep-2640.yaml b/src/seps/sep-2640.yaml index 1b0679e5..6a51ac8c 100644 --- a/src/seps/sep-2640.yaml +++ b/src/seps/sep-2640.yaml @@ -23,6 +23,16 @@ # archive rows) and four were reworded. Because the SEP is mid-vote, expect one # more pass at the CM-stamped head. # +# coverage: extracted against branch head a3e147ca2710 (2026-08-25), the commit +# the CM vote is running on. A keyword sweep of the source finds 97 sentences +# carrying an RFC 2119 term; this file declares 89 checks plus 7 excluded rows. +# The residual handful are restatements that fold into a declared row rather +# than standing alone. Of the 89 declared checks, 40 are exercised on the wire +# by the three server scenarios; the other 49 are host obligations (retrieval +# policy, approval binding, cache isolation, context handling) that a +# server-side harness cannot observe and that would need a host-side scenario +# set to test. +# # backing_scenarios: three server ClientScenarios under # src/scenarios/server/skills/ emit the check IDs below (a row is "tested" # once a scenario emits its check ID; see src/traceability/): @@ -62,6 +72,8 @@ requirements: # SKILL.md MUST NOT appear in any descendant directory. - check: sep-2640-nested-skills-permitted text: 'A `SKILL.md` MAY appear in a descendant directory of a skill — skills can nest.' + - check: sep-2640-nested-skillmd-not-acted-on + text: 'From the enclosing skill''s perspective, a nested skill''s directory and files are ordinary supporting files, and reading them is ordinary reading. A nested `SKILL.md` read this way is ordinary markdown: hosts MUST NOT act on its frontmatter.' - check: sep-2640-nested-publication-flat text: "A nested skill is published like any other: through its own `skills/list` entry, or by explicit reference. The listing remains flat — an entry for a nested skill is an ordinary entry whose `uri` happens to share a path prefix with the enclosing skill's, and nothing in the listing marks nesting." @@ -117,6 +129,10 @@ requirements: text: "The final `` segment of the entry's `uri` MUST equal `frontmatter.name`, per Resource Mapping." - check: sep-2640-metadata-reserved-prefix text: 'Within the frontmatter `metadata` object, keys prefixed with `io.modelcontextprotocol/` are reserved for metadata defined by MCP extensions. This extension currently defines no such keys. Implementations SHOULD ignore keys under this prefix that they do not recognize.' + - check: sep-2640-names-should-be-unique + text: "Within a server's listing, names SHOULD be unique, but they are not guaranteed to be." + - check: sep-2640-host-disambiguate-listing-collision + text: 'When two entries in one listing collide on `name`, hosts MUST disambiguate them — for example by their distinguishing path segments — rather than silently discarding or preferring one.' - check: sep-2640-names-not-unique text: "A skill's `name` is a label, not an identifier. Within a server's listing, names SHOULD be unique, but they are not guaranteed to be. Hosts MUST NOT assume name uniqueness. When two entries in one listing collide on `name`, hosts MUST disambiguate them — for example by their distinguishing path segments — rather than silently discarding or preferring one." @@ -161,6 +177,8 @@ requirements: # === Reading / integrity (host obligations) === - check: sep-2640-host-load-by-uri text: 'hosts MUST support loading a skill given only its URI' + - check: sep-2640-read-is-not-a-load + text: 'Hosts MUST NOT treat a `resources/read` of a `SKILL.md` that arrives by any other route as a load: it grants no approval, opens no window, and confers no standing on the skill''s supporting files.' - check: sep-2640-host-verify-digest text: 'When a host retrieves a file listed in a skill''s `resources`, it MUST verify the content against that entry''s digest. Whatever the cause, hosts MUST NOT use the unverified content.' - check: sep-2640-host-size-mismatch-failure @@ -186,6 +204,65 @@ requirements: - check: sep-2640-host-nested-fresh-consent text: 'Activating a nested skill — loading it as a skill in its own right — requires fresh, explicit user consent; approval of the enclosing skill does not substitute for it. A nested `SKILL.md` read as supporting content is ordinary markdown: hosts MUST NOT act on its frontmatter.' + - check: sep-2640-host-origin-tag-visible + text: 'Hosts MUST tag MCP-served skill content with its originating server identity at the point it enters model context, and MUST NOT present it as indistinguishable from a local filesystem skill.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-untrusted-input + text: 'Hosts MUST treat MCP-served skill content as untrusted model input, subject to the same prompt-injection defenses applied to any server-provided text. A server being connected does not make its skill content authoritative.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-skills-are-data-not-directives + text: 'Hosts MUST NOT treat skill resources as higher-authority than other context. Explicit user policy governs whether a skill is loaded at all.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-cache-path-encodes-origin + text: 'Any path at which a host materializes skill content, whether a cache directory or a virtual mount, MUST encode the server identity as well as the `uri`, so that same-URI skills from different servers land at distinct paths and the originating server is recoverable from the path.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-higher-risk-surface + text: 'Hosts MUST treat MCP-served skills as a higher-risk surface than remote tool invocation.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-no-implicit-local-execution + text: 'Hosts MUST NOT allow MCP-served skill content to cause host-side code execution without explicit per-skill user approval.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-exec-gate-while-acting + text: 'Hosts MUST apply the same approval gate to code-execution tool calls issued while the model is acting on an MCP-served skill.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-reads-bound-to-origin + text: 'Hosts MUST bind such reads to the skill''s originating server: a skill served by server A MUST NOT cause a `resources/read` against server B.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-cross-origin-read-approval + text: 'Any cross-origin read MUST be gated behind explicit per-call user approval naming both servers.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-label-not-serverinfo-name + text: 'Hosts MUST identify servers by a host-assigned label, not the server''s self-reported `serverInfo.name`.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-no-implicit-permission-grants + text: 'Hosts MUST NOT honor frontmatter fields that widen the model''s tool or filesystem permissions when the skill arrives over MCP.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-ignore-allowed-tools + text: 'The Agent Skills `allowed-tools` field, which a filesystem-sourced skill uses to declare the tools available while it runs, MUST be ignored for MCP-origin skills.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-intermediary-meta-prefix + text: 'Intermediaries MAY attach provenance or verification annotations via `_meta` under their own reverse-domain prefix — not the `io.modelcontextprotocol.skills/` prefix reserved for this extension.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-cache-write-isolation + text: 'Hosts that cache skill content on disk MUST do one of the following for every file served from the cache: keep the cache where nothing but the host can write to it, or re-hash the file against the entry on every read.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-cache-excluded-from-discovery + text: 'Hosts that cache MCP-served skill content on disk MUST also do so in a location excluded from every filesystem-skill discovery path, and MUST treat content loaded from there as MCP-origin.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-cache-removal-on-server-removal + text: "Hosts SHOULD remove a server's cached skill content when the user removes that server." + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + + # === Implementation Guidelines (host + SDK) === + - check: sep-2640-host-registry-no-fetch + text: 'Assembling the registry reads only the listing: the host MUST NOT fetch `SKILL.md` or any supporting file at this stage.' + - check: sep-2640-host-virtual-mount-lazy + text: 'A virtual mount resolves reads on access; it MUST NOT be populated by fetching the skill''s files in advance.' + - check: sep-2640-host-surface-directory-read + text: 'When the originating server declares `directoryRead`, the host SHOULD surface this capability to the model.' + - check: sep-2640-sdk-convenience-wrappers + text: 'SDK maintainers SHOULD provide affordances that wrap the underlying resource operations in skill-specific terms. The SDK handles: reading `SKILL.md` frontmatter to populate resource metadata, serving file content on `resources/read`, and answering `skills/get` — and, where the server''s skill set is bounded, `skills/list` — computing entry digests and sizes from the registered files, and warning when a registered skill exceeds the Limits.' + # === Directory Listing (resources/directory/read) === - check: sep-2640-directory-read-method-registered text: 'A server that declares `directoryRead` MUST support the method for every directory within the skill namespaces it serves as individual files.' @@ -205,18 +282,6 @@ requirements: # === Excluded: not observable on the MCP wire === - text: 'Per RFC 3986, the first segment of `` occupies the authority component. This carries no special semantics under this convention and clients MUST NOT attempt DNS or network resolution of it.' excluded: 'DNS and network resolution sit below the MCP wire layer; the harness cannot observe whether the client performed name lookups on URI authority components.' - - text: 'Hosts MUST treat MCP-served skill content as untrusted model input, subject to the same prompt-injection defenses applied to any server-provided text. A server being connected does not make its skill content authoritative.' - excluded: 'Internal host policy: "treats as untrusted" is an assertion about how content is reasoned over downstream of the read, not about wire traffic.' - url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications - - text: 'Hosts MUST NOT honor mechanisms in skill content that would cause local code execution without explicit user opt-in. This includes, non-exhaustively: hook declarations, pre/post-invocation scripts, shell commands embedded in frontmatter, or any field that a filesystem-sourced skill might use to register executable behavior on the host.' - excluded: 'Local code execution and explicit user opt-in are host-side filesystem and UX behaviors; not protocol-observable on the wire.' - url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications - - text: 'Hosts MUST either ignore such fields entirely when the skill arrives over MCP, or gate them behind an explicit per-skill user approval that states what will execute and where.' - excluded: 'Either branch (silent ignore vs. UI-gated approval) is a host-internal handling choice; not protocol-observable.' - url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications - - text: 'Hosts MUST NOT treat skill resources as higher-authority than other context. Explicit user policy governs whether a skill is loaded at all.' - excluded: 'Context-authority ordering is an internal prompting decision; not protocol-observable.' - url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications - text: "Hosts SHOULD indicate which server a skill originates from when presenting it, SHOULD let users inspect a skill's content before it is loaded into model context" excluded: 'UI presentation requirements (origin indicator, pre-load inspection); the harness cannot observe what the host displays to users.' url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications @@ -224,9 +289,6 @@ requirements: excluded: 'The window is defined by what is in the model''s context, which is host-internal state the harness cannot observe. It scopes several wire-observable rules but is not itself checkable.' - text: 'Hosts SHOULD instead cache what they do retrieve, and digests make that cache cheap to validate: a cached file whose digest matches the current entry can be served without fetching it again.' excluded: 'Cache hits are the absence of a request. A harness cannot distinguish a compliant cache from a host that simply did not need the file again.' - - text: 'Any path at which a host materializes skill content, whether a cache directory or a virtual mount, MUST encode the server identity as well as the `uri`, so that same-URI skills from different servers land at distinct paths.' - excluded: 'Disk layout and cache-path construction are host-local filesystem concerns with no wire representation.' - url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications - text: 'A host that declines a skill on this basis SHOULD tell the user why rather than fail silently on a later read.' excluded: 'User-facing messaging; not protocol-observable.' - text: 'For a skill whose `resources` is `"dynamic"`, the entry offers nothing to count. A host that chooses to load such a skill applies the total-size limit to what it actually retrieves and MAY stop loading the skill once that limit is reached.' From 22f4ecbe33c6680959f8254aa13a2d77a0025d19 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Sat, 29 Aug 2026 11:45:06 -0700 Subject: [PATCH 09/12] style: apply prettier to the SEP-2640 scenarios and yaml Quote-style normalisation in the yaml (escaped single quotes become double-quoted scalars) and one return-type wrap in helpers.ts. The parsed yaml is byte-identical in content; verified by comparing the parse before and after. --- src/scenarios/server/skills/enumeration.ts | 53 ++++++++++++++++------ src/scenarios/server/skills/helpers.ts | 7 +-- src/seps/sep-2640.yaml | 52 ++++++++++----------- 3 files changed, 69 insertions(+), 43 deletions(-) diff --git a/src/scenarios/server/skills/enumeration.ts b/src/scenarios/server/skills/enumeration.ts index 6dbc3fdb..60c2a234 100644 --- a/src/scenarios/server/skills/enumeration.ts +++ b/src/scenarios/server/skills/enumeration.ts @@ -368,7 +368,9 @@ function entryChecks(entries: SkillEntry[]): ConformanceCheck[] { entries.forEach((e, i) => { const fm = e.frontmatter; if (!fm || typeof fm !== 'object' || Array.isArray(fm)) { - fmErrs.push(`${entryLabel(e, i)}: frontmatter is missing or not an object`); + fmErrs.push( + `${entryLabel(e, i)}: frontmatter is missing or not an object` + ); return; } const obj = fm as Record; @@ -426,7 +428,9 @@ function entryChecks(entries: SkillEntry[]): ConformanceCheck[] { // skills under another scheme native to its domain and no scheme is // privileged, so a deviation is a WARNING rather than a failure. const otherScheme = entries - .filter((e) => typeof e.uri === 'string' && !e.uri.startsWith(SKILL_URI_SCHEME)) + .filter( + (e) => typeof e.uri === 'string' && !e.uri.startsWith(SKILL_URI_SCHEME) + ) .map((e) => String(e.uri)); checks.push( skillsCheck( @@ -469,8 +473,9 @@ function entryChecks(entries: SkillEntry[]): ConformanceCheck[] { // Entries carrying an array are the only ones the remaining checks apply to. const arrayEntries = entries .map((e, i) => ({ e, i, arr: resourcesArray(e) })) - .filter((x): x is { e: SkillEntry; i: number; arr: SkillResourceEntry[] } => - x.arr !== undefined + .filter( + (x): x is { e: SkillEntry; i: number; arr: SkillResourceEntry[] } => + x.arr !== undefined ); const dynamicOnlyReason = @@ -561,7 +566,10 @@ function entryChecks(entries: SkillEntry[]): ConformanceCheck[] { const digestErrs: string[] = []; for (const { e, i, arr } of arrayEntries) { for (const r of arr) { - if (typeof r.digest !== 'string' || !SKILL_DIGEST_PATTERN.test(r.digest)) { + if ( + typeof r.digest !== 'string' || + !SKILL_DIGEST_PATTERN.test(r.digest) + ) { digestErrs.push( `${entryLabel(e, i)}: ${String(r.uri)} digest=${JSON.stringify(r.digest)} is not sha256:{64 lowercase hex}` ); @@ -665,7 +673,9 @@ function entryChecks(entries: SkillEntry[]): ConformanceCheck[] { if (!meta || typeof meta !== 'object' || Array.isArray(meta)) return; for (const key of Object.keys(meta as Record)) { if (key.startsWith(FRONTMATTER_RESERVED_PREFIX)) { - reservedErrs.push(`${entryLabel(e, i)}: frontmatter.metadata["${key}"]`); + reservedErrs.push( + `${entryLabel(e, i)}: frontmatter.metadata["${key}"]` + ); } } }); @@ -772,7 +782,8 @@ async function readbackChecks( ): Promise { const checks: ConformanceCheck[] = []; const sample = entries.find( - (e) => typeof e.uri === 'string' && e.uri.endsWith(`/${SKILL_MANIFEST_FILENAME}`) + (e) => + typeof e.uri === 'string' && e.uri.endsWith(`/${SKILL_MANIFEST_FILENAME}`) ); const uri = sample?.uri as string | undefined; @@ -797,7 +808,10 @@ async function readbackChecks( { errorMessage: reason } ) ); - for (const id of ['sep-2640-skillmd-frontmatter', 'sep-2640-entry-frontmatter-identical']) { + for (const id of [ + 'sep-2640-skillmd-frontmatter', + 'sep-2640-entry-frontmatter-identical' + ]) { checks.push( skillsCheck(id, 'SKILL.md is unreadable.', 'SKIPPED', { errorMessage: reason @@ -817,7 +831,10 @@ async function readbackChecks( { errorMessage: reason } ) ); - for (const id of ['sep-2640-skillmd-frontmatter', 'sep-2640-entry-frontmatter-identical']) { + for (const id of [ + 'sep-2640-skillmd-frontmatter', + 'sep-2640-entry-frontmatter-identical' + ]) { checks.push( skillsCheck(id, 'No SKILL.md content to inspect.', 'SKIPPED', { errorMessage: reason @@ -862,7 +879,12 @@ async function readbackChecks( // === entry-frontmatter-identical === const declared = sample?.frontmatter as Record | undefined; - if (!fm || !declared || typeof declared !== 'object' || Array.isArray(declared)) { + if ( + !fm || + !declared || + typeof declared !== 'object' || + Array.isArray(declared) + ) { const reason = 'Either the file has no parseable frontmatter or the entry carries no frontmatter object, so the two cannot be compared.'; checks.push( @@ -912,9 +934,7 @@ async function getChecks( const reason = 'No listed entry carries a uri, so skills/get cannot be exercised against a known skill.'; for (const id of GET_IDS) { - checks.push( - skillsCheck(id, reason, 'SKIPPED', { errorMessage: reason }) - ); + checks.push(skillsCheck(id, reason, 'SKIPPED', { errorMessage: reason })); } return checks; } @@ -1025,7 +1045,12 @@ async function getChecks( 'If the URI does not identify a skill the server serves, the server MUST return error -32602 (Invalid params).', ok ? 'SUCCESS' : 'FAILURE', ok - ? { details: { probedUri: UNKNOWN_SKILL_URI, code: unknown.error.code } } + ? { + details: { + probedUri: UNKNOWN_SKILL_URI, + code: unknown.error.code + } + } : { errorMessage: `skills/get on an unserved URI returned code ${unknown.error.code} (${unknown.error.message}); expected ${JSONRPC_INVALID_PARAMS}.` } diff --git a/src/scenarios/server/skills/helpers.ts b/src/scenarios/server/skills/helpers.ts index 7bd24943..699fc307 100644 --- a/src/scenarios/server/skills/helpers.ts +++ b/src/scenarios/server/skills/helpers.ts @@ -158,9 +158,10 @@ export async function skillsCapability( * at Final status, they do not. The envelope is a non-conformant shape emitted * by at least one SDK, so it is reported rather than silently accepted. */ -export function settingsAreInline( - skills: Record -): { inline: boolean; envelopeKeys: string[] } { +export function settingsAreInline(skills: Record): { + inline: boolean; + envelopeKeys: string[]; +} { const envelopeKeys = ['config', 'specVersion', 'stability', 'id'].filter( (k) => k in skills ); diff --git a/src/seps/sep-2640.yaml b/src/seps/sep-2640.yaml index 6a51ac8c..24e2ad03 100644 --- a/src/seps/sep-2640.yaml +++ b/src/seps/sep-2640.yaml @@ -73,7 +73,7 @@ requirements: - check: sep-2640-nested-skills-permitted text: 'A `SKILL.md` MAY appear in a descendant directory of a skill — skills can nest.' - check: sep-2640-nested-skillmd-not-acted-on - text: 'From the enclosing skill''s perspective, a nested skill''s directory and files are ordinary supporting files, and reading them is ordinary reading. A nested `SKILL.md` read this way is ordinary markdown: hosts MUST NOT act on its frontmatter.' + text: "From the enclosing skill's perspective, a nested skill's directory and files are ordinary supporting files, and reading them is ordinary reading. A nested `SKILL.md` read this way is ordinary markdown: hosts MUST NOT act on its frontmatter." - check: sep-2640-nested-publication-flat text: "A nested skill is published like any other: through its own `skills/list` entry, or by explicit reference. The listing remains flat — an entry for a nested skill is an ordinary entry whose `uri` happens to share a path prefix with the enclosing skill's, and nothing in the listing marks nesting." @@ -106,21 +106,21 @@ requirements: - check: sep-2640-skills-list-implemented text: 'A server declaring the `io.modelcontextprotocol/skills` extension MUST implement the `skills/list` method, which returns the skills it serves. The result MAY be empty.' - check: sep-2640-skills-list-pagination - text: 'Pagination mirrors the base protocol''s list methods: the request accepts an optional `cursor`, and when the result includes `nextCursor` the client passes it back to retrieve the next page.' + text: "Pagination mirrors the base protocol's list methods: the request accepts an optional `cursor`, and when the result includes `nextCursor` the client passes it back to retrieve the next page." - check: sep-2640-skills-list-entry-atomic text: "An entry is atomic — a skill's `resources` set is never split across pages." - check: sep-2640-skills-list-cache-attributes - text: 'In protocol versions 2026-07-28 and later, the result also carries the base protocol''s list-caching attributes — `ttlMs` and `cacheScope`, as defined for `tools/list` and `resources/list` (SEP-2549) — with the same semantics: a freshness hint for the listing and a cache-scope marker, not an integrity property.' + text: "In protocol versions 2026-07-28 and later, the result also carries the base protocol's list-caching attributes — `ttlMs` and `cacheScope`, as defined for `tools/list` and `resources/list` (SEP-2549) — with the same semantics: a freshness hint for the listing and a cache-scope marker, not an integrity property." - check: sep-2640-skills-list-may-be-partial text: 'A server whose skill catalog is large, generated on demand, or otherwise unenumerable MAY return an empty or partial listing.' - check: sep-2640-host-no-empty-listing-assumption text: 'Hosts MUST NOT treat an empty or partial listing as proof that a server has no skills.' - check: sep-2640-enumeration-scheme-uniform - text: 'The method serves entries for a server''s skills whatever URI scheme they use — enumeration is uniform across schemes.' + text: "The method serves entries for a server's skills whatever URI scheme they use — enumeration is uniform across schemes." # === The skill entry (shared by skills/list and skills/get) === - check: sep-2640-entry-uri-required - text: '`skills[].uri` (Yes): Resource URI of the skill''s `SKILL.md`.' + text: "`skills[].uri` (Yes): Resource URI of the skill's `SKILL.md`." - check: sep-2640-entry-frontmatter-required text: "`frontmatter` is the skill's `SKILL.md` YAML frontmatter rendered verbatim as a JSON object — every field the author wrote, not a curated subset. Because the Agent Skills specification requires `name` and `description`, those fields are always present." - check: sep-2640-entry-frontmatter-identical @@ -140,13 +140,13 @@ requirements: - check: sep-2640-entry-resources-required text: '`resources` is REQUIRED on every skill entry and takes one of two forms: an array enumerating the skill''s files — `SKILL.md` and every supporting file — as `{uri, digest, size}` triples, or the string `"dynamic"`.' - check: sep-2640-resources-complete - text: 'When present, `resources` MUST be complete: it lists every file of the skill, each exactly once, including an entry matching the skill''s top-level `uri` — that entry carries the digest and size of `SKILL.md` itself.' + text: "When present, `resources` MUST be complete: it lists every file of the skill, each exactly once, including an entry matching the skill's top-level `uri` — that entry carries the digest and size of `SKILL.md` itself." - check: sep-2640-resources-uri-within-skill - text: 'Each `uri` MUST be the skill''s `SKILL.md` or a file within the skill''s directory.' + text: "Each `uri` MUST be the skill's `SKILL.md` or a file within the skill's directory." - check: sep-2640-resources-digest-format - text: 'Digests are SHA-256 hashes of an artifact''s raw bytes, formatted as `sha256:{hex}` where `{hex}` is 64 lowercase hexadecimal characters. Each entry in a skill''s `resources` carries the digest of the file at its `uri`.' + text: "Digests are SHA-256 hashes of an artifact's raw bytes, formatted as `sha256:{hex}` where `{hex}` is 64 lowercase hexadecimal characters. Each entry in a skill's `resources` carries the digest of the file at its `uri`." - check: sep-2640-resources-size-required - text: 'Each entry MUST carry `size`: the length in bytes of the file''s raw content — the same bytes the `digest` covers.' + text: "Each entry MUST carry `size`: the length in bytes of the file's raw content — the same bytes the `digest` covers." - check: sep-2640-resources-dynamic-marker text: 'When a skill''s content is generated dynamically, such that stable digests cannot be published, the server MUST set `"resources": "dynamic"` instead of an array.' - check: sep-2640-resources-invalid-entry @@ -156,9 +156,9 @@ requirements: # === Limits === - check: sep-2640-limit-resources-per-skill - text: 'Resources per skill: 512 entries, counted over the entries of the skill''s `resources`, `SKILL.md` included.' + text: "Resources per skill: 512 entries, counted over the entries of the skill's `resources`, `SKILL.md` included." - check: sep-2640-limit-total-size - text: 'Total file size per skill: 16 MiB (16,777,216 bytes), counted over the sum of `size` over the skill''s `resources`.' + text: "Total file size per skill: 16 MiB (16,777,216 bytes), counted over the sum of `size` over the skill's `resources`." - check: sep-2640-limit-host-support text: 'Hosts MUST support skills up to and including these limits, and MAY support larger ones. Servers SHOULD NOT serve a skill that exceeds either limit; a skill that does is not guaranteed to be loadable by any conforming host.' @@ -178,27 +178,27 @@ requirements: - check: sep-2640-host-load-by-uri text: 'hosts MUST support loading a skill given only its URI' - check: sep-2640-read-is-not-a-load - text: 'Hosts MUST NOT treat a `resources/read` of a `SKILL.md` that arrives by any other route as a load: it grants no approval, opens no window, and confers no standing on the skill''s supporting files.' + text: "Hosts MUST NOT treat a `resources/read` of a `SKILL.md` that arrives by any other route as a load: it grants no approval, opens no window, and confers no standing on the skill's supporting files." - check: sep-2640-host-verify-digest - text: 'When a host retrieves a file listed in a skill''s `resources`, it MUST verify the content against that entry''s digest. Whatever the cause, hosts MUST NOT use the unverified content.' + text: "When a host retrieves a file listed in a skill's `resources`, it MUST verify the content against that entry's digest. Whatever the cause, hosts MUST NOT use the unverified content." - check: sep-2640-host-size-mismatch-failure - text: 'A read whose byte length differs from the entry''s `size` is a verification failure equivalent to a digest mismatch, whether or not the host goes on to compute the digest.' + text: "A read whose byte length differs from the entry's `size` is a verification failure equivalent to a digest mismatch, whether or not the host goes on to compute the digest." - check: sep-2640-host-unlisted-read-failure - text: 'While acting on a skill, a host MUST resolve reads of the skill''s files only to URIs listed in that entry''s `resources`, and MUST treat a read of an unlisted file within the skill as a verification failure equivalent to a digest mismatch.' + text: "While acting on a skill, a host MUST resolve reads of the skill's files only to URIs listed in that entry's `resources`, and MUST treat a read of an unlisted file within the skill as a verification failure equivalent to a digest mismatch." - check: sep-2640-host-no-prefetch - text: 'Hosts MUST NOT retrieve a skill''s files ahead of need — not on connection, not on listing, and not at approval. A `SKILL.md` is fetched when the skill is loaded, and a supporting file when it is read.' + text: "Hosts MUST NOT retrieve a skill's files ahead of need — not on connection, not on listing, and not at approval. A `SKILL.md` is fetched when the skill is loaded, and a supporting file when it is read." - check: sep-2640-host-frontmatter-comparison - text: 'After fetching a `SKILL.md` for which the host holds an entry, hosts MUST parse its YAML frontmatter and compare it field-by-field against the entry''s `frontmatter`. Any discrepancy MUST be treated as a verification failure equivalent to a digest mismatch, and the skill MUST NOT be loaded.' + text: "After fetching a `SKILL.md` for which the host holds an entry, hosts MUST parse its YAML frontmatter and compare it field-by-field against the entry's `frontmatter`. Any discrepancy MUST be treated as a verification failure equivalent to a digest mismatch, and the skill MUST NOT be loaded." - check: sep-2640-host-digest-not-security-boundary text: 'Digests are unsigned and supplied by the same server that supplies the content. A match proves the two are consistent, not that either is trustworthy. Hosts MUST NOT treat a digest match as a security boundary.' - check: sep-2640-host-skill-identity-pair - text: 'The identity of an MCP-served skill is the pair of the host''s identity for the originating server and the skill''s `uri`. Hosts MUST preserve both halves wherever a skill is recorded or addressed — the registry, persisted approvals, the cache, and any tool or path through which the model reaches the skill — and MUST NOT key any of these on the `uri` alone.' + text: "The identity of an MCP-served skill is the pair of the host's identity for the originating server and the skill's `uri`. Hosts MUST preserve both halves wherever a skill is recorded or addressed — the registry, persisted approvals, the cache, and any tool or path through which the model reaches the skill — and MUST NOT key any of these on the `uri` alone." - check: sep-2640-host-not-skill-by-scheme text: 'A host MUST NOT conclude that a resource is a skill merely because its URI carries a particular scheme.' - check: sep-2640-host-cross-origin-no-shadow text: 'When skills from different origins collide on `name`, hosts MUST resolve the name within a per-origin namespace, identifying servers by a host-assigned label; an MCP-served skill MUST NOT silently shadow, or be silently substituted for, a same-named skill from any other origin.' - check: sep-2640-host-content-bound-approval - text: 'When a host persists any per-skill user approval, it MUST be bound to the entry''s `resources` set — every `uri` and `digest` — observed at the moment of approval. If a subsequent entry for that skill advertises a different set, the host MUST treat the prior approval as revoked and re-prompt before loading or executing.' + text: "When a host persists any per-skill user approval, it MUST be bound to the entry's `resources` set — every `uri` and `digest` — observed at the moment of approval. If a subsequent entry for that skill advertises a different set, the host MUST treat the prior approval as revoked and re-prompt before loading or executing." - check: sep-2640-host-dynamic-not-content-bound text: 'A skill whose `resources` is `"dynamic"` cannot be content-bound: hosts MAY decline to load it, and MUST NOT treat a persisted approval as covering whatever content the server currently serves.' - check: sep-2640-host-nested-fresh-consent @@ -226,16 +226,16 @@ requirements: text: 'Hosts MUST apply the same approval gate to code-execution tool calls issued while the model is acting on an MCP-served skill.' url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications - check: sep-2640-host-reads-bound-to-origin - text: 'Hosts MUST bind such reads to the skill''s originating server: a skill served by server A MUST NOT cause a `resources/read` against server B.' + text: "Hosts MUST bind such reads to the skill's originating server: a skill served by server A MUST NOT cause a `resources/read` against server B." url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications - check: sep-2640-host-cross-origin-read-approval text: 'Any cross-origin read MUST be gated behind explicit per-call user approval naming both servers.' url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications - check: sep-2640-host-label-not-serverinfo-name - text: 'Hosts MUST identify servers by a host-assigned label, not the server''s self-reported `serverInfo.name`.' + text: "Hosts MUST identify servers by a host-assigned label, not the server's self-reported `serverInfo.name`." url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications - check: sep-2640-host-no-implicit-permission-grants - text: 'Hosts MUST NOT honor frontmatter fields that widen the model''s tool or filesystem permissions when the skill arrives over MCP.' + text: "Hosts MUST NOT honor frontmatter fields that widen the model's tool or filesystem permissions when the skill arrives over MCP." url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications - check: sep-2640-host-ignore-allowed-tools text: 'The Agent Skills `allowed-tools` field, which a filesystem-sourced skill uses to declare the tools available while it runs, MUST be ignored for MCP-origin skills.' @@ -257,11 +257,11 @@ requirements: - check: sep-2640-host-registry-no-fetch text: 'Assembling the registry reads only the listing: the host MUST NOT fetch `SKILL.md` or any supporting file at this stage.' - check: sep-2640-host-virtual-mount-lazy - text: 'A virtual mount resolves reads on access; it MUST NOT be populated by fetching the skill''s files in advance.' + text: "A virtual mount resolves reads on access; it MUST NOT be populated by fetching the skill's files in advance." - check: sep-2640-host-surface-directory-read text: 'When the originating server declares `directoryRead`, the host SHOULD surface this capability to the model.' - check: sep-2640-sdk-convenience-wrappers - text: 'SDK maintainers SHOULD provide affordances that wrap the underlying resource operations in skill-specific terms. The SDK handles: reading `SKILL.md` frontmatter to populate resource metadata, serving file content on `resources/read`, and answering `skills/get` — and, where the server''s skill set is bounded, `skills/list` — computing entry digests and sizes from the registered files, and warning when a registered skill exceeds the Limits.' + text: "SDK maintainers SHOULD provide affordances that wrap the underlying resource operations in skill-specific terms. The SDK handles: reading `SKILL.md` frontmatter to populate resource metadata, serving file content on `resources/read`, and answering `skills/get` — and, where the server's skill set is bounded, `skills/list` — computing entry digests and sizes from the registered files, and warning when a registered skill exceeds the Limits." # === Directory Listing (resources/directory/read) === - check: sep-2640-directory-read-method-registered @@ -285,8 +285,8 @@ requirements: - text: "Hosts SHOULD indicate which server a skill originates from when presenting it, SHOULD let users inspect a skill's content before it is loaded into model context" excluded: 'UI presentation requirements (origin indicator, pre-load inspection); the harness cannot observe what the host displays to users.' url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications - - text: 'A host is _acting on_ a skill from the moment it loads the skill''s `SKILL.md` into the model''s context until, at the earliest, that `SKILL.md` leaves context; hosts MAY hold the window open longer, never shorter.' - excluded: 'The window is defined by what is in the model''s context, which is host-internal state the harness cannot observe. It scopes several wire-observable rules but is not itself checkable.' + - text: "A host is _acting on_ a skill from the moment it loads the skill's `SKILL.md` into the model's context until, at the earliest, that `SKILL.md` leaves context; hosts MAY hold the window open longer, never shorter." + excluded: "The window is defined by what is in the model's context, which is host-internal state the harness cannot observe. It scopes several wire-observable rules but is not itself checkable." - text: 'Hosts SHOULD instead cache what they do retrieve, and digests make that cache cheap to validate: a cached file whose digest matches the current entry can be served without fetching it again.' excluded: 'Cache hits are the absence of a request. A harness cannot distinguish a compliant cache from a host that simply did not need the file again.' - text: 'A host that declines a skill on this basis SHOULD tell the user why rather than fail silently on a later read.' From e78284749678f9f39bf997a4b60511ec9f3c148f Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Sat, 29 Aug 2026 11:54:39 -0700 Subject: [PATCH 10/12] docs(sep-2640): note why resultType is not declared in this yaml It appears in all three result examples but is a base-protocol field on the common Result interface, not a SEP-2640 requirement; its caching-hint obligations are already owned by sep-2549.yaml. --- src/seps/sep-2640.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/seps/sep-2640.yaml b/src/seps/sep-2640.yaml index 24e2ad03..65ce6ad8 100644 --- a/src/seps/sep-2640.yaml +++ b/src/seps/sep-2640.yaml @@ -33,6 +33,13 @@ # server-side harness cannot observe and that would need a host-side scenario # set to test. # +# deliberately not declared here: `resultType`, which appears in all three +# result examples in this SEP, is a base-protocol field on the common `Result` +# interface (schema/2026-07-28/schema.ts), not a SEP-2640 requirement. Servers +# MUST include it and clients treat an absent value as "complete". Its +# caching-hint obligations belong to sep-2549.yaml. Declaring it here would +# double-count a base-protocol rule against this extension. +# # backing_scenarios: three server ClientScenarios under # src/scenarios/server/skills/ emit the check IDs below (a row is "tested" # once a scenario emits its check ID; see src/traceability/): From 1f21fbaf3c388d9091149e07609729c00afd5bfe Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Fri, 4 Sep 2026 15:49:28 -0700 Subject: [PATCH 11/12] fix(sep-2640): follow nextCursor on resources/directory/read The directory scenario issued one resources/directory/read and every check below it inspected that single page: the result-shape check, the subdirectory-mimeType check, and the target selection for the -32602 probe. SEP-2640 says directory-read pagination mirrors resources/list, so a conformant server MAY split a directory across pages, and reading only the first made "no subdirectory here" indistinguishable from "the subdirectory is on page two". Reported by Sam Bloomberg against the go-sdk implementation, which exposes DirectoryOptions.PageSize. Rebuilding that server with PageSize 1 fails sep-2640-directory-read-subdir-mimetype; with this change it passes 7/7, and both servers still pass at their default page size. Only findable against a second implementation. mcpkit hardcoded its page size to zero with no override, so it never emitted a cursor and the defect was invisible to every run against it. listAllResources already did this for resources/list. directoryReadAll applies the same shape to directory reads, with a page bound and a repeated-cursor guard. --- src/scenarios/server/skills/directory.ts | 5 ++++ src/scenarios/server/skills/helpers.ts | 34 ++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/scenarios/server/skills/directory.ts b/src/scenarios/server/skills/directory.ts index 4c5c2b02..46ec603a 100644 --- a/src/scenarios/server/skills/directory.ts +++ b/src/scenarios/server/skills/directory.ts @@ -33,6 +33,7 @@ import { skillsCheck, listAllResources, skillsListAll, + directoryReadAll, skillNameFromManifestUri } from './helpers'; @@ -210,6 +211,10 @@ export class SkillsDirectoryReadScenario implements ClientScenario { 'resources/directory/read', { uri: target.dirUri } ); + // Every check below inspects the directory's children, so they need + // the whole directory rather than whichever slice fits one page. + const all = await directoryReadAll(conn, target.dirUri); + happy = { ...happy, resources: all.resources }; } catch (e) { happyErr = e; } diff --git a/src/scenarios/server/skills/helpers.ts b/src/scenarios/server/skills/helpers.ts index 699fc307..d682b32c 100644 --- a/src/scenarios/server/skills/helpers.ts +++ b/src/scenarios/server/skills/helpers.ts @@ -376,3 +376,37 @@ export function resourcesArray( export function entryLabel(entry: SkillEntry, i: number): string { return typeof entry.uri === 'string' ? entry.uri : `skills[${i}]`; } + +/** + * Every child of a directory, following `nextCursor` until it clears. + * + * SEP-2640 says directory-read pagination mirrors `resources/list`, so a + * conformant server MAY split a directory across pages. Reading only the + * first page makes "no subdirectory here" indistinguishable from "the + * subdirectory is on page two". + */ +export async function directoryReadAll( + conn: Connection, + uri: string, + maxPages = 50 +): Promise<{ resources: SkillResource[]; pages: number; truncated: boolean }> { + const resources: SkillResource[] = []; + const seen = new Set(); + let cursor: string | undefined; + + for (let i = 0; i < maxPages; i++) { + const page = await conn.request<{ + resources?: SkillResource[]; + nextCursor?: string; + }>('resources/directory/read', cursor ? { uri, cursor } : { uri }); + resources.push(...(page.resources ?? [])); + const next = page.nextCursor; + if (typeof next !== 'string' || next.length === 0) { + return { resources, pages: i + 1, truncated: false }; + } + if (seen.has(next)) return { resources, pages: i + 1, truncated: true }; + seen.add(next); + cursor = next; + } + return { resources, pages: maxPages, truncated: true }; +} From fa8aaec3611e387249da6adc826b02dd77372a3e Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Fri, 4 Sep 2026 18:59:44 -0700 Subject: [PATCH 12/12] feat(sep-2640): client scenario for the no-prefetch MUST NOT The traceability file filed retrieval policy under "host obligations a harness cannot observe", alongside 48 other rows. That was wrong for this one: "hosts MUST NOT retrieve a skill's files ahead of need" reduces to whether a request arrived, which makes it among the most wire-observable rules in the SEP. Adds a Scenario (harness stands up the server, client is the SUT) that serves a listing naming a SKILL.md and a supporting file, then fails if either is read. The client's contract, keyed on MCP_CONFORMANCE_SCENARIO, is to connect, enumerate, and stop. SKIPs rather than passes when the client never calls skills/list, since without a listing there is no window in which prefetching is possible and a pass would prove nothing. Verified both directions against mcpkit's client: SUCCESS as written, and FAILURE with a deliberate ReadFromEntry added to the driver. A check that has never been shown to fail is the same trap as the directory-pagination false negative this suite shipped for months. --- src/scenarios/client/skills/no-prefetch.ts | 172 +++++++++++++++++++++ src/scenarios/index.ts | 8 +- 2 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 src/scenarios/client/skills/no-prefetch.ts diff --git a/src/scenarios/client/skills/no-prefetch.ts b/src/scenarios/client/skills/no-prefetch.ts new file mode 100644 index 00000000..68bbc864 --- /dev/null +++ b/src/scenarios/client/skills/no-prefetch.ts @@ -0,0 +1,172 @@ +/** + * SEP-2640 client scenario: hosts MUST NOT retrieve a skill's files ahead of + * need. + * + * This is a `Scenario` rather than a `ClientScenario`: the harness stands up + * the server and the client is the system under test. The rule reduces to + * "did a request arrive", which makes it one of the most wire-observable + * obligations in the SEP despite the first traceability pass filing it as an + * unobservable host obligation. + * + * Contract for the client under test, keyed on MCP_CONFORMANCE_SCENARIO: + * connect, call `skills/list`, then exit. Do not load a skill. A client that + * prefetches will read `SKILL.md` or a supporting file during that window and + * fail the check. + */ + +import http from 'http'; +import { ConformanceCheck } from '../../../types.js'; +import { BaseHttpScenario } from '../http-base.js'; + +const SPEC_REFERENCE = { + id: 'SEP-2640-Lazy-Retrieval', + url: 'https://modelcontextprotocol.io/seps/2640-skills-extension#integrity-and-verification' +}; + +const SKILLS_EXTENSION_ID = 'io.modelcontextprotocol/skills'; + +/** One skill with a supporting file, so a prefetch has something to grab. */ +const SKILL_URI = 'skill://pdf-processing/SKILL.md'; +const SUPPORTING_URI = 'skill://pdf-processing/references/FORMS.md'; + +const SKILL_MD = `--- +name: pdf-processing +description: Extract, fill, and assemble PDF documents +--- + +Body the client has no business fetching yet. +`; + +const SUPPORTING = + 'Supporting content the client has no business fetching yet.\n'; + +/** sha256 of the two bodies, computed at module load so the entry is honest. */ +import { createHash } from 'crypto'; +const digestOf = (s: string) => + 'sha256:' + createHash('sha256').update(s, 'utf8').digest('hex'); + +export class SkillsNoPrefetchScenario extends BaseHttpScenario { + name = 'sep-2640-client-no-prefetch'; + description = + 'A client MUST NOT retrieve a skill file before the skill is loaded'; + readonly source = { extensionId: SKILLS_EXTENSION_ID } as const; + + /** Every resources/read URI the client asked for, in order. */ + private readsRequested: string[] = []; + private listCalled = false; + + protected handlePost( + _req: http.IncomingMessage, + res: http.ServerResponse, + request: any + ): void { + switch (request.method) { + case 'initialize': + this.sendInitialize(res, request, { + resources: { listChanged: false }, + extensions: { [SKILLS_EXTENSION_ID]: {} } + }); + return; + + case 'skills/list': + this.listCalled = true; + this.sendJson(res, { + jsonrpc: '2.0', + id: request.id, + result: { + resultType: 'complete', + skills: [ + { + uri: SKILL_URI, + frontmatter: { + name: 'pdf-processing', + description: 'Extract, fill, and assemble PDF documents' + }, + resources: [ + { + uri: SKILL_URI, + digest: digestOf(SKILL_MD), + size: Buffer.byteLength(SKILL_MD) + }, + { + uri: SUPPORTING_URI, + digest: digestOf(SUPPORTING), + size: Buffer.byteLength(SUPPORTING) + } + ] + } + ] + } + }); + return; + + // Served, but reaching it during this scenario is the failure. + case 'resources/read': { + const uri = request.params?.uri; + if (typeof uri === 'string') this.readsRequested.push(uri); + const body = uri === SUPPORTING_URI ? SUPPORTING : SKILL_MD; + this.sendJson(res, { + jsonrpc: '2.0', + id: request.id, + result: { + resultType: 'complete', + contents: [{ uri, mimeType: 'text/markdown', text: body }] + } + }); + return; + } + + default: + if (request.id === undefined) { + this.sendNotificationAck(res); + return; + } + this.sendGenericResult(res, request); + } + } + + getChecks(): ConformanceCheck[] { + const DESC = + "Hosts MUST NOT retrieve a skill's files ahead of need, not on connection, not on listing, and not at approval."; + + // Without a listing there is no window in which prefetching is even + // possible, so the run proves nothing rather than passing. + if (!this.listCalled) { + return [ + { + id: 'sep-2640-host-no-prefetch', + name: 'SkillsClientNoPrefetch', + description: DESC, + status: 'SKIPPED', + timestamp: new Date().toISOString(), + errorMessage: + 'the client never called skills/list, so no retrieval window was opened', + specReferences: [SPEC_REFERENCE] + } + ]; + } + + const prefetched = this.readsRequested.filter( + (u) => u === SKILL_URI || u === SUPPORTING_URI + ); + + return [ + { + id: 'sep-2640-host-no-prefetch', + name: 'SkillsClientNoPrefetch', + description: DESC, + status: prefetched.length === 0 ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: + prefetched.length === 0 + ? undefined + : `client read ${prefetched.length} skill file(s) without loading a skill: ${prefetched.join(', ')}`, + specReferences: [SPEC_REFERENCE], + details: { + skillsListCalled: true, + fileReads: prefetched.length + } + } + ]; + } +} diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 52c8ed0f..6c5330bb 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -10,6 +10,7 @@ import { DRAFT_PROTOCOL_VERSION } from '../types'; import { InitializeScenario } from './client/initialize'; +import { SkillsNoPrefetchScenario } from './client/skills/no-prefetch'; import { ToolsCallScenario } from './client/tools_call'; import { ElicitationClientDefaultsScenario } from './client/elicitation-defaults'; import { SSERetryScenario } from './client/sse-retry'; @@ -335,7 +336,12 @@ const scenariosList: Scenario[] = [ new JsonSchemaRefDerefScenario(), // JSON Schema 2020-12 client-side keyword preservation (SEP-1613, SEP-2106) - new JsonSchema2020_12PreservationScenario() + new JsonSchema2020_12PreservationScenario(), + + // SEP-2640 skills, client side. The harness is the server and grades what + // the client requests, which is how the retrieval-policy MUSTs become + // observable at all. + new SkillsNoPrefetchScenario() ]; // Core scenarios (tier 1 requirements)