diff --git a/src/scenarios/client/skills/no-prefetch.ts b/src/scenarios/client/skills/no-prefetch.ts new file mode 100644 index 00000000..68bbc864 --- /dev/null +++ b/src/scenarios/client/skills/no-prefetch.ts @@ -0,0 +1,172 @@ +/** + * SEP-2640 client scenario: hosts MUST NOT retrieve a skill's files ahead of + * need. + * + * This is a `Scenario` rather than a `ClientScenario`: the harness stands up + * the server and the client is the system under test. The rule reduces to + * "did a request arrive", which makes it one of the most wire-observable + * obligations in the SEP despite the first traceability pass filing it as an + * unobservable host obligation. + * + * Contract for the client under test, keyed on MCP_CONFORMANCE_SCENARIO: + * connect, call `skills/list`, then exit. Do not load a skill. A client that + * prefetches will read `SKILL.md` or a supporting file during that window and + * fail the check. + */ + +import http from 'http'; +import { ConformanceCheck } from '../../../types.js'; +import { BaseHttpScenario } from '../http-base.js'; + +const SPEC_REFERENCE = { + id: 'SEP-2640-Lazy-Retrieval', + url: 'https://modelcontextprotocol.io/seps/2640-skills-extension#integrity-and-verification' +}; + +const SKILLS_EXTENSION_ID = 'io.modelcontextprotocol/skills'; + +/** One skill with a supporting file, so a prefetch has something to grab. */ +const SKILL_URI = 'skill://pdf-processing/SKILL.md'; +const SUPPORTING_URI = 'skill://pdf-processing/references/FORMS.md'; + +const SKILL_MD = `--- +name: pdf-processing +description: Extract, fill, and assemble PDF documents +--- + +Body the client has no business fetching yet. +`; + +const SUPPORTING = + 'Supporting content the client has no business fetching yet.\n'; + +/** sha256 of the two bodies, computed at module load so the entry is honest. */ +import { createHash } from 'crypto'; +const digestOf = (s: string) => + 'sha256:' + createHash('sha256').update(s, 'utf8').digest('hex'); + +export class SkillsNoPrefetchScenario extends BaseHttpScenario { + name = 'sep-2640-client-no-prefetch'; + description = + 'A client MUST NOT retrieve a skill file before the skill is loaded'; + readonly source = { extensionId: SKILLS_EXTENSION_ID } as const; + + /** Every resources/read URI the client asked for, in order. */ + private readsRequested: string[] = []; + private listCalled = false; + + protected handlePost( + _req: http.IncomingMessage, + res: http.ServerResponse, + request: any + ): void { + switch (request.method) { + case 'initialize': + this.sendInitialize(res, request, { + resources: { listChanged: false }, + extensions: { [SKILLS_EXTENSION_ID]: {} } + }); + return; + + case 'skills/list': + this.listCalled = true; + this.sendJson(res, { + jsonrpc: '2.0', + id: request.id, + result: { + resultType: 'complete', + skills: [ + { + uri: SKILL_URI, + frontmatter: { + name: 'pdf-processing', + description: 'Extract, fill, and assemble PDF documents' + }, + resources: [ + { + uri: SKILL_URI, + digest: digestOf(SKILL_MD), + size: Buffer.byteLength(SKILL_MD) + }, + { + uri: SUPPORTING_URI, + digest: digestOf(SUPPORTING), + size: Buffer.byteLength(SUPPORTING) + } + ] + } + ] + } + }); + return; + + // Served, but reaching it during this scenario is the failure. + case 'resources/read': { + const uri = request.params?.uri; + if (typeof uri === 'string') this.readsRequested.push(uri); + const body = uri === SUPPORTING_URI ? SUPPORTING : SKILL_MD; + this.sendJson(res, { + jsonrpc: '2.0', + id: request.id, + result: { + resultType: 'complete', + contents: [{ uri, mimeType: 'text/markdown', text: body }] + } + }); + return; + } + + default: + if (request.id === undefined) { + this.sendNotificationAck(res); + return; + } + this.sendGenericResult(res, request); + } + } + + getChecks(): ConformanceCheck[] { + const DESC = + "Hosts MUST NOT retrieve a skill's files ahead of need, not on connection, not on listing, and not at approval."; + + // Without a listing there is no window in which prefetching is even + // possible, so the run proves nothing rather than passing. + if (!this.listCalled) { + return [ + { + id: 'sep-2640-host-no-prefetch', + name: 'SkillsClientNoPrefetch', + description: DESC, + status: 'SKIPPED', + timestamp: new Date().toISOString(), + errorMessage: + 'the client never called skills/list, so no retrieval window was opened', + specReferences: [SPEC_REFERENCE] + } + ]; + } + + const prefetched = this.readsRequested.filter( + (u) => u === SKILL_URI || u === SUPPORTING_URI + ); + + return [ + { + id: 'sep-2640-host-no-prefetch', + name: 'SkillsClientNoPrefetch', + description: DESC, + status: prefetched.length === 0 ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: + prefetched.length === 0 + ? undefined + : `client read ${prefetched.length} skill file(s) without loading a skill: ${prefetched.join(', ')}`, + specReferences: [SPEC_REFERENCE], + details: { + skillsListCalled: true, + fileReads: prefetched.length + } + } + ]; + } +} diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 03cf55f5..6c5330bb 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -10,6 +10,7 @@ import { DRAFT_PROTOCOL_VERSION } from '../types'; import { InitializeScenario } from './client/initialize'; +import { SkillsNoPrefetchScenario } from './client/skills/no-prefetch'; import { ToolsCallScenario } from './client/tools_call'; import { ElicitationClientDefaultsScenario } from './client/elicitation-defaults'; import { SSERetryScenario } from './client/sse-retry'; @@ -58,6 +59,10 @@ import { ResourcesNotFoundErrorScenario } from './server/resources'; +import { SkillsDirectoryReadScenario } from './server/skills/directory'; +import { SkillsEnumerationScenario } from './server/skills/enumeration'; +import { SkillsManifestScenario } from './server/skills/manifest'; + import { PromptsListScenario, PromptsGetSimpleScenario, @@ -151,7 +156,15 @@ const pendingClientScenariosList: ClientScenario[] = [ new TasksDispatchScenario(), new TasksStatusNotificationsScenario(), new TasksRequiredTaskErrorScenario(), - new TasksMrtrCompositionScenario() + new TasksMrtrCompositionScenario(), + + // SEP-2640 Skills extension. Pending because the everything-server does not + // implement io.modelcontextprotocol/skills; targeted runs point at a + // SEP-2640-conformant fixture via + // `npm start -- server --scenario sep-2640-skills-* --url `. + new SkillsDirectoryReadScenario(), + new SkillsEnumerationScenario(), + new SkillsManifestScenario() ]; // All client scenarios @@ -203,6 +216,12 @@ const allClientScenariosList: ClientScenario[] = [ // Resources error handling (SEP-2164) new ResourcesNotFoundErrorScenario(), + // Skills extension (SEP-2640). Fixture-dependent (needs a SEP-2640 server); + // each scenario SKIPs cleanly when the extension is not declared. + new SkillsDirectoryReadScenario(), + new SkillsEnumerationScenario(), + new SkillsManifestScenario(), + // Prompts scenarios new PromptsListScenario(), new PromptsGetSimpleScenario(), @@ -317,7 +336,12 @@ const scenariosList: Scenario[] = [ new JsonSchemaRefDerefScenario(), // JSON Schema 2020-12 client-side keyword preservation (SEP-1613, SEP-2106) - new JsonSchema2020_12PreservationScenario() + new JsonSchema2020_12PreservationScenario(), + + // SEP-2640 skills, client side. The harness is the server and grades what + // the client requests, which is how the retrieval-policy MUSTs become + // observable at all. + new SkillsNoPrefetchScenario() ]; // Core scenarios (tier 1 requirements) diff --git a/src/scenarios/server/skills/directory.ts b/src/scenarios/server/skills/directory.ts new file mode 100644 index 00000000..46ec603a --- /dev/null +++ b/src/scenarios/server/skills/directory.ts @@ -0,0 +1,393 @@ +/** + * SEP-2640 Skills extension — the `resources/directory/read` surface (added in + * spec commit 2e04c48d, 2026-06-09). + * + * One scenario, six checks (per AGENTS.md "fewer scenarios, more checks"). + * Each check's verbatim spec quote lives next to its check ID in + * src/seps/sep-2640.yaml. + * + * Capability gating reads the declared capability from `server/discover` + * (mirrors `tasks/capability.ts`): the checks run only when the server declares + * `io.modelcontextprotocol/skills.directoryRead: true`. An undeclared optional + * capability is a SKIP (not a failure); a declared-but-broken one fails. + * + * Discovery is dynamic and brand-neutral: the directory to exercise is derived + * from `skill://index.json` or `resources/list`, hardcoding no fixture URI, so + * the scenario passes against any conformant SEP-2640 server. When no directory + * (or no subdirectory) can be discovered, that check reports the missing + * prerequisite via untestableCheck (issue #248), never a silent green. + */ + +import { ClientScenario, ConformanceCheck } from '../../../types'; +import { Connection, JsonRpcError, type RunContext } from '../../../connection'; +import { untestableCheck } from '../../untestable'; +import { + SKILLS_EXTENSION_ID, + SKILL_MANIFEST_FILENAME, + SEP_2640_REF, + JSONRPC_METHOD_NOT_FOUND, + JSONRPC_INVALID_PARAMS, + type SkillResource, + skillsCapability, + directoryReadDeclared, + skillsCheck, + listAllResources, + skillsListAll, + directoryReadAll, + skillNameFromManifestUri +} from './helpers'; + +const DIRECTORY_MIME = 'inode/directory'; + +const CAPABILITY_ID = 'sep-2640-capability-directory-read-flag'; +const METHOD_ID = 'sep-2640-directory-read-method-registered'; +const SHAPE_ID = 'sep-2640-directory-read-result-resources-shape'; +const SUBDIR_ID = 'sep-2640-directory-read-subdir-mimetype'; +const INVALID_PARAMS_ID = 'sep-2640-directory-read-invalid-params'; +const PAGINATION_ID = 'sep-2640-directory-read-pagination'; + +const ALL_IDS = [ + CAPABILITY_ID, + METHOD_ID, + SHAPE_ID, + SUBDIR_ID, + INVALID_PARAMS_ID, + PAGINATION_ID +]; + +interface DirectoryReadResult { + resources?: SkillResource[]; + nextCursor?: string; +} + +/** A directory to exercise plus, when known, a non-directory resource under it. */ +interface DirectoryTarget { + dirUri: string; + /** A known file (non-directory) resource, used for the -32602 negative path. */ + fileUri?: string; +} + +/** The skill root directory URI for a SKILL.md URI (strip the trailing file). */ +function skillRootFromManifestUri(uri: string): string | undefined { + if (skillNameFromManifestUri(uri) === undefined) return undefined; + return uri.slice(0, uri.length - `/${SKILL_MANIFEST_FILENAME}`.length); +} + +/** + * Discover a directory resource to exercise, brand-neutrally: prefer a skill + * root derived from a skill-md SKILL.md (index first, then resources/list), + * then any `inode/directory` resource in resources/list. + */ +async function discoverDirectory( + conn: Connection +): Promise { + // 1. A skills/list entry — its SKILL.md URI gives us both a directory (the + // skill root) and a known file (the SKILL.md itself). An unenumerable + // catalog returns nothing here, so discovery falls through. + const listed = await skillsListAll(conn); + if (!('error' in listed)) { + const entry = listed.entries.find( + (e) => + typeof e.uri === 'string' && + skillRootFromManifestUri(e.uri) !== undefined + ); + const uri = entry?.uri as string | undefined; + if (uri) { + return { dirUri: skillRootFromManifestUri(uri)!, fileUri: uri }; + } + } + + const resources = await listAllResources(conn); + + // 2. A SKILL.md in resources/list — derive the skill root the same way. + const manifest = resources.find( + (r) => skillRootFromManifestUri(r.uri) !== undefined + ); + if (manifest) { + return { + dirUri: skillRootFromManifestUri(manifest.uri)!, + fileUri: manifest.uri + }; + } + + // 3. Any directory resource, using a non-directory sibling for the -32602 + // path when one is listed. + const dir = resources.find((r) => r.mimeType === DIRECTORY_MIME); + if (dir) { + const file = resources.find((r) => r.mimeType !== DIRECTORY_MIME); + return { dirUri: dir.uri, fileUri: file?.uri }; + } + + return undefined; +} + +export class SkillsDirectoryReadScenario implements ClientScenario { + name = 'sep-2640-skills-directory'; + readonly source = { extensionId: SKILLS_EXTENSION_ID } as const; + description = `SEP-2640 Skills extension: resources/directory/read surface (added in spec commit 2e04c48d, 2026-06-09). + +**Endpoint**: \`resources/directory/read\` (gated by \`io.modelcontextprotocol/skills.directoryRead: true\`) + +**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-2640.yaml): + +- \`sep-2640-capability-directory-read-flag\` — server declared directoryRead (read from server/discover) +- \`sep-2640-directory-read-method-registered\` — a declaring server supports the method on a served directory (MUST) +- \`sep-2640-directory-read-result-resources-shape\` — result has resources[] of direct children (MUST) +- \`sep-2640-directory-read-subdir-mimetype\` — subdirectory children carry \`inode/directory\` (MUST) +- \`sep-2640-directory-read-invalid-params\` — a non-directory URI returns \`-32602\` (MUST) +- \`sep-2640-directory-read-pagination\` — \`nextCursor\` round-trips per resources/list (single-page is conformant) + +**Gating & discovery**: the checks SKIP when the skills extension or its \`directoryRead\` flag is undeclared. The directory to exercise is discovered dynamically from \`skill://index.json\` / \`resources/list\` — no fixture URI is hardcoded.`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect(); + try { + // === Capability gating via server/discover (not error-inference) === + const skills = await skillsCapability(conn); + if (!skills) { + const reason = + 'Server did not declare the io.modelcontextprotocol/skills extension; directoryRead checks not applicable.'; + return ALL_IDS.map((id) => + skillsCheck(id, reason, 'SKIPPED', { errorMessage: reason }) + ); + } + if (!directoryReadDeclared(skills)) { + const reason = + 'Server declared the skills extension but not directoryRead: true; the resources/directory/read checks are optional and not applicable.'; + return ALL_IDS.map((id) => + skillsCheck(id, reason, 'SKIPPED', { errorMessage: reason }) + ); + } + + const checks: ConformanceCheck[] = []; + + // Check 1: capability declared (observed directly from server/discover). + checks.push( + skillsCheck( + CAPABILITY_ID, + 'Server declared io.modelcontextprotocol/skills.directoryRead: true under capabilities.extensions.', + 'SUCCESS', + { details: { directoryRead: true } } + ) + ); + + // === Discover a directory to exercise (brand-neutral) === + const target = await discoverDirectory(conn); + if (!target) { + const reason = + 'no directory resource discoverable via skill://index.json or resources/list to exercise resources/directory/read'; + const rest: Array<[string, string]> = [ + [ + METHOD_ID, + 'A declaring server MUST support the method on a served directory.' + ], + [ + SHAPE_ID, + 'Result carries resources[] of the directory’s direct children.' + ], + [SUBDIR_ID, 'Subdirectory children carry mimeType inode/directory.'], + [ + INVALID_PARAMS_ID, + 'A non-directory URI yields -32602 Invalid params.' + ], + [ + PAGINATION_ID, + 'nextCursor round-trips per the resources/list contract.' + ] + ]; + for (const [id, desc] of rest) { + checks.push( + untestableCheck(id, id, desc, reason, [SEP_2640_REF], 'FAILURE') + ); + } + return checks; + } + + // === Happy path: list the discovered directory === + let happy: DirectoryReadResult | undefined; + let happyErr: unknown; + try { + happy = await conn.request( + 'resources/directory/read', + { uri: target.dirUri } + ); + // Every check below inspects the directory's children, so they need + // the whole directory rather than whichever slice fits one page. + const all = await directoryReadAll(conn, target.dirUri); + happy = { ...happy, resources: all.resources }; + } catch (e) { + happyErr = e; + } + + // Check 2: method registered (declared -> MUST be supported). + const methodNotFound = + happyErr instanceof JsonRpcError && + happyErr.code === JSONRPC_METHOD_NOT_FOUND; + checks.push( + skillsCheck( + METHOD_ID, + 'A server that declares directoryRead MUST support resources/directory/read on a served skill directory.', + happy !== undefined ? 'SUCCESS' : 'FAILURE', + happy !== undefined + ? { details: { uri: target.dirUri } } + : { + errorMessage: methodNotFound + ? `resources/directory/read returned -32601 for ${target.dirUri} despite the server declaring directoryRead: true` + : `resources/directory/read on ${target.dirUri} failed: ${ + happyErr instanceof Error + ? happyErr.message + : String(happyErr) + }` + } + ) + ); + + // Check 3: result shape — resources[] of Resource objects. + const shapeErrs: string[] = []; + if (!Array.isArray(happy?.resources)) { + shapeErrs.push('result.resources is not an array'); + } else { + happy.resources.forEach((r, i) => { + if (typeof r.uri !== 'string') { + shapeErrs.push(`resources[${i}].uri is not a string`); + } + }); + } + checks.push( + skillsCheck( + SHAPE_ID, + 'The result contains resources[] listing the directory’s direct children, each with at least a uri.', + happy === undefined + ? 'FAILURE' + : shapeErrs.length === 0 + ? 'SUCCESS' + : 'FAILURE', + happy === undefined + ? { errorMessage: 'directory read did not return a result' } + : shapeErrs.length === 0 + ? { details: { childCount: happy.resources?.length ?? 0 } } + : { errorMessage: shapeErrs.join('; ') } + ) + ); + + // Check 4: subdirectory mime marker. A directory whose fixture exposes no + // child subdirectory cannot exercise this — report it untestable, not a + // pass and not a failure of the server. + const subdirChild = Array.isArray(happy?.resources) + ? happy.resources.find((r) => r.mimeType === DIRECTORY_MIME) + : undefined; + if (subdirChild) { + checks.push( + skillsCheck( + SUBDIR_ID, + 'A subdirectory child is listed as a directory resource (mimeType inode/directory) so clients can descend.', + 'SUCCESS', + { details: { subdirectoryUri: subdirChild.uri } } + ) + ); + } else { + checks.push( + untestableCheck( + SUBDIR_ID, + SUBDIR_ID, + 'A subdirectory child is listed with mimeType inode/directory.', + `no child with mimeType ${DIRECTORY_MIME} under ${target.dirUri}; the served directory exposes no subdirectory to exercise this check`, + [SEP_2640_REF], + 'FAILURE' + ) + ); + } + + // Check 5: non-directory URI -> -32602. Needs a known non-directory + // resource; prefer the discovered fileUri, else a non-directory child. + const nonDirUri = + target.fileUri ?? + (Array.isArray(happy?.resources) + ? happy.resources.find( + (r) => typeof r.uri === 'string' && r.mimeType !== DIRECTORY_MIME + )?.uri + : undefined); + if (nonDirUri === undefined) { + checks.push( + untestableCheck( + INVALID_PARAMS_ID, + INVALID_PARAMS_ID, + 'A non-directory URI yields -32602 Invalid params.', + 'no non-directory resource discoverable to probe the -32602 path', + [SEP_2640_REF], + 'FAILURE' + ) + ); + } else { + let invalidOk = false; + let invalidDetail = ''; + try { + await conn.request('resources/directory/read', { + uri: nonDirUri + }); + invalidDetail = `expected -32602 for non-directory URI ${nonDirUri}, got a successful result`; + } catch (e) { + if (e instanceof JsonRpcError && e.code === JSONRPC_INVALID_PARAMS) { + invalidOk = true; + } else if (e instanceof JsonRpcError) { + invalidDetail = `expected -32602 for ${nonDirUri}, got ${e.code}: ${e.message}`; + } else { + invalidDetail = `expected -32602, got non-JsonRpcError: ${ + e instanceof Error ? e.message : String(e) + }`; + } + } + checks.push( + skillsCheck( + INVALID_PARAMS_ID, + 'resources/directory/read on a non-directory URI MUST return -32602 (Invalid params).', + invalidOk ? 'SUCCESS' : 'FAILURE', + invalidOk + ? { details: { nonDirectoryUri: nonDirUri } } + : { errorMessage: invalidDetail } + ) + ); + } + + // Check 6: pagination contract (single-page is conformant). + let paginationOk = false; + let paginationDetail = ''; + const firstCursor = happy?.nextCursor; + if (happy === undefined) { + paginationDetail = 'no directory result to evaluate pagination'; + } else if (!firstCursor) { + paginationOk = true; + paginationDetail = 'single-page response (no nextCursor)'; + } else { + try { + const second = await conn.request( + 'resources/directory/read', + { uri: target.dirUri, cursor: firstCursor } + ); + paginationOk = Array.isArray(second.resources); + paginationDetail = paginationOk + ? `nextCursor round-tripped: ${firstCursor}` + : 'follow-up call returned non-array resources'; + } catch (e) { + paginationDetail = `follow-up call with cursor failed: ${ + e instanceof Error ? e.message : String(e) + }`; + } + } + checks.push( + skillsCheck( + PAGINATION_ID, + 'nextCursor round-trips per the resources/list contract (single-page responses are conformant).', + paginationOk ? 'SUCCESS' : 'FAILURE', + paginationOk + ? { details: { paginationDetail } } + : { errorMessage: paginationDetail } + ) + ); + + return checks; + } finally { + await conn.close(); + } + } +} diff --git a/src/scenarios/server/skills/enumeration.ts b/src/scenarios/server/skills/enumeration.ts new file mode 100644 index 00000000..60c2a234 --- /dev/null +++ b/src/scenarios/server/skills/enumeration.ts @@ -0,0 +1,1073 @@ +/** + * SEP-2640 Skills extension — the `skills/list` and `skills/get` surface. + * + * Replaces the former `skill://index.json` scenario. The 2026-08-21 revision of + * the SEP removed that well-known resource entirely (it appears nowhere in the + * current text) and replaced it with two methods that every server declaring the + * extension MUST implement. + * + * One scenario, many checks (per AGENTS.md "fewer scenarios, more checks"). + * Each check's verbatim spec quote lives next to its check ID in + * src/seps/sep-2640.yaml, keeping the YAML and this scenario in lock-step. + * + * All discovery is dynamic and brand-neutral: the scenario enumerates whatever + * the server serves and validates the entries it finds, hardcoding no + * fixture-specific skill name or URI. When the server does not declare the + * skills extension the checks are SKIPPED (an optional, undeclared capability). + * An empty listing is explicitly permitted, so entry-level checks SKIP rather + * than fail against a server with an unenumerable catalog. + */ + +import { ClientScenario, ConformanceCheck } from '../../../types'; +import type { RunContext } from '../../../connection'; +import { + SKILLS_EXTENSION_ID, + SKILLS_LIST_METHOD, + SKILLS_GET_METHOD, + SKILL_URI_SCHEME, + SKILL_MANIFEST_FILENAME, + SKILL_DIGEST_PATTERN, + RESOURCES_DYNAMIC, + MAX_RESOURCES_PER_SKILL, + MAX_TOTAL_SIZE_PER_SKILL, + FRONTMATTER_RESERVED_PREFIX, + JSONRPC_INVALID_PARAMS, + type SkillEntry, + type SkillResourceEntry, + skillsCapability, + skillsCheck, + skillsListAll, + skillsGet, + settingsAreInline, + skillNameFromManifestUri, + skillRootFromManifestUri, + isDynamicResources, + resourcesArray, + entryLabel, + readResourceText, + parseFrontmatter +} from './helpers'; + +const CAPABILITY_IDS = [ + 'sep-2640-capability-declaration-inline', + 'sep-2640-capability-commits-to-methods', + 'sep-2640-capability-empty-object' +] as const; + +const LIST_IDS = [ + 'sep-2640-skills-list-implemented', + 'sep-2640-skills-list-pagination', + 'sep-2640-skills-list-entry-atomic', + 'sep-2640-skills-list-cache-attributes' +] as const; + +const ENTRY_IDS = [ + 'sep-2640-entry-uri-required', + 'sep-2640-entry-frontmatter-required', + 'sep-2640-entry-uri-matches-frontmatter-name', + 'sep-2640-skill-uri-scheme', + 'sep-2640-entry-resources-required', + 'sep-2640-resources-complete', + 'sep-2640-resources-uri-within-skill', + 'sep-2640-resources-digest-format', + 'sep-2640-resources-size-required', + 'sep-2640-limit-resources-per-skill', + 'sep-2640-limit-total-size', + 'sep-2640-metadata-reserved-prefix', + 'sep-2640-name-naming-rules', + 'sep-2640-authority-reg-name', + 'sep-2640-names-should-be-unique' +] as const; + +const GET_IDS = [ + 'sep-2640-skills-get-implemented', + 'sep-2640-skills-get-entry-shape', + 'sep-2640-skills-get-no-cursor', + 'sep-2640-skills-get-unknown-uri-invalid-params' +] as const; + +/** Emitted by the read-back pass, which fetches one listed SKILL.md. */ +const READBACK_IDS = [ + 'sep-2640-skillmd-required', + 'sep-2640-skillmd-frontmatter', + 'sep-2640-entry-frontmatter-identical' +] as const; + +const ALL_CHECK_IDS = [ + ...CAPABILITY_IDS, + ...LIST_IDS, + ...ENTRY_IDS, + ...GET_IDS, + ...READBACK_IDS +]; + +/** + * Agent Skills naming rules as the SEP defers to them: 1-64 characters, + * lowercase alphanumeric and hyphens. + */ +const SKILL_NAME_PATTERN = /^[a-z0-9]([a-z0-9-]{0,62}[a-z0-9])?$/; + +/** RFC 3986 reg-name: unreserved / pct-encoded / sub-delims, case-insensitive. */ +const REG_NAME_PATTERN = /^(?:[A-Za-z0-9\-._~!$&'()*+,;=]|%[0-9A-Fa-f]{2})*$/; + +/** A URI that no conformant server should serve, for the -32602 probe. */ +const UNKNOWN_SKILL_URI = + 'skill://mcp-conformance-nonexistent-skill-9f3a2b/SKILL.md'; + +function joinErrs(errs: string[], limit = 5): string { + const shown = errs.slice(0, limit).join('; '); + return errs.length > limit + ? `${shown} (+${errs.length - limit} more)` + : shown; +} + +export class SkillsEnumerationScenario implements ClientScenario { + name = 'sep-2640-skills-enumeration'; + readonly source = { extensionId: SKILLS_EXTENSION_ID } as const; + description = `SEP-2640 Skills extension: \`skills/list\` enumeration and \`skills/get\` retrieval. + +**Methods**: \`skills/list\`, \`skills/get\` (both mandatory for a server declaring \`io.modelcontextprotocol/skills\`) + +**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-2640.yaml): + +- \`sep-2640-capability-declaration-inline\` — extension settings sit inline under the identifier, per SEP-2133 (no \`config\` envelope) +- \`sep-2640-capability-commits-to-methods\` — declaring the extension commits the server to both methods +- \`sep-2640-skills-list-implemented\` — \`skills/list\` is implemented and returns a \`skills\` array +- \`sep-2640-skills-list-pagination\` — \`nextCursor\` is honoured as a cursor on the next request +- \`sep-2640-skills-list-entry-atomic\` — no skill entry is split across pages +- \`sep-2640-entry-uri-required\` / \`sep-2640-entry-frontmatter-required\` / \`sep-2640-entry-resources-required\` — the three required entry fields +- \`sep-2640-entry-uri-matches-frontmatter-name\` — the final skill-path segment equals \`frontmatter.name\` +- \`sep-2640-resources-complete\` — \`resources\` includes an entry matching the skill's own \`uri\`, each file once +- \`sep-2640-resources-uri-within-skill\` / \`sep-2640-resources-digest-format\` / \`sep-2640-resources-size-required\` — the \`{uri, digest, size}\` triple +- \`sep-2640-limit-resources-per-skill\` / \`sep-2640-limit-total-size\` — 512 entries, 16 MiB +- \`sep-2640-metadata-reserved-prefix\` — frontmatter \`metadata\` keys under \`io.modelcontextprotocol/\` are reserved +- \`sep-2640-skills-get-*\` — \`skills/get\` returns a list-shaped entry, carries no cursor, and answers \`-32602\` for an unknown URI + +**Discovery is dynamic**: an undeclared extension SKIPs everything; an empty or partial listing is permitted and SKIPs the entry-level checks.`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect(); + try { + const skills = await skillsCapability(conn); + if (!skills) { + const reason = + 'Server did not declare the io.modelcontextprotocol/skills extension; enumeration checks not applicable.'; + return ALL_CHECK_IDS.map((id) => + skillsCheck(id, reason, 'SKIPPED', { errorMessage: reason }) + ); + } + + const checks: ConformanceCheck[] = []; + + // === capability-declaration-inline === + // SEP-2133 (Final) maps an extension identifier straight to its settings + // object. An envelope hides settings from any spec-following client. + const { inline, envelopeKeys } = settingsAreInline(skills); + checks.push( + skillsCheck( + 'sep-2640-capability-declaration-inline', + 'Extension settings are a map of extension identifiers to per-extension settings objects; the settings sit directly under the identifier.', + inline ? 'SUCCESS' : 'FAILURE', + inline + ? { details: { settingKeys: Object.keys(skills) } } + : { + errorMessage: `capabilities.extensions["${SKILLS_EXTENSION_ID}"] carries envelope key(s) ${envelopeKeys.join(', ')} instead of the settings object itself. SEP-2133 (Final) defines no envelope, and SEP-2640's capability block places directoryRead inline.`, + details: { envelopeKeys, observed: skills } + } + ) + ); + + // === capability-empty-object === + // "An empty object indicates support for the extension with no optional + // features." Observable as: the declared value is a JSON object, and an + // empty one is a valid declaration rather than a malformed capability. + checks.push( + skillsCheck( + 'sep-2640-capability-empty-object', + 'An empty object indicates support for the extension with no optional features.', + 'SUCCESS', + { + details: { + declaredKeys: Object.keys(skills), + empty: Object.keys(skills).length === 0 + } + } + ) + ); + + // === skills/list === + const listed = await skillsListAll(conn); + if ('error' in listed) { + const reason = `${SKILLS_LIST_METHOD} failed with code ${listed.error.code}: ${listed.error.message}. A server declaring the extension MUST implement it.`; + checks.push( + skillsCheck( + 'sep-2640-capability-commits-to-methods', + 'Declaring the extension commits the server to skills/list and skills/get.', + 'FAILURE', + { errorMessage: reason } + ), + skillsCheck( + 'sep-2640-skills-list-implemented', + 'A server declaring the extension MUST implement the skills/list method.', + 'FAILURE', + { errorMessage: reason } + ) + ); + for (const id of [ + 'sep-2640-skills-list-pagination', + 'sep-2640-skills-list-entry-atomic', + ...ENTRY_IDS, + ...GET_IDS, + ...READBACK_IDS + ]) { + checks.push( + skillsCheck(id, 'skills/list is unavailable.', 'SKIPPED', { + errorMessage: reason + }) + ); + } + return checks; + } + + const { entries, pages, truncated } = listed; + + checks.push( + skillsCheck( + 'sep-2640-skills-list-implemented', + 'A server declaring the extension MUST implement the skills/list method, which returns the skills it serves. The result MAY be empty.', + 'SUCCESS', + { + details: { + pages: pages.length, + entries: entries.length, + emptyListingPermitted: entries.length === 0 + } + } + ) + ); + + // === skills-list-pagination === + // Multi-page runs prove the cursor round-trips. A single page is a clean + // pass: the contract is "when nextCursor is present, pass it back", and + // skillsListAll did exactly that to reach the end. + checks.push( + skillsCheck( + 'sep-2640-skills-list-pagination', + 'Pagination mirrors the base protocol: the request accepts an optional cursor, and when the result includes nextCursor the client passes it back.', + truncated ? 'FAILURE' : 'SUCCESS', + truncated + ? { + errorMessage: `skills/list did not terminate: the server kept returning a nextCursor (or repeated one) across ${pages.length} pages.` + } + : { details: { pages: pages.length } } + ) + ); + + // === skills-list-entry-atomic === + // "An entry is atomic — a skill's resources set is never split across + // pages." Observable as a URI appearing in more than one page. + const uriPages = new Map(); + pages.forEach((page, pageIdx) => { + for (const e of page.entries) { + if (typeof e.uri !== 'string') continue; + const seen = uriPages.get(e.uri) ?? []; + if (!seen.includes(pageIdx)) seen.push(pageIdx); + uriPages.set(e.uri, seen); + } + }); + const split = [...uriPages.entries()] + .filter(([, p]) => p.length > 1) + .map(([uri, p]) => `${uri} appears on pages ${p.join(', ')}`); + checks.push( + skillsCheck( + 'sep-2640-skills-list-entry-atomic', + "An entry is atomic — a skill's resources set is never split across pages.", + split.length === 0 ? 'SUCCESS' : 'FAILURE', + split.length === 0 + ? { details: { pages: pages.length, distinctUris: uriPages.size } } + : { errorMessage: joinErrs(split) } + ) + ); + + // === skills-list-cache-attributes === + // SEP-2549 attributes are required only on protocol 2026-07-28 and later. + // The harness does not gate on the negotiated version here, so an absent + // attribute is reported as a WARNING rather than a failure. + const first = pages[0]?.result ?? {}; + const hasTtl = first.ttlMs !== undefined; + const hasScope = first.cacheScope !== undefined; + checks.push( + skillsCheck( + 'sep-2640-skills-list-cache-attributes', + "In protocol versions 2026-07-28 and later, the skills/list result carries the base protocol's list-caching attributes ttlMs and cacheScope (SEP-2549).", + hasTtl && hasScope ? 'SUCCESS' : 'WARNING', + hasTtl && hasScope + ? { details: { ttlMs: first.ttlMs, cacheScope: first.cacheScope } } + : { + errorMessage: `skills/list result omits ${[!hasTtl && 'ttlMs', !hasScope && 'cacheScope'].filter(Boolean).join(' and ')}. Required only on protocol 2026-07-28 and later; on an earlier negotiated version this is expected.` + } + ) + ); + + if (entries.length === 0) { + const reason = + 'skills/list returned no entries; entry-level checks not applicable. A server whose catalog is large, generated on demand, or otherwise unenumerable MAY return an empty listing.'; + for (const id of [...ENTRY_IDS, ...GET_IDS, ...READBACK_IDS]) { + checks.push( + skillsCheck(id, reason, 'SKIPPED', { errorMessage: reason }) + ); + } + checks.push( + skillsCheck( + 'sep-2640-capability-commits-to-methods', + 'Declaring the extension commits the server to skills/list and skills/get.', + 'SUCCESS', + { details: { note: 'skills/list answered; listing is empty.' } } + ) + ); + return checks; + } + + checks.push(...entryChecks(entries)); + checks.push(...(await getChecks(conn, entries))); + checks.push(...(await readbackChecks(conn, entries))); + + return checks; + } finally { + await conn.close(); + } + } +} + +/** Validate every `skills[]` entry against the §Discovery entry schema. */ +function entryChecks(entries: SkillEntry[]): ConformanceCheck[] { + const checks: ConformanceCheck[] = []; + + // === entry-uri-required === + const uriErrs = entries + .map((e, i) => + typeof e.uri === 'string' && e.uri.length > 0 + ? null + : `skills[${i}].uri is missing or not a string` + ) + .filter((x): x is string => x !== null); + checks.push( + skillsCheck( + 'sep-2640-entry-uri-required', + "Every entry carries uri, the full resource URI of the skill's SKILL.md.", + uriErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + uriErrs.length === 0 + ? { details: { entryCount: entries.length } } + : { errorMessage: joinErrs(uriErrs) } + ) + ); + + // === entry-frontmatter-required === + // Verbatim frontmatter, so name and description are always present. + const fmErrs: string[] = []; + entries.forEach((e, i) => { + const fm = e.frontmatter; + if (!fm || typeof fm !== 'object' || Array.isArray(fm)) { + fmErrs.push( + `${entryLabel(e, i)}: frontmatter is missing or not an object` + ); + return; + } + const obj = fm as Record; + if (typeof obj.name !== 'string' || obj.name.length === 0) { + fmErrs.push(`${entryLabel(e, i)}: frontmatter.name is missing or empty`); + } + if (typeof obj.description !== 'string' || obj.description.length === 0) { + fmErrs.push( + `${entryLabel(e, i)}: frontmatter.description is missing or empty` + ); + } + }); + checks.push( + skillsCheck( + 'sep-2640-entry-frontmatter-required', + "frontmatter is the skill's SKILL.md YAML frontmatter rendered verbatim as a JSON object; because the Agent Skills specification requires name and description, those fields are always present.", + fmErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + fmErrs.length === 0 + ? { details: { entryCount: entries.length } } + : { errorMessage: joinErrs(fmErrs) } + ) + ); + + // === entry-uri-matches-frontmatter-name === + const nameErrs: string[] = []; + entries.forEach((e) => { + if (typeof e.uri !== 'string') return; + const fm = e.frontmatter as Record | undefined; + const declared = fm && typeof fm.name === 'string' ? fm.name : undefined; + if (declared === undefined) return; + const fromUri = skillNameFromManifestUri(e.uri); + if (fromUri === undefined) { + nameErrs.push( + `${e.uri}: does not end in /${SKILL_MANIFEST_FILENAME}, so the skill name is not recoverable from the URI` + ); + } else if (fromUri !== declared) { + nameErrs.push( + `${e.uri}: final skill-path segment "${fromUri}" !== frontmatter.name "${declared}"` + ); + } + }); + checks.push( + skillsCheck( + 'sep-2640-entry-uri-matches-frontmatter-name', + "The final segment of the entry's uri MUST equal frontmatter.name.", + nameErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + nameErrs.length === 0 + ? { details: { entryCount: entries.length } } + : { errorMessage: joinErrs(nameErrs) } + ) + ); + + // === skill-uri-scheme (SHOULD) === + // "Servers SHOULD use the skill:// URI scheme", but a server MAY serve + // skills under another scheme native to its domain and no scheme is + // privileged, so a deviation is a WARNING rather than a failure. + const otherScheme = entries + .filter( + (e) => typeof e.uri === 'string' && !e.uri.startsWith(SKILL_URI_SCHEME) + ) + .map((e) => String(e.uri)); + checks.push( + skillsCheck( + 'sep-2640-skill-uri-scheme', + 'Servers SHOULD use the skill:// URI scheme for the resources of a skill.', + otherScheme.length === 0 ? 'SUCCESS' : 'WARNING', + otherScheme.length === 0 + ? { details: { entryCount: entries.length } } + : { + errorMessage: `Entries served under another scheme (explicitly permitted; no scheme is privileged): ${joinErrs(otherScheme)}` + } + ) + ); + + // === entry-resources-required === + const resErrs: string[] = []; + entries.forEach((e, i) => { + if (isDynamicResources(e)) return; + if (resourcesArray(e) !== undefined) return; + resErrs.push( + `${entryLabel(e, i)}: resources is ${JSON.stringify(e.resources)}, neither an array nor "${RESOURCES_DYNAMIC}"` + ); + }); + checks.push( + skillsCheck( + 'sep-2640-entry-resources-required', + 'resources is REQUIRED on every skill entry and takes one of two forms: an array of {uri, digest, size} triples, or the string "dynamic". An entry with no resources at all, or with any other value, is invalid.', + resErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + resErrs.length === 0 + ? { + details: { + entryCount: entries.length, + dynamicEntries: entries.filter(isDynamicResources).length + } + } + : { errorMessage: joinErrs(resErrs) } + ) + ); + + // Entries carrying an array are the only ones the remaining checks apply to. + const arrayEntries = entries + .map((e, i) => ({ e, i, arr: resourcesArray(e) })) + .filter( + (x): x is { e: SkillEntry; i: number; arr: SkillResourceEntry[] } => + x.arr !== undefined + ); + + const dynamicOnlyReason = + 'Every entry declares "resources": "dynamic", which publishes no file manifest; the resources-array checks are not applicable.'; + + if (arrayEntries.length === 0) { + for (const id of [ + 'sep-2640-resources-complete', + 'sep-2640-resources-uri-within-skill', + 'sep-2640-resources-digest-format', + 'sep-2640-resources-size-required', + 'sep-2640-limit-resources-per-skill', + 'sep-2640-limit-total-size' + ]) { + checks.push( + skillsCheck(id, dynamicOnlyReason, 'SKIPPED', { + errorMessage: dynamicOnlyReason + }) + ); + } + } else { + // === resources-complete === + // Observable half: an entry matching the skill's own uri, and no file + // listed twice. Full completeness (every file of the skill) cannot be + // confirmed from the wire without a second source of truth. + const completeErrs: string[] = []; + for (const { e, i, arr } of arrayEntries) { + const own = typeof e.uri === 'string' ? e.uri : undefined; + if (own && !arr.some((r) => r.uri === own)) { + completeErrs.push( + `${entryLabel(e, i)}: resources has no entry matching the skill's own uri (the SKILL.md digest and size)` + ); + } + const seen = new Set(); + const dupes = new Set(); + for (const r of arr) { + if (typeof r.uri !== 'string') continue; + if (seen.has(r.uri)) dupes.add(r.uri); + seen.add(r.uri); + } + if (dupes.size > 0) { + completeErrs.push( + `${entryLabel(e, i)}: resources lists ${[...dupes].join(', ')} more than once` + ); + } + } + checks.push( + skillsCheck( + 'sep-2640-resources-complete', + 'When present, resources MUST be complete: it lists every file of the skill, each exactly once, including an entry matching the skill top-level uri — that entry carries the digest and size of SKILL.md itself.', + completeErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + completeErrs.length === 0 + ? { details: { entriesChecked: arrayEntries.length } } + : { errorMessage: joinErrs(completeErrs) } + ) + ); + + // === resources-uri-within-skill === + const containErrs: string[] = []; + for (const { e, i, arr } of arrayEntries) { + const own = typeof e.uri === 'string' ? e.uri : undefined; + const root = own ? skillRootFromManifestUri(own) : undefined; + if (!root) continue; + for (const r of arr) { + if (typeof r.uri !== 'string') { + containErrs.push(`${entryLabel(e, i)}: a resources entry has no uri`); + continue; + } + if (r.uri !== own && !r.uri.startsWith(`${root}/`)) { + containErrs.push( + `${entryLabel(e, i)}: ${r.uri} is outside the skill directory ${root}` + ); + } + } + } + checks.push( + skillsCheck( + 'sep-2640-resources-uri-within-skill', + "Each uri MUST be the skill's SKILL.md or a file within the skill's directory.", + containErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + containErrs.length === 0 + ? { details: { entriesChecked: arrayEntries.length } } + : { errorMessage: joinErrs(containErrs) } + ) + ); + + // === resources-digest-format === + const digestErrs: string[] = []; + for (const { e, i, arr } of arrayEntries) { + for (const r of arr) { + if ( + typeof r.digest !== 'string' || + !SKILL_DIGEST_PATTERN.test(r.digest) + ) { + digestErrs.push( + `${entryLabel(e, i)}: ${String(r.uri)} digest=${JSON.stringify(r.digest)} is not sha256:{64 lowercase hex}` + ); + } + } + } + checks.push( + skillsCheck( + 'sep-2640-resources-digest-format', + "Digests are SHA-256 hashes of an artifact's raw bytes, formatted as sha256:{hex} where {hex} is 64 lowercase hexadecimal characters.", + digestErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + digestErrs.length === 0 + ? { details: { entriesChecked: arrayEntries.length } } + : { errorMessage: joinErrs(digestErrs) } + ) + ); + + // === resources-size-required === + const sizeErrs: string[] = []; + for (const { e, i, arr } of arrayEntries) { + for (const r of arr) { + if ( + typeof r.size !== 'number' || + !Number.isInteger(r.size) || + r.size < 0 + ) { + sizeErrs.push( + `${entryLabel(e, i)}: ${String(r.uri)} size=${JSON.stringify(r.size)} is not a non-negative integer` + ); + } + } + } + checks.push( + skillsCheck( + 'sep-2640-resources-size-required', + "Each entry MUST carry size: the length in bytes of the file's raw content — the same bytes the digest covers.", + sizeErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + sizeErrs.length === 0 + ? { details: { entriesChecked: arrayEntries.length } } + : { errorMessage: joinErrs(sizeErrs) } + ) + ); + + // === limit-resources-per-skill (SHOULD NOT exceed) === + const overCount = arrayEntries + .filter(({ arr }) => arr.length > MAX_RESOURCES_PER_SKILL) + .map(({ e, i, arr }) => `${entryLabel(e, i)}: ${arr.length} entries`); + checks.push( + skillsCheck( + 'sep-2640-limit-resources-per-skill', + 'Servers SHOULD NOT serve a skill exceeding 512 resource entries, counted over the entries of the skill resources, SKILL.md included.', + overCount.length === 0 ? 'SUCCESS' : 'WARNING', + overCount.length === 0 + ? { + details: { + maxObserved: Math.max( + ...arrayEntries.map(({ arr }) => arr.length) + ), + limit: MAX_RESOURCES_PER_SKILL + } + } + : { errorMessage: joinErrs(overCount) } + ) + ); + + // === limit-total-size (SHOULD NOT exceed) === + const sums = arrayEntries.map(({ e, i, arr }) => ({ + label: entryLabel(e, i), + total: arr.reduce( + (acc, r) => acc + (typeof r.size === 'number' ? r.size : 0), + 0 + ) + })); + const overSize = sums + .filter((s) => s.total > MAX_TOTAL_SIZE_PER_SKILL) + .map((s) => `${s.label}: ${s.total} bytes`); + checks.push( + skillsCheck( + 'sep-2640-limit-total-size', + 'Servers SHOULD NOT serve a skill whose total file size exceeds 16 MiB (16,777,216 bytes), summed over the skill resources.', + overSize.length === 0 ? 'SUCCESS' : 'WARNING', + overSize.length === 0 + ? { + details: { + maxObservedBytes: Math.max(...sums.map((s) => s.total)), + limit: MAX_TOTAL_SIZE_PER_SKILL + } + } + : { errorMessage: joinErrs(overSize) } + ) + ); + } + + // === metadata-reserved-prefix === + // This extension currently defines no keys under the reserved prefix, so a + // server publishing one is squatting on a namespace reserved for MCP. + const reservedErrs: string[] = []; + entries.forEach((e, i) => { + const fm = e.frontmatter as Record | undefined; + const meta = fm?.metadata; + if (!meta || typeof meta !== 'object' || Array.isArray(meta)) return; + for (const key of Object.keys(meta as Record)) { + if (key.startsWith(FRONTMATTER_RESERVED_PREFIX)) { + reservedErrs.push( + `${entryLabel(e, i)}: frontmatter.metadata["${key}"]` + ); + } + } + }); + checks.push( + skillsCheck( + 'sep-2640-metadata-reserved-prefix', + 'Within the frontmatter metadata object, keys prefixed with io.modelcontextprotocol/ are reserved for metadata defined by MCP extensions. This extension currently defines no such keys.', + reservedErrs.length === 0 ? 'SUCCESS' : 'WARNING', + reservedErrs.length === 0 + ? { details: { entryCount: entries.length } } + : { + errorMessage: `Keys under the reserved prefix, which this extension does not currently define: ${joinErrs(reservedErrs)}` + } + ) + ); + + // === name-naming-rules === + // The name is recoverable from the URI alone, so this is checkable without + // fetching anything. + const badNames: string[] = []; + entries.forEach((e, i) => { + if (typeof e.uri !== 'string') return; + const name = skillNameFromManifestUri(e.uri); + if (name !== undefined && !SKILL_NAME_PATTERN.test(name)) { + badNames.push(`${entryLabel(e, i)}: name "${name}"`); + } + }); + checks.push( + skillsCheck( + 'sep-2640-name-naming-rules', + "The final segment, being the skill name, MUST satisfy the Agent Skills specification's naming rules (1-64 characters, lowercase alphanumeric and hyphens).", + badNames.length === 0 ? 'SUCCESS' : 'FAILURE', + badNames.length === 0 + ? { details: { entryCount: entries.length } } + : { errorMessage: joinErrs(badNames) } + ) + ); + + // === authority-reg-name (SHOULD) === + const badAuthority: string[] = []; + entries.forEach((e, i) => { + if (typeof e.uri !== 'string') return; + const schemeEnd = e.uri.indexOf('://'); + if (schemeEnd < 0) return; + const segments = e.uri + .slice(schemeEnd + 3) + .split('/') + .filter((x) => x.length > 0); + const authority = segments[0]; + if (authority !== undefined && !REG_NAME_PATTERN.test(authority)) { + badAuthority.push(`${entryLabel(e, i)}: authority "${authority}"`); + } + }); + checks.push( + skillsCheck( + 'sep-2640-authority-reg-name', + 'The first segment occupies the authority component and SHOULD be a valid reg-name per RFC 3986.', + badAuthority.length === 0 ? 'SUCCESS' : 'WARNING', + badAuthority.length === 0 + ? { details: { entryCount: entries.length } } + : { errorMessage: joinErrs(badAuthority) } + ) + ); + + // === names-should-be-unique (SHOULD) === + // A collision is explicitly permitted — two skills at different paths may + // share a final segment — so this is a WARNING that tells a host operator the + // listing will need disambiguating, not a failure. + const byName = new Map(); + entries.forEach((e) => { + if (typeof e.uri !== 'string') return; + const n = skillNameFromManifestUri(e.uri); + if (n === undefined) return; + byName.set(n, [...(byName.get(n) ?? []), e.uri]); + }); + const collisions = [...byName.entries()] + .filter(([, uris]) => uris.length > 1) + .map(([n, uris]) => `"${n}" served at ${uris.join(' and ')}`); + checks.push( + skillsCheck( + 'sep-2640-names-should-be-unique', + "Within a server's listing, names SHOULD be unique, but they are not guaranteed to be.", + collisions.length === 0 ? 'SUCCESS' : 'WARNING', + collisions.length === 0 + ? { details: { distinctNames: byName.size } } + : { + errorMessage: `Names collide within one listing, so hosts MUST disambiguate them: ${joinErrs(collisions)}` + } + ) + ); + + return checks; +} + +/** + * Fetch one listed `SKILL.md` and check it against the entry that advertised + * it. This is the server-side half of the host's frontmatter-comparison MUST: + * if the entry's `frontmatter` does not match the file, no conforming host can + * load the skill. + */ +async function readbackChecks( + conn: Parameters[0], + entries: SkillEntry[] +): Promise { + const checks: ConformanceCheck[] = []; + const sample = entries.find( + (e) => + typeof e.uri === 'string' && e.uri.endsWith(`/${SKILL_MANIFEST_FILENAME}`) + ); + const uri = sample?.uri as string | undefined; + + if (!uri) { + const reason = `No listed entry has a uri ending in /${SKILL_MANIFEST_FILENAME}, so no SKILL.md can be read back.`; + for (const id of READBACK_IDS) { + checks.push(skillsCheck(id, reason, 'SKIPPED', { errorMessage: reason })); + } + return checks; + } + + let body: Awaited>; + try { + body = await readResourceText(conn, uri); + } catch (e) { + const reason = `resources/read on ${uri} failed: ${e instanceof Error ? e.message : String(e)}`; + checks.push( + skillsCheck( + 'sep-2640-skillmd-required', + 'Every skill MUST contain a SKILL.md file at its root.', + 'FAILURE', + { errorMessage: reason } + ) + ); + for (const id of [ + 'sep-2640-skillmd-frontmatter', + 'sep-2640-entry-frontmatter-identical' + ]) { + checks.push( + skillsCheck(id, 'SKILL.md is unreadable.', 'SKIPPED', { + errorMessage: reason + }) + ); + } + return checks; + } + + if (!body) { + const reason = `resources/read on ${uri} returned no text content, so the listed SKILL.md is not retrievable.`; + checks.push( + skillsCheck( + 'sep-2640-skillmd-required', + 'Every skill MUST contain a SKILL.md file at its root.', + 'FAILURE', + { errorMessage: reason } + ) + ); + for (const id of [ + 'sep-2640-skillmd-frontmatter', + 'sep-2640-entry-frontmatter-identical' + ]) { + checks.push( + skillsCheck(id, 'No SKILL.md content to inspect.', 'SKIPPED', { + errorMessage: reason + }) + ); + } + return checks; + } + + checks.push( + skillsCheck( + 'sep-2640-skillmd-required', + 'Every skill MUST contain a SKILL.md file at its root.', + 'SUCCESS', + { details: { uri, bytes: body.text.length } } + ) + ); + + // === skillmd-frontmatter === + const fm = parseFrontmatter(body.text); + const fmErrs: string[] = []; + if (!fm) { + fmErrs.push('SKILL.md has no leading --- delimited YAML frontmatter block'); + } else { + if (typeof fm.name !== 'string' || fm.name.length === 0) { + fmErrs.push('frontmatter has no non-empty name'); + } + if (typeof fm.description !== 'string' || fm.description.length === 0) { + fmErrs.push('frontmatter has no non-empty description'); + } + } + checks.push( + skillsCheck( + 'sep-2640-skillmd-frontmatter', + 'SKILL.md MUST begin with YAML frontmatter containing at minimum the name and description fields.', + fmErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + fmErrs.length === 0 + ? { details: { uri } } + : { errorMessage: `${uri}: ${fmErrs.join('; ')}` } + ) + ); + + // === entry-frontmatter-identical === + const declared = sample?.frontmatter as Record | undefined; + if ( + !fm || + !declared || + typeof declared !== 'object' || + Array.isArray(declared) + ) { + const reason = + 'Either the file has no parseable frontmatter or the entry carries no frontmatter object, so the two cannot be compared.'; + checks.push( + skillsCheck( + 'sep-2640-entry-frontmatter-identical', + 'The frontmatter object MUST be identical in content to the frontmatter of the SKILL.md it describes.', + 'SKIPPED', + { errorMessage: reason } + ) + ); + return checks; + } + + const diffs: string[] = []; + const keys = new Set([...Object.keys(fm), ...Object.keys(declared)]); + for (const k of keys) { + const a = JSON.stringify(fm[k] ?? null); + const b = JSON.stringify(declared[k] ?? null); + if (a !== b) diffs.push(`${k}: file=${a} entry=${b}`); + } + checks.push( + skillsCheck( + 'sep-2640-entry-frontmatter-identical', + 'The frontmatter object MUST be identical in content to the frontmatter of the SKILL.md it describes.', + diffs.length === 0 ? 'SUCCESS' : 'FAILURE', + diffs.length === 0 + ? { details: { uri, fields: [...keys] } } + : { + errorMessage: `${uri}: entry frontmatter differs from the file's: ${joinErrs(diffs)}` + } + ) + ); + + return checks; +} + +/** Exercise `skills/get` against a real entry and against an unknown URI. */ +async function getChecks( + conn: Parameters[0], + entries: SkillEntry[] +): Promise { + const checks: ConformanceCheck[] = []; + const sample = entries.find((e) => typeof e.uri === 'string'); + const sampleUri = sample?.uri as string | undefined; + + if (!sampleUri) { + const reason = + 'No listed entry carries a uri, so skills/get cannot be exercised against a known skill.'; + for (const id of GET_IDS) { + checks.push(skillsCheck(id, reason, 'SKIPPED', { errorMessage: reason })); + } + return checks; + } + + const got = await skillsGet(conn, sampleUri); + if ('error' in got) { + const reason = `${SKILLS_GET_METHOD} failed for a skill the server itself listed (${sampleUri}) with code ${got.error.code}: ${got.error.message}.`; + checks.push( + skillsCheck( + 'sep-2640-capability-commits-to-methods', + 'Declaring the extension commits the server to skills/list and skills/get.', + 'FAILURE', + { errorMessage: reason } + ), + skillsCheck( + 'sep-2640-skills-get-implemented', + 'A server declaring the extension MUST also implement the skills/get method, which returns the entry for a single skill named by its URI.', + 'FAILURE', + { errorMessage: reason } + ) + ); + for (const id of [ + 'sep-2640-skills-get-entry-shape', + 'sep-2640-skills-get-no-cursor', + 'sep-2640-skills-get-unknown-uri-invalid-params' + ]) { + checks.push( + skillsCheck(id, 'skills/get is unavailable.', 'SKIPPED', { + errorMessage: reason + }) + ); + } + return checks; + } + + checks.push( + skillsCheck( + 'sep-2640-capability-commits-to-methods', + 'Declaring the extension itself commits the server to skills/list and skills/get.', + 'SUCCESS', + { details: { methods: [SKILLS_LIST_METHOD, SKILLS_GET_METHOD] } } + ), + skillsCheck( + 'sep-2640-skills-get-implemented', + 'A server declaring the extension MUST also implement the skills/get method, which returns the entry for a single skill named by its URI.', + 'SUCCESS', + { details: { probedUri: sampleUri } } + ) + ); + + // === skills-get-entry-shape === + const skill = got.result.skill as SkillEntry | undefined; + const shapeErrs: string[] = []; + if (!skill || typeof skill !== 'object' || Array.isArray(skill)) { + shapeErrs.push('result.skill is missing or not an object'); + } else { + if (skill.uri !== sampleUri) { + shapeErrs.push( + `result.skill.uri=${JSON.stringify(skill.uri)} does not echo the requested uri ${sampleUri}` + ); + } + if ( + !skill.frontmatter || + typeof skill.frontmatter !== 'object' || + Array.isArray(skill.frontmatter) + ) { + shapeErrs.push('result.skill.frontmatter is missing or not an object'); + } + if (!isDynamicResources(skill) && resourcesArray(skill) === undefined) { + shapeErrs.push( + `result.skill.resources=${JSON.stringify(skill.resources)} is neither an array nor "${RESOURCES_DYNAMIC}"` + ); + } + } + checks.push( + skillsCheck( + 'sep-2640-skills-get-entry-shape', + 'The skill object is a skill entry, identical in shape and meaning to an entry of skills/list — the same uri, frontmatter, and resources fields, under the same rules.', + shapeErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + shapeErrs.length === 0 + ? { details: { probedUri: sampleUri } } + : { errorMessage: joinErrs(shapeErrs) } + ) + ); + + // === skills-get-no-cursor === + const hasCursor = got.result.nextCursor !== undefined; + checks.push( + skillsCheck( + 'sep-2640-skills-get-no-cursor', + 'The result carries no pagination cursor: a single entry is not a list.', + hasCursor ? 'FAILURE' : 'SUCCESS', + hasCursor + ? { + errorMessage: `skills/get returned nextCursor=${JSON.stringify(got.result.nextCursor)}; a single entry is not a list.` + } + : { details: { probedUri: sampleUri } } + ) + ); + + // === skills-get-unknown-uri-invalid-params === + const unknown = await skillsGet(conn, UNKNOWN_SKILL_URI); + if ('error' in unknown) { + const ok = unknown.error.code === JSONRPC_INVALID_PARAMS; + checks.push( + skillsCheck( + 'sep-2640-skills-get-unknown-uri-invalid-params', + 'If the URI does not identify a skill the server serves, the server MUST return error -32602 (Invalid params).', + ok ? 'SUCCESS' : 'FAILURE', + ok + ? { + details: { + probedUri: UNKNOWN_SKILL_URI, + code: unknown.error.code + } + } + : { + errorMessage: `skills/get on an unserved URI returned code ${unknown.error.code} (${unknown.error.message}); expected ${JSONRPC_INVALID_PARAMS}.` + } + ) + ); + } else { + checks.push( + skillsCheck( + 'sep-2640-skills-get-unknown-uri-invalid-params', + 'If the URI does not identify a skill the server serves, the server MUST return error -32602 (Invalid params).', + 'FAILURE', + { + errorMessage: `skills/get returned a successful result for ${UNKNOWN_SKILL_URI}, which no conformant server should serve; expected error ${JSONRPC_INVALID_PARAMS}.` + } + ) + ); + } + + return checks; +} diff --git a/src/scenarios/server/skills/helpers.ts b/src/scenarios/server/skills/helpers.ts new file mode 100644 index 00000000..d682b32c --- /dev/null +++ b/src/scenarios/server/skills/helpers.ts @@ -0,0 +1,412 @@ +/** + * Shared helpers for the SEP-2640 (Skills extension) server-conformance + * scenarios under this directory. + * + * The scenarios treat the server-under-test as an arbitrary SEP-2640 server: + * capability is read from `server/discover` (never inferred from an error), and + * every skill is discovered dynamically through `skills/list` — no + * fixture-specific URI is hardcoded, so the checks pass against any conformant + * server, not just one implementation's fixture. + * + * Extracted against the 2026-08-21 revision of the SEP (branch + * `sep/skills-extension`), which replaced the `skill://index.json` well-known + * resource with the `skills/list` and `skills/get` methods, deferred archive + * distribution, and reshaped the skill entry to `{uri, frontmatter, resources}`. + */ + +import type { + CheckStatus, + ConformanceCheck, + SpecReference +} from '../../../types'; +import type { Connection } from '../../../connection'; +import { JsonRpcError } from '../../../connection'; +import { parse as parseYaml } from 'yaml'; + +export const SKILLS_EXTENSION_ID = 'io.modelcontextprotocol/skills'; +export const SKILL_URI_SCHEME = 'skill://'; +export const SKILL_MANIFEST_FILENAME = 'SKILL.md'; +export const SKILLS_META_PREFIX = 'io.modelcontextprotocol.skills/'; + +/** Reserved prefix for MCP-defined keys inside frontmatter `metadata`. */ +export const FRONTMATTER_RESERVED_PREFIX = 'io.modelcontextprotocol/'; + +export const SKILLS_LIST_METHOD = 'skills/list'; +export const SKILLS_GET_METHOD = 'skills/get'; +export const DIRECTORY_READ_METHOD = 'resources/directory/read'; + +/** `sha256:{hex}` with exactly 64 lowercase hex characters (SEP-2640). */ +export const SKILL_DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/; + +/** The `"dynamic"` sentinel a server sets in place of a `resources` array. */ +export const RESOURCES_DYNAMIC = 'dynamic'; + +/** Per-skill limits fixed by the SEP (§Limits). */ +export const MAX_RESOURCES_PER_SKILL = 512; +export const MAX_TOTAL_SIZE_PER_SKILL = 16 * 1024 * 1024; // 16 MiB + +export const JSONRPC_METHOD_NOT_FOUND = -32601; +export const JSONRPC_INVALID_PARAMS = -32602; + +export const SEP_2640_REF: SpecReference = { + id: 'SEP-2640', + url: 'https://modelcontextprotocol.io/seps/2640-skills-extension#specification' +}; + +/** A `resources/list` / directory-read entry (only the fields we inspect). */ +export interface SkillResource { + uri: string; + name?: string; + description?: string; + mimeType?: string; + _meta?: Record; +} + +/** One `{uri, digest, size}` triple of a skill entry's `resources` array. */ +export interface SkillResourceEntry { + uri?: unknown; + digest?: unknown; + size?: unknown; + [key: string]: unknown; +} + +/** + * One skill entry, as returned by `skills/list` (in `skills[]`) and by + * `skills/get` (as `skill`). The two are identical in shape and meaning. + */ +export interface SkillEntry { + uri?: unknown; + frontmatter?: unknown; + /** An array of `{uri, digest, size}`, or the string `"dynamic"`. */ + resources?: unknown; + [key: string]: unknown; +} + +export interface SkillsListResult { + skills?: unknown; + nextCursor?: string; + ttlMs?: unknown; + cacheScope?: unknown; + [key: string]: unknown; +} + +export interface SkillsGetResult { + skill?: unknown; + nextCursor?: unknown; + [key: string]: unknown; +} + +/** First text content of a `resources/read`, with its mimeType and `_meta`. */ +export interface ResourceText { + text: string; + mimeType?: string; + meta?: Record; +} + +/** + * Build a check carrying the SEP-2640 reference. Per AGENTS.md the same `id` + * flips `status` + `errorMessage` between SUCCESS and FAILURE rather than + * branching into distinct slugs. + */ +export function skillsCheck( + id: string, + description: string, + status: CheckStatus, + extras: Partial = {} +): ConformanceCheck { + return { + id, + name: id, + description, + status, + timestamp: new Date().toISOString(), + specReferences: [SEP_2640_REF], + ...extras + }; +} + +/** + * The skills extension object declared under `capabilities.extensions`, or + * `undefined` when the server did not declare it. Reads the declared capability + * from `server/discover` (mirrors `tasks/capability.ts`) — an undeclared + * optional extension is a SKIP, never inferred from a `-32601`. + */ +export async function skillsCapability( + conn: Connection +): Promise | undefined> { + const discovered = await conn.discover(); + const caps = (discovered.capabilities as Record) ?? {}; + const extensions = caps.extensions as Record | undefined; + const skills = extensions?.[SKILLS_EXTENSION_ID]; + return skills && typeof skills === 'object' + ? (skills as Record) + : undefined; +} + +/** + * Whether the declared extension object nests its settings inline, as both + * SEP-2133 and SEP-2640 require, rather than wrapping them in an envelope. + * + * SEP-2133 (status Final) defines `extensions` as "a map of extension + * identifiers to per-extension settings objects", and SEP-2640's capability + * block matches: `{"io.modelcontextprotocol/skills": {"directoryRead": true}}`. + * Neither SEP defines an envelope, and neither has a slot for `id`, + * `specVersion` or `stability`. + * + * An earlier revision of this helper accepted a `config` envelope alongside the + * inline form, on the belief that the two SEPs disagreed. Re-reading SEP-2133 + * at Final status, they do not. The envelope is a non-conformant shape emitted + * by at least one SDK, so it is reported rather than silently accepted. + */ +export function settingsAreInline(skills: Record): { + inline: boolean; + envelopeKeys: string[]; +} { + const envelopeKeys = ['config', 'specVersion', 'stability', 'id'].filter( + (k) => k in skills + ); + return { inline: envelopeKeys.length === 0, envelopeKeys }; +} + +/** + * Whether the skills extension declares `directoryRead: true`. + * + * Reads only the inline location the SEPs specify. A server that buries the + * flag inside an envelope fails `sep-2640-capability-declaration-inline` and is + * treated here as not having declared the optional method, which is the + * conservative reading: a client that follows the spec would not see the flag + * either, and "clients MUST NOT call resources/directory/read against a server + * that has not declared directoryRead: true". + */ +export function directoryReadDeclared( + skills: Record +): boolean { + return skills.directoryRead === true; +} + +/** Everything from `resources/list`, paginating until `nextCursor` clears. */ +export async function listAllResources( + conn: Connection +): Promise { + const out: SkillResource[] = []; + let cursor: string | undefined; + do { + const page = await conn.request<{ + resources?: SkillResource[]; + nextCursor?: string; + }>('resources/list', cursor ? { cursor } : undefined); + out.push(...(page.resources ?? [])); + cursor = page.nextCursor; + } while (cursor); + return out; +} + +/** One page of `skills/list`, kept separate so pagination can be inspected. */ +export interface SkillsListPage { + result: SkillsListResult; + entries: SkillEntry[]; +} + +/** + * Call `skills/list` once, optionally with a cursor. Returns a `JsonRpcError` + * rather than throwing so a scenario can distinguish "method not implemented" + * (a FAILURE, since the method is mandatory for a declaring server) from a + * transport fault. + */ +export async function skillsListPage( + conn: Connection, + cursor?: string +): Promise { + try { + const result = await conn.request( + SKILLS_LIST_METHOD, + cursor ? { cursor } : {} + ); + const entries = Array.isArray(result.skills) + ? (result.skills as SkillEntry[]) + : []; + return { result, entries }; + } catch (e) { + if (e instanceof JsonRpcError) return { error: e }; + throw e; + } +} + +/** + * Every entry from `skills/list`, following `nextCursor`. `pages` is retained + * so the atomic-entry and pagination checks can reason about page boundaries. + * Bounded to avoid looping forever against a server that returns a constant + * cursor. + */ +export async function skillsListAll( + conn: Connection, + maxPages = 50 +): Promise< + | { entries: SkillEntry[]; pages: SkillsListPage[]; truncated: boolean } + | { error: JsonRpcError } +> { + const pages: SkillsListPage[] = []; + const entries: SkillEntry[] = []; + let cursor: string | undefined; + const seenCursors = new Set(); + + for (let i = 0; i < maxPages; i++) { + const page = await skillsListPage(conn, cursor); + if ('error' in page) return page; + pages.push(page); + entries.push(...page.entries); + const next = page.result.nextCursor; + if (typeof next !== 'string' || next.length === 0) { + return { entries, pages, truncated: false }; + } + if (seenCursors.has(next)) { + // A repeating cursor is a server bug; stop rather than spin. + return { entries, pages, truncated: true }; + } + seenCursors.add(next); + cursor = next; + } + return { entries, pages, truncated: true }; +} + +/** Call `skills/get` for one skill URI. */ +export async function skillsGet( + conn: Connection, + uri: string +): Promise<{ result: SkillsGetResult } | { error: JsonRpcError }> { + try { + const result = await conn.request(SKILLS_GET_METHOD, { + uri + }); + return { result }; + } catch (e) { + if (e instanceof JsonRpcError) return { error: e }; + throw e; + } +} + +/** Read a resource's first text content plus its mimeType and `_meta`. */ +export async function readResourceText( + conn: Connection, + uri: string +): Promise { + const res = await conn.request<{ + contents?: Array<{ + text?: string; + mimeType?: string; + _meta?: Record; + }>; + }>('resources/read', { uri }); + const entry = (res.contents ?? []).find((c) => typeof c.text === 'string'); + if (!entry || typeof entry.text !== 'string') return undefined; + return { text: entry.text, mimeType: entry.mimeType, meta: entry._meta }; +} + +/** + * The skill name recoverable from a `SKILL.md` resource URI: the final segment + * of ``, i.e. the last path segment before the trailing + * `SKILL.md`. Returns `undefined` when the URI does not end in `/SKILL.md`. + * + * Scheme-agnostic by design: the SEP is explicit that "no scheme is + * privileged" and that the structural constraints "apply regardless of + * scheme", so a server serving skills under `github://` is judged by the same + * path rule as one using `skill://`. + * + * skill://org/team/deploy/SKILL.md -> "deploy" + * github://acme/repo/lint/SKILL.md -> "lint" + */ +export function skillNameFromManifestUri(uri: string): string | undefined { + const schemeEnd = uri.indexOf('://'); + if (schemeEnd < 0) return undefined; + const parts = uri + .slice(schemeEnd + 3) + .split('/') + .filter((p) => p.length > 0); + if (parts.length < 2) return undefined; + if (parts[parts.length - 1] !== SKILL_MANIFEST_FILENAME) return undefined; + return parts[parts.length - 2]; +} + +/** The skill's root directory URI: its `SKILL.md` URI with the file removed. */ +export function skillRootFromManifestUri(uri: string): string | undefined { + if (!uri.endsWith(`/${SKILL_MANIFEST_FILENAME}`)) return undefined; + return uri.slice(0, -`/${SKILL_MANIFEST_FILENAME}`.length); +} + +/** + * Extract and parse the YAML frontmatter block at the head of a `SKILL.md`. + * Returns `undefined` when there is no leading `---` delimited block or it does + * not parse to an object. + */ +export function parseFrontmatter( + markdown: string +): Record | undefined { + // Tolerate a leading UTF-8 BOM before the opening `---` fence. + const body = markdown.charCodeAt(0) === 0xfeff ? markdown.slice(1) : markdown; + const match = body.match(/^---\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/); + if (!match) return undefined; + try { + const parsed = parseYaml(match[1]) as unknown; + return parsed && typeof parsed === 'object' + ? (parsed as Record) + : undefined; + } catch { + return undefined; + } +} + +/** True when the entry's `resources` is the `"dynamic"` sentinel. */ +export function isDynamicResources(entry: SkillEntry): boolean { + return entry.resources === RESOURCES_DYNAMIC; +} + +/** + * The entry's `resources` array, or `undefined` when it is `"dynamic"`, absent, + * or any other value. Callers distinguish those cases via `isDynamicResources`. + */ +export function resourcesArray( + entry: SkillEntry +): SkillResourceEntry[] | undefined { + return Array.isArray(entry.resources) + ? (entry.resources as SkillResourceEntry[]) + : undefined; +} + +/** A short, stable label for an entry, for error messages. */ +export function entryLabel(entry: SkillEntry, i: number): string { + return typeof entry.uri === 'string' ? entry.uri : `skills[${i}]`; +} + +/** + * Every child of a directory, following `nextCursor` until it clears. + * + * SEP-2640 says directory-read pagination mirrors `resources/list`, so a + * conformant server MAY split a directory across pages. Reading only the + * first page makes "no subdirectory here" indistinguishable from "the + * subdirectory is on page two". + */ +export async function directoryReadAll( + conn: Connection, + uri: string, + maxPages = 50 +): Promise<{ resources: SkillResource[]; pages: number; truncated: boolean }> { + const resources: SkillResource[] = []; + const seen = new Set(); + let cursor: string | undefined; + + for (let i = 0; i < maxPages; i++) { + const page = await conn.request<{ + resources?: SkillResource[]; + nextCursor?: string; + }>('resources/directory/read', cursor ? { uri, cursor } : { uri }); + resources.push(...(page.resources ?? [])); + const next = page.nextCursor; + if (typeof next !== 'string' || next.length === 0) { + return { resources, pages: i + 1, truncated: false }; + } + if (seen.has(next)) return { resources, pages: i + 1, truncated: true }; + seen.add(next); + cursor = next; + } + return { resources, pages: maxPages, truncated: true }; +} diff --git a/src/scenarios/server/skills/manifest.ts b/src/scenarios/server/skills/manifest.ts new file mode 100644 index 00000000..42b19229 --- /dev/null +++ b/src/scenarios/server/skills/manifest.ts @@ -0,0 +1,358 @@ +/** + * SEP-2640 Skills extension — the `SKILL.md` manifest resource. + * + * One scenario, many checks (per AGENTS.md "fewer scenarios, more checks"). + * Each check's verbatim spec quote lives next to its check ID in + * src/seps/sep-2640.yaml. + * + * Discovery is dynamic and brand-neutral: the scenario finds a `skill-md` + * skill's `SKILL.md` resource from `resources/list` (preferred — it carries the + * Resource `name`/`description` metadata) or falls back to the first `skill-md` + * entry in `skill://index.json`, hardcoding no fixture skill. Undeclared + * extension SKIPs; a declared extension with no discoverable `SKILL.md` reports + * the missing prerequisite via untestableCheck (issue #248), never a silent + * green. + */ + +import { ClientScenario, ConformanceCheck } from '../../../types'; +import { JsonRpcError, type RunContext } from '../../../connection'; +import { untestableCheck } from '../../untestable'; +import { + SKILLS_EXTENSION_ID, + SKILLS_META_PREFIX, + SEP_2640_REF, + type SkillResource, + skillsCapability, + skillsCheck, + listAllResources, + skillsListAll, + readResourceText, + skillNameFromManifestUri, + parseFrontmatter +} from './helpers'; + +const MIMETYPE_ID = 'sep-2640-skillmd-mimetype'; +const METADATA_NAME_ID = 'sep-2640-skillmd-metadata-name'; +const METADATA_DESCRIPTION_ID = 'sep-2640-skillmd-metadata-description'; +const FINAL_SEGMENT_ID = 'sep-2640-final-segment-equals-name'; +const META_PREFIX_ID = 'sep-2640-meta-prefix'; + +const MARKDOWN_MIME = 'text/markdown'; + +/** A SKILL.md resource URI is skill:///SKILL.md. */ +function isManifestUri(uri: string): boolean { + return skillNameFromManifestUri(uri) !== undefined; +} + +/** A `_meta` key that already carries a reverse-domain namespace (`vendor.tld/…`). */ +function isNamespacedMetaKey(key: string): boolean { + return /^[a-z0-9-]+(\.[a-z0-9-]+)+\//i.test(key); +} + +export class SkillsManifestScenario implements ClientScenario { + name = 'sep-2640-skills-manifest'; + readonly source = { extensionId: SKILLS_EXTENSION_ID } as const; + description = `SEP-2640 Skills extension: the \`SKILL.md\` manifest resource. + +**Resource**: \`skill:///SKILL.md\` (read via \`resources/read\`) + +**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-2640.yaml): + +- \`sep-2640-skillmd-mimetype\` — the SKILL.md resource \`mimeType\` SHOULD be \`text/markdown\` +- \`sep-2640-skillmd-metadata-name\` — the resource \`name\` SHOULD be the frontmatter \`name\` +- \`sep-2640-skillmd-metadata-description\` — the resource \`description\` SHOULD be the frontmatter \`description\` +- \`sep-2640-final-segment-equals-name\` — the final \`\` segment MUST equal the frontmatter \`name\` +- \`sep-2640-meta-prefix\` — un-namespaced skill \`_meta\` keys SHOULD use the \`io.modelcontextprotocol.skills/\` prefix + +**Discovery is dynamic**: the scenario picks the first \`skill-md\` skill it finds. Undeclared extension SKIPs; a declared extension with no discoverable SKILL.md reports the missing prerequisite (not a silent skip).`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect(); + try { + const skills = await skillsCapability(conn); + const allIds = [ + MIMETYPE_ID, + METADATA_NAME_ID, + METADATA_DESCRIPTION_ID, + FINAL_SEGMENT_ID, + META_PREFIX_ID + ]; + if (!skills) { + const reason = + 'Server did not declare the io.modelcontextprotocol/skills extension; SKILL.md checks not applicable.'; + return allIds.map((id) => + skillsCheck(id, reason, 'SKIPPED', { errorMessage: reason }) + ); + } + + // === Dynamic discovery: resources/list first (carries Resource + // metadata), then skill://index.json. === + const resources = await listAllResources(conn); + const manifestResource: SkillResource | undefined = resources.find((r) => + isManifestUri(r.uri) + ); + let manifestUri = manifestResource?.uri; + if (!manifestUri) { + const listed = await skillsListAll(conn); + if (!('error' in listed)) { + const entry = listed.entries.find( + (e) => typeof e.uri === 'string' && isManifestUri(e.uri) + ); + manifestUri = entry?.uri as string | undefined; + } + } + + if (!manifestUri) { + const reason = + 'no skill:///SKILL.md resource found via resources/list or skill://index.json'; + return [ + untestableCheck( + MIMETYPE_ID, + MIMETYPE_ID, + 'SKILL.md resource mimeType SHOULD be text/markdown.', + reason, + [SEP_2640_REF], + 'WARNING' + ), + untestableCheck( + METADATA_NAME_ID, + METADATA_NAME_ID, + 'SKILL.md resource name SHOULD match the frontmatter name.', + reason, + [SEP_2640_REF], + 'WARNING' + ), + untestableCheck( + METADATA_DESCRIPTION_ID, + METADATA_DESCRIPTION_ID, + 'SKILL.md resource description SHOULD match the frontmatter description.', + reason, + [SEP_2640_REF], + 'WARNING' + ), + untestableCheck( + FINAL_SEGMENT_ID, + FINAL_SEGMENT_ID, + 'The final segment MUST equal the frontmatter name.', + reason, + [SEP_2640_REF], + 'FAILURE' + ), + untestableCheck( + META_PREFIX_ID, + META_PREFIX_ID, + 'Skill _meta keys SHOULD use the io.modelcontextprotocol.skills/ prefix.', + reason, + [SEP_2640_REF], + 'WARNING' + ) + ]; + } + + const checks: ConformanceCheck[] = []; + + // Read the manifest content (for mimeType, frontmatter, and _meta). + let content: + | { text: string; mimeType?: string; meta?: Record } + | undefined; + let readError: string | undefined; + try { + content = await readResourceText(conn, manifestUri); + if (!content) readError = 'resources/read returned no text content'; + } catch (e) { + readError = + e instanceof JsonRpcError + ? `resources/read failed: code ${e.code}: ${e.message}` + : e instanceof Error + ? e.message + : String(e); + } + + // === skillmd-mimetype (SHOULD) === + // Prefer the read content's mimeType; fall back to the resources/list + // Resource metadata mimeType. + const mimeType = content?.mimeType ?? manifestResource?.mimeType; + if (mimeType === undefined) { + checks.push( + untestableCheck( + MIMETYPE_ID, + MIMETYPE_ID, + 'SKILL.md resource mimeType SHOULD be text/markdown.', + `no mimeType observable for ${manifestUri}${readError ? ` (${readError})` : ''}`, + [SEP_2640_REF], + 'WARNING' + ) + ); + } else { + checks.push( + skillsCheck( + MIMETYPE_ID, + 'SKILL.md resource mimeType SHOULD be text/markdown.', + mimeType === MARKDOWN_MIME ? 'SUCCESS' : 'WARNING', + mimeType === MARKDOWN_MIME + ? { details: { uri: manifestUri, mimeType } } + : { + errorMessage: `expected mimeType "${MARKDOWN_MIME}", got ${JSON.stringify(mimeType)}` + } + ) + ); + } + + // Parse the frontmatter once for the name/description/final-segment checks. + const frontmatter = content ? parseFrontmatter(content.text) : undefined; + const fmName = + typeof frontmatter?.name === 'string' ? frontmatter.name : undefined; + const fmDescription = + typeof frontmatter?.description === 'string' + ? frontmatter.description + : undefined; + + // === final-segment-equals-name (MUST) === + const uriName = skillNameFromManifestUri(manifestUri); + if (fmName === undefined || uriName === undefined) { + const missing = + fmName === undefined + ? `SKILL.md frontmatter has no string "name"${readError ? ` (${readError})` : ''}` + : `could not derive the skill name from URI ${manifestUri}`; + checks.push( + untestableCheck( + FINAL_SEGMENT_ID, + FINAL_SEGMENT_ID, + 'The final segment MUST equal the frontmatter name.', + missing, + [SEP_2640_REF], + 'FAILURE' + ) + ); + } else { + checks.push( + skillsCheck( + FINAL_SEGMENT_ID, + 'The final segment of the SKILL.md URI MUST equal the frontmatter name.', + uriName === fmName ? 'SUCCESS' : 'FAILURE', + uriName === fmName + ? { details: { uri: manifestUri, name: fmName } } + : { + errorMessage: `final path segment "${uriName}" != frontmatter name "${fmName}"` + } + ) + ); + } + + // === skillmd-metadata-name (SHOULD) — needs the Resource metadata === + if (!manifestResource) { + checks.push( + untestableCheck( + METADATA_NAME_ID, + METADATA_NAME_ID, + 'SKILL.md resource name SHOULD match the frontmatter name.', + `SKILL.md ${manifestUri} is not listed in resources/list, so its Resource name metadata is not observable`, + [SEP_2640_REF], + 'WARNING' + ) + ); + } else if (fmName === undefined) { + checks.push( + untestableCheck( + METADATA_NAME_ID, + METADATA_NAME_ID, + 'SKILL.md resource name SHOULD match the frontmatter name.', + `SKILL.md frontmatter has no string "name" to compare against${readError ? ` (${readError})` : ''}`, + [SEP_2640_REF], + 'WARNING' + ) + ); + } else { + checks.push( + skillsCheck( + METADATA_NAME_ID, + 'The SKILL.md resource name SHOULD be set from the frontmatter name.', + manifestResource.name === fmName ? 'SUCCESS' : 'WARNING', + manifestResource.name === fmName + ? { details: { name: fmName } } + : { + errorMessage: `resource name ${JSON.stringify(manifestResource.name)} != frontmatter name ${JSON.stringify(fmName)}` + } + ) + ); + } + + // === skillmd-metadata-description (SHOULD) — needs Resource metadata === + if (!manifestResource) { + checks.push( + untestableCheck( + METADATA_DESCRIPTION_ID, + METADATA_DESCRIPTION_ID, + 'SKILL.md resource description SHOULD match the frontmatter description.', + `SKILL.md ${manifestUri} is not listed in resources/list, so its Resource description metadata is not observable`, + [SEP_2640_REF], + 'WARNING' + ) + ); + } else if (fmDescription === undefined) { + checks.push( + untestableCheck( + METADATA_DESCRIPTION_ID, + METADATA_DESCRIPTION_ID, + 'SKILL.md resource description SHOULD match the frontmatter description.', + `SKILL.md frontmatter has no string "description" to compare against${readError ? ` (${readError})` : ''}`, + [SEP_2640_REF], + 'WARNING' + ) + ); + } else { + checks.push( + skillsCheck( + METADATA_DESCRIPTION_ID, + 'The SKILL.md resource description SHOULD be set from the frontmatter description.', + manifestResource.description === fmDescription + ? 'SUCCESS' + : 'WARNING', + manifestResource.description === fmDescription + ? { details: { description: fmDescription } } + : { + errorMessage: `resource description ${JSON.stringify(manifestResource.description)} != frontmatter description ${JSON.stringify(fmDescription)}` + } + ) + ); + } + + // === meta-prefix (SHOULD, conditional on _meta keys being present) === + // Union the _meta of the read content and the resources/list Resource. + const metaKeys = new Set([ + ...Object.keys(content?.meta ?? {}), + ...Object.keys(manifestResource?._meta ?? {}) + ]); + if (metaKeys.size === 0) { + checks.push( + skillsCheck( + META_PREFIX_ID, + 'When _meta keys are used for skill resources, they SHOULD use the io.modelcontextprotocol.skills/ reverse-domain prefix.', + 'SUCCESS', + { details: { note: 'skill resource exposes no _meta keys' } } + ) + ); + } else { + // Only bare (un-namespaced) keys are flagged: a key already carrying a + // reverse-domain namespace is the intended shape, whichever vendor. + const bareKeys = [...metaKeys].filter((k) => !isNamespacedMetaKey(k)); + checks.push( + skillsCheck( + META_PREFIX_ID, + 'When _meta keys are used for skill resources, they SHOULD use the io.modelcontextprotocol.skills/ reverse-domain prefix.', + bareKeys.length === 0 ? 'SUCCESS' : 'WARNING', + bareKeys.length === 0 + ? { details: { metaKeys: [...metaKeys] } } + : { + errorMessage: `un-namespaced skill _meta keys SHOULD use the ${SKILLS_META_PREFIX} prefix: ${bareKeys.join(', ')}` + } + ) + ); + } + + return checks; + } finally { + await conn.close(); + } + } +} diff --git a/src/seps/sep-2640.yaml b/src/seps/sep-2640.yaml new file mode 100644 index 00000000..65ce6ad8 --- /dev/null +++ b/src/seps/sep-2640.yaml @@ -0,0 +1,304 @@ +# spec_source: modelcontextprotocol/modelcontextprotocol@sep/skills-extension seps/2640-skills-extension.md +# extracted: 2026-08-28 +# +# provenance: re-extracted against the 2026-08-21 revision, which the core +# maintainers produced after their second review round and which went back up +# for CM vote on 2026-08-25. The prior extraction held at 556154c (2026-06-05) +# by an explicit decision to wait for mcpkit#780; that hold expired badly. +# The 08-21 revision: +# - removed the `skill://index.json` well-known resource entirely (it now +# appears nowhere in the SEP) and replaced it with the `skills/list` and +# `skills/get` methods, both mandatory for a declaring server; +# - deferred archive distribution to an appendix ("Appendix: Deferred +# Features"), retiring the `type` enum and every archive requirement; +# - reversed the nesting rule: a `SKILL.md` MAY now appear in a descendant +# directory, where the June text forbade it; +# - reshaped the entry to `{uri, frontmatter, resources}`, where `resources` +# is a complete `{uri, digest, size}` array or the string `"dynamic"`; +# - fixed per-skill limits of 512 resource entries and 16 MiB; +# - added host obligations around lazy retrieval, content-bound approval and +# frontmatter re-verification. +# Eight rows from the prior extraction were removed as no longer normative +# (server-expose-index, index-entry-type-enum, no-nested-skills, and the five +# archive rows) and four were reworded. Because the SEP is mid-vote, expect one +# more pass at the CM-stamped head. +# +# coverage: extracted against branch head a3e147ca2710 (2026-08-25), the commit +# the CM vote is running on. A keyword sweep of the source finds 97 sentences +# carrying an RFC 2119 term; this file declares 89 checks plus 7 excluded rows. +# The residual handful are restatements that fold into a declared row rather +# than standing alone. Of the 89 declared checks, 40 are exercised on the wire +# by the three server scenarios; the other 49 are host obligations (retrieval +# policy, approval binding, cache isolation, context handling) that a +# server-side harness cannot observe and that would need a host-side scenario +# set to test. +# +# deliberately not declared here: `resultType`, which appears in all three +# result examples in this SEP, is a base-protocol field on the common `Result` +# interface (schema/2026-07-28/schema.ts), not a SEP-2640 requirement. Servers +# MUST include it and clients treat an absent value as "complete". Its +# caching-hint obligations belong to sep-2549.yaml. Declaring it here would +# double-count a base-protocol rule against this extension. +# +# backing_scenarios: three server ClientScenarios under +# src/scenarios/server/skills/ emit the check IDs below (a row is "tested" +# once a scenario emits its check ID; see src/traceability/): +# enumeration.ts (sep-2640-skills-enumeration) — the capability rows, the +# sep-2640-skills-list-* rows, the entry-schema and resources rows, the two +# limit rows, and the sep-2640-skills-get-* rows. Replaces the retired +# index.ts, which tested `skill://index.json`. +# manifest.ts (sep-2640-skills-manifest) — sep-2640-skillmd-mimetype, +# sep-2640-skillmd-metadata-name, sep-2640-skillmd-metadata-description, +# sep-2640-final-segment-equals-name, sep-2640-meta-prefix. +# directory.ts (sep-2640-skills-directory) — +# sep-2640-capability-directory-read-flag and the +# sep-2640-directory-read-* rows. +# The remaining rows are host-internal or off-wire (host retrieval policy, +# digest and frontmatter verification, approval binding, cache isolation) and +# stay traceability-only for this server-scenario set. A host-side scenario +# set would be needed to exercise them. +sep: 2640 +spec_url: https://modelcontextprotocol.io/seps/2640-skills-extension#specification +requirements: + # === Skill Format === + - check: sep-2640-skillmd-required + text: 'Every skill MUST contain a `SKILL.md` file at its root.' + - check: sep-2640-skillmd-frontmatter + text: '`SKILL.md` MUST begin with YAML frontmatter containing at minimum the `name` and `description` fields as defined by the Agent Skills specification.' + + # === Resource Mapping === + - check: sep-2640-skill-uri-scheme + text: 'Each file within a skill directory is exposed as an MCP resource. Servers SHOULD use the `skill://` URI scheme, under which the resource URI has the form: `skill:///`' + - check: sep-2640-final-segment-equals-name + text: "The final segment of `` MUST equal the skill's `name` as declared in its `SKILL.md` frontmatter." + - check: sep-2640-name-naming-rules + text: "The final `` segment, being the skill `name`, MUST satisfy the Agent Skills specification's naming rules." + - check: sep-2640-authority-reg-name + text: 'The first `` segment occupies the authority component and SHOULD be a valid `reg-name` per RFC 3986; any other prefix segments SHOULD be valid URI path segments; no further constraints are imposed on them.' + # Reverses the retired sep-2640-no-nested-skills row: the June text said a + # SKILL.md MUST NOT appear in any descendant directory. + - check: sep-2640-nested-skills-permitted + text: 'A `SKILL.md` MAY appear in a descendant directory of a skill — skills can nest.' + - check: sep-2640-nested-skillmd-not-acted-on + text: "From the enclosing skill's perspective, a nested skill's directory and files are ordinary supporting files, and reading them is ordinary reading. A nested `SKILL.md` read this way is ordinary markdown: hosts MUST NOT act on its frontmatter." + - check: sep-2640-nested-publication-flat + text: "A nested skill is published like any other: through its own `skills/list` entry, or by explicit reference. The listing remains flat — an entry for a nested skill is an ordinary entry whose `uri` happens to share a path prefix with the enclosing skill's, and nothing in the listing marks nesting." + + # === Resource Metadata === + - check: sep-2640-skillmd-mimetype + text: 'For each `skill:///SKILL.md` resource: `mimeType` SHOULD be `text/markdown`.' + - check: sep-2640-skillmd-metadata-name + text: 'For each `skill:///SKILL.md` resource: `name` SHOULD be set from the `name` field of the `SKILL.md` YAML frontmatter. By the path constraint above, this will equal the final segment of ``.' + - check: sep-2640-skillmd-metadata-description + text: 'For each `skill:///SKILL.md` resource: `description` SHOULD be set from the `description` field of the `SKILL.md` YAML frontmatter.' + - check: sep-2640-meta-prefix + text: 'When `_meta` keys are used for skill resources, implementations SHOULD use the `io.modelcontextprotocol.skills/` reverse-domain prefix.' + + # === Capability Declaration === + # The inline-settings row is the check that would have caught panyam/mcpkit#1334. + # SEP-2133 (status Final) defines `extensions` as "a map of extension + # identifiers to per-extension settings objects" and defines no envelope; the + # SEP-2640 capability block matches it. + - check: sep-2640-capability-declaration-inline + text: 'Per SEP-2133 extension negotiation, servers declare support for this extension in their `initialize` response, mapping the extension identifier directly to its per-extension settings object: `{"capabilities": {"extensions": {"io.modelcontextprotocol/skills": {"directoryRead": true}}}}`.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#capability-declaration + - check: sep-2640-capability-commits-to-methods + text: 'Declaring the extension itself commits the server to `skills/list` and `skills/get`.' + - check: sep-2640-capability-empty-object + text: 'An empty object indicates support for the extension with no optional features.' + - check: sep-2640-capability-directory-read-flag + text: 'Clients MUST NOT call `resources/directory/read` against a server that has not declared `directoryRead: true`.' + + # === Discovery: enumeration via skills/list === + - check: sep-2640-skills-list-implemented + text: 'A server declaring the `io.modelcontextprotocol/skills` extension MUST implement the `skills/list` method, which returns the skills it serves. The result MAY be empty.' + - check: sep-2640-skills-list-pagination + text: "Pagination mirrors the base protocol's list methods: the request accepts an optional `cursor`, and when the result includes `nextCursor` the client passes it back to retrieve the next page." + - check: sep-2640-skills-list-entry-atomic + text: "An entry is atomic — a skill's `resources` set is never split across pages." + - check: sep-2640-skills-list-cache-attributes + text: "In protocol versions 2026-07-28 and later, the result also carries the base protocol's list-caching attributes — `ttlMs` and `cacheScope`, as defined for `tools/list` and `resources/list` (SEP-2549) — with the same semantics: a freshness hint for the listing and a cache-scope marker, not an integrity property." + - check: sep-2640-skills-list-may-be-partial + text: 'A server whose skill catalog is large, generated on demand, or otherwise unenumerable MAY return an empty or partial listing.' + - check: sep-2640-host-no-empty-listing-assumption + text: 'Hosts MUST NOT treat an empty or partial listing as proof that a server has no skills.' + - check: sep-2640-enumeration-scheme-uniform + text: "The method serves entries for a server's skills whatever URI scheme they use — enumeration is uniform across schemes." + + # === The skill entry (shared by skills/list and skills/get) === + - check: sep-2640-entry-uri-required + text: "`skills[].uri` (Yes): Resource URI of the skill's `SKILL.md`." + - check: sep-2640-entry-frontmatter-required + text: "`frontmatter` is the skill's `SKILL.md` YAML frontmatter rendered verbatim as a JSON object — every field the author wrote, not a curated subset. Because the Agent Skills specification requires `name` and `description`, those fields are always present." + - check: sep-2640-entry-frontmatter-identical + text: 'The `frontmatter` object MUST be identical in content to the frontmatter of the `SKILL.md` it describes.' + - check: sep-2640-entry-uri-matches-frontmatter-name + text: "The final `` segment of the entry's `uri` MUST equal `frontmatter.name`, per Resource Mapping." + - check: sep-2640-metadata-reserved-prefix + text: 'Within the frontmatter `metadata` object, keys prefixed with `io.modelcontextprotocol/` are reserved for metadata defined by MCP extensions. This extension currently defines no such keys. Implementations SHOULD ignore keys under this prefix that they do not recognize.' + - check: sep-2640-names-should-be-unique + text: "Within a server's listing, names SHOULD be unique, but they are not guaranteed to be." + - check: sep-2640-host-disambiguate-listing-collision + text: 'When two entries in one listing collide on `name`, hosts MUST disambiguate them — for example by their distinguishing path segments — rather than silently discarding or preferring one.' + - check: sep-2640-names-not-unique + text: "A skill's `name` is a label, not an identifier. Within a server's listing, names SHOULD be unique, but they are not guaranteed to be. Hosts MUST NOT assume name uniqueness. When two entries in one listing collide on `name`, hosts MUST disambiguate them — for example by their distinguishing path segments — rather than silently discarding or preferring one." + + # === resources === + - check: sep-2640-entry-resources-required + text: '`resources` is REQUIRED on every skill entry and takes one of two forms: an array enumerating the skill''s files — `SKILL.md` and every supporting file — as `{uri, digest, size}` triples, or the string `"dynamic"`.' + - check: sep-2640-resources-complete + text: "When present, `resources` MUST be complete: it lists every file of the skill, each exactly once, including an entry matching the skill's top-level `uri` — that entry carries the digest and size of `SKILL.md` itself." + - check: sep-2640-resources-uri-within-skill + text: "Each `uri` MUST be the skill's `SKILL.md` or a file within the skill's directory." + - check: sep-2640-resources-digest-format + text: "Digests are SHA-256 hashes of an artifact's raw bytes, formatted as `sha256:{hex}` where `{hex}` is 64 lowercase hexadecimal characters. Each entry in a skill's `resources` carries the digest of the file at its `uri`." + - check: sep-2640-resources-size-required + text: "Each entry MUST carry `size`: the length in bytes of the file's raw content — the same bytes the `digest` covers." + - check: sep-2640-resources-dynamic-marker + text: 'When a skill''s content is generated dynamically, such that stable digests cannot be published, the server MUST set `"resources": "dynamic"` instead of an array.' + - check: sep-2640-resources-invalid-entry + text: 'An entry with no `resources` at all, or with any value other than an array or `"dynamic"`, is invalid, and hosts MUST NOT load it.' + - check: sep-2640-resources-nested-completeness + text: "Completeness extends to nested skills: from the enclosing skill's perspective their files are supporting files, so the enclosing skill's `resources` lists them too, and the same file may appear in both the enclosing and the nested skill's entries." + + # === Limits === + - check: sep-2640-limit-resources-per-skill + text: "Resources per skill: 512 entries, counted over the entries of the skill's `resources`, `SKILL.md` included." + - check: sep-2640-limit-total-size + text: "Total file size per skill: 16 MiB (16,777,216 bytes), counted over the sum of `size` over the skill's `resources`." + - check: sep-2640-limit-host-support + text: 'Hosts MUST support skills up to and including these limits, and MAY support larger ones. Servers SHOULD NOT serve a skill that exceeds either limit; a skill that does is not guaranteed to be loadable by any conforming host.' + + # === Retrieval via skills/get === + - check: sep-2640-skills-get-implemented + text: 'A server declaring the `io.modelcontextprotocol/skills` extension MUST also implement the `skills/get` method, which returns the entry for a single skill named by its URI.' + - check: sep-2640-skills-get-entry-shape + text: 'The `skill` object is a skill entry, identical in shape and meaning to an entry of `skills/list` — the same `uri`, `frontmatter`, and `resources` fields, under the same rules.' + - check: sep-2640-skills-get-unknown-uri-invalid-params + text: 'If the URI does not identify a skill the server serves, the server MUST return error `-32602` (Invalid params) — the same code `resources/read` uses for unknown resources.' + - check: sep-2640-skills-get-answers-unlisted + text: 'A server MUST answer for every skill it serves, whether or not that skill appears in its `skills/list` result. A skill absent from a partial listing is still retrievable by URI.' + - check: sep-2640-skills-get-no-cursor + text: 'The result carries no pagination cursor: a single entry is not a list.' + + # === Reading / integrity (host obligations) === + - check: sep-2640-host-load-by-uri + text: 'hosts MUST support loading a skill given only its URI' + - check: sep-2640-read-is-not-a-load + text: "Hosts MUST NOT treat a `resources/read` of a `SKILL.md` that arrives by any other route as a load: it grants no approval, opens no window, and confers no standing on the skill's supporting files." + - check: sep-2640-host-verify-digest + text: "When a host retrieves a file listed in a skill's `resources`, it MUST verify the content against that entry's digest. Whatever the cause, hosts MUST NOT use the unverified content." + - check: sep-2640-host-size-mismatch-failure + text: "A read whose byte length differs from the entry's `size` is a verification failure equivalent to a digest mismatch, whether or not the host goes on to compute the digest." + - check: sep-2640-host-unlisted-read-failure + text: "While acting on a skill, a host MUST resolve reads of the skill's files only to URIs listed in that entry's `resources`, and MUST treat a read of an unlisted file within the skill as a verification failure equivalent to a digest mismatch." + - check: sep-2640-host-no-prefetch + text: "Hosts MUST NOT retrieve a skill's files ahead of need — not on connection, not on listing, and not at approval. A `SKILL.md` is fetched when the skill is loaded, and a supporting file when it is read." + - check: sep-2640-host-frontmatter-comparison + text: "After fetching a `SKILL.md` for which the host holds an entry, hosts MUST parse its YAML frontmatter and compare it field-by-field against the entry's `frontmatter`. Any discrepancy MUST be treated as a verification failure equivalent to a digest mismatch, and the skill MUST NOT be loaded." + - check: sep-2640-host-digest-not-security-boundary + text: 'Digests are unsigned and supplied by the same server that supplies the content. A match proves the two are consistent, not that either is trustworthy. Hosts MUST NOT treat a digest match as a security boundary.' + - check: sep-2640-host-skill-identity-pair + text: "The identity of an MCP-served skill is the pair of the host's identity for the originating server and the skill's `uri`. Hosts MUST preserve both halves wherever a skill is recorded or addressed — the registry, persisted approvals, the cache, and any tool or path through which the model reaches the skill — and MUST NOT key any of these on the `uri` alone." + - check: sep-2640-host-not-skill-by-scheme + text: 'A host MUST NOT conclude that a resource is a skill merely because its URI carries a particular scheme.' + - check: sep-2640-host-cross-origin-no-shadow + text: 'When skills from different origins collide on `name`, hosts MUST resolve the name within a per-origin namespace, identifying servers by a host-assigned label; an MCP-served skill MUST NOT silently shadow, or be silently substituted for, a same-named skill from any other origin.' + - check: sep-2640-host-content-bound-approval + text: "When a host persists any per-skill user approval, it MUST be bound to the entry's `resources` set — every `uri` and `digest` — observed at the moment of approval. If a subsequent entry for that skill advertises a different set, the host MUST treat the prior approval as revoked and re-prompt before loading or executing." + - check: sep-2640-host-dynamic-not-content-bound + text: 'A skill whose `resources` is `"dynamic"` cannot be content-bound: hosts MAY decline to load it, and MUST NOT treat a persisted approval as covering whatever content the server currently serves.' + - check: sep-2640-host-nested-fresh-consent + text: 'Activating a nested skill — loading it as a skill in its own right — requires fresh, explicit user consent; approval of the enclosing skill does not substitute for it. A nested `SKILL.md` read as supporting content is ordinary markdown: hosts MUST NOT act on its frontmatter.' + + - check: sep-2640-host-origin-tag-visible + text: 'Hosts MUST tag MCP-served skill content with its originating server identity at the point it enters model context, and MUST NOT present it as indistinguishable from a local filesystem skill.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-untrusted-input + text: 'Hosts MUST treat MCP-served skill content as untrusted model input, subject to the same prompt-injection defenses applied to any server-provided text. A server being connected does not make its skill content authoritative.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-skills-are-data-not-directives + text: 'Hosts MUST NOT treat skill resources as higher-authority than other context. Explicit user policy governs whether a skill is loaded at all.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-cache-path-encodes-origin + text: 'Any path at which a host materializes skill content, whether a cache directory or a virtual mount, MUST encode the server identity as well as the `uri`, so that same-URI skills from different servers land at distinct paths and the originating server is recoverable from the path.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-higher-risk-surface + text: 'Hosts MUST treat MCP-served skills as a higher-risk surface than remote tool invocation.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-no-implicit-local-execution + text: 'Hosts MUST NOT allow MCP-served skill content to cause host-side code execution without explicit per-skill user approval.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-exec-gate-while-acting + text: 'Hosts MUST apply the same approval gate to code-execution tool calls issued while the model is acting on an MCP-served skill.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-reads-bound-to-origin + text: "Hosts MUST bind such reads to the skill's originating server: a skill served by server A MUST NOT cause a `resources/read` against server B." + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-cross-origin-read-approval + text: 'Any cross-origin read MUST be gated behind explicit per-call user approval naming both servers.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-label-not-serverinfo-name + text: "Hosts MUST identify servers by a host-assigned label, not the server's self-reported `serverInfo.name`." + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-no-implicit-permission-grants + text: "Hosts MUST NOT honor frontmatter fields that widen the model's tool or filesystem permissions when the skill arrives over MCP." + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-ignore-allowed-tools + text: 'The Agent Skills `allowed-tools` field, which a filesystem-sourced skill uses to declare the tools available while it runs, MUST be ignored for MCP-origin skills.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-intermediary-meta-prefix + text: 'Intermediaries MAY attach provenance or verification annotations via `_meta` under their own reverse-domain prefix — not the `io.modelcontextprotocol.skills/` prefix reserved for this extension.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-cache-write-isolation + text: 'Hosts that cache skill content on disk MUST do one of the following for every file served from the cache: keep the cache where nothing but the host can write to it, or re-hash the file against the entry on every read.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-cache-excluded-from-discovery + text: 'Hosts that cache MCP-served skill content on disk MUST also do so in a location excluded from every filesystem-skill discovery path, and MUST treat content loaded from there as MCP-origin.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - check: sep-2640-host-cache-removal-on-server-removal + text: "Hosts SHOULD remove a server's cached skill content when the user removes that server." + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + + # === Implementation Guidelines (host + SDK) === + - check: sep-2640-host-registry-no-fetch + text: 'Assembling the registry reads only the listing: the host MUST NOT fetch `SKILL.md` or any supporting file at this stage.' + - check: sep-2640-host-virtual-mount-lazy + text: "A virtual mount resolves reads on access; it MUST NOT be populated by fetching the skill's files in advance." + - check: sep-2640-host-surface-directory-read + text: 'When the originating server declares `directoryRead`, the host SHOULD surface this capability to the model.' + - check: sep-2640-sdk-convenience-wrappers + text: "SDK maintainers SHOULD provide affordances that wrap the underlying resource operations in skill-specific terms. The SDK handles: reading `SKILL.md` frontmatter to populate resource metadata, serving file content on `resources/read`, and answering `skills/get` — and, where the server's skill set is bounded, `skills/list` — computing entry digests and sizes from the registered files, and warning when a registered skill exceeds the Limits." + + # === Directory Listing (resources/directory/read) === + - check: sep-2640-directory-read-method-registered + text: 'A server that declares `directoryRead` MUST support the method for every directory within the skill namespaces it serves as individual files.' + - check: sep-2640-directory-read-subdir-mimetype + text: 'A _directory resource_ is a resource whose `mimeType` is `inode/directory`.' + - check: sep-2640-directory-read-result-resources-shape + text: 'The result contains every direct child of the directory: files with their ordinary resource metadata, subdirectories listed as directory resources (`mimeType: "inode/directory"`). The listing is not recursive; clients descend by calling the method again on a child directory.' + - check: sep-2640-directory-read-invalid-params + text: 'The method applies only to directory resources. If the URI does not exist, or exists but is not a directory resource, the server MUST return error `-32602` (Invalid params) — the same code `resources/read` uses for unknown resources.' + - check: sep-2640-directory-read-pagination + text: 'Pagination mirrors `resources/list`: when the result includes `nextCursor`, the client passes it back as `cursor` to retrieve the next page.' + - check: sep-2640-directory-read-empty-dir + text: 'An empty directory yields an empty `resources` array.' + - check: sep-2640-directory-read-not-manifest-extension + text: 'Hosts MUST NOT treat the directory result as extending the manifest. While acting on the skill under the held entry, the host MUST NOT read a newly listed child — an unlisted file is a verification failure, exactly as a digest mismatch is — and MUST NOT surface it to the model as a file of the skill.' + + # === Excluded: not observable on the MCP wire === + - text: 'Per RFC 3986, the first segment of `` occupies the authority component. This carries no special semantics under this convention and clients MUST NOT attempt DNS or network resolution of it.' + excluded: 'DNS and network resolution sit below the MCP wire layer; the harness cannot observe whether the client performed name lookups on URI authority components.' + - text: "Hosts SHOULD indicate which server a skill originates from when presenting it, SHOULD let users inspect a skill's content before it is loaded into model context" + excluded: 'UI presentation requirements (origin indicator, pre-load inspection); the harness cannot observe what the host displays to users.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - text: "A host is _acting on_ a skill from the moment it loads the skill's `SKILL.md` into the model's context until, at the earliest, that `SKILL.md` leaves context; hosts MAY hold the window open longer, never shorter." + excluded: "The window is defined by what is in the model's context, which is host-internal state the harness cannot observe. It scopes several wire-observable rules but is not itself checkable." + - text: 'Hosts SHOULD instead cache what they do retrieve, and digests make that cache cheap to validate: a cached file whose digest matches the current entry can be served without fetching it again.' + excluded: 'Cache hits are the absence of a request. A harness cannot distinguish a compliant cache from a host that simply did not need the file again.' + - text: 'A host that declines a skill on this basis SHOULD tell the user why rather than fail silently on a later read.' + excluded: 'User-facing messaging; not protocol-observable.' + - text: 'For a skill whose `resources` is `"dynamic"`, the entry offers nothing to count. A host that chooses to load such a skill applies the total-size limit to what it actually retrieves and MAY stop loading the skill once that limit is reached.' + excluded: 'A host stopping mid-load is indistinguishable on the wire from a host that needed no further files.' + - text: 'Hosts SHOULD expect this sequence and present it as such — a skill that has changed and needs re-approval — rather than as a read error.' + excluded: 'User-facing presentation of the stale-entry recovery path; not protocol-observable.' diff --git a/src/types.ts b/src/types.ts index 5960945b..ebe75a27 100644 --- a/src/types.ts +++ b/src/types.ts @@ -102,7 +102,8 @@ export const EXTENSION_IDS = [ 'io.modelcontextprotocol/enterprise-managed-authorization', 'io.modelcontextprotocol/auth/dpop', 'io.modelcontextprotocol/auth/wif', - 'io.modelcontextprotocol/tasks' + 'io.modelcontextprotocol/tasks', + 'io.modelcontextprotocol/skills' ] as const; export type ExtensionId = (typeof EXTENSION_IDS)[number];