diff --git a/examples/servers/typescript/tools-list-rotated-order.ts b/examples/servers/typescript/tools-list-rotated-order.ts new file mode 100644 index 00000000..df32529a --- /dev/null +++ b/examples/servers/typescript/tools-list-rotated-order.ts @@ -0,0 +1,76 @@ +#!/usr/bin/env node + +/** + * tools/list ordering negative test server. + * + * Speaks the sessionless 2026-07-28 wire (SEP-2575) and advertises the same + * four tools on every tools/list request, but rotates the list by one + * position each time (it keeps a call counter for that, nothing else). The set never changes, only the order, which violates the + * 2026-07-28 SHOULD "Servers SHOULD return tools in a deterministic order". + * The tools-list scenario should emit WARNING for + * tools-list-deterministic-order against this server while tools-list itself + * still passes, since every response is structurally valid. + */ + +import express from 'express'; + +const app = express(); +app.use(express.json()); + +const TOOLS = ['alpha', 'bravo', 'charlie', 'delta'].map((name) => ({ + name, + description: `Fixture tool ${name}`, + inputSchema: { type: 'object', properties: {} } +})); + +let listCalls = 0; + +app.post('/mcp', (req, res) => { + const body = req.body || {}; + const id = body.id ?? null; + const method = body.method; + + switch (method) { + case 'server/discover': + return res.json({ + jsonrpc: '2.0', + id, + result: { + resultType: 'complete', + ttlMs: 0, + cacheScope: 'private', + supportedVersions: ['2026-07-28'], + capabilities: { tools: {} }, + serverInfo: { name: 'tools-list-rotated-order', version: '1.0.0' } + } + }); + case 'tools/list': { + // Rotate by one position per call: the same set, never the same order. + const offset = listCalls++ % TOOLS.length; + const tools = [...TOOLS.slice(offset), ...TOOLS.slice(0, offset)]; + return res.json({ + jsonrpc: '2.0', + id, + result: { + resultType: 'complete', + ttlMs: 0, + cacheScope: 'private', + tools + } + }); + } + default: + return res.status(404).json({ + jsonrpc: '2.0', + id, + error: { code: -32601, message: 'Method not found' } + }); + } +}); + +const PORT = parseInt(process.env.PORT || '3008', 10); +app.listen(PORT, '127.0.0.1', () => { + console.log( + `tools/list rotated-order negative test server running on http://localhost:${PORT}/mcp` + ); +}); diff --git a/src/scenarios/server/negative.test.ts b/src/scenarios/server/negative.test.ts index 0d248272..e2748ab3 100644 --- a/src/scenarios/server/negative.test.ts +++ b/src/scenarios/server/negative.test.ts @@ -4,6 +4,7 @@ import path from 'path'; import { DNSRebindingProtectionScenario } from './dns-rebinding'; import { ResourcesNotFoundErrorScenario } from './resources'; import { CachingScenario } from './caching'; +import { ToolsListScenario } from './tools'; import { JsonSchema2020_12Scenario, sep2106KeywordCheckStatus @@ -216,6 +217,43 @@ describe('Server scenario negative tests', () => { }, 10000); }); + describe('tools-list-deterministic-order', () => { + let serverProcess: ChildProcess | null = null; + const PORT = 3008; + + beforeAll(async () => { + serverProcess = await startServer( + path.join( + process.cwd(), + 'examples/servers/typescript/tools-list-rotated-order.ts' + ), + PORT + ); + }, 35000); + + afterAll(async () => { + await stopServer(serverProcess); + }); + + it('emits WARNING for deterministic-order while tools-list still passes against a server that rotates its tool list', async () => { + const scenario = new ToolsListScenario(); + const checks = await scenario.run( + testContext(`http://localhost:${PORT}/mcp`, DRAFT_PROTOCOL_VERSION) + ); + + const list = checks.find((c) => c.id === 'tools-list'); + expect(list?.status).toBe('SUCCESS'); + + const order = checks.find( + (c) => c.id === 'tools-list-deterministic-order' + ); + expect(order?.status).toBe('WARNING'); + expect(order?.errorMessage).toMatch(/different order/); + expect(order?.details).toMatchObject({ toolCount: 4, probes: 3 }); + expect(order?.details?.untestable).toBeUndefined(); + }, 10000); + }); + describe('sep2106KeywordCheckStatus (soft version gate)', () => { it('passes preserved keywords at any target version', () => { expect(sep2106KeywordCheckStatus(true, DRAFT_PROTOCOL_VERSION)).toBe( diff --git a/src/scenarios/server/tools.test.ts b/src/scenarios/server/tools.test.ts index d54daa4c..0d748777 100644 --- a/src/scenarios/server/tools.test.ts +++ b/src/scenarios/server/tools.test.ts @@ -1,5 +1,9 @@ import { describe, it, expect } from 'vitest'; -import { buildToolsNameFormatCheck, validateToolNameFormat } from './tools.js'; +import { + buildToolsListDeterministicOrderCheck, + buildToolsNameFormatCheck, + validateToolNameFormat +} from './tools.js'; describe('validateToolNameFormat', () => { it('accepts a typical snake_case name', () => { @@ -105,3 +109,143 @@ describe('buildToolsNameFormatCheck', () => { expect(ids).toEqual(['MCP-Tools-List', 'SEP-986']); }); }); + +describe('buildToolsListDeterministicOrderCheck', () => { + const names = (...ns: string[]) => ns.map((name) => ({ name })); + + it('returns SUCCESS when every probe lists the same tools in the same order', () => { + const check = buildToolsListDeterministicOrderCheck([ + names('a', 'b', 'c'), + names('a', 'b', 'c'), + names('a', 'b', 'c') + ]); + expect(check.id).toBe('tools-list-deterministic-order'); + expect(check.status).toBe('SUCCESS'); + expect(check.errorMessage).toBeUndefined(); + expect(check.details).toEqual({ + toolCount: 3, + probes: 3, + orders: [ + ['a', 'b', 'c'], + ['a', 'b', 'c'], + ['a', 'b', 'c'] + ] + }); + expect(check.specReferences?.[0]?.url).toContain('2026-07-28/server/tools'); + }); + + it('returns WARNING when the same tools come back in a different order', () => { + const check = buildToolsListDeterministicOrderCheck([ + names('a', 'b', 'c'), + names('b', 'c', 'a'), + names('c', 'a', 'b') + ]); + expect(check.status).toBe('WARNING'); + expect(check.errorMessage).toMatch(/different order/); + expect(check.errorMessage).toMatch(/index 0/); + expect(check.details).toMatchObject({ toolCount: 3, probes: 3 }); + expect(check.details?.untestable).toBeUndefined(); + expect(check.details?.orders).toEqual([ + ['a', 'b', 'c'], + ['b', 'c', 'a'], + ['c', 'a', 'b'] + ]); + }); + + it('flags a divergence that only appears on the last probe', () => { + const check = buildToolsListDeterministicOrderCheck([ + names('a', 'b', 'c'), + names('a', 'b', 'c'), + names('a', 'c', 'b') + ]); + expect(check.status).toBe('WARNING'); + expect(check.errorMessage).toMatch(/probe 3/); + expect(check.errorMessage).toMatch(/index 1/); + }); + + it('reports untestable (WARNING) when the set of tools changed between probes', () => { + const check = buildToolsListDeterministicOrderCheck([ + names('a', 'b', 'c'), + names('a', 'b', 'd') + ]); + expect(check.status).toBe('WARNING'); + expect(check.errorMessage).toMatch(/^Not testable: /); + expect(check.errorMessage).toMatch(/added: d/); + expect(check.errorMessage).toMatch(/removed: c/); + expect(check.details).toMatchObject({ untestable: true }); + }); + + it('reports untestable (WARNING) when a probe returned no tools array', () => { + const check = buildToolsListDeterministicOrderCheck([ + names('a', 'b'), + undefined + ]); + expect(check.status).toBe('WARNING'); + expect(check.errorMessage).toMatch(/^Not testable: /); + expect(check.details).toMatchObject({ untestable: true }); + }); + + it('reports untestable (WARNING) with fewer than two probes', () => { + const check = buildToolsListDeterministicOrderCheck([names('a', 'b')]); + expect(check.status).toBe('WARNING'); + expect(check.errorMessage).toMatch(/^Not testable: /); + }); + + it('returns INFO when no probe saw two tools to order', () => { + const one = buildToolsListDeterministicOrderCheck([names('a'), names('a')]); + expect(one.status).toBe('INFO'); + expect(one.errorMessage).toMatch(/nothing to compare/); + expect(one.details).toEqual({ + toolCount: 1, + probes: 2, + orders: [['a'], ['a']] + }); + const none = buildToolsListDeterministicOrderCheck([[], []]); + expect(none.status).toBe('INFO'); + }); + + it('reports untestable when a probe grows from one tool to two', () => { + const check = buildToolsListDeterministicOrderCheck([ + names('a'), + names('a', 'b') + ]); + expect(check.status).toBe('WARNING'); + expect(check.errorMessage).toMatch(/^Not testable: /); + expect(check.errorMessage).toMatch(/added: b/); + }); + + it('treats a tool without a string name as one placeholder entry', () => { + const check = buildToolsListDeterministicOrderCheck([ + [{ name: 'a' }, { name: 42 }], + [{ name: 42 }, { name: 'a' }] + ]); + expect(check.status).toBe('WARNING'); + expect(check.errorMessage).toMatch(/different order/); + expect(check.details?.orders).toEqual([ + ['a', ''], + ['', 'a'] + ]); + }); + + it('treats duplicate names as a set change only when their multiplicity changes', () => { + const stable = buildToolsListDeterministicOrderCheck([ + names('a', 'a', 'b'), + names('a', 'a', 'b') + ]); + expect(stable.status).toBe('SUCCESS'); + const changed = buildToolsListDeterministicOrderCheck([ + names('a', 'a', 'b'), + names('a', 'b', 'b') + ]); + expect(changed.status).toBe('WARNING'); + expect(changed.errorMessage).toMatch(/^Not testable: /); + }); + + it('gates itself to the 2026-07-28 wire', () => { + const check = buildToolsListDeterministicOrderCheck([ + names('a', 'b'), + names('a', 'b') + ]); + expect(check.source).toEqual({ introducedIn: '2026-07-28' }); + }); +}); diff --git a/src/scenarios/server/tools.ts b/src/scenarios/server/tools.ts index f18df3cf..83ed934f 100644 --- a/src/scenarios/server/tools.ts +++ b/src/scenarios/server/tools.ts @@ -5,9 +5,11 @@ import { ClientScenario, ConformanceCheck, - DRAFT_PROTOCOL_VERSION + DRAFT_PROTOCOL_VERSION, + specVersionAtLeast } from '../../types'; import type { RunContext } from '../../connection'; +import { notTestable, untestableCheck } from '../untestable'; import type { ListToolsResult, CallToolResult @@ -98,6 +100,136 @@ export function buildToolsNameFormatCheck( }; } +export const TOOLS_LIST_ORDER_CHECK_ID = 'tools-list-deterministic-order'; + +/** The revision that introduced the deterministic-order SHOULD. A published, dated revision, so written as a literal. */ +export const TOOLS_LIST_ORDER_INTRODUCED_IN = '2026-07-28' as const; + +/** How many consecutive tools/list snapshots the ordering check compares. */ +const TOOLS_LIST_ORDER_PROBES = 3; + +const TOOLS_LIST_ORDER_SPEC_REFS = [ + { + id: 'MCP-Tools-Deterministic-Order', + url: 'https://modelcontextprotocol.io/specification/2026-07-28/server/tools#capabilities' + } +]; + +/** + * Build the tools-list-deterministic-order check from consecutive tools/list + * snapshots. + * + * 2026-07-28 server/tools.mdx: "Servers SHOULD return tools in a deterministic + * order (i.e., the same ordering across requests when the underlying set of + * tools has not changed)." SHOULD, so a violation is WARNING. + * + * The spec scopes the SHOULD to an unchanged set, so a set that differs + * between probes is reported as untestable (issue #248) rather than as a + * violation: from the outside, a sample cannot tell a nondeterministic server + * from one whose tools legitimately changed between two requests. + */ +export function buildToolsListDeterministicOrderCheck( + snapshots: ReadonlyArray | undefined> +): ConformanceCheck { + const timestamp = new Date().toISOString(); + const baseCheck = { + id: TOOLS_LIST_ORDER_CHECK_ID, + name: 'ToolsListDeterministicOrder', + description: + 'Consecutive tools/list requests return the same tools in the same order', + specReferences: TOOLS_LIST_ORDER_SPEC_REFS, + source: { introducedIn: TOOLS_LIST_ORDER_INTRODUCED_IN }, + timestamp + }; + const untestable = (reason: string): ConformanceCheck => ({ + ...baseCheck, + status: 'WARNING', + errorMessage: notTestable(reason), + details: { untestable: true, reason, probes: snapshots.length } + }); + + if (snapshots.length < 2) { + return untestable( + `needs at least two tools/list snapshots to compare, got ${snapshots.length}` + ); + } + const missing = snapshots.findIndex((tools) => !Array.isArray(tools)); + if (missing !== -1) { + return untestable( + `tools/list probe ${missing + 1} did not return a tools array` + ); + } + + // A position-independent placeholder, so that a nameless tool moving + // around reads as an order change rather than as a set change. + const orders = snapshots.map((tools) => + (tools as ReadonlyArray<{ name?: unknown }>).map((tool) => + typeof tool.name === 'string' ? tool.name : '' + ) + ); + + // Nothing to order when no probe saw two tools. + if (orders.every((order) => order.length < 2)) { + return { + ...baseCheck, + status: 'INFO', + errorMessage: `${orders[0].length} tool(s) advertised; nothing to compare`, + details: { toolCount: orders[0].length, probes: orders.length, orders } + }; + } + + // The SHOULD only binds while the set is unchanged: compare multisets first. + const countNames = (names: string[]): Map => { + const counts = new Map(); + for (const name of names) counts.set(name, (counts.get(name) ?? 0) + 1); + return counts; + }; + const baseline = countNames(orders[0]); + for (let probe = 1; probe < orders.length; probe++) { + const current = countNames(orders[probe]); + const added: string[] = []; + const removed: string[] = []; + for (const [name, count] of current) { + const before = baseline.get(name) ?? 0; + if (count > before) added.push(name); + } + for (const [name, count] of baseline) { + const now = current.get(name) ?? 0; + if (count > now) removed.push(name); + } + if (added.length > 0 || removed.length > 0) { + return untestable( + `the set of tools changed between tools/list probe 1 and probe ${probe + 1} ` + + `(added: ${added.join(', ') || 'none'}; removed: ${removed.join(', ') || 'none'}), ` + + 'so the deterministic-order SHOULD does not apply to this sample' + ); + } + } + + const toolCount = orders[0].length; + + for (let probe = 1; probe < orders.length; probe++) { + const index = orders[probe].findIndex((name, i) => name !== orders[0][i]); + if (index !== -1) { + return { + ...baseCheck, + status: 'WARNING', + errorMessage: + `tools/list returned the same ${toolCount} tools in a different order across ` + + `consecutive requests: probe ${probe + 1} diverges from probe 1 at index ${index} ` + + `(${orders[0][index]} vs ${orders[probe][index]})`, + details: { toolCount, probes: orders.length, orders } + }; + } + } + + return { + ...baseCheck, + status: 'SUCCESS', + details: { toolCount, probes: orders.length, orders } + }; +} + export class ToolsListScenario implements ClientScenario { name = 'tools-list'; readonly source = { introducedIn: '2025-06-18' } as const; @@ -112,7 +244,9 @@ export class ToolsListScenario implements ClientScenario { - Each tool MUST have: - \`name\` (string, 1-64 chars, matching \`^[A-Za-z0-9_./-]+$\`) - \`description\` (string) - - \`inputSchema\` (valid JSON Schema object)`; + - \`inputSchema\` (valid JSON Schema object) +- From 2026-07-28: return tools in a deterministic order across requests + when the set of tools has not changed (SHOULD)`; async run(ctx: RunContext): Promise { const checks: ConformanceCheck[] = []; @@ -163,6 +297,38 @@ export class ToolsListScenario implements ClientScenario { // names MUST be 1-64 chars matching ^[A-Za-z0-9_./-]+$ checks.push(buildToolsNameFormatCheck(result.tools)); + // 2026-07-28: tools SHOULD come back in a deterministic order across + // requests. Take two more consecutive tools/list snapshots and compare. + if (specVersionAtLeast(ctx.specVersion, TOOLS_LIST_ORDER_INTRODUCED_IN)) { + const snapshots: Array = [ + result.tools + ]; + let probeError: unknown; + try { + while (snapshots.length < TOOLS_LIST_ORDER_PROBES) { + const again = await conn.request('tools/list'); + snapshots.push(again.tools); + } + } catch (error) { + probeError = error; + } + checks.push( + probeError === undefined + ? buildToolsListDeterministicOrderCheck(snapshots) + : { + ...untestableCheck( + TOOLS_LIST_ORDER_CHECK_ID, + 'ToolsListDeterministicOrder', + 'Consecutive tools/list requests return the same tools in the same order', + `repeated tools/list request failed: ${probeError instanceof Error ? probeError.message : String(probeError)}`, + TOOLS_LIST_ORDER_SPEC_REFS, + 'WARNING' + ), + source: { introducedIn: TOOLS_LIST_ORDER_INTRODUCED_IN } + } + ); + } + await conn.close(); } catch (error) { checks.push({