From 6f169caf8688f3a8d42ed837fd048cbb73e5709b Mon Sep 17 00:00:00 2001 From: shahinyanm Date: Sun, 26 Jul 2026 13:54:00 +0400 Subject: [PATCH] fix: enforce dispatch routing to the matched specialist (0.50.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sending a task to general-purpose when a tp-* specialist covers it is the most expensive mistake available in a session. Same question, measured: 57,921 tokens through general-purpose against 18,677 through tp-run — 3.1x, ~39k wasted per launch, most of it startup cost a general agent pays before doing any work. The matcher was never the problem; it named the right agent on five of seven real descriptions. The suggestion just never landed. It went out as permissionDecision=allow with the advice in the reason, and an allow is something the model may ignore — it did, on ten of eleven dispatches in a measured session. Reads are disciplined because a wasteful read comes back denied. Dispatches were not, because they came back allowed. Deny mode (the default) now blocks a high-confidence match and names the agent to use. The dead `&& ctx.force` clause that made the deny tier unreachable is gone with it. Matching now reads the prompt as well as the description. Descriptions run two or three words ("Reuse check") and score low; the prompt carries the task and takes the same dispatch to high. Without it the blocking tier would almost never fire. containsEscape reads the prompt too, so an escape written there still works now that the prompt can trigger a block. Low-confidence matches still only advise, escapes still pass, TOKEN_PILOT_MODE=advisory still disables blocking, tp-* dispatches are untouched. The v0.31.0 test asserting the old advise-only behaviour is updated deliberately, not deleted. --- .claude-plugin/marketplace.json | 4 +- .claude-plugin/plugin.json | 2 +- CHANGELOG.md | 32 ++++++++ agents/tp-api-surface-tracker.md | 2 +- agents/tp-audit-scanner.md | 2 +- agents/tp-commit-writer.md | 2 +- agents/tp-context-engineer.md | 2 +- agents/tp-dead-code-finder.md | 2 +- agents/tp-debugger.md | 2 +- agents/tp-dep-health.md | 2 +- agents/tp-doc-writer.md | 2 +- agents/tp-history-explorer.md | 2 +- agents/tp-impact-analyzer.md | 2 +- agents/tp-incident-timeline.md | 2 +- agents/tp-incremental-builder.md | 2 +- agents/tp-migration-scout.md | 2 +- agents/tp-onboard.md | 2 +- agents/tp-performance-profiler.md | 2 +- agents/tp-pr-reviewer.md | 2 +- agents/tp-refactor-planner.md | 2 +- agents/tp-review-impact.md | 2 +- agents/tp-run.md | 2 +- agents/tp-session-restorer.md | 2 +- agents/tp-ship-coordinator.md | 2 +- agents/tp-spec-writer.md | 2 +- agents/tp-test-coverage-gapper.md | 2 +- agents/tp-test-triage.md | 2 +- agents/tp-test-writer.md | 2 +- package-lock.json | 4 +- package.json | 2 +- server.json | 4 +- src/hooks/pre-task.ts | 28 ++++++- tests/hooks/pre-task-routing.test.ts | 112 +++++++++++++++++++++++++++ tests/hooks/pre-task.test.ts | 15 ++-- 34 files changed, 212 insertions(+), 41 deletions(-) create mode 100644 tests/hooks/pre-task-routing.test.ts diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 1df40e3..ed192f2 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -6,14 +6,14 @@ }, "metadata": { "description": "Token Pilot — save 60-90% tokens when AI reads code", - "version": "0.49.0" + "version": "0.50.0" }, "plugins": [ { "name": "token-pilot", "source": "./", "description": "Reduces token consumption by 60-90% via AST-aware lazy file reading, structural symbol navigation, and cross-session tool-usage analytics. 25 MCP tools + 25 subagents + budget watchdog hooks.", - "version": "0.49.0", + "version": "0.50.0", "author": { "name": "Digital-Threads" }, diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 4f063ee..65c9f22 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "token-pilot", - "version": "0.49.0", + "version": "0.50.0", "description": "Saves 60-90% tokens on AI code reading. AST-aware lazy reads, symbol navigation, find_usages, structural git diff/log, edit-safety guard, Task-routing matcher, cross-session telemetry (errors + diagnostics), 25 tp-* subagents tiered to haiku/sonnet/opus with budget watchdog.", "author": { "name": "Digital-Threads", diff --git a/CHANGELOG.md b/CHANGELOG.md index 39d3592..9261d45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,38 @@ All notable changes to Token Pilot will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.50.0] - 2026-07-26 + +### Fixed — dispatch routing now enforces instead of suggesting + +Sending a task to `general-purpose` when a tp-* specialist covers it is +the single most expensive mistake in a session. Measured on the same +question: **57,921 tokens through `general-purpose` against 18,677 +through `tp-run`** — 3.1x, about 39k wasted per launch. Most of that is +startup: a general agent pays ~35k just to exist, before doing any work. + +token-pilot has matched dispatches to specialists since v0.31.0, and the +matcher works — it named the right agent on five of seven real +descriptions. The suggestion simply never landed. It was delivered as +`permissionDecision=allow` with the advice in the reason, and an allow is +something the model may ignore. It did: over one measured session, ten of +eleven dispatches went to `general-purpose` anyway. + +Reads are disciplined because a wasteful read comes back **denied**. +Dispatches were not, because they came back allowed. So deny mode — the +default — now blocks a high-confidence match and names the agent to use. + +Matching also reads the `prompt`, not just the `description`. Real +descriptions are two or three words ("Reuse check") and score as low +confidence; the prompt carries the actual task and takes the same +dispatch to high. Without that, the blocking tier would almost never fire. + +Unchanged: a low-confidence match still only advises, because a weak +keyword must never cost you a dispatch. Escape phrases ("ad-hoc", +"open-ended") still pass — now recognised in the prompt as well — +`TOKEN_PILOT_MODE=advisory` still turns blocking off entirely, and tp-* +dispatches are never touched. + ## [0.49.0] - 2026-07-26 ### Fixed — the tp-* response-budget watchdog was dead diff --git a/agents/tp-api-surface-tracker.md b/agents/tp-api-surface-tracker.md index c2de79b..31123c8 100644 --- a/agents/tp-api-surface-tracker.md +++ b/agents/tp-api-surface-tracker.md @@ -9,7 +9,7 @@ tools: - mcp__token-pilot__read_symbol - Bash model: haiku -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: dd184501203fa7f3c73f419c4ffbe33c4be75400cb64a7a51733a3fe23f6e085 requiredMcpServers: - "token-pilot" diff --git a/agents/tp-audit-scanner.md b/agents/tp-audit-scanner.md index c171b07..d7f72c1 100644 --- a/agents/tp-audit-scanner.md +++ b/agents/tp-audit-scanner.md @@ -11,7 +11,7 @@ tools: - Grep - Read model: sonnet -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: d172f600bf32277ea6eb4cbbee4542ddd698a986dcd96997d33930561964569b requiredMcpServers: - "token-pilot" diff --git a/agents/tp-commit-writer.md b/agents/tp-commit-writer.md index 57dca65..e5285e7 100644 --- a/agents/tp-commit-writer.md +++ b/agents/tp-commit-writer.md @@ -8,7 +8,7 @@ tools: - mcp__token-pilot__test_summary - mcp__token-pilot__outline - Bash -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: de64a406b5176de19f7422619c7de7949b1f28865f225402c9cea9255f377428 requiredMcpServers: - "token-pilot" diff --git a/agents/tp-context-engineer.md b/agents/tp-context-engineer.md index baa87b6..1b642aa 100644 --- a/agents/tp-context-engineer.md +++ b/agents/tp-context-engineer.md @@ -13,7 +13,7 @@ tools: - Edit - Glob model: sonnet -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: 68b32af2dacd82ebe52c4eec93edb903d452688274c3065218270627c564d8b0 requiredMcpServers: - "token-pilot" diff --git a/agents/tp-dead-code-finder.md b/agents/tp-dead-code-finder.md index 9fa9fec..e4acfa9 100644 --- a/agents/tp-dead-code-finder.md +++ b/agents/tp-dead-code-finder.md @@ -11,7 +11,7 @@ tools: - Grep - Read model: sonnet -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: d9b7f5b7ae6f4ae21305c775361bcab097cc774370a6d976c093571d46d55021 requiredMcpServers: - "token-pilot" diff --git a/agents/tp-debugger.md b/agents/tp-debugger.md index b6524a2..d7962f3 100644 --- a/agents/tp-debugger.md +++ b/agents/tp-debugger.md @@ -12,7 +12,7 @@ tools: - Read - Bash model: sonnet -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: 052413de8d92377edcde6ae5c823f5378db304baccfa29e8866467f42553a500 requiredMcpServers: - "token-pilot" diff --git a/agents/tp-dep-health.md b/agents/tp-dep-health.md index e20dc15..16c6659 100644 --- a/agents/tp-dep-health.md +++ b/agents/tp-dep-health.md @@ -9,7 +9,7 @@ tools: - Bash - Read model: haiku -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: e14dc57493d816f8c2e017963e2ef5f66bea50fd0b805a80e8a0d97c968427e7 requiredMcpServers: - "token-pilot" diff --git a/agents/tp-doc-writer.md b/agents/tp-doc-writer.md index 123b44a..cc06bb2 100644 --- a/agents/tp-doc-writer.md +++ b/agents/tp-doc-writer.md @@ -13,7 +13,7 @@ tools: - Edit - Glob model: haiku -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: 57d741794ab40e31a7ac49c68ea39a9088f5827cdef866ce81bfca1b7c9180cf requiredMcpServers: - "token-pilot" diff --git a/agents/tp-history-explorer.md b/agents/tp-history-explorer.md index 9711f19..f587721 100644 --- a/agents/tp-history-explorer.md +++ b/agents/tp-history-explorer.md @@ -10,7 +10,7 @@ tools: - Bash - Read model: haiku -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: 7b70fa76a60e3c58a1de4f56c32c0f166424137e203a0cf1c8654e7c9235d904 requiredMcpServers: - "token-pilot" diff --git a/agents/tp-impact-analyzer.md b/agents/tp-impact-analyzer.md index c2b4da2..fad3317 100644 --- a/agents/tp-impact-analyzer.md +++ b/agents/tp-impact-analyzer.md @@ -12,7 +12,7 @@ tools: - mcp__token-pilot__read_symbols - Read model: sonnet -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: 351a987e11eba63852f5431a16d8eb53104f4f689f82fdcc5a2bf4db948ba92f requiredMcpServers: - "token-pilot" diff --git a/agents/tp-incident-timeline.md b/agents/tp-incident-timeline.md index 6977ed5..d8bd4c0 100644 --- a/agents/tp-incident-timeline.md +++ b/agents/tp-incident-timeline.md @@ -8,7 +8,7 @@ tools: - mcp__token-pilot__read_symbol - Bash model: inherit -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: de5722bfea374eaab096c1ae635c37879e7a91370ee3cd0532f4240be03c91eb requiredMcpServers: - "token-pilot" diff --git a/agents/tp-incremental-builder.md b/agents/tp-incremental-builder.md index 0e75416..55f2a87 100644 --- a/agents/tp-incremental-builder.md +++ b/agents/tp-incremental-builder.md @@ -13,7 +13,7 @@ tools: - Edit - Bash model: sonnet -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: 375a824d0d847bb5453ec594c7a62ad566ee7e4d92717b0473f771f1a0477c60 requiredMcpServers: - "token-pilot" diff --git a/agents/tp-migration-scout.md b/agents/tp-migration-scout.md index 6a2f062..f0aa91b 100644 --- a/agents/tp-migration-scout.md +++ b/agents/tp-migration-scout.md @@ -11,7 +11,7 @@ tools: - Grep - Glob model: sonnet -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: 0334de1bf99b431b65359637d125cda7c44c6f780eb92c57cc538715b1939536 requiredMcpServers: - "token-pilot" diff --git a/agents/tp-onboard.md b/agents/tp-onboard.md index e68dec3..0bba0a4 100644 --- a/agents/tp-onboard.md +++ b/agents/tp-onboard.md @@ -10,7 +10,7 @@ tools: - mcp__token-pilot__smart_read - mcp__token-pilot__smart_read_many - mcp__token-pilot__read_section -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: 832e95633fbc8e9b0c10f3e540a327d4be062fb4b3f17a6cce6be13f414e2927 requiredMcpServers: - "token-pilot" diff --git a/agents/tp-performance-profiler.md b/agents/tp-performance-profiler.md index 03073f8..c53f2a5 100644 --- a/agents/tp-performance-profiler.md +++ b/agents/tp-performance-profiler.md @@ -11,7 +11,7 @@ tools: - Bash - Read model: sonnet -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: b61f06380d80798fa2e49d37bcba0653495bee04dd6bdbc1feff9a75607b0508 requiredMcpServers: - "token-pilot" diff --git a/agents/tp-pr-reviewer.md b/agents/tp-pr-reviewer.md index 7a5d8dd..c1c7a34 100644 --- a/agents/tp-pr-reviewer.md +++ b/agents/tp-pr-reviewer.md @@ -11,7 +11,7 @@ tools: - mcp__token-pilot__read_for_edit - Read model: sonnet -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: f83f50d05b4f70285ae7afed2b1a406fc436df56e61a0aedbfb31edc7f2b6e66 requiredMcpServers: - "token-pilot" diff --git a/agents/tp-refactor-planner.md b/agents/tp-refactor-planner.md index 06057b7..6b5e930 100644 --- a/agents/tp-refactor-planner.md +++ b/agents/tp-refactor-planner.md @@ -8,7 +8,7 @@ tools: - mcp__token-pilot__outline - mcp__token-pilot__read_symbol model: sonnet -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: c5f6fc122c89e16e5cf774045f92169ee3468555320b898171ba13eca5323550 requiredMcpServers: - "token-pilot" diff --git a/agents/tp-review-impact.md b/agents/tp-review-impact.md index f78f25f..e77b745 100644 --- a/agents/tp-review-impact.md +++ b/agents/tp-review-impact.md @@ -9,7 +9,7 @@ tools: - mcp__token-pilot__module_info - Bash model: sonnet -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: 8ef3c3341cbfed4eb8dd130126a9683edc57e378c92ff0ca764d584fd941c55c requiredMcpServers: - "token-pilot" diff --git a/agents/tp-run.md b/agents/tp-run.md index 5f17a2e..8ebae82 100644 --- a/agents/tp-run.md +++ b/agents/tp-run.md @@ -16,7 +16,7 @@ tools: - Glob - Bash model: haiku -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: 2b08618d34a61f00aafccbda9fed6d83243296dedb83440edbd2d5c28bb6dbc4 requiredMcpServers: - "token-pilot" diff --git a/agents/tp-session-restorer.md b/agents/tp-session-restorer.md index 72660d4..2a4b732 100644 --- a/agents/tp-session-restorer.md +++ b/agents/tp-session-restorer.md @@ -9,7 +9,7 @@ tools: - mcp__token-pilot__session_budget - Bash - Read -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: 529374ed728f5eed5b758b3be3da65624783c0bf0c1a253d7d661a843eb5f767 requiredMcpServers: - "token-pilot" diff --git a/agents/tp-ship-coordinator.md b/agents/tp-ship-coordinator.md index 21cfd2c..73097b4 100644 --- a/agents/tp-ship-coordinator.md +++ b/agents/tp-ship-coordinator.md @@ -11,7 +11,7 @@ tools: - Read - Grep model: sonnet -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: a60f6ae110eb3138064bce074e8ba26fa0ce5f4659df1624a9d9d3646803391b requiredMcpServers: - "token-pilot" diff --git a/agents/tp-spec-writer.md b/agents/tp-spec-writer.md index f79a804..4ee5547 100644 --- a/agents/tp-spec-writer.md +++ b/agents/tp-spec-writer.md @@ -9,7 +9,7 @@ tools: - Read - Write model: sonnet -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: c7a4e8b39228fd5158528f389c924c5ff2d98c4b9b05ee0106d54a26c5dc1350 requiredMcpServers: - "token-pilot" diff --git a/agents/tp-test-coverage-gapper.md b/agents/tp-test-coverage-gapper.md index 2e28b12..23b81c4 100644 --- a/agents/tp-test-coverage-gapper.md +++ b/agents/tp-test-coverage-gapper.md @@ -10,7 +10,7 @@ tools: - mcp__token-pilot__test_summary - Glob - Grep -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: be81eed53a3720d146cf89e4a14a7a56577633f7c84c234c412ab70d64c05b11 requiredMcpServers: - "token-pilot" diff --git a/agents/tp-test-triage.md b/agents/tp-test-triage.md index 75dc6c7..0a0117e 100644 --- a/agents/tp-test-triage.md +++ b/agents/tp-test-triage.md @@ -8,7 +8,7 @@ tools: - mcp__token-pilot__find_usages - mcp__token-pilot__read_symbol model: sonnet -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: 362ecf4cb03b059421ea26933473700900073dc38b3a7fe271208dfb1ae14f90 requiredMcpServers: - "token-pilot" diff --git a/agents/tp-test-writer.md b/agents/tp-test-writer.md index 5ecb028..c803159 100644 --- a/agents/tp-test-writer.md +++ b/agents/tp-test-writer.md @@ -13,7 +13,7 @@ tools: - Edit - Bash model: sonnet -token_pilot_version: "0.49.0" +token_pilot_version: "0.50.0" token_pilot_body_hash: 269f2fe22ff4517c277d3f56ca67d8a5527b93290ab21079a83ba7af22c1b5a9 requiredMcpServers: - "token-pilot" diff --git a/package-lock.json b/package-lock.json index 931d546..c9d18c8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "token-pilot", - "version": "0.49.0", + "version": "0.50.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "token-pilot", - "version": "0.49.0", + "version": "0.50.0", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index d0f7e00..22f8dea 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "token-pilot", - "version": "0.49.0", + "version": "0.50.0", "description": "Save up to 80% tokens when AI reads code \u2014 MCP server for token-efficient code navigation, AST-aware structural reading instead of dumping full files into context window", "type": "module", "main": "dist/index.js", diff --git a/server.json b/server.json index 38816d4..59de379 100644 --- a/server.json +++ b/server.json @@ -6,12 +6,12 @@ "url": "https://github.com/Digital-Threads/token-pilot", "source": "github" }, - "version": "0.49.0", + "version": "0.50.0", "packages": [ { "registryType": "npm", "identifier": "token-pilot", - "version": "0.49.0", + "version": "0.50.0", "transport": { "type": "stdio" }, diff --git a/src/hooks/pre-task.ts b/src/hooks/pre-task.ts index 486e8a6..c5885e5 100644 --- a/src/hooks/pre-task.ts +++ b/src/hooks/pre-task.ts @@ -133,6 +133,15 @@ export function decidePreTask( }; } + // v0.50.0 — match against the prompt as well as the description. + // Descriptions are a few words ("Reuse check") and score 1, which is + // always low confidence; the prompt carries the actual task and takes + // the same dispatch to 3. Without this the high-confidence tier below + // almost never fires on real traffic. + const prompt = + typeof input.tool_input?.prompt === "string" ? input.tool_input.prompt : ""; + const haystack = prompt ? `${description} ${prompt}` : description; + // No description → nothing to match against. Inject the generic // tool-guide so the subagent still picks tp-tools (B14). if (!description || description.length === 0) { @@ -141,11 +150,13 @@ export function decidePreTask( // Author-blessed escape clauses — user is explicitly saying // "this is broad". Inject the tool-guide but no agent suggestion. - if (containsEscape(description)) { + // Checked across the prompt too, so an escape written there is honoured + // now that the prompt can trigger a block. + if (containsEscape(haystack)) { return { kind: "advise", message: SUBAGENT_TOOL_GUIDE }; } - const hit = matchTpAgent(description, ctx.agentIndex); + const hit = matchTpAgent(haystack, ctx.agentIndex); if (!hit) { // No specific tp-* match. Still send the generic tool-guide so // the subagent learns about smart_read / read_symbol — covers the @@ -162,10 +173,21 @@ export function decidePreTask( `TOKEN_PILOT_MODE=advisory for warn-only behaviour.\n\n` + SUBAGENT_TOOL_GUIDE; + // v0.50.0 — deny mode now blocks a high-confidence match instead of + // only advising. An advisory rides along as permissionDecision=allow, + // which the model is free to ignore, and it did: measured over one + // session, ten of eleven dispatches went to general-purpose while the + // matcher named a specialist every time. The same question costs 57,921 + // tokens through general-purpose and 18,677 through tp-run. Reads are + // disciplined because they come back denied; dispatches were not, + // because they came back allowed. + // + // Low-confidence matches still only advise — a weak keyword must never + // cost someone their dispatch — and both escape routes are unchanged. const hardBlock = ctx.force || ctx.mode === "strict" || - (ctx.mode === "deny" && hit.confidence === "high" && ctx.force); + (ctx.mode === "deny" && hit.confidence === "high"); if (hardBlock) { return { diff --git a/tests/hooks/pre-task-routing.test.ts b/tests/hooks/pre-task-routing.test.ts new file mode 100644 index 0000000..11371a3 --- /dev/null +++ b/tests/hooks/pre-task-routing.test.ts @@ -0,0 +1,112 @@ +/** + * v0.50.0 — routing to the cheap specialist actually enforces. + * + * Measured on a real session: dispatching `general-purpose` for a question + * a tp-* agent covers costs 57,921 tokens against 18,677 for the same + * question through `tp-run` — 3.1x, ~39k wasted per launch. Ten of eleven + * launches that day went to general-purpose while the matcher was + * correctly identifying a specialist every time. + * + * The reason was the decision tier, not the matcher: deny mode only ever + * advised, and an advisory rides along as permissionDecision=allow, which + * the model does not have to act on. Reads work precisely because they + * come back denied. Two changes close the gap: + * + * 1. deny mode (the default) hard-blocks on a HIGH-confidence match. + * 2. matching reads the prompt as well as the description, because + * descriptions are short ("Reuse review" scored 1 — low) while the + * prompt carries the real signal (the same task scores 3 — high). + * + * Escapes are unchanged: an escape phrase, or TOKEN_PILOT_MODE=advisory, + * still gets you through with advice only. + */ +import { describe, expect, it } from "vitest"; +import { decidePreTask, type PreTaskInput } from "../../src/hooks/pre-task.ts"; +import { parseAgent, type AgentIndex } from "../../src/core/agent-matcher.ts"; +import type { EnforcementMode } from "../../src/server/enforcement-mode.ts"; + +/** A catalog with one reviewer whose quoted trigger is unambiguous. */ +const index: AgentIndex = { + agents: [ + parseAgent( + "tp-pr-reviewer", + `--- +name: tp-pr-reviewer +description: PROACTIVELY use this when the user asks to review a diff, PR, commit range, or changeset ("review these changes", "look at my PR", "is this safe to merge"). Verdict-first output. +--- +`, + )!, + ], +}; + +function input( + description: string, + prompt?: string, + subagentType = "general-purpose", +): PreTaskInput { + return { + tool_name: "Task", + tool_input: { subagent_type: subagentType, description, prompt }, + }; +} + +function ctx(mode: EnforcementMode = "deny", force = false) { + return { mode, agentIndex: index, force }; +} + +describe("pre-task routing enforcement", () => { + it("hard-blocks a high-confidence match in the default deny mode", () => { + const d = decidePreTask(input("review these changes"), ctx("deny")); + expect(d.kind).toBe("deny"); + if (d.kind === "deny") { + expect(d.reason).toContain("tp-pr-reviewer"); + // The block has to say how to get past it, or it is just a wall. + expect(d.reason).toContain("TOKEN_PILOT_MODE"); + } + }); + + it("still only advises on a low-confidence match", () => { + // A single weak keyword should never cost someone their dispatch. + const d = decidePreTask(input("review"), ctx("deny")); + expect(d.kind).not.toBe("deny"); + }); + + it("advisory mode never hard-blocks, however strong the match", () => { + const d = decidePreTask(input("review these changes"), ctx("advisory")); + expect(d.kind).toBe("advise"); + }); + + it("an escape phrase still gets through with advice only", () => { + const d = decidePreTask(input("ad-hoc review these changes"), ctx("deny")); + expect(d.kind).toBe("advise"); + }); + + it("leaves tp-* dispatches alone", () => { + const d = decidePreTask( + input("review these changes", undefined, "tp-pr-reviewer"), + ctx("deny"), + ); + expect(d.kind).toBe("allow"); + }); + + // The description alone is usually a few words and scores low; the + // prompt is where the real task lives. Without reading it, the + // high-confidence tier almost never triggers on real dispatches. + it("uses the prompt to reach high confidence when the description is thin", () => { + const thin = decidePreTask(input("Reuse check"), ctx("deny")); + const withPrompt = decidePreTask( + input("Reuse check", "review these changes for duplication before merge"), + ctx("deny"), + ); + expect(thin.kind).not.toBe("deny"); + expect(withPrompt.kind).toBe("deny"); + }); + + it("does not let a prompt escape phrase be ignored", () => { + const d = decidePreTask( + input("Reuse check", "ad-hoc: review these changes however you like"), + ctx("deny"), + ); + expect(d.kind).toBe("advise"); + }); +}); diff --git a/tests/hooks/pre-task.test.ts b/tests/hooks/pre-task.test.ts index a02219e..3f36ba7 100644 --- a/tests/hooks/pre-task.test.ts +++ b/tests/hooks/pre-task.test.ts @@ -119,15 +119,20 @@ describe("decidePreTask — allow cases", () => { }); describe("decidePreTask — advise cases (default deny-mode)", () => { - it("advises on clear match (high confidence) in deny mode", () => { + // v0.50.0 — this used to expect "advise". Deny mode now blocks a + // high-confidence match, because advising did not work: an advisory is + // delivered as permissionDecision=allow and was ignored on ten of + // eleven dispatches in a measured session, each one costing ~3x what + // the matched specialist would have. + it("hard-denies a clear match (high confidence) in deny mode", () => { const d = decidePreTask( input("general-purpose", "please review these changes"), ctx({ mode: "deny" }), ); - expect(d.kind).toBe("advise"); - if (d.kind === "advise") { - expect(d.message).toContain("tp-pr-reviewer"); - expect(d.message).toContain("confidence: high"); + expect(d.kind).toBe("deny"); + if (d.kind === "deny") { + expect(d.reason).toContain("tp-pr-reviewer"); + expect(d.reason).toContain("confidence: high"); } });