From 74af951b740e0a5ac353ab89049ce80c4ae1cad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=B1=CE=B7dr3=CE=B15=20=C9=AE=CE=B1=CE=B7=C9=A6=D6=85l?= =?UTF-8?q?=CA=903=CA=80?= Date: Fri, 5 Jun 2026 17:53:38 +0200 Subject: [PATCH 1/5] Wire --force + grading reload command Phase 2 (Memo 110): cache control on the CLI. - thread --force through gradingDeterministic -> DataPretest.run - surface fromCache + dataAt in the deterministic result - new `grading reload `: re-fetch + rewrite test-N.json only (force), decoupled from grading (no _gradings/grade.json writes) - tests for force-threading, data-stamp surfacing, reload routing Co-Authored-By: Claude Opus 4.8 (1M context) --- src/index.mjs | 18 +++- src/task/FlowMcpCli.mjs | 102 ++++++++++++++++++++-- tests/unit/grading-deterministic.test.mjs | 82 +++++++++++++++++ 3 files changed, 194 insertions(+), 8 deletions(-) diff --git a/src/index.mjs b/src/index.mjs index 9c8c859..ba787f2 100755 --- a/src/index.mjs +++ b/src/index.mjs @@ -431,13 +431,13 @@ const runCommand = async () => { // non-deterministic LLM-scoring path (emit + consume), formerly only reached // via `run --emit-prompts` / `run --consume-scores`. `run` is kept as the // internal mechanic (Never-delete-legacy). - const validSubCommands = [ 'deterministic', 'non-deterministic', 'export', 'run', 'state', 'worklist', 'doctor', 'config' ] + const validSubCommands = [ 'deterministic', 'non-deterministic', 'reload', 'export', 'run', 'state', 'worklist', 'doctor', 'config' ] if( !subCommand || !validSubCommands.includes( subCommand ) ) { const result = { 'status': false, 'error': 'Missing or unknown grading sub-command.', - 'fix': `Use: ${appConfig[ 'cliCommand' ]} grading deterministic | non-deterministic --emit-prompts | --consume-scores | export | state | worklist | doctor | config [--set-data-dir ] [--set-export-dir ]` + 'fix': `Use: ${appConfig[ 'cliCommand' ]} grading deterministic [--force] | non-deterministic --emit-prompts | --consume-scores | reload | export | state | worklist | doctor | config [--set-data-dir ] [--set-export-dir ]` } output( { result } ) @@ -460,9 +460,21 @@ const runCommand = async () => { // single internal switch dryRun. When set, grading performs but writes // NOTHING to the island (no pretest persist, no index/grade/state). const dryRun = values[ 'no-save' ] === true + // PRD-2.2 — --force bypasses the read-cache (PRD-2.1): re-fetch the test + // data instead of reusing the persisted test-N.json. + const force = values[ 'force' ] === true if( subCommand === 'deterministic' ) { - const { result } = await FlowMcpCli.gradingDeterministic( { cwd, target, gradingDataDir, gradingExportDir, withKeys, only, dryRun, json } ) + const { result } = await FlowMcpCli.gradingDeterministic( { cwd, target, gradingDataDir, gradingExportDir, withKeys, only, dryRun, force, json } ) + output( { result } ) + + return true + } + + if( subCommand === 'reload' ) { + // PRD-2.3 — re-fetch + rewrite the persisted test-N.json only (force), + // decoupled from grading: no _gradings/grade.json writes. + const { result } = await FlowMcpCli.gradingReload( { cwd, target, gradingDataDir, withKeys, json } ) output( { result } ) return true diff --git a/src/task/FlowMcpCli.mjs b/src/task/FlowMcpCli.mjs index c5b823f..539b6c2 100644 --- a/src/task/FlowMcpCli.mjs +++ b/src/task/FlowMcpCli.mjs @@ -12651,7 +12651,7 @@ allowlist, migrate-config, etc.). // selection-member run through the existing structural primitive check // (#runTypedTests + #aggregateByPrimitive). The same #validateOnlyFilter // allowlist applies (no duplication). - static async gradingDeterministic( { cwd, target, gradingDataDir, gradingExportDir = null, withKeys, only, dryRun = false, json, skipRollup = false } ) { + static async gradingDeterministic( { cwd, target, gradingDataDir, gradingExportDir = null, withKeys, only, dryRun = false, force = false, json, skipRollup = false } ) { const grading = await FlowMcpCli.#loadGradingModule() if( grading === null || grading[ 'DataPretest' ] === undefined ) { return { 'result': FlowMcpCli.#error( { 'error': 'grading module unavailable', 'fix': 'npm install / update the flowmcp-grading dependency' } ) } @@ -12680,7 +12680,7 @@ allowlist, migrate-config, etc.). // namespace rollup (index.json) + Provider-Proof (grade.json). Delegated so // the single-schema path below stays unchanged. if( parsed.type === 'namespace' ) { - return FlowMcpCli.#gradingDeterministicNamespace( { cwd, 'namespace': parsed.namespace, gradingDataDir, gradingExportDir, withKeys, only, dryRun, json } ) + return FlowMcpCli.#gradingDeterministicNamespace( { cwd, 'namespace': parsed.namespace, gradingDataDir, gradingExportDir, withKeys, only, dryRun, force, json } ) } if( parsed.type !== 'schema' && parsed.type !== 'tool' && parsed.type !== 'test' ) { return { 'result': FlowMcpCli.#error( { 'error': `Spec-ID type "${parsed.type}" is not supported by grading deterministic (only namespace, schema-ID, tool-ID or per-test).`, 'fix': 'Use "", "/", "/tool/" or "/tool//tests/".' } ) } @@ -12739,6 +12739,12 @@ allowlist, migrate-config, etc.). // PRD-012 — --no-save (dryRun) runs the pretest in full but persists NOTHING // to the island (no summary.json / test-N.json). The deterministic path has // no Stage-3 writes, so dryRun here only gates the DataPretest persist. + // PRD-2.2 — force threads the cache bypass into the pretest. Without it the + // pretest reuses the persisted test-N.json (read-cache, PRD-2.1); with it the + // data is re-fetched. A re-fetch that flips `deterministic-green` flows + // straight into the _gradings rewrite + rollup below, so the affected + // deterministic areas are re-evaluated (the grade itself still hangs on the + // schemaHash — data reuse never silently invalidates it). const pretestRaw = await grading[ 'DataPretest' ].run( { namespace, 'toolName': schemaName, @@ -12748,7 +12754,8 @@ allowlist, migrate-config, etc.). serverParams, sharedLists, 'gradingDataDir': gradingDataRoot, - dryRun + dryRun, + force } ) // Tool-ID: restrict the pretest view to the one addressed tool. The gate is @@ -12934,7 +12941,7 @@ allowlist, migrate-config, etc.). // Memo 107 PRD-004 — bare-namespace deterministic grade: run every schema of the // namespace (skipRollup, so each writes its own `_gradings/` but defers the rollup), // then build the namespace index.json + Provider-Proof grade.json EXACTLY ONCE. - static async #gradingDeterministicNamespace( { cwd, namespace, gradingDataDir, gradingExportDir, withKeys, only, dryRun, json } ) { + static async #gradingDeterministicNamespace( { cwd, namespace, gradingDataDir, gradingExportDir, withKeys, only, dryRun, force = false, json } ) { const resolved = await FlowMcpCli.#resolveSchemasForTarget( { namespace } ) if( resolved.status === false ) { return { 'result': FlowMcpCli.#error( { 'error': resolved.error, 'fix': resolved.fix } ) } @@ -12944,7 +12951,7 @@ allowlist, migrate-config, etc.). await resolved.schemas .reduce( ( promise, schema ) => promise.then( async () => { const sub = await FlowMcpCli.gradingDeterministic( { - cwd, 'target': `${namespace}/${schema.schemaName}`, gradingDataDir, gradingExportDir, withKeys, only, dryRun, json, 'skipRollup': true + cwd, 'target': `${namespace}/${schema.schemaName}`, gradingDataDir, gradingExportDir, withKeys, only, dryRun, force, json, 'skipRollup': true } ) const subResult = sub.result perSchema.push( { @@ -12982,6 +12989,87 @@ allowlist, migrate-config, etc.). } + // PRD-2.3 — `grading reload `: re-fetch + rewrite the persisted + // test-N.json (force semantics), DECOUPLED from grading. It runs the data + // pretest with force:true so the read-cache (PRD-2.1) is bypassed and the + // island test data is refreshed, but it writes NO `_gradings/` entries and NO + // grade.json/index.json — a pure data reload. Reports the per-schema rewritten + // test counts + the new data stamp (dataAt). NO SILENT DEFAULTS. + static async gradingReload( { cwd, target, gradingDataDir, withKeys, json } ) { + const grading = await FlowMcpCli.#loadGradingModule() + if( grading === null || grading[ 'DataPretest' ] === undefined ) { + return { 'result': FlowMcpCli.#error( { 'error': 'grading module unavailable', 'fix': 'npm install / update the flowmcp-grading dependency' } ) } + } + if( typeof target !== 'string' || target.length === 0 ) { + return { 'result': FlowMcpCli.#error( { 'error': 'Missing reload target.', 'fix': 'Usage: flowmcp grading reload | /' } ) } + } + + const parsed = FlowMcpCli.#parseSpecId( { 'specId': target } ) + if( parsed.valid !== true || ( parsed.type !== 'namespace' && parsed.type !== 'schema' ) ) { + return { 'result': FlowMcpCli.#error( { 'error': `grading reload accepts a namespace or a schema-ID, got "${target}".`, 'fix': 'Use "" or "/".' } ) } + } + + const namespace = parsed.namespace + const gradingDataRoot = await FlowMcpCli.#gradingDataRoot( { cwd, gradingDataDir } ) + const resolvedSchemas = await FlowMcpCli.#resolveSchemasForTarget( { namespace } ) + if( resolvedSchemas.status === false ) { + return { 'result': FlowMcpCli.#error( { 'error': resolvedSchemas.error, 'fix': resolvedSchemas.fix } ) } + } + + const targetSchemas = parsed.type === 'schema' + ? resolvedSchemas.schemas.filter( ( s ) => s.schemaName === parsed.name ) + : resolvedSchemas.schemas + if( targetSchemas.length === 0 ) { + return { 'result': FlowMcpCli.#error( { 'error': `Schema "${target}" not found in schemaFolders[].`, 'fix': 'Address an existing schema or namespace.' } ) } + } + + const { useKeys } = await FlowMcpCli.#gradingUseKeys( { withKeys } ) + const envObject = useKeys === true ? ( await FlowMcpCli.#resolveEnv( { cwd } ) ).envObject : {} + + const perSchema = [] + await targetSchemas + .reduce( ( promise, schema ) => promise.then( async () => { + const requiredServerParams = Array.isArray( schema.main[ 'requiredServerParams' ] ) ? schema.main[ 'requiredServerParams' ] : [] + const serverParams = useKeys === true + ? FlowMcpCli.#buildServerParams( { envObject, requiredServerParams } ).serverParams + : {} + const { sharedLists } = await FlowMcpCli.#resolveSharedListsForSchema( { 'main': schema.main, 'filePath': schema.sourcePath } ) + const pretest = await grading[ 'DataPretest' ].run( { + namespace, + 'toolName': schema.schemaName, + 'main': schema.main, + 'handlersFn': schema.handlersFn, + 'schemaSnapshotPath': schema.sourcePath, + serverParams, + sharedLists, + 'gradingDataDir': gradingDataRoot, + 'force': true + } ) + const testsWritten = ( Array.isArray( pretest.results ) ? pretest.results : [] ) + .filter( ( r ) => r[ 'primitive' ] === 'tool' || r[ 'primitive' ] === 'resource' ) + .length + perSchema.push( { + 'schema': schema.schemaName, + 'reloaded': pretest.fromCache === false, + 'testsWritten': testsWritten, + 'ok': pretest.ok === true, + 'keyGated': pretest.keyGated === true, + 'dataAt': pretest.dataAt === undefined ? null : pretest.dataAt + } ) + } ), Promise.resolve() ) + + return { + 'result': { + 'status': true, + 'mode': 'reload', + 'target': target, + 'schemaCount': perSchema.length, + 'schemas': perSchema + } + } + } + + // PRD-001 + PRD-003 (B2) — resolve the addressed schema from the LIVE // schemaFolders[] read (liveSchemas = { schemaName, main, handlersFn, // sourcePath }[]). A schema-ID names the folder directly (must exist). A @@ -13042,6 +13130,8 @@ allowlist, migrate-config, etc.). 'toolsBelowThreshold': Array.isArray( pretestRaw.toolsBelowThreshold ) ? pretestRaw.toolsBelowThreshold : [], 'perTool': pretestRaw.perTool === undefined || pretestRaw.perTool === null ? {} : pretestRaw.perTool, 'stopReason': pretestRaw.stopReason === undefined ? null : pretestRaw.stopReason, + 'fromCache': pretestRaw.fromCache === true, + 'dataAt': pretestRaw.dataAt === undefined ? null : pretestRaw.dataAt, 'results': baseResults, 'errors': Array.isArray( pretestRaw.errors ) ? pretestRaw.errors : [] } @@ -13094,6 +13184,8 @@ allowlist, migrate-config, etc.). : [], 'perTool': toolNode === null ? {} : { [ toolFilter ]: toolNode }, 'stopReason': pretestRaw.stopReason === undefined ? null : pretestRaw.stopReason, + 'fromCache': pretestRaw.fromCache === true, + 'dataAt': pretestRaw.dataAt === undefined ? null : pretestRaw.dataAt, 'results': filtered, 'errors': [ ...errors, ...carried ] } diff --git a/tests/unit/grading-deterministic.test.mjs b/tests/unit/grading-deterministic.test.mjs index 6de0952..6b2b519 100644 --- a/tests/unit/grading-deterministic.test.mjs +++ b/tests/unit/grading-deterministic.test.mjs @@ -326,3 +326,85 @@ describe( 'gradingDeterministic — det alias parity (dispatch level)', () => { expect( parsedAlias.error ).toBe( parsedFull.error ) } ) } ) + + +describe( 'gradingDeterministic — force / read-cache surfacing (Memo 110 P2)', () => { + afterEach( () => { FlowMcpCli.__testInjectGrading( { grading: null } ) } ) + + it( 'threads --force into DataPretest.run (cache bypass)', async () => { + const cwd = await freshCwd() + const grading = gradingWithStubbedPretest( { ok: true } ) + await importFixture( { cwd, grading } ) + + await FlowMcpCli.gradingDeterministic( { cwd, target: 'demoapi/demoapi', gradingDataDir: '.flowmcp/grading', withKeys: false, only: null, force: true, json: true } ) + expect( grading.__stub.lastCall.force ).toBe( true ) + } ) + + it( 'defaults force to false (read-cache reuse is the default)', async () => { + const cwd = await freshCwd() + const grading = gradingWithStubbedPretest( { ok: true } ) + await importFixture( { cwd, grading } ) + + await FlowMcpCli.gradingDeterministic( { cwd, target: 'demoapi/demoapi', gradingDataDir: '.flowmcp/grading', withKeys: false, only: null, json: true } ) + expect( grading.__stub.lastCall.force ).toBe( false ) + } ) + + it( 'surfaces the data stamp (fromCache + dataAt) from the pretest', async () => { + const cwd = await freshCwd() + const grading = { ...realGrading, DataPretest: { + getVersion: () => ( { version: 'stub' } ), + run: async ( params ) => ( { + ok: true, + passedDownloadable: 3, + required: 2, + toolsBelowThreshold: [], + perTool: {}, + fromCache: true, + dataAt: '2026-06-05T10:00:00.000Z', + schemaDir: null, + summaryPath: join( params.gradingDataDir, 'providers', params.namespace, params.toolName, 'summary.json' ), + results: [ + { primitive: 'tool', name: 'getThing', status: true, hasData: true, working: true, error: null }, + { primitive: 'tool', name: 'getThing', status: true, hasData: true, working: true, error: null } + ], + stopReason: null, + errors: [] + } ) + } } + await importFixture( { cwd, grading } ) + + const { result } = await FlowMcpCli.gradingDeterministic( { cwd, target: 'demoapi/demoapi', gradingDataDir: '.flowmcp/grading', withKeys: false, only: null, json: true } ) + expect( result.pretest.fromCache ).toBe( true ) + expect( result.pretest.dataAt ).toBe( '2026-06-05T10:00:00.000Z' ) + } ) +} ) + + +describe( 'gradingReload — data reload only (Memo 110 PRD-2.3)', () => { + afterEach( () => { FlowMcpCli.__testInjectGrading( { grading: null } ) } ) + + it( 'reloads a schema with force:true and writes no grade.json', async () => { + const cwd = await freshCwd() + const grading = gradingWithStubbedPretest( { ok: true } ) + await importFixture( { cwd, grading } ) + + const { result } = await FlowMcpCli.gradingReload( { cwd, target: 'demoapi/demoapi', gradingDataDir: '.flowmcp/grading', withKeys: false, json: true } ) + expect( result.status ).toBe( true ) + expect( result.mode ).toBe( 'reload' ) + expect( result.schemaCount ).toBeGreaterThanOrEqual( 1 ) + expect( grading.__stub.lastCall.force ).toBe( true ) + // reload never writes grade.json / index.json (no rollup fields) + expect( result.indexPath ).toBeUndefined() + expect( result.proofPath ).toBeUndefined() + } ) + + it( 'rejects a tool-ID target (namespace or schema only)', async () => { + const cwd = await freshCwd() + const grading = gradingWithStubbedPretest( { ok: true } ) + await importFixture( { cwd, grading } ) + + const { result } = await FlowMcpCli.gradingReload( { cwd, target: 'demoapi/tool/getThing', gradingDataDir: '.flowmcp/grading', withKeys: false, json: true } ) + expect( result.status ).toBe( false ) + expect( result.error ).toContain( 'namespace or a schema-ID' ) + } ) +} ) From be07a34eba612c77c15f77f30d2123e6f6459de8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=B1=CE=B7dr3=CE=B15=20=C9=AE=CE=B1=CE=B7=C9=A6=D6=85l?= =?UTF-8?q?=CA=903=CA=80?= Date: Fri, 5 Jun 2026 18:07:07 +0200 Subject: [PATCH 2/5] Emit ONE self-contained grading skill Phase 3 (Memo 110): the emit returns a single self-contained Emit-Skill. - #buildEmitSubstitutions feeds real repo-relative schema paths + tool/ namespace names into the composed area prompts (no torso) - #buildEmitSkill assembles ONE skill text: self-describing header, the bundled READY-stage areas, and the Task-ID + exact --consume-scores return command IN THE TEXT (Kap 10.1); gated stage-2 areas named for the follow-up emit - maxTurns configurable (--max-turns, default 25; was hardcoded) - tests for emit-skill text, maxTurns flow + malformed-value guard Co-Authored-By: Claude Opus 4.8 (1M context) --- src/index.mjs | 4 +- src/task/FlowMcpCli.mjs | 162 +++++++++++++++++++++-- tests/unit/grading-phase2-areas.test.mjs | 51 +++++++ 3 files changed, 208 insertions(+), 9 deletions(-) diff --git a/src/index.mjs b/src/index.mjs index ba787f2..259e619 100755 --- a/src/index.mjs +++ b/src/index.mjs @@ -54,6 +54,7 @@ const args = parseArgs( { 'grading-data': { type: 'string' }, 'export-dir': { type: 'string' }, 'max-iterations': { type: 'string' }, + 'max-turns': { type: 'string' }, 'with-keys': { type: 'boolean' }, 'set-data-dir': { type: 'string' }, 'set-export-dir': { type: 'string' } @@ -453,6 +454,7 @@ const runCommand = async () => { const gradingDataDir = values[ 'grading-data' ] === undefined ? null : values[ 'grading-data' ] const gradingExportDir = values[ 'export-dir' ] === undefined ? null : values[ 'export-dir' ] const maxIterations = values[ 'max-iterations' ] === undefined ? null : values[ 'max-iterations' ] + const maxTurns = values[ 'max-turns' ] === undefined ? null : values[ 'max-turns' ] const withKeys = values[ 'with-keys' ] === true const only = values[ 'only' ] === undefined ? null : values[ 'only' ] const json = values[ 'json' ] === true @@ -493,7 +495,7 @@ const runCommand = async () => { // mechanic. Both share the exact same gradingRun() implementation (no // code drift). The mode (--emit-prompts | --consume-scores) is still // explicit — no silent default. - const { result } = await FlowMcpCli.gradingRun( { cwd, target, phase, emitPrompts, consumeScores, onConflict, memberSource, gradingDataDir, gradingExportDir, maxIterations, withKeys, dryRun, json } ) + const { result } = await FlowMcpCli.gradingRun( { cwd, target, phase, emitPrompts, consumeScores, onConflict, memberSource, gradingDataDir, gradingExportDir, maxIterations, maxTurns, withKeys, dryRun, json } ) output( { result } ) return true diff --git a/src/task/FlowMcpCli.mjs b/src/task/FlowMcpCli.mjs index 539b6c2..94924e8 100644 --- a/src/task/FlowMcpCli.mjs +++ b/src/task/FlowMcpCli.mjs @@ -12538,7 +12538,7 @@ allowlist, migrate-config, etc.). } - static async gradingRun( { cwd, target, phase, emitPrompts, consumeScores, onConflict, memberSource, gradingDataDir, gradingExportDir, maxIterations, withKeys, dryRun = false, json } ) { + static async gradingRun( { cwd, target, phase, emitPrompts, consumeScores, onConflict, memberSource, gradingDataDir, gradingExportDir, maxIterations, maxTurns = null, withKeys, dryRun = false, json } ) { const grading = await FlowMcpCli.#loadGradingModule() if( grading === null ) { return { 'result': FlowMcpCli.#error( { 'error': 'grading module unavailable', 'fix': 'npm install / update the flowmcp-grading dependency' } ) } @@ -12551,6 +12551,14 @@ allowlist, migrate-config, etc.). return { 'result': FlowMcpCli.#error( { 'error': maxIterationsError, 'fix': 'Pass --max-iterations as a positive integer (default 1).' } ) } } + // PRD-3.5 — the Goal-Block turn bound is configurable (was hardcoded 25). NO + // SILENT DEFAULT: absent -> 25 (the documented default); a supplied value must + // parse to a positive integer. + const { maxTurns: maxTurnsResolved, error: maxTurnsError } = FlowMcpCli.#resolveMaxTurns( { maxTurns } ) + if( maxTurnsError !== null ) { + return { 'result': FlowMcpCli.#error( { 'error': maxTurnsError, 'fix': 'Pass --max-turns as a positive integer (default 25).' } ) } + } + if( typeof target !== 'string' || target.length === 0 ) { return { 'result': FlowMcpCli.#error( { 'error': 'Missing grading target.', 'fix': `Usage: ${appConfig[ 'cliCommand' ]} grading non-deterministic --emit-prompts | --consume-scores ` } ) } } @@ -12630,7 +12638,7 @@ allowlist, migrate-config, etc.). // deterministic pretest fires authenticated requests with live keys. const { useKeys } = await FlowMcpCli.#gradingUseKeys( { withKeys } ) - return FlowMcpCli.#gradingEmitPrompts( { cwd, grading, gradingDataRoot, 'flow': detected.flow, 'tier': detected.tier, 'maxGrade': detected.maxGrade, 'targetDir': detected.targetDir, target, areaSelector, conflict, 'maxIterations': maxIterationsResolved, useKeys, dryRun, 'dependencyChain': deps.chain } ) + return FlowMcpCli.#gradingEmitPrompts( { cwd, grading, gradingDataRoot, 'flow': detected.flow, 'tier': detected.tier, 'maxGrade': detected.maxGrade, 'targetDir': detected.targetDir, target, areaSelector, conflict, 'maxIterations': maxIterationsResolved, 'maxTurns': maxTurnsResolved, useKeys, dryRun, 'dependencyChain': deps.chain } ) } return FlowMcpCli.#gradingConsumeScores( { cwd, grading, gradingDataRoot, 'flow': detected.flow, 'targetDir': detected.targetDir, target, consumeScores, conflict, gradingDataDir, gradingExportDir, dryRun, 'dependencyChain': deps.chain } ) @@ -13272,6 +13280,22 @@ allowlist, migrate-config, etc.). } + // PRD-3.5 — resolve the configurable Goal-Block turn bound. Absent -> 25 (the + // documented default); a supplied value must be a positive integer. NO SILENT + // DEFAULT for a malformed value (it errors rather than falling back to 25). + static #resolveMaxTurns( { maxTurns } ) { + if( maxTurns === null || maxTurns === undefined ) { + return { 'maxTurns': 25, 'error': null } + } + const parsed = Number( maxTurns ) + if( Number.isInteger( parsed ) === false || parsed < 1 ) { + return { 'maxTurns': null, 'error': `Invalid --max-turns value: ${maxTurns}` } + } + + return { 'maxTurns': parsed, 'error': null } + } + + // PRD-004 — resolve the --phase flag into a multi-area selector. Three explicit // modes, no silent default: // absent -> { mode: 'default', areas: null } (all applicable) @@ -13320,22 +13344,124 @@ allowlist, migrate-config, etc.). // Compose one prompt per grading area via the AreaPromptLoader (Memo 097 // Kap. 9.0). The loader reuses PromptBuilder.build and resolves the package- // local prompts/ tree itself, so the CLI does not guess paths. - static async #composeGradingAreas( { grading, flow } ) { + static async #composeGradingAreas( { grading, flow, substitutions = null } ) { const AreaPromptLoader = grading[ 'AreaPromptLoader' ] if( AreaPromptLoader === undefined || AreaPromptLoader === null ) { throw new Error( 'AreaPromptLoader unavailable from flowmcp-grading — update the dependency.' ) } const { promptsRoot } = AreaPromptLoader.getPromptsRoot() - const { areas } = await AreaPromptLoader.loadAllAreas( { promptsRoot, flow } ) + // PRD-3.2: pass the substitution context so the composed prompts carry real + // schema paths + tool/namespace names (no torso). A null context keeps the + // legacy placeholder behaviour (back-compat for callers without schema data). + const { areas } = await AreaPromptLoader.loadAllAreas( { promptsRoot, flow, substitutions } ) return { areas } } + // PRD-3.2 — build the emit substitution context for a provider. Paths are + // REPO-RELATIVE (git-security: never leak an absolute path into the emitted + // artifact). The single-test/tools-aggregate areas are bundled across the + // namespace, so {{TOOL_NAME}} resolves to the joined declared tool list and + // {{SCHEMA_NAME}} to the schema name (single schema) or the namespace. + static #buildEmitSubstitutions( { cwd, namespace, liveSchemas, pretests } ) { + const allTools = liveSchemas + .flatMap( ( s ) => { + const tools = s.main[ 'tools' ] || s.main[ 'routes' ] || {} + return Object.keys( tools ) + } ) + const toolName = allTools.length > 0 ? allTools.join( ', ' ) : namespace + const schemaName = liveSchemas.length === 1 ? liveSchemas[ 0 ].schemaName : namespace + const firstSchema = liveSchemas[ 0 ] + const schemaPath = firstSchema !== undefined && typeof firstSchema.sourcePath === 'string' + ? FlowMcpCli.#toRepoRelativePath( { cwd, 'path': firstSchema.sourcePath } ) + : `providers/${namespace}` + const firstPretest = pretests.find( ( p ) => typeof p.summaryPath === 'string' ) + const responseFixturePath = firstPretest !== undefined + ? FlowMcpCli.#toRepoRelativePath( { cwd, 'path': firstPretest.summaryPath } ) + : `providers/${namespace}` + + return { namespace, schemaName, toolName, schemaPath, responseFixturePath } + } + + + // PRD-3.3/3.4 — assemble the ONE self-contained Emit-Skill text. Bundles every + // READY area (non-null prompt = the currently-emittable stage) into a single + // instruction a subagent works in one pass, and writes the Task-ID + the exact + // --consume-scores return command INTO the text (Kap 10.1). Hard-gated stage-2 + // areas are named so the operator knows a follow-up emit is due once every + // schema is deterministic-green. + static #buildEmitSkill( { target, flow, namespace, taskId, emittedAreas, gatedAreas, payloadSkeleton } ) { + const ready = emittedAreas + .filter( ( a ) => typeof a.prompt === 'string' && a.prompt.length > 0 ) + const deferred = emittedAreas + .filter( ( a ) => a.prompt === null || a.prompt === undefined ) + .map( ( a ) => a.area ) + + const header = [ + '# Grading Emit-Skill', + '', + 'This is a grading skill. Hand it to a subagent and follow the steps in', + 'order. Work the bundled areas below in a SINGLE pass, then return your', + 'results via the command at the end. Answer only from the files you read —', + 'no web research, no assumptions.', + '', + `- Target: \`${target}\` (flow: ${flow}, namespace: ${namespace})`, + `- Task-ID: \`${taskId}\``, + `- Ready areas this pass: ${ready.map( ( a ) => a.area ).join( ', ' ) || '(none)'}` + ].join( '\n' ) + + const gatedNote = ( Array.isArray( gatedAreas ) ? gatedAreas : [] ).length > 0 + ? [ + '', + '## Gated areas (NOT in this pass)', + '', + 'These stage-2 areas are emitted in a FOLLOW-UP skill once every schema', + 'of the namespace is deterministic-green — do not attempt them now:', + ...( gatedAreas.map( ( g ) => `- ${typeof g === 'string' ? g : ( g.area === undefined ? JSON.stringify( g ) : `${g.area} (${g.reason === undefined ? 'gated' : g.reason})` )}` ) ) + ].join( '\n' ) + : '' + + const deferredNote = deferred.length > 0 + ? `\n\n## Deferred areas\n\nComposed by the harness with the resolved persona (not in this text): ${deferred.join( ', ' )}.` + : '' + + const areaBlocks = ready + .map( ( a ) => `\n\n---\n\n## Area: ${a.area}\n\n${a.prompt}` ) + .join( '' ) + + const returnContract = [ + '', + '', + '---', + '', + '## Return your results', + '', + 'Produce ONE result object per area in this pass. Shape:', + '', + '```json', + JSON.stringify( payloadSkeleton, null, 2 ), + '```', + '', + 'When finished, return the filled array by running:', + '', + '```bash', + `flowmcp grading non-deterministic ${namespace} --consume-scores `, + '```', + '', + `The results file MUST carry this Task-ID (\`${taskId}\`) so consume-scores can`, + 'match your answers to this emit. Provide exactly the asked number of results', + 'per area — no more, no fewer.' + ].join( '\n' ) + + return `${header}${gatedNote}${deferredNote}${areaBlocks}${returnContract}\n` + } + + // Stage 1 — deterministic: Phase-0/1 wiring -> DataPretest.run -> emit the // /goal handoff (prompts.json + state.json baton). The CLI does NOT run // Agent() — Stage 2 lives in the harness. - static async #gradingEmitPrompts( { cwd, grading, gradingDataRoot, flow, tier, maxGrade, targetDir, target, areaSelector, conflict, maxIterations, useKeys, dryRun = false, dependencyChain } ) { + static async #gradingEmitPrompts( { cwd, grading, gradingDataRoot, flow, tier, maxGrade, targetDir, target, areaSelector, conflict, maxIterations, maxTurns = 25, useKeys, dryRun = false, dependencyChain } ) { const namespace = basename( targetDir ) const promptsPath = join( targetDir, 'prompts.json' ) const statePath = join( targetDir, 'state.json' ) @@ -13432,9 +13558,11 @@ allowlist, migrate-config, etc.). // Goal-Block (PromptBuilder) — the completion condition + surfacing // convention that drives the harness /goal loop (Stage 2). - const { goalBlock, condition, maxTurns } = grading[ 'PromptBuilder' ].buildGoalBlock( { + // PRD-3.5 — maxTurns is the configurable Goal-Block turn bound (default 25, + // resolved by the caller). buildGoalBlock echoes it back in `condition`. + const { goalBlock, condition } = grading[ 'PromptBuilder' ].buildGoalBlock( { 'condition': `Grade the ${flow} "${target}" (tier ${tier}, max grade ${maxGrade}) across all required areas until every area reaches a stable grade`, - 'maxTurns': 25 + 'maxTurns': maxTurns } ) // Memo 097 Kap. 9.0 fix #1: compose ONE prompt per area via the @@ -13442,7 +13570,13 @@ allowlist, migrate-config, etc.). // goalBlock. Neutral areas are composed deterministically here; persona- // required areas are surfaced as deferred entries (harness composes them // with the resolved domain/selection persona — no invented persona). - const { areas } = await FlowMcpCli.#composeGradingAreas( { grading, flow } ) + // PRD-3.2: a substitution context fills the real schema path + tool/namespace + // names into the neutral composed prompts (no {{…}} torso). Repo-relative + // paths only — never leak an absolute path into the emitted artifact. + const substitutions = flow === 'provider' + ? FlowMcpCli.#buildEmitSubstitutions( { cwd, namespace, liveSchemas, pretests } ) + : null + const { areas } = await FlowMcpCli.#composeGradingAreas( { grading, flow, substitutions } ) // PRD-005/006/004 — derive the FINAL emitted area set from the composed // areas: applicability pre-filter (optional-area precondition absent -> @@ -13470,6 +13604,15 @@ allowlist, migrate-config, etc.). const taskId = taskResult.taskId const payloadSkeleton = { taskId, 'areas': emittedAreas.map( ( a ) => ( { 'area': a.area, 'results': [] } ) ) } + // PRD-3.3/3.4 — assemble ONE self-contained Emit-Skill text: a self-describing + // header, the bundled READY (non-null prompt) areas, and the Task-ID + + // --consume-scores return contract IN THE TEXT (not only as JSON siblings). + // Hard-gated stage-2 areas are named so the operator knows a follow-up emit + // is needed once every schema is deterministic-green. + const emitSkill = FlowMcpCli.#buildEmitSkill( { + target, flow, namespace, taskId, emittedAreas, gatedAreas, payloadSkeleton + } ) + const now = new Date().toISOString() const promptsDoc = { target, @@ -13481,6 +13624,7 @@ allowlist, migrate-config, etc.). maxIterations, taskId, 'goal': { condition, maxTurns, goalBlock }, + emitSkill, 'areas': emittedAreas, skippedAreas, gatedAreas, @@ -13534,6 +13678,7 @@ allowlist, migrate-config, etc.). 'statePath': null, 'pretestCount': pretests.length, taskId, + emitSkill, 'areaSelector': { 'mode': areaSelector.mode, 'areas': areaSelector.areas }, 'emittedAreaSet': emittedAreaSet, skippedAreas, @@ -13564,6 +13709,7 @@ allowlist, migrate-config, etc.). statePath, 'pretestCount': pretests.length, taskId, + emitSkill, 'areaSelector': { 'mode': areaSelector.mode, 'areas': areaSelector.areas }, 'emittedAreaSet': emittedAreaSet, skippedAreas, diff --git a/tests/unit/grading-phase2-areas.test.mjs b/tests/unit/grading-phase2-areas.test.mjs index d9472ec..6f15545 100644 --- a/tests/unit/grading-phase2-areas.test.mjs +++ b/tests/unit/grading-phase2-areas.test.mjs @@ -394,3 +394,54 @@ describe( 'PRD-008 — grade.json produced from the consume-scores success path' expect( after.monitoring.githubIssue ).toBe( 4242 ) } ) } ) + + +// ---- Memo 110 P3: self-contained Emit-Skill + configurable maxTurns ------------ +describe( 'Memo 110 P3 — Emit-Skill text (PRD-3.3/3.4) + maxTurns (PRD-3.5)', () => { + afterEach( () => { FlowMcpCli.__testInjectGrading( { grading: null } ) } ) + + it( 'emits ONE self-contained emit-skill with Task-ID + consume command in the text', async () => { + const cwd = await freshCwd() + FlowMcpCli.__testInjectGrading( { grading: gradingWithStubbedPretest( { ok: true } ) } ) + + const { result } = await emit( { cwd, phase: 'single-test,tools-aggregate-schema' } ) + expect( result.status ).toBe( true ) + expect( typeof result.emitSkill ).toBe( 'string' ) + // self-describing header + Task-ID in the text + expect( result.emitSkill ).toContain( 'Grading Emit-Skill' ) + expect( result.emitSkill ).toContain( result.taskId ) + // the return contract names the exact consume command in the text + expect( result.emitSkill ).toContain( 'grading non-deterministic demoapi --consume-scores' ) + // the ready-area prompts are filled — no NAME torso survives + expect( result.emitSkill.includes( '{{NAMESPACE}}' ) ).toBe( false ) + expect( result.emitSkill.includes( '{{TOOL_NAME}}' ) ).toBe( false ) + expect( result.emitSkill.includes( '{{OUTPUT_SCHEMA_REF}}' ) ).toBe( false ) + } ) + + it( 'maxTurns is configurable (flows into the Goal-Block condition)', async () => { + const cwd = await freshCwd() + FlowMcpCli.__testInjectGrading( { grading: gradingWithStubbedPretest( { ok: true } ) } ) + + await FlowMcpCli.gradingRun( { + cwd, gradingDataDir: '.flowmcp/grading', target: 'demoapi', + phase: 'single-test', emitPrompts: true, consumeScores: null, onConflict: null, + maxIterations: null, maxTurns: '7', json: false + } ) + const prompts = JSON.parse( await readFile( join( cwd, '.flowmcp/grading/providers/demoapi/prompts.json' ), 'utf-8' ) ) + expect( prompts.goal.maxTurns ).toBe( 7 ) + expect( prompts.goal.condition ).toContain( 'stop after 7 turns' ) + } ) + + it( 'rejects a malformed --max-turns (no silent fallback to 25)', async () => { + const cwd = await freshCwd() + FlowMcpCli.__testInjectGrading( { grading: gradingWithStubbedPretest( { ok: true } ) } ) + + const { result } = await FlowMcpCli.gradingRun( { + cwd, gradingDataDir: '.flowmcp/grading', target: 'demoapi', + phase: 'single-test', emitPrompts: true, consumeScores: null, onConflict: null, + maxIterations: null, maxTurns: 'lots', json: false + } ) + expect( result.status ).toBe( false ) + expect( result.error ).toContain( 'Invalid --max-turns' ) + } ) +} ) From 4720a071150568fafb5c814787cad716a82d16fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=B1=CE=B7dr3=CE=B15=20=C9=AE=CE=B1=CE=B7=C9=A6=D6=85l?= =?UTF-8?q?=CA=903=CA=80?= Date: Fri, 5 Jun 2026 18:11:47 +0200 Subject: [PATCH 3/5] Add grading progress (stderr) + --quiet + summary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 4 (Memo 110): observability (Befund B). - #emitProgress ticks per-schema/per-tool to STDERR during the slow deterministic/emit/reload runs; --quiet silences it - stdout machine JSON is untouched (piped `... | jq` stays pure) - printDeterministicSummary: concise human summary to stderr, gated by !quiet && !json; no machine key dropped (rollup/proof consumers depend on them) — summary is additive - tests: stderr ticks vs --quiet, summary gating Co-Authored-By: Claude Opus 4.8 (1M context) --- src/index.mjs | 13 +++- src/task/FlowMcpCli.mjs | 88 ++++++++++++++++++++--- tests/unit/grading-deterministic.test.mjs | 48 +++++++++++++ 3 files changed, 135 insertions(+), 14 deletions(-) diff --git a/src/index.mjs b/src/index.mjs index 259e619..b1eb72d 100755 --- a/src/index.mjs +++ b/src/index.mjs @@ -39,6 +39,7 @@ const args = parseArgs( { 'consume-scores': { type: 'string' }, 'on-conflict': { type: 'string' }, 'no-save': { type: 'boolean' }, + 'quiet': { type: 'boolean' }, 'help': { type: 'boolean', short: 'h' }, 'strict': { type: 'boolean' }, 'fix-template': { type: 'boolean' }, @@ -465,10 +466,16 @@ const runCommand = async () => { // PRD-2.2 — --force bypasses the read-cache (PRD-2.1): re-fetch the test // data instead of reusing the persisted test-N.json. const force = values[ 'force' ] === true + // PRD-4.1 — --quiet silences the stderr progress; stdout JSON is unaffected. + const quiet = values[ 'quiet' ] === true if( subCommand === 'deterministic' ) { - const { result } = await FlowMcpCli.gradingDeterministic( { cwd, target, gradingDataDir, gradingExportDir, withKeys, only, dryRun, force, json } ) + const { result } = await FlowMcpCli.gradingDeterministic( { cwd, target, gradingDataDir, gradingExportDir, withKeys, only, dryRun, force, quiet, json } ) output( { result } ) + // PRD-4.2 — a concise human summary to STDERR (not on stdout, so a piped + // `... | jq` stays pure machine JSON). Suppressed by --quiet and by --json + // (pure machine mode). + FlowMcpCli.printDeterministicSummary( { result, quiet, json } ) return true } @@ -476,7 +483,7 @@ const runCommand = async () => { if( subCommand === 'reload' ) { // PRD-2.3 — re-fetch + rewrite the persisted test-N.json only (force), // decoupled from grading: no _gradings/grade.json writes. - const { result } = await FlowMcpCli.gradingReload( { cwd, target, gradingDataDir, withKeys, json } ) + const { result } = await FlowMcpCli.gradingReload( { cwd, target, gradingDataDir, withKeys, quiet, json } ) output( { result } ) return true @@ -495,7 +502,7 @@ const runCommand = async () => { // mechanic. Both share the exact same gradingRun() implementation (no // code drift). The mode (--emit-prompts | --consume-scores) is still // explicit — no silent default. - const { result } = await FlowMcpCli.gradingRun( { cwd, target, phase, emitPrompts, consumeScores, onConflict, memberSource, gradingDataDir, gradingExportDir, maxIterations, maxTurns, withKeys, dryRun, json } ) + const { result } = await FlowMcpCli.gradingRun( { cwd, target, phase, emitPrompts, consumeScores, onConflict, memberSource, gradingDataDir, gradingExportDir, maxIterations, maxTurns, withKeys, dryRun, quiet, json } ) output( { result } ) return true diff --git a/src/task/FlowMcpCli.mjs b/src/task/FlowMcpCli.mjs index 94924e8..61711de 100644 --- a/src/task/FlowMcpCli.mjs +++ b/src/task/FlowMcpCli.mjs @@ -9665,6 +9665,56 @@ allowlist, migrate-config, etc.). } + // PRD-4.1 — progress sink. Grading runs are slow (per-schema live pretests), so + // by default we tick to STDERR while running (Befund B: no progress). The machine + // JSON stays on STDOUT untouched. --quiet (quiet === true) silences the ticks. + // Stderr is chosen so a piped `... | jq` on stdout is never polluted. + static #emitProgress( { quiet, message } ) { + if( quiet === true ) { return } + process.stderr.write( `[grading] ${message}\n` ) + } + + + // PRD-4.2 — concise human summary for a deterministic result, to STDERR. + // JSON-shape audit conclusion: NO machine key is dropped/renamed (rollup + + // Provider-Proof consumers depend on them); this summary is ADDITIVE and lives + // on stderr so a piped stdout stays pure JSON. Suppressed by --quiet and by + // --json (pure machine mode). Handles both the single-schema and namespace shapes. + static printDeterministicSummary( { result, quiet, json } ) { + if( quiet === true || json === true || result === null || result === undefined ) { return } + if( result[ 'status' ] === false && result[ 'mode' ] === undefined ) { + process.stderr.write( `[grading] error: ${result[ 'error' ]}\n` ) + return + } + + const lines = [] + const verdict = result[ 'status' ] === true ? 'PASS' : 'FAIL' + lines.push( `[grading] ${result[ 'target' ]} — ${verdict}` ) + + if( Array.isArray( result[ 'schemas' ] ) === true ) { + const passed = result[ 'schemas' ].filter( ( s ) => s[ 'status' ] === true ).length + lines.push( `[grading] schemas: ${passed}/${result[ 'schemaCount' ]} green` ) + } else if( result[ 'pretest' ] !== undefined && result[ 'pretest' ] !== null ) { + const pretest = result[ 'pretest' ] + const validateOk = result[ 'validate' ] !== undefined && result[ 'validate' ] !== null ? result[ 'validate' ][ 'status' ] === true : null + lines.push( `[grading] validate: ${validateOk === null ? 'n/a' : ( validateOk ? 'ok' : 'fail' )} pretest: ${pretest[ 'ok' ] === true ? 'ok' : ( pretest[ 'keyGated' ] === true ? 'key-gated' : 'fail' )}` ) + const stamp = pretest[ 'fromCache' ] === true ? `cached (data ${pretest[ 'dataAt' ]})` : `fresh (data ${pretest[ 'dataAt' ]})` + lines.push( `[grading] data: ${stamp}` ) + const below = Array.isArray( pretest[ 'toolsBelowThreshold' ] ) ? pretest[ 'toolsBelowThreshold' ] : [] + if( below.length > 0 ) { lines.push( `[grading] below bar: ${below.join( ', ' )}` ) } + } + + if( result[ 'rollupGrade' ] !== undefined ) { + lines.push( `[grading] grade: ${result[ 'rollupGrade' ]} (${result[ 'rollupStatus' ]})` ) + } + if( result[ 'rollupError' ] !== undefined ) { + lines.push( `[grading] rollup error: ${result[ 'rollupError' ]}` ) + } + + process.stderr.write( lines.join( '\n' ) + '\n' ) + } + + static async #requireInit() { const globalConfigPath = FlowMcpCli.#globalConfigPath() const { data: globalConfig } = await FlowMcpCli.#readJson( { filePath: globalConfigPath } ) @@ -12538,7 +12588,7 @@ allowlist, migrate-config, etc.). } - static async gradingRun( { cwd, target, phase, emitPrompts, consumeScores, onConflict, memberSource, gradingDataDir, gradingExportDir, maxIterations, maxTurns = null, withKeys, dryRun = false, json } ) { + static async gradingRun( { cwd, target, phase, emitPrompts, consumeScores, onConflict, memberSource, gradingDataDir, gradingExportDir, maxIterations, maxTurns = null, withKeys, dryRun = false, quiet = false, json } ) { const grading = await FlowMcpCli.#loadGradingModule() if( grading === null ) { return { 'result': FlowMcpCli.#error( { 'error': 'grading module unavailable', 'fix': 'npm install / update the flowmcp-grading dependency' } ) } @@ -12638,7 +12688,7 @@ allowlist, migrate-config, etc.). // deterministic pretest fires authenticated requests with live keys. const { useKeys } = await FlowMcpCli.#gradingUseKeys( { withKeys } ) - return FlowMcpCli.#gradingEmitPrompts( { cwd, grading, gradingDataRoot, 'flow': detected.flow, 'tier': detected.tier, 'maxGrade': detected.maxGrade, 'targetDir': detected.targetDir, target, areaSelector, conflict, 'maxIterations': maxIterationsResolved, 'maxTurns': maxTurnsResolved, useKeys, dryRun, 'dependencyChain': deps.chain } ) + return FlowMcpCli.#gradingEmitPrompts( { cwd, grading, gradingDataRoot, 'flow': detected.flow, 'tier': detected.tier, 'maxGrade': detected.maxGrade, 'targetDir': detected.targetDir, target, areaSelector, conflict, 'maxIterations': maxIterationsResolved, 'maxTurns': maxTurnsResolved, useKeys, dryRun, quiet, 'dependencyChain': deps.chain } ) } return FlowMcpCli.#gradingConsumeScores( { cwd, grading, gradingDataRoot, 'flow': detected.flow, 'targetDir': detected.targetDir, target, consumeScores, conflict, gradingDataDir, gradingExportDir, dryRun, 'dependencyChain': deps.chain } ) @@ -12659,7 +12709,7 @@ allowlist, migrate-config, etc.). // selection-member run through the existing structural primitive check // (#runTypedTests + #aggregateByPrimitive). The same #validateOnlyFilter // allowlist applies (no duplication). - static async gradingDeterministic( { cwd, target, gradingDataDir, gradingExportDir = null, withKeys, only, dryRun = false, force = false, json, skipRollup = false } ) { + static async gradingDeterministic( { cwd, target, gradingDataDir, gradingExportDir = null, withKeys, only, dryRun = false, force = false, quiet = false, json, skipRollup = false } ) { const grading = await FlowMcpCli.#loadGradingModule() if( grading === null || grading[ 'DataPretest' ] === undefined ) { return { 'result': FlowMcpCli.#error( { 'error': 'grading module unavailable', 'fix': 'npm install / update the flowmcp-grading dependency' } ) } @@ -12688,7 +12738,7 @@ allowlist, migrate-config, etc.). // namespace rollup (index.json) + Provider-Proof (grade.json). Delegated so // the single-schema path below stays unchanged. if( parsed.type === 'namespace' ) { - return FlowMcpCli.#gradingDeterministicNamespace( { cwd, 'namespace': parsed.namespace, gradingDataDir, gradingExportDir, withKeys, only, dryRun, force, json } ) + return FlowMcpCli.#gradingDeterministicNamespace( { cwd, 'namespace': parsed.namespace, gradingDataDir, gradingExportDir, withKeys, only, dryRun, force, quiet, json } ) } if( parsed.type !== 'schema' && parsed.type !== 'tool' && parsed.type !== 'test' ) { return { 'result': FlowMcpCli.#error( { 'error': `Spec-ID type "${parsed.type}" is not supported by grading deterministic (only namespace, schema-ID, tool-ID or per-test).`, 'fix': 'Use "", "/", "/tool/" or "/tool//tests/".' } ) } @@ -12747,6 +12797,9 @@ allowlist, migrate-config, etc.). // PRD-012 — --no-save (dryRun) runs the pretest in full but persists NOTHING // to the island (no summary.json / test-N.json). The deterministic path has // no Stage-3 writes, so dryRun here only gates the DataPretest persist. + // PRD-4.1 — tick the slow part (live/cached pretest) to stderr. + FlowMcpCli.#emitProgress( { quiet, 'message': `${target}: structural validate + data pretest${force === true ? ' (--force re-fetch)' : ''}...` } ) + // PRD-2.2 — force threads the cache bypass into the pretest. Without it the // pretest reuses the persisted test-N.json (read-cache, PRD-2.1); with it the // data is re-fetched. A re-fetch that flips `deterministic-green` flows @@ -12809,6 +12862,10 @@ allowlist, migrate-config, etc.). const { hints } = FlowMcpCli.#deterministicHints( { 'validate': validate, pretest } ) const status = validate[ 'status' ] === true && pretest.ok === true + // PRD-4.1 — done-tick with the data stamp (cache reuse vs re-fetch). + const stamp = pretest.fromCache === true ? `cached, data ${pretest.dataAt}` : `fresh, data ${pretest.dataAt}` + FlowMcpCli.#emitProgress( { quiet, 'message': `${target}: ${status === true ? 'PASS' : 'FAIL'} (${stamp})` } ) + const result = { status, 'mode': 'deterministic', @@ -12949,17 +13006,21 @@ allowlist, migrate-config, etc.). // Memo 107 PRD-004 — bare-namespace deterministic grade: run every schema of the // namespace (skipRollup, so each writes its own `_gradings/` but defers the rollup), // then build the namespace index.json + Provider-Proof grade.json EXACTLY ONCE. - static async #gradingDeterministicNamespace( { cwd, namespace, gradingDataDir, gradingExportDir, withKeys, only, dryRun, force = false, json } ) { + static async #gradingDeterministicNamespace( { cwd, namespace, gradingDataDir, gradingExportDir, withKeys, only, dryRun, force = false, quiet = false, json } ) { const resolved = await FlowMcpCli.#resolveSchemasForTarget( { namespace } ) if( resolved.status === false ) { return { 'result': FlowMcpCli.#error( { 'error': resolved.error, 'fix': resolved.fix } ) } } + const total = resolved.schemas.length + FlowMcpCli.#emitProgress( { quiet, 'message': `namespace ${namespace}: ${total} schema(s) to grade deterministically` } ) + const perSchema = [] await resolved.schemas - .reduce( ( promise, schema ) => promise.then( async () => { + .reduce( ( promise, schema, index ) => promise.then( async () => { + FlowMcpCli.#emitProgress( { quiet, 'message': `[${index + 1}/${total}] ${schema.schemaName}` } ) const sub = await FlowMcpCli.gradingDeterministic( { - cwd, 'target': `${namespace}/${schema.schemaName}`, gradingDataDir, gradingExportDir, withKeys, only, dryRun, force, json, 'skipRollup': true + cwd, 'target': `${namespace}/${schema.schemaName}`, gradingDataDir, gradingExportDir, withKeys, only, dryRun, force, 'quiet': true, json, 'skipRollup': true } ) const subResult = sub.result perSchema.push( { @@ -13003,7 +13064,7 @@ allowlist, migrate-config, etc.). // island test data is refreshed, but it writes NO `_gradings/` entries and NO // grade.json/index.json — a pure data reload. Reports the per-schema rewritten // test counts + the new data stamp (dataAt). NO SILENT DEFAULTS. - static async gradingReload( { cwd, target, gradingDataDir, withKeys, json } ) { + static async gradingReload( { cwd, target, gradingDataDir, withKeys, quiet = false, json } ) { const grading = await FlowMcpCli.#loadGradingModule() if( grading === null || grading[ 'DataPretest' ] === undefined ) { return { 'result': FlowMcpCli.#error( { 'error': 'grading module unavailable', 'fix': 'npm install / update the flowmcp-grading dependency' } ) } @@ -13035,8 +13096,11 @@ allowlist, migrate-config, etc.). const envObject = useKeys === true ? ( await FlowMcpCli.#resolveEnv( { cwd } ) ).envObject : {} const perSchema = [] + const reloadTotal = targetSchemas.length + FlowMcpCli.#emitProgress( { quiet, 'message': `reload ${target}: re-fetch ${reloadTotal} schema(s)...` } ) await targetSchemas - .reduce( ( promise, schema ) => promise.then( async () => { + .reduce( ( promise, schema, index ) => promise.then( async () => { + FlowMcpCli.#emitProgress( { quiet, 'message': `[${index + 1}/${reloadTotal}] reload ${schema.schemaName}` } ) const requiredServerParams = Array.isArray( schema.main[ 'requiredServerParams' ] ) ? schema.main[ 'requiredServerParams' ] : [] const serverParams = useKeys === true ? FlowMcpCli.#buildServerParams( { envObject, requiredServerParams } ).serverParams @@ -13461,7 +13525,7 @@ allowlist, migrate-config, etc.). // Stage 1 — deterministic: Phase-0/1 wiring -> DataPretest.run -> emit the // /goal handoff (prompts.json + state.json baton). The CLI does NOT run // Agent() — Stage 2 lives in the harness. - static async #gradingEmitPrompts( { cwd, grading, gradingDataRoot, flow, tier, maxGrade, targetDir, target, areaSelector, conflict, maxIterations, maxTurns = 25, useKeys, dryRun = false, dependencyChain } ) { + static async #gradingEmitPrompts( { cwd, grading, gradingDataRoot, flow, tier, maxGrade, targetDir, target, areaSelector, conflict, maxIterations, maxTurns = 25, useKeys, dryRun = false, quiet = false, dependencyChain } ) { const namespace = basename( targetDir ) const promptsPath = join( targetDir, 'prompts.json' ) const statePath = join( targetDir, 'state.json' ) @@ -13512,9 +13576,11 @@ allowlist, migrate-config, etc.). // emitted AREA prompts later, not the pretest pass. const pretestUnits = liveSchemas + FlowMcpCli.#emitProgress( { quiet, 'message': `emit ${target}: data pretest over ${pretestUnits.length} schema(s)...` } ) await pretestUnits - .reduce( ( promise, unit ) => promise.then( async () => { + .reduce( ( promise, unit, index ) => promise.then( async () => { const { schemaName, main, handlersFn, sourcePath } = unit + FlowMcpCli.#emitProgress( { quiet, 'message': `[${index + 1}/${pretestUnits.length}] pretest ${schemaName}` } ) if( main === null || main === undefined ) { pretests.push( { schemaName, 'ok': false, 'errors': [ `cannot load schema source for ${schemaName}` ] } ) return diff --git a/tests/unit/grading-deterministic.test.mjs b/tests/unit/grading-deterministic.test.mjs index 6b2b519..0ff50f9 100644 --- a/tests/unit/grading-deterministic.test.mjs +++ b/tests/unit/grading-deterministic.test.mjs @@ -408,3 +408,51 @@ describe( 'gradingReload — data reload only (Memo 110 PRD-2.3)', () => { expect( result.error ).toContain( 'namespace or a schema-ID' ) } ) } ) + + +describe( 'gradingDeterministic — progress + summary (Memo 110 P4)', () => { + afterEach( () => { FlowMcpCli.__testInjectGrading( { grading: null } ) } ) + + it( 'ticks progress to STDERR by default, suppressed by --quiet', async () => { + const cwd = await freshCwd() + const grading = gradingWithStubbedPretest( { ok: true } ) + await importFixture( { cwd, grading } ) + + const writes = [] + const spy = ( chunk ) => { writes.push( String( chunk ) ); return true } + const original = process.stderr.write + process.stderr.write = spy + try { + await FlowMcpCli.gradingDeterministic( { cwd, target: 'demoapi/demoapi', gradingDataDir: '.flowmcp/grading', withKeys: false, only: null, json: true } ) + const loud = writes.join( '' ) + writes.length = 0 + await FlowMcpCli.gradingDeterministic( { cwd, target: 'demoapi/demoapi', gradingDataDir: '.flowmcp/grading', withKeys: false, only: null, quiet: true, json: true } ) + const quiet = writes.join( '' ) + + expect( loud ).toContain( '[grading]' ) + expect( loud ).toContain( 'demoapi/demoapi' ) + expect( quiet ).toBe( '' ) + } finally { + process.stderr.write = original + } + } ) + + it( 'printDeterministicSummary writes to stderr only when not quiet/json', () => { + const result = { status: true, mode: 'deterministic', target: 'demoapi/demoapi', validate: { status: true }, pretest: { ok: true, fromCache: true, dataAt: '2026-06-05T10:00:00.000Z', toolsBelowThreshold: [] } } + const writes = [] + const original = process.stderr.write + process.stderr.write = ( chunk ) => { writes.push( String( chunk ) ); return true } + try { + FlowMcpCli.printDeterministicSummary( { result, quiet: false, json: false } ) + FlowMcpCli.printDeterministicSummary( { result, quiet: true, json: false } ) + FlowMcpCli.printDeterministicSummary( { result, quiet: false, json: true } ) + } finally { + process.stderr.write = original + } + const out = writes.join( '' ) + expect( out ).toContain( 'demoapi/demoapi — PASS' ) + expect( out ).toContain( 'cached' ) + // only the first call (not quiet, not json) emitted -> exactly one summary block + expect( out.match( /— PASS/g ).length ).toBe( 1 ) + } ) +} ) From e5b2b98cfb4b06c816f2a05ea9e8443900194d1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=B1=CE=B7dr3=CE=B15=20=C9=AE=CE=B1=CE=B7=C9=A6=D6=85l?= =?UTF-8?q?=CA=903=CA=80?= Date: Fri, 5 Jun 2026 18:21:45 +0200 Subject: [PATCH 4/5] Route `both`-classified areas to both buckets Phase 5 (Memo 110, I-4): a `both`-classified area carries a free deterministic gate AND an LLM round, so nextAction lists it in deterministicNow AND the non-det area-set. Doctor test reflects the corrected below-deterministic-green behaviour. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/task/FlowMcpCli.mjs | 10 ++++++++-- tests/unit/grading-phase3-doctor.test.mjs | 10 ++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/task/FlowMcpCli.mjs b/src/task/FlowMcpCli.mjs index 61711de..7935493 100644 --- a/src/task/FlowMcpCli.mjs +++ b/src/task/FlowMcpCli.mjs @@ -14806,12 +14806,18 @@ allowlist, migrate-config, etc.). .filter( ( name ) => inFlowScope( name ) === true ) .filter( ( name ) => isApplicable( name ) === true ) - // Split ready areas by their data-driven classification. + // Split ready areas by their data-driven classification. Befund I-4: a + // `both`-classified area carries a deterministic gate (done for free by the + // CLI) AND a non-deterministic LLM round, so it appears in BOTH buckets — the + // free det part is surfaced as deterministicNow, the descriptive questions + // bundle into the non-det emit. `deterministic` -> det only, `non-deterministic` + // -> nonDet only. const classified = readyAreas .reduce( ( acc, name ) => { const c = AreaDependencyGraph.classifyArea( { 'graph': loaded.graph, 'area': name } ) if( c.errors.length > 0 ) { acc.errors.push( c.errors.join( '; ' ) ); return acc } - if( c.classification === 'deterministic' ) { acc.det.push( name ) } else { acc.nonDet.push( name ) } + if( c.classification === 'deterministic' || c.classification === 'both' ) { acc.det.push( name ) } + if( c.classification === 'non-deterministic' || c.classification === 'both' ) { acc.nonDet.push( name ) } return acc }, { 'det': [], 'nonDet': [], 'errors': [] } ) if( classified.errors.length > 0 ) { diff --git a/tests/unit/grading-phase3-doctor.test.mjs b/tests/unit/grading-phase3-doctor.test.mjs index d9ccd5d..a4504bb 100644 --- a/tests/unit/grading-phase3-doctor.test.mjs +++ b/tests/unit/grading-phase3-doctor.test.mjs @@ -200,17 +200,23 @@ describe( 'PRD-009 — worklist is a thin wrapper over the shared collector', () describe( 'PRD-010 — nextAction split (graph-driven, read-only, no emission)', () => { afterEach( () => { FlowMcpCli.__testInjectGrading( { grading: null } ) } ) - it( 'below deterministic-green: namespace areas are gated (cost guard), nonDeterministic is null', async () => { + it( 'below deterministic-green: namespace areas stay gated; the schema-areas form the ready non-det set (Befund I-4)', async () => { const cwd = await freshCwd() await importAndEmit( { cwd, ok: false } ) const { result } = await FlowMcpCli.gradingState( { cwd, gradingDataDir: '.flowmcp/grading', target: 'demoapi', json: true } ) const na = result.nextAction expect( na.status ).toBe( true ) - expect( na.nonDeterministic ).toBeNull() + // single-test / tools-aggregate-schema are `both` (deterministic gate + LLM + // round) and only need structural-valid, so even below deterministic-green + // they are the ready non-det set (this is exactly what --emit-prompts emits). + expect( na.nonDeterministic ).not.toBeNull() + expect( na.nonDeterministic.areaSet ).toContain( 'single-test' ) + // the gated NAMESPACE areas (requiredLevel deterministic-green) are still held. const gatedAreas = na.gated.map( ( g ) => g.area ) expect( gatedAreas ).toContain( 'namespace-description' ) expect( gatedAreas ).toContain( 'namespace-skills' ) + expect( na.nonDeterministic.areaSet ).not.toContain( 'namespace-description' ) na.gated.forEach( ( g ) => { expect( typeof g.reason ).toBe( 'string' ); expect( g.reason.length ).toBeGreaterThan( 0 ) } ) } ) From 452bf020bd1ae5d7ddec79dcff4ca343b6487802 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=B1=CE=B7dr3=CE=B15=20=C9=AE=CE=B1=CE=B7=C9=A6=D6=85l?= =?UTF-8?q?=CA=903=CA=80?= Date: Fri, 5 Jun 2026 18:40:20 +0200 Subject: [PATCH 5/5] Bump cli to 4.5.0 + pin grading v2.4.0 #98 Co-Authored-By: Claude Opus 4.8 (1M context) --- package-lock.json | 10 +++++----- package.json | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index fc87e31..d901c0a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "flowmcp-cli", - "version": "4.4.0", + "version": "4.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "flowmcp-cli", - "version": "4.3.0", + "version": "4.5.0", "license": "MIT", "dependencies": { "@modelcontextprotocol/sdk": "^1.0.0", @@ -15,7 +15,7 @@ "ethers": "^6.16.0", "figlet": "^1.10.0", "flowmcp": "git+https://github.com/FlowMCP/flowmcp-core.git#de3a08888fa3945a284203d3c6ecd1419caab2bc", - "flowmcp-grading": "github:FlowMCP/flowmcp-grading#v2.3.0", + "flowmcp-grading": "github:FlowMCP/flowmcp-grading#v2.4.0", "geo-csv-tsv-toolkit": "git+https://github.com/FlowMCP/geo-csv-tsv-toolkit.git#5d5e51f8d88a35a20dc44489958e5f376e1674eb", "geo-geojson-toolkit": "git+https://github.com/FlowMCP/geo-geojson-toolkit.git#af764638f215083da52be785b8e276fa98217b02", "geo-gtfs-toolkit": "git+https://github.com/FlowMCP/geo-gtfs-toolkit.git#b868e612dc74c70e6401c9568a044f61ce80cede", @@ -3584,8 +3584,8 @@ } }, "node_modules/flowmcp-grading": { - "version": "2.3.0", - "resolved": "git+ssh://git@github.com/FlowMCP/flowmcp-grading.git#78b9c0890716383316bbf83fff5bc2d2b7585265", + "version": "2.4.0", + "resolved": "git+ssh://git@github.com/FlowMCP/flowmcp-grading.git#f8bededa2328e09198930367046ca9e4885f4744", "license": "MIT", "dependencies": { "flowmcp": "github:FlowMCP/flowmcp-core#bca23b1fb8727fb4cd4877b1c0ea94f4ac14acb5", diff --git a/package.json b/package.json index e93e7fd..4d63ef6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flowmcp-cli", - "version": "4.4.0", + "version": "4.5.0", "description": "CLI for developing and validating FlowMCP schemas", "type": "module", "license": "MIT", @@ -33,7 +33,7 @@ "ethers": "^6.16.0", "figlet": "^1.10.0", "flowmcp": "git+https://github.com/FlowMCP/flowmcp-core.git#de3a08888fa3945a284203d3c6ecd1419caab2bc", - "flowmcp-grading": "github:FlowMCP/flowmcp-grading#v2.3.0", + "flowmcp-grading": "github:FlowMCP/flowmcp-grading#v2.4.0", "geo-csv-tsv-toolkit": "git+https://github.com/FlowMCP/geo-csv-tsv-toolkit.git#5d5e51f8d88a35a20dc44489958e5f376e1674eb", "geo-geojson-toolkit": "git+https://github.com/FlowMCP/geo-geojson-toolkit.git#af764638f215083da52be785b8e276fa98217b02", "geo-gtfs-toolkit": "git+https://github.com/FlowMCP/geo-gtfs-toolkit.git#b868e612dc74c70e6401c9568a044f61ce80cede",