From 39661c33fb5b5f7cd539379c24d8cb2b6a0d9984 Mon Sep 17 00:00:00 2001 From: EmersonBraun Date: Thu, 3 Sep 2026 16:12:29 -0300 Subject: [PATCH] fix: make review call estimates demand-driven --- CHANGELOG.md | 4 ++++ README.md | 4 ++-- action.yml | 2 +- agents/code-review/agent.ts | 11 ++++++----- docs/OPERATIONS.md | 2 +- llms-full.txt | 10 +++++++--- readme-standard-v1.json | 2 +- test/cli-smoke.test.mjs | 39 ++++++++++++++++++++++++++++++++----- 8 files changed, 56 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5f9224..e4abb0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ All notable changes will be documented here. This project follows Semantic Versi - Added the explicit `fast` profile with a single required-lens batch for lower latency and predictable calls. - Added CI Action inputs for profile, deadline, and health-check policy. +### Fixed + +- Made provider-call preflight estimates demand-driven for adversarial verification while preserving the hard, fail-closed runtime call ceiling. + ## [0.2.3] - 2026-08-30 ### Fixed diff --git a/README.md b/README.md index 9df5047..62b94f9 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ AgentsKit Code Review is built around a different contract: - **Low noise by design.** Findings are challenged by independent verification votes before they survive. - **Local first, CI ready.** Review a diff before pushing, inspect complete paths, read stdin, or comment directly on a GitHub PR. - **Control cost and policy.** Set file budgets, concurrency, thresholds, project conventions, and blocking severity. -- **See the cost before execution.** Use `--plan --json` to inspect files, lenses, retries, concurrency, deadline, and estimated provider calls without a model request. Plans label estimates as `bounded` when `thresholds.maxPerFile` is set; otherwise they are `best-effort` because model output volume is inherently variable. +- **See the cost before execution.** Use `--plan --json` to inspect files, lenses, retries, concurrency, deadline, and estimated provider calls without a model request. Estimates are `best-effort` because model output volume is inherently variable; the runtime call ceiling remains hard and fail-closed. ## Run your first review @@ -275,7 +275,7 @@ In shortened examples, replace `...` with `npx --yes github:AgentsKit-io/code-re | `--min-confidence ` | Minimum reported confidence | | `--max-files ` | Positive file budget; over-budget runs are refused before the provider | | `--max-calls ` | Provider-call budget; absolute ceiling `1000` | -| `--max-findings-per-file ` | Maximum verified findings per file; bounds adversarial verification calls | +| `--max-findings-per-file ` | Maximum verified findings per file; caps adversarial verification work | | `--concurrency ` | Parallel model calls; default `1` for CLI providers, `4` for API providers | | `--deadline-ms ` | Global run deadline; defaults to `600000` (`120000` for `fast`) | | `--health-check ` | Bounded provider smoke check before model fan-out | diff --git a/action.yml b/action.yml index cf442de..6247dde 100644 --- a/action.yml +++ b/action.yml @@ -50,7 +50,7 @@ inputs: required: false default: '1000' max-findings-per-file: - description: 'Maximum findings verified per file; keeps the provider-call estimate bounded.' + description: 'Maximum findings verified per file; caps adversarial verification work.' required: false default: '7' profile: diff --git a/agents/code-review/agent.ts b/agents/code-review/agent.ts index 6fe2afd..34e79bf 100644 --- a/agents/code-review/agent.ts +++ b/agents/code-review/agent.ts @@ -684,15 +684,16 @@ export function createCodeReviewAgent(config: CodeReviewConfig) { const enabledLenses = lenses.map((lens) => lens.key) const required = [...requiredLenses] const primaryCalls = files * (batched ? 1 : enabledLenses.length) * (1 + retries) - const maxFindingsPerFile = config.thresholds?.maxPerFile - const verificationCalls = files * (maxFindingsPerFile ?? enabledLenses.length) * auditVotes * (1 + retries) - const estimatedProviderCalls = primaryCalls + verificationCalls + (files && enabledLenses.length ? 1 : 0) + // Verification is demand-driven: reserve only the optional consolidation call here. + // The runtime counter remains the hard ceiling and fails closed if candidates exhaust it. + const consolidationReserve = files && enabledLenses.length ? 1 : 0 + const estimatedProviderCalls = primaryCalls + consolidationReserve const plan: ReviewPlan = { profile, batched, files, bytes, enabledLenses, requiredLenses: required, votes: auditVotes, retries, concurrency, estimatedProviderCalls, - providerCallEstimate: maxFindingsPerFile === undefined ? 'best-effort' : 'bounded', + providerCallEstimate: 'best-effort', maxCalls, unreviewedFiles: all.length - files, overBudget: [], suggestions: [], deadlineMs, } const maxFiles = config.budget?.maxFiles @@ -710,7 +711,7 @@ export function createCodeReviewAgent(config: CodeReviewConfig) { plan.suggestions.push('reduce scope with --paths or an isolated context pattern') } if (estimatedProviderCalls > maxCalls) { - const perFile = Math.max(1, (batched ? 1 : enabledLenses.length) * (1 + retries) + (maxFindingsPerFile ?? enabledLenses.length) * auditVotes * (1 + retries)) + const perFile = Math.max(1, (batched ? 1 : enabledLenses.length) * (1 + retries)) plan.overBudget.push(`${estimatedProviderCalls} estimated provider calls exceed maxCalls ${maxCalls}`) plan.suggestions.push(`reduce scope to at most ${Math.max(1, Math.floor((maxCalls - 1) / perFile))} files or lower --votes`) } diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 61165a8..54d56e5 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -232,7 +232,7 @@ Then require the workflow check in branch protection. CLI exit codes are: A model response that is malformed may drop one lens while other lenses continue; progress output and the final summary report successful and failed primary-lens counts. If any reviewable file cannot be ingested or has zero successful primary lenses, the pipeline stops before reporters run and exits `2`, including in advisory mode. Treat missing output or exit `2` as unavailable review, not approval. -Use `--plan --json` (or `--dry-run`) to run the source and budget preflight without a model request. The plan reports profile, batching, files, bytes, enabled and required lenses, votes, retries, concurrency, deadline, estimated provider calls, and concrete reductions when a limit would be exceeded. Estimates are `bounded` when `thresholds.maxPerFile` is set and `best-effort` otherwise, because model output volume is variable. The preflight refuses before the provider starts; `maxCalls` is capped at 1000 and unlimited mode is not supported. A required-lens failure is `INCOMPLETE` and exits `2`, including with `--no-fail`. +Use `--plan --json` (or `--dry-run`) to run the source and budget preflight without a model request. The plan reports profile, batching, files, bytes, enabled and required lenses, votes, retries, concurrency, deadline, estimated provider calls, and concrete reductions when a limit would be exceeded. Estimates are `best-effort` because verification calls depend on findings produced by the primary lenses; the runtime call ceiling remains hard and fails closed if exhausted. The preflight refuses before the provider starts when primary lens coverage itself cannot fit; `maxCalls` is capped at 1000 and unlimited mode is not supported. A required-lens failure is `INCOMPLETE` and exits `2`, including with `--no-fail`. ## Cost and latency controls diff --git a/llms-full.txt b/llms-full.txt index 478a5a6..94a233d 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -58,7 +58,7 @@ AgentsKit Code Review is built around a different contract: - **Low noise by design.** Findings are challenged by independent verification votes before they survive. - **Local first, CI ready.** Review a diff before pushing, inspect complete paths, read stdin, or comment directly on a GitHub PR. - **Control cost and policy.** Set file budgets, concurrency, thresholds, project conventions, and blocking severity. -- **See the cost before execution.** Use `--plan --json` to inspect files, lenses, retries, concurrency, deadline, and estimated provider calls without a model request. Plans label estimates as `bounded` when `thresholds.maxPerFile` is set; otherwise they are `best-effort` because model output volume is inherently variable. +- **See the cost before execution.** Use `--plan --json` to inspect files, lenses, retries, concurrency, deadline, and estimated provider calls without a model request. Estimates are `best-effort` because model output volume is inherently variable; the runtime call ceiling remains hard and fail-closed. ## Run your first review @@ -293,7 +293,7 @@ In shortened examples, replace `...` with `npx --yes github:AgentsKit-io/code-re | `--min-confidence ` | Minimum reported confidence | | `--max-files ` | Positive file budget; over-budget runs are refused before the provider | | `--max-calls ` | Provider-call budget; absolute ceiling `1000` | -| `--max-findings-per-file ` | Maximum verified findings per file; bounds adversarial verification calls | +| `--max-findings-per-file ` | Maximum verified findings per file; caps adversarial verification work | | `--concurrency ` | Parallel model calls; default `1` for CLI providers, `4` for API providers | | `--deadline-ms ` | Global run deadline; defaults to `600000` (`120000` for `fast`) | | `--health-check ` | Bounded provider smoke check before model fan-out | @@ -654,7 +654,7 @@ Then require the workflow check in branch protection. CLI exit codes are: A model response that is malformed may drop one lens while other lenses continue; progress output and the final summary report successful and failed primary-lens counts. If any reviewable file cannot be ingested or has zero successful primary lenses, the pipeline stops before reporters run and exits `2`, including in advisory mode. Treat missing output or exit `2` as unavailable review, not approval. -Use `--plan --json` (or `--dry-run`) to run the source and budget preflight without a model request. The plan reports profile, batching, files, bytes, enabled and required lenses, votes, retries, concurrency, deadline, estimated provider calls, and concrete reductions when a limit would be exceeded. Estimates are `bounded` when `thresholds.maxPerFile` is set and `best-effort` otherwise, because model output volume is variable. The preflight refuses before the provider starts; `maxCalls` is capped at 1000 and unlimited mode is not supported. A required-lens failure is `INCOMPLETE` and exits `2`, including with `--no-fail`. +Use `--plan --json` (or `--dry-run`) to run the source and budget preflight without a model request. The plan reports profile, batching, files, bytes, enabled and required lenses, votes, retries, concurrency, deadline, estimated provider calls, and concrete reductions when a limit would be exceeded. Estimates are `best-effort` because verification calls depend on findings produced by the primary lenses; the runtime call ceiling remains hard and fails closed if exhausted. The preflight refuses before the provider starts when primary lens coverage itself cannot fit; `maxCalls` is capped at 1000 and unlimited mode is not supported. A required-lens failure is `INCOMPLETE` and exits `2`, including with `--no-fail`. ## Cost and latency controls @@ -964,6 +964,10 @@ All notable changes will be documented here. This project follows Semantic Versi - Added the explicit `fast` profile with a single required-lens batch for lower latency and predictable calls. - Added CI Action inputs for profile, deadline, and health-check policy. +### Fixed + +- Made provider-call preflight estimates demand-driven for adversarial verification while preserving the hard, fail-closed runtime call ceiling. + ## [0.2.3] - 2026-08-30 ### Fixed diff --git a/readme-standard-v1.json b/readme-standard-v1.json index 53fe0f8..6001bdd 100644 --- a/readme-standard-v1.json +++ b/readme-standard-v1.json @@ -221,7 +221,7 @@ "docs/OPERATIONS.md", "test/cli-smoke.test.mjs" ], - "sourceHash": "sha256:d60d942075696d400f19636b9c3fa87ca09742b7b29e54028b6c74c9c036193c" + "sourceHash": "sha256:c2095d3b266dbfc347c94e4968eaf99c97161db2966353c2a1c14b93d75c6f2e" }, "exceptions": [] } diff --git a/test/cli-smoke.test.mjs b/test/cli-smoke.test.mjs index de8f4e0..8d6a316 100644 --- a/test/cli-smoke.test.mjs +++ b/test/cli-smoke.test.mjs @@ -1,11 +1,12 @@ import assert from 'node:assert/strict' -import { spawnSync } from 'node:child_process' -import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { execFileSync, spawnSync } from 'node:child_process' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { fileURLToPath } from 'node:url' import { dirname, join, resolve } from 'node:path' import { tmpdir } from 'node:os' import test from 'node:test' import { codexCli } from '../dist/src/codex-adapter.js' +import { createCodeReviewAgent } from '../dist/agents/code-review/agent.js' const root = resolve(dirname(fileURLToPath(import.meta.url)), '..') @@ -266,7 +267,7 @@ test('plan is provider-free and machine-readable', () => { assert.equal(plan.overBudget.length, 0) }) -test('plan supports a bounded findings-per-file limit', () => { +test('plan accepts a findings-per-file limit without reserving its worst case', () => { const run = spawnSync(process.execPath, [ 'dist/src/cli.js', '--provider', 'codex-cli', '--stdin', '--dry-run', '--json', '--max-findings-per-file', '2', ], { @@ -276,8 +277,36 @@ test('plan supports a bounded findings-per-file limit', () => { assert.equal(run.status, 0, run.stderr) const plan = JSON.parse(run.stdout) - assert.equal(plan.providerCallEstimate, 'bounded') - assert.equal(plan.estimatedProviderCalls, 27) + assert.equal(plan.providerCallEstimate, 'best-effort') + assert.equal(plan.estimatedProviderCalls, 15) +}) + +test('plan keeps verification demand-driven when primary coverage fits the budget', async () => { + const cwd = mkdtempSync(join(tmpdir(), 'agentskit-adaptive-plan-')) + try { + execFileSync('git', ['-C', cwd, 'init', '-q']) + for (let i = 0; i < 20; i++) writeFileSync(join(cwd, `file-${i}.ts`), 'export const value = 1\n') + execFileSync('git', ['-C', cwd, 'add', '.']) + execFileSync('git', ['-C', cwd, '-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'initial']) + for (let i = 0; i < 20; i++) writeFileSync(join(cwd, `file-${i}.ts`), 'export const value = 2\n') + execFileSync('git', ['-C', cwd, 'add', '.']) + execFileSync('git', ['-C', cwd, '-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'change']) + + const plan = await createCodeReviewAgent({ + source: { kind: 'git-diff', base: 'HEAD~1', cwd }, + auditVotes: 3, + retries: 1, + thresholds: { maxPerFile: 7 }, + budget: { maxFiles: 50, maxCalls: 1000 }, + reporters: [], + }).plan() + + assert.equal(plan.files, 20) + assert.equal(plan.unreviewedFiles, 0) + assert.equal(plan.providerCallEstimate, 'best-effort') + assert.equal(plan.estimatedProviderCalls, 281) + assert.equal(plan.overBudget.length, 0) + } finally { rmSync(cwd, { recursive: true, force: true }) } }) test('fast profile batches required lenses and stays within a small call budget', () => {