diff --git a/benchmarks/public-catch-rate/reviews-raw/ts-hardcoded-credentials.codevetter.raw.json b/benchmarks/public-catch-rate/reviews-raw/ts-hardcoded-credentials.codevetter.raw.json new file mode 100644 index 00000000..b4ba237e --- /dev/null +++ b/benchmarks/public-catch-rate/reviews-raw/ts-hardcoded-credentials.codevetter.raw.json @@ -0,0 +1,142 @@ +{ + "agent": "claude", + "changed_lines": 14, + "coordinator_used": false, + "diff_range": "HEAD~1..HEAD", + "duration_ms": 55313, + "evidence_candidates": [ + { + "affected_files": ["source.ts"], + "caveats": ["The user may have run verification outside indexed agent sessions."], + "confidence": 0.66, + "evidence_refs": [ + { + "detail": null, + "kind": "history", + "label": "No fresh command/test/browser proof found in compact history" + } + ], + "id": "no-fresh-verification-evidence", + "kind": "not_verified", + "open_questions": ["What is the smallest relevant command or browser task for this diff?"], + "scale": "1 changed file(s)", + "severity_hint": "medium", + "suggested_checks": [ + "Attach a fresh command, test, log, screenshot, or trace before marking the finding fixed." + ], + "why_it_matters": "A review without fresh verification evidence can only prove static plausibility, not that the change works." + } + ], + "evidence_procedure_steps": [ + { + "action": "Choose the smallest relevant repo command or browser task, rerun it, and attach the fresh output.", + "artifact": "command log, test report, screenshot, trace, or QA run artifact", + "blocked_on": ["fresh verification artifact"], + "candidate_ids": ["no-fresh-verification-evidence"], + "gate": "No candidate is marked confirmed or fixed using stale or missing verification.", + "id": "rerun_relevant_verification", + "input": "Failed, stale, or missing command/test/browser evidence.", + "output": "Fresh pass/fail evidence linked to the review.", + "procedure": "rerun_relevant_verification", + "status": "blocked" + } + ], + "findings": [ + { + "confidence": 0.99, + "filePath": "source.ts", + "line": 7, + "severity": "critical", + "suggestion": "Rotate the production password immediately (it is already in git history). Load credentials from environment/secret manager: `password: process.env.DB_PASSWORD` with no default, and fail fast if unset. Add secret-scanning to CI to prevent recurrence.", + "summary": "Line 7 commits `password: 'P@ssw0rd-prod-2024!'` for the production host `db.prod.internal` (line 3) directly into source. The line-6 comment self-labels this as a BUG, so the stated intent contradicts the committed code. The secret is already in git history (commit d354ed0 'agent change under review'), meaning it is exposed to anyone with repo access and must be rotated, not merely deleted from source. There is no .env or .gitignore to mitigate it.", + "title": "Hardcoded production database password committed in plaintext" + }, + { + "confidence": 0.93, + "filePath": "source.ts", + "line": 13, + "severity": "high", + "suggestion": "Use a real Postgres client instead of fetch, e.g. `const client = new (await import('pg')).Client(dbConfig); await client.connect(); return client;`. Then actually run connect() against a test/mocked DB and attach the output.", + "summary": "Lines 12-13 build a `postgres://...` URL and pass it to `fetch(url)`. `fetch` only supports http/https/data/blob/file schemes, so this throws `TypeError: ... URL scheme \"postgres\" is not supported` at runtime — the function cannot establish a DB connection. 0 callers (newly added, no regression), but the symbol's entire purpose is broken, and the throw shows it was never executed/verified.", + "title": "connect() uses fetch() on a postgres:// URL and can never connect" + } + ], + "findings_count": 2, + "qa_evidence": [], + "review_id": "07fce861-da5f-4ba3-8f7f-e4babb582f89", + "review_memory_graph": { + "edges": [ + { + "confidence": 0.74, + "from": "file-source-ts", + "kind": "has_history_context", + "to": "history-context" + }, + { + "confidence": 0.68, + "from": "file-source-ts", + "kind": "has_blast_radius", + "to": "blast-radius" + }, + { + "confidence": 0.66, + "from": "file-source-ts", + "kind": "raises_candidate", + "to": "candidate-no-fresh-verification-evidence" + }, + { + "confidence": 0.86, + "from": "candidate-no-fresh-verification-evidence", + "kind": "requires_gate", + "to": "gate-rerun-relevant-verification" + } + ], + "nodes": [ + { + "detail": "changed file", + "file_path": "source.ts", + "id": "file-source-ts", + "kind": "file", + "label": "source.ts" + }, + { + "detail": "306 chars in prompt section", + "file_path": null, + "id": "history-context", + "kind": "history_context", + "label": "Prior commits, decisions, agents, and command evidence" + }, + { + "detail": "computed from repo relationships", + "file_path": null, + "id": "blast-radius", + "kind": "blast_radius", + "label": "Blast-radius summary" + }, + { + "detail": "not_verified · confidence 0.66", + "file_path": "source.ts", + "id": "candidate-no-fresh-verification-evidence", + "kind": "evidence_candidate", + "label": "no-fresh-verification-evidence" + }, + { + "detail": "blocked · No candidate is marked confirmed or fixed using stale or missing verification.", + "file_path": null, + "id": "gate-rerun-relevant-verification", + "kind": "procedure_gate", + "label": "rerun_relevant_verification" + } + ], + "schema_version": 1, + "scope": "review_changed_files", + "truncated": false + }, + "review_mode": "specialist-lite", + "risk_tier": "lite", + "score": 70.0, + "sensitive_paths": [], + "specialists": ["assumption-integrity", "product-safety", "agent-handoff"], + "summary": "Risk-tiered review (lite) completed.\n\nNew file source.ts contains a critical, self-acknowledged security defect: a production database password hardcoded in plaintext (line 7) that is already committed to git history and requires rotation. Separately, connect() (lines 11–13) is non-functional because it passes a postgres:// URL to fetch(), which only supports http/https and will throw at runtime. Both findings are confirmed by static inspection; no fresh runtime verification was run but none is needed to prove either defect.\n\nNew 14-line file with two real defects. Critical: plaintext production DB credentials committed at source.ts:7 (author-flagged) — full prod access leak, now permanent in git history. Product-safety-scope correctness bug: connect() (source.ts:13) calls fetch() on a postgres:// URL, which throws at runtime because Fetch only supports http/https — the connect flow is non-functional. connect() currently has 0 callers, limiting live blast radius. The no-fresh-verification-evidence gate is still open; I did not execute the code — the credential leak is proven statically, the fetch/postgres failure is derived from the Fetch spec.\n\nNew file source.ts contains a critical committed secret (plaintext prod DB password, line 7) that must be rotated, not just deleted, and a broken connect() that calls fetch() on a postgres:// URL (lines 12-13) so it can never work. No test or run evidence exists — consistent with the blocked verification gate — and the code as written would throw on first execution.", + "talk_id": "a3efe414-b70e-46ed-ace1-f92ddcde85f0" +} diff --git a/benchmarks/public-catch-rate/reviews-raw/ts-insecure-cookie.codevetter.raw.json b/benchmarks/public-catch-rate/reviews-raw/ts-insecure-cookie.codevetter.raw.json new file mode 100644 index 00000000..72b41506 --- /dev/null +++ b/benchmarks/public-catch-rate/reviews-raw/ts-insecure-cookie.codevetter.raw.json @@ -0,0 +1,133 @@ +{ + "agent": "claude", + "changed_lines": 9, + "coordinator_used": false, + "diff_range": "HEAD~1..HEAD", + "duration_ms": 46578, + "evidence_candidates": [ + { + "affected_files": ["source.ts"], + "caveats": ["The user may have run verification outside indexed agent sessions."], + "confidence": 0.66, + "evidence_refs": [ + { + "detail": null, + "kind": "history", + "label": "No fresh command/test/browser proof found in compact history" + } + ], + "id": "no-fresh-verification-evidence", + "kind": "not_verified", + "open_questions": ["What is the smallest relevant command or browser task for this diff?"], + "scale": "1 changed file(s)", + "severity_hint": "medium", + "suggested_checks": [ + "Attach a fresh command, test, log, screenshot, or trace before marking the finding fixed." + ], + "why_it_matters": "A review without fresh verification evidence can only prove static plausibility, not that the change works." + } + ], + "evidence_procedure_steps": [ + { + "action": "Choose the smallest relevant repo command or browser task, rerun it, and attach the fresh output.", + "artifact": "command log, test report, screenshot, trace, or QA run artifact", + "blocked_on": ["fresh verification artifact"], + "candidate_ids": ["no-fresh-verification-evidence"], + "gate": "No candidate is marked confirmed or fixed using stale or missing verification.", + "id": "rerun_relevant_verification", + "input": "Failed, stale, or missing command/test/browser evidence.", + "output": "Fresh pass/fail evidence linked to the review.", + "procedure": "rerun_relevant_verification", + "status": "blocked" + } + ], + "findings": [ + { + "confidence": 0.98, + "filePath": "source.ts", + "line": 8, + "severity": "high", + "suggestion": "Set the security attributes: `res.cookie('session', token, { maxAge: 86400000, httpOnly: true, secure: true, sameSite: 'lax' })` (use `sameSite: 'strict'` if no cross-site flows require the cookie). Consider deriving `secure` from environment so local HTTP dev still works while production stays HTTPS-only.", + "summary": "Line 8 `res.cookie('session', token, { maxAge: 86400000 })` passes only `maxAge`; Express defaults `httpOnly`/`secure`/`sameSite` to off, so the session token is JS-readable (XSS theft), sent over plaintext HTTP (MITM), and attached cross-site (CSRF). The comment on lines 5–7 confirms this is the intended-to-be-fixed vulnerability, not desired behavior.", + "title": "Session cookie set without HttpOnly, Secure, or SameSite" + } + ], + "findings_count": 1, + "qa_evidence": [], + "review_id": "b80521d7-a9de-4606-a3a0-9570128242bb", + "review_memory_graph": { + "edges": [ + { + "confidence": 0.74, + "from": "file-source-ts", + "kind": "has_history_context", + "to": "history-context" + }, + { + "confidence": 0.68, + "from": "file-source-ts", + "kind": "has_blast_radius", + "to": "blast-radius" + }, + { + "confidence": 0.66, + "from": "file-source-ts", + "kind": "raises_candidate", + "to": "candidate-no-fresh-verification-evidence" + }, + { + "confidence": 0.86, + "from": "candidate-no-fresh-verification-evidence", + "kind": "requires_gate", + "to": "gate-rerun-relevant-verification" + } + ], + "nodes": [ + { + "detail": "changed file", + "file_path": "source.ts", + "id": "file-source-ts", + "kind": "file", + "label": "source.ts" + }, + { + "detail": "306 chars in prompt section", + "file_path": null, + "id": "history-context", + "kind": "history_context", + "label": "Prior commits, decisions, agents, and command evidence" + }, + { + "detail": "computed from repo relationships", + "file_path": null, + "id": "blast-radius", + "kind": "blast_radius", + "label": "Blast-radius summary" + }, + { + "detail": "not_verified · confidence 0.66", + "file_path": "source.ts", + "id": "candidate-no-fresh-verification-evidence", + "kind": "evidence_candidate", + "label": "no-fresh-verification-evidence" + }, + { + "detail": "blocked · No candidate is marked confirmed or fixed using stale or missing verification.", + "file_path": null, + "id": "gate-rerun-relevant-verification", + "kind": "procedure_gate", + "label": "rerun_relevant_verification" + } + ], + "schema_version": 1, + "scope": "review_changed_files", + "truncated": false + }, + "review_mode": "assumption-first", + "risk_tier": "trivial", + "score": 90.0, + "sensitive_paths": [], + "specialists": ["assumption-integrity", "general"], + "summary": "Risk-tiered review (trivial) completed.\n\nSingle newly-added function `setSessionCookie`. It sets a session-token cookie with only `maxAge`, omitting `httpOnly`, `secure`, and `sameSite` — a real session-security hole (XSS exfiltration, plaintext transmission, CSRF). The in-file comment explicitly flags it as a BUG, so stated intent and code agree; the fix is to add the missing attributes on line 8. Currently 0 callers, so no live path yet, but it is the intended session-cookie helper.\n\nSingle new file adding a session-cookie helper. It sets the 'session' cookie with only maxAge, omitting httpOnly, secure, and sameSite — a real, statically-verifiable session-security defect (XSS theft, plaintext transmission, CSRF). Currently 0 callers, so runtime blast radius is nil today, but this is an auth-path helper whose intended use makes the defect high severity. No other issues in the 9 changed lines.", + "talk_id": "6d5a3ccc-46fa-436f-91f7-cb681b5d6c8a" +} diff --git a/benchmarks/public-catch-rate/reviews-raw/ts-missing-await.codevetter.raw.json b/benchmarks/public-catch-rate/reviews-raw/ts-missing-await.codevetter.raw.json new file mode 100644 index 00000000..92886ef1 --- /dev/null +++ b/benchmarks/public-catch-rate/reviews-raw/ts-missing-await.codevetter.raw.json @@ -0,0 +1,142 @@ +{ + "agent": "claude", + "changed_lines": 10, + "coordinator_used": false, + "diff_range": "HEAD~1..HEAD", + "duration_ms": 50188, + "evidence_candidates": [ + { + "affected_files": ["source.ts"], + "caveats": ["The user may have run verification outside indexed agent sessions."], + "confidence": 0.66, + "evidence_refs": [ + { + "detail": null, + "kind": "history", + "label": "No fresh command/test/browser proof found in compact history" + } + ], + "id": "no-fresh-verification-evidence", + "kind": "not_verified", + "open_questions": ["What is the smallest relevant command or browser task for this diff?"], + "scale": "1 changed file(s)", + "severity_hint": "medium", + "suggested_checks": [ + "Attach a fresh command, test, log, screenshot, or trace before marking the finding fixed." + ], + "why_it_matters": "A review without fresh verification evidence can only prove static plausibility, not that the change works." + } + ], + "evidence_procedure_steps": [ + { + "action": "Choose the smallest relevant repo command or browser task, rerun it, and attach the fresh output.", + "artifact": "command log, test report, screenshot, trace, or QA run artifact", + "blocked_on": ["fresh verification artifact"], + "candidate_ids": ["no-fresh-verification-evidence"], + "gate": "No candidate is marked confirmed or fixed using stale or missing verification.", + "id": "rerun_relevant_verification", + "input": "Failed, stale, or missing command/test/browser evidence.", + "output": "Fresh pass/fail evidence linked to the review.", + "procedure": "rerun_relevant_verification", + "status": "blocked" + } + ], + "findings": [ + { + "confidence": 0.97, + "filePath": "source.ts", + "line": 8, + "severity": "high", + "suggestion": "Await the promise: `await deleteSession(userId);` before logging success. Optionally wrap in try/catch (or let the rejection propagate) so callers can detect and retry a failed session deletion instead of getting a false success.", + "summary": "source.ts:8 calls `deleteSession(userId)` without `await` inside an `async` function whose contract (name `logout`, `Promise`, and the `console.log('user logged out')` on line 9) promises the session is deleted. Because the promise floats: (1) `logout` resolves before deletion completes, so an awaiting caller proceeds while the server session may still be live — a lingering-auth/session-fixation risk on a security-sensitive path; (2) a rejection from `deleteSession` becomes an unhandled promise rejection while `logout` still resolves successfully and logs success, hiding the failure from every caller. The in-file comment (lines 5-7) accurately describes this bug.", + "title": "Floating promise: deleteSession not awaited on logout path" + }, + { + "confidence": 0.9, + "filePath": "source.ts", + "line": 8, + "severity": "high", + "suggestion": "Await the call: `await deleteSession(userId);` before logging/returning. If fire-and-forget is genuinely intended, at minimum attach a `.catch()` so the rejection is handled, but for a logout flow the deletion should be awaited so the caller can trust that logout() resolving means the session is gone.", + "summary": "source.ts:8 calls `deleteSession(userId)` without `await` inside an `async function logout(): Promise`. Because the promise is not awaited, (1) a rejection becomes an unhandled promise rejection instead of propagating to logout's caller, and (2) logout resolves before deletion completes, so any caller that awaits logout() and then assumes the session is destroyed races an in-flight delete — leaving stale sessions. The deleteSession contract (async store op over `./session`) is not enforced by logout even though logout's success is meant to imply the session was removed.", + "title": "Missing await on deleteSession swallows failures and races session cleanup" + } + ], + "findings_count": 2, + "qa_evidence": [], + "review_id": "2d0bd324-c25f-473b-b58d-0c0baffa4268", + "review_memory_graph": { + "edges": [ + { + "confidence": 0.74, + "from": "file-source-ts", + "kind": "has_history_context", + "to": "history-context" + }, + { + "confidence": 0.68, + "from": "file-source-ts", + "kind": "has_blast_radius", + "to": "blast-radius" + }, + { + "confidence": 0.66, + "from": "file-source-ts", + "kind": "raises_candidate", + "to": "candidate-no-fresh-verification-evidence" + }, + { + "confidence": 0.86, + "from": "candidate-no-fresh-verification-evidence", + "kind": "requires_gate", + "to": "gate-rerun-relevant-verification" + } + ], + "nodes": [ + { + "detail": "changed file", + "file_path": "source.ts", + "id": "file-source-ts", + "kind": "file", + "label": "source.ts" + }, + { + "detail": "306 chars in prompt section", + "file_path": null, + "id": "history-context", + "kind": "history_context", + "label": "Prior commits, decisions, agents, and command evidence" + }, + { + "detail": "computed from repo relationships", + "file_path": null, + "id": "blast-radius", + "kind": "blast_radius", + "label": "Blast-radius summary" + }, + { + "detail": "not_verified · confidence 0.66", + "file_path": "source.ts", + "id": "candidate-no-fresh-verification-evidence", + "kind": "evidence_candidate", + "label": "no-fresh-verification-evidence" + }, + { + "detail": "blocked · No candidate is marked confirmed or fixed using stale or missing verification.", + "file_path": null, + "id": "gate-rerun-relevant-verification", + "kind": "procedure_gate", + "label": "rerun_relevant_verification" + } + ], + "schema_version": 1, + "scope": "review_changed_files", + "truncated": false + }, + "review_mode": "assumption-first", + "risk_tier": "trivial", + "score": 80.0, + "sensitive_paths": [], + "specialists": ["assumption-integrity", "general"], + "summary": "Risk-tiered review (trivial) completed.\n\nNewly added exported `logout` has a genuine correctness/security bug: `deleteSession(userId)` on line 8 is a floating (un-awaited) promise, so logout reports success before the session is actually deleted and silently swallows any deletion failure as an unhandled rejection. The function's name, return type, and success log all assert the session was deleted, which the code does not guarantee — a stated-intent-vs-code contradiction. 0 callers today (weights live impact down) but it will bite the first caller; rated high due to the security-sensitive logout context. The code comment is accurate and won't mislead a follow-up agent; the misleading artifact is the unconditional success log. Could not run anything — the imported `./session` module does not exist in this benchmark repo, so the runtime-verification gate remains open, but the defect is statically provable.\n\nSingle new file `source.ts` defining `logout`. It contains one real high-severity correctness bug: `deleteSession(userId)` is called without `await`, so a rejected deletion is swallowed (unhandled rejection) and logout reports success before the session is actually deleted. The bug is self-documented in the comment and confirmed against the changed line. Blast radius is currently 0 callers (newly added), but as a logout primitive it will attract security-sensitive callers, so severity stays high.", + "talk_id": "546c7883-e87f-4806-a993-6013b1d3f483" +} diff --git a/benchmarks/public-catch-rate/reviews-raw/ts-prototype-pollution.codevetter.raw.json b/benchmarks/public-catch-rate/reviews-raw/ts-prototype-pollution.codevetter.raw.json new file mode 100644 index 00000000..6eace235 --- /dev/null +++ b/benchmarks/public-catch-rate/reviews-raw/ts-prototype-pollution.codevetter.raw.json @@ -0,0 +1,160 @@ +{ + "agent": "claude", + "changed_lines": 21, + "coordinator_used": false, + "diff_range": "HEAD~1..HEAD", + "duration_ms": 161761, + "evidence_candidates": [ + { + "affected_files": ["source.ts"], + "caveats": ["The user may have run verification outside indexed agent sessions."], + "confidence": 0.66, + "evidence_refs": [ + { + "detail": null, + "kind": "history", + "label": "No fresh command/test/browser proof found in compact history" + } + ], + "id": "no-fresh-verification-evidence", + "kind": "not_verified", + "open_questions": ["What is the smallest relevant command or browser task for this diff?"], + "scale": "1 changed file(s)", + "severity_hint": "medium", + "suggested_checks": [ + "Attach a fresh command, test, log, screenshot, or trace before marking the finding fixed." + ], + "why_it_matters": "A review without fresh verification evidence can only prove static plausibility, not that the change works." + } + ], + "evidence_procedure_steps": [ + { + "action": "Choose the smallest relevant repo command or browser task, rerun it, and attach the fresh output.", + "artifact": "command log, test report, screenshot, trace, or QA run artifact", + "blocked_on": ["fresh verification artifact"], + "candidate_ids": ["no-fresh-verification-evidence"], + "gate": "No candidate is marked confirmed or fixed using stale or missing verification.", + "id": "rerun_relevant_verification", + "input": "Failed, stale, or missing command/test/browser evidence.", + "output": "Fresh pass/fail evidence linked to the review.", + "procedure": "rerun_relevant_verification", + "status": "blocked" + } + ], + "findings": [ + { + "confidence": 0.97, + "filePath": "source.ts", + "line": 17, + "severity": "critical", + "suggestion": "Before assigning or recursing, reject dangerous keys: if (key==='__proto__'||key==='constructor'||key==='prototype') continue; Additionally guard the recursion/assignment with Object.prototype.hasOwnProperty.call(target,key) and consider null-prototype accumulators. Add a unit test asserting ({}).admin===undefined after merging a __proto__ payload.", + "summary": "merge() iterates Object.keys(source) (line 11) without excluding dangerous keys. For source parsed from untrusted JSON like {\"__proto__\":{\"admin\":true}}, __proto__ is an own enumerable key; at lines 12-15 tv=target['__proto__'] resolves to Object.prototype (isObject true) and sv={admin:true} (isObject true), so it recurses into merge(Object.prototype, sv) and line 17 executes Object.prototype['admin']=true. This pollutes Object.prototype globally, silently flipping authorization/feature checks and creating confusing/broken states across every user flow. (The code comment's constructor/prototype claim is inaccurate — isObject rejects functions so only the __proto__ vector fires — but that vector is confirmed.)", + "title": "Prototype pollution via merge() __proto__ key enables app-wide privilege escalation" + }, + { + "confidence": 0.97, + "filePath": "source.ts", + "line": 11, + "severity": "critical", + "suggestion": "At the top of the loop body (after line 11) skip dangerous keys: `if (key === '__proto__' || key === 'constructor' || key === 'prototype') continue;`. Also prefer Object.hasOwn(source, key) and consider setting values via Object.defineProperty or using a null-prototype target. Add a regression test asserting ({} as any).admin === undefined after merge({}, JSON.parse('{\"__proto__\":{\"admin\":true}}')).", + "summary": "source.ts:11-17 iterates Object.keys(source) and assigns/recurses without filtering dangerous keys. With a JSON-parsed payload {\"__proto__\":{\"admin\":true}}, Object.keys returns own key \"__proto__\" (line 11); tv=Object.prototype and sv={admin:true} are both objects (line 14), so merge recurses into Object.prototype and line 17 executes Object.prototype.admin=true — app-wide pollution. merge is exported (line 9), so any caller passing untrusted input is exploitable.", + "title": "Prototype pollution in exported merge() via __proto__ key" + }, + { + "confidence": 0.8, + "filePath": "source.ts", + "line": 6, + "severity": "low", + "suggestion": "Correct the comment to state that __proto__ is the vector that pollutes through this merge (constructor is not, because functions aren't treated as objects by isObject), while still recommending all three be blocked defensively.", + "summary": "Lines 6-7 claim __proto__, constructor, AND prototype are all walked/exploitable, but isObject (line 3) uses typeof==='object', so target['constructor'] (a function, typeof 'function') fails the isObject(tv) check at line 14 and falls to line 17 setting a harmless own key — same for a top-level 'prototype' key (undefined on plain targets). Only own-enumerable __proto__ with an object value reaches Object.prototype. The comment would steer a next agent to expect constructor/prototype pollution to reproduce through this function, which it does not.", + "title": "BUG comment overstates exploitable vectors (constructor/prototype not live through this merge)" + }, + { + "confidence": 0.8, + "filePath": "source.ts", + "line": 6, + "severity": "low", + "suggestion": "Do not close this as verified without an artifact. Add a unit test (e.g. `expect(({}).admin).toBeUndefined()` after a __proto__ payload merge) and run it; attach the failing-then-passing output as proof once the key-filter fix lands.", + "summary": "The file carries a self-labeled BUG: comment (source.ts:6-8) documenting the pollution but no fix and no test. Blast-radius scan shows merge has 0 callers and there is no fresh command/test evidence (procedure gate rerun_relevant_verification is blocked). The next agent could misread the annotation as 'known/handled' and mark this green.", + "title": "No test or fresh verification for the flagged bug (handoff gap)" + } + ], + "findings_count": 4, + "qa_evidence": [], + "review_id": "2b972065-c4af-42d8-b3f4-a9f151b5213c", + "review_memory_graph": { + "edges": [ + { + "confidence": 0.74, + "from": "file-source-ts", + "kind": "has_history_context", + "to": "history-context" + }, + { + "confidence": 0.68, + "from": "file-source-ts", + "kind": "has_blast_radius", + "to": "blast-radius" + }, + { + "confidence": 0.66, + "from": "file-source-ts", + "kind": "raises_candidate", + "to": "candidate-no-fresh-verification-evidence" + }, + { + "confidence": 0.86, + "from": "candidate-no-fresh-verification-evidence", + "kind": "requires_gate", + "to": "gate-rerun-relevant-verification" + } + ], + "nodes": [ + { + "detail": "changed file", + "file_path": "source.ts", + "id": "file-source-ts", + "kind": "file", + "label": "source.ts" + }, + { + "detail": "306 chars in prompt section", + "file_path": null, + "id": "history-context", + "kind": "history_context", + "label": "Prior commits, decisions, agents, and command evidence" + }, + { + "detail": "computed from repo relationships", + "file_path": null, + "id": "blast-radius", + "kind": "blast_radius", + "label": "Blast-radius summary" + }, + { + "detail": "not_verified · confidence 0.66", + "file_path": "source.ts", + "id": "candidate-no-fresh-verification-evidence", + "kind": "evidence_candidate", + "label": "no-fresh-verification-evidence" + }, + { + "detail": "blocked · No candidate is marked confirmed or fixed using stale or missing verification.", + "file_path": null, + "id": "gate-rerun-relevant-verification", + "kind": "procedure_gate", + "label": "rerun_relevant_verification" + } + ], + "schema_version": 1, + "scope": "review_changed_files", + "truncated": false + }, + "review_mode": "specialist-lite", + "risk_tier": "lite", + "score": 56.0, + "sensitive_paths": [], + "specialists": ["assumption-integrity", "product-safety", "agent-handoff"], + "summary": "Risk-tiered review (lite) completed.\n\nNew standalone file introducing an exported recursive merge with a genuine, well-understood prototype-pollution vulnerability (line 17 via unfiltered keys at line 11). The primary BUG comment's core claim (__proto__ pollutes Object.prototype) is accurate and matches the code; a secondary claim (constructor/prototype are equally live vectors) is contradicted by isObject's typeof==='object' check, which blocks the constructor path. No callers or tests exist yet, so the risk is latent but real for any future consumer of untrusted input. Fresh runtime verification could not be attached (sandbox denied node execution and file writes), so the rerun_relevant_verification gate remains open — the __proto__ finding rests on textbook JS semantics, not a captured artifact.\n\nSingle new file introduces a classic recursive-merge prototype pollution. The __proto__ vector is confirmed exploitable by static trace (lines 11-17): a JSON-parsed {\"__proto__\":{...}} payload recurses into Object.prototype and writes to it, escalating privileges app-wide — a critical cross-scope defect with clear product-safety fallout (authorization checks flipped, confusing/broken states everywhere). Exported with 0 current callers, so no live workflow breaks yet, but the exported contract is unsafe for untrusted input. The in-code comment overstates the constructor/prototype vectors (isObject rejects functions), which does not reduce severity.\n\nNew file introduces an exported recursive merge() with a genuine, exploitable prototype-pollution vulnerability via the __proto__ key on JSON-parsed input (lines 11-17). The in-code BUG: comment correctly describes the hole but no fix, test, or verification exists. Blocker: filter __proto__/constructor/prototype before assignment/recursion and add a regression test.", + "talk_id": "23bb1ceb-9c6a-4aa5-bcaf-9fe3305ef2ca" +} diff --git a/benchmarks/public-catch-rate/reviews-raw/ts-regex-dos.codevetter.raw.json b/benchmarks/public-catch-rate/reviews-raw/ts-regex-dos.codevetter.raw.json new file mode 100644 index 00000000..150d08b2 --- /dev/null +++ b/benchmarks/public-catch-rate/reviews-raw/ts-regex-dos.codevetter.raw.json @@ -0,0 +1,142 @@ +{ + "agent": "claude", + "changed_lines": 10, + "coordinator_used": false, + "diff_range": "HEAD~1..HEAD", + "duration_ms": 89824, + "evidence_candidates": [ + { + "affected_files": ["source.ts"], + "caveats": ["The user may have run verification outside indexed agent sessions."], + "confidence": 0.66, + "evidence_refs": [ + { + "detail": null, + "kind": "history", + "label": "No fresh command/test/browser proof found in compact history" + } + ], + "id": "no-fresh-verification-evidence", + "kind": "not_verified", + "open_questions": ["What is the smallest relevant command or browser task for this diff?"], + "scale": "1 changed file(s)", + "severity_hint": "medium", + "suggested_checks": [ + "Attach a fresh command, test, log, screenshot, or trace before marking the finding fixed." + ], + "why_it_matters": "A review without fresh verification evidence can only prove static plausibility, not that the change works." + } + ], + "evidence_procedure_steps": [ + { + "action": "Choose the smallest relevant repo command or browser task, rerun it, and attach the fresh output.", + "artifact": "command log, test report, screenshot, trace, or QA run artifact", + "blocked_on": ["fresh verification artifact"], + "candidate_ids": ["no-fresh-verification-evidence"], + "gate": "No candidate is marked confirmed or fixed using stale or missing verification.", + "id": "rerun_relevant_verification", + "input": "Failed, stale, or missing command/test/browser evidence.", + "output": "Fresh pass/fail evidence linked to the review.", + "procedure": "rerun_relevant_verification", + "status": "blocked" + } + ], + "findings": [ + { + "confidence": 0.97, + "filePath": "source.ts", + "line": 3, + "severity": "high", + "suggestion": "Remove the redundant outer quantifier: /^[a-zA-Z0-9._%+-]+$/ matches the same language in linear time. Alternatively cap input length before .test(), or use a non-backtracking/RE2-style matcher. Add a test asserting isEmailLike returns quickly on a long non-matching input.", + "summary": "source.ts:3 defines /^([a-zA-Z0-9._%+-]+)+$/ — the nested (charclass+)+ with no delimiter causes exponential backtracking on inputs that match the class but fail the anchor (e.g. 'a'.repeat(n)+'!'). isEmailLike (source.ts:8-10) exposes this over user-supplied strings per the line-2 comment, so a single crafted string blocks the single-threaded event loop and denies service to all concurrent requests. The line-5-7 comment documents this exact bug yet the vulnerable regex is shipped as the exported implementation.", + "title": "ReDoS: nested quantifier in emailLikePattern blocks event loop on user input" + }, + { + "confidence": 0.75, + "filePath": "source.ts", + "line": 2, + "severity": "low", + "suggestion": "Either fix the regex before this exported API is adopted, or adjust the comment to reflect that it is not yet in use; do not let the vulnerable version be wired up as-is.", + "summary": "source.ts:2 states 'This regex is used to validate user-supplied email-like strings', but the blast-radius scan shows isEmailLike (source.ts:8) has 0 callers — it is exported but unwired. The stated-intent-vs-reality gap can mislead the next agent into believing validation is live, or into wiring the exported function into a request path and inheriting the ReDoS from the same diff.", + "title": "Comment claims the regex 'is used' but isEmailLike has 0 callers" + } + ], + "findings_count": 2, + "qa_evidence": [], + "review_id": "eb5a3d8a-fae1-4a69-a88a-ae2a8fc1cd08", + "review_memory_graph": { + "edges": [ + { + "confidence": 0.74, + "from": "file-source-ts", + "kind": "has_history_context", + "to": "history-context" + }, + { + "confidence": 0.68, + "from": "file-source-ts", + "kind": "has_blast_radius", + "to": "blast-radius" + }, + { + "confidence": 0.66, + "from": "file-source-ts", + "kind": "raises_candidate", + "to": "candidate-no-fresh-verification-evidence" + }, + { + "confidence": 0.86, + "from": "candidate-no-fresh-verification-evidence", + "kind": "requires_gate", + "to": "gate-rerun-relevant-verification" + } + ], + "nodes": [ + { + "detail": "changed file", + "file_path": "source.ts", + "id": "file-source-ts", + "kind": "file", + "label": "source.ts" + }, + { + "detail": "306 chars in prompt section", + "file_path": null, + "id": "history-context", + "kind": "history_context", + "label": "Prior commits, decisions, agents, and command evidence" + }, + { + "detail": "computed from repo relationships", + "file_path": null, + "id": "blast-radius", + "kind": "blast_radius", + "label": "Blast-radius summary" + }, + { + "detail": "not_verified · confidence 0.66", + "file_path": "source.ts", + "id": "candidate-no-fresh-verification-evidence", + "kind": "evidence_candidate", + "label": "no-fresh-verification-evidence" + }, + { + "detail": "blocked · No candidate is marked confirmed or fixed using stale or missing verification.", + "file_path": null, + "id": "gate-rerun-relevant-verification", + "kind": "procedure_gate", + "label": "rerun_relevant_verification" + } + ], + "schema_version": 1, + "scope": "review_changed_files", + "truncated": false + }, + "review_mode": "assumption-first", + "risk_tier": "trivial", + "score": 88.0, + "sensitive_paths": [], + "specialists": ["assumption-integrity", "general"], + "summary": "Risk-tiered review (trivial) completed.\n\nNew file introduces an exported email validator whose regex /^([a-zA-Z0-9._%+-]+)+$/ is a classic catastrophic-backtracking ReDoS (nested + over the same character class). isEmailLike wraps it for user-supplied input, so a crafted long non-matching string blocks the event loop — a real DoS. The change even documents the bug in a comment yet ships the vulnerable pattern. Assumption-integrity concern: the comment claims the regex 'is used' while blast-radius shows 0 callers, a stated-intent-vs-code contradiction that could snowball if the next agent wires it up unfixed. Fix: drop the redundant outer quantifier (/^[a-zA-Z0-9._%+-]+$/) and/or length-cap input.\n\nSingle new file introducing an exported ReDoS-vulnerable regex. The nested-quantifier pattern /^([a-zA-Z0-9._%+-]+)+$/ used by isEmailLike() on user-supplied input causes exponential backtracking on crafted non-matching strings, stalling the event loop (DoS). Confirmed by static analysis; the runtime timing benchmark was blocked by the sandbox approval gate so the rerun_relevant_verification gate remains open, but the vulnerability is definitive regardless. Minimal fix is to drop the outer '+' quantifier.", + "talk_id": "c4ba7e3c-e2e7-49da-a8c3-b1e7f41f49e4" +} diff --git a/benchmarks/public-catch-rate/reviews-raw/ts-sql-injection.codevetter.raw.json b/benchmarks/public-catch-rate/reviews-raw/ts-sql-injection.codevetter.raw.json new file mode 100644 index 00000000..cac61361 --- /dev/null +++ b/benchmarks/public-catch-rate/reviews-raw/ts-sql-injection.codevetter.raw.json @@ -0,0 +1,133 @@ +{ + "agent": "claude", + "changed_lines": 16, + "coordinator_used": false, + "diff_range": "HEAD~1..HEAD", + "duration_ms": 49688, + "evidence_candidates": [ + { + "affected_files": ["source.ts"], + "caveats": ["The user may have run verification outside indexed agent sessions."], + "confidence": 0.66, + "evidence_refs": [ + { + "detail": null, + "kind": "history", + "label": "No fresh command/test/browser proof found in compact history" + } + ], + "id": "no-fresh-verification-evidence", + "kind": "not_verified", + "open_questions": ["What is the smallest relevant command or browser task for this diff?"], + "scale": "1 changed file(s)", + "severity_hint": "medium", + "suggested_checks": [ + "Attach a fresh command, test, log, screenshot, or trace before marking the finding fixed." + ], + "why_it_matters": "A review without fresh verification evidence can only prove static plausibility, not that the change works." + } + ], + "evidence_procedure_steps": [ + { + "action": "Choose the smallest relevant repo command or browser task, rerun it, and attach the fresh output.", + "artifact": "command log, test report, screenshot, trace, or QA run artifact", + "blocked_on": ["fresh verification artifact"], + "candidate_ids": ["no-fresh-verification-evidence"], + "gate": "No candidate is marked confirmed or fixed using stale or missing verification.", + "id": "rerun_relevant_verification", + "input": "Failed, stale, or missing command/test/browser evidence.", + "output": "Fresh pass/fail evidence linked to the review.", + "procedure": "rerun_relevant_verification", + "status": "blocked" + } + ], + "findings": [ + { + "confidence": 0.99, + "filePath": "source.ts", + "line": 13, + "severity": "critical", + "suggestion": "Use a parameterized/prepared query instead of interpolation, e.g. `db.query(\"SELECT id, email FROM users WHERE email = ? LIMIT 1\", [emailInput])`. Confirm the exact bind placeholder syntax against the real `db.query` signature (the `./db` module is not present in this repo). Add a test asserting `findUserByEmail(\"' OR '1'='1\")` returns null, and run `tsc --noEmit`.", + "summary": "Line 13 interpolates user-controlled `emailInput` directly into the SQL string (`WHERE email = '${emailInput}'`) and line 14 executes it via `db.query`. Input such as `' OR '1'='1' -- ` breaks out of the quoted literal; combined with `LIMIT 1` this returns an arbitrary user row (auth/enumeration bypass), and UNION/stacked payloads enable data exfiltration depending on the driver.", + "title": "SQL injection via string concatenation in findUserByEmail" + } + ], + "findings_count": 1, + "qa_evidence": [], + "review_id": "e6a906c7-912a-4a72-8a2a-150b0594b92e", + "review_memory_graph": { + "edges": [ + { + "confidence": 0.74, + "from": "file-source-ts", + "kind": "has_history_context", + "to": "history-context" + }, + { + "confidence": 0.68, + "from": "file-source-ts", + "kind": "has_blast_radius", + "to": "blast-radius" + }, + { + "confidence": 0.66, + "from": "file-source-ts", + "kind": "raises_candidate", + "to": "candidate-no-fresh-verification-evidence" + }, + { + "confidence": 0.86, + "from": "candidate-no-fresh-verification-evidence", + "kind": "requires_gate", + "to": "gate-rerun-relevant-verification" + } + ], + "nodes": [ + { + "detail": "changed file", + "file_path": "source.ts", + "id": "file-source-ts", + "kind": "file", + "label": "source.ts" + }, + { + "detail": "306 chars in prompt section", + "file_path": null, + "id": "history-context", + "kind": "history_context", + "label": "Prior commits, decisions, agents, and command evidence" + }, + { + "detail": "computed from repo relationships", + "file_path": null, + "id": "blast-radius", + "kind": "blast_radius", + "label": "Blast-radius summary" + }, + { + "detail": "not_verified · confidence 0.66", + "file_path": "source.ts", + "id": "candidate-no-fresh-verification-evidence", + "kind": "evidence_candidate", + "label": "no-fresh-verification-evidence" + }, + { + "detail": "blocked · No candidate is marked confirmed or fixed using stale or missing verification.", + "file_path": null, + "id": "gate-rerun-relevant-verification", + "kind": "procedure_gate", + "label": "rerun_relevant_verification" + } + ], + "schema_version": 1, + "scope": "review_changed_files", + "truncated": false + }, + "review_mode": "specialist-lite", + "risk_tier": "lite", + "score": 80.0, + "sensitive_paths": [], + "specialists": ["assumption-integrity", "product-safety", "agent-handoff"], + "summary": "Risk-tiered review (lite) completed.\n\nSingle new file introduces a classic SQL injection: user-controlled `emailInput` is interpolated into a raw SQL string (source.ts:13) and executed (line 14). The vulnerability is provable from the diff alone and is critical. Assumption-integrity check: the file/inline comments correctly describe the bug (no comment-vs-code contradiction), but the code relies on an unenforced 'emailInput is safe' invariant. The parameterized-query fix is standard, but the `db.query` signature could NOT be confirmed — the imported `./db` module does not exist in this repo — so the exact placeholder API is an open question left for the fixer.\n\nNew file introduces an exported `findUserByEmail` with a textbook SQL injection: `emailInput` is interpolated directly into the SQL string (line 13) and executed via `db.query` (line 14) with no parameterization or escaping. This is a critical security bug with concrete product-safety impact (auth/lookup bypass, data exposure, potential data loss). The inline `// BUG:` comment acknowledges the flaw but does not mitigate it. Everything else is trivial. Recommend blocking until the query is parameterized.\n\nSingle new file with a textbook, self-labeled SQL injection: user input is concatenated into a SQL string on line 13 and executed on line 14. Critical, high-confidence. Fix is a bounded change to a parameterized query, but the exact bind syntax and any verification are blocked because the imported `./db` module does not exist in the repo — so no compile/test evidence can be produced against this file as-is.", + "talk_id": "cbd01303-b0af-4ba3-b890-c2b414b5b703" +} diff --git a/benchmarks/public-catch-rate/reviews-raw/ts-type-confusion.codevetter.raw.json b/benchmarks/public-catch-rate/reviews-raw/ts-type-confusion.codevetter.raw.json new file mode 100644 index 00000000..a21ab8b9 --- /dev/null +++ b/benchmarks/public-catch-rate/reviews-raw/ts-type-confusion.codevetter.raw.json @@ -0,0 +1,133 @@ +{ + "agent": "claude", + "changed_lines": 20, + "coordinator_used": false, + "diff_range": "HEAD~1..HEAD", + "duration_ms": 59492, + "evidence_candidates": [ + { + "affected_files": ["source.ts"], + "caveats": ["The user may have run verification outside indexed agent sessions."], + "confidence": 0.66, + "evidence_refs": [ + { + "detail": null, + "kind": "history", + "label": "No fresh command/test/browser proof found in compact history" + } + ], + "id": "no-fresh-verification-evidence", + "kind": "not_verified", + "open_questions": ["What is the smallest relevant command or browser task for this diff?"], + "scale": "1 changed file(s)", + "severity_hint": "medium", + "suggested_checks": [ + "Attach a fresh command, test, log, screenshot, or trace before marking the finding fixed." + ], + "why_it_matters": "A review without fresh verification evidence can only prove static plausibility, not that the change works." + } + ], + "evidence_procedure_steps": [ + { + "action": "Choose the smallest relevant repo command or browser task, rerun it, and attach the fresh output.", + "artifact": "command log, test report, screenshot, trace, or QA run artifact", + "blocked_on": ["fresh verification artifact"], + "candidate_ids": ["no-fresh-verification-evidence"], + "gate": "No candidate is marked confirmed or fixed using stale or missing verification.", + "id": "rerun_relevant_verification", + "input": "Failed, stale, or missing command/test/browser evidence.", + "output": "Fresh pass/fail evidence linked to the review.", + "procedure": "rerun_relevant_verification", + "status": "blocked" + } + ], + "findings": [ + { + "confidence": 0.95, + "filePath": "source.ts", + "line": 18, + "severity": "high", + "suggestion": "Replace the double-cast with a real runtime type guard before use, e.g. `if (typeof (payload as any)?.total !== 'number' || !Array.isArray((payload as any)?.items)) throw new Error('processOrder: payload is not an Order');` then narrow. Do not use `as any` to silence the compiler — it removes the only safety the `unknown` parameter provides.", + "summary": "processOrder (source.ts:17-20) takes `payload: unknown` but line 18 does `payload as any as Order`; the intermediate `as any` erases all type info so the narrowing to `Order` performs no structural check. The code then relies on the unenforced invariant that payload is an Order: line 19 `order.total + order.items.length` throws `TypeError: Cannot read properties of undefined (reading 'length')` for any payload without `items` (e.g. a Refund), and yields `NaN` when `total` is absent. The `unknown` signature explicitly admits arbitrary input, so this invariant is relied upon but never validated.", + "title": "`unknown` payload cast to `Order` via `as any` with no runtime validation — guaranteed throw / NaN on shape mismatch" + } + ], + "findings_count": 1, + "qa_evidence": [], + "review_id": "32565810-fc8a-458d-b499-59f301fed34b", + "review_memory_graph": { + "edges": [ + { + "confidence": 0.74, + "from": "file-source-ts", + "kind": "has_history_context", + "to": "history-context" + }, + { + "confidence": 0.68, + "from": "file-source-ts", + "kind": "has_blast_radius", + "to": "blast-radius" + }, + { + "confidence": 0.66, + "from": "file-source-ts", + "kind": "raises_candidate", + "to": "candidate-no-fresh-verification-evidence" + }, + { + "confidence": 0.86, + "from": "candidate-no-fresh-verification-evidence", + "kind": "requires_gate", + "to": "gate-rerun-relevant-verification" + } + ], + "nodes": [ + { + "detail": "changed file", + "file_path": "source.ts", + "id": "file-source-ts", + "kind": "file", + "label": "source.ts" + }, + { + "detail": "306 chars in prompt section", + "file_path": null, + "id": "history-context", + "kind": "history_context", + "label": "Prior commits, decisions, agents, and command evidence" + }, + { + "detail": "computed from repo relationships", + "file_path": null, + "id": "blast-radius", + "kind": "blast_radius", + "label": "Blast-radius summary" + }, + { + "detail": "not_verified · confidence 0.66", + "file_path": "source.ts", + "id": "candidate-no-fresh-verification-evidence", + "kind": "evidence_candidate", + "label": "no-fresh-verification-evidence" + }, + { + "detail": "blocked · No candidate is marked confirmed or fixed using stale or missing verification.", + "file_path": null, + "id": "gate-rerun-relevant-verification", + "kind": "procedure_gate", + "label": "rerun_relevant_verification" + } + ], + "schema_version": 1, + "scope": "review_changed_files", + "truncated": false + }, + "review_mode": "specialist-lite", + "risk_tier": "lite", + "score": 90.0, + "sensitive_paths": [], + "specialists": ["assumption-integrity", "product-safety", "agent-handoff"], + "summary": "Risk-tiered review (lite) completed.\n\nSingle new file with a deliberately unenforced invariant. processOrder accepts `unknown` but casts through `as any` to `Order`, defeating type checking, then dereferences `.items.length` and `.total` without any runtime guard — a guaranteed TypeError (missing items) or silent NaN (missing total) for non-Order payloads. The inline `// BUG:` comment accurately describes the defect and does not mislead a follow-up agent. 0 callers today, so live blast radius is low, but it is an exported (public) function so any future caller inherits the trap.\n\nSingle new file adds `processOrder`, which double-casts an `unknown` payload through `any` to `Order` and dereferences `.total`/`.items.length` with zero runtime validation. This is a genuine correctness/crash bug: any non-Order payload throws a TypeError (missing `items`) or produces NaN (missing/non-numeric `total`). Currently 0 callers so no live regression, but the shipped contract is broken and will fail the first real order/refund flow wired to it. No fresh verification artifact was produced — static reasoning is conclusive, but the evidence gate remains open.\n\nSingle newly-added exported function `processOrder` (0 callers) contains a real latent runtime bug: `payload as any as Order` on line 18 bypasses the type system and there is no runtime validation before `order.items.length` on line 19, so any non-Order payload (e.g. the sibling `Refund`) throws a TypeError or produces NaN. The defect is definite when exercised; severity is high rather than critical only because the function is not yet wired to any caller. Fix is bounded (add a type guard).", + "talk_id": "ea499c9e-cd6c-4490-805f-0165e950e107" +} diff --git a/benchmarks/public-catch-rate/reviews-raw/ts-xss.codevetter.raw.json b/benchmarks/public-catch-rate/reviews-raw/ts-xss.codevetter.raw.json new file mode 100644 index 00000000..7fd55f85 --- /dev/null +++ b/benchmarks/public-catch-rate/reviews-raw/ts-xss.codevetter.raw.json @@ -0,0 +1,190 @@ +{ + "agent": "claude", + "changed_lines": 13, + "coordinator_used": false, + "diff_range": "HEAD~1..HEAD", + "duration_ms": 46836, + "evidence_candidates": [ + { + "affected_files": ["source.tsx"], + "caveats": ["Static path matching cannot prove the UI is user-visible."], + "confidence": 0.72, + "evidence_refs": [ + { + "detail": null, + "kind": "changed_file", + "label": "UI-facing file changed" + } + ], + "id": "ui-change-needs-browser-proof", + "kind": "ui_without_browser_proof", + "open_questions": ["What route or user task proves the changed UI still works?"], + "scale": "UI surface changed", + "severity_hint": "medium", + "suggested_checks": ["Run or attach a browser/Playwright artifact for the affected route."], + "why_it_matters": "Agent-written UI changes often pass static review while breaking layout, loading, empty, or interaction states." + }, + { + "affected_files": ["source.tsx"], + "caveats": ["The user may have run verification outside indexed agent sessions."], + "confidence": 0.66, + "evidence_refs": [ + { + "detail": null, + "kind": "history", + "label": "No fresh command/test/browser proof found in compact history" + } + ], + "id": "no-fresh-verification-evidence", + "kind": "not_verified", + "open_questions": ["What is the smallest relevant command or browser task for this diff?"], + "scale": "1 changed file(s)", + "severity_hint": "medium", + "suggested_checks": [ + "Attach a fresh command, test, log, screenshot, or trace before marking the finding fixed." + ], + "why_it_matters": "A review without fresh verification evidence can only prove static plausibility, not that the change works." + } + ], + "evidence_procedure_steps": [ + { + "action": "Choose the smallest relevant repo command or browser task, rerun it, and attach the fresh output.", + "artifact": "command log, test report, screenshot, trace, or QA run artifact", + "blocked_on": ["fresh verification artifact"], + "candidate_ids": ["no-fresh-verification-evidence"], + "gate": "No candidate is marked confirmed or fixed using stale or missing verification.", + "id": "rerun_relevant_verification", + "input": "Failed, stale, or missing command/test/browser evidence.", + "output": "Fresh pass/fail evidence linked to the review.", + "procedure": "rerun_relevant_verification", + "status": "blocked" + }, + { + "action": "Open the affected route or run the nearest Playwright flow, then capture interaction, console, and network evidence.", + "artifact": "screenshot, trace, console/network log, or Playwright report", + "blocked_on": ["browser or Playwright artifact"], + "candidate_ids": ["ui-change-needs-browser-proof"], + "gate": "Changed UI has at least one fresh visual or interaction artifact, or remains needs_proof.", + "id": "verify_ui_route_change", + "input": "UI-facing changed files and the route or task they affect.", + "output": "Browser proof linked to the candidate and affected route.", + "procedure": "verify_ui_route_change", + "status": "blocked" + } + ], + "findings": [ + { + "confidence": 0.99, + "filePath": "source.tsx", + "line": 12, + "severity": "critical", + "suggestion": "Do not inject raw user content. Prefer rendering as text: `return
{body}
;`. If HTML rendering is genuinely required, sanitize first with DOMPurify: `const clean = DOMPurify.sanitize(body); return
;` and add a test that a `` payload is stripped.", + "summary": "source.tsx:12 renders `dangerouslySetInnerHTML={{ __html: body }}` where `body` is user-supplied comment HTML (line 5). Any `