From abe70e379cef20c04de6cf18f04f5882d2211031 Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Wed, 2 Sep 2026 22:11:49 +0530 Subject: [PATCH 1/2] refactor: remove migrated benchmark corpus 3/3 --- .../ts-type-confusion.codevetter.raw.json | 133 ------------ .../reviews-raw/ts-xss.codevetter.raw.json | 190 ------------------ 2 files changed, 323 deletions(-) delete mode 100644 benchmark/reviews-raw/ts-type-confusion.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/ts-xss.codevetter.raw.json diff --git a/benchmark/reviews-raw/ts-type-confusion.codevetter.raw.json b/benchmark/reviews-raw/ts-type-confusion.codevetter.raw.json deleted file mode 100644 index a21ab8b9..00000000 --- a/benchmark/reviews-raw/ts-type-confusion.codevetter.raw.json +++ /dev/null @@ -1,133 +0,0 @@ -{ - "agent": "claude", - "changed_lines": 20, - "coordinator_used": false, - "diff_range": "HEAD~1..HEAD", - "duration_ms": 59492, - "evidence_candidates": [ - { - "affected_files": ["source.ts"], - "caveats": ["The user may have run verification outside indexed agent sessions."], - "confidence": 0.66, - "evidence_refs": [ - { - "detail": null, - "kind": "history", - "label": "No fresh command/test/browser proof found in compact history" - } - ], - "id": "no-fresh-verification-evidence", - "kind": "not_verified", - "open_questions": ["What is the smallest relevant command or browser task for this diff?"], - "scale": "1 changed file(s)", - "severity_hint": "medium", - "suggested_checks": [ - "Attach a fresh command, test, log, screenshot, or trace before marking the finding fixed." - ], - "why_it_matters": "A review without fresh verification evidence can only prove static plausibility, not that the change works." - } - ], - "evidence_procedure_steps": [ - { - "action": "Choose the smallest relevant repo command or browser task, rerun it, and attach the fresh output.", - "artifact": "command log, test report, screenshot, trace, or QA run artifact", - "blocked_on": ["fresh verification artifact"], - "candidate_ids": ["no-fresh-verification-evidence"], - "gate": "No candidate is marked confirmed or fixed using stale or missing verification.", - "id": "rerun_relevant_verification", - "input": "Failed, stale, or missing command/test/browser evidence.", - "output": "Fresh pass/fail evidence linked to the review.", - "procedure": "rerun_relevant_verification", - "status": "blocked" - } - ], - "findings": [ - { - "confidence": 0.95, - "filePath": "source.ts", - "line": 18, - "severity": "high", - "suggestion": "Replace the double-cast with a real runtime type guard before use, e.g. `if (typeof (payload as any)?.total !== 'number' || !Array.isArray((payload as any)?.items)) throw new Error('processOrder: payload is not an Order');` then narrow. Do not use `as any` to silence the compiler — it removes the only safety the `unknown` parameter provides.", - "summary": "processOrder (source.ts:17-20) takes `payload: unknown` but line 18 does `payload as any as Order`; the intermediate `as any` erases all type info so the narrowing to `Order` performs no structural check. The code then relies on the unenforced invariant that payload is an Order: line 19 `order.total + order.items.length` throws `TypeError: Cannot read properties of undefined (reading 'length')` for any payload without `items` (e.g. a Refund), and yields `NaN` when `total` is absent. The `unknown` signature explicitly admits arbitrary input, so this invariant is relied upon but never validated.", - "title": "`unknown` payload cast to `Order` via `as any` with no runtime validation — guaranteed throw / NaN on shape mismatch" - } - ], - "findings_count": 1, - "qa_evidence": [], - "review_id": "32565810-fc8a-458d-b499-59f301fed34b", - "review_memory_graph": { - "edges": [ - { - "confidence": 0.74, - "from": "file-source-ts", - "kind": "has_history_context", - "to": "history-context" - }, - { - "confidence": 0.68, - "from": "file-source-ts", - "kind": "has_blast_radius", - "to": "blast-radius" - }, - { - "confidence": 0.66, - "from": "file-source-ts", - "kind": "raises_candidate", - "to": "candidate-no-fresh-verification-evidence" - }, - { - "confidence": 0.86, - "from": "candidate-no-fresh-verification-evidence", - "kind": "requires_gate", - "to": "gate-rerun-relevant-verification" - } - ], - "nodes": [ - { - "detail": "changed file", - "file_path": "source.ts", - "id": "file-source-ts", - "kind": "file", - "label": "source.ts" - }, - { - "detail": "306 chars in prompt section", - "file_path": null, - "id": "history-context", - "kind": "history_context", - "label": "Prior commits, decisions, agents, and command evidence" - }, - { - "detail": "computed from repo relationships", - "file_path": null, - "id": "blast-radius", - "kind": "blast_radius", - "label": "Blast-radius summary" - }, - { - "detail": "not_verified · confidence 0.66", - "file_path": "source.ts", - "id": "candidate-no-fresh-verification-evidence", - "kind": "evidence_candidate", - "label": "no-fresh-verification-evidence" - }, - { - "detail": "blocked · No candidate is marked confirmed or fixed using stale or missing verification.", - "file_path": null, - "id": "gate-rerun-relevant-verification", - "kind": "procedure_gate", - "label": "rerun_relevant_verification" - } - ], - "schema_version": 1, - "scope": "review_changed_files", - "truncated": false - }, - "review_mode": "specialist-lite", - "risk_tier": "lite", - "score": 90.0, - "sensitive_paths": [], - "specialists": ["assumption-integrity", "product-safety", "agent-handoff"], - "summary": "Risk-tiered review (lite) completed.\n\nSingle new file with a deliberately unenforced invariant. processOrder accepts `unknown` but casts through `as any` to `Order`, defeating type checking, then dereferences `.items.length` and `.total` without any runtime guard — a guaranteed TypeError (missing items) or silent NaN (missing total) for non-Order payloads. The inline `// BUG:` comment accurately describes the defect and does not mislead a follow-up agent. 0 callers today, so live blast radius is low, but it is an exported (public) function so any future caller inherits the trap.\n\nSingle new file adds `processOrder`, which double-casts an `unknown` payload through `any` to `Order` and dereferences `.total`/`.items.length` with zero runtime validation. This is a genuine correctness/crash bug: any non-Order payload throws a TypeError (missing `items`) or produces NaN (missing/non-numeric `total`). Currently 0 callers so no live regression, but the shipped contract is broken and will fail the first real order/refund flow wired to it. No fresh verification artifact was produced — static reasoning is conclusive, but the evidence gate remains open.\n\nSingle newly-added exported function `processOrder` (0 callers) contains a real latent runtime bug: `payload as any as Order` on line 18 bypasses the type system and there is no runtime validation before `order.items.length` on line 19, so any non-Order payload (e.g. the sibling `Refund`) throws a TypeError or produces NaN. The defect is definite when exercised; severity is high rather than critical only because the function is not yet wired to any caller. Fix is bounded (add a type guard).", - "talk_id": "ea499c9e-cd6c-4490-805f-0165e950e107" -} diff --git a/benchmark/reviews-raw/ts-xss.codevetter.raw.json b/benchmark/reviews-raw/ts-xss.codevetter.raw.json deleted file mode 100644 index 7fd55f85..00000000 --- a/benchmark/reviews-raw/ts-xss.codevetter.raw.json +++ /dev/null @@ -1,190 +0,0 @@ -{ - "agent": "claude", - "changed_lines": 13, - "coordinator_used": false, - "diff_range": "HEAD~1..HEAD", - "duration_ms": 46836, - "evidence_candidates": [ - { - "affected_files": ["source.tsx"], - "caveats": ["Static path matching cannot prove the UI is user-visible."], - "confidence": 0.72, - "evidence_refs": [ - { - "detail": null, - "kind": "changed_file", - "label": "UI-facing file changed" - } - ], - "id": "ui-change-needs-browser-proof", - "kind": "ui_without_browser_proof", - "open_questions": ["What route or user task proves the changed UI still works?"], - "scale": "UI surface changed", - "severity_hint": "medium", - "suggested_checks": ["Run or attach a browser/Playwright artifact for the affected route."], - "why_it_matters": "Agent-written UI changes often pass static review while breaking layout, loading, empty, or interaction states." - }, - { - "affected_files": ["source.tsx"], - "caveats": ["The user may have run verification outside indexed agent sessions."], - "confidence": 0.66, - "evidence_refs": [ - { - "detail": null, - "kind": "history", - "label": "No fresh command/test/browser proof found in compact history" - } - ], - "id": "no-fresh-verification-evidence", - "kind": "not_verified", - "open_questions": ["What is the smallest relevant command or browser task for this diff?"], - "scale": "1 changed file(s)", - "severity_hint": "medium", - "suggested_checks": [ - "Attach a fresh command, test, log, screenshot, or trace before marking the finding fixed." - ], - "why_it_matters": "A review without fresh verification evidence can only prove static plausibility, not that the change works." - } - ], - "evidence_procedure_steps": [ - { - "action": "Choose the smallest relevant repo command or browser task, rerun it, and attach the fresh output.", - "artifact": "command log, test report, screenshot, trace, or QA run artifact", - "blocked_on": ["fresh verification artifact"], - "candidate_ids": ["no-fresh-verification-evidence"], - "gate": "No candidate is marked confirmed or fixed using stale or missing verification.", - "id": "rerun_relevant_verification", - "input": "Failed, stale, or missing command/test/browser evidence.", - "output": "Fresh pass/fail evidence linked to the review.", - "procedure": "rerun_relevant_verification", - "status": "blocked" - }, - { - "action": "Open the affected route or run the nearest Playwright flow, then capture interaction, console, and network evidence.", - "artifact": "screenshot, trace, console/network log, or Playwright report", - "blocked_on": ["browser or Playwright artifact"], - "candidate_ids": ["ui-change-needs-browser-proof"], - "gate": "Changed UI has at least one fresh visual or interaction artifact, or remains needs_proof.", - "id": "verify_ui_route_change", - "input": "UI-facing changed files and the route or task they affect.", - "output": "Browser proof linked to the candidate and affected route.", - "procedure": "verify_ui_route_change", - "status": "blocked" - } - ], - "findings": [ - { - "confidence": 0.99, - "filePath": "source.tsx", - "line": 12, - "severity": "critical", - "suggestion": "Do not inject raw user content. Prefer rendering as text: `return
{body}
;`. If HTML rendering is genuinely required, sanitize first with DOMPurify: `const clean = DOMPurify.sanitize(body); return
;` and add a test that a `` payload is stripped.", - "summary": "source.tsx:12 renders `dangerouslySetInnerHTML={{ __html: body }}` where `body` is user-supplied comment HTML (line 5). Any `