From 6b90430cd5cf0574197ec185d633725e9d4d508e Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Wed, 2 Sep 2026 22:11:48 +0530 Subject: [PATCH 1/2] refactor: remove migrated benchmark corpus 2/3 --- benchmark/README.md | 154 --------------- benchmark/cases/ts-insecure-cookie/label.json | 16 -- benchmark/cases/ts-insecure-cookie/source.ts | 9 - benchmark/cases/ts-missing-await/label.json | 16 -- benchmark/cases/ts-missing-await/source.ts | 10 - .../cases/ts-prototype-pollution/label.json | 16 -- .../cases/ts-prototype-pollution/source.ts | 21 --- benchmark/cases/ts-regex-dos/label.json | 16 -- benchmark/cases/ts-regex-dos/source.ts | 10 - benchmark/cases/ts-sql-injection/label.json | 16 -- benchmark/cases/ts-sql-injection/source.ts | 16 -- benchmark/cases/ts-type-confusion/label.json | 16 -- benchmark/cases/ts-type-confusion/source.ts | 20 -- benchmark/cases/ts-xss/label.json | 16 -- benchmark/cases/ts-xss/source.tsx | 13 -- .../go-errcheck.codevetter.raw.json | 142 -------------- ...-hardcoded-credentials.codevetter.raw.json | 151 --------------- .../go-nil-pointer.codevetter.raw.json | 142 -------------- .../go-race-condition.codevetter.raw.json | 142 -------------- .../go-sql-injection.codevetter.raw.json | 142 -------------- .../java-insecure-random.codevetter.raw.json | 138 -------------- .../js-eval-injection.codevetter.raw.json | 142 -------------- .../js-open-redirect.codevetter.raw.json | 129 ------------- .../py-bare-except.codevetter.raw.json | 151 --------------- .../py-command-injection.codevetter.raw.json | 160 ---------------- .../py-hardcoded-secret.codevetter.raw.json | 178 ------------------ ...secure-deserialization.codevetter.raw.json | 142 -------------- .../py-path-traversal.codevetter.raw.json | 142 -------------- .../py-sql-injection.codevetter.raw.json | 133 ------------- .../reviews-raw/py-ssrf.codevetter.raw.json | 142 -------------- .../py-weak-hash.codevetter.raw.json | 151 --------------- .../py-zip-bomb.codevetter.raw.json | 133 ------------- .../rust-integer-overflow.codevetter.raw.json | 151 --------------- .../ts-dead-code.codevetter.raw.json | 178 ------------------ ...-hardcoded-credentials.codevetter.raw.json | 142 -------------- .../ts-insecure-cookie.codevetter.raw.json | 133 ------------- .../ts-missing-await.codevetter.raw.json | 142 -------------- ...ts-prototype-pollution.codevetter.raw.json | 160 ---------------- .../ts-regex-dos.codevetter.raw.json | 142 -------------- .../ts-sql-injection.codevetter.raw.json | 133 ------------- 40 files changed, 4006 deletions(-) delete mode 100644 benchmark/README.md delete mode 100644 benchmark/cases/ts-insecure-cookie/label.json delete mode 100644 benchmark/cases/ts-insecure-cookie/source.ts delete mode 100644 benchmark/cases/ts-missing-await/label.json delete mode 100644 benchmark/cases/ts-missing-await/source.ts delete mode 100644 benchmark/cases/ts-prototype-pollution/label.json delete mode 100644 benchmark/cases/ts-prototype-pollution/source.ts delete mode 100644 benchmark/cases/ts-regex-dos/label.json delete mode 100644 benchmark/cases/ts-regex-dos/source.ts delete mode 100644 benchmark/cases/ts-sql-injection/label.json delete mode 100644 benchmark/cases/ts-sql-injection/source.ts delete mode 100644 benchmark/cases/ts-type-confusion/label.json delete mode 100644 benchmark/cases/ts-type-confusion/source.ts delete mode 100644 benchmark/cases/ts-xss/label.json delete mode 100644 benchmark/cases/ts-xss/source.tsx delete mode 100644 benchmark/reviews-raw/go-errcheck.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/go-hardcoded-credentials.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/go-nil-pointer.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/go-race-condition.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/go-sql-injection.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/java-insecure-random.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/js-eval-injection.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/js-open-redirect.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/py-bare-except.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/py-command-injection.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/py-hardcoded-secret.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/py-insecure-deserialization.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/py-path-traversal.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/py-sql-injection.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/py-ssrf.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/py-weak-hash.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/py-zip-bomb.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/rust-integer-overflow.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/ts-dead-code.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/ts-hardcoded-credentials.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/ts-insecure-cookie.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/ts-missing-await.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/ts-prototype-pollution.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/ts-regex-dos.codevetter.raw.json delete mode 100644 benchmark/reviews-raw/ts-sql-injection.codevetter.raw.json diff --git a/benchmark/README.md b/benchmark/README.md deleted file mode 100644 index 4cdcf359..00000000 --- a/benchmark/README.md +++ /dev/null @@ -1,154 +0,0 @@ -# CodeVetter Public Benchmark - -A public, hand-labeled benchmark for measuring whether code review / security -analysis tools actually catch known issues. Each case is a small code snippet -with one or more **hand-labeled** expected findings. The cases are intentionally -synthetic and self-contained so anyone can reproduce a score: drop a tool's -output into `reviews/.json` and run the scorer. - -This exists so enterprise claims about CodeVetter (or any reviewer) are backed -by **external, repeatable proof** instead of internal fixtures that cannot be -audited. - -## Layout - -``` -benchmark/ - cases/ - / - source. # the code snippet with known issues - label.json # hand-labeled ground truth: type, severity, location, description - reviews/ # gitignored; drop a reviewer's output here per case - .json - README.md # this file -``` - -Each `label.json` has the shape: - -```json -{ - "id": "ts-sql-injection", - "title": "SQL injection via string concatenation in TypeScript", - "language": "typescript", - "source_file": "source.ts", - "category": "security", - "ground_truth": [ - { - "id": "sql-injection-email-concat", - "type": "sql_injection", - "severity": "high", - "location": { "file": "source.ts", "lines": [14, 14] }, - "description": "User-controlled emailInput is concatenated directly into the SQL query string ..." - } - ] -} -``` - -A reviewer output file (`reviews/.json`) has the shape: - -```json -{ - "case_id": "ts-sql-injection", - "reviewer": "codevetter", - "findings": [ - { - "id": "f-1", - "type": "sql_injection", - "severity": "high", - "file": "source.ts", - "lines": [14, 14], - "title": "SQL injection via string concatenation", - "matched_ground_truth": ["sql-injection-email-concat"], - "rationale": "Identifies the same concatenated user input into the SQL string." - } - ] -} -``` - -`matched_ground_truth` lists the ground-truth ids the finding catches. Findings -with an empty `matched_ground_truth` count as false positives. - -## Cases (27) - -| Case | Language | Category | Issue type | -| --- | --- | --- | --- | -| ts-sql-injection | TypeScript | security | sql_injection | -| py-hardcoded-secret | Python | security | hardcoded_secret | -| go-race-condition | Go | concurrency | race_condition | -| ts-xss | TypeScript | security | xss | -| py-path-traversal | Python | security | path_traversal | -| js-eval-injection | JavaScript | security | code_injection | -| rust-integer-overflow | Rust | bug | integer_overflow | -| ts-dead-code | TypeScript | maintainability | dead_code | -| py-command-injection | Python | security | command_injection | -| go-errcheck | Go | bug | unchecked_error | -| ts-hardcoded-credentials | TypeScript | security | hardcoded_secret | -| py-weak-hash | Python | security | weak_crypto | -| java-insecure-random | Java | security | insecure_random | -| ts-prototype-pollution | TypeScript | security | prototype_pollution | -| py-sql-injection | Python | security | sql_injection | -| go-sql-injection | Go | security | sql_injection | -| ts-missing-await | TypeScript | bug | missing_await | -| py-bare-except | Python | bug | swallowed_error | -| js-open-redirect | JavaScript | security | open_redirect | -| ts-insecure-cookie | TypeScript | security | insecure_cookie | -| py-ssrf | Python | security | ssrf | -| go-hardcoded-credentials | Go | security | hardcoded_secret | -| ts-regex-dos | TypeScript | security | regex_dos | -| py-zip-bomb | Python | security | resource_exhaustion | -| ts-type-confusion | TypeScript | bug | type_confusion | -| py-insecure-deserialization | Python | security | insecure_deserialization | -| go-nil-pointer | Go | bug | nil_dereference | - -Coverage spans TypeScript, JavaScript, Python, Go, Rust, and Java across -security, concurrency, bug, and maintainability categories. - -## Running the scorer - -From the repo root: - -```bash -# Validate every case and print a scorecard of the labeled ground truth. -# This requires no reviewer output and always works. -npm run bench:public - -# Score a reviewer's output after dropping files into benchmark/reviews/. -npm run bench:public -- --reviewer=codevetter - -# Emit a JSON scorecard. -npm run bench:public -- --reviewer=codevetter --json - -# Write a Markdown scorecard to disk. -npm run bench:public -- --reviewer=codevetter --format=markdown --out=artifacts/public-benchmark.md - -# Gate on minimum catch rate (exits non-zero when below threshold). -npm run bench:public -- --reviewer=codevetter --min-rate=0.8 -``` - -## How to evaluate a tool against this benchmark - -1. For each case in `benchmark/cases//`, feed `source.` to your - reviewer (CodeVetter or any comparator). -2. Normalize the reviewer's findings into the `reviews/.json` shape - above, filling `matched_ground_truth` with the ground-truth ids each finding - catches (leave empty for findings that do not match any labeled issue). -3. Run `npm run bench:public -- --reviewer=` to get catch-rate, precision, - F1, false-positive, and per-severity metrics, plus a per-case breakdown. - -## Metrics - -- **Catch rate**: matched ground-truth issues / total expected issues. -- **Precision**: matched issues / (matched + false positives + redundant matches). -- **F1**: harmonic mean of catch rate and precision. -- **False positives**: reviewer findings with empty `matched_ground_truth`. -- **Redundant matches**: repeated matches to an issue already caught in the same case. -- **By-severity catch rate**: catch rate grouped by `severity`. - -## Notes - -- Cases are synthetic and self-contained; they are not tied to a specific PR or - repo. They exist to make the benchmark reproducible by anyone, anywhere. -- The sibling `benchmarks/agent-prs/` harness measures catch rate on real public - agent-generated PRs with preserved review artifacts. This `benchmark/` set - complements it with broad, language- and issue-type coverage that is cheap to - re-run. diff --git a/benchmark/cases/ts-insecure-cookie/label.json b/benchmark/cases/ts-insecure-cookie/label.json deleted file mode 100644 index 50b110a7..00000000 --- a/benchmark/cases/ts-insecure-cookie/label.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "id": "ts-insecure-cookie", - "title": "Insecure session cookie attributes in TypeScript", - "language": "typescript", - "source_file": "source.ts", - "category": "security", - "ground_truth": [ - { - "id": "cookie-missing-secure-httponly-samesite", - "type": "insecure_cookie", - "severity": "high", - "location": { "file": "source.ts", "lines": [8, 8] }, - "description": "The session cookie is set without Secure, HttpOnly, or SameSite attributes, so it is transmitted over HTTP, readable by JavaScript/XSS, and vulnerable to CSRF." - } - ] -} diff --git a/benchmark/cases/ts-insecure-cookie/source.ts b/benchmark/cases/ts-insecure-cookie/source.ts deleted file mode 100644 index 6a775001..00000000 --- a/benchmark/cases/ts-insecure-cookie/source.ts +++ /dev/null @@ -1,9 +0,0 @@ -// Case: Session cookie set without Secure, HttpOnly, or SameSite attributes. -import type { Response } from 'express'; - -export function setSessionCookie(res: Response, token: string): void { - // BUG: the cookie is set without Secure (sent over HTTP), HttpOnly (readable - // by JS/XSS), and SameSite (vulnerable to CSRF). A stolen cookie value is a - // stolen session. - res.cookie('session', token, { maxAge: 86400000 }); -} diff --git a/benchmark/cases/ts-missing-await/label.json b/benchmark/cases/ts-missing-await/label.json deleted file mode 100644 index 15185e94..00000000 --- a/benchmark/cases/ts-missing-await/label.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "id": "ts-missing-await", - "title": "Missing await on async operation in TypeScript", - "language": "typescript", - "source_file": "source.ts", - "category": "bug", - "ground_truth": [ - { - "id": "fire-and-forget-delete-session", - "type": "missing_await", - "severity": "medium", - "location": { "file": "source.ts", "lines": [8, 8] }, - "description": "deleteSession returns a promise but is not awaited, so a rejection becomes an unhandled promise rejection and logout resolves before the session is actually deleted, leaving stale sessions." - } - ] -} diff --git a/benchmark/cases/ts-missing-await/source.ts b/benchmark/cases/ts-missing-await/source.ts deleted file mode 100644 index 31216954..00000000 --- a/benchmark/cases/ts-missing-await/source.ts +++ /dev/null @@ -1,10 +0,0 @@ -// Case: Missing await on a rejected promise swallows an error. -import { deleteSession } from './session'; - -export async function logout(userId: string): Promise { - // BUG: deleteSession returns a promise but is not awaited. If it rejects, - // the rejection becomes an unhandled promise rejection and logout resolves - // as if the session were deleted, leaving stale sessions behind. - deleteSession(userId); - console.log('user logged out'); -} diff --git a/benchmark/cases/ts-prototype-pollution/label.json b/benchmark/cases/ts-prototype-pollution/label.json deleted file mode 100644 index 751f45d4..00000000 --- a/benchmark/cases/ts-prototype-pollution/label.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "id": "ts-prototype-pollution", - "title": "Prototype pollution via recursive merge in TypeScript", - "language": "typescript", - "source_file": "source.ts", - "category": "security", - "ground_truth": [ - { - "id": "proto-pollution-merge", - "type": "prototype_pollution", - "severity": "high", - "location": { "file": "source.ts", "lines": [11, 19] }, - "description": "The recursive merge walks user-supplied keys without blocking __proto__/constructor/prototype, so a payload like {\"__proto__\": {\"admin\": true}} pollutes Object.prototype and escalates privileges across the application." - } - ] -} diff --git a/benchmark/cases/ts-prototype-pollution/source.ts b/benchmark/cases/ts-prototype-pollution/source.ts deleted file mode 100644 index ecac270f..00000000 --- a/benchmark/cases/ts-prototype-pollution/source.ts +++ /dev/null @@ -1,21 +0,0 @@ -// Case: Prototype pollution via recursive object merge. -function isObject(v: unknown): v is Record { - return typeof v === 'object' && v !== null; -} - -// BUG: the merge walks user-supplied keys without blocking __proto__, -// constructor, or prototype. A payload like {"__proto__": {"admin": true}} -// pollutes Object.prototype and escalates privileges app-wide. -export function merge(target: Record, source: unknown): Record { - if (!isObject(source)) return target; - for (const key of Object.keys(source)) { - const tv = target[key]; - const sv = source[key]; - if (isObject(tv) && isObject(sv)) { - merge(tv, sv); - } else { - target[key] = sv; - } - } - return target; -} diff --git a/benchmark/cases/ts-regex-dos/label.json b/benchmark/cases/ts-regex-dos/label.json deleted file mode 100644 index 61629df1..00000000 --- a/benchmark/cases/ts-regex-dos/label.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "id": "ts-regex-dos", - "title": "Catastrophic backtracking regex (ReDoS) in TypeScript", - "language": "typescript", - "source_file": "source.ts", - "category": "security", - "ground_truth": [ - { - "id": "redos-nested-quantifier", - "type": "regex_dos", - "severity": "high", - "location": { "file": "source.ts", "lines": [4, 4] }, - "description": "The regex uses a nested + quantifier ((...+)+) causing exponential backtracking on non-matching inputs; a long crafted string hangs the event loop and denies service to all other requests." - } - ] -} diff --git a/benchmark/cases/ts-regex-dos/source.ts b/benchmark/cases/ts-regex-dos/source.ts deleted file mode 100644 index 57171e5e..00000000 --- a/benchmark/cases/ts-regex-dos/source.ts +++ /dev/null @@ -1,10 +0,0 @@ -// Case: Catastrophic backtracking regex (ReDoS). -// This regex is used to validate user-supplied email-like strings. -export const emailLikePattern = /^([a-zA-Z0-9._%+-]+)+$/; - -// BUG: the nested + quantifier ((...+)+) creates exponential backtracking on -// non-matching inputs. A long string like "a".repeat(30) + "!" hangs the event -// loop and denies service to all other requests. -export function isEmailLike(input: string): boolean { - return emailLikePattern.test(input); -} diff --git a/benchmark/cases/ts-sql-injection/label.json b/benchmark/cases/ts-sql-injection/label.json deleted file mode 100644 index e8e18a77..00000000 --- a/benchmark/cases/ts-sql-injection/label.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "id": "ts-sql-injection", - "title": "SQL injection via string concatenation in TypeScript", - "language": "typescript", - "source_file": "source.ts", - "category": "security", - "ground_truth": [ - { - "id": "sql-injection-email-concat", - "type": "sql_injection", - "severity": "high", - "location": { "file": "source.ts", "lines": [14, 14] }, - "description": "User-controlled emailInput is concatenated directly into the SQL query string, allowing injection of arbitrary SQL by breaking out of the single-quoted value." - } - ] -} diff --git a/benchmark/cases/ts-sql-injection/source.ts b/benchmark/cases/ts-sql-injection/source.ts deleted file mode 100644 index ef347c45..00000000 --- a/benchmark/cases/ts-sql-injection/source.ts +++ /dev/null @@ -1,16 +0,0 @@ -// Case: SQL injection via string concatenation in a TypeScript query builder. -import { db } from './db'; - -interface User { - id: number; - email: string; -} - -export async function findUserByEmail(emailInput: string): Promise { - // BUG: user-controlled emailInput is concatenated directly into the SQL - // string, allowing an attacker to break out of the quoted value and append - // arbitrary SQL (e.g. "' OR '1'='1"). - const sql = `SELECT id, email FROM users WHERE email = '${emailInput}' LIMIT 1`; - const rows = await db.query(sql); - return rows[0] ?? null; -} diff --git a/benchmark/cases/ts-type-confusion/label.json b/benchmark/cases/ts-type-confusion/label.json deleted file mode 100644 index 519bbaa8..00000000 --- a/benchmark/cases/ts-type-confusion/label.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "id": "ts-type-confusion", - "title": "Type-safety bypass via `as any` in TypeScript", - "language": "typescript", - "source_file": "source.ts", - "category": "bug", - "ground_truth": [ - { - "id": "as-any-type-confusion", - "type": "type_confusion", - "severity": "medium", - "location": { "file": "source.ts", "lines": [19, 19] }, - "description": "The payload is cast through `as any` to Order, bypassing the compiler so a Refund-shaped object is treated as an Order; at runtime items is undefined and the .length access throws or silently corrupts downstream totals." - } - ] -} diff --git a/benchmark/cases/ts-type-confusion/source.ts b/benchmark/cases/ts-type-confusion/source.ts deleted file mode 100644 index 628805e6..00000000 --- a/benchmark/cases/ts-type-confusion/source.ts +++ /dev/null @@ -1,20 +0,0 @@ -// Case: Type-safety bypass via `as any` that hides a real shape mismatch. -interface Order { - id: string; - total: number; - items: string[]; -} - -interface Refund { - id: string; - amount: number; - reason: string; -} - -// BUG: the caller casts the payload to `any` so the compiler cannot catch that -// a Refund is being treated as an Order. At runtime `items` is undefined and -// the .length access throws, or worse, silently corrupts downstream totals. -export function processOrder(payload: unknown): number { - const order = payload as any as Order; - return order.total + order.items.length; -} diff --git a/benchmark/cases/ts-xss/label.json b/benchmark/cases/ts-xss/label.json deleted file mode 100644 index 78b558de..00000000 --- a/benchmark/cases/ts-xss/label.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "id": "ts-xss", - "title": "Reflected XSS via dangerouslySetInnerHTML in React", - "language": "typescript", - "source_file": "source.tsx", - "category": "security", - "ground_truth": [ - { - "id": "xss-dangerously-set-inner-html", - "type": "xss", - "severity": "high", - "location": { "file": "source.tsx", "lines": [12, 12] }, - "description": "User-supplied comment body is rendered as raw HTML via dangerouslySetInnerHTML without any sanitization, enabling stored/reflected XSS." - } - ] -} diff --git a/benchmark/cases/ts-xss/source.tsx b/benchmark/cases/ts-xss/source.tsx deleted file mode 100644 index e66ca5b6..00000000 --- a/benchmark/cases/ts-xss/source.tsx +++ /dev/null @@ -1,13 +0,0 @@ -// Case: Reflected XSS via dangerouslySetInnerHTML in a React component. -import React from 'react'; - -interface CommentProps { - body: string; // user-supplied comment markdown/html -} - -export const Comment: React.FC = ({ body }) => { - // BUG: raw user-supplied content is rendered as HTML without sanitization. - // An attacker can inject