From baec91c75e0f67e23d0f174a6b4b6609ef2af2e8 Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Wed, 2 Sep 2026 21:51:13 +0530 Subject: [PATCH] test: stage consolidated benchmark corpus 2/3 --- benchmarks/public-catch-rate/README.md | 154 +++++++++++++++ .../cases/ts-dead-code/label.json | 23 +++ .../cases/ts-dead-code/source.ts | 21 +++ .../cases/ts-hardcoded-credentials/label.json | 16 ++ .../cases/ts-hardcoded-credentials/source.ts | 14 ++ .../cases/ts-insecure-cookie/label.json | 16 ++ .../cases/ts-insecure-cookie/source.ts | 9 + .../cases/ts-missing-await/label.json | 16 ++ .../cases/ts-missing-await/source.ts | 10 + .../cases/ts-prototype-pollution/label.json | 16 ++ .../cases/ts-prototype-pollution/source.ts | 21 +++ .../cases/ts-regex-dos/label.json | 16 ++ .../cases/ts-regex-dos/source.ts | 10 + .../cases/ts-sql-injection/label.json | 16 ++ .../cases/ts-sql-injection/source.ts | 16 ++ .../cases/ts-type-confusion/label.json | 16 ++ .../cases/ts-type-confusion/source.ts | 20 ++ .../public-catch-rate/cases/ts-xss/label.json | 16 ++ .../public-catch-rate/cases/ts-xss/source.tsx | 13 ++ .../go-errcheck.codevetter.raw.json | 142 ++++++++++++++ ...-hardcoded-credentials.codevetter.raw.json | 151 +++++++++++++++ .../go-nil-pointer.codevetter.raw.json | 142 ++++++++++++++ .../go-race-condition.codevetter.raw.json | 142 ++++++++++++++ .../go-sql-injection.codevetter.raw.json | 142 ++++++++++++++ .../java-insecure-random.codevetter.raw.json | 138 ++++++++++++++ .../js-eval-injection.codevetter.raw.json | 142 ++++++++++++++ .../js-open-redirect.codevetter.raw.json | 129 +++++++++++++ .../py-bare-except.codevetter.raw.json | 151 +++++++++++++++ .../py-command-injection.codevetter.raw.json | 160 ++++++++++++++++ .../py-hardcoded-secret.codevetter.raw.json | 178 ++++++++++++++++++ ...secure-deserialization.codevetter.raw.json | 142 ++++++++++++++ .../py-path-traversal.codevetter.raw.json | 142 ++++++++++++++ .../py-sql-injection.codevetter.raw.json | 133 +++++++++++++ .../reviews-raw/py-ssrf.codevetter.raw.json | 142 ++++++++++++++ .../py-weak-hash.codevetter.raw.json | 151 +++++++++++++++ .../py-zip-bomb.codevetter.raw.json | 133 +++++++++++++ .../rust-integer-overflow.codevetter.raw.json | 151 +++++++++++++++ .../ts-dead-code.codevetter.raw.json | 178 ++++++++++++++++++ 38 files changed, 3228 insertions(+) create mode 100644 benchmarks/public-catch-rate/README.md create mode 100644 benchmarks/public-catch-rate/cases/ts-dead-code/label.json create mode 100644 benchmarks/public-catch-rate/cases/ts-dead-code/source.ts create mode 100644 benchmarks/public-catch-rate/cases/ts-hardcoded-credentials/label.json create mode 100644 benchmarks/public-catch-rate/cases/ts-hardcoded-credentials/source.ts create mode 100644 benchmarks/public-catch-rate/cases/ts-insecure-cookie/label.json create mode 100644 benchmarks/public-catch-rate/cases/ts-insecure-cookie/source.ts create mode 100644 benchmarks/public-catch-rate/cases/ts-missing-await/label.json create mode 100644 benchmarks/public-catch-rate/cases/ts-missing-await/source.ts create mode 100644 benchmarks/public-catch-rate/cases/ts-prototype-pollution/label.json create mode 100644 benchmarks/public-catch-rate/cases/ts-prototype-pollution/source.ts create mode 100644 benchmarks/public-catch-rate/cases/ts-regex-dos/label.json create mode 100644 benchmarks/public-catch-rate/cases/ts-regex-dos/source.ts create mode 100644 benchmarks/public-catch-rate/cases/ts-sql-injection/label.json create mode 100644 benchmarks/public-catch-rate/cases/ts-sql-injection/source.ts create mode 100644 benchmarks/public-catch-rate/cases/ts-type-confusion/label.json create mode 100644 benchmarks/public-catch-rate/cases/ts-type-confusion/source.ts create mode 100644 benchmarks/public-catch-rate/cases/ts-xss/label.json create mode 100644 benchmarks/public-catch-rate/cases/ts-xss/source.tsx create mode 100644 benchmarks/public-catch-rate/reviews-raw/go-errcheck.codevetter.raw.json create mode 100644 benchmarks/public-catch-rate/reviews-raw/go-hardcoded-credentials.codevetter.raw.json create mode 100644 benchmarks/public-catch-rate/reviews-raw/go-nil-pointer.codevetter.raw.json create mode 100644 benchmarks/public-catch-rate/reviews-raw/go-race-condition.codevetter.raw.json create mode 100644 benchmarks/public-catch-rate/reviews-raw/go-sql-injection.codevetter.raw.json create mode 100644 benchmarks/public-catch-rate/reviews-raw/java-insecure-random.codevetter.raw.json create mode 100644 benchmarks/public-catch-rate/reviews-raw/js-eval-injection.codevetter.raw.json create mode 100644 benchmarks/public-catch-rate/reviews-raw/js-open-redirect.codevetter.raw.json create mode 100644 benchmarks/public-catch-rate/reviews-raw/py-bare-except.codevetter.raw.json create mode 100644 benchmarks/public-catch-rate/reviews-raw/py-command-injection.codevetter.raw.json create mode 100644 benchmarks/public-catch-rate/reviews-raw/py-hardcoded-secret.codevetter.raw.json create mode 100644 benchmarks/public-catch-rate/reviews-raw/py-insecure-deserialization.codevetter.raw.json create mode 100644 benchmarks/public-catch-rate/reviews-raw/py-path-traversal.codevetter.raw.json create mode 100644 benchmarks/public-catch-rate/reviews-raw/py-sql-injection.codevetter.raw.json create mode 100644 benchmarks/public-catch-rate/reviews-raw/py-ssrf.codevetter.raw.json create mode 100644 benchmarks/public-catch-rate/reviews-raw/py-weak-hash.codevetter.raw.json create mode 100644 benchmarks/public-catch-rate/reviews-raw/py-zip-bomb.codevetter.raw.json create mode 100644 benchmarks/public-catch-rate/reviews-raw/rust-integer-overflow.codevetter.raw.json create mode 100644 benchmarks/public-catch-rate/reviews-raw/ts-dead-code.codevetter.raw.json diff --git a/benchmarks/public-catch-rate/README.md b/benchmarks/public-catch-rate/README.md new file mode 100644 index 00000000..555f838b --- /dev/null +++ b/benchmarks/public-catch-rate/README.md @@ -0,0 +1,154 @@ +# CodeVetter Public Benchmark + +A public, hand-labeled benchmark for measuring whether code review / security +analysis tools actually catch known issues. Each case is a small code snippet +with one or more **hand-labeled** expected findings. The cases are intentionally +synthetic and self-contained so anyone can reproduce a score: drop a tool's +output into `reviews/.json` and run the scorer. + +This exists so enterprise claims about CodeVetter (or any reviewer) are backed +by **external, repeatable proof** instead of internal fixtures that cannot be +audited. + +## Layout + +``` +benchmarks/public-catch-rate/ + cases/ + / + source. # the code snippet with known issues + label.json # hand-labeled ground truth: type, severity, location, description + reviews/ # gitignored; drop a reviewer's output here per case + .json + README.md # this file +``` + +Each `label.json` has the shape: + +```json +{ + "id": "ts-sql-injection", + "title": "SQL injection via string concatenation in TypeScript", + "language": "typescript", + "source_file": "source.ts", + "category": "security", + "ground_truth": [ + { + "id": "sql-injection-email-concat", + "type": "sql_injection", + "severity": "high", + "location": { "file": "source.ts", "lines": [14, 14] }, + "description": "User-controlled emailInput is concatenated directly into the SQL query string ..." + } + ] +} +``` + +A reviewer output file (`reviews/.json`) has the shape: + +```json +{ + "case_id": "ts-sql-injection", + "reviewer": "codevetter", + "findings": [ + { + "id": "f-1", + "type": "sql_injection", + "severity": "high", + "file": "source.ts", + "lines": [14, 14], + "title": "SQL injection via string concatenation", + "matched_ground_truth": ["sql-injection-email-concat"], + "rationale": "Identifies the same concatenated user input into the SQL string." + } + ] +} +``` + +`matched_ground_truth` lists the ground-truth ids the finding catches. Findings +with an empty `matched_ground_truth` count as false positives. + +## Cases (27) + +| Case | Language | Category | Issue type | +| --- | --- | --- | --- | +| ts-sql-injection | TypeScript | security | sql_injection | +| py-hardcoded-secret | Python | security | hardcoded_secret | +| go-race-condition | Go | concurrency | race_condition | +| ts-xss | TypeScript | security | xss | +| py-path-traversal | Python | security | path_traversal | +| js-eval-injection | JavaScript | security | code_injection | +| rust-integer-overflow | Rust | bug | integer_overflow | +| ts-dead-code | TypeScript | maintainability | dead_code | +| py-command-injection | Python | security | command_injection | +| go-errcheck | Go | bug | unchecked_error | +| ts-hardcoded-credentials | TypeScript | security | hardcoded_secret | +| py-weak-hash | Python | security | weak_crypto | +| java-insecure-random | Java | security | insecure_random | +| ts-prototype-pollution | TypeScript | security | prototype_pollution | +| py-sql-injection | Python | security | sql_injection | +| go-sql-injection | Go | security | sql_injection | +| ts-missing-await | TypeScript | bug | missing_await | +| py-bare-except | Python | bug | swallowed_error | +| js-open-redirect | JavaScript | security | open_redirect | +| ts-insecure-cookie | TypeScript | security | insecure_cookie | +| py-ssrf | Python | security | ssrf | +| go-hardcoded-credentials | Go | security | hardcoded_secret | +| ts-regex-dos | TypeScript | security | regex_dos | +| py-zip-bomb | Python | security | resource_exhaustion | +| ts-type-confusion | TypeScript | bug | type_confusion | +| py-insecure-deserialization | Python | security | insecure_deserialization | +| go-nil-pointer | Go | bug | nil_dereference | + +Coverage spans TypeScript, JavaScript, Python, Go, Rust, and Java across +security, concurrency, bug, and maintainability categories. + +## Running the scorer + +From the repo root: + +```bash +# Validate every case and print a scorecard of the labeled ground truth. +# This requires no reviewer output and always works. +npm run bench:public + +# Score a reviewer's output after dropping files into benchmarks/public-catch-rate/reviews/. +npm run bench:public -- --reviewer=codevetter + +# Emit a JSON scorecard. +npm run bench:public -- --reviewer=codevetter --json + +# Write a Markdown scorecard to disk. +npm run bench:public -- --reviewer=codevetter --format=markdown --out=artifacts/public-benchmark.md + +# Gate on minimum catch rate (exits non-zero when below threshold). +npm run bench:public -- --reviewer=codevetter --min-rate=0.8 +``` + +## How to evaluate a tool against this benchmark + +1. For each case in `benchmarks/public-catch-rate/cases//`, feed `source.` to your + reviewer (CodeVetter or any comparator). +2. Normalize the reviewer's findings into the `reviews/.json` shape + above, filling `matched_ground_truth` with the ground-truth ids each finding + catches (leave empty for findings that do not match any labeled issue). +3. Run `npm run bench:public -- --reviewer=` to get catch-rate, precision, + F1, false-positive, and per-severity metrics, plus a per-case breakdown. + +## Metrics + +- **Catch rate**: matched ground-truth issues / total expected issues. +- **Precision**: matched issues / (matched + false positives + redundant matches). +- **F1**: harmonic mean of catch rate and precision. +- **False positives**: reviewer findings with empty `matched_ground_truth`. +- **Redundant matches**: repeated matches to an issue already caught in the same case. +- **By-severity catch rate**: catch rate grouped by `severity`. + +## Notes + +- Cases are synthetic and self-contained; they are not tied to a specific PR or + repo. They exist to make the benchmark reproducible by anyone, anywhere. +- The sibling `benchmarks/agent-prs/` harness measures catch rate on real public + agent-generated PRs with preserved review artifacts. This `benchmarks/public-catch-rate/` set + complements it with broad, language- and issue-type coverage that is cheap to + re-run. diff --git a/benchmarks/public-catch-rate/cases/ts-dead-code/label.json b/benchmarks/public-catch-rate/cases/ts-dead-code/label.json new file mode 100644 index 00000000..a106da7b --- /dev/null +++ b/benchmarks/public-catch-rate/cases/ts-dead-code/label.json @@ -0,0 +1,23 @@ +{ + "id": "ts-dead-code", + "title": "Unreachable code after unconditional return", + "language": "typescript", + "source_file": "source.ts", + "category": "maintainability", + "ground_truth": [ + { + "id": "unreachable-branch-after-return", + "type": "dead_code", + "severity": "medium", + "location": { "file": "source.ts", "lines": [12, 15] }, + "description": "The if (score >= 70) branch and final return are unreachable because an unconditional return at line 8 always exits the function first." + }, + { + "id": "unused-helper-function", + "type": "dead_code", + "severity": "low", + "location": { "file": "source.ts", "lines": [18, 20] }, + "description": "neverCalled is defined but never referenced anywhere, so it is dead code that should be removed." + } + ] +} diff --git a/benchmarks/public-catch-rate/cases/ts-dead-code/source.ts b/benchmarks/public-catch-rate/cases/ts-dead-code/source.ts new file mode 100644 index 00000000..aa04a459 --- /dev/null +++ b/benchmarks/public-catch-rate/cases/ts-dead-code/source.ts @@ -0,0 +1,21 @@ +// Case: Dead/unreachable code after an unconditional return. +export function classify(score: number): string { + if (score >= 90) { + return 'A'; + } + if (score >= 80) { + return 'B'; + } + return 'C'; + + // BUG: everything below this point is unreachable. The unconditional return + // above means this branch can never execute, and the helper is never used. + if (score >= 70) { + return 'D'; + } + return 'F'; +} + +function neverCalled(): void { + console.log('this function has no callers'); +} diff --git a/benchmarks/public-catch-rate/cases/ts-hardcoded-credentials/label.json b/benchmarks/public-catch-rate/cases/ts-hardcoded-credentials/label.json new file mode 100644 index 00000000..01da3d62 --- /dev/null +++ b/benchmarks/public-catch-rate/cases/ts-hardcoded-credentials/label.json @@ -0,0 +1,16 @@ +{ + "id": "ts-hardcoded-credentials", + "title": "Hardcoded database password in TypeScript", + "language": "typescript", + "source_file": "source.ts", + "category": "security", + "ground_truth": [ + { + "id": "hardcoded-db-password", + "type": "hardcoded_secret", + "severity": "high", + "location": { "file": "source.ts", "lines": [8, 8] }, + "description": "The production database password is committed in plaintext inside the source file, exposing credentials to anyone with repository access." + } + ] +} diff --git a/benchmarks/public-catch-rate/cases/ts-hardcoded-credentials/source.ts b/benchmarks/public-catch-rate/cases/ts-hardcoded-credentials/source.ts new file mode 100644 index 00000000..7406c01c --- /dev/null +++ b/benchmarks/public-catch-rate/cases/ts-hardcoded-credentials/source.ts @@ -0,0 +1,14 @@ +// Case: Hardcoded database credentials in a TypeScript service config. +export const dbConfig = { + host: 'db.prod.internal', + port: 5432, + user: 'admin', + // BUG: the production database password is committed in plaintext. + password: 'P@ssw0rd-prod-2024!', + database: 'orders', +}; + +export async function connect() { + const url = `postgres://${dbConfig.user}:${dbConfig.password}@${dbConfig.host}:${dbConfig.port}/${dbConfig.database}`; + return fetch(url); +} diff --git a/benchmarks/public-catch-rate/cases/ts-insecure-cookie/label.json b/benchmarks/public-catch-rate/cases/ts-insecure-cookie/label.json new file mode 100644 index 00000000..50b110a7 --- /dev/null +++ b/benchmarks/public-catch-rate/cases/ts-insecure-cookie/label.json @@ -0,0 +1,16 @@ +{ + "id": "ts-insecure-cookie", + "title": "Insecure session cookie attributes in TypeScript", + "language": "typescript", + "source_file": "source.ts", + "category": "security", + "ground_truth": [ + { + "id": "cookie-missing-secure-httponly-samesite", + "type": "insecure_cookie", + "severity": "high", + "location": { "file": "source.ts", "lines": [8, 8] }, + "description": "The session cookie is set without Secure, HttpOnly, or SameSite attributes, so it is transmitted over HTTP, readable by JavaScript/XSS, and vulnerable to CSRF." + } + ] +} diff --git a/benchmarks/public-catch-rate/cases/ts-insecure-cookie/source.ts b/benchmarks/public-catch-rate/cases/ts-insecure-cookie/source.ts new file mode 100644 index 00000000..6a775001 --- /dev/null +++ b/benchmarks/public-catch-rate/cases/ts-insecure-cookie/source.ts @@ -0,0 +1,9 @@ +// Case: Session cookie set without Secure, HttpOnly, or SameSite attributes. +import type { Response } from 'express'; + +export function setSessionCookie(res: Response, token: string): void { + // BUG: the cookie is set without Secure (sent over HTTP), HttpOnly (readable + // by JS/XSS), and SameSite (vulnerable to CSRF). A stolen cookie value is a + // stolen session. + res.cookie('session', token, { maxAge: 86400000 }); +} diff --git a/benchmarks/public-catch-rate/cases/ts-missing-await/label.json b/benchmarks/public-catch-rate/cases/ts-missing-await/label.json new file mode 100644 index 00000000..15185e94 --- /dev/null +++ b/benchmarks/public-catch-rate/cases/ts-missing-await/label.json @@ -0,0 +1,16 @@ +{ + "id": "ts-missing-await", + "title": "Missing await on async operation in TypeScript", + "language": "typescript", + "source_file": "source.ts", + "category": "bug", + "ground_truth": [ + { + "id": "fire-and-forget-delete-session", + "type": "missing_await", + "severity": "medium", + "location": { "file": "source.ts", "lines": [8, 8] }, + "description": "deleteSession returns a promise but is not awaited, so a rejection becomes an unhandled promise rejection and logout resolves before the session is actually deleted, leaving stale sessions." + } + ] +} diff --git a/benchmarks/public-catch-rate/cases/ts-missing-await/source.ts b/benchmarks/public-catch-rate/cases/ts-missing-await/source.ts new file mode 100644 index 00000000..31216954 --- /dev/null +++ b/benchmarks/public-catch-rate/cases/ts-missing-await/source.ts @@ -0,0 +1,10 @@ +// Case: Missing await on a rejected promise swallows an error. +import { deleteSession } from './session'; + +export async function logout(userId: string): Promise { + // BUG: deleteSession returns a promise but is not awaited. If it rejects, + // the rejection becomes an unhandled promise rejection and logout resolves + // as if the session were deleted, leaving stale sessions behind. + deleteSession(userId); + console.log('user logged out'); +} diff --git a/benchmarks/public-catch-rate/cases/ts-prototype-pollution/label.json b/benchmarks/public-catch-rate/cases/ts-prototype-pollution/label.json new file mode 100644 index 00000000..751f45d4 --- /dev/null +++ b/benchmarks/public-catch-rate/cases/ts-prototype-pollution/label.json @@ -0,0 +1,16 @@ +{ + "id": "ts-prototype-pollution", + "title": "Prototype pollution via recursive merge in TypeScript", + "language": "typescript", + "source_file": "source.ts", + "category": "security", + "ground_truth": [ + { + "id": "proto-pollution-merge", + "type": "prototype_pollution", + "severity": "high", + "location": { "file": "source.ts", "lines": [11, 19] }, + "description": "The recursive merge walks user-supplied keys without blocking __proto__/constructor/prototype, so a payload like {\"__proto__\": {\"admin\": true}} pollutes Object.prototype and escalates privileges across the application." + } + ] +} diff --git a/benchmarks/public-catch-rate/cases/ts-prototype-pollution/source.ts b/benchmarks/public-catch-rate/cases/ts-prototype-pollution/source.ts new file mode 100644 index 00000000..ecac270f --- /dev/null +++ b/benchmarks/public-catch-rate/cases/ts-prototype-pollution/source.ts @@ -0,0 +1,21 @@ +// Case: Prototype pollution via recursive object merge. +function isObject(v: unknown): v is Record { + return typeof v === 'object' && v !== null; +} + +// BUG: the merge walks user-supplied keys without blocking __proto__, +// constructor, or prototype. A payload like {"__proto__": {"admin": true}} +// pollutes Object.prototype and escalates privileges app-wide. +export function merge(target: Record, source: unknown): Record { + if (!isObject(source)) return target; + for (const key of Object.keys(source)) { + const tv = target[key]; + const sv = source[key]; + if (isObject(tv) && isObject(sv)) { + merge(tv, sv); + } else { + target[key] = sv; + } + } + return target; +} diff --git a/benchmarks/public-catch-rate/cases/ts-regex-dos/label.json b/benchmarks/public-catch-rate/cases/ts-regex-dos/label.json new file mode 100644 index 00000000..61629df1 --- /dev/null +++ b/benchmarks/public-catch-rate/cases/ts-regex-dos/label.json @@ -0,0 +1,16 @@ +{ + "id": "ts-regex-dos", + "title": "Catastrophic backtracking regex (ReDoS) in TypeScript", + "language": "typescript", + "source_file": "source.ts", + "category": "security", + "ground_truth": [ + { + "id": "redos-nested-quantifier", + "type": "regex_dos", + "severity": "high", + "location": { "file": "source.ts", "lines": [4, 4] }, + "description": "The regex uses a nested + quantifier ((...+)+) causing exponential backtracking on non-matching inputs; a long crafted string hangs the event loop and denies service to all other requests." + } + ] +} diff --git a/benchmarks/public-catch-rate/cases/ts-regex-dos/source.ts b/benchmarks/public-catch-rate/cases/ts-regex-dos/source.ts new file mode 100644 index 00000000..57171e5e --- /dev/null +++ b/benchmarks/public-catch-rate/cases/ts-regex-dos/source.ts @@ -0,0 +1,10 @@ +// Case: Catastrophic backtracking regex (ReDoS). +// This regex is used to validate user-supplied email-like strings. +export const emailLikePattern = /^([a-zA-Z0-9._%+-]+)+$/; + +// BUG: the nested + quantifier ((...+)+) creates exponential backtracking on +// non-matching inputs. A long string like "a".repeat(30) + "!" hangs the event +// loop and denies service to all other requests. +export function isEmailLike(input: string): boolean { + return emailLikePattern.test(input); +} diff --git a/benchmarks/public-catch-rate/cases/ts-sql-injection/label.json b/benchmarks/public-catch-rate/cases/ts-sql-injection/label.json new file mode 100644 index 00000000..e8e18a77 --- /dev/null +++ b/benchmarks/public-catch-rate/cases/ts-sql-injection/label.json @@ -0,0 +1,16 @@ +{ + "id": "ts-sql-injection", + "title": "SQL injection via string concatenation in TypeScript", + "language": "typescript", + "source_file": "source.ts", + "category": "security", + "ground_truth": [ + { + "id": "sql-injection-email-concat", + "type": "sql_injection", + "severity": "high", + "location": { "file": "source.ts", "lines": [14, 14] }, + "description": "User-controlled emailInput is concatenated directly into the SQL query string, allowing injection of arbitrary SQL by breaking out of the single-quoted value." + } + ] +} diff --git a/benchmarks/public-catch-rate/cases/ts-sql-injection/source.ts b/benchmarks/public-catch-rate/cases/ts-sql-injection/source.ts new file mode 100644 index 00000000..ef347c45 --- /dev/null +++ b/benchmarks/public-catch-rate/cases/ts-sql-injection/source.ts @@ -0,0 +1,16 @@ +// Case: SQL injection via string concatenation in a TypeScript query builder. +import { db } from './db'; + +interface User { + id: number; + email: string; +} + +export async function findUserByEmail(emailInput: string): Promise { + // BUG: user-controlled emailInput is concatenated directly into the SQL + // string, allowing an attacker to break out of the quoted value and append + // arbitrary SQL (e.g. "' OR '1'='1"). + const sql = `SELECT id, email FROM users WHERE email = '${emailInput}' LIMIT 1`; + const rows = await db.query(sql); + return rows[0] ?? null; +} diff --git a/benchmarks/public-catch-rate/cases/ts-type-confusion/label.json b/benchmarks/public-catch-rate/cases/ts-type-confusion/label.json new file mode 100644 index 00000000..519bbaa8 --- /dev/null +++ b/benchmarks/public-catch-rate/cases/ts-type-confusion/label.json @@ -0,0 +1,16 @@ +{ + "id": "ts-type-confusion", + "title": "Type-safety bypass via `as any` in TypeScript", + "language": "typescript", + "source_file": "source.ts", + "category": "bug", + "ground_truth": [ + { + "id": "as-any-type-confusion", + "type": "type_confusion", + "severity": "medium", + "location": { "file": "source.ts", "lines": [19, 19] }, + "description": "The payload is cast through `as any` to Order, bypassing the compiler so a Refund-shaped object is treated as an Order; at runtime items is undefined and the .length access throws or silently corrupts downstream totals." + } + ] +} diff --git a/benchmarks/public-catch-rate/cases/ts-type-confusion/source.ts b/benchmarks/public-catch-rate/cases/ts-type-confusion/source.ts new file mode 100644 index 00000000..628805e6 --- /dev/null +++ b/benchmarks/public-catch-rate/cases/ts-type-confusion/source.ts @@ -0,0 +1,20 @@ +// Case: Type-safety bypass via `as any` that hides a real shape mismatch. +interface Order { + id: string; + total: number; + items: string[]; +} + +interface Refund { + id: string; + amount: number; + reason: string; +} + +// BUG: the caller casts the payload to `any` so the compiler cannot catch that +// a Refund is being treated as an Order. At runtime `items` is undefined and +// the .length access throws, or worse, silently corrupts downstream totals. +export function processOrder(payload: unknown): number { + const order = payload as any as Order; + return order.total + order.items.length; +} diff --git a/benchmarks/public-catch-rate/cases/ts-xss/label.json b/benchmarks/public-catch-rate/cases/ts-xss/label.json new file mode 100644 index 00000000..78b558de --- /dev/null +++ b/benchmarks/public-catch-rate/cases/ts-xss/label.json @@ -0,0 +1,16 @@ +{ + "id": "ts-xss", + "title": "Reflected XSS via dangerouslySetInnerHTML in React", + "language": "typescript", + "source_file": "source.tsx", + "category": "security", + "ground_truth": [ + { + "id": "xss-dangerously-set-inner-html", + "type": "xss", + "severity": "high", + "location": { "file": "source.tsx", "lines": [12, 12] }, + "description": "User-supplied comment body is rendered as raw HTML via dangerouslySetInnerHTML without any sanitization, enabling stored/reflected XSS." + } + ] +} diff --git a/benchmarks/public-catch-rate/cases/ts-xss/source.tsx b/benchmarks/public-catch-rate/cases/ts-xss/source.tsx new file mode 100644 index 00000000..e66ca5b6 --- /dev/null +++ b/benchmarks/public-catch-rate/cases/ts-xss/source.tsx @@ -0,0 +1,13 @@ +// Case: Reflected XSS via dangerouslySetInnerHTML in a React component. +import React from 'react'; + +interface CommentProps { + body: string; // user-supplied comment markdown/html +} + +export const Comment: React.FC = ({ body }) => { + // BUG: raw user-supplied content is rendered as HTML without sanitization. + // An attacker can inject