diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 006ed60..db8cbac 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/claude-code-marketplace.json", "name": "clean-code-toolkit", - "version": "3.5.2", + "version": "3.6.0", "description": "Clean-code and product-handoff tools for AI-assisted builders.", "owner": { "name": "Tarik Moody" @@ -10,7 +10,7 @@ { "name": "clean-code-toolkit", "description": "Review code, assess product readiness, refactor safely, and prepare a developer handoff.", - "version": "3.5.2", + "version": "3.6.0", "author": { "name": "Tarik Moody" }, diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 5c3a7c7..c3b343f 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "clean-code-toolkit", - "version": "3.5.2", + "version": "3.6.0", "description": "Practical clean-code, product-readiness, and developer-handoff workflows for AI-assisted projects.", "author": { "name": "Tarik Moody" diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index bc3c58b..b55e048 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -26,5 +26,8 @@ jobs: - name: Run prod-readiness-coach tests run: python -m unittest discover skills/prod-readiness-coach/tests + - name: Coverage grid must stay full + run: python skills/prod-readiness-coach/scripts/coverage_grid.py --fail-under 100 + - name: Coach audits this repository run: python skills/prod-readiness-coach/scripts/prod_audit.py --repo . --profile library --fail-on critical --output /dev/null diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a63ebc..beaa096 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## 3.6.0 + +Three review rounds in a row each found new defects, and the question was asked plainly: is there an end to this. There was no way to answer, because nobody could say how much had been checked. This release makes that a number. + +**The coverage grid.** Every check must now have a fixture that makes it fire and a fixture that makes it stay quiet. Checks a profile can skip must also have a fixture that skips them. That is 83 cells across 34 checks, and it stands at 83 of 83. `scripts/coverage_grid.py` prints it, and CI fails if it drops below 100 percent. The grid is computed by running the audit over the fixtures, so it cannot drift from the code: add a check with no fixture and the build fails with that check's id in the message. + +Before this, all 68 tests asked only whether a check fires. None asked whether it stays quiet. That is the exact hole `auth-2` fell through, wrong about half the time it spoke, for three releases, with a green suite the whole way. + +**Two false negatives the grid found in its first hour, both invisible to three rounds of human review.** +- **Sentry installed as `@sentry/nextjs` did not count as error tracking.** The dependency matcher knew `sentry` and `@sentry` but not the scope prefix, so a Next.js app with Sentry correctly wired up was told at CRITICAL that it had no error tracking at all. That is the most common error-tracker and framework pairing this tool audits. `@vercel/otel`, `@sentry/node` and every other scoped package were equally invisible. Type stubs like `@types/pino` still correctly do not count. +- **A plain `migrations/` folder was never read for destructive SQL.** The irreversible-migration check knew Prisma, Drizzle, Alembic and Rails, and nothing else. A `DROP TABLE` in the folder that dbmate, golang-migrate, node-pg-migrate, sqlx and Supabase all use went unreported. + +**Five invariants, each with a test.** Each closes a whole class of defect so it cannot come back one instance at a time: never report about a file that is not there; never claim a control exists from a text match without saying the evidence is weak; never grade what it did not scan; never ship text its own linter rejects; every check has a firing case and a quiet case. Two more properties are now enforced across every fixture: a failing check always carries a recommendation, and no finding text uses an em or en dash. + +**The stop rule.** The engine is done when the grid is full, the invariants have tests, and CI is green. After that a review hunts for a new class of defect, not a new instance of an old one, and a review that finds nothing counts as a pass. See `docs/decisions/009`. + +Tests: 114. + ## 3.5.2 A developer-experience audit of the toolkit itself: install it fresh, run the CLI, break it on purpose, read every doc link. Four things it found. diff --git a/README.md b/README.md index b2b2569..ceb39ff 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,8 @@ Repository layout: The root `CLAUDE.md` holds contributor instructions for this repository. User-facing behavior lives in the skills, commands, and the installable template. Maintainers follow the [release checklist](docs/release-checklist.md) before tagging. +The audit engine keeps a coverage grid: every check has to prove it fires when a control is missing and stays quiet when the control is there. It sits at 83 of 83 and CI fails if it slips. Run `python3 skills/prod-readiness-coach/scripts/coverage_grid.py` to see it. + Design principles, in one line each: product intent first; read before editing; evidence over confidence; the framework's conventions beat generic advice; line counts are prompts to look, not failures; small focused diffs; unknowns stay unknown. [How It Works](docs/how-it-works.md) describes each tool's boundaries. [Start Here for Vibe Coders](docs/start-here-vibe-coders.md) is the longer plain-English guide. ## Scope diff --git a/docs/decisions/009-coverage-grid-and-stop-rule.md b/docs/decisions/009-coverage-grid-and-stop-rule.md new file mode 100644 index 0000000..38116a9 --- /dev/null +++ b/docs/decisions/009-coverage-grid-and-stop-rule.md @@ -0,0 +1,23 @@ +# 009: A coverage grid, and a rule for when to stop reviewing + +**Decision.** Every check must have a fixture that makes it fire and a fixture that makes it stay quiet. Checks a profile can skip must also have a fixture that skips them. The count is computed by running the audit over the fixtures, printed by `scripts/coverage_grid.py`, and CI fails if it drops below 100 percent. Once the grid is full and the invariants have tests, only a real user's bug report changes the engine. + +**Why this came up.** Three review rounds in a row each found new defects, and the question was asked plainly: is there any end to this. There was no way to answer, because nobody could say how much had been checked. Sixty-eight tests existed and every one asked only whether a check fires. None asked whether it stays quiet when it should. That is the exact hole `auth-2` fell through: it was wrong about half the time it spoke, for three releases, with a full green suite. Without a denominator, every finding looks like proof of infinite findings, and the honest answer to "are we done" is "nobody can tell". + +**Options.** +1. Keep reviewing. Each round finds real things, and the severity has been dropping. Cost: no finish line, and no way to distinguish a review that found nothing because the code is good from one that found nothing because the reviewer looked in the wrong place. +2. Rules as data. Move every check into a YAML or Rego file carrying its own applicability and its own fixtures, the way Semgrep, KICS and Trivy do. Cost: those projects did that at thousands of rules. This engine has thirty-four. The machinery would cost more than the checks. +3. Keep the checks as Python. Add fixtures and count the grid mechanically. Borrow the contract from KICS, which refuses a query that ships without both a vulnerable and a safe fixture, and from Checkov, which requires both a PASSED and a FAILED test case. Cost: fixtures are work, and a fixture that is wrong teaches the wrong lesson confidently. + +**What we chose and why.** Option 3. Tarik's call, after research into how OpenSSF Scorecard, Semgrep, Checkov, Trivy, KICS, SonarQube and Datree each handle it. The deciding detail is that the grid is derived, not maintained. Nobody writes down which checks have which coverage. The number comes from running the audit over the fixtures, so it cannot drift from the code, and adding a check with no fixture fails the build with that check's id in the message. + +It paid immediately. Demanding a passing fixture for every check surfaced two false negatives that three rounds of human review had missed: a Sentry package installed as `@sentry/nextjs` was invisible to the error-tracking check, so a correctly instrumented Next.js app was told at CRITICAL that it had no error tracking, and a plain `migrations/` folder was never read for destructive SQL, so a `DROP TABLE` went unreported outside four specific ORMs. + +**What we gave up.** Fixtures are a maintenance surface of their own. A fixture that misrepresents a real repository will make a check look covered when it is not, and nothing in the grid can detect that. The count says a behaviour is exercised; it does not say the behaviour is right. + +**The stop rule.** The engine is done when the grid is at 100 percent, the five invariants have tests, and CI is green. After that, a review looks for a new class of defect, not a new instance of an old one, and a review that finds nothing counts as a pass. Only a bug report from a real user reopens the engine. + +**How we'll know if this was right.** The next review round finds either nothing or something in a class nobody had named before. If it finds another instance of a class already on the invariant list, the invariant was written too narrowly and that is the thing to fix. + +**What actually happened.** +(Tarik fills this in.) diff --git a/skills/prod-readiness-coach/scripts/audit/checks_runtime.py b/skills/prod-readiness-coach/scripts/audit/checks_runtime.py index 1320f29..adef809 100644 --- a/skills/prod-readiness-coach/scripts/audit/checks_runtime.py +++ b/skills/prod-readiness-coach/scripts/audit/checks_runtime.py @@ -33,9 +33,25 @@ def check_structured_logging(repo: Repo) -> list[CheckResult]: # A dependency named @types/pino is a type stub, not a logger. Match the package # name (allowing a scope prefix), never a substring of a serialized blob. def _installed(lib: str) -> bool: + """True when a dependency is this library, under any packaging style. + + Sentry ships as `@sentry/nextjs`, not `sentry`, and the scope prefix was + matched by nothing here: a Next.js app with Sentry correctly installed was + told at CRITICAL that it had no error tracking at all. That is the most + common error-tracker and framework pairing this tool audits. + """ lib = lib.lower() - return any(n == lib or (n.endswith(f"/{lib}") and not n.startswith("@types/")) - for n in dep_names) + # A type stub is a description of a library, not the library. + real = [n for n in dep_names if not n.startswith("@types/")] + return any( + n == lib + # `@foo/pino` is pino. + or n.endswith(f"/{lib}") + # `@sentry` is `@sentry/nextjs`, `@sentry/node`, `@sentry/react`. + or (lib.startswith("@") and n.startswith(f"{lib}/")) + # `sentry` is also `@sentry/nextjs`; `otel` is also `@vercel/otel`. + or (n.startswith("@") and "/" in n and n.split("/", 1)[1] == lib) + for n in real) matched_logging_libs = [lib for lib in LOGGING_LIBS if _installed(lib)] if matched_logging_libs: @@ -498,6 +514,12 @@ def check_multi_surface_deployment(repo: Repo, fp: StackFingerprint) -> list[Che "drizzle/*.sql", "**/drizzle/*.sql", "drizzle/**/*.sql", "**/drizzle/**/*.sql", "alembic/versions/*.py", "**/alembic/versions/*.py", "db/migrate/*.rb", "**/db/migrate/*.rb", + # A plain migrations folder is what dbmate, golang-migrate, node-pg-migrate, + # sqlx, Supabase and hand-rolled setups all use. Knowing only the four ORMs + # above meant a `DROP TABLE` in `migrations/` was never read. + "migrations/*.sql", "**/migrations/*.sql", "migrations/**/*.sql", + "**/migrations/**/*.sql", "db/migrations/*.sql", "supabase/migrations/*.sql", + "migrations/*.js", "migrations/*.ts", "**/migrations/*.js", "**/migrations/*.ts", ]) destructive_hits = [] for f in migration_paths: diff --git a/skills/prod-readiness-coach/scripts/coverage_grid.py b/skills/prod-readiness-coach/scripts/coverage_grid.py new file mode 100644 index 0000000..95318b2 --- /dev/null +++ b/skills/prod-readiness-coach/scripts/coverage_grid.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Print the coverage grid, so 'is there an end to this' has a number. + +Every check must prove two things: that it fires when the control is missing, +and that it stays quiet when the control is there. Checks that a profile can +skip must also have a fixture that skips them. This counts those cells. + +Run: python3 skills/prod-readiness-coach/scripts/coverage_grid.py + python3 skills/prod-readiness-coach/scripts/coverage_grid.py --fail-under 100 +""" +import argparse +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +sys.path.insert(0, str(HERE.parent / "tests")) + +import prod_audit # noqa: E402 +from test_coverage_grid import ALL_CHECK_IDS, FIXTURES, FIRING, build_grid # noqa: E402 + +MARK = {True: " yes", False: " NO"} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--fail-under", type=float, default=None, + help="Exit 1 if the filled percentage is below this number.") + parser.add_argument("--quiet", action="store_true", help="Print the summary only.") + args = parser.parse_args() + + grid = build_grid() + skippable = set(prod_audit.CHECK_SKIPS_BY_PROFILE["library"]) + + filled = required = 0 + rows = [] + for cid in ALL_CHECK_IDS: + seen = grid.get(cid, set()) + fires, quiet = bool(seen & FIRING), "pass" in seen + needs_na = cid in skippable + has_na = "n/a" in seen + required += 2 + (1 if needs_na else 0) + filled += int(fires) + int(quiet) + (int(has_na) if needs_na else 0) + rows.append((cid, fires, quiet, needs_na, has_na)) + + pct = 100.0 * filled / required if required else 100.0 + if not args.quiet: + print(f"Coverage grid, {len(FIXTURES)} fixtures over {len(ALL_CHECK_IDS)} checks\n") + print(f"{'check':10}{'fires':>7}{'quiet':>7}{'skips':>7}") + print("-" * 31) + for cid, fires, quiet, needs_na, has_na in rows: + skip = MARK[has_na] if needs_na else " ." + print(f"{cid:10}{MARK[fires]:>7}{MARK[quiet]:>7}{skip:>7}") + print("-" * 31) + + gaps = [cid for cid, fires, quiet, needs_na, has_na in rows + if not fires or not quiet or (needs_na and not has_na)] + print(f"\n{filled} of {required} cells filled ({pct:.0f}%). {len(gaps)} check(s) with a gap.") + if gaps: + print("Gaps: " + ", ".join(gaps)) + print("A gap means no fixture proves that behaviour. Add one to tests/fixtures.py.") + + if args.fail_under is not None and pct < args.fail_under: + print(f"\nBelow the floor of {args.fail_under}%.", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/prod-readiness-coach/tests/fixtures.py b/skills/prod-readiness-coach/tests/fixtures.py new file mode 100644 index 0000000..b7c99f0 --- /dev/null +++ b/skills/prod-readiness-coach/tests/fixtures.py @@ -0,0 +1,144 @@ +"""Named repository fixtures. Together these are the coverage grid. + +Every check has to prove two things: that it fires when a control is missing, +and that it stays quiet when the control is there. Only the first half was ever +tested, which is how auth-2 shipped wrong about half the time it spoke. + +The grid is computed by running the audit over these fixtures, not by anyone +maintaining a list. Add a fixture and cells fill in on their own. Add a check +with no fixture that exercises it and test_coverage_grid fails. +""" +import json + +# -------------------------------------------------------------------------- +# Node / Next.js +# -------------------------------------------------------------------------- + +NEXTJS_BARE = { + "package.json": json.dumps({"dependencies": {"next": "15.0.0"}}), + "tsconfig.json": "{}", + "package-lock.json": "{}", +} + +_COMPLETE_CI = """ +name: CI +on: + pull_request: + push: + branches: [main] +jobs: + build: + runs-on: ubuntu-latest + steps: + - run: npm ci + - run: npm run lint + - run: tsc --noEmit + - run: npx vitest run --coverage + - run: npm audit --audit-level=high + - run: npx playwright test + - run: vercel deploy --prod +""" + +_COMPLETE_RUNBOOK = """ +# Runbook + +## Rollback across every surface + +This app deploys to two places that can drift apart, so a rollback has to cover +both. Vercel hosts the web app; Convex holds the database and the functions. + +1. Vercel: open the deployment list, pick the last good build, use Instant + Rollback. This swaps the routing layer and takes about a minute. +2. Convex: run `npx convex deploy` from the matching git tag so the functions and + the schema go back together with the frontend. +3. If a migration ran, apply the reverse migration in `migrations/` before the + Convex step, because rolling back code does not undo a schema change. + +## When the database is unreachable + +The health route at /api/health returns 503 and the load balancer drains the +instance. Page the on-call engineer when the error rate passes two percent for +five minutes running. + +## Backups + +Convex keeps daily snapshots. To restore, use the dashboard export from the day +before the incident and re-import it. Confirm row counts before pointing traffic +back. +""" + +NEXTJS_COMPLETE = { + "package.json": json.dumps({ + "dependencies": {"next": "15.0.0", "convex": "1.0.0", "@clerk/nextjs": "6.0.0", + "pino": "9.0.0", "@sentry/nextjs": "8.0.0", "@upstash/ratelimit": "2.0.0"}, + "devDependencies": {"vitest": "2.0.0", "@playwright/test": "1.0.0", "typescript": "5.0.0"}, + "scripts": {"test": "vitest run", "lint": "eslint .", "build": "next build"}, + }), + "package-lock.json": "{}", + "tsconfig.json": json.dumps({"compilerOptions": {"strict": True}}), + "vitest.config.ts": "export default { test: { coverage: { lines: 80 } } }", + "playwright.config.ts": "export default { testDir: './e2e' }", + "vercel.json": json.dumps({"env": {"NEXT_PUBLIC_CONVEX_URL": "@convex-url"}}), + "convex/schema.ts": "export default {}", + "next.config.ts": 'export default { env: { STAGE: process.env.VERCEL_ENV } }', + ".github/workflows/ci.yml": _COMPLETE_CI, + ".github/dependabot.yml": 'version: 2\nupdates:\n - package-ecosystem: npm\n directory: "/"\n', + ".env.example": "CONVEX_URL=\nCLERK_SECRET_KEY=\nSENTRY_DSN=\n", + ".env.production.example": "CONVEX_URL=\n", + ".env.development.example": "CONVEX_URL=\n", + ".gitignore": ".env\n.env.local\n!.env.example\nnode_modules\n", + "CLAUDE.md": ("Next.js app on Vercel with Convex behind it. Build with npm run build. " + "Test with npm test. Lint with npm run lint. Rollback needs both surfaces, " + "see docs/runbook.md. Auth is Clerk, enforced in src/middleware.ts, and every " + "route not on the public list is protected before the handler runs."), + "docs/runbook.md": _COMPLETE_RUNBOOK, + "README.md": ("# App\n\nDeployed to Vercel with a Convex backend. To roll back, follow the " + "coordinated procedure in docs/runbook.md, which covers Vercel and Convex " + "together and tells you when to reverse a migration first. Backups are the " + "Convex daily snapshots.\n"), + "src/middleware.ts": ( + 'import { clerkMiddleware, createRouteMatcher } from "@clerk/nextjs/server";\n' + 'const isPublicRoute = createRouteMatcher(["/", "/sign-in(.*)"]);\n' + 'export default clerkMiddleware(async (auth, req) => {\n' + ' if (!isPublicRoute(req)) await auth.protect();\n});\n'), + "src/app/api/health/route.ts": ( + 'export async function GET() {\n return Response.json({ status: "ok" });\n}\n'), + "src/app/api/items/route.ts": ( + 'import { auth } from "@clerk/nextjs/server";\n' + 'import { ratelimit } from "@/lib/ratelimit";\n' + 'import { logger } from "@/lib/logger";\n\n' + 'export async function GET(req) {\n' + ' const { userId } = await auth();\n' + ' if (!userId) return new Response("no", { status: 401 });\n' + ' const requestId = req.headers.get("x-request-id") ?? crypto.randomUUID();\n' + ' const { success } = await ratelimit.limit(userId);\n' + ' if (!success) return new Response("slow down", { status: 429 });\n' + ' try {\n logger.info({ requestId }, "items");\n return Response.json([]);\n' + ' } catch (e) {\n logger.error({ requestId, e }, "items failed");\n' + ' return new Response("error", { status: 500 });\n }\n}\n'), + "src/lib/ratelimit.ts": ( + 'import { Ratelimit } from "@upstash/ratelimit";\n' + 'export const ratelimit = new Ratelimit({ limiter: Ratelimit.slidingWindow(10, "10 s") });\n'), + "src/lib/logger.ts": 'import pino from "pino";\nexport const logger = pino();\n', + "src/lib/owner.ts": ('const owner = process.env.OWNER_EMAIL;\n' + 'export function isOwner(email) {\n' + ' if (!owner) return false;\n return email === owner;\n}\n'), + "migrations/0001_add_items.sql": "ALTER TABLE items ADD COLUMN label text;\n", + "tests/items.test.ts": 'import { test } from "vitest";\ntest("items", () => {});\n', + "e2e/checkout.spec.ts": 'import { test } from "@playwright/test";\ntest("checkout", async () => {});\n', +} + +# A repo where every control is a costume: the file exists, the substance does not. +HOLLOW = { + "package.json": json.dumps({"dependencies": {"next": "15.0.0"}, + "scripts": {"test": "echo 'test skipped'"}}), + "package-lock.json": "{}", + "tsconfig.json": "{}", + ".github/workflows/ci.yml": ("on: push\njobs:\n t:\n runs-on: ubuntu-latest\n" + " steps:\n - run: echo \"running tests\"\n"), + "docs/runbook.md": "# Runbook\n\nTODO: write this.\n", + "README.md": "# App\n\n- [ ] document the rollback\n", + ".gitignore": "node_modules\n", + "src/app/api/thing/route.ts": "export async function GET() { return Response.json({}) }\n", + "migrations/0002_drop.sql": "DROP TABLE customers;\n", +} diff --git a/skills/prod-readiness-coach/tests/test_coverage_grid.py b/skills/prod-readiness-coach/tests/test_coverage_grid.py new file mode 100644 index 0000000..707a49b --- /dev/null +++ b/skills/prod-readiness-coach/tests/test_coverage_grid.py @@ -0,0 +1,209 @@ +"""The coverage grid. + +Every check must prove it fires when the control is missing AND stays quiet when +the control is there. Counting that is the point: without a denominator, every +review feels like proof of infinite findings. + +The grid is computed by running the audit over the fixtures, so it cannot drift +from the code. Add a check with no fixture that exercises both sides and this +file fails with the check's id. +""" +import json +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import prod_audit # noqa: E402 +import fixtures # noqa: E402 +from test_prod_audit import ( # noqa: E402 + GoRepoIsNotLiedTo, PythonServiceRepo, make_repo, +) + +ALL_CHECK_IDS = [ + "agent-1", + "auth-1", "auth-2", "auth-3", + "ci-1", "ci-2", "ci-3", "ci-4", "ci-5", "ci-6", + "log-1", "log-2", "log-3", "log-4", + "sec-1", "sec-2", "sec-3", "sec-4", "sec-5", + "res-1", "res-2", "res-3", "res-4", "res-5", + "ms-1", "ms-2", "ms-3", + "test-1", "test-2", "test-3", "test-4", + "dep-1", "dep-2", "dep-3", +] + +FAKE_AWS_KEY = "AKIA" + "ABCDEFGHIJKLMNOP" + +# Small fixtures that exist only to push one check to one side. +GUARDED_ROUTES = { + **fixtures.NEXTJS_BARE, + "package.json": json.dumps({"dependencies": {"next": "15.0.0", "@clerk/nextjs": "6.0.0"}}), + "src/app/api/items/route.ts": ( + 'import { auth } from "@clerk/nextjs/server";\n' + 'export async function GET() { const { userId } = await auth(); return Response.json([]); }\n'), +} +FAIL_OPEN_GUARD = { + **fixtures.NEXTJS_BARE, + "src/lib/owner.ts": ('const owner = process.env.OWNER_EMAIL;\n' + 'export function isOwner(e) { if (!owner) return true; return e === owner; }\n'), +} +LEAKY_SECRETS = { + **fixtures.NEXTJS_BARE, + "src/config.ts": f'export const key = "{FAKE_AWS_KEY}";\n' + 'const url = process.env.DATABASE_URL;\n', + ".env": "DATABASE_URL=postgres://real:secret@host/db\n", +} +UNSAFE_ROUTE = { + **fixtures.NEXTJS_BARE, + "src/app/api/pay/route.ts": ( + 'export async function POST(req) {\n' + ' const body = await req.json();\n return Response.json(charge(body));\n}\n'), +} +PLAIN_MIGRATIONS = { + **fixtures.NEXTJS_BARE, + "migrations/0001_drop_customers.sql": "DROP TABLE customers;\n", +} + +CRON_NO_HANDLING = { + **fixtures.NEXTJS_BARE, + "src/cron/cleanup.ts": ("export async function cleanup() {\n" + " const rows = await db.stale();\n" + " await db.remove(rows);\n}\n"), +} +CRON_WITH_HANDLING = { + **fixtures.NEXTJS_BARE, + "src/cron/cleanup.ts": ("export async function cleanup() {\n try {\n" + " const rows = await db.stale();\n await db.remove(rows);\n" + " } catch (e) {\n logger.error(e);\n }\n}\n"), +} +MULTI_SURFACE_UNCOORDINATED = { + **fixtures.NEXTJS_BARE, + "package.json": json.dumps({"dependencies": {"next": "15.0.0", "convex": "1.0.0"}}), + "convex/schema.ts": "export default {}", + "vercel.json": "{}", + "README.md": "# App\n\nRun npm run dev to start it locally.\n", +} + +FIXTURES = { + "nextjs-bare": (fixtures.NEXTJS_BARE, None), + "cron-no-error-handling": (CRON_NO_HANDLING, None), + "cron-with-error-handling": (CRON_WITH_HANDLING, None), + "cron-in-a-library": (CRON_NO_HANDLING, "library"), + "multi-surface-uncoordinated": (MULTI_SURFACE_UNCOORDINATED, None), + "nextjs-complete": (fixtures.NEXTJS_COMPLETE, None), + "hollow": (fixtures.HOLLOW, None), + "python-service": (PythonServiceRepo.FILES, None), + "go-service": (GoRepoIsNotLiedTo.FILES, None), + "library-profile": (GUARDED_ROUTES, "library"), + "guarded-routes": (GUARDED_ROUTES, None), + "fail-open-guard": (FAIL_OPEN_GUARD, None), + "leaky-secrets": (LEAKY_SECRETS, None), + "unsafe-route": (UNSAFE_ROUTE, None), + "plain-migrations": (PLAIN_MIGRATIONS, None), +} + +FIRING = {"fail", "warn"} + + +def build_grid() -> dict[str, set[str]]: + grid: dict[str, set[str]] = {cid: set() for cid in ALL_CHECK_IDS} + for _, (files, profile) in FIXTURES.items(): + categories, _ = prod_audit.run_audit(make_repo(files), profile) + for cat in categories: + for check in cat.checks: + grid.setdefault(check.id, set()).add(check.status) + return grid + + +GRID = build_grid() + + +class CoverageGrid(unittest.TestCase): + def test_the_registered_id_list_matches_what_the_engine_emits(self): + emitted = {cid for cid, seen in GRID.items() if seen} + self.assertEqual(sorted(emitted), sorted(ALL_CHECK_IDS), + "a check was added or removed without updating ALL_CHECK_IDS") + + def test_every_check_has_a_fixture_where_it_fires(self): + missing = sorted(c for c in ALL_CHECK_IDS if not GRID[c] & FIRING) + self.assertEqual(missing, [], f"no fixture makes these fire: {missing}") + + def test_every_check_has_a_fixture_where_it_stays_quiet(self): + missing = sorted(c for c in ALL_CHECK_IDS if "pass" not in GRID[c]) + self.assertEqual(missing, [], f"no fixture makes these pass: {missing}") + + def test_checks_that_can_be_skipped_have_a_fixture_that_skips_them(self): + skippable = sorted(prod_audit.CHECK_SKIPS_BY_PROFILE["library"]) + missing = [c for c in skippable if "n/a" not in GRID.get(c, set())] + self.assertEqual(missing, [], f"no fixture marks these n/a: {missing}") + + +class FalseNegativesTheGridFound(unittest.TestCase): + """Both of these were invisible until the grid demanded a passing fixture.""" + + def check(self, files, cid, profile=None): + categories, _ = prod_audit.run_audit(make_repo(files), profile) + return next(c for cat in categories for c in cat.checks if c.id == cid) + + def test_scoped_sentry_package_counts_as_error_tracking(self): + files = {**fixtures.NEXTJS_BARE, + "package.json": json.dumps({"dependencies": {"next": "15.0.0", + "@sentry/nextjs": "8.0.0"}})} + c = self.check(files, "log-2", profile="web-app") + self.assertEqual(c.status, "pass", c.detail) + + def test_vercel_otel_counts_as_error_tracking(self): + files = {**fixtures.NEXTJS_BARE, + "package.json": json.dumps({"dependencies": {"next": "15.0.0", + "@vercel/otel": "1.0.0"}})} + self.assertEqual(self.check(files, "log-2", profile="web-app").status, "pass") + + def test_a_plain_migrations_folder_is_read_for_destructive_sql(self): + c = self.check(PLAIN_MIGRATIONS, "ms-3", profile="web-app") + self.assertEqual(c.status, "fail", c.detail) + self.assertTrue(any("drop_customers" in e for e in c.evidence), c.evidence) + + +class Invariants(unittest.TestCase): + """Five rules the engine must never break. Each one closes a whole class of + bug, so it cannot come back one instance at a time. + + 1. Never report about a file that is not there. (test_prod_audit: Go fixture) + 2. Never claim a control exists from a text match (here) + without saying the evidence is weak. + 3. Never grade what it did not scan. (test_prod_audit: RefusesToGradeNothing) + 4. Never ship text its own linter rejects. (test_prod_audit: HouseStyle) + 5. Every check has a firing case and a quiet case. (CoverageGrid, above) + """ + + def all_checks(self): + for name, (files, profile) in FIXTURES.items(): + categories, _ = prod_audit.run_audit(make_repo(files), profile) + for cat in categories: + for check in cat.checks: + yield name, check + + def test_a_pass_with_no_evidence_is_never_called_verified(self): + bad = [f"{n}:{c.id}" for n, c in self.all_checks() + if c.status == "pass" and not c.evidence and c.confidence != "weak"] + self.assertEqual(bad, [], f"passing with nothing to show, but marked verified: {bad}") + + def test_every_failing_check_tells_the_reader_what_to_do(self): + bad = [f"{n}:{c.id}" for n, c in self.all_checks() + if c.status == "fail" and not c.recommendation.strip()] + self.assertEqual(bad, [], f"a failure with no recommendation: {bad}") + + def test_no_check_ever_crashes_on_any_fixture(self): + ids = {c.id for _, c in self.all_checks()} + self.assertTrue(ids.issubset(set(ALL_CHECK_IDS)), ids - set(ALL_CHECK_IDS)) + + def test_no_shipped_finding_text_uses_an_em_or_en_dash(self): + bad = [f"{n}:{c.id}" for n, c in self.all_checks() + if "\u2014" in (c.detail + c.recommendation) + or "\u2013" in (c.detail + c.recommendation)] + self.assertEqual(bad, [], f"em or en dash in a finding: {bad}") + + +if __name__ == "__main__": + unittest.main()