From bf2105528bc7f8a2964d67f887eb3727bffaeb2a Mon Sep 17 00:00:00 2001 From: omsherikar Date: Thu, 14 May 2026 16:36:47 +0530 Subject: [PATCH 1/5] feat(cli): per-transform extended after-context for source excerpts (gauntlet G3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Function-level transforms (callback_to_async_await, class_to_dataclass, promise_chains_to_async, promise_constructor_to_async, commonjs_to_esm, manual_typecheck_to_hints) fire on the *declaration* line, so the default ±1 window only shows the signature + docstring and hides the actual code being refactored. Per-transform overrides give each pattern the right amount of after-context without bloating output for single-line transforms. Single-line transforms (format_to_fstring, var_to_const_let, etc.) keep the tight default ±1 window — verified by a regression test that asserts unrelated body lines are NOT pulled into the excerpt for a format_to_fstring finding. Two new tests in format-analysis.test.ts (8 → cover both 'extends context for function-level transforms' AND 'keeps tight context for single-line transforms — no over-pull'). --- src/cli/format-analysis.ts | 18 ++++++- tests/unit/cli/format-analysis.test.ts | 71 ++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/src/cli/format-analysis.ts b/src/cli/format-analysis.ts index 413092f..02db43c 100644 --- a/src/cli/format-analysis.ts +++ b/src/cli/format-analysis.ts @@ -87,6 +87,21 @@ class SourceCache { } } +// Some transforms fire on the *declaration* line of a function/class, so a +// ±1 window only shows the signature + docstring and hides the actual code +// being refactored (e.g. callback_to_async_await fires on `def fn(callback):` +// but the user needs to see the `callback(result)` invocation a few lines +// below). Per-transform `linesAfter` overrides give the right context for +// each pattern without bloating output for single-line transforms. +const EXTENDED_AFTER_BY_TRANSFORM: Partial> = { + callback_to_async_await: 6, + class_to_dataclass: 5, + promise_chains_to_async: 5, + promise_constructor_to_async: 6, + commonjs_to_esm: 3, + manual_typecheck_to_hints: 4, +}; + function excerptFor( lines: string[], targetLine: number, @@ -161,7 +176,8 @@ export async function formatAnalysisReport( if (sourceLines === null) { out.push({ text: ` (source unavailable)`, color: theme.colors.textDim }); } else { - const excerpt = excerptFor(sourceLines, finding.line, linesBefore, linesAfter); + const effAfter = EXTENDED_AFTER_BY_TRANSFORM[finding.transformId] ?? linesAfter; + const excerpt = excerptFor(sourceLines, finding.line, linesBefore, effAfter); for (const e of excerpt) { out.push({ text: ` ${e.text}`, diff --git a/tests/unit/cli/format-analysis.test.ts b/tests/unit/cli/format-analysis.test.ts index 7cb851d..c7f4c22 100644 --- a/tests/unit/cli/format-analysis.test.ts +++ b/tests/unit/cli/format-analysis.test.ts @@ -132,6 +132,77 @@ describe('formatAnalysisReport', () => { expect(text).toMatch(/1 medium/); }); + it('extends after-context for function-level transforms (gauntlet G3)', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'fa-')); + tmp.push(root); + // 8 lines: def header at line 1, body lines 2-7 (the actual code being + // refactored is on line 5), blank at 8. With default linesAfter=1 the + // user only sees lines 1-2 (def + first body line) and misses the + // callback invocation. With per-transform extended context, we should + // see line 5 in the excerpt. + await fs.writeFile( + path.join(root, 'a.py'), + [ + 'def fetch_user(user_id, callback):', + ' """Docstring."""', + ' # comment', + ' result = lookup(user_id)', + ' callback(result)', + ' metric.tick()', + ' return None', + '', + ].join('\n'), + ); + const report = synthReport(root, [ + { + id: '1', + file: 'a.py', + line: 1, + transformId: 'callback_to_async_await', + remediationMinutes: 7, + confidence: 'high', + }, + ]); + const lines = await formatAnalysisReport(report, { projectRoot: root }); + const text = lines.map((l) => l.text).join('\n'); + expect(text).toContain('callback(result)'); + expect(text).toContain('metric.tick()'); + }); + + it('keeps tight context for single-line transforms (gauntlet G3 — no over-pull)', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'fa-')); + tmp.push(root); + // For format_to_fstring (single-line transform) the default ±1 window + // should NOT be extended — we don't want to bloat output by pulling 6 + // unrelated body lines for a one-line refactor. + await fs.writeFile( + path.join(root, 'b.py'), + [ + 'def greet(name):', + ' return "hello %s" % name', + ' unrelated_line_three()', + ' unrelated_line_four()', + ' unrelated_line_five()', + ' unrelated_line_six()', + ' unrelated_line_seven()', + ].join('\n'), + ); + const report = synthReport(root, [ + { + id: '1', + file: 'b.py', + line: 2, + transformId: 'format_to_fstring', + remediationMinutes: 1, + confidence: 'high', + }, + ]); + const lines = await formatAnalysisReport(report, { projectRoot: root }); + const text = lines.map((l) => l.text).join('\n'); + expect(text).toContain('return "hello %s" % name'); + expect(text).not.toContain('unrelated_line_six()'); + }); + it('sorts findings within a file by line number (gauntlet G2)', async () => { const root = await fs.mkdtemp(path.join(os.tmpdir(), 'fa-')); tmp.push(root); From 0fbff66d8d7104a89200680f47f0c07810453706 Mon Sep 17 00:00:00 2001 From: omsherikar Date: Thu, 14 May 2026 16:37:05 +0530 Subject: [PATCH 2/5] docs(readme): document the self-test paradox and the exclude workaround (gauntlet G4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a user clones Refactron and runs 'refactron run --apply' on the repo itself, the test gate fails by design — the meta-tests in tests/unit/transform/transforms/ exercise the transforms on the fixtures, and refactoring those fixtures breaks the meta-tests' input. This is exactly the safety guarantee firing as intended (no files written when the test gate fails) but it confuses first-time users on Refactron's own checkout. Add a 'Known Limitations' section before Contributing with the explanation and the .refactronrc.json { exclude: ['fixtures/**'] } workaround. --- README.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/README.md b/README.md index a7334dd..ba474a8 100644 --- a/README.md +++ b/README.md @@ -258,6 +258,33 @@ Extend `BaseFixer` from `src/autofix/fixers/base.ts`, declare `supportedIssueTyp --- +## Known Limitations + +### Running Refactron on the Refactron repo (self-test paradox) + +If you `git clone` Refactron and run `refactron run --apply` on the repo +itself, the test gate **will** fail and **no files will be written**. + +That's working as designed. Refactron's own test suite includes meta-tests +that exercise the transforms on `fixtures/python-legacy-mini/` and +`fixtures/ts-legacy-mini/` — fixtures that are deliberately full of legacy +patterns. Running the transforms on those fixtures produces refactored code, +which then no longer matches what the meta-tests expect as input. The +verification engine catches the regression and refuses to write — exactly +what would happen on any project where a refactor breaks downstream tests. + +To self-analyze without triggering this, exclude the fixtures via +`.refactronrc.json`: + +```json +{ "exclude": ["fixtures/**"] } +``` + +`refactron analyze .` then returns `No findings.` and `run --apply .` +becomes a no-op. + +--- + ## Contributing See [CONTRIBUTING.md](./CONTRIBUTING.md). From 96a44a6e0425b5c7de32a855f8a4df3722f3dacd Mon Sep 17 00:00:00 2001 From: omsherikar Date: Thu, 14 May 2026 16:37:31 +0530 Subject: [PATCH 3/5] feat(bench): synthetic fixture generator + perf bench scaffolding for week 7 day 49 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bench/gen-fixture.ts produces a directory tree of mixed Python + TypeScript files at a target LOC count, with every Refactron transform pattern represented so analyze finds work to do. 50/50 split between Python and TypeScript; ~100 files per subdirectory to avoid pathological single-dir sizes. Generated trees gitignored — they're large (a 100k-LOC tree is ~6 MB, a 500k-LOC tree is ~30 MB) and trivially regenerable. README documents the bench protocol + the Week 7 perf targets. Bench results captured in dev-docs/decisions/09-week-7-architecture.md: - 10k LOC analyze: 1.31s (target 6s, 4.6x headroom) - 100k LOC analyze: 11.48s (target 60s, 5.2x headroom) - 500k LOC: deferred to local-only; targets met without profiling needed. --- .gitignore | 1 + bench/README.md | 36 +++++++++++++ bench/gen-fixture.ts | 118 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 155 insertions(+) create mode 100644 bench/README.md create mode 100644 bench/gen-fixture.ts diff --git a/.gitignore b/.gitignore index b5e9882..f929509 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ YRC/ __pycache__/ *.pyc .pytest_cache/ +bench/*-loc/ diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 0000000..7048d60 --- /dev/null +++ b/bench/README.md @@ -0,0 +1,36 @@ +# bench/ + +Synthetic fixture generator for the Week 7 perf bench. + +The generated trees are NOT committed (they're large and easy to regenerate). +Run the generator locally before benchmarking. + +## Usage + +```bash +# Generate a synthetic 10k-LOC fixture mixing Python + TypeScript files +# sprinkled with every Refactron transform pattern. +npx tsx bench/gen-fixture.ts 10000 bench/10k-loc + +# Run analyze and time it. +time REFACTRON_TOKEN=dummy node dist/cli/index.js analyze bench/10k-loc + +# Cleanup +rm -rf bench/10k-loc +``` + +## Targets (Week 7 binary gate) + +| Tree size | Target | +|---|---| +| 10k LOC | < 6s for `analyze` | +| 100k LOC | < 60s for `analyze` | +| 500k LOC | < 5min for `analyze` | +| 100k LOC + run --apply | < 5min including test gate | + +The 500k tree generation requires ~25 MB of disk and ~30s wall-clock on a +modern dev machine. Skip it on CI; run locally before release. + +## Most recent results + +See `dev-docs/decisions/09-week-7-architecture.md` for benchmark snapshots. diff --git a/bench/gen-fixture.ts b/bench/gen-fixture.ts new file mode 100644 index 0000000..3cf838d --- /dev/null +++ b/bench/gen-fixture.ts @@ -0,0 +1,118 @@ +// bench/gen-fixture.ts +// Synthetic fixture generator for the Week 7 perf bench. Produces a directory +// tree of mixed Python + TypeScript files at a target line count, sprinkled +// with one of each Refactron transform pattern so `analyze` finds work to do. +// +// Usage: +// tsx bench/gen-fixture.ts +// node --loader ts-node/esm bench/gen-fixture.ts 10000 bench/10k-loc +// +// We DON'T commit the generated trees — they're large and easy to regenerate. +// Run this script locally before invoking the bench script. + +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; + +const PY_TEMPLATE = (i: number): string => `# Generated module ${i} +import requests + +def fetch_${i}(user_id, callback): + """Generated callback-style fetch fixture #${i}.""" + result = requests.get("/users/%s" % user_id) + payload = "Loaded user %s" % user_id + callback(payload) + return result + + +class User${i}: + def __init__(self, id, name, email): + self.id = id + self.name = name + self.email = email + + +def check_${i}(value): + if isinstance(value, str): + return value.upper() + if isinstance(value, int): + return str(value) + return None +`; + +const TS_TEMPLATE = (i: number): string => `// Generated module ${i} +const path = require('path'); + +export function makeGreeting${i}(name) { + var greeting = 'hi-${i}'; + return greeting + ', ' + name; +} + +export function chain${i}(input: any) { + return Promise.resolve(input) + .then((v) => v + 1) + .then((v) => v * 2); +} + +export function build${i}() { + return new Promise((resolve, reject) => { + setTimeout(() => resolve(${i}), 10); + }); +} + +module.exports = { makeGreeting${i}, chain${i}, build${i} }; +`; + +// Approximate LOC per template (counted from the strings above). +const PY_LOC_PER_FILE = 24; +const TS_LOC_PER_FILE = 21; + +async function generate(outDir: string, targetLoc: number): Promise { + await fs.rm(outDir, { recursive: true, force: true }); + await fs.mkdir(outDir, { recursive: true }); + + // 50/50 split between python and typescript. + const halfLoc = targetLoc / 2; + const pyCount = Math.ceil(halfLoc / PY_LOC_PER_FILE); + const tsCount = Math.ceil(halfLoc / TS_LOC_PER_FILE); + + // Spread into ~100 files per directory to avoid pathological dir sizes. + const pyDir = path.join(outDir, 'src_py'); + const tsDir = path.join(outDir, 'src_ts'); + await fs.mkdir(pyDir, { recursive: true }); + await fs.mkdir(tsDir, { recursive: true }); + + for (let i = 0; i < pyCount; i++) { + const sub = path.join(pyDir, `pkg_${Math.floor(i / 100)}`); + await fs.mkdir(sub, { recursive: true }); + await fs.writeFile(path.join(sub, `mod_${i}.py`), PY_TEMPLATE(i)); + } + for (let i = 0; i < tsCount; i++) { + const sub = path.join(tsDir, `pkg_${Math.floor(i / 100)}`); + await fs.mkdir(sub, { recursive: true }); + await fs.writeFile(path.join(sub, `mod_${i}.ts`), TS_TEMPLATE(i)); + } + + const totalFiles = pyCount + tsCount; + const totalLoc = pyCount * PY_LOC_PER_FILE + tsCount * TS_LOC_PER_FILE; + process.stdout.write( + `generated ${totalFiles} files (~${totalLoc} LOC) in ${outDir}\n` + + ` python: ${pyCount} files\n` + + ` typescript: ${tsCount} files\n`, + ); +} + +async function main(): Promise { + const args = process.argv.slice(2); + const targetLoc = Number(args[0]); + const outDir = args[1]; + if (!Number.isFinite(targetLoc) || targetLoc <= 0 || !outDir) { + process.stderr.write('usage: gen-fixture.ts \n'); + process.exit(1); + } + await generate(path.resolve(outDir), targetLoc); +} + +main().catch((err) => { + process.stderr.write(`gen-fixture failed: ${err}\n`); + process.exit(1); +}); From f70a77b30e4959672fbef272ca07825001e16a17 Mon Sep 17 00:00:00 2001 From: omsherikar Date: Thu, 14 May 2026 16:37:33 +0530 Subject: [PATCH 4/5] docs: ADR-009 records week 7 architecture, gauntlet outcomes, and perf bench results --- dev-docs/decisions/09-week-7-architecture.md | 120 +++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 dev-docs/decisions/09-week-7-architecture.md diff --git a/dev-docs/decisions/09-week-7-architecture.md b/dev-docs/decisions/09-week-7-architecture.md new file mode 100644 index 0000000..4106cb5 --- /dev/null +++ b/dev-docs/decisions/09-week-7-architecture.md @@ -0,0 +1,120 @@ +# ADR 009 — Week 7 Architecture (CLI Output Redesign + Polish + Perf) + +## Status +Accepted, 2026-05 (Week 7). + +## Context +Through Week 6 the v2.0 pipeline shipped functional but with summary-only +output that stripped the detail users needed to act on. The acute trigger +was the test-gate failure surface — vitest FAIL lines dropped by a +front-slice in `src/verify/gates/tests.ts:62`. The broader picture: +`analyze`, `run --dry-run`, and `run --apply` all showed less than what the +engines knew. + +Week 7 also surfaced two real bugs from the gauntlet-#1 self-repo run that +predated Week 7: +- `RefactronRc.exclude` was defined and validated since Week 5 but never + plumbed into discovery (dead config). +- The REPL never loaded `.refactronrc.json` at all (a secondary gap that + became obvious while wiring `exclude`). + +## Decision + +### Three presentation-layer modules under `src/cli/` +- **`format-analysis.ts`** — by-file findings + source excerpts + per-transform + suggestions + expanded summary block. Replaces the legacy severity-grouped + flat list with 55-char-truncated messages. +- **`format-plan.ts`** — per-file unified diffs for `run --dry-run` with + truncation, +/- counts, header per file, `--diff-context=N` and + `--files=GLOB` flags. Replaces the bullet-list-of-paths in the REPL and + the raw `generateUnifiedDiff` dump in the one-shot CLI. +- **`format-verify.ts`** — gate-by-gate progress, per-file atomic-write + list on success, structured failure surface with failing tests + in-flight + plan + reproduce hint + culprit hint on failure. Replaces the one-line + success and 4000-char-truncated failure blob. + +All three return `RenderedLine[]` consumed by both surfaces (REPL via +`onLine`, one-shot via `process.stdout.write` + `applyColor`). `--json` +output unchanged. + +### Engine-side hooks (no LOCKED contract changes) +- `RefactronVerifier` constructor opts gain `onGateComplete` and + `onShadowRoot` callbacks. The CLI uses them to stream per-gate progress + and capture the shadow tree path for the failure UX. +- `src/verify/gates/tests.ts` flips its `.slice(0, 4000)` front-slice to + `.slice(-4000)` (vitest writes FAIL at the END of output), and embeds a + structured `summarizeVitestFailures()` summary ahead of the raw tail. +- `src/verify/runners/run.ts` derives `timedOut` from observable wall-clock + + signal instead of trusting `execa.r.timedOut` — that field is unreliable + on Node 18 with `reject:false`. + +### Gauntlet-#1 fixes +- `discovery.ts` now accepts `excludeGlobs?: string[]` and merges it into the + loaded gitignore. `RefactronAnalyzer` plumbs `config.exclude` through. +- The REPL `analyze` and `run` branches now load `.refactronrc.json` (the + REPL was previously ignoring it entirely). +- `format-analysis.ts:groupByFile` sorts each file's findings by line so + rendering matches source order. +- Per-transform extended-context map in `format-analysis.ts`: function-level + transforms (callback_to_async_await, class_to_dataclass, + promise_chains_to_async, promise_constructor_to_async, commonjs_to_esm, + manual_typecheck_to_hints) get more after-context so the user can see the + body code being refactored, not just the declaration line. Single-line + transforms (format_to_fstring, var_to_const_let) keep the tight ±1 window. + +### Cross-platform consistency +- `format-types.ts:toPosix(p)` helper applied to every display-bound + `path.relative` call in the formatters. Windows users see `src/foo.py` not + `src\foo.py`. Internal uses of `relPath` (map keys, import resolution) + keep native separators. + +### Perf bench +- `bench/gen-fixture.ts` generates synthetic Python + TypeScript trees at a + target LOC count with every transform pattern represented. +- Generated trees are gitignored (regenerate locally before each run). + +## Bench results (2026-05-14, M-series macOS, Node 22) + +| Tree size | Files | `analyze` wall-clock | Target | Headroom | +|---|---|---|---|---| +| 10k LOC | 448 | 1.31s | 6s | 4.6× | +| 100k LOC | 4 465 | 11.48s | 60s | 5.2× | +| 500k LOC | — | not run | 5min | — | + +500k LOC run requires generating ~25 MB of fixture; deferred. Both measured +sizes pass with healthy headroom on first attempt — no profiling needed. + +## Consequences +- The redesign removes the test-gate front-slice bug (failure surface used + to drop the FAIL section); both `--apply` failure paths now produce + ~30-line readable output instead of a 4000-char truncated dump. +- `.refactronrc.json` is now actually consultable from both surfaces with + both `confidence` and `exclude` honored — closing a Week-5 dead-code gap. +- Cross-platform reliability improved: Windows path separators and Node 18 + execa quirks were caught and fixed during PR #19/#20 CI failures. +- Bench infrastructure exists; any future perf regression is detectable in + one command. +- `RefactorPlan`, `VerificationResult`, `Verifier`, `Documenter` interfaces in + `src/contracts.ts` — all untouched. + +## Future work (deferred from Week 7) +- **Days 46-48 gauntlets #2-#5** with real beta users. Schedule when ≥3 + external developers commit. Bug-fix day from those gauntlets follows. +- **500k LOC bench** — needs ~25 MB of generated fixture; trivially runnable + locally with `bench/gen-fixture.ts 500000`. +- **G3 polish** — current per-transform after-context is heuristic. v2.1 + could derive a real body line range from the analyzer (extending + `DetectorFinding` with `bodyEndLine?: number`). +- **`--keep-shadow` flag** for the verifier so the user can `cd` into the + shadow tree after a failure (currently it's cleaned up before the + reproduce hint is read). +- **Per-test bisection** ("transform X on file Y caused test Z to fail") + via running the test gate with progressively smaller subsets of the plan. + +## References +- Source-of-truth: `dev-docs/Refactron_Detailed_Execution_Plan.md` §Week 7. +- LOCKED contract: `src/contracts.ts` (untouched). +- Existing infrastructure: `src/infrastructure/diff.ts` (Week 1), + `src/cli/v2-adapters.ts` (Week 5), `src/ui/theme.ts`. +- Gauntlet-#1 PR: #20 (G1 + G2 + bonus Node 18 / Windows fixes). +- Output redesign PRs: #19 (Days 43-45), #18 (REPL document output routing). From 42e621279417814b54240b006b14969338b2bdef Mon Sep 17 00:00:00 2001 From: omsherikar Date: Thu, 14 May 2026 16:48:40 +0530 Subject: [PATCH 5/5] feat(bench): reproducible perf bench script with N=5 iterations and evidence file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The original single-run bench results were misleading — they implied exact numbers (1.31s / 11.48s) that single-iteration timing can't support. Replaced with a reproducible N=5-iterations-with-warmup script and saved the raw evidence to bench/results-2026-05-14.txt. Methodology (bench/run-bench.sh): - 1 warm-up run (discarded — primes Node module cache, tree-sitter wasm load, OS file cache) - 5 measured runs via /usr/bin/time -p - Median + min + max reported - Hardware + Node version captured at top of results file Real results (2026-05-14, Apple M2, Node 24): - 10k LOC (448 files): median 1.31s range 1.16-1.64s - 100k LOC (4465 files): median 20.58s range 14.99-38.65s - 500k LOC: not run (~30 MB fixture; runnable locally) Honest correction: the prior single-run 11.48s for 100k was below the bench's measured MIN (14.99s) — that run probably had a fully-warm FS cache from the immediately-preceding fixture generation. The N=5 median (20.58s) is the correct number to claim. 100k LOC shows 2.6x run-to-run variance. Even the worst run (38.65s) beats the 60s target. Variance source likely OS file-cache churn under 8 GB RAM + GC pauses. Investigation deferred unless it becomes a ship-blocker. --- bench/results-2026-05-14.txt | 27 +++++ bench/run-bench.sh | 102 +++++++++++++++++++ dev-docs/decisions/09-week-7-architecture.md | 39 +++++-- 3 files changed, 158 insertions(+), 10 deletions(-) create mode 100644 bench/results-2026-05-14.txt create mode 100755 bench/run-bench.sh diff --git a/bench/results-2026-05-14.txt b/bench/results-2026-05-14.txt new file mode 100644 index 0000000..44f0f3f --- /dev/null +++ b/bench/results-2026-05-14.txt @@ -0,0 +1,27 @@ +Refactron analyze perf bench — 2026-05-14 +==================================== + +Hardware: + Apple M2 (8 physical cores) + 8 GB RAM + Darwin 25.4.0 arm64 + macOS 26.4.1 + +Versions: + node: v24.2.0 + npm: 11.3.0 + refactron: 0.1.0-beta.2 + +Methodology: + 1 warm-up run (discarded), then 5 measured runs per size. + Wall-clock seconds via /usr/bin/time -p. + Report: median (middle), min, max. + +Size: 10000 LOC (448 files) + Runs: 1.23 1.43 1.31 1.64 1.16 + Median: 1.31s Min: 1.16s Max: 1.64s + +Size: 100000 LOC (4465 files) + Runs: 24.66 38.65 17.58 20.58 14.99 + Median: 20.58s Min: 14.99s Max: 38.65s + diff --git a/bench/run-bench.sh b/bench/run-bench.sh new file mode 100755 index 0000000..f9c5ea9 --- /dev/null +++ b/bench/run-bench.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# bench/run-bench.sh +# Reproducible perf benchmark for Refactron's analyze step. +# Generates the requested fixture sizes fresh, runs N timed iterations, +# and saves median + min + max to bench/results-.txt. +# +# Methodology: +# 1. One warm-up run per size (discarded — primes Node's module cache, +# tree-sitter wasm load, OS file cache). +# 2. N=5 measured runs per size (default), captured via /usr/bin/time -p. +# 3. Report median (3rd of 5), min, max wall-clock seconds. +# 4. Hardware + Node version recorded at the top of the results file. + +set -euo pipefail +cd "$(dirname "$0")/.." + +# Default sizes; override with: SIZES="10000 100000" ./bench/run-bench.sh +SIZES="${SIZES:-10000 100000}" +ITERATIONS="${ITERATIONS:-5}" +DATE="$(date +%Y-%m-%d)" +OUT="bench/results-${DATE}.txt" + +echo "==> Building dist/ (if stale)..." +npm run build > /dev/null 2>&1 + +{ + echo "Refactron analyze perf bench — $DATE" + echo "====================================" + echo + echo "Hardware:" + echo " $(sysctl -n machdep.cpu.brand_string 2>/dev/null || echo 'unknown CPU') ($(sysctl -n hw.physicalcpu 2>/dev/null || echo '?') physical cores)" + echo " $(($(sysctl -n hw.memsize 2>/dev/null || echo 0) / 1024 / 1024 / 1024)) GB RAM" + echo " $(uname -srm)" + if command -v sw_vers > /dev/null; then + echo " $(sw_vers -productName) $(sw_vers -productVersion)" + fi + echo + echo "Versions:" + echo " node: $(node --version)" + echo " npm: $(npm --version)" + echo " refactron: $(node -e "console.log(require('./package.json').version)")" + echo + echo "Methodology:" + echo " 1 warm-up run (discarded), then $ITERATIONS measured runs per size." + echo " Wall-clock seconds via /usr/bin/time -p." + echo " Report: median (middle), min, max." + echo +} > "$OUT" + +run_one() { + local size="$1" + local dir="bench/${size}-loc" + + echo "==> Generating $size-LOC fixture..." + npx tsx bench/gen-fixture.ts "$size" "$dir" > /dev/null + + echo "==> Warming up..." + REFACTRON_TOKEN=dummy node dist/cli/index.js analyze "$dir" > /dev/null 2>&1 || true + + echo "==> Running $ITERATIONS measured iterations..." + local times=() + for i in $(seq 1 "$ITERATIONS"); do + local elapsed + # Discard analyze's stdout. time -p writes "real X.XX" to stderr; merge + # it onto stdout (after analyze's stdout is silenced) so awk can parse it. + elapsed=$( { /usr/bin/time -p env REFACTRON_TOKEN=dummy node dist/cli/index.js analyze "$dir" > /dev/null; } 2>&1 | awk '/^real/{print $2}') + times+=("$elapsed") + echo " run $i: ${elapsed}s" + done + + # Compute median, min, max. + local sorted + sorted=$(printf '%s\n' "${times[@]}" | sort -n) + local mid_idx=$(( (ITERATIONS + 1) / 2 )) + local median min max + median=$(echo "$sorted" | sed -n "${mid_idx}p") + min=$(echo "$sorted" | head -1) + max=$(echo "$sorted" | tail -1) + + # File count. + local files + files=$(find "$dir" -type f \( -name '*.py' -o -name '*.ts' \) | wc -l | tr -d ' ') + + { + echo "Size: $size LOC ($files files)" + echo " Runs: ${times[*]}" + echo " Median: ${median}s Min: ${min}s Max: ${max}s" + echo + } >> "$OUT" + + echo "==> Cleaning up..." + rm -rf "$dir" +} + +for size in $SIZES; do + run_one "$size" +done + +echo +echo "==> Results saved to $OUT" +echo +cat "$OUT" diff --git a/dev-docs/decisions/09-week-7-architecture.md b/dev-docs/decisions/09-week-7-architecture.md index 4106cb5..ef6feb6 100644 --- a/dev-docs/decisions/09-week-7-architecture.md +++ b/dev-docs/decisions/09-week-7-architecture.md @@ -73,16 +73,35 @@ output unchanged. target LOC count with every transform pattern represented. - Generated trees are gitignored (regenerate locally before each run). -## Bench results (2026-05-14, M-series macOS, Node 22) - -| Tree size | Files | `analyze` wall-clock | Target | Headroom | -|---|---|---|---|---| -| 10k LOC | 448 | 1.31s | 6s | 4.6× | -| 100k LOC | 4 465 | 11.48s | 60s | 5.2× | -| 500k LOC | — | not run | 5min | — | - -500k LOC run requires generating ~25 MB of fixture; deferred. Both measured -sizes pass with healthy headroom on first attempt — no profiling needed. +## Bench results (2026-05-14) + +**Hardware:** Apple M2, 8 physical cores, 8 GB RAM, macOS 26.4.1. +**Versions:** Node 24.2.0, npm 11.3.0, refactron 0.1.0-beta.2. + +**Methodology:** 1 warm-up run (discarded — primes Node's module cache, +tree-sitter wasm load, OS file cache), then **5 measured iterations** per +size via `/usr/bin/time -p`. Reproducible via `bash bench/run-bench.sh`; +raw evidence saved to `bench/results-.txt`. + +| Tree size | Files | Runs (s) | Median | Min – Max | Target | Median headroom | +|---|---|---|---|---|---|---| +| 10k LOC | 448 | 1.23 / 1.43 / 1.31 / 1.64 / 1.16 | **1.31s** | 1.16s – 1.64s | 6s | 4.6× | +| 100k LOC | 4 465 | 24.66 / 38.65 / 17.58 / 20.58 / 14.99 | **20.58s** | 14.99s – 38.65s | 60s | 2.9× | +| 500k LOC | — | not run | — | — | 5min | — | + +**Both measured sizes pass the targets at the median.** The 100k case shows +~2.6× run-to-run variance (15s vs 39s). Likely sources: OS file-cache +churn under 8 GB RAM, GC pauses, background daemons. Even the worst +observed run (38.65s) sits comfortably under the 60s target. Raw evidence: +[`bench/results-2026-05-14.txt`](../../bench/results-2026-05-14.txt). + +**500k LOC run** requires generating ~30 MB of fixture; deferred from CI +because the fixture generation alone takes ~30s wall-clock on this machine. +Trivially runnable locally with `SIZES=500000 bash bench/run-bench.sh`. + +**Variance investigation deferred** to a follow-up if 100k variance becomes a +ship-blocker. Current data is sufficient to claim "runs in reasonable time +on a 4 465-file project" without overclaiming a precise number. ## Consequences - The redesign removes the test-gate front-slice bug (failure surface used