Week 7 close-out: gauntlet G3+G4, perf bench, ADR-009

.gitignore

@@ -29,3 +29,4 @@ YRC/
 __pycache__/
 *.pyc
 .pytest_cache/
+bench/*-loc/

README.md

@@ -258,6 +258,33 @@ Extend `BaseFixer` from `src/autofix/fixers/base.ts`, declare `supportedIssueTyp
 
 ---
 
+## Known Limitations
+
+### Running Refactron on the Refactron repo (self-test paradox)
+
+If you `git clone` Refactron and run `refactron run --apply` on the repo
+itself, the test gate **will** fail and **no files will be written**.
+
+That's working as designed. Refactron's own test suite includes meta-tests
+that exercise the transforms on `fixtures/python-legacy-mini/` and
+`fixtures/ts-legacy-mini/` — fixtures that are deliberately full of legacy
+patterns. Running the transforms on those fixtures produces refactored code,
+which then no longer matches what the meta-tests expect as input. The
+verification engine catches the regression and refuses to write — exactly
+what would happen on any project where a refactor breaks downstream tests.
+
+To self-analyze without triggering this, exclude the fixtures via
+`.refactronrc.json`:
+
+```json
+{ "exclude": ["fixtures/**"] }
+```
+
+`refactron analyze .` then returns `No findings.` and `run --apply .`
+becomes a no-op.
+
+---
+
 ## Contributing
 
 See [CONTRIBUTING.md](./CONTRIBUTING.md).

bench/README.md

@@ -0,0 +1,36 @@
+# bench/
+
+Synthetic fixture generator for the Week 7 perf bench.
+
+The generated trees are NOT committed (they're large and easy to regenerate).
+Run the generator locally before benchmarking.
+
+## Usage
+
+```bash
+# Generate a synthetic 10k-LOC fixture mixing Python + TypeScript files
+# sprinkled with every Refactron transform pattern.
+npx tsx bench/gen-fixture.ts 10000 bench/10k-loc
+
+# Run analyze and time it.
+time REFACTRON_TOKEN=dummy node dist/cli/index.js analyze bench/10k-loc
+
+# Cleanup
+rm -rf bench/10k-loc
+```
+
+## Targets (Week 7 binary gate)
+
+| Tree size | Target |
+|---|---|
+| 10k LOC | < 6s for `analyze` |
+| 100k LOC | < 60s for `analyze` |
+| 500k LOC | < 5min for `analyze` |
+| 100k LOC + run --apply | < 5min including test gate |
+
+The 500k tree generation requires ~25 MB of disk and ~30s wall-clock on a
+modern dev machine. Skip it on CI; run locally before release.
+
+## Most recent results
+
+See `dev-docs/decisions/09-week-7-architecture.md` for benchmark snapshots.

bench/gen-fixture.ts

@@ -0,0 +1,118 @@
+// bench/gen-fixture.ts
+// Synthetic fixture generator for the Week 7 perf bench. Produces a directory
+// tree of mixed Python + TypeScript files at a target line count, sprinkled
+// with one of each Refactron transform pattern so `analyze` finds work to do.
+//
+// Usage:
+//   tsx bench/gen-fixture.ts <target-loc> <out-dir>
+//   node --loader ts-node/esm bench/gen-fixture.ts 10000 bench/10k-loc
+//
+// We DON'T commit the generated trees — they're large and easy to regenerate.
+// Run this script locally before invoking the bench script.
+
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+
+const PY_TEMPLATE = (i: number): string => `# Generated module ${i}
+import requests
+
+def fetch_${i}(user_id, callback):
+    """Generated callback-style fetch fixture #${i}."""
+    result = requests.get("/users/%s" % user_id)
+    payload = "Loaded user %s" % user_id
+    callback(payload)
+    return result
+
+
+class User${i}:
+    def __init__(self, id, name, email):
+        self.id = id
+        self.name = name
+        self.email = email
+
+
+def check_${i}(value):
+    if isinstance(value, str):
+        return value.upper()
+    if isinstance(value, int):
+        return str(value)
+    return None
+`;
+
+const TS_TEMPLATE = (i: number): string => `// Generated module ${i}
+const path = require('path');
+
+export function makeGreeting${i}(name) {
+  var greeting = 'hi-${i}';
+  return greeting + ', ' + name;
+}
+
+export function chain${i}(input: any) {
+  return Promise.resolve(input)
+    .then((v) => v + 1)
+    .then((v) => v * 2);
+}
+
+export function build${i}() {
+  return new Promise((resolve, reject) => {
+    setTimeout(() => resolve(${i}), 10);
+  });
+}
+
+module.exports = { makeGreeting${i}, chain${i}, build${i} };
+`;
+
+// Approximate LOC per template (counted from the strings above).
+const PY_LOC_PER_FILE = 24;
+const TS_LOC_PER_FILE = 21;
+
+async function generate(outDir: string, targetLoc: number): Promise<void> {
+  await fs.rm(outDir, { recursive: true, force: true });
+  await fs.mkdir(outDir, { recursive: true });
+
+  // 50/50 split between python and typescript.
+  const halfLoc = targetLoc / 2;
+  const pyCount = Math.ceil(halfLoc / PY_LOC_PER_FILE);
+  const tsCount = Math.ceil(halfLoc / TS_LOC_PER_FILE);
+
+  // Spread into ~100 files per directory to avoid pathological dir sizes.
+  const pyDir = path.join(outDir, 'src_py');
+  const tsDir = path.join(outDir, 'src_ts');
+  await fs.mkdir(pyDir, { recursive: true });
+  await fs.mkdir(tsDir, { recursive: true });
+
+  for (let i = 0; i < pyCount; i++) {
+    const sub = path.join(pyDir, `pkg_${Math.floor(i / 100)}`);
+    await fs.mkdir(sub, { recursive: true });
+    await fs.writeFile(path.join(sub, `mod_${i}.py`), PY_TEMPLATE(i));
+  }
+  for (let i = 0; i < tsCount; i++) {
+    const sub = path.join(tsDir, `pkg_${Math.floor(i / 100)}`);
+    await fs.mkdir(sub, { recursive: true });
+    await fs.writeFile(path.join(sub, `mod_${i}.ts`), TS_TEMPLATE(i));
+  }
+
+  const totalFiles = pyCount + tsCount;
+  const totalLoc = pyCount * PY_LOC_PER_FILE + tsCount * TS_LOC_PER_FILE;
+  process.stdout.write(
+    `generated ${totalFiles} files (~${totalLoc} LOC) in ${outDir}\n` +
+      `  python: ${pyCount} files\n` +
+      `  typescript: ${tsCount} files\n`,
+  );
+}
+
+async function main(): Promise<void> {
+  const args = process.argv.slice(2);
+  const targetLoc = Number(args[0]);
+  const outDir = args[1];
+  if (!Number.isFinite(targetLoc) || targetLoc <= 0 || !outDir) {
+    process.stderr.write('usage: gen-fixture.ts <target-loc> <out-dir>\n');
+    process.exit(1);
+  }
+  await generate(path.resolve(outDir), targetLoc);
+}
+
+main().catch((err) => {
+  process.stderr.write(`gen-fixture failed: ${err}\n`);
+  process.exit(1);
+});

bench/results-2026-05-14.txt

@@ -0,0 +1,27 @@
+Refactron analyze perf bench — 2026-05-14
+====================================
+
+Hardware:
+  Apple M2 (8 physical cores)
+  8 GB RAM
+  Darwin 25.4.0 arm64
+  macOS 26.4.1
+
+Versions:
+  node: v24.2.0
+  npm: 11.3.0
+  refactron: 0.1.0-beta.2
+
+Methodology:
+  1 warm-up run (discarded), then 5 measured runs per size.
+  Wall-clock seconds via /usr/bin/time -p.
+  Report: median (middle), min, max.
+
+Size: 10000 LOC (448 files)
+  Runs: 1.23 1.43 1.31 1.64 1.16
+  Median: 1.31s   Min: 1.16s   Max: 1.64s
+
+Size: 100000 LOC (4465 files)
+  Runs: 24.66 38.65 17.58 20.58 14.99
+  Median: 20.58s   Min: 14.99s   Max: 38.65s
+

bench/run-bench.sh

@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+# bench/run-bench.sh
+# Reproducible perf benchmark for Refactron's analyze step.
+# Generates the requested fixture sizes fresh, runs N timed iterations,
+# and saves median + min + max to bench/results-<DATE>.txt.
+#
+# Methodology:
+#   1. One warm-up run per size (discarded — primes Node's module cache,
+#      tree-sitter wasm load, OS file cache).
+#   2. N=5 measured runs per size (default), captured via /usr/bin/time -p.
+#   3. Report median (3rd of 5), min, max wall-clock seconds.
+#   4. Hardware + Node version recorded at the top of the results file.
+
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+# Default sizes; override with: SIZES="10000 100000" ./bench/run-bench.sh
+SIZES="${SIZES:-10000 100000}"
+ITERATIONS="${ITERATIONS:-5}"
+DATE="$(date +%Y-%m-%d)"
+OUT="bench/results-${DATE}.txt"
+
+echo "==> Building dist/ (if stale)..."
+npm run build > /dev/null 2>&1
+
+{
+  echo "Refactron analyze perf bench — $DATE"
+  echo "===================================="
+  echo
+  echo "Hardware:"
+  echo "  $(sysctl -n machdep.cpu.brand_string 2>/dev/null || echo 'unknown CPU') ($(sysctl -n hw.physicalcpu 2>/dev/null || echo '?') physical cores)"
+  echo "  $(($(sysctl -n hw.memsize 2>/dev/null || echo 0) / 1024 / 1024 / 1024)) GB RAM"
+  echo "  $(uname -srm)"
+  if command -v sw_vers > /dev/null; then
+    echo "  $(sw_vers -productName) $(sw_vers -productVersion)"
+  fi
+  echo
+  echo "Versions:"
+  echo "  node: $(node --version)"
+  echo "  npm: $(npm --version)"
+  echo "  refactron: $(node -e "console.log(require('./package.json').version)")"
+  echo
+  echo "Methodology:"
+  echo "  1 warm-up run (discarded), then $ITERATIONS measured runs per size."
+  echo "  Wall-clock seconds via /usr/bin/time -p."
+  echo "  Report: median (middle), min, max."
+  echo
+} > "$OUT"
+
+run_one() {
+  local size="$1"
+  local dir="bench/${size}-loc"
+
+  echo "==> Generating $size-LOC fixture..."
+  npx tsx bench/gen-fixture.ts "$size" "$dir" > /dev/null
+
+  echo "==> Warming up..."
+  REFACTRON_TOKEN=dummy node dist/cli/index.js analyze "$dir" > /dev/null 2>&1 || true
+
+  echo "==> Running $ITERATIONS measured iterations..."
+  local times=()
+  for i in $(seq 1 "$ITERATIONS"); do
+    local elapsed
+    # Discard analyze's stdout. time -p writes "real X.XX" to stderr; merge
+    # it onto stdout (after analyze's stdout is silenced) so awk can parse it.
+    elapsed=$( { /usr/bin/time -p env REFACTRON_TOKEN=dummy node dist/cli/index.js analyze "$dir" > /dev/null; } 2>&1 | awk '/^real/{print $2}')
+    times+=("$elapsed")
+    echo "    run $i: ${elapsed}s"
+  done
+
+  # Compute median, min, max.
+  local sorted
+  sorted=$(printf '%s\n' "${times[@]}" | sort -n)
+  local mid_idx=$(( (ITERATIONS + 1) / 2 ))
+  local median min max
+  median=$(echo "$sorted" | sed -n "${mid_idx}p")
+  min=$(echo "$sorted" | head -1)
+  max=$(echo "$sorted" | tail -1)
+
+  # File count.
+  local files
+  files=$(find "$dir" -type f \( -name '*.py' -o -name '*.ts' \) | wc -l | tr -d ' ')
+
+  {
+    echo "Size: $size LOC ($files files)"
+    echo "  Runs: ${times[*]}"
+    echo "  Median: ${median}s   Min: ${min}s   Max: ${max}s"
+    echo
+  } >> "$OUT"
+
+  echo "==> Cleaning up..."
+  rm -rf "$dir"
+}
+
+for size in $SIZES; do
+  run_one "$size"
+done
+
+echo
+echo "==> Results saved to $OUT"
+echo
+cat "$OUT"